From e73ddd1e6f9db6e7e36c617f39ce7fbdf170afd1 Mon Sep 17 00:00:00 2001 From: wpan Date: Mon, 21 Jun 2021 09:23:13 +0800 Subject: [PATCH 1/7] fix bug for disk full --- src/query/src/qExecutor.c | 6 +++++- src/query/src/qTsbuf.c | 30 +++++++++++++++++++++++++----- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index f97a0c4a74..69242adc2a 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6720,7 +6720,11 @@ int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *p if (pQueryMsg->tsLen > 0) { // open new file to save the result char *tsBlock = (char *) pQueryMsg + pQueryMsg->tsOffset; pTsBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder, vgId); - + if (pTsBuf == NULL) { + code = TSDB_CODE_QRY_NO_DISKSPACE; + goto _error; + } + tsBufResetPos(pTsBuf); bool ret = tsBufNextPos(pTsBuf); diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 1f43c5b33c..f4c6e24290 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -2,6 +2,7 @@ #include "taoserror.h" #include "tscompression.h" #include "tutil.h" +#include "queryLog.h" static int32_t getDataStartOffset(); static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo); @@ -633,10 +634,14 @@ int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) { int32_t r = fseek(pTSBuf->f, 0, SEEK_SET); if (r != 0) { + qError("fseek failed, errno:%d", errno); return -1; } - fwrite(pHeader, sizeof(STSBufFileHeader), 1, pTSBuf->f); + if (fwrite(pHeader, sizeof(STSBufFileHeader), 1, pTSBuf->f) != sizeof(STSBufFileHeader)) { + qError("fwrite failed, errno:%d", errno); + return -1; + } return 0; } @@ -856,9 +861,17 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, pBlockInfo); int32_t ret = fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET); - UNUSED(ret); + if (ret == -1) { + qError("fseek failed, errno:%d", errno); + tsBufDestroy(pTSBuf); + return NULL; + } size_t sz = fwrite((void*)pData, 1, len, pTSBuf->f); - UNUSED(sz); + if (sz != len) { + qError("fwrite failed, errno:%d", errno); + tsBufDestroy(pTSBuf); + return NULL; + } pTSBuf->fileSize += len; pTSBuf->tsOrder = order; @@ -866,9 +879,16 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ STSBufFileHeader header = { .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder}; - STSBufUpdateHeader(pTSBuf, &header); + if (STSBufUpdateHeader(pTSBuf, &header) < 0) { + tsBufDestroy(pTSBuf); + return NULL; + } - fsync(fileno(pTSBuf->f)); + if (fsync(fileno(pTSBuf->f)) == -1) { + qError("fsync failed, errno:%d", errno); + tsBufDestroy(pTSBuf); + return NULL; + } return pTSBuf; } From 7b2a1265309f677ccc76052b67a99e1435abf4f4 Mon Sep 17 00:00:00 2001 From: wpan Date: Mon, 21 Jun 2021 09:41:22 +0800 Subject: [PATCH 2/7] fix bug --- src/query/src/qTsbuf.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index f4c6e24290..81ad0af736 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -637,9 +637,10 @@ int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) { qError("fseek failed, errno:%d", errno); return -1; } - - if (fwrite(pHeader, sizeof(STSBufFileHeader), 1, pTSBuf->f) != sizeof(STSBufFileHeader)) { - qError("fwrite failed, errno:%d", errno); + + size_t ws = fwrite(pHeader, sizeof(STSBufFileHeader), 1, pTSBuf->f); + if (ws != sizeof(STSBufFileHeader)) { + qError("ts update header fwrite failed, size:%d, expected size:%d", (int32_t)ws, (int32_t)sizeof(STSBufFileHeader)); return -1; } return 0; @@ -868,7 +869,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ } size_t sz = fwrite((void*)pData, 1, len, pTSBuf->f); if (sz != len) { - qError("fwrite failed, errno:%d", errno); + qError("ts data fwrite failed, write size:%d, expected size:%d", (int32_t)sz, len); tsBufDestroy(pTSBuf); return NULL; } From 1a458d66db33dd3cdaee074fa18bda07d1a9a19d Mon Sep 17 00:00:00 2001 From: wpan Date: Mon, 21 Jun 2021 09:45:26 +0800 Subject: [PATCH 3/7] fix bug --- src/query/src/qTsbuf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 81ad0af736..9c04c7c929 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -639,7 +639,7 @@ int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) { } size_t ws = fwrite(pHeader, sizeof(STSBufFileHeader), 1, pTSBuf->f); - if (ws != sizeof(STSBufFileHeader)) { + if (ws != 1) { qError("ts update header fwrite failed, size:%d, expected size:%d", (int32_t)ws, (int32_t)sizeof(STSBufFileHeader)); return -1; } From 612b6c84521e3fc0c762d7446657e1e3cfed7496 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 22 Jun 2021 16:33:00 +0800 Subject: [PATCH 4/7] [TD-4805]: fix startup os config display --- src/dnode/src/dnodeMain.c | 2 ++ src/os/inc/osSysinfo.h | 1 + src/os/src/darwin/darwinSysInfo.c | 11 ++++++++--- src/os/src/detail/osSysinfo.c | 11 ++++++++--- src/os/src/windows/wSysinfo.c | 7 ++++++- 5 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 410e6bb188..22ce6c995a 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -253,6 +253,8 @@ static int32_t dnodeInitStorage() { dnodeCheckDataDirOpenned(tsDnodeDir); + taosGetDisk(); + taosPrintDiskInfo(); dInfo("dnode storage is initialized at %s", tsDnodeDir); return 0; } diff --git a/src/os/inc/osSysinfo.h b/src/os/inc/osSysinfo.h index 895b5dd499..1be2e94dc5 100644 --- a/src/os/inc/osSysinfo.h +++ b/src/os/inc/osSysinfo.h @@ -36,6 +36,7 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) ; bool taosGetProcMemory(float *memoryUsedMB) ; bool taosGetSysMemory(float *memoryUsedMB); void taosPrintOsInfo(); +void taosPrintDiskInfo(); int taosSystem(const char * cmd) ; void taosKillSystem(); bool taosGetSystemUid(char *uid); diff --git a/src/os/src/darwin/darwinSysInfo.c b/src/os/src/darwin/darwinSysInfo.c index 6e70043779..3c80ee4260 100644 --- a/src/os/src/darwin/darwinSysInfo.c +++ b/src/os/src/darwin/darwinSysInfo.c @@ -137,9 +137,6 @@ void taosPrintOsInfo() { // uInfo(" os openMax: %" PRId64, tsOpenMax); // uInfo(" os streamMax: %" PRId64, tsStreamMax); uInfo(" os numOfCores: %d", tsNumOfCores); - uInfo(" os totalDisk: %f(GB)", tsTotalDataDirGB); - uInfo(" os usedDisk: %f(GB)", tsUsedDataDirGB); - uInfo(" os availDisk: %f(GB)", tsAvailDataDirGB); uInfo(" os totalMemory: %d(MB)", tsTotalMemoryMB); struct utsname buf; @@ -155,6 +152,14 @@ void taosPrintOsInfo() { uInfo("=================================="); } +void taosPrintDiskInfo() { + uInfo("=================================="); + uInfo(" os totalDisk: %f(GB)", tsTotalDataDirGB); + uInfo(" os usedDisk: %f(GB)", tsUsedDataDirGB); + uInfo(" os availDisk: %f(GB)", tsAvailDataDirGB); + uInfo("=================================="); +} + void taosKillSystem() { uError("function taosKillSystem, exit!"); exit(0); diff --git a/src/os/src/detail/osSysinfo.c b/src/os/src/detail/osSysinfo.c index bbda08aa25..a2a560b296 100644 --- a/src/os/src/detail/osSysinfo.c +++ b/src/os/src/detail/osSysinfo.c @@ -506,9 +506,6 @@ void taosPrintOsInfo() { uInfo(" os openMax: %" PRId64, tsOpenMax); uInfo(" os streamMax: %" PRId64, tsStreamMax); uInfo(" os numOfCores: %d", tsNumOfCores); - uInfo(" os totalDisk: %f(GB)", tsTotalDataDirGB); - uInfo(" os usedDisk: %f(GB)", tsUsedDataDirGB); - uInfo(" os availDisk: %f(GB)", tsAvailDataDirGB); uInfo(" os totalMemory: %d(MB)", tsTotalMemoryMB); struct utsname buf; @@ -523,6 +520,14 @@ void taosPrintOsInfo() { uInfo(" os machine: %s", buf.machine); } +void taosPrintDiskInfo() { + uInfo("=================================="); + uInfo(" os totalDisk: %f(GB)", tsTotalDataDirGB); + uInfo(" os usedDisk: %f(GB)", tsUsedDataDirGB); + uInfo(" os availDisk: %f(GB)", tsAvailDataDirGB); + uInfo("=================================="); +} + void taosKillSystem() { // SIGINT uInfo("taosd will shut down soon"); diff --git a/src/os/src/windows/wSysinfo.c b/src/os/src/windows/wSysinfo.c index 8a81e3079a..72793a1049 100644 --- a/src/os/src/windows/wSysinfo.c +++ b/src/os/src/windows/wSysinfo.c @@ -205,10 +205,15 @@ void taosGetSystemInfo() { void taosPrintOsInfo() { uInfo(" os numOfCores: %d", tsNumOfCores); + uInfo(" os totalMemory: %d(MB)", tsTotalMemoryMB); + uInfo("=================================="); +} + +void taosPrintDiskInfo() { + uInfo("=================================="); uInfo(" os totalDisk: %f(GB)", tsTotalDataDirGB); uInfo(" os usedDisk: %f(GB)", tsUsedDataDirGB); uInfo(" os availDisk: %f(GB)", tsAvailDataDirGB); - uInfo(" os totalMemory: %d(MB)", tsTotalMemoryMB); uInfo("=================================="); } From f1de963dc54cf9f961c57fb20e49914e325f497b Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 22 Jun 2021 18:49:20 +0800 Subject: [PATCH 5/7] [TD-4823]: taosdemo getTableName return empty. (#6570) * [TD-4823]: taosdemo getTableName return empty. * fix typo. --- src/kit/taosdemo/taosdemo.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 4202a833ea..6b12e66cb9 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -6000,6 +6000,12 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { verbosePrint("%s() LN%d: tid=%d seq=%"PRId64" tableName=%s\n", __func__, __LINE__, pThreadInfo->threadID, tableSeq, tableName); + if (0 == strlen(tableName)) { + errorPrint("[%d] %s() LN%d, getTableName return null\n", + pThreadInfo->threadID, __func__, __LINE__); + free(pThreadInfo->buffer); + return NULL; + } int64_t remainderBufLen = maxSqlLen; char *pstr = pThreadInfo->buffer; From 2763345e778c8807879c9a368bd81f3a8e832463 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 22 Jun 2021 21:48:01 +0800 Subject: [PATCH 6/7] [TD-4838]: taosdump binary length bug. (#6580) --- src/kit/taosdump/taosdump.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index f80ac069a0..8525496b0e 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -1858,13 +1858,13 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* converStringToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE); //pstr = stpcpy(pstr, tbuf); //*(pstr++) = '\''; - pstr += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); break; } case TSDB_DATA_TYPE_NCHAR: { char tbuf[COMMAND_SIZE] = {0}; convertNCharToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE); - pstr += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); break; } case TSDB_DATA_TYPE_TIMESTAMP: @@ -1897,7 +1897,8 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* total_sqlstr_len += curr_sqlstr_len; - if ((count >= arguments->data_batch) || (sql_buf_len - total_sqlstr_len < TSDB_MAX_BYTES_PER_ROW)) { + if ((count >= arguments->data_batch) + || (sql_buf_len - total_sqlstr_len < TSDB_MAX_BYTES_PER_ROW)) { fprintf(fp, ";\n"); count = 0; } //else { @@ -1905,6 +1906,8 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* //} } + printf("total_sqlstr_len: %d\n", total_sqlstr_len); + fprintf(fp, "\n"); atomic_add_fetch_64(&totalDumpOutRows, totalRows); From e1505aceb160b174df0b31d1110b0a1b55a0f707 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 23 Jun 2021 15:51:47 +0800 Subject: [PATCH 7/7] [TD-4874]: reset customScore to 0 --- src/mnode/src/mnodeDnode.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 70a63517ca..ce21af49c2 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -101,6 +101,8 @@ static int32_t mnodeDnodeActionInsert(SSdbRow *pRow) { pDnode->offlineReason = TAOS_DN_OFF_STATUS_NOT_RECEIVED; } + pDnode->customScore = 0; + dnodeUpdateEp(pDnode->dnodeId, pDnode->dnodeEp, pDnode->dnodeFqdn, &pDnode->dnodePort); mnodeUpdateDnodeEps();