From c8acfc4d54530cc05736696454aa386f312cadbe Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 10 Feb 2023 10:48:26 +0800 Subject: [PATCH 01/10] fix: stream meta --- source/libs/stream/src/streamMeta.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 2f991288ff..184c1c8abf 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -44,7 +44,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF goto _err; } - pMeta->pTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + pMeta->pTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (pMeta->pTasks == NULL) { goto _err; } From 5d526d926978f851c60c52120058527f451ea2d7 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 10 Feb 2023 10:57:50 +0800 Subject: [PATCH 02/10] fix: taosd crash issue --- source/libs/transport/src/thttp.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index cd508f6fe9..8e5f79137f 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -420,7 +420,13 @@ static void transHttpEnvInit() { uv_loop_init(http->loop); http->asyncPool = transAsyncPoolCreate(http->loop, 1, http, httpAsyncCb); - + if (NULL == http->asyncPool) { + taosMemoryFree(http->loop); + taosMemoryFree(http); + http = NULL; + return; + } + int err = taosThreadCreate(&http->thread, NULL, httpThread, (void*)http); if (err != 0) { taosMemoryFree(http->loop); From 4908c563ccd3fd9347c217f97db06df6f2312c02 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 10 Feb 2023 14:35:18 +0800 Subject: [PATCH 03/10] ehn(query): dynamic set the initial buffer size for data block from disk. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 6400e7534b..af3da87b17 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3837,7 +3837,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL int32_t capacity = 0; if (pResBlock == NULL) { - capacity = 4096; + capacity = pVnode->config.tsdbCfg.maxRows; } else { capacity = pResBlock->info.capacity; } From 17f05c0020c2b285dd5d928ed4260081f36cccb9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 10 Feb 2023 14:40:37 +0800 Subject: [PATCH 04/10] refactor: disable the limitation of maximum buffer size. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index af3da87b17..b8d0ea28da 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -423,7 +423,9 @@ static STimeWindow updateQueryTimeWindow(STsdb* pTsdb, STimeWindow* pWindow) { return win; } +// note: currently not need this limitation static void limitOutputBufferSize(const SQueryTableDataCond* pCond, int32_t* capacity) { +#if 0 int32_t rowLen = 0; for (int32_t i = 0; i < pCond->numOfCols; ++i) { rowLen += pCond->colList[i].bytes; @@ -434,6 +436,7 @@ static void limitOutputBufferSize(const SQueryTableDataCond* pCond, int32_t* cap if ((*capacity) * rowLen > TWOMB) { (*capacity) = TWOMB / rowLen; } +#endif } // init file iterator @@ -618,7 +621,6 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd goto _end; } - // todo refactor. limitOutputBufferSize(pCond, &pReader->capacity); // allocate buffer in order to load data blocks from file From c8f1c3b6e4973521ca35cbd7564b2341daa208d5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 10 Feb 2023 16:18:30 +0800 Subject: [PATCH 05/10] fix(query): set the max rows as the default block buffer rows. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 26 +++---------------------- source/libs/executor/src/scanoperator.c | 6 +++++- 2 files changed, 8 insertions(+), 24 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index b8d0ea28da..e9bb7e3d09 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -423,22 +423,6 @@ static STimeWindow updateQueryTimeWindow(STsdb* pTsdb, STimeWindow* pWindow) { return win; } -// note: currently not need this limitation -static void limitOutputBufferSize(const SQueryTableDataCond* pCond, int32_t* capacity) { -#if 0 - int32_t rowLen = 0; - for (int32_t i = 0; i < pCond->numOfCols; ++i) { - rowLen += pCond->colList[i].bytes; - } - - // make sure the output SSDataBlock size be less than 2MB. - const int32_t TWOMB = 2 * 1024 * 1024; - if ((*capacity) * rowLen > TWOMB) { - (*capacity) = TWOMB / rowLen; - } -#endif -} - // init file iterator static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdbReader* pReader) { size_t numOfFileset = taosArrayGetSize(aDFileSet); @@ -621,8 +605,6 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd goto _end; } - limitOutputBufferSize(pCond, &pReader->capacity); - // allocate buffer in order to load data blocks from file SBlockLoadSuppInfo* pSup = &pReader->suppInfo; pSup->pColAgg = taosArrayInit(pCond->numOfCols, sizeof(SColumnDataAgg)); @@ -3837,11 +3819,9 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL pCond->twindows.ekey -= 1; } - int32_t capacity = 0; - if (pResBlock == NULL) { - capacity = pVnode->config.tsdbCfg.maxRows; - } else { - capacity = pResBlock->info.capacity; + int32_t capacity = pVnode->config.tsdbCfg.maxRows; + if (pResBlock != NULL) { + blockDataEnsureCapacity(pResBlock, capacity); } int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, capacity, pResBlock, idstr); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 6d3eb67f16..ac76f179a0 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -781,6 +781,10 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } + + if (pInfo->pResBlock->info.capacity > pOperator->resultInfo.capacity) { + pOperator->resultInfo.capacity = pInfo->pResBlock->info.capacity; + } } SSDataBlock* result = doGroupedTableScan(pOperator); @@ -884,7 +888,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pResBlock = createDataBlockFromDescNode(pDescNode); - blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity); +// blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity); code = filterInitFromNode((SNode*)pTableScanNode->scan.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); if (code != TSDB_CODE_SUCCESS) { From f6b2da8c064d9c96be1fc54f36fc4e9b6da4cf70 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 10 Feb 2023 16:53:11 +0800 Subject: [PATCH 06/10] fix(query): add check for table/super table dropped, when retrieving cache rows. --- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 9 +++++++-- source/libs/executor/src/cachescanoperator.c | 11 +++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index a837543e62..240dbb5d0c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -117,8 +117,13 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, return TSDB_CODE_SUCCESS; } - STableKeyInfo* pKeyInfo = &((STableKeyInfo*)pTableIdList)[0]; - p->pSchema = metaGetTbTSchema(p->pVnode->pMeta, pKeyInfo->uid, -1, 1); + p->pSchema = metaGetTbTSchema(p->pVnode->pMeta, suid, -1, 1); + if (p->pSchema == NULL) { + taosMemoryFree(p); + tsdbWarn("stable:%"PRIu64" has been dropped, failed to retrieve cached rows, %s", suid, idstr); + return TSDB_CODE_PAR_TABLE_NOT_EXIST; + } + p->pTableList = pTableIdList; p->numOfTables = numOfTables; diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 294424746a..7ee186511a 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -215,8 +215,15 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, code); } - tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num, - taosArrayGetSize(pInfo->matchInfo.pList), suid, &pInfo->pLastrowReader, pTaskInfo->id.str); + code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num, + taosArrayGetSize(pInfo->matchInfo.pList), suid, &pInfo->pLastrowReader, + pTaskInfo->id.str); + if (code != TSDB_CODE_SUCCESS) { + pInfo->currentGroupIndex += 1; + taosArrayClear(pInfo->pUidList); + continue; + } + taosArrayClear(pInfo->pUidList); code = tsdbRetrieveCacheRows(pInfo->pLastrowReader, pInfo->pRes, pInfo->pSlotIds, pInfo->pUidList); From 6122a2fb0c281a8b43dfc5e94831ec1de0aa600b Mon Sep 17 00:00:00 2001 From: xinsheng Ren <285808407@qq.com> Date: Fri, 10 Feb 2023 18:31:09 +0800 Subject: [PATCH 07/10] fix stackflow on mac (#19592) * fix stackflow on mac * ALL UPPER CASE for macro definition --------- Co-authored-by: facetosea <25808407@qq.com> --- source/os/src/osSysinfo.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index aeaa4fcafd..897e68a126 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -18,6 +18,7 @@ #include "taoserror.h" #define PROCESS_ITEM 12 +#define UUIDLEN37 37 typedef struct { uint64_t user; @@ -830,7 +831,8 @@ int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { return 0; #elif defined(_TD_DARWIN_64) uuid_t uuid = {0}; - char buf[37] = {0}; + char buf[UUIDLEN37]; + memset(buf, 0, UUIDLEN37); uuid_generate(uuid); // it's caller's responsibility to make enough space for `uid`, that's 36-char + 1-null uuid_unparse_lower(uuid, buf); From 5dfa0e69a9745c1b863bc46a9613d45aca0740d9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 10 Feb 2023 19:06:23 +0800 Subject: [PATCH 08/10] fix(query): set correct schema info for normal table. --- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 45 +++++++++++++++++---- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 240dbb5d0c..fd5e8eb6e0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -99,6 +99,38 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p return TSDB_CODE_SUCCESS; } +static int32_t setTableSchema(SCacheRowsReader* p, uint64_t suid, const char* idstr) { + int32_t numOfTables = p->numOfTables; + + if (suid != 0) { + p->pSchema = metaGetTbTSchema(p->pVnode->pMeta, suid, -1, 1); + if (p->pSchema == NULL) { + taosMemoryFree(p); + tsdbWarn("stable:%" PRIu64 " has been dropped, failed to retrieve cached rows, %s", suid, idstr); + return TSDB_CODE_PAR_TABLE_NOT_EXIST; + } + } else { + for (int32_t i = 0; i < numOfTables; ++i) { + uint64_t uid = p->pTableList[i].uid; + p->pSchema = metaGetTbTSchema(p->pVnode->pMeta, uid, -1, 1); + if (p->pSchema != NULL) { + break; + } + + tsdbWarn("table:%" PRIu64 " has been dropped, failed to retrieve cached rows, %s", uid, idstr); + } + + // all queried tables have been dropped already, return immediately. + if (p->pSchema == NULL) { + taosMemoryFree(p); + tsdbWarn("all queried tables has been dropped, try next group, %s", idstr); + return TSDB_CODE_PAR_TABLE_NOT_EXIST; + } + } + + return TSDB_CODE_SUCCESS; +} + int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, int32_t numOfTables, int32_t numOfCols, uint64_t suid, void** pReader, const char* idstr) { *pReader = NULL; @@ -117,16 +149,15 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, return TSDB_CODE_SUCCESS; } - p->pSchema = metaGetTbTSchema(p->pVnode->pMeta, suid, -1, 1); - if (p->pSchema == NULL) { - taosMemoryFree(p); - tsdbWarn("stable:%"PRIu64" has been dropped, failed to retrieve cached rows, %s", suid, idstr); - return TSDB_CODE_PAR_TABLE_NOT_EXIST; - } - p->pTableList = pTableIdList; p->numOfTables = numOfTables; + int32_t code = setTableSchema(p, suid, idstr); + if (code != TSDB_CODE_SUCCESS) { + tsdbCacherowsReaderClose(p); + return code; + } + p->transferBuf = taosMemoryCalloc(p->pSchema->numOfCols, POINTER_BYTES); if (p->transferBuf == NULL) { tsdbCacherowsReaderClose(p); From 18cad4f33a9b8c6dd7fb992eb5e122f06b21b95b Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 11 Feb 2023 00:45:42 +0800 Subject: [PATCH 09/10] fix: taosbenchmark fix sml-rest mem leak for main (#19916) * fix: taosbenchmark fix sml-rest mem leak for main * fix: update taos-tools 4097420 --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 48f31c6daf..19895a7f05 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG e04f39b + GIT_TAG 4097420 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 7a3fc004f29f5c0d9c973a91957916371ba29837 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 11 Feb 2023 20:18:29 +0800 Subject: [PATCH 10/10] fix: taosbenchmark invalid read when ctrl-c (#19927) --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 19895a7f05..3d10a0921e 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 4097420 + GIT_TAG 74c0357 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE