From 492b5d5f2a6fcb6ea036bcd1c813fcaacdbef942 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 16 Oct 2022 02:46:46 +0000 Subject: [PATCH 01/82] docs: refine taosbenchmark json drop --- docs/en/14-reference/05-taosbenchmark.md | 2 +- docs/zh/14-reference/05-taosbenchmark.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/14-reference/05-taosbenchmark.md b/docs/en/14-reference/05-taosbenchmark.md index d95086d4c4..6295d8553d 100644 --- a/docs/en/14-reference/05-taosbenchmark.md +++ b/docs/en/14-reference/05-taosbenchmark.md @@ -231,7 +231,7 @@ The parameters related to database creation are configured in `dbinfo` in the js - **name**: specify the name of the database. -- **drop**: indicate whether to delete the database before inserting. The default is true. +- **drop**: indicate whether to delete the database before inserting. The value can be 'yes' or 'no'. No means do not drop. The default is to drop. #### Stream processing related configuration parameters diff --git a/docs/zh/14-reference/05-taosbenchmark.md b/docs/zh/14-reference/05-taosbenchmark.md index 9f4f728f78..e067c646b2 100644 --- a/docs/zh/14-reference/05-taosbenchmark.md +++ b/docs/zh/14-reference/05-taosbenchmark.md @@ -231,7 +231,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) - **name** : 数据库名。 -- **drop** : 插入前是否删除数据库，默认为 true。 +- **drop** : 插入前是否删除数据库，可选项为 "yes" 或者 "no", 为 "no" 时不创建。默认删除。 #### 流式计算相关配置参数 From a96534d94589e56387b6490fa80666f7b6b25622 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 1 Nov 2022 17:07:24 +0800 Subject: [PATCH 02/82] fix: invalid read problem of tsdb path --- include/os/osDef.h | 2 +- source/dnode/vnode/src/tsdb/tsdbOpen.c | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/include/os/osDef.h b/include/os/osDef.h index 07cd197ad7..f5a79934bf 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -245,7 +245,7 @@ void syslog(int unused, const char *format, ...); #define TD_TIMEZONE_LEN 96 #ifdef WINDOWS -#define TD_PATH_MAX 260 +#define TD_PATH_MAX _MAX_PATH #elif defined(PATH_MAX) #define TD_PATH_MAX PATH_MAX #elif defined(_XOPEN_PATH_MAX) diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index bb20a9b012..c5ab751c17 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -38,7 +38,14 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee int slen = 0; *ppTsdb = NULL; - slen = strlen(pVnode->path) + strlen(dir) + 2; + slen = TD_PATH_MAX; //strlen(pVnode->path) + strlen(dir) + 3; + + // if (slen > TD_PATH_MAX) { + // terrno = TSDB_CODE_OUT_OF_RANGE; + // tsdbError("vgId:%d, tsdb open failed since %s, path: %s%s%s", TD_VID(pVnode), terrstr(), pVnode->path, TD_DIRSEP, + // dir); + // return -1; + // } // create handle pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen); From cd12ed33a2ee595ddece72a4c0cf5c8f0c56ba1e Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 1 Nov 2022 17:09:56 +0800 Subject: [PATCH 03/82] enh: code optimization for tsdb path --- source/dnode/vnode/src/tsdb/tsdbOpen.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index c5ab751c17..ffeaea0ce9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -38,14 +38,14 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee int slen = 0; *ppTsdb = NULL; - slen = TD_PATH_MAX; //strlen(pVnode->path) + strlen(dir) + 3; + slen = strlen(pVnode->path) + strlen(dir) + 3; - // if (slen > TD_PATH_MAX) { - // terrno = TSDB_CODE_OUT_OF_RANGE; - // tsdbError("vgId:%d, tsdb open failed since %s, path: %s%s%s", TD_VID(pVnode), terrstr(), pVnode->path, TD_DIRSEP, - // dir); - // return -1; - // } + if (slen > TD_PATH_MAX) { + terrno = TSDB_CODE_OUT_OF_RANGE; + tsdbError("vgId:%d, tsdb open failed since %s, path: %s%s%s", TD_VID(pVnode), terrstr(), pVnode->path, TD_DIRSEP, + dir); + return -1; + } // create handle pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen); From 4ef8676aaf20c8a7ffbdec5f9bf930199171ace3 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 2 Nov 2022 17:11:12 +0800 Subject: [PATCH 04/82] fix: invalid read of tsdb path --- include/os/osDef.h | 10 ---------- include/os/osFile.h | 10 ++++++++++ source/dnode/vnode/src/tsdb/tsdbOpen.c | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/include/os/osDef.h b/include/os/osDef.h index f5a79934bf..297d19e21a 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -244,16 +244,6 @@ void syslog(int unused, const char *format, ...); #define TD_CHARSET_LEN 64 #define TD_TIMEZONE_LEN 96 -#ifdef WINDOWS -#define TD_PATH_MAX _MAX_PATH -#elif defined(PATH_MAX) -#define TD_PATH_MAX PATH_MAX -#elif defined(_XOPEN_PATH_MAX) -#define TD_PATH_MAX _XOPEN_PATH_MAX -#else -#define TD_PATH_MAX _POSIX_PATH_MAX -#endif - #ifdef __cplusplus } #endif diff --git a/include/os/osFile.h b/include/os/osFile.h index 21e3d2e6cf..f6759d19a7 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -42,6 +42,16 @@ extern "C" { #define PATH_MAX 256 #endif +#ifdef WINDOWS +#define TD_PATH_MAX _MAX_PATH +#elif defined(PATH_MAX) +#define TD_PATH_MAX PATH_MAX +#elif defined(_XOPEN_PATH_MAX) +#define TD_PATH_MAX _XOPEN_PATH_MAX +#else +#define TD_PATH_MAX _POSIX_PATH_MAX +#endif + typedef struct TdFile *TdFilePtr; #define TD_FILE_CREATE 0x0001 diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index ffeaea0ce9..e109edca11 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -38,7 +38,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee int slen = 0; *ppTsdb = NULL; - slen = strlen(pVnode->path) + strlen(dir) + 3; + slen = TD_PATH_MAX; if (slen > TD_PATH_MAX) { terrno = TSDB_CODE_OUT_OF_RANGE; From 11c4878ca13de5e510c329a0fab9f5dc25163817 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Nov 2022 10:01:11 +0800 Subject: [PATCH 05/82] refactor(query): do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 1 - source/libs/executor/src/executil.c | 4 -- source/libs/executor/src/sortoperator.c | 49 +++++++++++++-------- 3 files changed, 31 insertions(+), 23 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 0b614fd2db..01fbcf657f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -279,7 +279,6 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t (*pIter)->pBlockLoadInfo = pBlockLoadInfo; -// size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); if (!pBlockLoadInfo->sttBlockLoaded) { int64_t st = taosGetTimestampUs(); pBlockLoadInfo->sttBlockLoaded = true; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 67405aad92..77b90e8728 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1482,10 +1482,6 @@ void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray while (i < numOfSrcCols && j < taosArrayGetSize(pColMatchInfo)) { SColumnInfoData* p = taosArrayGet(pCols, i); SColMatchItem* pmInfo = taosArrayGet(pColMatchInfo, j); - /* if (!outputEveryColumn && pmInfo->reserved) { - j++; - continue; - }*/ if (p->info.colId == pmInfo->colId) { SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, pmInfo->dstSlotId); diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 26f1932b12..5b05b3b2ed 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -579,21 +579,8 @@ int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } -SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, - SArray* pColMatchInfo, SOperatorInfo* pOperator) { - SMultiwayMergeOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; +static void doGetSortedBlockData(SMultiwayMergeOperatorInfo* pInfo, SSortHandle* pHandle, int32_t capacity, SSDataBlock* p) { - blockDataCleanup(pDataBlock); - - SSDataBlock* p = tsortGetSortedDataBlock(pHandle); - if (p == NULL) { - return NULL; - } - - blockDataEnsureCapacity(p, capacity); - -_retry: while (1) { STupleHandle* pTupleHandle = NULL; if (pInfo->groupSort) { @@ -638,22 +625,48 @@ _retry: pInfo->hasGroupId = false; } - if (p->info.rows > 0) { // todo extract method - applyLimitOffset(&pInfo->limitInfo, p, pTaskInfo, pOperator); +} + +SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, + SArray* pColMatchInfo, SOperatorInfo* pOperator) { + SMultiwayMergeOperatorInfo* pInfo = pOperator->info; + + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + blockDataCleanup(pDataBlock); + + SSDataBlock* p = tsortGetSortedDataBlock(pHandle); + if (p == NULL) { + return NULL; + } + + blockDataEnsureCapacity(p, capacity); + + while (1) { + doGetSortedBlockData(pInfo, pHandle, capacity, p); if (p->info.rows == 0) { - goto _retry; + break; } + if (p->info.rows > 0) { + applyLimitOffset(&pInfo->limitInfo, p, pTaskInfo, pOperator); + if (p->info.rows > 0) { + break; + } + } + } + + if (p->info.rows > 0) { blockDataEnsureCapacity(pDataBlock, p->info.rows); + int32_t numOfCols = taosArrayGetSize(pColMatchInfo); for (int32_t i = 0; i < numOfCols; ++i) { SColMatchItem* pmInfo = taosArrayGet(pColMatchInfo, i); - // ASSERT(pColMatchInfo-> == COL_MATCH_FROM_SLOT_ID); SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->dstSlotId); colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info); } + pInfo->limitInfo.numOfOutputRows += p->info.rows; pDataBlock->info.rows = p->info.rows; pDataBlock->info.groupId = pInfo->groupId; From 53e1c19acb6763c3de9f2806fdb4e3fb81079c0f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Nov 2022 10:28:52 +0800 Subject: [PATCH 06/82] enh(query): remove the memset. --- source/common/src/tdatablock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 0b27d9b527..322d9a1f75 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1182,7 +1182,7 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* return TSDB_CODE_OUT_OF_MEMORY; } - memset(tmp + pColumn->info.bytes * existedRows, 0, pColumn->info.bytes * (numOfRows - existedRows)); +// memset(tmp + pColumn->info.bytes * existedRows, 0, pColumn->info.bytes * (numOfRows - existedRows)); pColumn->pData = tmp; } From e83159f0d97e08d2c0e001d2dae22388982e20ea Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Nov 2022 10:47:55 +0800 Subject: [PATCH 07/82] refactor: increase the buffer for each agg operator, and make the pointer attributes in SColumnInfoData to be aligned to 8bit in memory to speedup the execution of memmove & memset. --- include/common/tcommon.h | 6 +++--- source/libs/executor/src/executorimpl.c | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 3bfbb85958..674bdcf171 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -225,13 +225,13 @@ typedef struct SVarColAttr { // pBlockAgg->numOfNull == info.rows, all data are null // pBlockAgg->numOfNull == 0, no data are null. typedef struct SColumnInfoData { - SColumnInfo info; // column info - bool hasNull; // if current column data has null value. - char* pData; // the corresponding block data in memory + char* pData; // the corresponding block data in memory union { char* nullbitmap; // bitmap, one bit for each item in the list SVarColAttr varmeta; }; + SColumnInfo info; // column info + bool hasNull; // if current column data has null value. } SColumnInfoData; typedef struct SQueryTableDataCond { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 13a3712f0c..59266716e1 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2783,8 +2783,10 @@ int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaul *defaultPgsz <<= 1u; } + // The default buffer for each operator in query is 10MB. // at least four pages need to be in buffer - *defaultBufsz = 4096 * 256; + // TODO: make this variable to be configurable. + *defaultBufsz = 4096 * 2560; if ((*defaultBufsz) <= (*defaultPgsz)) { (*defaultBufsz) = (*defaultPgsz) * 4; } From 7a14bc05ea11ae6ebba40d50d75569090b6e6646 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Nov 2022 10:57:05 +0800 Subject: [PATCH 08/82] fix(query): fix some issues about the initialization of SColumnInfoData. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- source/libs/executor/src/executil.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5dbeaa3fc7..89d07911d9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -534,7 +534,7 @@ static SSDataBlock* createResBlock(SQueryTableDataCond* pCond, int32_t capacity) } for (int32_t i = 0; i < pCond->numOfCols; ++i) { - SColumnInfoData colInfo = {{0}, 0}; + SColumnInfoData colInfo = {0, {0}}; colInfo.info = pCond->colList[i]; blockDataAppendColInfo(pResBlock, &colInfo); } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 32386a72fd..cb65fb9197 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -421,7 +421,7 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray* } for (int32_t i = 0; i < taosArrayGetSize(ctx.cInfoList); ++i) { - SColumnInfoData colInfo = {{0}, 0}; + SColumnInfoData colInfo = {0, {0}}; colInfo.info = *(SColumnInfo*)taosArrayGet(ctx.cInfoList, i); blockDataAppendColInfo(pResBlock, &colInfo); } @@ -582,7 +582,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis } for (int32_t i = 0; i < taosArrayGetSize(ctx.cInfoList); ++i) { - SColumnInfoData colInfo = {{0}, 0}; + SColumnInfoData colInfo = {0, {0}}; colInfo.info = *(SColumnInfo*)taosArrayGet(ctx.cInfoList, i); blockDataAppendColInfo(pResBlock, &colInfo); } From d282e511e3b30f6223f975325d138c6b81126107 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 3 Nov 2022 11:47:15 +0800 Subject: [PATCH 09/82] fix: fix taosd memory leak issue --- source/libs/executor/src/groupoperator.c | 8 ++++++++ source/libs/qworker/src/qworker.c | 6 ++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 98f8d57fc6..e07a3475e0 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -750,6 +750,14 @@ static void destroyPartitionOperatorInfo(void* param) { taosArrayDestroy(pInfo->pGroupColVals); taosMemoryFree(pInfo->keyBuf); taosArrayDestroy(pInfo->sortedGroupArray); + + void* pGroupIter = taosHashIterate(pInfo->pGroupSet, NULL); + while (pGroupIter != NULL) { + SDataGroupInfo* pGroupInfo = pGroupIter; + taosArrayDestroy(pGroupInfo->pPageList); + pGroupIter = taosHashIterate(pInfo->pGroupSet, pGroupIter); + } + taosHashCleanup(pInfo->pGroupSet); taosMemoryFree(pInfo->columnOffset); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 001a1972a0..a7cd3db824 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -644,11 +644,13 @@ _return: input.msgType = qwMsg->msgType; code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL); - if (ctx != NULL && QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { + if (QUERY_RSP_POLICY_QUICK == tsQueryRspPolicy && ctx != NULL && QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { void *rsp = NULL; int32_t dataLen = 0; SOutputData sOutput = {0}; - QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); + if (qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)) { + return TSDB_CODE_SUCCESS; + } if (rsp) { bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); From f17d1d1e312d3373b4ffc463e281853eb33cb7e4 Mon Sep 17 00:00:00 2001 From: Huo Linhe Date: Thu, 3 Nov 2022 12:25:01 +0800 Subject: [PATCH 10/82] fix: fix packaging condition test in the scripts Close [TD-20142](https://jira.taosdata.com:18080/browse/TD-20142) --- packaging/tools/makepkg.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index e9546ba103..f687abb123 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -285,10 +285,14 @@ if [[ $dbName == "taos" ]]; then fi # Add web files - if [ -d "${web_dir}/admin" ]; then - mkdir -p ${install_dir}/share/ - cp ${web_dir}/admin ${install_dir}/share/ -r - cp ${web_dir}/png/taos.png ${install_dir}/share/admin/images/taos.png + if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then + if [ -d "${web_dir}/admin" ] ; then + mkdir -p ${install_dir}/share/ + cp ${web_dir}/admin ${install_dir}/share/ -r + cp ${web_dir}/png/taos.png ${install_dir}/share/admin/images/taos.png + else + echo "directory not found for enterprise release: ${web_dir}/admin" + fi fi fi From 2fbc8e630698ce6742910be6bf06731cd0a73c6c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 3 Nov 2022 13:22:23 +0800 Subject: [PATCH 11/82] refactor(stream): stream deploy and state transfer --- include/common/tmsgdef.h | 1 + include/libs/stream/tstream.h | 94 ++- include/util/taoserror.h | 7 +- source/client/src/clientTmq.c | 2 +- source/dnode/mgmt/mgmt_snode/src/smWorker.c | 2 +- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 4 +- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 7 +- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 2 +- source/dnode/mnode/impl/src/mndStream.c | 3 +- source/dnode/mnode/impl/src/mndSubscribe.c | 6 +- source/dnode/snode/src/snode.c | 4 +- source/dnode/vnode/inc/vnode.h | 4 +- source/dnode/vnode/src/inc/vnodeInt.h | 30 +- source/dnode/vnode/src/tq/tq.c | 174 ++++-- source/dnode/vnode/src/vnd/vnodeSvr.c | 24 +- source/libs/executor/src/scanoperator.c | 12 +- source/libs/stream/inc/streamInc.h | 2 + source/libs/stream/src/stream.c | 11 +- source/libs/stream/src/streamDispatch.c | 45 +- source/libs/stream/src/streamExec.c | 34 +- source/libs/stream/src/streamRecover.c | 581 ++++++------------ source/util/src/terror.c | 3 + tests/script/jenkins/basic.txt | 2 + .../script/tsim/stream/fillHistoryBasic1.sim | 7 +- .../script/tsim/stream/fillHistoryBasic2.sim | 277 +++++++++ .../script/tsim/stream/fillHistoryBasic3.sim | 203 ++++++ 26 files changed, 986 insertions(+), 555 deletions(-) create mode 100644 tests/script/tsim/stream/fillHistoryBasic2.sim create mode 100644 tests/script/tsim/stream/fillHistoryBasic3.sim diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 70145b434d..36966779de 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -241,6 +241,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_STREAM_UNUSED1, "stream-unused1", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE, "stream-retrieve", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_RECOVER_FINISH, "vnode-stream-finish", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_CHECK, "vnode-stream-task-check", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL) TD_NEW_MSG_SEG(TDMT_MON_MSG) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index e2616567f5..1c3f905e23 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -46,7 +46,7 @@ enum { TASK_STATUS__DROPPING, TASK_STATUS__FAIL, TASK_STATUS__STOP, - TASK_STATUS__RECOVER_DOWNSTREAM, + TASK_STATUS__WAIT_DOWNSTREAM, TASK_STATUS__RECOVER_PREPARE, TASK_STATUS__RECOVER1, TASK_STATUS__RECOVER2, @@ -332,7 +332,10 @@ typedef struct SStreamTask { SStreamState* pState; // do not serialize - int32_t recoverWaitingChild; + int32_t recoverTryingDownstream; + int32_t recoverWaitingUpstream; + int64_t checkReqId; + SArray* checkReqIds; // shuffle } SStreamTask; @@ -418,7 +421,10 @@ typedef struct { typedef struct { int64_t streamId; - int32_t taskId; + int32_t upstreamNodeId; + int32_t upstreamTaskId; + int32_t downstreamNodeId; + int32_t downstreamTaskId; int8_t inputStatus; } SStreamDispatchRsp; @@ -440,6 +446,27 @@ typedef struct { int32_t rspToTaskId; } SStreamRetrieveRsp; +typedef struct { + int64_t reqId; + int64_t streamId; + int32_t upstreamNodeId; + int32_t upstreamTaskId; + int32_t downstreamNodeId; + int32_t downstreamTaskId; + int32_t childId; +} SStreamTaskCheckReq; + +typedef struct { + int64_t reqId; + int64_t streamId; + int32_t upstreamNodeId; + int32_t upstreamTaskId; + int32_t downstreamNodeId; + int32_t downstreamTaskId; + int32_t childId; + int8_t status; +} SStreamTaskCheckRsp; + typedef struct { SMsgHead msgHead; int64_t streamId; @@ -455,47 +482,6 @@ typedef struct { int32_t tEncodeSStreamRecoverFinishReq(SEncoder* pEncoder, const SStreamRecoverFinishReq* pReq); int32_t tDecodeSStreamRecoverFinishReq(SDecoder* pDecoder, SStreamRecoverFinishReq* pReq); -#if 0 -typedef struct { - int64_t streamId; - int32_t taskId; - int32_t upstreamTaskId; - int32_t upstreamNodeId; -} SStreamTaskRecoverReq; - -typedef struct { - int64_t streamId; - int32_t rspTaskId; - int32_t reqTaskId; - int8_t inputStatus; -} SStreamTaskRecoverRsp; - -int32_t tEncodeStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamTaskRecoverReq* pReq); -int32_t tDecodeStreamTaskRecoverReq(SDecoder* pDecoder, SStreamTaskRecoverReq* pReq); - -int32_t tEncodeStreamTaskRecoverRsp(SEncoder* pEncoder, const SStreamTaskRecoverRsp* pRsp); -int32_t tDecodeStreamTaskRecoverRsp(SDecoder* pDecoder, SStreamTaskRecoverRsp* pRsp); - -typedef struct { - int64_t streamId; - int32_t taskId; -} SMStreamTaskRecoverReq; - -typedef struct { - int64_t streamId; - int32_t taskId; -} SMStreamTaskRecoverRsp; - -int32_t tEncodeSMStreamTaskRecoverReq(SEncoder* pEncoder, const SMStreamTaskRecoverReq* pReq); -int32_t tDecodeSMStreamTaskRecoverReq(SDecoder* pDecoder, SMStreamTaskRecoverReq* pReq); - -int32_t tEncodeSMStreamTaskRecoverRsp(SEncoder* pEncoder, const SMStreamTaskRecoverRsp* pRsp); -int32_t tDecodeSMStreamTaskRecoverRsp(SDecoder* pDecoder, SMStreamTaskRecoverRsp* pRsp); - -int32_t streamProcessRecoverReq(SStreamTask* pTask, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg); -int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp); -#endif - typedef struct { int64_t streamId; int32_t downstreamTaskId; @@ -509,20 +495,18 @@ typedef struct { SArray* checkpointVer; // SArray } SStreamRecoverDownstreamRsp; +int32_t tEncodeSStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq* pReq); +int32_t tDecodeSStreamTaskCheckReq(SDecoder* pDecoder, SStreamTaskCheckReq* pReq); + +int32_t tEncodeSStreamTaskCheckRsp(SEncoder* pEncoder, const SStreamTaskCheckRsp* pRsp); +int32_t tDecodeSStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp); + int32_t tEncodeSStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamRecoverDownstreamReq* pReq); int32_t tDecodeSStreamTaskRecoverReq(SDecoder* pDecoder, SStreamRecoverDownstreamReq* pReq); int32_t tEncodeSStreamTaskRecoverRsp(SEncoder* pEncoder, const SStreamRecoverDownstreamRsp* pRsp); int32_t tDecodeSStreamTaskRecoverRsp(SDecoder* pDecoder, SStreamRecoverDownstreamRsp* pRsp); -typedef struct { - int64_t streamId; - int32_t taskId; - int32_t waitingRspCnt; - int32_t totReq; - SArray* info; // SArray*> -} SStreamRecoverStatus; - int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq); int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq); void tDeleteStreamRetrieveReq(SStreamRetrieveReq* pReq); @@ -533,7 +517,7 @@ int32_t streamSetupTrigger(SStreamTask* pTask); int32_t streamProcessRunReq(SStreamTask* pTask); int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pMsg, bool exec); -int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp); +int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code); int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pMsg); int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp); @@ -544,6 +528,10 @@ int32_t streamSchedExec(SStreamTask* pTask); int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz); // recover and fill history +int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version); +int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version); +int32_t streamProcessTaskCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* pReq); +int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp, int64_t version); // common int32_t streamSetParamForRecover(SStreamTask* pTask); int32_t streamRestoreParam(SStreamTask* pTask); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 563820f60b..2365469066 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -635,9 +635,12 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TMQ_CONSUMER_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x4001) #define TSDB_CODE_TMQ_CONSUMER_CLOSED TAOS_DEF_ERROR_CODE(0, 0x4002) +// stream +#define TSDB_CODE_STREAM_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x4100) + // TDLite -#define TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS TAOS_DEF_ERROR_CODE(0, 0x4100) -#define TSDB_CODE_TDLITE_IVLD_OPEN_DIR TAOS_DEF_ERROR_CODE(0, 0x4101) +#define TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS TAOS_DEF_ERROR_CODE(0, 0x5100) +#define TSDB_CODE_TDLITE_IVLD_OPEN_DIR TAOS_DEF_ERROR_CODE(0, 0x5101) #ifdef __cplusplus } diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index ba639476d8..8e7faf48f6 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1756,7 +1756,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { rspObj = tmqHandleAllRsp(tmq, timeout, false); if (rspObj) { - tscDebug("consumer:%" PRId64 ", return rsp", tmq->consumerId); + tscDebug("consumer:%" PRId64 ", return rsp %p", tmq->consumerId, rspObj); return (TAOS_RES*)rspObj; } else if (terrno == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) { tscDebug("consumer:%" PRId64 ", return null since no committed offset", tmq->consumerId); diff --git a/source/dnode/mgmt/mgmt_snode/src/smWorker.c b/source/dnode/mgmt/mgmt_snode/src/smWorker.c index 6a70527541..8300b13476 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smWorker.c +++ b/source/dnode/mgmt/mgmt_snode/src/smWorker.c @@ -58,7 +58,7 @@ static void smProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { dTrace("msg:%p, get from snode-stream queue", pMsg); int32_t code = sndProcessStreamMsg(pMgmt->pSnode, pMsg); if (code < 0) { - dGError("snd, msg:%p failed to process stream since %s", pMsg, terrstr(code)); + dGError("snd, msg:%p failed to process stream msg %s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr(code)); smSendRsp(pMsg, terrno); } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index e2812e0d31..6faa3354ee 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -403,7 +403,6 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_COMMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_HEARTBEAT, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TRIGGER, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; @@ -412,6 +411,9 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RECOVER_FINISH, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECK, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECK_RSP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TRIGGER, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 71e2ece733..3dfcb8a708 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -86,7 +86,8 @@ static void vmProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo); if (code != 0) { if (terrno != 0) code = terrno; - dGError("vgId:%d, msg:%p failed to process stream since %s", pVnode->vgId, pMsg, terrstr(code)); + dGError("vgId:%d, msg:%p failed to process stream msg %s since %s", pVnode->vgId, pMsg, TMSG_INFO(pMsg->msgType), + terrstr(code)); vmSendRsp(pMsg, code); } @@ -218,7 +219,9 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp return code; } -int32_t vmPutMsgToSyncCtrlQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, SYNC_CTRL_QUEUE); } +int32_t vmPutMsgToSyncCtrlQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return vmPutMsgToQueue(pMgmt, pMsg, SYNC_CTRL_QUEUE); +} int32_t vmPutMsgToSyncQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, SYNC_QUEUE); } diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 7ad24be258..4334bcfb48 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -155,7 +155,7 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { _OVER: if (code != 0) { if (terrno != 0) code = terrno; - dGTrace("msg:%p, failed to process since %s", pMsg, terrstr()); + dGTrace("msg:%p, failed to process %s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr()); if (IsReq(pRpc)) { SRpcMsg rsp = {.code = code, .info = pRpc->info}; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 9f433f3322..36ba0aaf87 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -705,7 +705,8 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { return -1; } - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-stream"); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, "drop-stream"); + mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb); if (pTrans == NULL) { mError("stream:%s, failed to drop since %s", dropReq.name, terrstr()); sdbRelease(pMnode->pSdb, pStream); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 74f2b1288e..a62f15f978 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -612,14 +612,14 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { mndReleaseSubscribe(pMnode, pSub); } - // TODO replace assert with error check - ASSERT(mndDoRebalance(pMnode, &rebInput, &rebOutput) == 0); + if (mndDoRebalance(pMnode, &rebInput, &rebOutput) < 0) { + mError("mq rebalance internal error"); + } // if add more consumer to balanced subscribe, // possibly no vg is changed /*ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0);*/ - // TODO replace assert with error check if (mndPersistRebResult(pMnode, pMsg, &rebOutput) < 0) { mError("mq rebalance persist rebalance output error, possibly vnode splitted or dropped"); } diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index d34159d312..08ea4808c5 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -231,10 +231,10 @@ int32_t sndProcessTaskRetrieveReq(SSnode *pSnode, SRpcMsg *pMsg) { int32_t sndProcessTaskDispatchRsp(SSnode *pSnode, SRpcMsg *pMsg) { SStreamDispatchRsp *pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t taskId = pRsp->taskId; + int32_t taskId = pRsp->upstreamTaskId; SStreamTask *pTask = streamMetaGetTask(pSnode->pMeta, taskId); if (pTask) { - streamProcessDispatchRsp(pTask, pRsp); + streamProcessDispatchRsp(pTask, pRsp, pMsg->code); return 0; } else { return -1; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 020d2b6049..370103c222 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -158,7 +158,7 @@ int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableL void tsdbReaderClose(STsdbReader *pReader); bool tsdbNextDataBlock(STsdbReader *pReader); bool tsdbTableNextDataBlock(STsdbReader *pReader, uint64_t uid); -void tsdbRetrieveDataBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64_t* uid, STimeWindow* pWindow); +void tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow); int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave); SArray *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); @@ -240,7 +240,7 @@ bool tqNextDataBlock(STqReader *pReader); bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader); -void vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg); +int32_t vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg); // sma int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 9ec8bb2cfd..44ecf64419 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -150,19 +150,19 @@ typedef struct { int32_t metaGetStbStats(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo); // tsdb -int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback); -int tsdbClose(STsdb** pTsdb); -int32_t tsdbBegin(STsdb* pTsdb); -int32_t tsdbCommit(STsdb* pTsdb); -int32_t tsdbFinishCommit(STsdb* pTsdb); -int32_t tsdbRollbackCommit(STsdb* pTsdb); -int32_t tsdbDoRetention(STsdb* pTsdb, int64_t now); -int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg); -int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); -int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, - SSubmitBlkRsp* pRsp); -int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey); -int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg); +int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback); +int tsdbClose(STsdb** pTsdb); +int32_t tsdbBegin(STsdb* pTsdb); +int32_t tsdbCommit(STsdb* pTsdb); +int32_t tsdbFinishCommit(STsdb* pTsdb); +int32_t tsdbRollbackCommit(STsdb* pTsdb); +int32_t tsdbDoRetention(STsdb* pTsdb, int64_t now); +int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg); +int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); +int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, + SSubmitBlkRsp* pRsp); +int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey); +int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg); // tq int tqInit(); @@ -183,13 +183,13 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); // tq-stream int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); +int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* data, int64_t ver); int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver); int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec); int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg); -// int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg); -// int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 094db9ebd0..9c377fe7f5 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -896,6 +896,10 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { pTask->startVer = ver; // expand executor + if (pTask->fillHistory) { + pTask->taskStatus = TASK_STATUS__WAIT_DOWNSTREAM; + } + if (pTask->taskLevel == TASK_LEVEL__SOURCE) { pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1); if (pTask->pState == NULL) { @@ -911,9 +915,6 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); ASSERT(pTask->exec.executor); - if (pTask->fillHistory) { - pTask->taskStatus = TASK_STATUS__RECOVER_PREPARE; - } } else if (pTask->taskLevel == TASK_LEVEL__AGG) { pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1); if (pTask->pState == NULL) { @@ -947,11 +948,90 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { streamSetupTrigger(pTask); - tqInfo("expand stream task on vg %d, task id %d, child id %d", TD_VID(pTq->pVnode), pTask->taskId, - pTask->selfChildId); + tqInfo("expand stream task on vg %d, task id %d, child id %d, level %d", TD_VID(pTq->pVnode), pTask->taskId, + pTask->selfChildId, pTask->taskLevel); return 0; } +int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { + char* msgStr = pMsg->pCont; + char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + SStreamTaskCheckReq req; + SDecoder decoder; + tDecoderInit(&decoder, msgBody, msgLen); + tDecodeSStreamTaskCheckReq(&decoder, &req); + tDecoderClear(&decoder); + int32_t taskId = req.downstreamTaskId; + SStreamTaskCheckRsp rsp = { + .reqId = req.reqId, + .streamId = req.streamId, + .childId = req.childId, + .downstreamNodeId = req.downstreamNodeId, + .downstreamTaskId = req.downstreamTaskId, + .upstreamNodeId = req.upstreamNodeId, + .upstreamTaskId = req.upstreamTaskId, + }; + SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); + if (pTask && atomic_load_8(&pTask->taskStatus) == TASK_STATUS__NORMAL) { + rsp.status = 1; + } else { + rsp.status = 0; + } + + tqDebug("tq recv task check req(reqId: %" PRId64 ") %d at node %d check req from task %d at node %d, status %d", + rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); + + SEncoder encoder; + int32_t code; + int32_t len; + tEncodeSize(tEncodeSStreamTaskCheckRsp, &rsp, len, code); + if (code < 0) { + ASSERT(0); + } + void* buf = rpcMallocCont(sizeof(SMsgHead) + len); + ((SMsgHead*)buf)->vgId = htonl(req.upstreamNodeId); + + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tEncoderInit(&encoder, (uint8_t*)abuf, len); + tEncodeSStreamTaskCheckRsp(&encoder, &rsp); + tEncoderClear(&encoder); + + SRpcMsg rspMsg = { + .code = 0, + .pCont = buf, + .contLen = sizeof(SMsgHead) + len, + .info = pMsg->info, + }; + + tmsgSendRsp(&rspMsg); + return 0; +} + +int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t version, char* msg, int32_t msgLen) { + int32_t code; + SStreamTaskCheckRsp rsp; + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msg, msgLen); + code = tDecodeSStreamTaskCheckRsp(&decoder, &rsp); + if (code < 0) { + tDecoderClear(&decoder); + return -1; + } + tDecoderClear(&decoder); + + tqDebug("tq recv task check rsp(reqId: %" PRId64 ") %d at node %d check req from task %d at node %d, status %d", + rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); + + SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, rsp.upstreamTaskId); + if (pTask == NULL) { + return -1; + } + + return streamProcessTaskCheckRsp(pTask, &rsp, version); +} + int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) { int32_t code; #if 0 @@ -982,37 +1062,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msg // 3.go through recover steps to fill history if (pTask->fillHistory) { - if (pTask->taskLevel == TASK_LEVEL__SOURCE) { - streamSetParamForRecover(pTask); - streamSourceRecoverPrepareStep1(pTask, version); - - SStreamRecoverStep1Req req; - streamBuildSourceRecover1Req(pTask, &req); - int32_t len = sizeof(SStreamRecoverStep1Req); - - void* serializedReq = rpcMallocCont(len); - if (serializedReq == NULL) { - return -1; - } - - memcpy(serializedReq, &req, len); - - SRpcMsg rpcMsg = { - .contLen = len, - .pCont = serializedReq, - .msgType = TDMT_VND_STREAM_RECOVER_STEP1, - }; - - if (tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &rpcMsg) < 0) { - /*ASSERT(0);*/ - } - - } else if (pTask->taskLevel == TASK_LEVEL__AGG) { - streamSetParamForRecover(pTask); - streamAggRecoverPrepare(pTask); - } else if (pTask->taskLevel == TASK_LEVEL__SINK) { - // do nothing - } + streamTaskCheckDownstream(pTask, version); } return 0; @@ -1268,7 +1318,7 @@ int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* pReq, int64_t ver) { if (pIter == NULL) break; SStreamTask* pTask = *(SStreamTask**)pIter; if (pTask->taskLevel != TASK_LEVEL__SOURCE) continue; - if (pTask->taskStatus == TASK_STATUS__RECOVER_PREPARE) { + if (pTask->taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) { tqDebug("skip push task %d, task status %d", pTask->taskId, pTask->taskStatus); continue; } @@ -1335,10 +1385,11 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) { int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t taskId = pRsp->taskId; + int32_t taskId = ntohl(pRsp->upstreamTaskId); SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); + tqDebug("recv dispatch rsp, code: %x", pMsg->code); if (pTask) { - streamProcessDispatchRsp(pTask, pRsp); + streamProcessDispatchRsp(pTask, pRsp, pMsg->code); return 0; } else { return -1; @@ -1379,12 +1430,12 @@ int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg) { return 0; } -void vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) { - STQ* pTq = pVnode->pTq; - char* msgStr = pMsg->pCont; - char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); - int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); - int32_t code = 0; +int32_t vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) { + STQ* pTq = pVnode->pTq; + SMsgHead* msgStr = pMsg->pCont; + char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + int32_t code = 0; SStreamDispatchReq req; SDecoder decoder; @@ -1407,16 +1458,45 @@ void vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) { streamProcessDispatchReq(pTask, &req, &rsp, false); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); - return; + return 0; } + code = TSDB_CODE_STREAM_TASK_NOT_EXIST; + FAIL: - if (pMsg->info.handle == NULL) return; + if (pMsg->info.handle == NULL) return -1; + + SMsgHead* pRspHead = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamDispatchRsp)); + if (pRspHead == NULL) { + SRpcMsg rsp = { + .code = TSDB_CODE_OUT_OF_MEMORY, + .info = pMsg->info, + }; + tqDebug("send dispatch error rsp, code: %x", code); + tmsgSendRsp(&rsp); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); + return -1; + } + + pRspHead->vgId = htonl(req.upstreamNodeId); + SStreamDispatchRsp* pRsp = POINTER_SHIFT(pRspHead, sizeof(SMsgHead)); + pRsp->streamId = htobe64(req.streamId); + pRsp->upstreamTaskId = htonl(req.upstreamTaskId); + pRsp->upstreamNodeId = htonl(req.upstreamNodeId); + pRsp->downstreamNodeId = htonl(pVnode->config.vgId); + pRsp->downstreamTaskId = htonl(req.taskId); + pRsp->inputStatus = TASK_OUTPUT_STATUS__NORMAL; + SRpcMsg rsp = { .code = code, .info = pMsg->info, + .contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp), + .pCont = pRspHead, }; + tqDebug("send dispatch error rsp, code: %x", code); tmsgSendRsp(&rsp); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); + return -1; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index d6c77afa66..f1c4ab76d1 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -229,8 +229,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp break; /* TQ */ case TDMT_VND_TMQ_SUBSCRIBE: - if (tqProcessSubscribeReq(pVnode->pTq, version, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), - pMsg->contLen - sizeof(SMsgHead)) < 0) { + if (tqProcessSubscribeReq(pVnode->pTq, version, pReq, len) < 0) { goto _err; } break; @@ -240,26 +239,22 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp } break; case TDMT_VND_TMQ_COMMIT_OFFSET: - if (tqProcessOffsetCommitReq(pVnode->pTq, version, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), - pMsg->contLen - sizeof(SMsgHead)) < 0) { + if (tqProcessOffsetCommitReq(pVnode->pTq, version, pReq, pMsg->contLen - sizeof(SMsgHead)) < 0) { goto _err; } break; case TDMT_VND_TMQ_ADD_CHECKINFO: - if (tqProcessAddCheckInfoReq(pVnode->pTq, version, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), - pMsg->contLen - sizeof(SMsgHead)) < 0) { + if (tqProcessAddCheckInfoReq(pVnode->pTq, version, pReq, len) < 0) { goto _err; } break; case TDMT_VND_TMQ_DEL_CHECKINFO: - if (tqProcessDelCheckInfoReq(pVnode->pTq, version, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), - pMsg->contLen - sizeof(SMsgHead)) < 0) { + if (tqProcessDelCheckInfoReq(pVnode->pTq, version, pReq, len) < 0) { goto _err; } break; case TDMT_STREAM_TASK_DEPLOY: { - if (tqProcessTaskDeployReq(pVnode->pTq, version, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), - pMsg->contLen - sizeof(SMsgHead)) < 0) { + if (tqProcessTaskDeployReq(pVnode->pTq, version, pReq, len) < 0) { goto _err; } } break; @@ -273,6 +268,11 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp goto _err; } } break; + case TDMT_STREAM_TASK_CHECK_RSP: { + if (tqProcessStreamTaskCheckRsp(pVnode->pTq, version, pReq, len) < 0) { + goto _err; + } + } break; case TDMT_VND_ALTER_CONFIRM: vnodeProcessAlterConfirmReq(pVnode, version, pReq, len, pRsp); break; @@ -388,10 +388,12 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { return tqProcessPollReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_RUN: return tqProcessTaskRunReq(pVnode->pTq, pMsg); -#if 0 +#if 1 case TDMT_STREAM_TASK_DISPATCH: return tqProcessTaskDispatchReq(pVnode->pTq, pMsg, true); #endif + case TDMT_STREAM_TASK_CHECK: + return tqProcessStreamTaskCheckReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_DISPATCH_RSP: return tqProcessTaskDispatchRsp(pVnode->pTq, pMsg); case TDMT_STREAM_RETRIEVE: diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e54b889a7a..d596eab2f8 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1884,11 +1884,15 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; memcpy(&pTSInfo->cond, &pTaskInfo->streamInfo.tableCond, sizeof(SQueryTableDataCond)); if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE1) { - pTSInfo->cond.startVersion = -1; + pTSInfo->cond.startVersion = 0; pTSInfo->cond.endVersion = pTaskInfo->streamInfo.fillHistoryVer1; + qDebug("stream recover step 1, from %" PRId64 " to %" PRId64, pTSInfo->cond.startVersion, + pTSInfo->cond.endVersion); } else { pTSInfo->cond.startVersion = pTaskInfo->streamInfo.fillHistoryVer1 + 1; pTSInfo->cond.endVersion = pTaskInfo->streamInfo.fillHistoryVer2; + qDebug("stream recover step 2, from %" PRId64 " to %" PRId64, pTSInfo->cond.startVersion, + pTSInfo->cond.endVersion); } /*resetTableScanInfo(pTSInfo, pWin);*/ @@ -1905,11 +1909,15 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { if (pBlock != NULL) { calBlockTbName(&pInfo->tbnameCalSup, pBlock); updateInfoFillBlockData(pInfo->pUpdateInfo, pBlock, pInfo->primaryTsIndex); + qDebug("stream recover scan get block, rows %d", pBlock->info.rows); return pBlock; } pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE; STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - pTSInfo->cond.startVersion = 0; + tsdbReaderClose(pTSInfo->dataReader); + pTSInfo->dataReader = NULL; + + pTSInfo->cond.startVersion = -1; pTSInfo->cond.endVersion = -1; return NULL; diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h index a8f7184bb2..6a3bdb59c9 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -43,6 +43,8 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq); +int32_t streamDispatchOneCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet); + int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecoverFinishReq* pReq, int32_t vgId, SEpSet* pEpSet); diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index ee317d0751..b71562cf45 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -135,8 +135,11 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, const SStreamDispatchReq* pReq, SR ((SMsgHead*)buf)->vgId = htonl(pReq->upstreamNodeId); SStreamDispatchRsp* pCont = POINTER_SHIFT(buf, sizeof(SMsgHead)); pCont->inputStatus = status; - pCont->streamId = pReq->streamId; - pCont->taskId = pReq->upstreamTaskId; + pCont->streamId = htobe64(pReq->streamId); + pCont->upstreamNodeId = htonl(pReq->upstreamNodeId); + pCont->upstreamTaskId = htonl(pReq->upstreamTaskId); + pCont->downstreamNodeId = htonl(pTask->nodeId); + pCont->downstreamTaskId = htonl(pTask->taskId); pRsp->pCont = buf; pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp); tmsgSendRsp(pRsp); @@ -203,10 +206,10 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S return 0; } -int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp) { +int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code) { ASSERT(pRsp->inputStatus == TASK_OUTPUT_STATUS__NORMAL || pRsp->inputStatus == TASK_OUTPUT_STATUS__BLOCKED); - qDebug("task %d receive dispatch rsp", pTask->taskId); + qDebug("task %d receive dispatch rsp, code: %x", pTask->taskId, code); if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { int32_t leftRsp = atomic_sub_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index d2876a22c6..ad342edfa0 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -210,6 +210,46 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis return 0; } +int32_t streamDispatchOneCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet) { + void* buf = NULL; + int32_t code = -1; + SRpcMsg msg = {0}; + + int32_t tlen; + tEncodeSize(tEncodeSStreamTaskCheckReq, pReq, tlen, code); + if (code < 0) { + return -1; + } + + buf = rpcMallocCont(sizeof(SMsgHead) + tlen); + if (buf == NULL) { + return -1; + } + + ((SMsgHead*)buf)->vgId = htonl(nodeId); + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + + SEncoder encoder; + tEncoderInit(&encoder, abuf, tlen); + if ((code = tEncodeSStreamTaskCheckReq(&encoder, pReq)) < 0) { + goto FAIL; + } + tEncoderClear(&encoder); + + msg.contLen = tlen + sizeof(SMsgHead); + msg.pCont = buf; + msg.msgType = TDMT_STREAM_TASK_CHECK; + + qDebug("dispatch from task %d to task %d node %d: check msg", pTask->taskId, pReq->downstreamTaskId, nodeId); + + tmsgSendReq(pEpSet, &msg); + + return 0; +FAIL: + if (buf) rpcFreeCont(buf); + return code; +} + int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecoverFinishReq* pReq, int32_t vgId, SEpSet* pEpSet) { void* buf = NULL; @@ -243,7 +283,8 @@ int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecov tmsgSendReq(pEpSet, &msg); - code = 0; + qDebug("dispatch from task %d to task %d node %d: recover finish msg", pTask->taskId, pReq->taskId, vgId); + return 0; FAIL: if (buf) rpcFreeCont(buf); @@ -279,7 +320,7 @@ int32_t streamDispatchOneDataReq(SStreamTask* pTask, const SStreamDispatchReq* p msg.pCont = buf; msg.msgType = pTask->dispatchMsgType; - qDebug("dispatch from task %d to task %d node %d", pTask->taskId, pReq->taskId, vgId); + qDebug("dispatch from task %d to task %d node %d: data msg", pTask->taskId, pReq->taskId, vgId); tmsgSendReq(pEpSet, &msg); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 629333b4b4..46fab53659 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -202,83 +202,83 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum, bool dispatch) int32_t streamExecForAll(SStreamTask* pTask) { while (1) { int32_t batchCnt = 1; - void* data = NULL; + void* input = NULL; while (1) { SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue); if (qItem == NULL) { qDebug("stream task exec over, queue empty, task: %d", pTask->taskId); break; } - if (data == NULL) { - data = qItem; + if (input == NULL) { + input = qItem; streamQueueProcessSuccess(pTask->inputQueue); if (pTask->taskLevel == TASK_LEVEL__SINK) { break; } } else { void* newRet; - if ((newRet = streamMergeQueueItem(data, qItem)) == NULL) { + if ((newRet = streamMergeQueueItem(input, qItem)) == NULL) { streamQueueProcessFail(pTask->inputQueue); break; } else { batchCnt++; - data = newRet; + input = newRet; streamQueueProcessSuccess(pTask->inputQueue); } } } if (pTask->taskStatus == TASK_STATUS__DROPPING) { - if (data) streamFreeQitem(data); + if (input) streamFreeQitem(input); return 0; } - if (data == NULL) { + if (input == NULL) { break; } if (pTask->taskLevel == TASK_LEVEL__SINK) { - ASSERT(((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_BLOCK); - streamTaskOutput(pTask, data); + ASSERT(((SStreamQueueItem*)input)->type == STREAM_INPUT__DATA_BLOCK); + streamTaskOutput(pTask, input); continue; } SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); qDebug("stream task %d exec begin, msg batch: %d", pTask->taskId, batchCnt); - streamTaskExecImpl(pTask, data, pRes); + streamTaskExecImpl(pTask, input, pRes); qDebug("stream task %d exec end", pTask->taskId); if (taosArrayGetSize(pRes) != 0) { SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); if (qRes == NULL) { taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - streamFreeQitem(data); + streamFreeQitem(input); return -1; } qRes->type = STREAM_INPUT__DATA_BLOCK; qRes->blocks = pRes; - if (((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; + if (((SStreamQueueItem*)input)->type == STREAM_INPUT__DATA_SUBMIT) { + SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)input; qRes->childId = pTask->selfChildId; qRes->sourceVer = pSubmit->ver; - } else if (((SStreamQueueItem*)data)->type == STREAM_INPUT__MERGED_SUBMIT) { - SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)data; + } else if (((SStreamQueueItem*)input)->type == STREAM_INPUT__MERGED_SUBMIT) { + SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)input; qRes->childId = pTask->selfChildId; qRes->sourceVer = pMerged->ver; } if (streamTaskOutput(pTask, qRes) < 0) { taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - streamFreeQitem(data); + streamFreeQitem(input); taosFreeQitem(qRes); return -1; } } else { taosArrayDestroy(pRes); } - streamFreeQitem(data); + streamFreeQitem(input); } return 0; } diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index adeb797721..2a2784afea 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -15,6 +15,148 @@ #include "streamInc.h" +int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version) { + qDebug("task %d at node %d launch recover", pTask->taskId, pTask->nodeId); + if (pTask->taskLevel == TASK_LEVEL__SOURCE) { + atomic_store_8(&pTask->taskStatus, TASK_STATUS__RECOVER_PREPARE); + streamSetParamForRecover(pTask); + streamSourceRecoverPrepareStep1(pTask, version); + + SStreamRecoverStep1Req req; + streamBuildSourceRecover1Req(pTask, &req); + int32_t len = sizeof(SStreamRecoverStep1Req); + + void* serializedReq = rpcMallocCont(len); + if (serializedReq == NULL) { + return -1; + } + + memcpy(serializedReq, &req, len); + + SRpcMsg rpcMsg = { + .contLen = len, + .pCont = serializedReq, + .msgType = TDMT_VND_STREAM_RECOVER_STEP1, + }; + + if (tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &rpcMsg) < 0) { + /*ASSERT(0);*/ + } + + } else if (pTask->taskLevel == TASK_LEVEL__AGG) { + atomic_store_8(&pTask->taskStatus, TASK_STATUS__NORMAL); + streamSetParamForRecover(pTask); + streamAggRecoverPrepare(pTask); + } else if (pTask->taskLevel == TASK_LEVEL__SINK) { + atomic_store_8(&pTask->taskStatus, TASK_STATUS__NORMAL); + } + return 0; +} + +// checkstatus +int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) { + SStreamTaskCheckReq req = { + .streamId = pTask->streamId, + .upstreamTaskId = pTask->taskId, + .upstreamNodeId = pTask->nodeId, + .childId = pTask->selfChildId, + }; + // serialize + if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { + req.reqId = tGenIdPI64(); + req.downstreamNodeId = pTask->fixedEpDispatcher.nodeId; + req.downstreamTaskId = pTask->fixedEpDispatcher.taskId; + pTask->checkReqId = req.reqId; + + qDebug("task %d at node %d check downstream task %d at node %d", pTask->taskId, pTask->nodeId, req.downstreamTaskId, + req.downstreamNodeId); + streamDispatchOneCheckReq(pTask, &req, pTask->fixedEpDispatcher.nodeId, &pTask->fixedEpDispatcher.epSet); + } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; + int32_t vgSz = taosArrayGetSize(vgInfo); + pTask->recoverTryingDownstream = vgSz; + pTask->checkReqIds = taosArrayInit(vgSz, sizeof(int64_t)); + + for (int32_t i = 0; i < vgSz; i++) { + SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); + req.reqId = tGenIdPI64(); + taosArrayPush(pTask->checkReqIds, &req.reqId); + req.downstreamNodeId = pVgInfo->vgId; + req.downstreamTaskId = pVgInfo->taskId; + qDebug("task %d at node %d check downstream task %d at node %d (shuffle)", pTask->taskId, pTask->nodeId, + req.downstreamTaskId, req.downstreamNodeId); + streamDispatchOneCheckReq(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); + } + } else { + qDebug("task %d at node %d direct launch recover since no downstream", pTask->taskId, pTask->nodeId); + streamTaskLaunchRecover(pTask, version); + } + return 0; +} + +int32_t streamRecheckOneDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp) { + SStreamTaskCheckReq req = { + .reqId = pRsp->reqId, + .streamId = pRsp->streamId, + .upstreamTaskId = pRsp->upstreamTaskId, + .upstreamNodeId = pRsp->upstreamNodeId, + .downstreamTaskId = pRsp->downstreamTaskId, + .downstreamNodeId = pRsp->downstreamNodeId, + .childId = pRsp->childId, + }; + qDebug("task %d at node %d check downstream task %d at node %d (recheck)", pTask->taskId, pTask->nodeId, + req.downstreamTaskId, req.downstreamNodeId); + if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { + streamDispatchOneCheckReq(pTask, &req, pRsp->downstreamNodeId, &pTask->fixedEpDispatcher.epSet); + } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; + int32_t vgSz = taosArrayGetSize(vgInfo); + for (int32_t i = 0; i < vgSz; i++) { + SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); + if (pVgInfo->taskId == req.downstreamTaskId) { + streamDispatchOneCheckReq(pTask, &req, pRsp->downstreamNodeId, &pVgInfo->epSet); + } + } + } + return 0; +} + +int32_t streamProcessTaskCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* pReq) { + return atomic_load_8(&pTask->taskStatus) == TASK_STATUS__NORMAL; +} + +int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp, int64_t version) { + qDebug("task %d at node %d recv check rsp from task %d at node %d: status %d", pRsp->upstreamTaskId, + pRsp->upstreamNodeId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->status); + if (pRsp->status == 1) { + if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + bool found = false; + for (int32_t i = 0; i < taosArrayGetSize(pTask->checkReqIds); i++) { + int64_t reqId = *(int64_t*)taosArrayGet(pTask->checkReqIds, i); + if (reqId == pRsp->reqId) { + found = true; + break; + } + } + if (!found) return -1; + int32_t left = atomic_sub_fetch_32(&pTask->recoverTryingDownstream, 1); + ASSERT(left >= 0); + if (left == 0) { + taosArrayDestroy(pTask->checkReqIds); + streamTaskLaunchRecover(pTask, version); + } + } else if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { + if (pRsp->reqId != pTask->checkReqId) return -1; + streamTaskLaunchRecover(pTask, version); + } else { + ASSERT(0); + } + } else { + streamRecheckOneDownstream(pTask, pRsp); + } + return 0; +} + // common int32_t streamSetParamForRecover(SStreamTask* pTask) { void* exec = pTask->exec.executor; @@ -86,10 +228,7 @@ int32_t streamDispatchRecoverFinishReq(SStreamTask* pTask) { // agg int32_t streamAggRecoverPrepare(SStreamTask* pTask) { void* exec = pTask->exec.executor; - /*if (qStreamSetParamForRecover(exec) < 0) {*/ - /*return -1;*/ - /*}*/ - pTask->recoverWaitingChild = taosArrayGetSize(pTask->childEpInfo); + pTask->recoverWaitingUpstream = taosArrayGetSize(pTask->childEpInfo); return 0; } @@ -107,7 +246,7 @@ int32_t streamAggChildrenRecoverFinish(SStreamTask* pTask) { int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t childId) { if (pTask->taskLevel == TASK_LEVEL__AGG) { - int32_t left = atomic_sub_fetch_32(&pTask->recoverWaitingChild, 1); + int32_t left = atomic_sub_fetch_32(&pTask->recoverWaitingUpstream, 1); ASSERT(left >= 0); if (left == 0) { streamAggChildrenRecoverFinish(pTask); @@ -116,6 +255,60 @@ int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t childId) { return 0; } +int32_t tEncodeSStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq* pReq) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->reqId) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->upstreamNodeId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->upstreamTaskId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->downstreamNodeId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->downstreamTaskId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->childId) < 0) return -1; + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeSStreamTaskCheckReq(SDecoder* pDecoder, SStreamTaskCheckReq* pReq) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->reqId) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->upstreamNodeId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->upstreamTaskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->downstreamNodeId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->downstreamTaskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->childId) < 0) return -1; + tEndDecode(pDecoder); + return 0; +} + +int32_t tEncodeSStreamTaskCheckRsp(SEncoder* pEncoder, const SStreamTaskCheckRsp* pRsp) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->reqId) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->upstreamNodeId) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->upstreamTaskId) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->downstreamNodeId) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->downstreamTaskId) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->childId) < 0) return -1; + if (tEncodeI8(pEncoder, pRsp->status) < 0) return -1; + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeSStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->reqId) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->upstreamNodeId) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->upstreamTaskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->downstreamNodeId) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->downstreamTaskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->childId) < 0) return -1; + if (tDecodeI8(pDecoder, &pRsp->status) < 0) return -1; + tEndDecode(pDecoder); + return 0; +} + int32_t tEncodeSStreamRecoverFinishReq(SEncoder* pEncoder, const SStreamRecoverFinishReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; @@ -132,79 +325,6 @@ int32_t tDecodeSStreamRecoverFinishReq(SDecoder* pDecoder, SStreamRecoverFinishR tEndDecode(pDecoder); return 0; } -#if 0 -int32_t tEncodeStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamTaskRecoverReq* pReq) { - if (tStartEncode(pEncoder) < 0) return -1; - if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; - if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1; - if (tEncodeI32(pEncoder, pReq->upstreamTaskId) < 0) return -1; - if (tEncodeI32(pEncoder, pReq->upstreamNodeId) < 0) return -1; - tEndEncode(pEncoder); - return pEncoder->pos; -} - -int32_t tDecodeStreamTaskRecoverReq(SDecoder* pDecoder, SStreamTaskRecoverReq* pReq) { - if (tStartDecode(pDecoder) < 0) return -1; - if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; - if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1; - if (tDecodeI32(pDecoder, &pReq->upstreamTaskId) < 0) return -1; - if (tDecodeI32(pDecoder, &pReq->upstreamNodeId) < 0) return -1; - tEndDecode(pDecoder); - return 0; -} - -int32_t tEncodeStreamTaskRecoverRsp(SEncoder* pEncoder, const SStreamTaskRecoverRsp* pRsp) { - if (tStartEncode(pEncoder) < 0) return -1; - if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1; - if (tEncodeI32(pEncoder, pRsp->reqTaskId) < 0) return -1; - if (tEncodeI32(pEncoder, pRsp->rspTaskId) < 0) return -1; - if (tEncodeI8(pEncoder, pRsp->inputStatus) < 0) return -1; - tEndEncode(pEncoder); - return pEncoder->pos; -} - -int32_t tDecodeStreamTaskRecoverRsp(SDecoder* pDecoder, SStreamTaskRecoverRsp* pReq) { - if (tStartDecode(pDecoder) < 0) return -1; - if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; - if (tDecodeI32(pDecoder, &pReq->reqTaskId) < 0) return -1; - if (tDecodeI32(pDecoder, &pReq->rspTaskId) < 0) return -1; - if (tDecodeI8(pDecoder, &pReq->inputStatus) < 0) return -1; - tEndDecode(pDecoder); - return 0; -} - -int32_t tEncodeSMStreamTaskRecoverReq(SEncoder* pEncoder, const SMStreamTaskRecoverReq* pReq) { - if (tStartEncode(pEncoder) < 0) return -1; - if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; - if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1; - tEndEncode(pEncoder); - return pEncoder->pos; -} - -int32_t tDecodeSMStreamTaskRecoverReq(SDecoder* pDecoder, SMStreamTaskRecoverReq* pReq) { - if (tStartDecode(pDecoder) < 0) return -1; - if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; - if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1; - tEndDecode(pDecoder); - return 0; -} - -int32_t tEncodeSMStreamTaskRecoverRsp(SEncoder* pEncoder, const SMStreamTaskRecoverRsp* pRsp) { - if (tStartEncode(pEncoder) < 0) return -1; - if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1; - if (tEncodeI32(pEncoder, pRsp->taskId) < 0) return -1; - tEndEncode(pEncoder); - return pEncoder->pos; -} - -int32_t tDecodeSMStreamTaskRecoverRsp(SDecoder* pDecoder, SMStreamTaskRecoverRsp* pReq) { - if (tStartDecode(pDecoder) < 0) return -1; - if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; - if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1; - tEndDecode(pDecoder); - return 0; -} -#endif int32_t tEncodeSStreamCheckpointInfo(SEncoder* pEncoder, const SStreamCheckpointInfo* pCheckpoint) { if (tEncodeI32(pEncoder, pCheckpoint->srcNodeId) < 0) return -1; @@ -248,308 +368,3 @@ int32_t tDecodeSStreamMultiVgCheckpointInfo(SDecoder* pDecoder, SStreamMultiVgCh } return 0; } - -#if 0 -int32_t tEncodeSStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamRecoverDownstreamReq* pReq) { - if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; - if (tEncodeI32(pEncoder, pReq->downstreamTaskId) < 0) return -1; - if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1; - return 0; -} - -int32_t tDecodeSStreamTaskRecoverReq(SDecoder* pDecoder, SStreamRecoverDownstreamReq* pReq) { - if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; - if (tDecodeI32(pDecoder, &pReq->downstreamTaskId) < 0) return -1; - if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1; - return 0; -} - -int32_t tEncodeSStreamTaskRecoverRsp(SEncoder* pEncoder, const SStreamRecoverDownstreamRsp* pRsp) { - if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1; - if (tEncodeI32(pEncoder, pRsp->downstreamTaskId) < 0) return -1; - if (tEncodeI32(pEncoder, pRsp->taskId) < 0) return -1; - int32_t sz = taosArrayGetSize(pRsp->checkpointVer); - if (tEncodeI32(pEncoder, sz) < 0) return -1; - for (int32_t i = 0; i < sz; i++) { - SStreamCheckpointInfo* pInfo = taosArrayGet(pRsp->checkpointVer, i); - if (tEncodeSStreamCheckpointInfo(pEncoder, pInfo) < 0) return -1; - } - return 0; -} - -int32_t tDecodeSStreamTaskRecoverRsp(SDecoder* pDecoder, SStreamRecoverDownstreamRsp* pRsp) { - if (tDecodeI64(pDecoder, &pRsp->streamId) < 0) return -1; - if (tDecodeI32(pDecoder, &pRsp->downstreamTaskId) < 0) return -1; - if (tDecodeI32(pDecoder, &pRsp->taskId) < 0) return -1; - int32_t sz; - if (tDecodeI32(pDecoder, &sz) < 0) return -1; - pRsp->checkpointVer = taosArrayInit(sz, sizeof(SStreamCheckpointInfo)); - if (pRsp->checkpointVer == NULL) return -1; - for (int32_t i = 0; i < sz; i++) { - SStreamCheckpointInfo info; - if (tDecodeSStreamCheckpointInfo(pDecoder, &info) < 0) return -1; - taosArrayPush(pRsp->checkpointVer, &info); - } - return 0; -} -#endif - -int32_t streamSaveStateInfo(SStreamMeta* pMeta, SStreamTask* pTask) { -#if 0 - void* buf = NULL; - - ASSERT(pTask->taskLevel == TASK_LEVEL__SINK); - - SStreamMultiVgCheckpointInfo checkpoint; - checkpoint.checkpointId = atomic_fetch_add_32(&pTask->nextCheckId, 1); - checkpoint.checkTs = taosGetTimestampMs(); - checkpoint.streamId = pTask->streamId; - checkpoint.taskId = pTask->taskId; - checkpoint.checkpointVer = pTask->checkpointInfo; - - int32_t len; - int32_t code; - tEncodeSize(tEncodeSStreamMultiVgCheckpointInfo, &checkpoint, len, code); - if (code < 0) { - return -1; - } - - buf = taosMemoryCalloc(1, len); - if (buf == NULL) { - return -1; - } - SEncoder encoder; - tEncoderInit(&encoder, buf, len); - tEncodeSStreamMultiVgCheckpointInfo(&encoder, &checkpoint); - tEncoderClear(&encoder); - - SStreamCheckpointKey key = { - .taskId = pTask->taskId, - .checkpointId = checkpoint.checkpointId, - }; - - if (tdbTbUpsert(pMeta->pStateDb, &key, sizeof(SStreamCheckpointKey), buf, len, &pMeta->txn) < 0) { - ASSERT(0); - goto FAIL; - } - - int32_t sz = taosArrayGetSize(pTask->checkpointInfo); - for (int32_t i = 0; i < sz; i++) { - SStreamCheckpointInfo* pCheck = taosArrayGet(pTask->checkpointInfo, i); - pCheck->stateSaveVer = pCheck->stateProcessedVer; - } - - taosMemoryFree(buf); - return 0; -FAIL: - if (buf) taosMemoryFree(buf); - return -1; -#endif - return 0; -} - -int32_t streamLoadStateInfo(SStreamMeta* pMeta, SStreamTask* pTask) { -#if 0 - void* pVal = NULL; - int32_t vLen = 0; - if (tdbTbGet(pMeta->pStateDb, &pTask->taskId, sizeof(void*), &pVal, &vLen) < 0) { - return -1; - } - SDecoder decoder; - tDecoderInit(&decoder, pVal, vLen); - SStreamMultiVgCheckpointInfo aggCheckpoint; - tDecodeSStreamMultiVgCheckpointInfo(&decoder, &aggCheckpoint); - tDecoderClear(&decoder); - - pTask->nextCheckId = aggCheckpoint.checkpointId + 1; - pTask->checkpointInfo = aggCheckpoint.checkpointVer; -#endif - return 0; -} - -int32_t streamSaveSinkLevel(SStreamMeta* pMeta, SStreamTask* pTask) { - ASSERT(pTask->taskLevel == TASK_LEVEL__SINK); - return streamSaveStateInfo(pMeta, pTask); -} - -int32_t streamRecoverSinkLevel(SStreamMeta* pMeta, SStreamTask* pTask) { - ASSERT(pTask->taskLevel == TASK_LEVEL__SINK); - return streamLoadStateInfo(pMeta, pTask); -} - -int32_t streamSaveAggLevel(SStreamMeta* pMeta, SStreamTask* pTask) { - ASSERT(pTask->taskLevel == TASK_LEVEL__AGG); - // TODO save and copy state - - // save state info - if (streamSaveStateInfo(pMeta, pTask) < 0) { - return -1; - } - return 0; -} - -#if 0 -int32_t streamFetchRecoverStatus(SStreamTask* pTask, const SVgroupInfo* pVgInfo) { - int32_t taskId = pVgInfo->taskId; - int32_t nodeId = pVgInfo->vgId; - SStreamRecoverDownstreamReq req = { - .streamId = pTask->taskId, - .downstreamTaskId = taskId, - .taskId = pTask->taskId, - }; - int32_t tlen; - int32_t code; - tEncodeSize(tEncodeSStreamTaskRecoverReq, &req, tlen, code); - if (code < 0) { - return -1; - } - void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + tlen); - if (buf == NULL) { - return -1; - } - void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - SEncoder encoder; - tEncoderInit(&encoder, abuf, tlen); - if (tEncodeSStreamTaskRecoverReq(&encoder, &req) < 0) { - tEncoderClear(&encoder); - taosMemoryFree(buf); - return -1; - } - tEncoderClear(&encoder); - - ((SMsgHead*)buf)->vgId = htonl(nodeId); - SRpcMsg msg = { - .pCont = buf, .contLen = sizeof(SMsgHead) + tlen, - /*.msgType = */ - }; - tmsgSendReq(&pVgInfo->epSet, &msg); - - return 0; -} - -int32_t streamFetchDownstreamStatus(SStreamMeta* pMeta, SStreamTask* pTask) { - // set self status to recover_phase1 - SStreamRecoverStatus* pRecover; - atomic_store_8(&pTask->taskStatus, TASK_STATUS__RECOVER_DOWNSTREAM); - pRecover = taosHashGet(pMeta->pRecoverStatus, &pTask->taskId, sizeof(int32_t)); - if (pRecover == NULL) { - pRecover = taosMemoryCalloc(1, sizeof(SStreamRecoverStatus)); - if (pRecover == NULL) { - return -1; - } - pRecover->info = taosArrayInit(0, sizeof(void*)); - if (pRecover->info == NULL) { - taosMemoryFree(pRecover); - return -1; - } - taosHashPut(pMeta->pRecoverStatus, &pTask->taskId, sizeof(int32_t), &pRecover, sizeof(void*)); - } - - if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { - pRecover->totReq = 1; - } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { - int32_t numOfDownstream = taosArrayGetSize(pTask->shuffleDispatcher.dbInfo.pVgroupInfos); - pRecover->totReq = numOfDownstream; - for (int32_t i = 0; i < numOfDownstream; i++) { - SVgroupInfo* pVgInfo = taosArrayGet(pTask->shuffleDispatcher.dbInfo.pVgroupInfos, i); - streamFetchRecoverStatus(pTask, pVgInfo); - } - } else { - ASSERT(0); - } - return 0; -} -#endif - -#if 0 -int32_t streamProcessFetchStatusRsp(SStreamMeta* pMeta, SStreamTask* pTask, SStreamRecoverDownstreamRsp* pRsp) { - // if failed, set timer and retry - // if successful - int32_t taskId = pTask->taskId; - SStreamRecoverStatus* pRecover = taosHashGet(pMeta->pRecoverStatus, &taskId, sizeof(int32_t)); - if (pRecover == NULL) { - return -1; - } - - taosArrayPush(pRecover->info, &pRsp->checkpointVer); - - int32_t leftRsp = atomic_sub_fetch_32(&pRecover->waitingRspCnt, 1); - ASSERT(leftRsp >= 0); - - if (leftRsp == 0) { - ASSERT(taosArrayGetSize(pRecover->info) == pRecover->totReq); - - // srcNodeId -> SStreamCheckpointInfo* - SHashObj* pFinalChecks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); - if (pFinalChecks == NULL) return -1; - - for (int32_t i = 0; i < pRecover->totReq; i++) { - SArray* pChecks = taosArrayGetP(pRecover->info, i); - int32_t sz = taosArrayGetSize(pChecks); - for (int32_t j = 0; j < sz; j++) { - SStreamCheckpointInfo* pOneCheck = taosArrayGet(pChecks, j); - SStreamCheckpointInfo* pCheck = taosHashGet(pFinalChecks, &pOneCheck->srcNodeId, sizeof(int32_t)); - if (pCheck == NULL) { - pCheck = taosMemoryCalloc(1, sizeof(SStreamCheckpointInfo)); - pCheck->srcNodeId = pOneCheck->srcNodeId; - pCheck->srcChildId = pOneCheck->srcChildId; - pCheck->stateProcessedVer = pOneCheck->stateProcessedVer; - taosHashPut(pFinalChecks, &pCheck->srcNodeId, sizeof(int32_t), &pCheck, sizeof(void*)); - } else { - pCheck->stateProcessedVer = TMIN(pCheck->stateProcessedVer, pOneCheck->stateProcessedVer); - } - } - } - // load local state - // - // recover - // - if (pTask->taskLevel == TASK_LEVEL__SOURCE) { - qStreamPrepareRecover(pTask->exec.executor, pTask->startVer, pTask->recoverSnapVer); - if (streamPipelineExec(pTask, 10000, true) < 0) { - return -1; - } - } - taosHashCleanup(pFinalChecks); - taosHashRemove(pMeta->pRecoverStatus, &taskId, sizeof(int32_t)); - atomic_store_8(&pTask->taskStatus, TASK_STATUS__NORMAL); - } - return 0; -} - -int32_t streamRecoverAggLevel(SStreamMeta* pMeta, SStreamTask* pTask) { - ASSERT(pTask->taskLevel == TASK_LEVEL__AGG); - // recover sink level - // after all sink level recovered - // choose suitable state to recover - return 0; -} - -int32_t streamSaveSourceLevel(SStreamMeta* pMeta, SStreamTask* pTask) { - ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); - // TODO: save and copy state - return 0; -} - -int32_t streamRecoverSourceLevel(SStreamMeta* pMeta, SStreamTask* pTask) { - ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); - // if totLevel == 3 - // fetch agg state - // recover from local state to agg state, not send msg - // recover from agg state to most recent log v1 - // enable input queue, set status recover_phase2 - // recover from v1 to queue msg v2, set status normal - - // if totLevel == 2 - // fetch sink state - // recover from local state to sink state v1, send msg - // enable input queue, set status recover_phase2 - // recover from v1 to queue msg v2, set status normal - return 0; -} - -int32_t streamRecoverTask(SStreamTask* pTask) { - // - return 0; -} -#endif diff --git a/source/util/src/terror.c b/source/util/src/terror.c index bbdf43c85e..7afb3c7260 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -633,6 +633,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_MSG, "Invalid message") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_MISMATCH, "Consumer mismatch") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_CLOSED, "Consumer closed") +// stream +TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_TASK_NOT_EXIST, "Stream task not exist") + // TDLite TAOS_DEFINE_ERROR(TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS, "Invalid TDLite open flags") TAOS_DEFINE_ERROR(TSDB_CODE_TDLITE_IVLD_OPEN_DIR, "Invalid TDLite open directory") diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 1a80aa0681..8584555333 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -235,6 +235,8 @@ ./test.sh -f tsim/stream/basic2.sim ./test.sh -f tsim/stream/drop_stream.sim ./test.sh -f tsim/stream/fillHistoryBasic1.sim +./test.sh -f tsim/stream/fillHistoryBasic2.sim +./test.sh -f tsim/stream/fillHistoryBasic3.sim ./test.sh -f tsim/stream/distributeInterval0.sim ./test.sh -f tsim/stream/distributeIntervalRetrive0.sim ./test.sh -f tsim/stream/distributeSession0.sim diff --git a/tests/script/tsim/stream/fillHistoryBasic1.sim b/tests/script/tsim/stream/fillHistoryBasic1.sim index 5bbaf1b712..4f4fa96bea 100644 --- a/tests/script/tsim/stream/fillHistoryBasic1.sim +++ b/tests/script/tsim/stream/fillHistoryBasic1.sim @@ -26,7 +26,7 @@ sql insert into t1 values(1648791243003,4,2,3,3.1); sql insert into t1 values(1648791213004,4,2,3,4.1); -sleep 1000 +sleep 5000 sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; if $rows != 4 then @@ -466,9 +466,6 @@ if $data25 != 3 then endi - - - sql create database test2 vgroups 1; sql select * from information_schema.ins_databases; @@ -484,7 +481,7 @@ sql insert into t1 values(1648791213004,4,2,3,4.1); sql create stream stream2 trigger at_once fill_history 1 into streamt as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s); -sleep 1000 +sleep 5000 sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; if $rows != 4 then diff --git a/tests/script/tsim/stream/fillHistoryBasic2.sim b/tests/script/tsim/stream/fillHistoryBasic2.sim new file mode 100644 index 0000000000..3af198259d --- /dev/null +++ b/tests/script/tsim/stream/fillHistoryBasic2.sim @@ -0,0 +1,277 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 + +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +sql create dnode $hostname2 port 7200 + +system sh/exec.sh -n dnode2 -s start + +print ===== step1 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql select * from information_schema.ins_dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 2 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi + +print ===== step2 +sql drop stream if exists stream_t1; +sql drop database if exists test; +sql create database test vgroups 4; +sql use test; +sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); +sql create table ts2 using st tags(2,2,2); +sql create table ts3 using st tags(3,2,2); +sql create table ts4 using st tags(4,2,2); + +sql insert into ts1 values(1648791213001,1,12,3,1.0); +sql insert into ts2 values(1648791213001,1,12,3,1.0); + +sql insert into ts3 values(1648791213001,1,12,3,1.0); +sql insert into ts4 values(1648791213001,1,12,3,1.0); + +sql insert into ts1 values(1648791213002,NULL,NULL,NULL,NULL); +sql insert into ts2 values(1648791213002,NULL,NULL,NULL,NULL); + +sql insert into ts3 values(1648791213002,NULL,NULL,NULL,NULL); +sql insert into ts4 values(1648791213002,NULL,NULL,NULL,NULL); + +sql insert into ts1 values(1648791223002,2,2,3,1.1); +sql insert into ts1 values(1648791233003,3,2,3,2.1); +sql insert into ts2 values(1648791243004,4,2,43,73.1); +sql insert into ts1 values(1648791213002,24,22,23,4.1); +sql insert into ts1 values(1648791243005,4,20,3,3.1); +sql insert into ts2 values(1648791243006,4,2,3,3.1) (1648791243007,4,2,3,3.1) ; +sql insert into ts1 values(1648791243008,4,2,30,3.1) (1648791243009,4,2,3,3.1) (1648791243010,4,2,3,3.1) ; +sql insert into ts2 values(1648791243011,4,2,3,3.1) (1648791243012,34,32,33,3.1) (1648791243013,4,2,3,3.1) (1648791243014,4,2,13,3.1); +sql insert into ts1 values(1648791243005,4,42,3,3.1) (1648791243003,4,2,33,3.1) (1648791243006,4,2,3,3.1) (1648791213001,1,52,13,1.0) (1648791223001,22,22,83,1.1) ; +sql insert into ts2 values(1648791243005,4,42,3,3.1) (1648791243003,4,2,33,3.1) (1648791243006,4,2,3,3.1) (1648791213001,1,52,13,1.0) (1648791223001,22,22,83,1.1) (1648791233004,13,12,13,2.1) ; +sql insert into ts1 values(1648791243006,4,2,3,3.1) (1648791213001,1,52,13,1.0) (1648791223001,22,22,83,1.1) ; + +sql insert into ts3 values(1648791223002,2,2,3,1.1); +sql insert into ts4 values(1648791233003,3,2,3,2.1); +sql insert into ts3 values(1648791243004,4,2,43,73.1); +sql insert into ts4 values(1648791213002,24,22,23,4.1); +sql insert into ts3 values(1648791243005,4,20,3,3.1); +sql insert into ts4 values(1648791243006,4,2,3,3.1) (1648791243007,4,2,3,3.1) ; +sql insert into ts3 values(1648791243008,4,2,30,3.1) (1648791243009,4,2,3,3.1) (1648791243010,4,2,3,3.1) ; +sql insert into ts4 values(1648791243011,4,2,3,3.1) (1648791243012,34,32,33,3.1) (1648791243013,4,2,3,3.1) (1648791243014,4,2,13,3.1); +sql insert into ts3 values(1648791243005,4,42,3,3.1) (1648791243003,4,2,33,3.1) (1648791243006,4,2,3,3.1) (1648791213001,1,52,13,1.0) (1648791223001,22,22,83,1.1) ; +sql insert into ts4 values(1648791243005,4,42,3,3.1) (1648791243003,4,2,33,3.1) (1648791243006,4,2,3,3.1) (1648791213001,1,52,13,1.0) (1648791223001,22,22,83,1.1) (1648791233004,13,12,13,2.1) ; +sql insert into ts3 values(1648791243006,4,2,3,3.1) (1648791213001,1,52,13,1.0) (1648791223001,22,22,83,1.1) ; + +sql create stream stream_t1 trigger at_once fill_history 1 watermark 1d into streamtST1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); + +sleep 1000 + +$loop_count = 0 +loop1: +sleep 300 +sql select * from streamtST1; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $data01 != 8 then + print =====data01=$data01 + goto loop1 +endi + +if $data02 != 6 then + print =====data02=$data02 + goto loop1 +endi + +if $data03 != 52 then + print ======data03=$data03 + goto loop1 +endi + +if $data04 != 52 then + print ======data04=$data04 + goto loop1 +endi + +if $data05 != 13 then + print ======data05=$data05 + goto loop1 +endi + +# row 1 +if $data11 != 6 then + print =====data11=$data11 + goto loop1 +endi + +if $data12 != 6 then + print =====data12=$data12 + goto loop1 +endi + +if $data13 != 92 then + print ======$data13 + return -1 +endi + +if $data14 != 22 then + print ======$data14 + return -1 +endi + +if $data15 != 3 then + print ======$data15 + return -1 +endi + +# row 2 +if $data21 != 4 then + print =====data21=$data21 + goto loop1 +endi + +if $data22 != 4 then + print =====data22=$data22 + goto loop1 +endi + +if $data23 != 32 then + print ======$data23 + return -1 +endi + +if $data24 != 12 then + print ======$data24 + return -1 +endi + +if $data25 != 3 then + print ======$data25 + return -1 +endi + +# row 3 +if $data31 != 30 then + print =====data31=$data31 + goto loop1 +endi + +if $data32 != 30 then + print =====data32=$data32 + goto loop1 +endi + +if $data33 != 180 then + print ======$data33 + return -1 +endi + +if $data34 != 42 then + print ======$data34 + return -1 +endi + +if $data35 != 3 then + print ======$data35 + return -1 +endi + +sql select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5, avg(d) from st interval(10s); + + +sql create database test1 vgroups 4; +sql use test1; +sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); +sql create table ts2 using st tags(2,2,2); + +sql insert into ts1 values(1648791211000,1,2,3); +sql insert into ts1 values(1648791222001,2,2,3); +sql insert into ts2 values(1648791211000,1,2,3); +sql insert into ts2 values(1648791222001,2,2,3); + +sql create stream stream_t2 trigger at_once fill_history 1 watermark 20s into streamtST1 as select _wstart, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6 from st interval(10s) ; + +$loop_count = 0 +loop2: +sql select * from streamtST1; + +sleep 300 +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $data01 != 2 then + print =====data01=$data01 + goto loop2 +endi + +#rows 1 +if $data11 != 2 then + print =====data11=$data11 + goto loop2 +endi + +#max,min selectivity +sql create database test3 vgroups 4; +sql use test3; +sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); +sql create table ts2 using st tags(2,2,2); +sql create stream stream_t3 trigger at_once into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from st interval(10s) ; + +sql insert into ts1 values(1648791211000,1,2,3); +sleep 50 +sql insert into ts1 values(1648791222001,2,2,3); +sleep 50 +sql insert into ts2 values(1648791211000,1,2,3); +sleep 50 +sql insert into ts2 values(1648791222001,2,2,3); +sleep 50 + +$loop_count = 0 +loop3: +sql select * from streamtST3; + +sleep 300 +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $data02 != 1 then + print =====data02=$data02 + goto loop3 +endi + +# row 1 +if $data12 != 2 then + print =====data12=$data12 + goto loop3 +endi + +system sh/stop_dnodes.sh diff --git a/tests/script/tsim/stream/fillHistoryBasic3.sim b/tests/script/tsim/stream/fillHistoryBasic3.sim new file mode 100644 index 0000000000..db8d6bc2d0 --- /dev/null +++ b/tests/script/tsim/stream/fillHistoryBasic3.sim @@ -0,0 +1,203 @@ +$loop_all = 0 +looptest: + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +sql create database test vgroups 1; +sql create database test2 vgroups 4; +sql use test2; +sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); + +sql create stream streams2 trigger at_once fill_history 1 into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a interval(10s); + +sleep 3000 + +$loop_count = 0 + +loop7: +sleep 50 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop7 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop7 +endi + +sql insert into t1 values(1648791213000,1,2,3,1.0); +sql insert into t2 values(1648791213000,1,2,3,1.0); + +$loop_count = 0 + +loop8: +sleep 50 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop8 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop8 +endi + +sql insert into t1 values(1648791213000,2,2,3,1.0); +sql insert into t2 values(1648791213000,2,2,3,1.0); + +$loop_count = 0 + +loop9: +sleep 50 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop9 +endi + +if $data02 != 2 then + print =====data02=$data02 + goto loop9 +endi + +sql insert into t1 values(1648791213000,2,2,3,1.0); +sql insert into t1 values(1648791213001,2,2,3,1.0); +sql insert into t1 values(1648791213002,2,2,3,1.0); +sql insert into t1 values(1648791213002,1,2,3,1.0); +sql insert into t2 values(1648791213000,2,2,3,1.0); +sql insert into t2 values(1648791213001,2,2,3,1.0); +sql insert into t2 values(1648791213002,2,2,3,1.0); +sql insert into t2 values(1648791213002,1,2,3,1.0); + +$loop_count = 0 + +loop10: +sleep 50 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop10 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop10 +endi + +if $data11 != 4 thenloop4 + print =====data11=$data11 + goto loop10 +endi + +if $data12 != 2 then + print =====data12=$data12 + goto loop10 +endi + +sql insert into t1 values(1648791223000,1,2,3,1.0); +sql insert into t1 values(1648791223001,1,2,3,1.0); +sql insert into t1 values(1648791223002,3,2,3,1.0); +sql insert into t1 values(1648791223003,3,2,3,1.0); +sql insert into t1 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0); +sql insert into t2 values(1648791223000,1,2,3,1.0); +sql insert into t2 values(1648791223001,1,2,3,1.0); +sql insert into t2 values(1648791223002,3,2,3,1.0); +sql insert into t2 values(1648791223003,3,2,3,1.0); +sql insert into t2 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0); + +$loop_count = 0 + +loop11: +sleep 50 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop11 +endi + +if $data02 != 2 then + print =====data02=$data02 + goto loop11 +endi + +if $data11 != 4 then + print =====data11=$data11 + goto loop11 +endi + +if $data12 != 1 then + print =====data12=$data12 + goto loop11 +endi + +if $data21 != 2 then + print =====data21=$data21 + goto loop11 +endi + +if $data22 != 2 then + print =====data22=$data22 + goto loop11 +endi + +if $data31 != 2 then + print =====data31=$data31 + goto loop11 +endi + +if $data32 != 3 then + print =====data32=$data32 + goto loop11 +endi + +if $data41 != 4 then + print =====data41=$data41 + goto loop11 +endi + +if $data42 != 1 then + print =====data42=$data42 + goto loop11 +endi From 9f7f11c362f3ab152d8e4734383f9e5b90db8c93 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 3 Nov 2022 13:57:07 +0800 Subject: [PATCH 12/82] docs/monitor --- docs/en/14-reference/12-config/index.md | 2 +- docs/zh/14-reference/12-config/index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 8053c4d25d..769d19e7f9 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -106,7 +106,7 @@ The parameters described in this document by the effect that they have on the sy | Applicable | Server only | | Meaning | The switch for monitoring inside server. The main object of monitoring is to collect information about load on physical nodes, including CPU usage, memory usage, disk usage, and network bandwidth. Monitoring information is sent over HTTP to the taosKeeper service specified by `monitorFqdn` and `monitorProt`. | Value Range | 0: monitoring disabled, 1: monitoring enabled | -| Default | 1 | +| Default | 0 | ### monitorFqdn diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 74af793f9f..d587dd2f2c 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -106,7 +106,7 @@ taos --dump-config | 适用范围 | 仅服务端适用 | | 含义 | 服务器内部的系统监控开关。监控主要负责收集物理节点的负载状况，包括 CPU、内存、硬盘、网络带宽的监控记录，监控信息将通过 HTTP 协议发送给由 `monitorFqdn` 和 `monitorProt` 指定的 TaosKeeper 监控服务 | | 取值范围 | 0：关闭监控服务， 1：激活监控服务。 | -| 缺省值 | 1 | +| 缺省值 | 0 | ### monitorFqdn From a4b54c4f0e8ef51fe5ef08e56f0f3055bb24e57a Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 3 Nov 2022 13:57:57 +0800 Subject: [PATCH 13/82] refactor(sync): add syncIsReadyForRead --- include/libs/sync/sync.h | 1 + source/dnode/vnode/src/vnd/vnodeSvr.c | 6 +++-- source/libs/sync/src/syncMain.c | 33 +++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index b6ff93ec85..8db3d89e39 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -219,6 +219,7 @@ int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex); int32_t syncEndSnapshot(int64_t rid); int32_t syncLeaderTransfer(int64_t rid); int32_t syncStepDown(int64_t rid, SyncTerm newTerm); +bool syncIsReadyForRead(int64_t rid); SSyncState syncGetState(int64_t rid); void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 521566b36d..435dbde30a 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -334,7 +334,8 @@ int32_t vnodePreprocessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { vTrace("message in vnode query queue is processing"); - if ((pMsg->msgType == TDMT_SCH_QUERY) && !vnodeIsLeader(pVnode)) { + // if ((pMsg->msgType == TDMT_SCH_QUERY) && !vnodeIsLeader(pVnode)) { + if ((pMsg->msgType == TDMT_SCH_QUERY) && !syncIsReadyForRead(pVnode->sync)) { vnodeRedirectRpcMsg(pVnode, pMsg); return 0; } @@ -356,7 +357,8 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { vTrace("vgId:%d, msg:%p in fetch queue is processing", pVnode->config.vgId, pMsg); if ((pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_VND_TABLE_META || pMsg->msgType == TDMT_VND_TABLE_CFG || pMsg->msgType == TDMT_VND_BATCH_META) && - !vnodeIsLeader(pVnode)) { + !syncIsReadyForRead(pVnode->sync)) { + // !vnodeIsLeader(pVnode)) { vnodeRedirectRpcMsg(pVnode, pMsg); return 0; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 5cd1ba3025..268173e795 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -437,6 +437,39 @@ int32_t syncStepDown(int64_t rid, SyncTerm newTerm) { return 0; } +bool syncIsReadyForRead(int64_t rid) { + SSyncNode* pSyncNode = syncNodeAcquire(rid); + if (pSyncNode == NULL) { + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + return false; + } + ASSERT(rid == pSyncNode->rid); + + if (pSyncNode->state == TAOS_SYNC_STATE_LEADER && pSyncNode->restoreFinish) { + syncNodeRelease(pSyncNode); + return true; + } + + bool ready = false; + if (pSyncNode->state == TAOS_SYNC_STATE_LEADER && !pSyncNode->restoreFinish) { + if (!pSyncNode->pLogStore->syncLogIsEmpty(pSyncNode->pLogStore)) { + SSyncRaftEntry* pEntry = NULL; + int32_t code = pSyncNode->pLogStore->syncLogGetEntry( + pSyncNode->pLogStore, pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore), &pEntry); + if (code == 0 && pEntry != NULL) { + if (pEntry->originalRpcType == TDMT_SYNC_NOOP && pEntry->term == pSyncNode->pRaftStore->currentTerm) { + ready = true; + } + + syncEntryDestory(pEntry); + } + } + } + + syncNodeRelease(pSyncNode); + return ready; +} + int32_t syncNodeLeaderTransfer(SSyncNode* pSyncNode) { if (pSyncNode->peersNum == 0) { sDebug("only one replica, cannot leader transfer"); From 31f574359db02c8f921a1c130f0a0c897ce61085 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 3 Nov 2022 13:44:34 +0800 Subject: [PATCH 14/82] fix:log empty msg --- source/dnode/mgmt/mgmt_snode/src/smWorker.c | 6 +++- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 8 ++++-- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 6 +++- source/dnode/snode/src/snode.c | 2 +- .../script/tsim/stream/fillHistoryBasic1.sim | 28 +++++++++---------- 5 files changed, 31 insertions(+), 19 deletions(-) diff --git a/source/dnode/mgmt/mgmt_snode/src/smWorker.c b/source/dnode/mgmt/mgmt_snode/src/smWorker.c index 8300b13476..dbd081338a 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smWorker.c +++ b/source/dnode/mgmt/mgmt_snode/src/smWorker.c @@ -58,7 +58,11 @@ static void smProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { dTrace("msg:%p, get from snode-stream queue", pMsg); int32_t code = sndProcessStreamMsg(pMgmt->pSnode, pMsg); if (code < 0) { - dGError("snd, msg:%p failed to process stream msg %s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr(code)); + if (pMsg) { + dGError("snd, msg:%p failed to process stream msg %s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr(code)); + } else { + dGError("snd, msg:%p failed to process stream empty msg since %s", pMsg, terrstr(code)); + } smSendRsp(pMsg, terrno); } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index f1e867a78a..40d0e32c2b 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -86,8 +86,12 @@ static void vmProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo); if (code != 0) { if (terrno != 0) code = terrno; - dGError("vgId:%d, msg:%p failed to process stream msg %s since %s", pVnode->vgId, pMsg, TMSG_INFO(pMsg->msgType), - terrstr(code)); + if (pMsg) { + dGError("vgId:%d, msg:%p failed to process stream msg %s since %s", pVnode->vgId, pMsg, TMSG_INFO(pMsg->msgType), + terrstr(code)); + } else { + dGError("vgId:%d, msg:%p failed to process stream empty msg since %s", pVnode->vgId, pMsg, terrstr(code)); + } vmSendRsp(pMsg, code); } diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 408244bfb5..1e5f3139aa 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -157,7 +157,11 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { _OVER: if (code != 0) { if (terrno != 0) code = terrno; - dGTrace("msg:%p, failed to process %s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr()); + if (pMsg) { + dGTrace("msg:%p, failed to process %s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr()); + } else { + dGTrace("msg:%p, failed to process empty msg since %s", pMsg, terrstr()); + } if (IsReq(pRpc)) { SRpcMsg rsp = {.code = code, .info = pRpc->info}; diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 08ea4808c5..f53350c10b 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -231,7 +231,7 @@ int32_t sndProcessTaskRetrieveReq(SSnode *pSnode, SRpcMsg *pMsg) { int32_t sndProcessTaskDispatchRsp(SSnode *pSnode, SRpcMsg *pMsg) { SStreamDispatchRsp *pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t taskId = pRsp->upstreamTaskId; + int32_t taskId = ntohl(pRsp->upstreamTaskId); SStreamTask *pTask = streamMetaGetTask(pSnode->pMeta, taskId); if (pTask) { streamProcessDispatchRsp(pTask, pRsp, pMsg->code); diff --git a/tests/script/tsim/stream/fillHistoryBasic1.sim b/tests/script/tsim/stream/fillHistoryBasic1.sim index 4f4fa96bea..772a09c017 100644 --- a/tests/script/tsim/stream/fillHistoryBasic1.sim +++ b/tests/script/tsim/stream/fillHistoryBasic1.sim @@ -139,7 +139,7 @@ if $data35 != 3 then endi sql insert into t1 values(1648791223001,12,14,13,11.1); -sleep 500 +sleep 1000 sql select * from streamt; print count(*) , count(d) , sum(a) , max(b) , min(c) @@ -256,7 +256,7 @@ if $data35 != 3 then endi sql insert into t1 values(1648791223002,12,14,13,11.1); -sleep 100 +sleep 1000 sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; # row 1 @@ -286,7 +286,7 @@ if $data15 != 13 then endi sql insert into t1 values(1648791223003,12,14,13,11.1); -sleep 100 +sleep 1000 sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; # row 1 @@ -318,7 +318,7 @@ endi sql insert into t1 values(1648791223001,1,1,1,1.1); sql insert into t1 values(1648791223002,2,2,2,2.1); sql insert into t1 values(1648791223003,3,3,3,3.1); -sleep 100 +sleep 1000 sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; # row 1 @@ -350,7 +350,7 @@ endi sql insert into t1 values(1648791233003,3,2,3,2.1); sql insert into t1 values(1648791233002,5,6,7,8.1); sql insert into t1 values(1648791233002,3,2,3,2.1); -sleep 100 +sleep 1000 sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; # row 2 @@ -380,7 +380,7 @@ if $data25 != 3 then endi sql insert into t1 values(1648791213004,4,2,3,4.1) (1648791213006,5,4,7,9.1) (1648791213004,40,20,30,40.1) (1648791213005,4,2,3,4.1); -sleep 100 +sleep 1000 sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; # row 0 @@ -410,7 +410,7 @@ if $data05 != 3 then endi sql insert into t1 values(1648791223004,4,2,3,4.1) (1648791233006,5,4,7,9.1) (1648791223004,40,20,30,40.1) (1648791233005,4,2,3,4.1); -sleep 100 +sleep 1000 sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; # row 1 @@ -594,7 +594,7 @@ if $data35 != 3 then endi sql insert into t1 values(1648791223001,12,14,13,11.1); -sleep 500 +sleep 1000 sql select * from streamt; print count(*) , count(d) , sum(a) , max(b) , min(c) @@ -711,7 +711,7 @@ if $data35 != 3 then endi sql insert into t1 values(1648791223002,12,14,13,11.1); -sleep 100 +sleep 1000 sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; # row 1 @@ -741,7 +741,7 @@ if $data15 != 13 then endi sql insert into t1 values(1648791223003,12,14,13,11.1); -sleep 100 +sleep 1000 sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; # row 1 @@ -773,7 +773,7 @@ endi sql insert into t1 values(1648791223001,1,1,1,1.1); sql insert into t1 values(1648791223002,2,2,2,2.1); sql insert into t1 values(1648791223003,3,3,3,3.1); -sleep 100 +sleep 1000 sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; # row 1 @@ -805,7 +805,7 @@ endi sql insert into t1 values(1648791233003,3,2,3,2.1); sql insert into t1 values(1648791233002,5,6,7,8.1); sql insert into t1 values(1648791233002,3,2,3,2.1); -sleep 100 +sleep 1000 sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; # row 2 @@ -835,7 +835,7 @@ if $data25 != 3 then endi sql insert into t1 values(1648791213004,4,2,3,4.1) (1648791213006,5,4,7,9.1) (1648791213004,40,20,30,40.1) (1648791213005,4,2,3,4.1); -sleep 100 +sleep 1000 sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; # row 0 @@ -865,7 +865,7 @@ if $data05 != 3 then endi sql insert into t1 values(1648791223004,4,2,3,4.1) (1648791233006,5,4,7,9.1) (1648791223004,40,20,30,40.1) (1648791233005,4,2,3,4.1); -sleep 100 +sleep 1000 sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; # row 1 From 0838afaa186ff79aea0e3f0b9e5251bb45faf426 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 3 Nov 2022 14:35:42 +0800 Subject: [PATCH 15/82] refactor(sync): add syncIsReadyForRead, set error code --- source/libs/sync/src/syncMain.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 268173e795..27f4eeedb2 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -466,6 +466,14 @@ bool syncIsReadyForRead(int64_t rid) { } } + if (!ready) { + if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { + terrno = TSDB_CODE_SYN_NOT_LEADER; + } else { + terrno = TSDB_CODE_APP_NOT_READY; + } + } + syncNodeRelease(pSyncNode); return ready; } From f09c7eed5d8296a5c50255041ff10e337e04839e Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Thu, 3 Nov 2022 15:07:01 +0800 Subject: [PATCH 16/82] build: release ver-3.0.1.6 --- packaging/release.bat | 4 ++-- packaging/tools/makepkg.sh | 14 +++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/packaging/release.bat b/packaging/release.bat index b87ae68e2b..4c82c5ead5 100644 --- a/packaging/release.bat +++ b/packaging/release.bat @@ -40,7 +40,7 @@ if not exist %work_dir%\debug\ver-%2-x86 ( ) cd %work_dir%\debug\ver-%2-x64 call vcvarsall.bat x64 -cmake ../../ -G "NMake Makefiles JOM" -DCMAKE_MAKE_PROGRAM=jom -DBUILD_TOOLS=true -DBUILD_HTTP=false -DBUILD_TEST=false -DVERNUMBER=%2 -DCPUTYPE=x64 +cmake ../../ -G "NMake Makefiles JOM" -DCMAKE_MAKE_PROGRAM=jom -DBUILD_TOOLS=true -DWEBSOCKET=true -DBUILD_HTTP=false -DBUILD_TEST=false -DVERNUMBER=%2 -DCPUTYPE=x64 cmake --build . rd /s /Q C:\TDengine cmake --install . @@ -63,4 +63,4 @@ exit /b :RUNFAILED echo %* cd %package_dir% -goto :eof \ No newline at end of file +goto :eof diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index e9546ba103..2776683a24 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -285,10 +285,14 @@ if [[ $dbName == "taos" ]]; then fi # Add web files - if [ -d "${web_dir}/admin" ]; then - mkdir -p ${install_dir}/share/ - cp ${web_dir}/admin ${install_dir}/share/ -r - cp ${web_dir}/png/taos.png ${install_dir}/share/admin/images/taos.png + if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then + if [ -d "${web_dir}/admin" ] ; then + mkdir -p ${install_dir}/share/ + cp ${web_dir}/admin ${install_dir}/share/ -r + cp ${web_dir}/png/taos.png ${install_dir}/share/admin/images/taos.png + else + echo "directory not found for enterprise release: ${web_dir}/admin" + fi fi fi @@ -375,4 +379,4 @@ if [ -n "${taostools_bin_files}" ]; then fi fi -cd ${curr_dir} +cd ${curr_dir} \ No newline at end of file From 93190003a3bd2dc2af5420199e64fd34c5c858de Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 3 Nov 2022 15:19:21 +0800 Subject: [PATCH 17/82] fix: reset last load info when close dataf reader for stt last files --- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 1396b822bf..2ae3115c0a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -338,8 +338,7 @@ _end: tsdbDataFReaderClose(&pr->pDataFReader); tsdbUntakeReadSnap(pr->pVnode->pTsdb, pr->pReadSnap, "cache-l"); - pr->pDataFReaderLast = NULL; - pr->pDataFReader = NULL; + resetLastBlockLoadInfo(pr->pLoadInfo); for (int32_t j = 0; j < pr->numOfCols; ++j) { taosMemoryFree(pRes[j]); From ae5f6ef1752832028276870c550187130068293c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 3 Nov 2022 16:28:48 +0800 Subject: [PATCH 18/82] enh: add show rewrite mask in query msg --- include/common/tmsg.h | 5 +++++ source/common/src/tglobal.c | 3 +++ source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 3 ++- source/libs/scheduler/src/schRemote.c | 1 + 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 7bc56daab0..1781374165 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -216,9 +216,14 @@ typedef struct SEp { uint16_t port; } SEp; +#define SHOW_REWRITE_MASK() (1 << 0) + +#define TEST_SHOW_REWRITE_MASK(m) ((m) & SHOW_REWRITE_MASK() != 0) + typedef struct { int32_t contLen; int32_t vgId; + int32_t msgMask; } SMsgHead; // Submit message for one table diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 7f4a826c5e..de1347a52c 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -786,6 +786,9 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { case 'd': { if (strcasecmp("dDebugFlag", name) == 0) { dDebugFlag = cfgGetItem(pCfg, "dDebugFlag")->i32; + } else if (strcasecmp("debugFlag", name) == 0) { + int32_t flag = cfgGetItem(pCfg, "debugFlag")->i32; + taosSetAllDebugFlag(flag, true); } break; } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 40d0e32c2b..0c9f09644f 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -145,6 +145,7 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp pHead->contLen = ntohl(pHead->contLen); pHead->vgId = ntohl(pHead->vgId); + pHead->msgMask = ntohl(pHead->msgMask); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); if (pVnode == NULL) { @@ -155,7 +156,7 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp switch (qtype) { case QUERY_QUEUE: - if ((pMsg->msgType == TDMT_SCH_QUERY) && (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS)) { + if ((pMsg->msgType == TDMT_SCH_QUERY) && (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS) && !TEST_SHOW_REWRITE_MASK(pHead->msgMask)) { terrno = TSDB_CODE_GRANT_EXPIRED; code = terrno; dDebug("vgId:%d, msg:%p put into vnode-query queue failed since %s", pVnode->vgId, pMsg, terrstr(code)); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 47de2528fa..7aa70175da 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -1047,6 +1047,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SSubQueryMsg *pMsg = msg; pMsg->header.vgId = htonl(addr->nodeId); + pMsg->header.msgMask = htonl((pTask->plan->showRewrite) ? SHOW_REWRITE_MASK() : 0); pMsg->sId = htobe64(schMgmt.sId); pMsg->queryId = htobe64(pJob->queryId); pMsg->taskId = htobe64(pTask->taskId); From 2c4d90aad4773b91d45cb064724e1a095f402d61 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 3 Nov 2022 16:55:48 +0800 Subject: [PATCH 19/82] handle trans except --- source/libs/transport/src/transCli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 21444018cd..e0821b8ca6 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -427,7 +427,7 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) { if (pCtx == NULL || pCtx->pSem == NULL) { if (transMsg.info.ahandle == NULL) { - if (REQUEST_NO_RESP(&pMsg->msg) || pMsg->type == Release) destroyCmsg(pMsg); + if (pMsg == NULL || REQUEST_NO_RESP(&pMsg->msg) || pMsg->type == Release) destroyCmsg(pMsg); once = true; continue; } From 5030218dcf328cff1d6c59017e21ded408659f4f Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Thu, 3 Nov 2022 17:11:57 +0800 Subject: [PATCH 20/82] Update 09-udf.md --- docs/en/07-develop/09-udf.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/07-develop/09-udf.md b/docs/en/07-develop/09-udf.md index deb9c4cdb5..8092035241 100644 --- a/docs/en/07-develop/09-udf.md +++ b/docs/en/07-develop/09-udf.md @@ -218,7 +218,7 @@ After compiling your function into a DLL, you add it to TDengine. For more infor ## Sample Code -### Sample scalar function: [bit_and](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/bit_and.c) +### Sample scalar function: [bit_and](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/bit_and.c) The bit_add function implements bitwise addition for multiple columns. If there is only one column, the column is returned. The bit_add function ignores null values. @@ -231,7 +231,7 @@ The bit_add function implements bitwise addition for multiple columns. If there -### Sample aggregate function: [l2norm](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/l2norm.c) +### Sample aggregate function: [l2norm](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/l2norm.c) The l2norm function finds the second-order norm for all data in the input column. This squares the values, takes a cumulative sum, and finds the square root. From 53052a298fb57c89e1797075bb94564cbebeca6b Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Thu, 3 Nov 2022 17:13:28 +0800 Subject: [PATCH 21/82] Update 09-udf.md --- docs/en/07-develop/09-udf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/07-develop/09-udf.md b/docs/en/07-develop/09-udf.md index 8092035241..253f1270f5 100644 --- a/docs/en/07-develop/09-udf.md +++ b/docs/en/07-develop/09-udf.md @@ -220,7 +220,7 @@ After compiling your function into a DLL, you add it to TDengine. For more infor ### Sample scalar function: [bit_and](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/bit_and.c) -The bit_add function implements bitwise addition for multiple columns. If there is only one column, the column is returned. The bit_add function ignores null values. +The bit_and function implements bitwise addition for multiple columns. If there is only one column, the column is returned. The bit_and function ignores null values.
bit_and.c From fcf306eb8d665623ffcec8f1dcd54c9054a3a4f4 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 3 Nov 2022 17:22:46 +0800 Subject: [PATCH 22/82] fix: correct cfgdir param meaning --- docs/en/14-reference/05-taosbenchmark.md | 2 +- docs/zh/14-reference/05-taosbenchmark.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/14-reference/05-taosbenchmark.md b/docs/en/14-reference/05-taosbenchmark.md index d21f1829b0..6e08671e34 100644 --- a/docs/en/14-reference/05-taosbenchmark.md +++ b/docs/en/14-reference/05-taosbenchmark.md @@ -217,7 +217,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) The parameters listed in this section apply to all function modes. - **filetype** : The function to be tested, with optional values `insert`, `query` and `subscribe`. These correspond to the insert, query, and subscribe functions, respectively. Users can specify only one of these in each configuration file. -**cfgdir**: specify the TDengine cluster configuration file's directory. The default path is /etc/taos. +**cfgdir**: specify the TDengine client configuration file's directory. The default path is /etc/taos. - **host**: Specify the FQDN of the TDengine server to connect. The default value is `localhost`. diff --git a/docs/zh/14-reference/05-taosbenchmark.md b/docs/zh/14-reference/05-taosbenchmark.md index 13530923b8..76dd5f12d8 100644 --- a/docs/zh/14-reference/05-taosbenchmark.md +++ b/docs/zh/14-reference/05-taosbenchmark.md @@ -217,7 +217,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) 本节所列参数适用于所有功能模式。 - **filetype** : 要测试的功能，可选值为 `insert`, `query` 和 `subscribe`。分别对应插入、查询和订阅功能。每个配置文件中只能指定其中之一。 -- **cfgdir** : TDengine 集群配置文件所在的目录，默认路径是 /etc/taos 。 +- **cfgdir** : TDengine 客户端配置文件所在的目录，默认路径是 /etc/taos 。 - **host** : 指定要连接的 TDengine 服务端的 FQDN，默认值为 localhost。 From d7bd87ea4cc4b643d41fc9cb6ea092c9a940a6bd Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Thu, 3 Nov 2022 17:47:30 +0800 Subject: [PATCH 23/82] release:update_docs of version 3.0.1.6 --- docs/en/28-releases/01-tdengine.md | 4 ++++ docs/en/28-releases/02-tools.md | 4 ++++ docs/zh/28-releases/01-tdengine.md | 4 ++++ docs/zh/28-releases/02-tools.md | 4 ++++ 4 files changed, 16 insertions(+) diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index 14096bd400..74eeeb5efb 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -6,6 +6,10 @@ description: TDengine release history, Release Notes and download links. import Release from "/components/ReleaseV3"; +## 3.0.1.6 + + + ## 3.0.1.5 diff --git a/docs/en/28-releases/02-tools.md b/docs/en/28-releases/02-tools.md index a7446be4e3..0a96c776e0 100644 --- a/docs/en/28-releases/02-tools.md +++ b/docs/en/28-releases/02-tools.md @@ -6,6 +6,10 @@ description: taosTools release history, Release Notes, download links. import Release from "/components/ReleaseV3"; +## 2.2.7 + + + ## 2.2.6 diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index a5b3534250..31093ce557 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -6,6 +6,10 @@ description: TDengine 发布历史、Release Notes 及下载链接 import Release from "/components/ReleaseV3"; +## 3.0.1.6 + + + ## 3.0.1.5 diff --git a/docs/zh/28-releases/02-tools.md b/docs/zh/28-releases/02-tools.md index e86481435c..2623391fb9 100644 --- a/docs/zh/28-releases/02-tools.md +++ b/docs/zh/28-releases/02-tools.md @@ -6,6 +6,10 @@ description: taosTools 的发布历史、Release Notes 和下载链接 import Release from "/components/ReleaseV3"; +## 2.2.7 + + + ## 2.2.6 From 32cbaa25a8c7c8f6897e5996a8b989784ee6871f Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 3 Nov 2022 17:48:16 +0800 Subject: [PATCH 24/82] test:add parameter for mixed deployed mnodes in testcase framework --- tests/pytest/util/cluster.py | 11 ++- tests/system-test/0-others/compa4096.json | 77 +++++++++++++++++++ tests/system-test/0-others/compatibility.py | 10 ++- .../6-cluster/5dnode3mnodeAdd1Ddnoe.py | 12 ++- .../6-cluster/5dnode3mnodeRecreateMnode.py | 13 ++-- .../5dnode3mnodeRestartDnodeInsertData.py | 13 ++-- ...5dnode3mnodeRestartDnodeInsertDataAsync.py | 13 ++-- .../5dnode3mnodeSep1VnodeStopDnodeCreateDb.py | 12 +-- ...5dnode3mnodeSep1VnodeStopDnodeCreateStb.py | 12 +-- ...dnode3mnodeSep1VnodeStopDnodeInsertData.py | 12 +-- .../5dnode3mnodeSep1VnodeStopMnodeCreateDb.py | 14 +--- ...ode3mnodeSep1VnodeStopMnodeCreateDbRep3.py | 12 +-- ...5dnode3mnodeSep1VnodeStopMnodeCreateStb.py | 12 +-- .../5dnode3mnodeSep1VnodeStopVnodeCreateDb.py | 12 +-- ...5dnode3mnodeSep1VnodeStopVnodeCreateStb.py | 12 +-- .../system-test/6-cluster/5dnode3mnodeStop.py | 17 ++-- .../6-cluster/5dnode3mnodeStop2Follower.py | 9 +-- .../6-cluster/5dnode3mnodeStopConnect.py | 14 ++-- .../5dnode3mnodeStopFollowerLeader.py | 14 ++-- .../6-cluster/5dnode3mnodeStopLoop.py | 10 +-- tests/system-test/test.py | 17 +++- 21 files changed, 185 insertions(+), 143 deletions(-) create mode 100644 tests/system-test/0-others/compa4096.json diff --git a/tests/pytest/util/cluster.py b/tests/pytest/util/cluster.py index 2b9af8de25..72b7e1fddf 100644 --- a/tests/pytest/util/cluster.py +++ b/tests/pytest/util/cluster.py @@ -36,7 +36,7 @@ class ConfigureyCluster: self.portStep = 100 self.mnodeNums = 0 - def configure_cluster(self ,dnodeNums=5,mnodeNums=0,startPort=6030,portStep=100,hostname="%s"%hostname): + def configure_cluster(self ,dnodeNums=5,mnodeNums=0,independentMnode=True,startPort=6030,portStep=100,hostname="%s"%hostname): self.startPort=int(startPort) self.portStep=int(portStep) self.hostname=hostname @@ -52,7 +52,7 @@ class ConfigureyCluster: dnode.addExtraCfg("secondEp", f"{hostname}:{startPort_sec}") # configure dnoe of independent mnodes - if num <= self.mnodeNums and self.mnodeNums != 0 : + if num <= self.mnodeNums and self.mnodeNums != 0 and independentMnode == True : dnode.addExtraCfg("supportVnodes", 0) # print(dnode) self.dnodes.append(dnode) @@ -67,6 +67,13 @@ class ConfigureyCluster: tdSql.execute(" create dnode '%s';"%dnode_id) + def create_mnode(self,conn,mnodeNums): + tdSql.init(conn.cursor()) + mnodeNums=int(mnodeNums) + for i in range(2,mnodeNums+1): + tdSql.execute(" create mnode on dnode %d;"%i) + + def check_dnode(self,conn): tdSql.init(conn.cursor()) diff --git a/tests/system-test/0-others/compa4096.json b/tests/system-test/0-others/compa4096.json new file mode 100644 index 0000000000..85b63ed35f --- /dev/null +++ b/tests/system-test/0-others/compa4096.json @@ -0,0 +1,77 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "localhost", + "port": 6030, + "rest_port": 6041, + "user": "root", + "password": "taosdata", + "thread_count": 100, + "create_table_thread_count": 24, + "result_file": "taosBenchmark_result.log", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 100, + "max_sql_len": 1024000, + "databases": [ + { + "dbinfo": { + "name": "taosc_sql", + "drop": "yes", + "replica": 1, + "duration": 10, + "precision": "ms", + "keep": 3650, + "minRows": 100, + "maxRows": 4096, + "comp": 2, + "vgroups": 30 + }, + "super_tables": [ + { + "name": "stb0", + "child_table_exists": "no", + "childtable_count": 100, + "childtable_prefix": "ctb0", + "escape_character": "no", + "auto_create_table": "no", + "batch_create_tbl_num": 500, + "data_source": "rand", + "insert_mode": "taosc", + "rollup": null, + "interlace_rows": 0, + "line_protocol": null, + "tcp_transfer": "no", + "insert_rows": 10000, + "childtable_limit": 0, + "childtable_offset": 0, + "rows_per_tbl": 0, + "max_sql_len": 1048576, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1000, + "start_timestamp": "2022-10-22 17:20:36", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [ + { + "type": "INT", + "count": 4094 + } + ], + "tags": [ + { + "type": "TINYINT", + "count": 1 + } + ] + } + ] + } + ], + "prepare_rand": 10000, + "chinese": "no", + "streams": false, + "test_log": "/root/testlog/" +} diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index 619031fc29..827bb5906e 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -93,6 +93,7 @@ class TDTestCase: oldServerVersion=tdsqlF.queryResult[0][0] tdLog.info(f"Base server version is {oldServerVersion}") tdsqlF.query(f"SELECT CLIENT_VERSION();") + # the oldClientVersion can't be updated in the same python process,so the version is new compiled verison oldClientVersion=tdsqlF.queryResult[0][0] tdLog.info(f"Base client version is {oldClientVersion}") @@ -105,7 +106,14 @@ class TDTestCase: # tdsqlF.query(f"select count(*) from {stb}") # tdsqlF.checkData(0,0,tableNumbers*recordNumbers1) os.system("pkill taosd") - sleep(1) + sleep(2) + + print(f"start taosd: nohup taosd -c {cPath} & ") + os.system(f" nohup taosd -c {cPath} & " ) + sleep(2) + tdLog.info(f" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f compa4096.json -y ") + + tdLog.printNoPrefix("==========step2:update new version ") self.buildTaosd(bPath) diff --git a/tests/system-test/6-cluster/5dnode3mnodeAdd1Ddnoe.py b/tests/system-test/6-cluster/5dnode3mnodeAdd1Ddnoe.py index f174975a8e..d084432a1a 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeAdd1Ddnoe.py +++ b/tests/system-test/6-cluster/5dnode3mnodeAdd1Ddnoe.py @@ -126,14 +126,12 @@ class TDTestCase: tdSql.checkData(0,1,'%s:6030'%self.host) tdSql.checkData(4,1,'%s:6430'%self.host) clusterComCheck.checkDnodes(dnodeNumbers) - clusterComCheck.checkMnodeStatus(1) - # fisr add three mnodes; - tdLog.info("fisr add three mnodes and check mnode status") - tdSql.execute("create mnode on dnode 2") - clusterComCheck.checkMnodeStatus(2) - tdSql.execute("create mnode on dnode 3") - clusterComCheck.checkMnodeStatus(3) + + #check mnode status + tdLog.info("check mnode status") + clusterComCheck.checkMnodeStatus(mnodeNums) + # add some error operations and tdLog.info("Confirm the status of the dnode again") diff --git a/tests/system-test/6-cluster/5dnode3mnodeRecreateMnode.py b/tests/system-test/6-cluster/5dnode3mnodeRecreateMnode.py index 1590a5948b..41082baa3d 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRecreateMnode.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRecreateMnode.py @@ -126,14 +126,11 @@ class TDTestCase: tdSql.checkData(0,1,'%s:6030'%self.host) tdSql.checkData(4,1,'%s:6430'%self.host) clusterComCheck.checkDnodes(dnodeNumbers) - clusterComCheck.checkMnodeStatus(1) - - # fisr add three mnodes; - tdLog.info("fisr add three mnodes and check mnode status") - tdSql.execute("create mnode on dnode 2") - clusterComCheck.checkMnodeStatus(2) - tdSql.execute("create mnode on dnode 3") - clusterComCheck.checkMnodeStatus(3) + + #check mnode status + tdLog.info("check mnode status") + clusterComCheck.checkMnodeStatus(mnodeNums) + # add some error operations and tdLog.info("Confirm the status of the dnode again") diff --git a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py index 5af9e55472..94e02b77b3 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py @@ -126,14 +126,11 @@ class TDTestCase: tdSql.checkData(0,1,'%s:6030'%self.host) tdSql.checkData(4,1,'%s:6430'%self.host) clusterComCheck.checkDnodes(dnodeNumbers) - clusterComCheck.checkMnodeStatus(1) - - # fisr add three mnodes; - tdLog.info("fisr add three mnodes and check mnode status") - tdSql.execute("create mnode on dnode 2") - clusterComCheck.checkMnodeStatus(2) - tdSql.execute("create mnode on dnode 3") - clusterComCheck.checkMnodeStatus(3) + + #check mnode status + tdLog.info("check mnode status") + clusterComCheck.checkMnodeStatus(mnodeNums) + # add some error operations and tdLog.info("Confirm the status of the dnode again") diff --git a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py index 1c488cab5f..d6d06446a1 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py @@ -126,14 +126,11 @@ class TDTestCase: tdSql.checkData(0,1,'%s:6030'%self.host) tdSql.checkData(4,1,'%s:6430'%self.host) clusterComCheck.checkDnodes(dnodeNumbers) - clusterComCheck.checkMnodeStatus(1) - - # fisr add three mnodes; - tdLog.info("fisr add three mnodes and check mnode status") - tdSql.execute("create mnode on dnode 2") - clusterComCheck.checkMnodeStatus(2) - tdSql.execute("create mnode on dnode 3") - clusterComCheck.checkMnodeStatus(3) + + #check mnode status + tdLog.info("check mnode status") + clusterComCheck.checkMnodeStatus(mnodeNums) + # add some error operations and tdLog.info("Confirm the status of the dnode again") diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py index 93d60e6561..d3ec429bdb 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py @@ -120,14 +120,10 @@ class TDTestCase: tdSql.checkData(0,1,'%s:6030'%self.host) tdSql.checkData(4,1,'%s:6430'%self.host) clusterComCheck.checkDnodes(dnodeNumbers) - clusterComCheck.checkMnodeStatus(1) - - # fisr add three mnodes; - tdLog.info("fisr add three mnodes and check mnode status") - tdSql.execute("create mnode on dnode 2") - clusterComCheck.checkMnodeStatus(2) - tdSql.execute("create mnode on dnode 3") - clusterComCheck.checkMnodeStatus(3) + + #check mnode status + tdLog.info("check mnode status") + clusterComCheck.checkMnodeStatus(mnodeNums) # add some error operations and tdLog.info("Confirm the status of the dnode again") diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py index 62f4b248f9..3a2492c89d 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py @@ -94,14 +94,10 @@ class TDTestCase: tdSql.checkData(0,1,'%s:6030'%self.host) tdSql.checkData(4,1,'%s:6430'%self.host) clusterComCheck.checkDnodes(dnodeNumbers) - clusterComCheck.checkMnodeStatus(1) - - # fisr add three mnodes; - tdLog.info("fisr add three mnodes and check mnode status") - tdSql.execute("create mnode on dnode 2") - clusterComCheck.checkMnodeStatus(2) - tdSql.execute("create mnode on dnode 3") - clusterComCheck.checkMnodeStatus(3) + + #check mnode status + tdLog.info("check mnode status") + clusterComCheck.checkMnodeStatus(mnodeNums) # add some error operations and tdLog.info("Confirm the status of the dnode again") diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py index 382144b69c..265000bdc9 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py @@ -124,14 +124,10 @@ class TDTestCase: tdSql.checkData(0,1,'%s:6030'%self.host) tdSql.checkData(4,1,'%s:6430'%self.host) clusterComCheck.checkDnodes(dnodeNumbers) - clusterComCheck.checkMnodeStatus(1) - - # fisr add three mnodes; - tdLog.info("fisr add three mnodes and check mnode status") - tdSql.execute("create mnode on dnode 2") - clusterComCheck.checkMnodeStatus(2) - tdSql.execute("create mnode on dnode 3") - clusterComCheck.checkMnodeStatus(3) + + #check mnode status + tdLog.info("check mnode status") + clusterComCheck.checkMnodeStatus(mnodeNums) # add some error operations and tdLog.info("Confirm the status of the dnode again") diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py index 397c6f5ccc..a5f8810a25 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py @@ -95,16 +95,10 @@ class TDTestCase: tdSql.checkData(0,1,'%s:6030'%self.host) tdSql.checkData(4,1,'%s:6430'%self.host) clusterComCheck.checkDnodes(dnodeNumbers) - clusterComCheck.checkMnodeStatus(1) - - # fisr add three mnodes; - tdLog.info("fisr add three mnodes and check mnode status") - tdLog.info("create mnode on dnode 2") - tdSql.execute("create mnode on dnode 2") - clusterComCheck.checkMnodeStatus(2) - tdLog.info("create mnode on dnode 3") - tdSql.execute("create mnode on dnode 3") - clusterComCheck.checkMnodeStatus(3) + + #check mnode status + tdLog.info("check mnode status") + clusterComCheck.checkMnodeStatus(mnodeNums) # add some error operations and tdLog.info("Confirm the status of the dnode again") diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py index e8e3b217a1..98842e3358 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py @@ -94,14 +94,10 @@ class TDTestCase: tdSql.checkData(0,1,'%s:6030'%self.host) tdSql.checkData(4,1,'%s:6430'%self.host) clusterComCheck.checkDnodes(dnodeNumbers) - clusterComCheck.checkMnodeStatus(1) - - # fisr add three mnodes; - tdLog.info("fisr add three mnodes and check mnode status") - tdSql.execute("create mnode on dnode 2") - clusterComCheck.checkMnodeStatus(2) - tdSql.execute("create mnode on dnode 3") - clusterComCheck.checkMnodeStatus(3) + + #check mnode status + tdLog.info("check mnode status") + clusterComCheck.checkMnodeStatus(mnodeNums) # add some error operations and tdLog.info("Confirm the status of the dnode again") diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py index e3f1a91de8..cb16059524 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py @@ -119,14 +119,10 @@ class TDTestCase: tdSql.checkData(0,1,'%s:6030'%self.host) tdSql.checkData(4,1,'%s:6430'%self.host) clusterComCheck.checkDnodes(dnodeNumbers) - clusterComCheck.checkMnodeStatus(1) - - # fisr add three mnodes; - tdLog.info("fisr add three mnodes and check mnode status") - tdSql.execute("create mnode on dnode 2") - clusterComCheck.checkMnodeStatus(2) - tdSql.execute("create mnode on dnode 3") - clusterComCheck.checkMnodeStatus(3) + + #check mnode status + tdLog.info("check mnode status") + clusterComCheck.checkMnodeStatus(mnodeNums) # add some error operations and tdLog.info("Confirm the status of the dnode again") diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py index 4ff6bffc07..a1ebef9709 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py @@ -96,14 +96,10 @@ class TDTestCase: tdSql.checkData(0,1,'%s:6030'%self.host) tdSql.checkData(4,1,'%s:6430'%self.host) clusterComCheck.checkDnodes(dnodeNumbers) - clusterComCheck.checkMnodeStatus(1) - - # fisr add three mnodes; - tdLog.info("fisr add three mnodes and check mnode status") - tdSql.execute("create mnode on dnode 2") - clusterComCheck.checkMnodeStatus(2) - tdSql.execute("create mnode on dnode 3") - clusterComCheck.checkMnodeStatus(3) + + #check mnode status + tdLog.info("check mnode status") + clusterComCheck.checkMnodeStatus(mnodeNums) # add some error operations and tdLog.info("Confirm the status of the dnode again") diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py index 4bd8628a66..dc8e600f29 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py @@ -119,14 +119,10 @@ class TDTestCase: tdSql.checkData(0,1,'%s:6030'%self.host) tdSql.checkData(4,1,'%s:6430'%self.host) clusterComCheck.checkDnodes(dnodeNumbers) - clusterComCheck.checkMnodeStatus(1) - - # fisr add three mnodes; - tdLog.info("fisr add three mnodes and check mnode status") - tdSql.execute("create mnode on dnode 2") - clusterComCheck.checkMnodeStatus(2) - tdSql.execute("create mnode on dnode 3") - clusterComCheck.checkMnodeStatus(3) + + #check mnode status + tdLog.info("check mnode status") + clusterComCheck.checkMnodeStatus(mnodeNums) # add some error operations and tdLog.info("Confirm the status of the dnode again") diff --git a/tests/system-test/6-cluster/5dnode3mnodeStop.py b/tests/system-test/6-cluster/5dnode3mnodeStop.py index 531688710a..522ba4c2fc 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStop.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStop.py @@ -75,14 +75,17 @@ class TDTestCase: tdSql.checkData(0,1,'%s:6030'%self.host) tdSql.checkData(4,1,'%s:6430'%self.host) clusterComCheck.checkDnodes(dnodenumbers) - clusterComCheck.checkMnodeStatus(1) + + #check mnode status + tdLog.info("check mnode status") + clusterComCheck.checkMnodeStatus(mnodeNums) - # fisr add three mnodes; - tdLog.info("fisr add three mnodes and check mnode status") - tdSql.execute("create mnode on dnode 2") - clusterComCheck.checkMnodeStatus(2) - tdSql.execute("create mnode on dnode 3") - clusterComCheck.checkMnodeStatus(3) + # # fisr add three mnodes; + # tdLog.info("check mnode status") + # # tdSql.execute("create mnode on dnode 2") + # clusterComCheck.checkMnodeStatus(2) + # # tdSql.execute("create mnode on dnode 3") + # clusterComCheck.checkMnodeStatus(3) # add some error operations and tdLog.info("Confirm the status of the dnode again") diff --git a/tests/system-test/6-cluster/5dnode3mnodeStop2Follower.py b/tests/system-test/6-cluster/5dnode3mnodeStop2Follower.py index 76ff746b2e..0596dd84ed 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStop2Follower.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStop2Follower.py @@ -75,14 +75,9 @@ class TDTestCase: tdSql.checkData(0,1,'%s:6030'%self.host) tdSql.checkData(4,1,'%s:6430'%self.host) clusterComCheck.checkDnodes(dnodenumbers) - clusterComCheck.checkMnodeStatus(1) - # fisr add three mnodes; - tdLog.info("fisr add three mnodes and check mnode status") - tdSql.execute("create mnode on dnode 2") - clusterComCheck.checkMnodeStatus(2) - tdSql.execute("create mnode on dnode 3") - clusterComCheck.checkMnodeStatus(3) + tdLog.info("check mnode status") + clusterComCheck.checkMnodeStatus(mnodeNums) # add some error operations and tdLog.info("Confirm the status of the dnode again") diff --git a/tests/system-test/6-cluster/5dnode3mnodeStopConnect.py b/tests/system-test/6-cluster/5dnode3mnodeStopConnect.py index 142f88c0d9..2c735ed9b6 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStopConnect.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStopConnect.py @@ -75,15 +75,11 @@ class TDTestCase: tdSql.checkData(0,1,'%s:6030'%self.host) tdSql.checkData(4,1,'%s:6430'%self.host) clusterComCheck.checkDnodes(dnodenumbers) - clusterComCheck.checkMnodeStatus(1) - - # fisr add three mnodes; - tdLog.info("fisr add three mnodes and check mnode status") - tdSql.execute("create mnode on dnode 2") - clusterComCheck.checkMnodeStatus(2) - tdSql.execute("create mnode on dnode 3") - clusterComCheck.checkMnodeStatus(3) - + + #check mnode status + tdLog.info("check mnode status") + clusterComCheck.checkMnodeStatus(mnodeNums) + # add some error operations and tdLog.info("Confirm the status of the dnode again") tdSql.error("create mnode on dnode 2") diff --git a/tests/system-test/6-cluster/5dnode3mnodeStopFollowerLeader.py b/tests/system-test/6-cluster/5dnode3mnodeStopFollowerLeader.py index afa1c3dcc3..d7176e142f 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStopFollowerLeader.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStopFollowerLeader.py @@ -75,15 +75,11 @@ class TDTestCase: tdSql.checkData(0,1,'%s:6030'%self.host) tdSql.checkData(4,1,'%s:6430'%self.host) clusterComCheck.checkDnodes(dnodenumbers) - clusterComCheck.checkMnodeStatus(1) - - # fisr add three mnodes; - tdLog.info("fisr add three mnodes and check mnode status") - tdSql.execute("create mnode on dnode 2") - clusterComCheck.checkMnodeStatus(2) - tdSql.execute("create mnode on dnode 3") - clusterComCheck.checkMnodeStatus(3) - + + #check mnode status + tdLog.info("check mnode status") + clusterComCheck.checkMnodeStatus(mnodeNums) + # add some error operations and tdLog.info("Confirm the status of the dnode again") tdSql.error("create mnode on dnode 2") diff --git a/tests/system-test/6-cluster/5dnode3mnodeStopLoop.py b/tests/system-test/6-cluster/5dnode3mnodeStopLoop.py index 97134ac2d1..52d61fb529 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStopLoop.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStopLoop.py @@ -75,14 +75,10 @@ class TDTestCase: tdSql.checkData(0,1,'%s:6030'%self.host) tdSql.checkData(4,1,'%s:6430'%self.host) clusterComCheck.checkDnodes(dnodenumbers) - clusterComCheck.checkMnodeStatus(1) - # fisr add three mnodes; - tdLog.info("fisr add three mnodes and check mnode status") - tdSql.execute("create mnode on dnode 2") - clusterComCheck.checkMnodeStatus(2) - tdSql.execute("create mnode on dnode 3") - clusterComCheck.checkMnodeStatus(3) + #check mnode status + tdLog.info("check mnode status") + clusterComCheck.checkMnodeStatus(mnodeNums) # add some error operations and tdLog.info("Confirm the status of the dnode again") diff --git a/tests/system-test/test.py b/tests/system-test/test.py index b25bda4a3b..e171baa656 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -73,8 +73,9 @@ if __name__ == "__main__": createDnodeNums = 1 restful = False replicaVar = 1 - opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RD:n:', [ - 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','adaptercfgupdate','replicaVar']) + independentMnode = True + opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RD:n:i:', [ + 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','adaptercfgupdate','replicaVar','independentMnode']) for key, value in opts: if key in ['-h', '--help']: tdLog.printNoPrefix( @@ -97,6 +98,8 @@ if __name__ == "__main__": tdLog.printNoPrefix('-R restful realization form') tdLog.printNoPrefix('-D taosadapter update cfg dict ') tdLog.printNoPrefix('-n the number of replicas') + tdLog.printNoPrefix('-i independentMnode Mnode') + sys.exit(0) if key in ['-r', '--restart']: @@ -158,6 +161,9 @@ if __name__ == "__main__": if key in ['-C', '--createDnodeNums']: createDnodeNums = value + if key in ['-i', '--independentMnode']: + independentMnode = value + if key in ['-R', '--restful']: restful = True @@ -313,7 +319,7 @@ if __name__ == "__main__": tdLog.exit(f"alter queryPolicy to {queryPolicy} failed") else : tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums)) - dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums,mnodeNums=mnodeNums) + dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode) tdDnodes = ClusterDnodes(dnodeslist) tdDnodes.init(deployPath, masterIp) tdDnodes.setTestCluster(testCluster) @@ -339,6 +345,7 @@ if __name__ == "__main__": else: createDnodeNums=createDnodeNums cluster.create_dnode(conn,createDnodeNums) + cluster.create_mnode(conn,mnodeNums) try: if cluster.check_dnode(conn) : print("check dnode ready") @@ -446,7 +453,7 @@ if __name__ == "__main__": else : tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums)) - dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums,mnodeNums=mnodeNums) + dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode) tdDnodes = ClusterDnodes(dnodeslist) tdDnodes.init(deployPath, masterIp) tdDnodes.setTestCluster(testCluster) @@ -472,6 +479,8 @@ if __name__ == "__main__": else: createDnodeNums=createDnodeNums cluster.create_dnode(conn,createDnodeNums) + cluster.create_mnode(conn,mnodeNums) + try: if cluster.check_dnode(conn) : print("check dnode ready") From 49595291277013d1ae79a5c99d754d77c738e720 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Nov 2022 19:00:13 +0800 Subject: [PATCH 25/82] fix(query): set the last key of each tablescan info --- source/dnode/vnode/src/tsdb/tsdbRead.c | 45 +- source/libs/executor/src/executorimpl.c | 5 +- tests/system-test/1-insert/delete_data.py | 614 +++++++++++----------- 3 files changed, 340 insertions(+), 324 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 89d07911d9..4099dafa26 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -185,11 +185,11 @@ static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STabl SRowMerger* pMerger, SVersionRange* pVerRange); static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger, STsdbReader* pReader); -static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, uint64_t uid); +static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, STableBlockScanInfo* pInfo); static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, int32_t rowIndex); static void setComposedBlockFlag(STsdbReader* pReader, bool composed); -static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order, SVersionRange* pRange); +static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order, SVersionRange* pVerRange); static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, STsdbReader* pReader, bool* freeTSRow); @@ -208,7 +208,6 @@ static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader); static int32_t doBuildDataBlock(STsdbReader* pReader); static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader); static bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo); -static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader); static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } @@ -1529,8 +1528,8 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB // opt version // 1. it is not a border point // 2. the direct next point is not an duplicated timestamp - if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && pReader->order == TSDB_ORDER_ASC) || - (pDumpInfo->rowIndex > 0 && pReader->order == TSDB_ORDER_DESC)) { + bool asc = (pReader->order == TSDB_ORDER_ASC); + if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && asc) || (pDumpInfo->rowIndex > 0 && (!asc))) { int32_t step = pReader->order == TSDB_ORDER_ASC ? 1 : -1; int64_t nextKey = pBlockData->aTSKEY[pDumpInfo->rowIndex + step]; @@ -1749,7 +1748,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); tRowMergerClear(&merge); @@ -1770,6 +1769,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, // only last block exists if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) { if (tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader)) { + pBlockScanInfo->lastKey = tsLastBlock; return TSDB_CODE_SUCCESS; } else { int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); @@ -1786,7 +1786,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, return code; } - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); tRowMergerClear(&merge); @@ -1810,7 +1810,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, return code; } - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); tRowMergerClear(&merge); @@ -1858,7 +1858,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader return code; } - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); tRowMergerClear(&merge); @@ -2082,7 +2082,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); tRowMergerClear(&merge); @@ -2233,6 +2233,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc STsdbReader* pReader) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { + pBlockScanInfo->lastKey = key; return TSDB_CODE_SUCCESS; } else { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); @@ -2251,7 +2252,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc return code; } - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); tRowMergerClear(&merge); @@ -2299,29 +2300,32 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; + bool asc = ASCENDING_TRAVERSE(pReader->order); int64_t st = taosGetTimestampUs(); + int32_t step = asc ? 1 : -1; STableBlockScanInfo* pBlockScanInfo = NULL; if (pBlockInfo != NULL) { - pBlockScanInfo = *(STableBlockScanInfo**)taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); - if (pBlockScanInfo == NULL) { + void* p = taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + if (p == NULL) { code = TSDB_CODE_INVALID_PARA; tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", pBlockInfo->uid, taosHashGetSize(pReader->status.pTableMap), pReader->idStr); goto _end; } + pBlockScanInfo = *(STableBlockScanInfo**) p; + SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader); // it is a clean block, load it directly if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader)) { - if (pReader->order == TSDB_ORDER_ASC || - (pReader->order == TSDB_ORDER_DESC && (!hasDataInLastBlock(pLastBlockReader)))) { + if (asc || ((!asc) && (!hasDataInLastBlock(pLastBlockReader)))) { copyBlockDataToSDataBlock(pReader, pBlockScanInfo); // record the last key value - pBlockScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->order)? pBlock->maxKey.ts:pBlock->minKey.ts; + pBlockScanInfo->lastKey = asc? pBlock->maxKey.ts:pBlock->minKey.ts; goto _end; } } @@ -2331,7 +2335,6 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBlockData* pBlockData = &pReader->status.fileBlockData; - int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1; while (1) { bool hasBlockData = false; @@ -3220,7 +3223,6 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, SRowMerger* pMerger, SVersionRange* pVerRange) { - pScanInfo->lastKey = ts; while (nextRowFromLastBlocks(pLastBlockReader, pScanInfo, pVerRange)) { int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); if (next1 == ts) { @@ -3413,9 +3415,10 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR return TSDB_CODE_SUCCESS; } -int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, uint64_t uid) { +int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, STableBlockScanInfo* pScanInfo) { int32_t numOfRows = pBlock->info.rows; int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); + int64_t uid = pScanInfo->uid; SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; STSchema* pSchema = doGetSchemaForTSRow(pTSRow->sver, pReader, uid); @@ -3454,6 +3457,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* } pBlock->info.rows += 1; + pScanInfo->lastKey = pTSRow->ts; return TSDB_CODE_SUCCESS; } @@ -3517,7 +3521,8 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e break; } - doAppendRowFromTSRow(pBlock, pReader, pTSRow, pBlockScanInfo->uid); + doAppendRowFromTSRow(pBlock, pReader, pTSRow, pBlockScanInfo); + if (freeTSRow) { taosMemoryFree(pTSRow); } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 59266716e1..26abc2b90b 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2973,7 +2973,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN return pOperator; - _error: +_error: if (pInfo != NULL) { destroyAggOperatorInfo(pInfo); } @@ -3189,11 +3189,12 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* code = appendDownstream(pOperator, &downstream, 1); return pOperator; - _error: +_error: if (pInfo != NULL) { destroyFillOperatorInfo(pInfo); } + pTaskInfo->code = code; taosMemoryFreeClear(pOperator); return NULL; } diff --git a/tests/system-test/1-insert/delete_data.py b/tests/system-test/1-insert/delete_data.py index 95f7649ba1..c085d2763a 100644 --- a/tests/system-test/1-insert/delete_data.py +++ b/tests/system-test/1-insert/delete_data.py @@ -1,302 +1,312 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- - -import random -import string - -from numpy import logspace -from util import constant -from util.log import * -from util.cases import * -from util.sql import * -from util.common import * -from util.sqlset import TDSetSql - -class TDTestCase: - def init(self, conn, logSql, replicaVar=1): - tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), True) - self.dbname = 'db_test' - self.setsql = TDSetSql() - self.stbname = 'stb' - self.ntbname = 'ntb' - self.rowNum = 5 - self.tbnum = 2 - self.ts = 1537146000000 - self.binary_str = 'taosdata' - self.nchar_str = '涛思数据' - self.str_length = 20 - self.column_dict = { - 'col1': 'tinyint', - 'col2': 'smallint', - 'col3': 'int', - 'col4': 'bigint', - 'col5': 'tinyint unsigned', - 'col6': 'smallint unsigned', - 'col7': 'int unsigned', - 'col8': 'bigint unsigned', - 'col9': 'float', - 'col10': 'double', - 'col11': 'bool', - 'col12': f'binary({self.str_length})', - 'col13': f'nchar({self.str_length})', - - } - - self.tinyint_val = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) - self.smallint_val = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) - self.int_val = random.randint(constant.INT_MIN,constant.INT_MAX) - self.bigint_val = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX) - self.untingint_val = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX) - self.unsmallint_val = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX) - self.unint_val = random.randint(constant.INT_UN_MIN,constant.INT_MAX) - self.unbigint_val = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX) - self.float_val = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) - self.double_val = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) - self.bool_val = random.randint(0,100)%2 - self.binary_val = tdCom.getLongName(random.randint(0,self.str_length)) - self.nchar_val = tdCom.getLongName(random.randint(0,self.str_length)) - self.base_data = { - 'tinyint':self.tinyint_val, - 'smallint':self.smallint_val, - 'int':self.int_val, - 'bigint':self.bigint_val, - 'tinyint unsigned':self.untingint_val, - 'smallint unsigned':self.unsmallint_val, - 'int unsigned':self.unint_val, - 'bigint unsigned':self.unbigint_val, - 'bool':self.bool_val, - 'float':self.float_val, - 'double':self.double_val, - 'binary':self.binary_val, - 'nchar':self.nchar_val - } - - def insert_base_data(self,col_type,tbname,rows,base_data): - for i in range(rows): - if col_type.lower() == 'tinyint': - tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint"]})') - elif col_type.lower() == 'smallint': - tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint"]})') - elif col_type.lower() == 'int': - tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int"]})') - elif col_type.lower() == 'bigint': - tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint"]})') - elif col_type.lower() == 'tinyint unsigned': - tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint unsigned"]})') - elif col_type.lower() == 'smallint unsigned': - tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint unsigned"]})') - elif col_type.lower() == 'int unsigned': - tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int unsigned"]})') - elif col_type.lower() == 'bigint unsigned': - tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint unsigned"]})') - elif col_type.lower() == 'bool': - tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bool"]})') - elif col_type.lower() == 'float': - tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["float"]})') - elif col_type.lower() == 'double': - tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["double"]})') - elif 'binary' in col_type.lower(): - tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['binary']}")''') - elif 'nchar' in col_type.lower(): - tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['nchar']}")''') - def delete_all_data(self,tbname,col_type,row_num,base_data,dbname,tb_type,tb_num=1): - tdSql.execute(f'delete from {tbname}') - tdSql.execute(f'flush database {dbname}') - tdSql.execute('reset query cache') - tdSql.query(f'select * from {tbname}') - tdSql.checkRows(0) - if tb_type == 'ntb' or tb_type == 'ctb': - self.insert_base_data(col_type,tbname,row_num,base_data) - elif tb_type == 'stb': - for i in range(tb_num): - self.insert_base_data(col_type,f'{tbname}_{i}',row_num,base_data) - tdSql.execute(f'flush database {dbname}') - tdSql.execute('reset query cache') - tdSql.query(f'select * from {tbname}') - if tb_type == 'ntb' or tb_type == 'ctb': - tdSql.checkRows(row_num) - elif tb_type =='stb': - tdSql.checkRows(row_num*tb_num) - def delete_one_row(self,tbname,column_type,column_name,base_data,row_num,dbname,tb_type,tb_num=1): - tdSql.execute(f'delete from {tbname} where ts={self.ts}') - tdSql.execute(f'flush database {dbname}') - tdSql.execute('reset query cache') - tdSql.query(f'select {column_name} from {tbname}') - if tb_type == 'ntb' or tb_type == 'ctb': - tdSql.checkRows(row_num-1) - elif tb_type == 'stb': - tdSql.checkRows((row_num-1)*tb_num) - tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}') - tdSql.checkRows(0) - if tb_type == 'ntb' or tb_type == 'ctb': - if 'binary' in column_type.lower(): - tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['binary']}")''') - elif 'nchar' in column_type.lower(): - tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['nchar']}")''') - else: - tdSql.execute(f'insert into {tbname} values({self.ts},{base_data[column_type]})') - elif tb_type == 'stb': - for i in range(tb_num): - if 'binary' in column_type.lower(): - tdSql.execute(f'''insert into {tbname}_{i} values({self.ts},"{base_data['binary']}")''') - elif 'nchar' in column_type.lower(): - tdSql.execute(f'''insert into {tbname}_{i} values({self.ts},"{base_data['nchar']}")''') - else: - tdSql.execute(f'insert into {tbname}_{i} values({self.ts},{base_data[column_type]})') - tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}') - if column_type.lower() == 'float' or column_type.lower() == 'double': - if abs(tdSql.queryResult[0][0] - base_data[column_type]) / base_data[column_type] <= 0.0001: - tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0]) - else: - tdLog.exit(f'{column_type} data check failure') - elif 'binary' in column_type.lower(): - tdSql.checkEqual(tdSql.queryResult[0][0],base_data['binary']) - elif 'nchar' in column_type.lower(): - tdSql.checkEqual(tdSql.queryResult[0][0],base_data['nchar']) - else: - tdSql.checkEqual(tdSql.queryResult[0][0],base_data[column_type]) - def delete_rows(self,dbname,tbname,col_name,col_type,base_data,row_num,tb_type,tb_num=1): - for i in range(row_num): - tdSql.execute(f'delete from {tbname} where ts>{self.ts+i}') - tdSql.execute(f'flush database {dbname}') - tdSql.execute('reset query cache') - tdSql.query(f'select {col_name} from {tbname}') - if tb_type == 'ntb' or tb_type == 'ctb': - tdSql.checkRows(i+1) - self.insert_base_data(col_type,tbname,row_num,base_data) - elif tb_type == 'stb': - tdSql.checkRows((i+1)*tb_num) - for j in range(tb_num): - self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) - for i in range(row_num): - tdSql.execute(f'delete from {tbname} where ts>={self.ts+i}') - tdSql.execute(f'flush database {dbname}') - tdSql.execute('reset query cache') - tdSql.query(f'select {col_name} from {tbname}') - if tb_type == 'ntb' or tb_type == 'ctb': - tdSql.checkRows(i) - self.insert_base_data(col_type,tbname,row_num,base_data) - elif tb_type == 'stb': - tdSql.checkRows(i*tb_num) - for j in range(tb_num): - self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) - for i in range(row_num): - tdSql.execute(f'delete from {tbname} where ts<={self.ts+i}') - tdSql.execute(f'flush database {dbname}') - tdSql.execute('reset query cache') - tdSql.query(f'select {col_name} from {tbname}') - if tb_type == 'ntb' or tb_type == 'ctb': - tdSql.checkRows(row_num-i-1) - self.insert_base_data(col_type,tbname,row_num,base_data) - elif tb_type == 'stb': - tdSql.checkRows((row_num-i-1)*tb_num) - for j in range(tb_num): - self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) - for i in range(row_num): - tdSql.execute(f'delete from {tbname} where ts<{self.ts+i}') - tdSql.execute(f'flush database {dbname}') - tdSql.execute('reset query cache') - tdSql.query(f'select {col_name} from {tbname}') - if tb_type == 'ntb' or tb_type == 'ctb': - tdSql.checkRows(row_num-i) - self.insert_base_data(col_type,tbname,row_num,base_data) - elif tb_type == 'stb': - tdSql.checkRows((row_num-i)*tb_num) - for j in range(tb_num): - self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) - for i in range(row_num): - tdSql.execute(f'delete from {tbname} where ts between {self.ts} and {self.ts+i}') - tdSql.execute(f'flush database {dbname}') - tdSql.execute('reset query cache') - tdSql.query(f'select {col_name} from {tbname}') - if tb_type == 'ntb' or tb_type == 'ctb': - tdSql.checkRows(row_num - i-1) - self.insert_base_data(col_type,tbname,row_num,base_data) - elif tb_type == 'stb': - tdSql.checkRows(tb_num*(row_num - i-1)) - for j in range(tb_num): - self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) - tdSql.execute(f'delete from {tbname} where ts between {self.ts+i+1} and {self.ts}') - tdSql.query(f'select {col_name} from {tbname}') - if tb_type == 'ntb' or tb_type == 'ctb': - tdSql.checkRows(row_num) - elif tb_type == 'stb': - tdSql.checkRows(tb_num*row_num) - def delete_error(self,tbname,column_name,column_type,base_data): - for error_list in ['',f'ts = {self.ts} and',f'ts = {self.ts} or']: - if 'binary' in column_type.lower(): - tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['binary']}"''') - elif 'nchar' in column_type.lower(): - tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['nchar']}"''') - else: - tdSql.error(f'delete from {tbname} where {error_list} {column_name} = {base_data[column_type]}') - - def delete_data_ntb(self): - tdSql.execute(f'create database if not exists {self.dbname}') - tdSql.execute(f'use {self.dbname}') - for col_name,col_type in self.column_dict.items(): - tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') - self.insert_base_data(col_type,self.ntbname,self.rowNum,self.base_data) - self.delete_one_row(self.ntbname,col_type,col_name,self.base_data,self.rowNum,self.dbname,'ntb') - self.delete_all_data(self.ntbname,col_type,self.rowNum,self.base_data,self.dbname,'ntb') - self.delete_error(self.ntbname,col_name,col_type,self.base_data) - self.delete_rows(self.dbname,self.ntbname,col_name,col_type,self.base_data,self.rowNum,'ntb') - for func in ['first','last']: - tdSql.query(f'select {func}(*) from {self.ntbname}') - tdSql.execute(f'drop table {self.ntbname}') - tdSql.execute(f'drop database {self.dbname}') - def delete_data_ctb(self): - tdSql.execute(f'create database if not exists {self.dbname}') - tdSql.execute(f'use {self.dbname}') - for col_name,col_type in self.column_dict.items(): - tdSql.execute(f'create table {self.stbname} (ts timestamp,{col_name} {col_type}) tags(t1 int)') - for i in range(self.tbnum): - tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags(1)') - self.insert_base_data(col_type,f'{self.stbname}_{i}',self.rowNum,self.base_data) - self.delete_one_row(f'{self.stbname}_{i}',col_type,col_name,self.base_data,self.rowNum,self.dbname,'ctb') - self.delete_all_data(f'{self.stbname}_{i}',col_type,self.rowNum,self.base_data,self.dbname,'ctb') - self.delete_error(f'{self.stbname}_{i}',col_name,col_type,self.base_data) - self.delete_rows(self.dbname,f'{self.stbname}_{i}',col_name,col_type,self.base_data,self.rowNum,'ctb') - for func in ['first','last']: - tdSql.query(f'select {func}(*) from {self.stbname}_{i}') - tdSql.execute(f'drop table {self.stbname}') - def delete_data_stb(self): - tdSql.execute(f'create database if not exists {self.dbname}') - tdSql.execute(f'use {self.dbname}') - for col_name,col_type in self.column_dict.items(): - tdSql.execute(f'create table {self.stbname} (ts timestamp,{col_name} {col_type}) tags(t1 int)') - for i in range(self.tbnum): - tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags(1)') - self.insert_base_data(col_type,f'{self.stbname}_{i}',self.rowNum,self.base_data) - self.delete_error(self.stbname,col_name,col_type,self.base_data) - self.delete_one_row(self.stbname,col_type,col_name,self.base_data,self.rowNum,self.dbname,'stb',self.tbnum) - self.delete_all_data(self.stbname,col_type,self.rowNum,self.base_data,self.dbname,'stb',self.tbnum) - self.delete_rows(self.dbname,self.stbname,col_name,col_type,self.base_data,self.rowNum,'stb',self.tbnum) - for func in ['first','last']: - tdSql.query(f'select {func}(*) from {self.stbname}') - tdSql.execute(f'drop table {self.stbname}') - tdSql.execute(f'drop database {self.dbname}') - def run(self): - self.delete_data_ntb() - self.delete_data_ctb() - self.delete_data_stb() - tdDnodes.stoptaosd(1) - tdDnodes.starttaosd(1) - self.delete_data_ntb() - def stop(self): - tdSql.close() - tdLog.success("%s successfully executed" % __file__) - -tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string + +from numpy import logspace +from util import constant +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import TDSetSql + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.dbname = 'db_test' + self.setsql = TDSetSql() + self.stbname = 'stb' + self.ntbname = 'ntb' + self.rowNum = 10 + self.tbnum = 3 + self.ts = 1537146000000 + self.binary_str = 'taosdata' + self.nchar_str = '涛思数据' + self.str_length = 20 + self.column_dict = { + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': f'binary({self.str_length})', + 'col13': f'nchar({self.str_length})', + + } + + self.tinyint_val = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) + self.smallint_val = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) + self.int_val = random.randint(constant.INT_MIN,constant.INT_MAX) + self.bigint_val = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX) + self.untingint_val = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX) + self.unsmallint_val = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX) + self.unint_val = random.randint(constant.INT_UN_MIN,constant.INT_MAX) + self.unbigint_val = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX) + self.float_val = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) + self.double_val = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) + self.bool_val = random.randint(0,100)%2 + self.binary_val = tdCom.getLongName(random.randint(0,self.str_length)) + self.nchar_val = tdCom.getLongName(random.randint(0,self.str_length)) + self.base_data = { + 'tinyint':self.tinyint_val, + 'smallint':self.smallint_val, + 'int':self.int_val, + 'bigint':self.bigint_val, + 'tinyint unsigned':self.untingint_val, + 'smallint unsigned':self.unsmallint_val, + 'int unsigned':self.unint_val, + 'bigint unsigned':self.unbigint_val, + 'bool':self.bool_val, + 'float':self.float_val, + 'double':self.double_val, + 'binary':self.binary_val, + 'nchar':self.nchar_val + } + + def insert_base_data(self,col_type,tbname,rows,base_data): + for i in range(rows): + if col_type.lower() == 'tinyint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint"]})') + elif col_type.lower() == 'smallint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint"]})') + elif col_type.lower() == 'int': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int"]})') + elif col_type.lower() == 'bigint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint"]})') + elif col_type.lower() == 'tinyint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint unsigned"]})') + elif col_type.lower() == 'smallint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint unsigned"]})') + elif col_type.lower() == 'int unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int unsigned"]})') + elif col_type.lower() == 'bigint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint unsigned"]})') + elif col_type.lower() == 'bool': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bool"]})') + elif col_type.lower() == 'float': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["float"]})') + elif col_type.lower() == 'double': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["double"]})') + elif 'binary' in col_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['binary']}")''') + elif 'nchar' in col_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['nchar']}")''') + def delete_all_data(self,tbname,col_type,row_num,base_data,dbname,tb_type,tb_num=1,stbname=''): + tdSql.query(f'select count(*) from {tbname}') + tdSql.execute(f'delete from {tbname}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select * from {tbname}') + tdSql.checkRows(0) + if tb_type == 'ntb' or tb_type == 'ctb': + if tb_type == 'ctb': + tdSql.query(f'select count(*) from {stbname}') + if tb_num <= 1: + if len(tdSql.queryResult) != 0: + tdLog.exit('delete case failure!') + else: + tdSql.checkEqual(tdSql.queryResult[0][0],(tb_num-1)*row_num) + + self.insert_base_data(col_type,tbname,row_num,base_data) + elif tb_type == 'stb': + for i in range(tb_num): + self.insert_base_data(col_type,f'{tbname}_{i}',row_num,base_data) + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select * from {tbname}') + if tb_type == 'ntb' or tb_type == 'ctb': + tdSql.checkRows(row_num) + elif tb_type =='stb': + tdSql.checkRows(row_num*tb_num) + def delete_one_row(self,tbname,column_type,column_name,base_data,row_num,dbname,tb_type,tb_num=1): + tdSql.execute(f'delete from {tbname} where ts={self.ts}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {column_name} from {tbname}') + if tb_type == 'ntb' or tb_type == 'ctb': + tdSql.checkRows(row_num-1) + elif tb_type == 'stb': + tdSql.checkRows((row_num-1)*tb_num) + tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}') + tdSql.checkRows(0) + if tb_type == 'ntb' or tb_type == 'ctb': + if 'binary' in column_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['binary']}")''') + elif 'nchar' in column_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['nchar']}")''') + else: + tdSql.execute(f'insert into {tbname} values({self.ts},{base_data[column_type]})') + elif tb_type == 'stb': + for i in range(tb_num): + if 'binary' in column_type.lower(): + tdSql.execute(f'''insert into {tbname}_{i} values({self.ts},"{base_data['binary']}")''') + elif 'nchar' in column_type.lower(): + tdSql.execute(f'''insert into {tbname}_{i} values({self.ts},"{base_data['nchar']}")''') + else: + tdSql.execute(f'insert into {tbname}_{i} values({self.ts},{base_data[column_type]})') + tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}') + if column_type.lower() == 'float' or column_type.lower() == 'double': + if abs(tdSql.queryResult[0][0] - base_data[column_type]) / base_data[column_type] <= 0.0001: + tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0]) + else: + tdLog.exit(f'{column_type} data check failure') + elif 'binary' in column_type.lower(): + tdSql.checkEqual(tdSql.queryResult[0][0],base_data['binary']) + elif 'nchar' in column_type.lower(): + tdSql.checkEqual(tdSql.queryResult[0][0],base_data['nchar']) + else: + tdSql.checkEqual(tdSql.queryResult[0][0],base_data[column_type]) + def delete_rows(self,dbname,tbname,col_name,col_type,base_data,row_num,tb_type,tb_num=1): + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts>{self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + if tb_type == 'ntb' or tb_type == 'ctb': + tdSql.checkRows(i+1) + self.insert_base_data(col_type,tbname,row_num,base_data) + elif tb_type == 'stb': + tdSql.checkRows((i+1)*tb_num) + for j in range(tb_num): + self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts>={self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + if tb_type == 'ntb' or tb_type == 'ctb': + tdSql.checkRows(i) + self.insert_base_data(col_type,tbname,row_num,base_data) + elif tb_type == 'stb': + tdSql.checkRows(i*tb_num) + for j in range(tb_num): + self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts<={self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + if tb_type == 'ntb' or tb_type == 'ctb': + tdSql.checkRows(row_num-i-1) + self.insert_base_data(col_type,tbname,row_num,base_data) + elif tb_type == 'stb': + tdSql.checkRows((row_num-i-1)*tb_num) + for j in range(tb_num): + self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts<{self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + if tb_type == 'ntb' or tb_type == 'ctb': + tdSql.checkRows(row_num-i) + self.insert_base_data(col_type,tbname,row_num,base_data) + elif tb_type == 'stb': + tdSql.checkRows((row_num-i)*tb_num) + for j in range(tb_num): + self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts between {self.ts} and {self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + if tb_type == 'ntb' or tb_type == 'ctb': + tdSql.checkRows(row_num - i-1) + self.insert_base_data(col_type,tbname,row_num,base_data) + elif tb_type == 'stb': + tdSql.checkRows(tb_num*(row_num - i-1)) + for j in range(tb_num): + self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) + tdSql.execute(f'delete from {tbname} where ts between {self.ts+i+1} and {self.ts}') + tdSql.query(f'select {col_name} from {tbname}') + if tb_type == 'ntb' or tb_type == 'ctb': + tdSql.checkRows(row_num) + elif tb_type == 'stb': + tdSql.checkRows(tb_num*row_num) + def delete_error(self,tbname,column_name,column_type,base_data): + for error_list in ['',f'ts = {self.ts} and',f'ts = {self.ts} or']: + if 'binary' in column_type.lower(): + tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['binary']}"''') + elif 'nchar' in column_type.lower(): + tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['nchar']}"''') + else: + tdSql.error(f'delete from {tbname} where {error_list} {column_name} = {base_data[column_type]}') + + def delete_data_ntb(self): + tdSql.execute(f'create database if not exists {self.dbname}') + tdSql.execute(f'use {self.dbname}') + for col_name,col_type in self.column_dict.items(): + tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') + self.insert_base_data(col_type,self.ntbname,self.rowNum,self.base_data) + self.delete_one_row(self.ntbname,col_type,col_name,self.base_data,self.rowNum,self.dbname,'ntb') + self.delete_all_data(self.ntbname,col_type,self.rowNum,self.base_data,self.dbname,'ntb') + self.delete_error(self.ntbname,col_name,col_type,self.base_data) + self.delete_rows(self.dbname,self.ntbname,col_name,col_type,self.base_data,self.rowNum,'ntb') + for func in ['first','last']: + tdSql.query(f'select {func}(*) from {self.ntbname}') + tdSql.execute(f'drop table {self.ntbname}') + tdSql.execute(f'drop database {self.dbname}') + def delete_data_ctb(self): + tdSql.execute(f'create database if not exists {self.dbname}') + tdSql.execute(f'use {self.dbname}') + for col_name,col_type in self.column_dict.items(): + tdSql.execute(f'create table {self.stbname} (ts timestamp,{col_name} {col_type}) tags(t1 int)') + for i in range(self.tbnum): + tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags(1)') + self.insert_base_data(col_type,f'{self.stbname}_{i}',self.rowNum,self.base_data) + self.delete_one_row(f'{self.stbname}_{i}',col_type,col_name,self.base_data,self.rowNum,self.dbname,'ctb') + self.delete_all_data(f'{self.stbname}_{i}',col_type,self.rowNum,self.base_data,self.dbname,'ctb',i+1,self.stbname) + self.delete_error(f'{self.stbname}_{i}',col_name,col_type,self.base_data) + self.delete_rows(self.dbname,f'{self.stbname}_{i}',col_name,col_type,self.base_data,self.rowNum,'ctb') + for func in ['first','last']: + tdSql.query(f'select {func}(*) from {self.stbname}_{i}') + tdSql.execute(f'drop table {self.stbname}') + def delete_data_stb(self): + tdSql.execute(f'create database if not exists {self.dbname}') + tdSql.execute(f'use {self.dbname}') + for col_name,col_type in self.column_dict.items(): + tdSql.execute(f'create table {self.stbname} (ts timestamp,{col_name} {col_type}) tags(t1 int)') + for i in range(self.tbnum): + tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags(1)') + self.insert_base_data(col_type,f'{self.stbname}_{i}',self.rowNum,self.base_data) + self.delete_error(self.stbname,col_name,col_type,self.base_data) + self.delete_one_row(self.stbname,col_type,col_name,self.base_data,self.rowNum,self.dbname,'stb',self.tbnum) + self.delete_all_data(self.stbname,col_type,self.rowNum,self.base_data,self.dbname,'stb',self.tbnum) + self.delete_rows(self.dbname,self.stbname,col_name,col_type,self.base_data,self.rowNum,'stb',self.tbnum) + for func in ['first','last']: + tdSql.query(f'select {func}(*) from {self.stbname}') + tdSql.execute(f'drop table {self.stbname}') + tdSql.execute(f'drop database {self.dbname}') + def run(self): + self.delete_data_ntb() + self.delete_data_ctb() + self.delete_data_stb() + tdDnodes.stoptaosd(1) + tdDnodes.starttaosd(1) + self.delete_data_ntb() + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From abc17227ab47fcfa522ceb1442bf873fde2edab0 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 3 Nov 2022 19:45:14 +0800 Subject: [PATCH 26/82] enh: code optimization for PATH_MAX --- source/dnode/vnode/src/tsdb/tsdbOpen.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index e109edca11..bb20a9b012 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -38,14 +38,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee int slen = 0; *ppTsdb = NULL; - slen = TD_PATH_MAX; - - if (slen > TD_PATH_MAX) { - terrno = TSDB_CODE_OUT_OF_RANGE; - tsdbError("vgId:%d, tsdb open failed since %s, path: %s%s%s", TD_VID(pVnode), terrstr(), pVnode->path, TD_DIRSEP, - dir); - return -1; - } + slen = strlen(pVnode->path) + strlen(dir) + 2; // create handle pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen); From 5312b4b996c1fa10f4a335b1801084a01950c0fc Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 3 Nov 2022 20:18:15 +0800 Subject: [PATCH 27/82] test:add compatibility cases: 4096 columns of table data and tests that need to recover data from wal files --- tests/system-test/0-others/compa4096.json | 11 +- tests/system-test/0-others/compatibility.py | 4 +- .../5dnode3mnodeSep1VnodeStopDnodeCreateDb.py | 2 +- tests/system-test/fulltest.sh | 672 +++++++++--------- 4 files changed, 346 insertions(+), 343 deletions(-) diff --git a/tests/system-test/0-others/compa4096.json b/tests/system-test/0-others/compa4096.json index 85b63ed35f..46f98be8bd 100644 --- a/tests/system-test/0-others/compa4096.json +++ b/tests/system-test/0-others/compa4096.json @@ -11,27 +11,26 @@ "result_file": "taosBenchmark_result.log", "confirm_parameter_prompt": "no", "insert_interval": 0, - "num_of_records_per_req": 100, + "num_of_records_per_req": 1000000, "max_sql_len": 1024000, "databases": [ { "dbinfo": { - "name": "taosc_sql", + "name": "db4096", "drop": "yes", "replica": 1, "duration": 10, "precision": "ms", "keep": 3650, - "minRows": 100, - "maxRows": 4096, "comp": 2, - "vgroups": 30 + "vgroups": 3, + "buffer": 1000 }, "super_tables": [ { "name": "stb0", "child_table_exists": "no", - "childtable_count": 100, + "childtable_count": 5, "childtable_prefix": "ctb0", "escape_character": "no", "auto_create_table": "no", diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index 827bb5906e..e70eab64bf 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -111,7 +111,9 @@ class TDTestCase: print(f"start taosd: nohup taosd -c {cPath} & ") os.system(f" nohup taosd -c {cPath} & " ) sleep(2) - tdLog.info(f" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f compa4096.json -y ") + tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ") + os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y") + os.system("pkill -9 taosd") diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py index d3ec429bdb..9f365440e2 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py @@ -195,7 +195,7 @@ class TDTestCase: def run(self): # print(self.master_dnode.cfgDict) - self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=10,stopRole='dnode') + self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=4,stopRole='dnode') def stop(self): tdSql.close() diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index febcc4b728..cd65b395f8 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -232,9 +232,11 @@ python3 ./test.py -f 2-query/function_diff.py python3 ./test.py -f 2-query/queryQnode.py python3 ./test.py -f 6-cluster/5dnode1mnode.py -python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 -i False python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 -i False python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 -n 3 @@ -291,351 +293,351 @@ python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_query python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py -N 4 -M 1 -python3 ./test.py -f 7-tmq/create_wrong_topic.py -python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 -python3 ./test.py -f 7-tmq/basic5.py -python3 ./test.py -f 7-tmq/subscribeDb.py -python3 ./test.py -f 7-tmq/subscribeDb0.py -python3 ./test.py -f 7-tmq/subscribeDb1.py -python3 ./test.py -f 7-tmq/subscribeDb2.py -python3 ./test.py -f 7-tmq/subscribeDb3.py -python3 ./test.py -f 7-tmq/subscribeDb4.py -python3 ./test.py -f 7-tmq/subscribeStb.py -python3 ./test.py -f 7-tmq/subscribeStb0.py -python3 ./test.py -f 7-tmq/subscribeStb1.py -python3 ./test.py -f 7-tmq/subscribeStb2.py -python3 ./test.py -f 7-tmq/subscribeStb3.py -python3 ./test.py -f 7-tmq/subscribeStb4.py -python3 ./test.py -f 7-tmq/db.py -python3 ./test.py -f 7-tmq/tmqError.py -python3 ./test.py -f 7-tmq/schema.py -python3 ./test.py -f 7-tmq/stbFilter.py -python3 ./test.py -f 7-tmq/tmqCheckData.py -python3 ./test.py -f 7-tmq/tmqCheckData1.py -#python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 5 -python3 ./test.py -f 7-tmq/tmqConsumerGroup.py -python3 ./test.py -f 7-tmq/tmqShow.py -python3 ./test.py -f 7-tmq/tmqAlterSchema.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py -python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py -python3 ./test.py -f 7-tmq/tmqDnodeRestart.py -python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py -python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py -python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py -python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py -python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py -python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py -python3 ./test.py -f 7-tmq/tmqDropStb.py -python3 ./test.py -f 7-tmq/tmqDropStbCtb.py -python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py -python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py -python3 ./test.py -f 7-tmq/tmqUdf.py -python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py -python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py -python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py -python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py -python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py -python3 ./test.py -f 7-tmq/tmq_taosx.py -python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py +# python3 ./test.py -f 7-tmq/create_wrong_topic.py +# python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 +# python3 ./test.py -f 7-tmq/basic5.py +# python3 ./test.py -f 7-tmq/subscribeDb.py +# python3 ./test.py -f 7-tmq/subscribeDb0.py +# python3 ./test.py -f 7-tmq/subscribeDb1.py +# python3 ./test.py -f 7-tmq/subscribeDb2.py +# python3 ./test.py -f 7-tmq/subscribeDb3.py +# python3 ./test.py -f 7-tmq/subscribeDb4.py +# python3 ./test.py -f 7-tmq/subscribeStb.py +# python3 ./test.py -f 7-tmq/subscribeStb0.py +# python3 ./test.py -f 7-tmq/subscribeStb1.py +# python3 ./test.py -f 7-tmq/subscribeStb2.py +# python3 ./test.py -f 7-tmq/subscribeStb3.py +# python3 ./test.py -f 7-tmq/subscribeStb4.py +# python3 ./test.py -f 7-tmq/db.py +# python3 ./test.py -f 7-tmq/tmqError.py +# python3 ./test.py -f 7-tmq/schema.py +# python3 ./test.py -f 7-tmq/stbFilter.py +# python3 ./test.py -f 7-tmq/tmqCheckData.py +# python3 ./test.py -f 7-tmq/tmqCheckData1.py +# #python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 5 +# python3 ./test.py -f 7-tmq/tmqConsumerGroup.py +# python3 ./test.py -f 7-tmq/tmqShow.py +# python3 ./test.py -f 7-tmq/tmqAlterSchema.py +# python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py +# python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py +# python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg.py +# python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg.py +# python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb.py +# python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb.py +# python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py +# python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py +# python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py +# python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py +# python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py +# python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py +# python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py +# python3 ./test.py -f 7-tmq/tmqDnodeRestart.py +# python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py +# python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py +# python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py +# python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py +# python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py +# python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py +# python3 ./test.py -f 7-tmq/tmqDropStb.py +# python3 ./test.py -f 7-tmq/tmqDropStbCtb.py +# python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py +# python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py +# python3 ./test.py -f 7-tmq/tmqUdf.py +# python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py +# python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py +# python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py +# python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py +# python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py +# python3 ./test.py -f 7-tmq/tmq_taosx.py +# python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py -python3 ./test.py -f 99-TDcase/TD-19201.py +# python3 ./test.py -f 99-TDcase/TD-19201.py -#------------querPolicy 2----------- +# #------------querPolicy 2----------- -python3 ./test.py -f 2-query/between.py -Q 2 -python3 ./test.py -f 2-query/distinct.py -Q 2 -python3 ./test.py -f 2-query/varchar.py -Q 2 -python3 ./test.py -f 2-query/ltrim.py -Q 2 -python3 ./test.py -f 2-query/rtrim.py -Q 2 -python3 ./test.py -f 2-query/length.py -Q 2 -python3 ./test.py -f 2-query/char_length.py -Q 2 -python3 ./test.py -f 2-query/upper.py -Q 2 -python3 ./test.py -f 2-query/lower.py -Q 2 -python3 ./test.py -f 2-query/join.py -Q 2 -python3 ./test.py -f 2-query/join2.py -Q 2 -python3 ./test.py -f 2-query/cast.py -Q 2 -python3 ./test.py -f 2-query/substr.py -Q 2 -python3 ./test.py -f 2-query/union.py -Q 2 -python3 ./test.py -f 2-query/union1.py -Q 2 -python3 ./test.py -f 2-query/concat.py -Q 2 -python3 ./test.py -f 2-query/concat2.py -Q 2 -python3 ./test.py -f 2-query/concat_ws.py -Q 2 -python3 ./test.py -f 2-query/concat_ws2.py -Q 2 -python3 ./test.py -f 2-query/check_tsdb.py -Q 2 -python3 ./test.py -f 2-query/spread.py -Q 2 -python3 ./test.py -f 2-query/hyperloglog.py -Q 2 -python3 ./test.py -f 2-query/explain.py -Q 2 -python3 ./test.py -f 2-query/leastsquares.py -Q 2 -python3 ./test.py -f 2-query/timezone.py -Q 2 -python3 ./test.py -f 2-query/Now.py -Q 2 -python3 ./test.py -f 2-query/Today.py -Q 2 -python3 ./test.py -f 2-query/max.py -Q 2 -python3 ./test.py -f 2-query/min.py -Q 2 -python3 ./test.py -f 2-query/count.py -Q 2 -python3 ./test.py -f 2-query/last.py -Q 2 -python3 ./test.py -f 2-query/first.py -Q 2 -python3 ./test.py -f 2-query/To_iso8601.py -Q 2 -python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 2 -python3 ./test.py -f 2-query/timetruncate.py -Q 2 -python3 ./test.py -f 2-query/diff.py -Q 2 -python3 ./test.py -f 2-query/Timediff.py -Q 2 -python3 ./test.py -f 2-query/json_tag.py -Q 2 -python3 ./test.py -f 2-query/top.py -Q 2 -python3 ./test.py -f 2-query/bottom.py -Q 2 -python3 ./test.py -f 2-query/percentile.py -Q 2 -python3 ./test.py -f 2-query/apercentile.py -Q 2 -python3 ./test.py -f 2-query/abs.py -Q 2 -python3 ./test.py -f 2-query/ceil.py -Q 2 -python3 ./test.py -f 2-query/floor.py -Q 2 -python3 ./test.py -f 2-query/round.py -Q 2 -python3 ./test.py -f 2-query/log.py -Q 2 -python3 ./test.py -f 2-query/pow.py -Q 2 -python3 ./test.py -f 2-query/sqrt.py -Q 2 -python3 ./test.py -f 2-query/sin.py -Q 2 -python3 ./test.py -f 2-query/cos.py -Q 2 -python3 ./test.py -f 2-query/tan.py -Q 2 -python3 ./test.py -f 2-query/arcsin.py -Q 2 -python3 ./test.py -f 2-query/arccos.py -Q 2 -python3 ./test.py -f 2-query/arctan.py -Q 2 -python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 2 -python3 ./test.py -f 2-query/interp.py -Q 2 +# python3 ./test.py -f 2-query/between.py -Q 2 +# python3 ./test.py -f 2-query/distinct.py -Q 2 +# python3 ./test.py -f 2-query/varchar.py -Q 2 +# python3 ./test.py -f 2-query/ltrim.py -Q 2 +# python3 ./test.py -f 2-query/rtrim.py -Q 2 +# python3 ./test.py -f 2-query/length.py -Q 2 +# python3 ./test.py -f 2-query/char_length.py -Q 2 +# python3 ./test.py -f 2-query/upper.py -Q 2 +# python3 ./test.py -f 2-query/lower.py -Q 2 +# python3 ./test.py -f 2-query/join.py -Q 2 +# python3 ./test.py -f 2-query/join2.py -Q 2 +# python3 ./test.py -f 2-query/cast.py -Q 2 +# python3 ./test.py -f 2-query/substr.py -Q 2 +# python3 ./test.py -f 2-query/union.py -Q 2 +# python3 ./test.py -f 2-query/union1.py -Q 2 +# python3 ./test.py -f 2-query/concat.py -Q 2 +# python3 ./test.py -f 2-query/concat2.py -Q 2 +# python3 ./test.py -f 2-query/concat_ws.py -Q 2 +# python3 ./test.py -f 2-query/concat_ws2.py -Q 2 +# python3 ./test.py -f 2-query/check_tsdb.py -Q 2 +# python3 ./test.py -f 2-query/spread.py -Q 2 +# python3 ./test.py -f 2-query/hyperloglog.py -Q 2 +# python3 ./test.py -f 2-query/explain.py -Q 2 +# python3 ./test.py -f 2-query/leastsquares.py -Q 2 +# python3 ./test.py -f 2-query/timezone.py -Q 2 +# python3 ./test.py -f 2-query/Now.py -Q 2 +# python3 ./test.py -f 2-query/Today.py -Q 2 +# python3 ./test.py -f 2-query/max.py -Q 2 +# python3 ./test.py -f 2-query/min.py -Q 2 +# python3 ./test.py -f 2-query/count.py -Q 2 +# python3 ./test.py -f 2-query/last.py -Q 2 +# python3 ./test.py -f 2-query/first.py -Q 2 +# python3 ./test.py -f 2-query/To_iso8601.py -Q 2 +# python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 2 +# python3 ./test.py -f 2-query/timetruncate.py -Q 2 +# python3 ./test.py -f 2-query/diff.py -Q 2 +# python3 ./test.py -f 2-query/Timediff.py -Q 2 +# python3 ./test.py -f 2-query/json_tag.py -Q 2 +# python3 ./test.py -f 2-query/top.py -Q 2 +# python3 ./test.py -f 2-query/bottom.py -Q 2 +# python3 ./test.py -f 2-query/percentile.py -Q 2 +# python3 ./test.py -f 2-query/apercentile.py -Q 2 +# python3 ./test.py -f 2-query/abs.py -Q 2 +# python3 ./test.py -f 2-query/ceil.py -Q 2 +# python3 ./test.py -f 2-query/floor.py -Q 2 +# python3 ./test.py -f 2-query/round.py -Q 2 +# python3 ./test.py -f 2-query/log.py -Q 2 +# python3 ./test.py -f 2-query/pow.py -Q 2 +# python3 ./test.py -f 2-query/sqrt.py -Q 2 +# python3 ./test.py -f 2-query/sin.py -Q 2 +# python3 ./test.py -f 2-query/cos.py -Q 2 +# python3 ./test.py -f 2-query/tan.py -Q 2 +# python3 ./test.py -f 2-query/arcsin.py -Q 2 +# python3 ./test.py -f 2-query/arccos.py -Q 2 +# python3 ./test.py -f 2-query/arctan.py -Q 2 +# python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 2 +# python3 ./test.py -f 2-query/interp.py -Q 2 -python3 ./test.py -f 2-query/nestedQuery.py -Q 2 -python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 -python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 -python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 -python3 ./test.py -f 2-query/stablity.py -Q 2 -python3 ./test.py -f 2-query/stablity_1.py -Q 2 +# python3 ./test.py -f 2-query/nestedQuery.py -Q 2 +# python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 +# python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 +# python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 +# python3 ./test.py -f 2-query/stablity.py -Q 2 +# python3 ./test.py -f 2-query/stablity_1.py -Q 2 -python3 ./test.py -f 2-query/avg.py -Q 2 -python3 ./test.py -f 2-query/elapsed.py -Q 2 -python3 ./test.py -f 2-query/csum.py -Q 2 -python3 ./test.py -f 2-query/mavg.py -Q 2 -python3 ./test.py -f 2-query/sample.py -Q 2 -python3 ./test.py -f 2-query/function_diff.py -Q 2 -python3 ./test.py -f 2-query/unique.py -Q 2 -python3 ./test.py -f 2-query/stateduration.py -Q 2 -python3 ./test.py -f 2-query/function_stateduration.py -Q 2 -python3 ./test.py -f 2-query/statecount.py -Q 2 -python3 ./test.py -f 2-query/tail.py -Q 2 -python3 ./test.py -f 2-query/ttl_comment.py -Q 2 -python3 ./test.py -f 2-query/distribute_agg_count.py -Q 2 -python3 ./test.py -f 2-query/distribute_agg_max.py -Q 2 -python3 ./test.py -f 2-query/distribute_agg_min.py -Q 2 -python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 2 -python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 2 -python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 2 -python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 2 -python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 2 -python3 ./test.py -f 2-query/twa.py -Q 2 -python3 ./test.py -f 2-query/irate.py -Q 2 -python3 ./test.py -f 2-query/function_null.py -Q 2 -python3 ./test.py -f 2-query/count_partition.py -Q 2 -python3 ./test.py -f 2-query/max_partition.py -Q 2 -python3 ./test.py -f 2-query/last_row.py -Q 2 -python3 ./test.py -f 2-query/tsbsQuery.py -Q 2 -python3 ./test.py -f 2-query/sml.py -Q 2 +# python3 ./test.py -f 2-query/avg.py -Q 2 +# python3 ./test.py -f 2-query/elapsed.py -Q 2 +# python3 ./test.py -f 2-query/csum.py -Q 2 +# python3 ./test.py -f 2-query/mavg.py -Q 2 +# python3 ./test.py -f 2-query/sample.py -Q 2 +# python3 ./test.py -f 2-query/function_diff.py -Q 2 +# python3 ./test.py -f 2-query/unique.py -Q 2 +# python3 ./test.py -f 2-query/stateduration.py -Q 2 +# python3 ./test.py -f 2-query/function_stateduration.py -Q 2 +# python3 ./test.py -f 2-query/statecount.py -Q 2 +# python3 ./test.py -f 2-query/tail.py -Q 2 +# python3 ./test.py -f 2-query/ttl_comment.py -Q 2 +# python3 ./test.py -f 2-query/distribute_agg_count.py -Q 2 +# python3 ./test.py -f 2-query/distribute_agg_max.py -Q 2 +# python3 ./test.py -f 2-query/distribute_agg_min.py -Q 2 +# python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 2 +# python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 2 +# python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 2 +# python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 2 +# python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 2 +# python3 ./test.py -f 2-query/twa.py -Q 2 +# python3 ./test.py -f 2-query/irate.py -Q 2 +# python3 ./test.py -f 2-query/function_null.py -Q 2 +# python3 ./test.py -f 2-query/count_partition.py -Q 2 +# python3 ./test.py -f 2-query/max_partition.py -Q 2 +# python3 ./test.py -f 2-query/last_row.py -Q 2 +# python3 ./test.py -f 2-query/tsbsQuery.py -Q 2 +# python3 ./test.py -f 2-query/sml.py -Q 2 -#------------querPolicy 3----------- -python3 ./test.py -f 2-query/between.py -Q 3 -python3 ./test.py -f 2-query/distinct.py -Q 3 -python3 ./test.py -f 2-query/varchar.py -Q 3 -python3 ./test.py -f 2-query/ltrim.py -Q 3 -python3 ./test.py -f 2-query/rtrim.py -Q 3 -python3 ./test.py -f 2-query/length.py -Q 3 -python3 ./test.py -f 2-query/char_length.py -Q 3 -python3 ./test.py -f 2-query/upper.py -Q 3 -python3 ./test.py -f 2-query/lower.py -Q 3 -python3 ./test.py -f 2-query/join.py -Q 3 -python3 ./test.py -f 2-query/join2.py -Q 3 -python3 ./test.py -f 2-query/cast.py -Q 3 -python3 ./test.py -f 2-query/substr.py -Q 3 -python3 ./test.py -f 2-query/union.py -Q 3 -python3 ./test.py -f 2-query/union1.py -Q 3 -python3 ./test.py -f 2-query/concat.py -Q 3 -python3 ./test.py -f 2-query/concat2.py -Q 3 -python3 ./test.py -f 2-query/concat_ws.py -Q 3 -python3 ./test.py -f 2-query/concat_ws2.py -Q 3 -python3 ./test.py -f 2-query/check_tsdb.py -Q 3 -python3 ./test.py -f 2-query/spread.py -Q 3 -python3 ./test.py -f 2-query/hyperloglog.py -Q 3 -python3 ./test.py -f 2-query/explain.py -Q 3 -python3 ./test.py -f 2-query/leastsquares.py -Q 3 -python3 ./test.py -f 2-query/timezone.py -Q 3 -python3 ./test.py -f 2-query/Now.py -Q 3 -python3 ./test.py -f 2-query/Today.py -Q 3 -python3 ./test.py -f 2-query/max.py -Q 3 -python3 ./test.py -f 2-query/min.py -Q 3 -python3 ./test.py -f 2-query/count.py -Q 3 -python3 ./test.py -f 2-query/last.py -Q 3 -python3 ./test.py -f 2-query/first.py -Q 3 -python3 ./test.py -f 2-query/To_iso8601.py -Q 3 -python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 3 -python3 ./test.py -f 2-query/timetruncate.py -Q 3 -python3 ./test.py -f 2-query/diff.py -Q 3 -python3 ./test.py -f 2-query/Timediff.py -Q 3 -python3 ./test.py -f 2-query/json_tag.py -Q 3 -python3 ./test.py -f 2-query/top.py -Q 3 -python3 ./test.py -f 2-query/bottom.py -Q 3 -python3 ./test.py -f 2-query/percentile.py -Q 3 -python3 ./test.py -f 2-query/apercentile.py -Q 3 -python3 ./test.py -f 2-query/abs.py -Q 3 -python3 ./test.py -f 2-query/ceil.py -Q 3 -python3 ./test.py -f 2-query/floor.py -Q 3 -python3 ./test.py -f 2-query/round.py -Q 3 -python3 ./test.py -f 2-query/log.py -Q 3 -python3 ./test.py -f 2-query/pow.py -Q 3 -python3 ./test.py -f 2-query/sqrt.py -Q 3 -python3 ./test.py -f 2-query/sin.py -Q 3 -python3 ./test.py -f 2-query/cos.py -Q 3 -python3 ./test.py -f 2-query/tan.py -Q 3 -python3 ./test.py -f 2-query/arcsin.py -Q 3 -python3 ./test.py -f 2-query/arccos.py -Q 3 -python3 ./test.py -f 2-query/arctan.py -Q 3 -python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 3 +# #------------querPolicy 3----------- +# python3 ./test.py -f 2-query/between.py -Q 3 +# python3 ./test.py -f 2-query/distinct.py -Q 3 +# python3 ./test.py -f 2-query/varchar.py -Q 3 +# python3 ./test.py -f 2-query/ltrim.py -Q 3 +# python3 ./test.py -f 2-query/rtrim.py -Q 3 +# python3 ./test.py -f 2-query/length.py -Q 3 +# python3 ./test.py -f 2-query/char_length.py -Q 3 +# python3 ./test.py -f 2-query/upper.py -Q 3 +# python3 ./test.py -f 2-query/lower.py -Q 3 +# python3 ./test.py -f 2-query/join.py -Q 3 +# python3 ./test.py -f 2-query/join2.py -Q 3 +# python3 ./test.py -f 2-query/cast.py -Q 3 +# python3 ./test.py -f 2-query/substr.py -Q 3 +# python3 ./test.py -f 2-query/union.py -Q 3 +# python3 ./test.py -f 2-query/union1.py -Q 3 +# python3 ./test.py -f 2-query/concat.py -Q 3 +# python3 ./test.py -f 2-query/concat2.py -Q 3 +# python3 ./test.py -f 2-query/concat_ws.py -Q 3 +# python3 ./test.py -f 2-query/concat_ws2.py -Q 3 +# python3 ./test.py -f 2-query/check_tsdb.py -Q 3 +# python3 ./test.py -f 2-query/spread.py -Q 3 +# python3 ./test.py -f 2-query/hyperloglog.py -Q 3 +# python3 ./test.py -f 2-query/explain.py -Q 3 +# python3 ./test.py -f 2-query/leastsquares.py -Q 3 +# python3 ./test.py -f 2-query/timezone.py -Q 3 +# python3 ./test.py -f 2-query/Now.py -Q 3 +# python3 ./test.py -f 2-query/Today.py -Q 3 +# python3 ./test.py -f 2-query/max.py -Q 3 +# python3 ./test.py -f 2-query/min.py -Q 3 +# python3 ./test.py -f 2-query/count.py -Q 3 +# python3 ./test.py -f 2-query/last.py -Q 3 +# python3 ./test.py -f 2-query/first.py -Q 3 +# python3 ./test.py -f 2-query/To_iso8601.py -Q 3 +# python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 3 +# python3 ./test.py -f 2-query/timetruncate.py -Q 3 +# python3 ./test.py -f 2-query/diff.py -Q 3 +# python3 ./test.py -f 2-query/Timediff.py -Q 3 +# python3 ./test.py -f 2-query/json_tag.py -Q 3 +# python3 ./test.py -f 2-query/top.py -Q 3 +# python3 ./test.py -f 2-query/bottom.py -Q 3 +# python3 ./test.py -f 2-query/percentile.py -Q 3 +# python3 ./test.py -f 2-query/apercentile.py -Q 3 +# python3 ./test.py -f 2-query/abs.py -Q 3 +# python3 ./test.py -f 2-query/ceil.py -Q 3 +# python3 ./test.py -f 2-query/floor.py -Q 3 +# python3 ./test.py -f 2-query/round.py -Q 3 +# python3 ./test.py -f 2-query/log.py -Q 3 +# python3 ./test.py -f 2-query/pow.py -Q 3 +# python3 ./test.py -f 2-query/sqrt.py -Q 3 +# python3 ./test.py -f 2-query/sin.py -Q 3 +# python3 ./test.py -f 2-query/cos.py -Q 3 +# python3 ./test.py -f 2-query/tan.py -Q 3 +# python3 ./test.py -f 2-query/arcsin.py -Q 3 +# python3 ./test.py -f 2-query/arccos.py -Q 3 +# python3 ./test.py -f 2-query/arctan.py -Q 3 +# python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 3 -python3 ./test.py -f 2-query/nestedQuery.py -Q 3 -python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 -python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 -python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 -python3 ./test.py -f 2-query/stablity.py -Q 3 -python3 ./test.py -f 2-query/stablity_1.py -Q 3 +# python3 ./test.py -f 2-query/nestedQuery.py -Q 3 +# python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 +# python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 +# python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 +# python3 ./test.py -f 2-query/stablity.py -Q 3 +# python3 ./test.py -f 2-query/stablity_1.py -Q 3 -python3 ./test.py -f 2-query/avg.py -Q 3 -python3 ./test.py -f 2-query/elapsed.py -Q 3 -python3 ./test.py -f 2-query/csum.py -Q 3 -python3 ./test.py -f 2-query/mavg.py -Q 3 -python3 ./test.py -f 2-query/sample.py -Q 3 -python3 ./test.py -f 2-query/function_diff.py -Q 3 -python3 ./test.py -f 2-query/unique.py -Q 3 -python3 ./test.py -f 2-query/stateduration.py -Q 3 -python3 ./test.py -f 2-query/function_stateduration.py -Q 3 -python3 ./test.py -f 2-query/statecount.py -Q 3 -python3 ./test.py -f 2-query/tail.py -Q 3 -python3 ./test.py -f 2-query/ttl_comment.py -Q 3 -python3 ./test.py -f 2-query/distribute_agg_count.py -Q 3 -python3 ./test.py -f 2-query/distribute_agg_max.py -Q 3 -python3 ./test.py -f 2-query/distribute_agg_min.py -Q 3 -python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 3 -python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 3 -python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 3 -python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 3 -python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 3 -python3 ./test.py -f 2-query/twa.py -Q 3 -python3 ./test.py -f 2-query/irate.py -Q 3 -python3 ./test.py -f 2-query/function_null.py -Q 3 -python3 ./test.py -f 2-query/count_partition.py -Q 3 -python3 ./test.py -f 2-query/max_partition.py -Q 3 -python3 ./test.py -f 2-query/last_row.py -Q 3 -python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 -python3 ./test.py -f 2-query/sml.py -Q 3 -python3 ./test.py -f 2-query/interp.py -Q 3 +# python3 ./test.py -f 2-query/avg.py -Q 3 +# python3 ./test.py -f 2-query/elapsed.py -Q 3 +# python3 ./test.py -f 2-query/csum.py -Q 3 +# python3 ./test.py -f 2-query/mavg.py -Q 3 +# python3 ./test.py -f 2-query/sample.py -Q 3 +# python3 ./test.py -f 2-query/function_diff.py -Q 3 +# python3 ./test.py -f 2-query/unique.py -Q 3 +# python3 ./test.py -f 2-query/stateduration.py -Q 3 +# python3 ./test.py -f 2-query/function_stateduration.py -Q 3 +# python3 ./test.py -f 2-query/statecount.py -Q 3 +# python3 ./test.py -f 2-query/tail.py -Q 3 +# python3 ./test.py -f 2-query/ttl_comment.py -Q 3 +# python3 ./test.py -f 2-query/distribute_agg_count.py -Q 3 +# python3 ./test.py -f 2-query/distribute_agg_max.py -Q 3 +# python3 ./test.py -f 2-query/distribute_agg_min.py -Q 3 +# python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 3 +# python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 3 +# python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 3 +# python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 3 +# python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 3 +# python3 ./test.py -f 2-query/twa.py -Q 3 +# python3 ./test.py -f 2-query/irate.py -Q 3 +# python3 ./test.py -f 2-query/function_null.py -Q 3 +# python3 ./test.py -f 2-query/count_partition.py -Q 3 +# python3 ./test.py -f 2-query/max_partition.py -Q 3 +# python3 ./test.py -f 2-query/last_row.py -Q 3 +# python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 +# python3 ./test.py -f 2-query/sml.py -Q 3 +# python3 ./test.py -f 2-query/interp.py -Q 3 -#------------querPolicy 4----------- +# #------------querPolicy 4----------- -python3 ./test.py -f 2-query/between.py -Q 4 -python3 ./test.py -f 2-query/distinct.py -Q 4 -python3 ./test.py -f 2-query/varchar.py -Q 4 -python3 ./test.py -f 2-query/ltrim.py -Q 4 -python3 ./test.py -f 2-query/rtrim.py -Q 4 -python3 ./test.py -f 2-query/length.py -Q 4 -python3 ./test.py -f 2-query/char_length.py -Q 4 -python3 ./test.py -f 2-query/upper.py -Q 4 -python3 ./test.py -f 2-query/lower.py -Q 4 -python3 ./test.py -f 2-query/join.py -Q 4 -python3 ./test.py -f 2-query/join2.py -Q 4 -#python3 ./test.py -f 2-query/cast.py -Q 4 -python3 ./test.py -f 2-query/substr.py -Q 4 -python3 ./test.py -f 2-query/union.py -Q 4 -python3 ./test.py -f 2-query/union1.py -Q 4 -python3 ./test.py -f 2-query/concat.py -Q 4 -python3 ./test.py -f 2-query/concat2.py -Q 4 -python3 ./test.py -f 2-query/concat_ws.py -Q 4 -python3 ./test.py -f 2-query/concat_ws2.py -Q 4 -python3 ./test.py -f 2-query/check_tsdb.py -Q 4 -python3 ./test.py -f 2-query/spread.py -Q 4 -python3 ./test.py -f 2-query/hyperloglog.py -Q 4 -python3 ./test.py -f 2-query/explain.py -Q 4 -python3 ./test.py -f 2-query/leastsquares.py -Q 4 -python3 ./test.py -f 2-query/timezone.py -Q 4 -python3 ./test.py -f 2-query/Now.py -Q 4 -python3 ./test.py -f 2-query/Today.py -Q 4 -python3 ./test.py -f 2-query/max.py -Q 4 -python3 ./test.py -f 2-query/min.py -Q 4 -python3 ./test.py -f 2-query/count.py -Q 4 -python3 ./test.py -f 2-query/last.py -Q 4 -python3 ./test.py -f 2-query/first.py -Q 4 -python3 ./test.py -f 2-query/To_iso8601.py -Q 4 -python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 4 -python3 ./test.py -f 2-query/timetruncate.py -Q 4 -python3 ./test.py -f 2-query/diff.py -Q 4 -python3 ./test.py -f 2-query/Timediff.py -Q 4 -python3 ./test.py -f 2-query/json_tag.py -Q 4 -python3 ./test.py -f 2-query/top.py -Q 4 -python3 ./test.py -f 2-query/bottom.py -Q 4 -python3 ./test.py -f 2-query/percentile.py -Q 4 -python3 ./test.py -f 2-query/apercentile.py -Q 4 -python3 ./test.py -f 2-query/abs.py -Q 4 -python3 ./test.py -f 2-query/ceil.py -Q 4 -python3 ./test.py -f 2-query/floor.py -Q 4 -python3 ./test.py -f 2-query/round.py -Q 4 -python3 ./test.py -f 2-query/log.py -Q 4 -python3 ./test.py -f 2-query/pow.py -Q 4 -python3 ./test.py -f 2-query/sqrt.py -Q 4 -python3 ./test.py -f 2-query/sin.py -Q 4 -python3 ./test.py -f 2-query/cos.py -Q 4 -python3 ./test.py -f 2-query/tan.py -Q 4 -python3 ./test.py -f 2-query/arcsin.py -Q 4 -python3 ./test.py -f 2-query/arccos.py -Q 4 -python3 ./test.py -f 2-query/arctan.py -Q 4 -python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 4 +# python3 ./test.py -f 2-query/between.py -Q 4 +# python3 ./test.py -f 2-query/distinct.py -Q 4 +# python3 ./test.py -f 2-query/varchar.py -Q 4 +# python3 ./test.py -f 2-query/ltrim.py -Q 4 +# python3 ./test.py -f 2-query/rtrim.py -Q 4 +# python3 ./test.py -f 2-query/length.py -Q 4 +# python3 ./test.py -f 2-query/char_length.py -Q 4 +# python3 ./test.py -f 2-query/upper.py -Q 4 +# python3 ./test.py -f 2-query/lower.py -Q 4 +# python3 ./test.py -f 2-query/join.py -Q 4 +# python3 ./test.py -f 2-query/join2.py -Q 4 +# #python3 ./test.py -f 2-query/cast.py -Q 4 +# python3 ./test.py -f 2-query/substr.py -Q 4 +# python3 ./test.py -f 2-query/union.py -Q 4 +# python3 ./test.py -f 2-query/union1.py -Q 4 +# python3 ./test.py -f 2-query/concat.py -Q 4 +# python3 ./test.py -f 2-query/concat2.py -Q 4 +# python3 ./test.py -f 2-query/concat_ws.py -Q 4 +# python3 ./test.py -f 2-query/concat_ws2.py -Q 4 +# python3 ./test.py -f 2-query/check_tsdb.py -Q 4 +# python3 ./test.py -f 2-query/spread.py -Q 4 +# python3 ./test.py -f 2-query/hyperloglog.py -Q 4 +# python3 ./test.py -f 2-query/explain.py -Q 4 +# python3 ./test.py -f 2-query/leastsquares.py -Q 4 +# python3 ./test.py -f 2-query/timezone.py -Q 4 +# python3 ./test.py -f 2-query/Now.py -Q 4 +# python3 ./test.py -f 2-query/Today.py -Q 4 +# python3 ./test.py -f 2-query/max.py -Q 4 +# python3 ./test.py -f 2-query/min.py -Q 4 +# python3 ./test.py -f 2-query/count.py -Q 4 +# python3 ./test.py -f 2-query/last.py -Q 4 +# python3 ./test.py -f 2-query/first.py -Q 4 +# python3 ./test.py -f 2-query/To_iso8601.py -Q 4 +# python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 4 +# python3 ./test.py -f 2-query/timetruncate.py -Q 4 +# python3 ./test.py -f 2-query/diff.py -Q 4 +# python3 ./test.py -f 2-query/Timediff.py -Q 4 +# python3 ./test.py -f 2-query/json_tag.py -Q 4 +# python3 ./test.py -f 2-query/top.py -Q 4 +# python3 ./test.py -f 2-query/bottom.py -Q 4 +# python3 ./test.py -f 2-query/percentile.py -Q 4 +# python3 ./test.py -f 2-query/apercentile.py -Q 4 +# python3 ./test.py -f 2-query/abs.py -Q 4 +# python3 ./test.py -f 2-query/ceil.py -Q 4 +# python3 ./test.py -f 2-query/floor.py -Q 4 +# python3 ./test.py -f 2-query/round.py -Q 4 +# python3 ./test.py -f 2-query/log.py -Q 4 +# python3 ./test.py -f 2-query/pow.py -Q 4 +# python3 ./test.py -f 2-query/sqrt.py -Q 4 +# python3 ./test.py -f 2-query/sin.py -Q 4 +# python3 ./test.py -f 2-query/cos.py -Q 4 +# python3 ./test.py -f 2-query/tan.py -Q 4 +# python3 ./test.py -f 2-query/arcsin.py -Q 4 +# python3 ./test.py -f 2-query/arccos.py -Q 4 +# python3 ./test.py -f 2-query/arctan.py -Q 4 +# python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 4 -python3 ./test.py -f 2-query/nestedQuery.py -Q 4 -python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 -python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 -python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 -python3 ./test.py -f 2-query/stablity.py -Q 4 -python3 ./test.py -f 2-query/stablity_1.py -Q 4 +# python3 ./test.py -f 2-query/nestedQuery.py -Q 4 +# python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 +# python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 +# python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 +# python3 ./test.py -f 2-query/stablity.py -Q 4 +# python3 ./test.py -f 2-query/stablity_1.py -Q 4 -python3 ./test.py -f 2-query/avg.py -Q 4 -python3 ./test.py -f 2-query/elapsed.py -Q 4 -python3 ./test.py -f 2-query/csum.py -Q 4 -python3 ./test.py -f 2-query/mavg.py -Q 4 -python3 ./test.py -f 2-query/sample.py -Q 4 -python3 ./test.py -f 2-query/function_diff.py -Q 4 -python3 ./test.py -f 2-query/unique.py -Q 4 -python3 ./test.py -f 2-query/stateduration.py -Q 4 -python3 ./test.py -f 2-query/function_stateduration.py -Q 4 -python3 ./test.py -f 2-query/statecount.py -Q 4 -python3 ./test.py -f 2-query/tail.py -Q 4 -python3 ./test.py -f 2-query/ttl_comment.py -Q 4 -python3 ./test.py -f 2-query/distribute_agg_count.py -Q 4 -python3 ./test.py -f 2-query/distribute_agg_max.py -Q 4 -python3 ./test.py -f 2-query/distribute_agg_min.py -Q 4 -python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 4 -python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 4 -python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 4 -python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 4 -python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 4 -python3 ./test.py -f 2-query/twa.py -Q 4 -python3 ./test.py -f 2-query/irate.py -Q 4 -python3 ./test.py -f 2-query/function_null.py -Q 4 -python3 ./test.py -f 2-query/count_partition.py -Q 4 -python3 ./test.py -f 2-query/max_partition.py -Q 4 -python3 ./test.py -f 2-query/last_row.py -Q 4 -python3 ./test.py -f 2-query/tsbsQuery.py -Q 4 -python3 ./test.py -f 2-query/sml.py -Q 4 -python3 ./test.py -f 2-query/interp.py -Q 4 +# python3 ./test.py -f 2-query/avg.py -Q 4 +# python3 ./test.py -f 2-query/elapsed.py -Q 4 +# python3 ./test.py -f 2-query/csum.py -Q 4 +# python3 ./test.py -f 2-query/mavg.py -Q 4 +# python3 ./test.py -f 2-query/sample.py -Q 4 +# python3 ./test.py -f 2-query/function_diff.py -Q 4 +# python3 ./test.py -f 2-query/unique.py -Q 4 +# python3 ./test.py -f 2-query/stateduration.py -Q 4 +# python3 ./test.py -f 2-query/function_stateduration.py -Q 4 +# python3 ./test.py -f 2-query/statecount.py -Q 4 +# python3 ./test.py -f 2-query/tail.py -Q 4 +# python3 ./test.py -f 2-query/ttl_comment.py -Q 4 +# python3 ./test.py -f 2-query/distribute_agg_count.py -Q 4 +# python3 ./test.py -f 2-query/distribute_agg_max.py -Q 4 +# python3 ./test.py -f 2-query/distribute_agg_min.py -Q 4 +# python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 4 +# python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 4 +# python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 4 +# python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 4 +# python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 4 +# python3 ./test.py -f 2-query/twa.py -Q 4 +# python3 ./test.py -f 2-query/irate.py -Q 4 +# python3 ./test.py -f 2-query/function_null.py -Q 4 +# python3 ./test.py -f 2-query/count_partition.py -Q 4 +# python3 ./test.py -f 2-query/max_partition.py -Q 4 +# python3 ./test.py -f 2-query/last_row.py -Q 4 +# python3 ./test.py -f 2-query/tsbsQuery.py -Q 4 +# python3 ./test.py -f 2-query/sml.py -Q 4 +# python3 ./test.py -f 2-query/interp.py -Q 4 From 139a078900aaa24bb1a55537187d72143f7bcd06 Mon Sep 17 00:00:00 2001 From: Xuefeng Tan <1172915550@qq.com> Date: Thu, 3 Nov 2022 20:34:35 +0800 Subject: [PATCH 28/82] enh(taosAdapter): support Windows service (#17885) --- cmake/taosadapter_CMakeLists.txt.in | 2 +- packaging/tools/make_install.bat | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index 5b8192831e..c507ae2536 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG a11131c + GIT_TAG 8c3d57d SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/packaging/tools/make_install.bat b/packaging/tools/make_install.bat index f777d10918..d875b783ba 100644 --- a/packaging/tools/make_install.bat +++ b/packaging/tools/make_install.bat @@ -67,3 +67,4 @@ if exist C:\\TDengine\\driver\\taosws.dll ( copy /y C:\\TDengine\\driver\\taosws.dll C:\\Windows\\System32 > nul ) sc query "taosd" >nul || sc create "taosd" binPath= "C:\\TDengine\\taosd.exe --win_service" start= DEMAND +sc query "taosadapter" >nul || sc create "taosadapter" binPath= "C:\\TDengine\\taosadapter.exe" start= DEMAND From abfa8c1d87d148e460f4e514a07983d2c6205cee Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Nov 2022 22:05:08 +0800 Subject: [PATCH 29/82] fix(query): add api to memset buffer for filtering data. --- include/common/tdatablock.h | 2 ++ source/client/CMakeLists.txt | 4 ++-- source/client/test/clientTests.cpp | 13 ++++++++----- source/common/src/tdatablock.c | 10 ++++++++++ source/libs/scalar/src/scalar.c | 1 + 5 files changed, 23 insertions(+), 7 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 24dfa5958d..c24a462ea7 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -217,6 +217,8 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); +void colInfoDataMemset(SColumnInfoData* pColumn, uint32_t numOfRows); + void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows); void blockDataCleanup(SSDataBlock* pDataBlock); diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index 0c445c7fbf..a14fc650d8 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -60,6 +60,6 @@ target_link_libraries( PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom ) -if(${BUILD_TEST}) +#if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) -endif(${BUILD_TEST}) \ No newline at end of file +#endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 21a52a4b57..85814305bd 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -112,7 +112,7 @@ void createNewTable(TAOS* pConn, int32_t index) { } taos_free_result(pRes); - for(int32_t i = 0; i < 10000; i += 20) { + for(int32_t i = 0; i < 20; i += 20) { char sql[1024] = {0}; sprintf(sql, "insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" @@ -692,6 +692,7 @@ TEST(testCase, insert_test) { taos_free_result(pRes); taos_close(pConn); } +#endif TEST(testCase, projection_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -703,7 +704,7 @@ TEST(testCase, projection_query_tables) { // } // taos_free_result(pRes); - TAOS_RES* pRes = taos_query(pConn, "use benchmarkcpu"); + TAOS_RES* pRes = taos_query(pConn, "use abc2"); taos_free_result(pRes); pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); @@ -725,7 +726,7 @@ TEST(testCase, projection_query_tables) { } taos_free_result(pRes); - for (int32_t i = 0; i < 2; ++i) { + for (int32_t i = 0; i < 200000; ++i) { printf("create table :%d\n", i); createNewTable(pConn, i); } @@ -750,7 +751,9 @@ TEST(testCase, projection_query_tables) { taos_free_result(pRes); taos_close(pConn); } -#endif + + +#if 0 TEST(testCase, tsbs_perf_test) { TdThread qid[20] = {0}; @@ -761,7 +764,7 @@ TEST(testCase, tsbs_perf_test) { getchar(); } -#if 0 + TEST(testCase, projection_query_stables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 322d9a1f75..69b270d46c 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1208,6 +1208,16 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) return doEnsureCapacity(pColumn, &info, numOfRows); } +void colInfoDataMemset(SColumnInfoData* pColumn, uint32_t numOfRows) { + if (!IS_VAR_DATA_TYPE(pColumn->info.type)) { + memset(pColumn->pData, 0, pColumn->info.bytes * numOfRows); + } else { + if (pColumn->varmeta.length > 0) { + memset(pColumn->pData, 0, pColumn->varmeta.length); + } + } +} + int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { int32_t code = 0; if (numOfRows == 0) { diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index ea1ce175e3..9d7c0f978f 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -56,6 +56,7 @@ int32_t sclCreateColumnInfoData(SDataType *pType, int32_t numOfRows, SScalarPara return terrno; } + colInfoDataMemset(pColumnData, numOfRows); pParam->columnData = pColumnData; pParam->colAlloced = true; return TSDB_CODE_SUCCESS; From 869b49158d3fc2f4d2c9536aab2f5bb209d04bcd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Nov 2022 22:05:55 +0800 Subject: [PATCH 30/82] test: disable unit test. --- source/client/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index a14fc650d8..0c445c7fbf 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -60,6 +60,6 @@ target_link_libraries( PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom ) -#if(${BUILD_TEST}) +if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) -#endif(${BUILD_TEST}) \ No newline at end of file +endif(${BUILD_TEST}) \ No newline at end of file From e7d58625b939c2ae4c74832aafca0b2a34cd9dce Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 4 Nov 2022 09:09:37 +0800 Subject: [PATCH 31/82] fix(query): add memset after malloc memory. --- source/common/src/tdatablock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 69b270d46c..d4aec8198f 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1182,7 +1182,7 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* return TSDB_CODE_OUT_OF_MEMORY; } -// memset(tmp + pColumn->info.bytes * existedRows, 0, pColumn->info.bytes * (numOfRows - existedRows)); + memset(tmp + pColumn->info.bytes * existedRows, 0, pColumn->info.bytes * (numOfRows - existedRows)); pColumn->pData = tmp; } From 9c33d10ad926e5681096508cc4dfbbbf554f9019 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 4 Nov 2022 09:15:14 +0800 Subject: [PATCH 32/82] refactor: remove the unnecessary function. --- include/common/tdatablock.h | 2 -- source/common/src/tdatablock.c | 10 ---------- source/libs/scalar/src/scalar.c | 1 - 3 files changed, 13 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index c24a462ea7..24dfa5958d 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -217,8 +217,6 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); -void colInfoDataMemset(SColumnInfoData* pColumn, uint32_t numOfRows); - void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows); void blockDataCleanup(SSDataBlock* pDataBlock); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index d4aec8198f..0b27d9b527 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1208,16 +1208,6 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) return doEnsureCapacity(pColumn, &info, numOfRows); } -void colInfoDataMemset(SColumnInfoData* pColumn, uint32_t numOfRows) { - if (!IS_VAR_DATA_TYPE(pColumn->info.type)) { - memset(pColumn->pData, 0, pColumn->info.bytes * numOfRows); - } else { - if (pColumn->varmeta.length > 0) { - memset(pColumn->pData, 0, pColumn->varmeta.length); - } - } -} - int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { int32_t code = 0; if (numOfRows == 0) { diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 9d7c0f978f..ea1ce175e3 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -56,7 +56,6 @@ int32_t sclCreateColumnInfoData(SDataType *pType, int32_t numOfRows, SScalarPara return terrno; } - colInfoDataMemset(pColumnData, numOfRows); pParam->columnData = pColumnData; pParam->colAlloced = true; return TSDB_CODE_SUCCESS; From 1f329f30252274fbd48bca25cbdf4f368a0f4a16 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 4 Nov 2022 10:41:36 +0800 Subject: [PATCH 33/82] refactor(sync): delete assert --- source/libs/sync/src/syncRaftLog.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 23d076cfbc..04228d4759 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -219,7 +219,7 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr pEntry->index, err, err, errStr, sysErr, sysErrStr); syncNodeErrorLog(pData->pSyncNode, logBuf); - ASSERT(0); + // ASSERT(0); return -1; } pEntry->index = index; @@ -327,7 +327,7 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn sError("vgId:%d, wal truncate error, from-index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId, fromIndex, err, err, errStr, sysErr, sysErrStr); - ASSERT(0); + // ASSERT(0); } // event log @@ -376,6 +376,7 @@ int32_t raftLogUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { pData->pSyncNode->vgId, index, err, err, errStr, sysErr, sysErrStr); ASSERT(0); + return -1; } return 0; } From 65636154ea38cba8e0ba8f30e84d4cb5d41144df Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 4 Nov 2022 10:47:49 +0800 Subject: [PATCH 34/82] fix: fix msg head issue --- source/dnode/mnode/impl/src/mndStb.c | 5 +++-- source/libs/sync/src/syncRaftEntry.c | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 641a8af437..b6f7e31638 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -466,7 +466,7 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt contLen += sizeof(SMsgHead); - SMsgHead *pHead = taosMemoryMalloc(contLen); + SMsgHead *pHead = taosMemoryCalloc(1, contLen); if (pHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -519,6 +519,7 @@ static void *mndBuildVDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, pHead->contLen = htonl(contLen); pHead->vgId = htonl(pVgroup->vgId); + pHead->msgMask = htonl(0); void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); @@ -2596,4 +2597,4 @@ const char *mndGetStbStr(const char *src) { if (posStb != NULL) ++posStb; if (posStb == NULL) return posDb; return posStb; -} \ No newline at end of file +} diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index 6d372acf2f..aba61edf0d 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -72,6 +72,7 @@ SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId) SMsgHead head; head.vgId = vgId; head.contLen = sizeof(SMsgHead); + head.msgMask = 0; SRpcMsg rpcMsg; memset(&rpcMsg, 0, sizeof(SRpcMsg)); rpcMsg.contLen = head.contLen; From a0983cf8c149bb02b9c1026d99805cde1273c164 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 4 Nov 2022 10:27:36 +0800 Subject: [PATCH 35/82] enh: adjust tqueue and tworker log --- include/util/tqueue.h | 41 ++++++++++++ source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 20 ++++-- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 17 +++-- source/dnode/vnode/src/vnd/vnodeSync.c | 69 --------------------- source/util/src/tqueue.c | 40 ------------ source/util/src/tworker.c | 15 +++-- 6 files changed, 77 insertions(+), 125 deletions(-) diff --git a/include/util/tqueue.h b/include/util/tqueue.h index da409a90bb..8b46bbd064 100644 --- a/include/util/tqueue.h +++ b/include/util/tqueue.h @@ -59,6 +59,47 @@ typedef enum { typedef void (*FItem)(SQueueInfo *pInfo, void *pItem); typedef void (*FItems)(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfItems); +typedef struct STaosQnode STaosQnode; + +typedef struct STaosQnode { + STaosQnode *next; + STaosQueue *queue; + int64_t timestamp; + int32_t size; + int8_t itype; + int8_t reserved[3]; + char item[]; +} STaosQnode; + +typedef struct STaosQueue { + STaosQnode *head; + STaosQnode *tail; + STaosQueue *next; // for queue set + STaosQset *qset; // for queue set + void *ahandle; // for queue set + FItem itemFp; + FItems itemsFp; + TdThreadMutex mutex; + int64_t memOfItems; + int32_t numOfItems; + int64_t threadId; +} STaosQueue; + +typedef struct STaosQset { + STaosQueue *head; + STaosQueue *current; + TdThreadMutex mutex; + tsem_t sem; + int32_t numOfQueues; + int32_t numOfItems; +} STaosQset; + +typedef struct STaosQall { + STaosQnode *current; + STaosQnode *start; + int32_t numOfItems; +} STaosQall; + STaosQueue *taosOpenQueue(); void taosCloseQueue(STaosQueue *queue); void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index f36604eb27..16cfa480b1 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -86,22 +86,34 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { taosThreadRwlockUnlock(&pMgmt->lock); vmReleaseVnode(pMgmt, pVnode); - dTrace("vgId:%d, wait for vnode ref become 0", pVnode->vgId); + dInfo("vgId:%d, wait for vnode ref become 0", pVnode->vgId); while (pVnode->refCount > 0) taosMsleep(10); - dTrace("vgId:%d, wait for vnode queue is empty", pVnode->vgId); + dInfo("vgId:%d, wait for vnode write queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteQ, + pVnode->pWriteQ->threadId); while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10); + dInfo("vgId:%d, wait for vnode sync queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncQ, + pVnode->pWriteQ->threadId); while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10); + dInfo("vgId:%d, wait for vnode sync ctrl queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncCtrlQ, + pVnode->pWriteQ->threadId); + while (!taosQueueEmpty(pVnode->pSyncCtrlQ)) taosMsleep(10); + dInfo("vgId:%d, wait for vnode apply queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyQ, + pVnode->pWriteQ->threadId); while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10); + dInfo("vgId:%d, wait for vnode query queue:%p is empty", pVnode->vgId, pVnode->pQueryQ); while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10); + dInfo("vgId:%d, wait for vnode fetch queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pFetchQ, + pVnode->pWriteQ->threadId); while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10); + dInfo("vgId:%d, wait for vnode stream queue:%p is empty", pVnode->vgId, pVnode->pStreamQ); while (!taosQueueEmpty(pVnode->pStreamQ)) taosMsleep(10); - dTrace("vgId:%d, vnode queue is empty", pVnode->vgId); + dInfo("vgId:%d, all vnode queues is empty", pVnode->vgId); vmFreeQueue(pMgmt, pVnode); vnodeClose(pVnode->pImpl); pVnode->pImpl = NULL; - dDebug("vgId:%d, vnode is closed", pVnode->vgId); + dInfo("vgId:%d, vnode is closed", pVnode->vgId); if (pVnode->dropped) { dInfo("vgId:%d, vnode is destroyed, dropped:%d", pVnode->vgId, pVnode->dropped); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 40d0e32c2b..a5070e09aa 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -320,12 +320,17 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { return -1; } - dDebug("vgId:%d, write-queue:%p is alloced", pVnode->vgId, pVnode->pWriteQ); - dDebug("vgId:%d, sync-queue:%p is alloced", pVnode->vgId, pVnode->pSyncQ); - dDebug("vgId:%d, apply-queue:%p is alloced", pVnode->vgId, pVnode->pApplyQ); - dDebug("vgId:%d, query-queue:%p is alloced", pVnode->vgId, pVnode->pQueryQ); - dDebug("vgId:%d, stream-queue:%p is alloced", pVnode->vgId, pVnode->pStreamQ); - dDebug("vgId:%d, fetch-queue:%p is alloced", pVnode->vgId, pVnode->pFetchQ); + dInfo("vgId:%d, write-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteQ, + pVnode->pWriteQ->threadId); + dInfo("vgId:%d, sync-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncQ, pVnode->pSyncQ->threadId); + dInfo("vgId:%d, sync-ctrl-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncCtrlQ, + pVnode->pSyncCtrlQ->threadId); + dInfo("vgId:%d, apply-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyQ, + pVnode->pApplyQ->threadId); + dInfo("vgId:%d, query-queue:%p is alloced", pVnode->vgId, pVnode->pQueryQ); + dInfo("vgId:%d, fetch-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pFetchQ, + pVnode->pFetchQ->threadId); + dInfo("vgId:%d, stream-queue:%p is alloced", pVnode->vgId, pVnode->pStreamQ); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index e27ae07460..7f3b1db7c3 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -244,16 +244,6 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } static int32_t vnodeSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { - if (pMsg == NULL || pMsg->pCont == NULL) { - return -1; - } - - if (msgcb == NULL || msgcb->putToQueueFp == NULL) { - rpcFreeCont(pMsg->pCont); - pMsg->pCont = NULL; - return -1; - } - int32_t code = tmsgPutToQueue(msgcb, SYNC_CTRL_QUEUE, pMsg); if (code != 0) { rpcFreeCont(pMsg->pCont); @@ -263,16 +253,6 @@ static int32_t vnodeSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { } static int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { - if (pMsg == NULL || pMsg->pCont == NULL) { - return -1; - } - - if (msgcb == NULL || msgcb->putToQueueFp == NULL) { - rpcFreeCont(pMsg->pCont); - pMsg->pCont = NULL; - return -1; - } - int32_t code = tmsgPutToQueue(msgcb, SYNC_QUEUE, pMsg); if (code != 0) { rpcFreeCont(pMsg->pCont); @@ -342,52 +322,26 @@ static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, cons TMSG_INFO(pMsg->msgType)); } -#define USE_TSDB_SNAPSHOT - static int32_t vnodeSnapshotStartRead(const SSyncFSM *pFsm, void *pParam, void **ppReader) { -#ifdef USE_TSDB_SNAPSHOT SVnode *pVnode = pFsm->data; SSnapshotParam *pSnapshotParam = pParam; int32_t code = vnodeSnapReaderOpen(pVnode, pSnapshotParam->start, pSnapshotParam->end, (SVSnapReader **)ppReader); return code; -#else - *ppReader = taosMemoryMalloc(32); - return 0; -#endif } static int32_t vnodeSnapshotStopRead(const SSyncFSM *pFsm, void *pReader) { -#ifdef USE_TSDB_SNAPSHOT SVnode *pVnode = pFsm->data; int32_t code = vnodeSnapReaderClose(pReader); return code; -#else - taosMemoryFree(pReader); - return 0; -#endif } static int32_t vnodeSnapshotDoRead(const SSyncFSM *pFsm, void *pReader, void **ppBuf, int32_t *len) { -#ifdef USE_TSDB_SNAPSHOT SVnode *pVnode = pFsm->data; int32_t code = vnodeSnapRead(pReader, (uint8_t **)ppBuf, len); return code; -#else - static int32_t times = 0; - if (times++ < 5) { - *len = 64; - *ppBuf = taosMemoryMalloc(*len); - snprintf(*ppBuf, *len, "snapshot block %d", times); - } else { - *len = 0; - *ppBuf = NULL; - } - return 0; -#endif } static int32_t vnodeSnapshotStartWrite(const SSyncFSM *pFsm, void *pParam, void **ppWriter) { -#ifdef USE_TSDB_SNAPSHOT SVnode *pVnode = pFsm->data; SSnapshotParam *pSnapshotParam = pParam; @@ -404,14 +358,9 @@ static int32_t vnodeSnapshotStartWrite(const SSyncFSM *pFsm, void *pParam, void int32_t code = vnodeSnapWriterOpen(pVnode, pSnapshotParam->start, pSnapshotParam->end, (SVSnapWriter **)ppWriter); return code; -#else - *ppWriter = taosMemoryMalloc(32); - return 0; -#endif } static int32_t vnodeSnapshotStopWrite(const SSyncFSM *pFsm, void *pWriter, bool isApply, SSnapshot *pSnapshot) { -#ifdef USE_TSDB_SNAPSHOT SVnode *pVnode = pFsm->data; vInfo("vgId:%d, stop write vnode snapshot, apply:%d, index:%" PRId64 " term:%" PRIu64 " config:%" PRId64, pVnode->config.vgId, isApply, pSnapshot->lastApplyIndex, pSnapshot->lastApplyTerm, pSnapshot->lastConfigIndex); @@ -419,22 +368,14 @@ static int32_t vnodeSnapshotStopWrite(const SSyncFSM *pFsm, void *pWriter, bool int32_t code = vnodeSnapWriterClose(pWriter, !isApply, pSnapshot); vInfo("vgId:%d, apply vnode snapshot finished, code:0x%x", pVnode->config.vgId, code); return code; -#else - taosMemoryFree(pWriter); - return 0; -#endif } static int32_t vnodeSnapshotDoWrite(const SSyncFSM *pFsm, void *pWriter, void *pBuf, int32_t len) { -#ifdef USE_TSDB_SNAPSHOT SVnode *pVnode = pFsm->data; vDebug("vgId:%d, continue write vnode snapshot, len:%d", pVnode->config.vgId, len); int32_t code = vnodeSnapWrite(pWriter, pBuf, len); vDebug("vgId:%d, continue write vnode snapshot finished, len:%d", pVnode->config.vgId, len); return code; -#else - return 0; -#endif } static void vnodeRestoreFinish(const SSyncFSM *pFsm) { @@ -461,7 +402,6 @@ static void vnodeBecomeFollower(const SSyncFSM *pFsm) { SVnode *pVnode = pFsm->data; vDebug("vgId:%d, become follower", pVnode->config.vgId); - // clear old leader resource taosThreadMutexLock(&pVnode->lock); if (pVnode->blocked) { pVnode->blocked = false; @@ -474,15 +414,6 @@ static void vnodeBecomeFollower(const SSyncFSM *pFsm) { static void vnodeBecomeLeader(const SSyncFSM *pFsm) { SVnode *pVnode = pFsm->data; vDebug("vgId:%d, become leader", pVnode->config.vgId); - -#if 0 - taosThreadMutexLock(&pVnode->lock); - if (pVnode->blocked) { - pVnode->blocked = false; - tsem_post(&pVnode->syncSem); - } - taosThreadMutexUnlock(&pVnode->lock); -#endif } static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 19b9b89cab..c8f128e666 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -21,46 +21,6 @@ int64_t tsRpcQueueMemoryAllowed = 0; int64_t tsRpcQueueMemoryUsed = 0; -typedef struct STaosQnode STaosQnode; - -typedef struct STaosQnode { - STaosQnode *next; - STaosQueue *queue; - int64_t timestamp; - int32_t size; - int8_t itype; - int8_t reserved[3]; - char item[]; -} STaosQnode; - -typedef struct STaosQueue { - STaosQnode *head; - STaosQnode *tail; - STaosQueue *next; // for queue set - STaosQset *qset; // for queue set - void *ahandle; // for queue set - FItem itemFp; - FItems itemsFp; - TdThreadMutex mutex; - int64_t memOfItems; - int32_t numOfItems; -} STaosQueue; - -typedef struct STaosQset { - STaosQueue *head; - STaosQueue *current; - TdThreadMutex mutex; - tsem_t sem; - int32_t numOfQueues; - int32_t numOfItems; -} STaosQset; - -typedef struct STaosQall { - STaosQnode *current; - STaosQnode *start; - int32_t numOfItems; -} STaosQall; - STaosQueue *taosOpenQueue() { STaosQueue *queue = taosMemoryCalloc(1, sizeof(STaosQueue)); if (queue == NULL) { diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index f7d4173d3f..27c641555f 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -36,7 +36,7 @@ int32_t tQWorkerInit(SQWorkerPool *pool) { worker->pool = pool; } - uInfo("worker:%s is initialized, min:%d max:%d", pool->name, pool->min, pool->max); + uDebug("worker:%s is initialized, min:%d max:%d", pool->name, pool->min, pool->max); return 0; } @@ -73,11 +73,12 @@ static void *tQWorkerThreadFp(SQWorker *worker) { taosBlockSIGPIPE(); setThreadName(pool->name); - uDebug("worker:%s:%d is running", pool->name, worker->id); + uInfo("worker:%s:%d is running, thread:%08" PRId64, pool->name, worker->id, taosGetSelfPthreadId()); while (1) { if (taosReadQitemFromQset(pool->qset, (void **)&msg, &qinfo) == 0) { - uDebug("worker:%s:%d qset:%p, got no message and exiting", pool->name, worker->id, pool->qset); + uInfo("worker:%s:%d qset:%p, got no message and exiting, thread:%08" PRId64, pool->name, worker->id, pool->qset, + taosGetSelfPthreadId()); break; } @@ -191,12 +192,13 @@ static void *tWWorkerThreadFp(SWWorker *worker) { taosBlockSIGPIPE(); setThreadName(pool->name); - uDebug("worker:%s:%d is running", pool->name, worker->id); + uInfo("worker:%s:%d is running, thread:%08" PRId64, pool->name, worker->id, taosGetSelfPthreadId()); while (1) { numOfMsgs = taosReadAllQitemsFromQset(worker->qset, worker->qall, &qinfo); if (numOfMsgs == 0) { - uDebug("worker:%s:%d qset:%p, got no message and exiting", pool->name, worker->id, worker->qset); + uInfo("worker:%s:%d qset:%p, got no message and exiting, thread:%08" PRId64, pool->name, worker->id, worker->qset, + taosGetSelfPthreadId()); break; } @@ -244,7 +246,8 @@ STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) { pool->nextId = (pool->nextId + 1) % pool->max; } - uDebug("worker:%s, queue:%p is allocated, ahandle:%p", pool->name, queue, ahandle); + queue->threadId = taosGetPthreadId(worker->thread); + uDebug("worker:%s, queue:%p is allocated, ahandle:%p thread:%08" PRId64, pool->name, queue, ahandle, queue->threadId); code = 0; _OVER: From 988b16f6e9efd7b116192e8e5686a3b00f341c7c Mon Sep 17 00:00:00 2001 From: Hui Li <52318143+plum-lihui@users.noreply.github.com> Date: Fri, 4 Nov 2022 11:01:23 +0800 Subject: [PATCH 36/82] Update taosdShell.py --- tests/system-test/0-others/taosdShell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/taosdShell.py b/tests/system-test/0-others/taosdShell.py index f38efdb03c..b743783a4f 100644 --- a/tests/system-test/0-others/taosdShell.py +++ b/tests/system-test/0-others/taosdShell.py @@ -91,7 +91,7 @@ class TDTestCase: break else: tdLog.info( "wait start taosd ,times: %d "%i) - sleep + time.sleep(1) i+= 1 else : tdLog.exit("taosd %s is not running "%startAction) From 880a34dc25a816293fa0af27c9dd51c478cdcb04 Mon Sep 17 00:00:00 2001 From: Alex Duan <51781608+DuanKuanJun@users.noreply.github.com> Date: Fri, 4 Nov 2022 11:46:28 +0800 Subject: [PATCH 37/82] Update 01-deploy.md --- docs/zh/10-deployment/01-deploy.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/zh/10-deployment/01-deploy.md b/docs/zh/10-deployment/01-deploy.md index eecb86ce41..bcfc153327 100644 --- a/docs/zh/10-deployment/01-deploy.md +++ b/docs/zh/10-deployment/01-deploy.md @@ -190,3 +190,14 @@ DROP DNODE dnodeId; dnodeID 是集群自动分配的，不得人工指定。它在生成时是递增的，不会重复。 ::: + +## 常见问题 + +1、建立集群时使用 CREATE DNODE 增加新节点后，新节点始终显示 offline 状态？ + 1）首先要检查增加的新节点上的 taosd 服务是否已经正常启动 + 2）如果已经启动，再检查到新节点的网络是否通畅，可以使用 ping fqdn 验证下 + 3）如果前面两步都没有问题，这一步要检查新节点做为独立集群在运行了，可以使用 taos -h fqdn 连接上后，show dnodes; 命令查看，如果显示的列表与你主节点上显示的不一致，说明此节点自己单独成立了一个集群，解决的方法是停止新节点上的服务，然后清空新节点上 taos.cfg 中配置的 dataDir 目录下的所有文件，重新启动新节点服务即可解决。 + + + + From 8e10e32bd05b9e1211e3db767d96a4dede66dc45 Mon Sep 17 00:00:00 2001 From: Alex Duan <51781608+DuanKuanJun@users.noreply.github.com> Date: Fri, 4 Nov 2022 13:40:47 +0800 Subject: [PATCH 38/82] Update 01-deploy.md adjust format --- docs/zh/10-deployment/01-deploy.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/zh/10-deployment/01-deploy.md b/docs/zh/10-deployment/01-deploy.md index bcfc153327..304ed7153b 100644 --- a/docs/zh/10-deployment/01-deploy.md +++ b/docs/zh/10-deployment/01-deploy.md @@ -194,10 +194,14 @@ dnodeID 是集群自动分配的，不得人工指定。它在生成时是递增 ## 常见问题 1、建立集群时使用 CREATE DNODE 增加新节点后，新节点始终显示 offline 状态？ + 1）首先要检查增加的新节点上的 taosd 服务是否已经正常启动 + 2）如果已经启动，再检查到新节点的网络是否通畅，可以使用 ping fqdn 验证下 + 3）如果前面两步都没有问题，这一步要检查新节点做为独立集群在运行了，可以使用 taos -h fqdn 连接上后，show dnodes; 命令查看，如果显示的列表与你主节点上显示的不一致，说明此节点自己单独成立了一个集群，解决的方法是停止新节点上的服务，然后清空新节点上 taos.cfg 中配置的 dataDir 目录下的所有文件，重新启动新节点服务即可解决。 + From 69d545fb8dd886ac8a082c4684064b634a3cc5f5 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 4 Nov 2022 13:55:54 +0800 Subject: [PATCH 39/82] fix: fix interval query memory leak --- source/libs/executor/src/timewindowoperator.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 6be9a0402f..a0d3de5676 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -985,7 +985,8 @@ void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInf // current result is done in computing final results. if (pInfo->timeWindowInterpo && isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) { closeResultRow(pResult); - tdListPopHead(pResultRowInfo->openWindow); + SListNode *pNode = tdListPopHead(pResultRowInfo->openWindow); + taosMemoryFree(pNode); } } From 47d9fe6422bdb90dcadbb5e88eb4dfbac6864f56 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 4 Nov 2022 15:10:08 +0800 Subject: [PATCH 40/82] fix(sync): do not vote when last-index < my cmt index --- source/libs/sync/src/syncRequestVote.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index 074e4fca64..3b4acea582 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -47,6 +47,19 @@ static bool syncNodeOnRequestVoteLogOK(SSyncNode* pSyncNode, SyncRequestVote* pM SyncTerm myLastTerm = syncNodeGetLastTerm(pSyncNode); SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode); + if (pMsg->lastLogIndex < pSyncNode->commitIndex) { + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), + "logok:0, {my-lterm:%" PRIu64 ", my-lindex:%" PRId64 ", recv-lterm:%" PRIu64 ", recv-lindex:%" PRId64 + ", recv-term:%" PRIu64 "}", + myLastTerm, myLastIndex, pMsg->lastLogTerm, pMsg->lastLogIndex, pMsg->term); + syncNodeEventLog(pSyncNode, logBuf); + } while (0); + + return false; + } + if (myLastTerm == SYNC_TERM_INVALID) { do { char logBuf[128]; From 85520d8c70ebc43c9d6b98224ce6b1d510680c3a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 4 Nov 2022 15:10:42 +0800 Subject: [PATCH 41/82] fix: fix msg mask check issue --- include/common/tmsg.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 1781374165..c0ac7da5bf 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -218,7 +218,7 @@ typedef struct SEp { #define SHOW_REWRITE_MASK() (1 << 0) -#define TEST_SHOW_REWRITE_MASK(m) ((m) & SHOW_REWRITE_MASK() != 0) +#define TEST_SHOW_REWRITE_MASK(m) (((m) & SHOW_REWRITE_MASK()) != 0) typedef struct { int32_t contLen; From d2a675dbcb8b12e90218cc77c717eb138dcea07b Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 4 Nov 2022 15:39:11 +0800 Subject: [PATCH 42/82] update query script --- tests/pytest/concurrent_inquiry.py | 734 ++++++++++++++++------------- 1 file changed, 395 insertions(+), 339 deletions(-) diff --git a/tests/pytest/concurrent_inquiry.py b/tests/pytest/concurrent_inquiry.py index 1bb2081d7f..0d281674f5 100644 --- a/tests/pytest/concurrent_inquiry.py +++ b/tests/pytest/concurrent_inquiry.py @@ -21,43 +21,52 @@ import argparse import datetime import string from requests.auth import HTTPBasicAuth -func_list=['avg','count','twa','sum','stddev','leastsquares','min', -'max','first','last','top','bottom','percentile','apercentile', -'last_row','diff','spread','distinct'] -condition_list=[ + +func_list = ['abs', 'acos', 'asin', 'atan', 'ceil', 'cos', 'floor', 'log', 'pow', 'round', 'sin', 'sqrt', 'tan', + 'char_length', 'concat', 'concat_ws', 'length', 'lower', 'ltrim', 'rtrim', 'substr', 'upper', + 'cast', 'to_iso8601', 'to_json', 'to_unixtimestamp', 'now', 'timediff', 'timetruncate', 'timezone', 'today', + 'apercentile', 'avg', 'count', 'elapsed', 'leastsquares', 'spread', 'stddev', 'sum', 'hyperloglog', 'histogram', 'percentile', + 'bottom', 'first', 'interp', 'last', 'last_row', 'max', 'min', 'mode', 'sample', 'tail', 'top', 'unique', + 'csum', 'derivative', 'diff', 'irate', 'mavg', 'statecount', 'stateduration', 'twa', + 'database', 'client_version', 'server_version', 'server_status'] + +condition_list = [ "where _c0 > now -10d ", 'interval(10s)', 'limit 10', 'group by', + 'partition by', 'order by', 'fill(null)' - ] -where_list = ['_c0>now-10d',' <50','like',' is null','in'] + +where_list = ['_c0>now-10d', ' <50', 'like', ' is null', 'in'] + + class ConcurrentInquiry: # def __init__(self,ts=1500000001000,host='127.0.0.1',user='root',password='taosdata',dbname='test', # stb_prefix='st',subtb_prefix='t',n_Therads=10,r_Therads=10,probabilities=0.05,loop=5, - # stableNum = 2,subtableNum = 1000,insertRows = 100): - def __init__(self,ts,host,user,password,dbname, - stb_prefix,subtb_prefix,n_Therads,r_Therads,probabilities,loop, - stableNum ,subtableNum ,insertRows ,mix_table, replay): + # stableNum = 2,subtableNum = 1000,insertRows = 100): + def __init__(self, ts, host, user, password, dbname, + stb_prefix, subtb_prefix, n_Therads, r_Therads, probabilities, loop, + stableNum, subtableNum, insertRows, mix_table, replay): self.n_numOfTherads = n_Therads self.r_numOfTherads = r_Therads - self.ts=ts + self.ts = ts self.host = host self.user = user self.password = password - self.dbname=dbname + self.dbname = dbname self.stb_prefix = stb_prefix self.subtb_prefix = subtb_prefix - self.stb_list=[] - self.subtb_list=[] - self.stb_stru_list=[] - self.subtb_stru_list=[] - self.stb_tag_list=[] - self.subtb_tag_list=[] - self.probabilities = [1-probabilities,probabilities] - self.ifjoin = [1,0] + self.stb_list = [] + self.subtb_list = [] + self.stb_stru_list = [] + self.subtb_stru_list = [] + self.stb_tag_list = [] + self.subtb_tag_list = [] + self.probabilities = [1-probabilities, probabilities] + self.ifjoin = [1, 0] self.loop = loop self.stableNum = stableNum self.subtableNum = subtableNum @@ -66,253 +75,276 @@ class ConcurrentInquiry: self.max_ts = datetime.datetime.now() self.min_ts = datetime.datetime.now() - datetime.timedelta(days=5) self.replay = replay - def SetThreadsNum(self,num): - self.numOfTherads=num - def ret_fcol(self,cl,sql): #返回结果的第一列 + def SetThreadsNum(self, num): + self.numOfTherads = num + + def ret_fcol(self, cl, sql): # 返回结果的第一列 cl.execute(sql) - fcol_list=[] + fcol_list = [] for data in cl: fcol_list.append(data[0]) return fcol_list - def r_stb_list(self,cl): #返回超级表列表 - sql='show '+self.dbname+'.stables' - self.stb_list=self.ret_fcol(cl,sql) + def r_stb_list(self, cl): # 返回超级表列表 + sql = 'show '+self.dbname+'.stables' + self.stb_list = self.ret_fcol(cl, sql) - def r_subtb_list(self,cl,stablename): #每个超级表返回2个子表 - sql='select tbname from '+self.dbname+'.'+stablename+' limit 2;' - self.subtb_list+=self.ret_fcol(cl,sql) + def r_subtb_list(self, cl, stablename): # 每个超级表返回2个子表 + sql = 'select tbname from '+self.dbname+'.'+stablename+' limit 2;' + self.subtb_list += self.ret_fcol(cl, sql) - def cal_struct(self,cl,tbname): #查看表结构 - tb=[] - tag=[] - sql='describe '+self.dbname+'.'+tbname+';' + def cal_struct(self, cl, tbname): # 查看表结构 + tb = [] + tag = [] + sql = 'describe '+self.dbname+'.'+tbname+';' cl.execute(sql) for data in cl: if data[3]: tag.append(data[0]) else: tb.append(data[0]) - return tb,tag + return tb, tag - def r_stb_stru(self,cl): #获取所有超级表的表结构 + def r_stb_stru(self, cl): # 获取所有超级表的表结构 for i in self.stb_list: - tb,tag=self.cal_struct(cl,i) + tb, tag = self.cal_struct(cl, i) self.stb_stru_list.append(tb) self.stb_tag_list.append(tag) - def r_subtb_stru(self,cl): #返回所有子表的表结构 + def r_subtb_stru(self, cl): # 返回所有子表的表结构 for i in self.subtb_list: - tb,tag=self.cal_struct(cl,i) + tb, tag = self.cal_struct(cl, i) self.subtb_stru_list.append(tb) self.subtb_tag_list.append(tag) - def get_timespan(self,cl): #获取时间跨度(仅第一个超级表) - sql = 'select first(_c0),last(_c0) from ' + self.dbname + '.' + self.stb_list[0] + ';' + def get_timespan(self, cl): # 获取时间跨度(仅第一个超级表) + sql = 'select first(_c0),last(_c0) from ' + \ + self.dbname + '.' + self.stb_list[0] + ';' print(sql) cl.execute(sql) for data in cl: self.max_ts = data[1] self.min_ts = data[0] - def get_full(self): #获取所有的表、表结构 + def get_full(self): # 获取所有的表、表结构 host = self.host user = self.user password = self.password conn = taos.connect( - host, - user, - password, - ) + host='%s' % host, + user='%s' % user, + password='%s' % password, + ) cl = conn.cursor() self.r_stb_list(cl) for i in self.stb_list: - self.r_subtb_list(cl,i) + self.r_subtb_list(cl, i) self.r_stb_stru(cl) self.r_subtb_stru(cl) self.get_timespan(cl) cl.close() - conn.close() - - #query condition - def con_where(self,tlist,col_list,tag_list): - l=[] - for i in range(random.randint(0,len(tlist))): + conn.close() + + # query condition + def con_where(self, tlist, col_list, tag_list): + l = [] + for i in range(random.randint(0, len(tlist))): c = random.choice(where_list) if c == '_c0>now-10d': - rdate = self.min_ts + (self.max_ts - self.min_ts)/10 * random.randint(-11,11) - conlist = ' _c0 ' + random.choice(['<','>','>=','<=','<>']) + "'" + str(rdate) + "'" + rdate = self.min_ts + \ + (self.max_ts - self.min_ts)/10 * random.randint(-11, 11) + conlist = ' _c0 ' + \ + random.choice(['<', '>', '>=', '<=', '<>'] + ) + "'" + str(rdate) + "'" if self.random_pick(): l.append(conlist) - else: l.append(c) + else: + l.append(c) elif '<50' in c: - conlist = ' ' + random.choice(tlist) + random.choice(['<','>','>=','<=','<>']) + str(random.randrange(-100,100)) - l.append(conlist) + conlist = ' ' + random.choice(tlist) + random.choice( + ['<', '>', '>=', '<=', '<>']) + str(random.randrange(-100, 100)) + l.append(conlist) elif 'is null' in c: - conlist = ' ' + random.choice(tlist) + random.choice([' is null',' is not null']) - l.append(conlist) + conlist = ' ' + \ + random.choice(tlist) + \ + random.choice([' is null', ' is not null']) + l.append(conlist) elif 'in' in c: in_list = [] temp = [] - for i in range(random.randint(0,100)): - temp.append(random.randint(-10000,10000)) + for i in range(random.randint(0, 100)): + temp.append(random.randint(-10000, 10000)) temp = (str(i) for i in temp) in_list.append(temp) temp1 = [] - for i in range(random.randint(0,100)): - temp1.append("'" + ''.join(random.sample(string.ascii_letters, random.randint(0,10))) + "'") - in_list.append(temp1) - in_list.append(['NULL','NULL']) - conlist = ' ' + random.choice(tlist) + ' in (' + ','.join(random.choice(in_list)) + ')' + for i in range(random.randint(0, 100)): + temp1.append( + "'" + ''.join(random.sample(string.ascii_letters, random.randint(0, 10))) + "'") + in_list.append(temp1) + in_list.append(['NULL', 'NULL']) + conlist = ' ' + \ + random.choice(tlist) + ' in (' + \ + ','.join(random.choice(in_list)) + ')' l.append(conlist) else: s_all = string.ascii_letters - conlist = ' ' + random.choice(tlist) + " like \'%" + random.choice(s_all) + "%\' " + conlist = ' ' + \ + random.choice(tlist) + " like \'%" + \ + random.choice(s_all) + "%\' " l.append(conlist) - return 'where '+random.choice([' and ',' or ']).join(l) + return 'where '+random.choice([' and ', ' or ']).join(l) - def con_interval(self,tlist,col_list,tag_list): - interval = 'interval(' + str(random.randint(0,20)) + random.choice(['a','s','d','w','n','y']) + ')' + def con_interval(self, tlist, col_list, tag_list): + interval = 'interval(' + str(random.randint(0, 20)) + \ + random.choice(['a', 's', 'd', 'w', 'n', 'y']) + ')' return interval - def con_limit(self,tlist,col_list,tag_list): - rand1 = str(random.randint(0,1000)) - rand2 = str(random.randint(0,1000)) - return random.choice(['limit ' + rand1,'limit ' + rand1 + ' offset '+rand2, - ' slimit ' + rand1,' slimit ' + rand1 + ' offset ' + rand2,'limit '+rand1 + ' slimit '+ rand2, - 'limit '+ rand1 + ' offset' + rand2 + ' slimit '+ rand1 + ' soffset ' + rand2 ]) - - def con_fill(self,tlist,col_list,tag_list): - return random.choice(['fill(null)','fill(prev)','fill(none)','fill(LINEAR)']) - - def con_group(self,tlist,col_list,tag_list): - rand_tag = random.randint(0,5) - rand_col = random.randint(0,1) - if len(tag_list): - return 'group by '+','.join(random.sample(col_list,rand_col) + random.sample(tag_list,rand_tag)) - else: - return 'group by '+','.join(random.sample(col_list,rand_col)) + def con_limit(self, tlist, col_list, tag_list): + rand1 = str(random.randint(0, 1000)) + rand2 = str(random.randint(0, 1000)) + return random.choice(['limit ' + rand1, 'limit ' + rand1 + ' offset '+rand2, + ' slimit ' + rand1, ' slimit ' + rand1 + ' offset ' + + rand2, 'limit '+rand1 + ' slimit ' + rand2, + 'limit ' + rand1 + ' offset' + rand2 + ' slimit ' + rand1 + ' soffset ' + rand2]) - def con_order(self,tlist,col_list,tag_list): + def con_fill(self, tlist, col_list, tag_list): + return random.choice(['fill(null)', 'fill(prev)', 'fill(none)', 'fill(LINEAR)']) + + def con_group(self, tlist, col_list, tag_list): + rand_tag = random.randint(0, 5) + rand_col = random.randint(0, 1) + if len(tag_list): + return 'group by '+','.join(random.sample(col_list, rand_col) + random.sample(tag_list, rand_tag)) + else: + return 'group by '+','.join(random.sample(col_list, rand_col)) + + def con_order(self, tlist, col_list, tag_list): return 'order by '+random.choice(tlist) - def con_state_window(self,tlist,col_list,tag_list): + def con_state_window(self, tlist, col_list, tag_list): return 'state_window(' + random.choice(tlist + tag_list) + ')' - def con_session_window(self,tlist,col_list,tag_list): - session_window = 'session_window(' + random.choice(tlist + tag_list) + ',' + str(random.randint(0,20)) + random.choice(['a','s','d','w','n','y']) + ')' + def con_session_window(self, tlist, col_list, tag_list): + session_window = 'session_window(' + random.choice(tlist + tag_list) + ',' + str( + random.randint(0, 20)) + random.choice(['a', 's', 'd', 'w', 'n', 'y']) + ')' return session_window def gen_subquery_sql(self): - subsql ,col_num = self.gen_query_sql(1) + subsql, col_num = self.gen_query_sql(1) if col_num == 0: return 0 - col_list=[] - tag_list=[] + col_list = [] + tag_list = [] for i in range(col_num): - col_list.append("taosd%d"%i) + col_list.append("taosd%d" % i) - tlist=col_list+['abc'] #增加不存在的域'abc'，是否会引起新bug - con_rand=random.randint(0,len(condition_list)) - func_rand=random.randint(0,len(func_list)) - col_rand=random.randint(0,len(col_list)) - t_rand=random.randint(0,len(tlist)) - sql='select ' #select + tlist = col_list+['abc'] # 增加不存在的域'abc'，是否会引起新bug + con_rand = random.randint(0, len(condition_list)) + func_rand = random.randint(0, len(func_list)) + col_rand = random.randint(0, len(col_list)) + t_rand = random.randint(0, len(tlist)) + sql = 'select ' # select random.shuffle(col_list) random.shuffle(func_list) - sel_col_list=[] - col_rand=random.randint(0,len(col_list)) + sel_col_list = [] + col_rand = random.randint(0, len(col_list)) loop = 0 - for i,j in zip(col_list[0:col_rand],func_list): #决定每个被查询col的函数 - alias = ' as '+ 'sub%d ' % loop + for i, j in zip(col_list[0:col_rand], func_list): # 决定每个被查询col的函数 + alias = ' as ' + 'sub%d ' % loop loop += 1 pick_func = '' if j == 'leastsquares': - pick_func=j+'('+i+',1,1)' + pick_func = j+'('+i+',1,1)' elif j == 'top' or j == 'bottom' or j == 'percentile' or j == 'apercentile': - pick_func=j+'('+i+',1)' + pick_func = j+'('+i+',1)' else: - pick_func=j+'('+i+')' - if bool(random.getrandbits(1)) : - pick_func+=alias + pick_func = j+'('+i+')' + if bool(random.getrandbits(1)): + pick_func += alias sel_col_list.append(pick_func) if col_rand == 0: - sql = sql + '*' - else: - sql=sql+','.join(sel_col_list) #select col & func - sql = sql + ' from ('+ subsql +') ' - con_func=[self.con_where,self.con_interval,self.con_limit,self.con_group,self.con_order,self.con_fill,self.con_state_window,self.con_session_window] - sel_con=random.sample(con_func,random.randint(0,len(con_func))) - sel_con_list=[] - for i in sel_con: - sel_con_list.append(i(tlist,col_list,tag_list)) #获取对应的条件函数 - sql+=' '.join(sel_con_list) # condition - #print(sql) - return sql - - def gen_query_sql(self,subquery=0): #生成查询语句 - tbi=random.randint(0,len(self.subtb_list)+len(self.stb_list)) #随机决定查询哪张表 - tbname='' - col_list=[] - tag_list=[] - is_stb=0 - if tbi>len(self.stb_list) : - tbi=tbi-len(self.stb_list) - tbname=self.subtb_list[tbi-1] - col_list=self.subtb_stru_list[tbi-1] - tag_list=self.subtb_tag_list[tbi-1] + sql = sql + '*' else: - tbname=self.stb_list[tbi-1] - col_list=self.stb_stru_list[tbi-1] - tag_list=self.stb_tag_list[tbi-1] - is_stb=1 - tlist=col_list+tag_list+['abc'] #增加不存在的域'abc'，是否会引起新bug - con_rand=random.randint(0,len(condition_list)) - func_rand=random.randint(0,len(func_list)) - col_rand=random.randint(0,len(col_list)) - tag_rand=random.randint(0,len(tag_list)) - t_rand=random.randint(0,len(tlist)) - sql='select ' #select + sql = sql+','.join(sel_col_list) # select col & func + sql = sql + ' from (' + subsql + ') ' + con_func = [self.con_where, self.con_interval, self.con_limit, self.con_group, + self.con_order, self.con_fill, self.con_state_window, self.con_session_window] + sel_con = random.sample(con_func, random.randint(0, len(con_func))) + sel_con_list = [] + for i in sel_con: + sel_con_list.append(i(tlist, col_list, tag_list)) # 获取对应的条件函数 + # condition + sql += ' '.join(sel_con_list) + # print(sql) + return sql + + def gen_query_sql(self, subquery=0): # 生成查询语句 + tbi = random.randint(0, len(self.subtb_list) + + len(self.stb_list)) # 随机决定查询哪张表 + tbname = '' + col_list = [] + tag_list = [] + is_stb = 0 + if tbi > len(self.stb_list): + tbi = tbi-len(self.stb_list) + tbname = self.subtb_list[tbi-1] + col_list = self.subtb_stru_list[tbi-1] + tag_list = self.subtb_tag_list[tbi-1] + else: + tbname = self.stb_list[tbi-1] + col_list = self.stb_stru_list[tbi-1] + tag_list = self.stb_tag_list[tbi-1] + is_stb = 1 + tlist = col_list+tag_list+['abc'] # 增加不存在的域'abc'，是否会引起新bug + con_rand = random.randint(0, len(condition_list)) + func_rand = random.randint(0, len(func_list)) + col_rand = random.randint(0, len(col_list)) + tag_rand = random.randint(0, len(tag_list)) + t_rand = random.randint(0, len(tlist)) + sql = 'select ' # select random.shuffle(col_list) random.shuffle(func_list) - sel_col_list=[] - col_rand=random.randint(0,len(col_list)) + sel_col_list = [] + col_rand = random.randint(0, len(col_list)) loop = 0 - for i,j in zip(col_list[0:col_rand],func_list): #决定每个被查询col的函数 - alias = ' as '+ 'taos%d ' % loop + for i, j in zip(col_list[0:col_rand], func_list): # 决定每个被查询col的函数 + alias = ' as ' + 'taos%d ' % loop loop += 1 pick_func = '' if j == 'leastsquares': - pick_func=j+'('+i+',1,1)' + pick_func = j+'('+i+',1,1)' elif j == 'top' or j == 'bottom' or j == 'percentile' or j == 'apercentile': - pick_func=j+'('+i+',1)' + pick_func = j+'('+i+',1)' else: - pick_func=j+'('+i+')' - if bool(random.getrandbits(1)) | subquery : - pick_func+=alias + pick_func = j+'('+i+')' + if bool(random.getrandbits(1)) | subquery: + pick_func += alias sel_col_list.append(pick_func) - if col_rand == 0 & subquery : - sql = sql + '*' - else: - sql=sql+','.join(sel_col_list) #select col & func + if col_rand == 0 & subquery: + sql = sql + '*' + else: + sql = sql+','.join(sel_col_list) # select col & func if self.mix_table == 0: - sql = sql + ' from '+random.choice(self.stb_list+self.subtb_list)+' ' + sql = sql + ' from ' + \ + random.choice(self.stb_list+self.subtb_list)+' ' elif self.mix_table == 1: sql = sql + ' from '+random.choice(self.subtb_list)+' ' else: - sql = sql + ' from '+random.choice(self.stb_list)+' ' - con_func=[self.con_where,self.con_interval,self.con_limit,self.con_group,self.con_order,self.con_fill,self.con_state_window,self.con_session_window] - sel_con=random.sample(con_func,random.randint(0,len(con_func))) - sel_con_list=[] + sql = sql + ' from '+random.choice(self.stb_list)+' ' + con_func = [self.con_where, self.con_interval, self.con_limit, self.con_group, + self.con_order, self.con_fill, self.con_state_window, self.con_session_window] + sel_con = random.sample(con_func, random.randint(0, len(con_func))) + sel_con_list = [] for i in sel_con: - sel_con_list.append(i(tlist,col_list,tag_list)) #获取对应的条件函数 - sql+=' '.join(sel_con_list) # condition - #print(sql) - return (sql,loop) + sel_con_list.append(i(tlist, col_list, tag_list)) # 获取对应的条件函数 + # condition + sql += ' '.join(sel_con_list) + # print(sql) + return (sql, loop) - def gen_query_join(self): #生成join查询语句 - tbname = [] + def gen_query_join(self): # 生成join查询语句 + tbname = [] col_list = [] tag_list = [] col_intersection = [] @@ -321,88 +353,105 @@ class ConcurrentInquiry: if self.mix_table == 0: if bool(random.getrandbits(1)): subtable = True - tbname = random.sample(self.subtb_list,2) + tbname = random.sample(self.subtb_list, 2) for i in tbname: - col_list.append(self.subtb_stru_list[self.subtb_list.index(i)]) - tag_list.append(self.subtb_stru_list[self.subtb_list.index(i)]) - col_intersection = list(set(col_list[0]).intersection(set(col_list[1]))) - tag_intersection = list(set(tag_list[0]).intersection(set(tag_list[1]))) + col_list.append( + self.subtb_stru_list[self.subtb_list.index(i)]) + tag_list.append( + self.subtb_stru_list[self.subtb_list.index(i)]) + col_intersection = list( + set(col_list[0]).intersection(set(col_list[1]))) + tag_intersection = list( + set(tag_list[0]).intersection(set(tag_list[1]))) else: - tbname = random.sample(self.stb_list,2) + tbname = random.sample(self.stb_list, 2) for i in tbname: col_list.append(self.stb_stru_list[self.stb_list.index(i)]) tag_list.append(self.stb_stru_list[self.stb_list.index(i)]) - col_intersection = list(set(col_list[0]).intersection(set(col_list[1]))) - tag_intersection = list(set(tag_list[0]).intersection(set(tag_list[1]))) + col_intersection = list( + set(col_list[0]).intersection(set(col_list[1]))) + tag_intersection = list( + set(tag_list[0]).intersection(set(tag_list[1]))) elif self.mix_table == 1: subtable = True - tbname = random.sample(self.subtb_list,2) + tbname = random.sample(self.subtb_list, 2) for i in tbname: col_list.append(self.subtb_stru_list[self.subtb_list.index(i)]) tag_list.append(self.subtb_stru_list[self.subtb_list.index(i)]) - col_intersection = list(set(col_list[0]).intersection(set(col_list[1]))) - tag_intersection = list(set(tag_list[0]).intersection(set(tag_list[1]))) + col_intersection = list( + set(col_list[0]).intersection(set(col_list[1]))) + tag_intersection = list( + set(tag_list[0]).intersection(set(tag_list[1]))) else: - tbname = random.sample(self.stb_list,2) + tbname = random.sample(self.stb_list, 2) for i in tbname: col_list.append(self.stb_stru_list[self.stb_list.index(i)]) tag_list.append(self.stb_stru_list[self.stb_list.index(i)]) - col_intersection = list(set(col_list[0]).intersection(set(col_list[1]))) - tag_intersection = list(set(tag_list[0]).intersection(set(tag_list[1]))) - con_rand=random.randint(0,len(condition_list)) - col_rand=random.randint(0,len(col_list)) - tag_rand=random.randint(0,len(tag_list)) - sql='select ' #select - - sel_col_tag=[] - col_rand=random.randint(0,len(col_list)) + col_intersection = list( + set(col_list[0]).intersection(set(col_list[1]))) + tag_intersection = list( + set(tag_list[0]).intersection(set(tag_list[1]))) + con_rand = random.randint(0, len(condition_list)) + col_rand = random.randint(0, len(col_list)) + tag_rand = random.randint(0, len(tag_list)) + sql = 'select ' # select + + sel_col_tag = [] + col_rand = random.randint(0, len(col_list)) if bool(random.getrandbits(1)): sql += '*' else: - sel_col_tag.append('t1.' + str(random.choice(col_list[0] + tag_list[0]))) - sel_col_tag.append('t2.' + str(random.choice(col_list[1] + tag_list[1]))) + sel_col_tag.append( + 't1.' + str(random.choice(col_list[0] + tag_list[0]))) + sel_col_tag.append( + 't2.' + str(random.choice(col_list[1] + tag_list[1]))) sel_col_list = [] random.shuffle(func_list) if self.random_pick(): loop = 0 - for i,j in zip(sel_col_tag,func_list): #决定每个被查询col的函数 - alias = ' as '+ 'taos%d ' % loop + for i, j in zip(sel_col_tag, func_list): # 决定每个被查询col的函数 + alias = ' as ' + 'taos%d ' % loop loop += 1 pick_func = '' if j == 'leastsquares': - pick_func=j+'('+i+',1,1)' + pick_func = j+'('+i+',1,1)' elif j == 'top' or j == 'bottom' or j == 'percentile' or j == 'apercentile': - pick_func=j+'('+i+',1)' + pick_func = j+'('+i+',1)' else: - pick_func=j+'('+i+')' + pick_func = j+'('+i+')' if bool(random.getrandbits(1)): - pick_func+=alias + pick_func += alias sel_col_list.append(pick_func) sql += ','.join(sel_col_list) else: sql += ','.join(sel_col_tag) - sql = sql + ' from '+ str(tbname[0]) +' t1,' + str(tbname[1]) + ' t2 ' #select col & func + sql = sql + ' from ' + \ + str(tbname[0]) + ' t1,' + str(tbname[1]) + \ + ' t2 ' # select col & func join_section = None temp = None if subtable: temp = random.choices(col_intersection) join_section = temp.pop() - sql += 'where t1._c0 = t2._c0 and ' + 't1.' + str(join_section) + '=t2.' + str(join_section) + sql += 'where t1._c0 = t2._c0 and ' + 't1.' + \ + str(join_section) + '=t2.' + str(join_section) else: temp = random.choices(col_intersection+tag_intersection) join_section = temp.pop() - sql += 'where t1._c0 = t2._c0 and ' + 't1.' + str(join_section) + '=t2.' + str(join_section) + sql += 'where t1._c0 = t2._c0 and ' + 't1.' + \ + str(join_section) + '=t2.' + str(join_section) return sql - def random_pick(self): - x = random.uniform(0,1) - cumulative_probability = 0.0 - for item, item_probability in zip(self.ifjoin, self.probabilities): - cumulative_probability += item_probability - if x < cumulative_probability:break + def random_pick(self): + x = random.uniform(0, 1) + cumulative_probability = 0.0 + for item, item_probability in zip(self.ifjoin, self.probabilities): + cumulative_probability += item_probability + if x < cumulative_probability: + break return item - + def gen_data(self): stableNum = self.stableNum subtableNum = self.subtableNum @@ -412,52 +461,54 @@ class ConcurrentInquiry: user = self.user password = self.password conn = taos.connect( - host, - user, - password, - ) + host='%s' % host, + user='%s' % user, + password='%s' % password, + ) cl = conn.cursor() - cl.execute("drop database if exists %s;" %self.dbname) - cl.execute("create database if not exists %s;" %self.dbname) + cl.execute("drop database if exists %s;" % self.dbname) + cl.execute("create database if not exists %s;" % self.dbname) cl.execute("use %s" % self.dbname) for k in range(stableNum): - sql="create table %s (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool,c8 binary(20),c9 nchar(20),c11 int unsigned,c12 smallint unsigned,c13 tinyint unsigned,c14 bigint unsigned) \ + sql = "create table %s (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool,c8 binary(20),c9 nchar(20),c11 int unsigned,c12 smallint unsigned,c13 tinyint unsigned,c14 bigint unsigned) \ tags(t1 int, t2 float, t3 bigint, t4 smallint, t5 tinyint, t6 double, t7 bool,t8 binary(20),t9 nchar(20), t11 int unsigned , t12 smallint unsigned , t13 tinyint unsigned , t14 bigint unsigned)" % (self.stb_prefix+str(k)) cl.execute(sql) for j in range(subtableNum): if j % 100 == 0: sql = "create table %s using %s tags(NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)" % \ - (self.subtb_prefix+str(k)+'_'+str(j),self.stb_prefix+str(k)) + (self.subtb_prefix+str(k)+'_' + + str(j), self.stb_prefix+str(k)) else: sql = "create table %s using %s tags(%d,%d,%d,%d,%d,%d,%d,'%s','%s',%d,%d,%d,%d)" % \ - (self.subtb_prefix+str(k)+'_'+str(j),self.stb_prefix+str(k),j,j/2.0,j%41,j%51,j%53,j*1.0,j%2,'taos'+str(j),'涛思'+str(j), j%43, j%23 , j%17 , j%3167) + (self.subtb_prefix+str(k)+'_'+str(j), self.stb_prefix+str(k), j, j/2.0, j % 41, j % + 51, j % 53, j*1.0, j % 2, 'taos'+str(j), '涛思'+str(j), j % 43, j % 23, j % 17, j % 3167) print(sql) cl.execute(sql) for i in range(insertRows): - if i % 100 == 0 : + if i % 100 == 0: ret = cl.execute( - "insert into %s values (%d , NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)" % - (self.subtb_prefix+str(k)+'_'+str(j), t0+i)) + "insert into %s values (%d , NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)" % + (self.subtb_prefix+str(k)+'_'+str(j), t0+i)) else: ret = cl.execute( "insert into %s values (%d , %d,%d,%d,%d,%d,%d,%d,'%s','%s',%d,%d,%d,%d)" % - (self.subtb_prefix+str(k)+'_'+str(j), t0+i, i%100, i/2.0, i%41, i%51, i%53, i*1.0, i%2,'taos'+str(i),'涛思'+str(i), i%43, i%23 , i%17 , i%3167)) + (self.subtb_prefix+str(k)+'_'+str(j), t0+i, i % 100, i/2.0, i % 41, i % 51, i % 53, i*1.0, i % 2, 'taos'+str(i), '涛思'+str(i), i % 43, i % 23, i % 17, i % 3167)) cl.close() conn.close() - - def rest_query(self,sql): #rest 接口 + + def rest_query(self, sql): # rest 接口 host = self.host user = self.user password = self.password - port =6041 - url = "http://{}:{}/rest/sql".format(host, port ) + port = 6041 + url = "http://{}:{}/rest/sql".format(host, port) try: - r = requests.post(url, - data = 'use %s' % self.dbname, - auth = HTTPBasicAuth('root', 'taosdata')) - r = requests.post(url, - data = sql, - auth = HTTPBasicAuth('root', 'taosdata')) + r = requests.post(url, + data='use %s' % self.dbname, + auth=HTTPBasicAuth('root', 'taosdata')) + r = requests.post(url, + data=sql, + auth=HTTPBasicAuth('root', 'taosdata')) except: print("REST API Failure (TODO: more info here)") raise @@ -481,165 +532,171 @@ class ConcurrentInquiry: nRows = rj['rows'] if ('rows' in rj) else 0 return nRows - - def query_thread_n(self,threadID): #使用原生python接口查询 + def query_thread_n(self, threadID): # 使用原生python接口查询 host = self.host user = self.user password = self.password conn = taos.connect( - host, - user, - password, - ) + host='%s' % host, + user='%s' % user, + password='%s' % password, + ) cl = conn.cursor() cl.execute("use %s;" % self.dbname) - fo = open('bak_sql_n_%d'%threadID,'w+') + fo = open('bak_sql_n_%d' % threadID, 'w+') print("Thread %d: starting" % threadID) loop = self.loop while loop: - - try: - if self.random_pick(): - if self.random_pick(): - sql,temp=self.gen_query_sql() - else: - sql = self.gen_subquery_sql() - else: - sql = self.gen_query_join() - print("sql is ",sql) - fo.write(sql+'\n') - start = time.time() - cl.execute(sql) - cl.fetchall() - end = time.time() - print("time cost :",end-start) - except Exception as e: - print('-'*40) - print( - "Failure thread%d, sql: %s \nexception: %s" % - (threadID, str(sql),str(e))) - err_uec='Unable to establish connection' - if err_uec in str(e) and loop >0: - exit(-1) - loop -= 1 - if loop == 0: break - fo.close() - cl.close() - conn.close() - print("Thread %d: finishing" % threadID) - def query_thread_nr(self,threadID): #使用原生python接口进行重放 - host = self.host - user = self.user - password = self.password - conn = taos.connect( - host, - user, - password, - ) - cl = conn.cursor() - cl.execute("use %s;" % self.dbname) - replay_sql = [] - with open('bak_sql_n_%d'%threadID,'r') as f: - replay_sql = f.readlines() - print("Replay Thread %d: starting" % threadID) - for sql in replay_sql: try: - print("sql is ",sql) + if self.random_pick(): + if self.random_pick(): + sql, temp = self.gen_query_sql() + else: + sql = self.gen_subquery_sql() + else: + sql = self.gen_query_join() + print("sql is ", sql) + fo.write(sql+'\n') start = time.time() cl.execute(sql) cl.fetchall() end = time.time() - print("time cost :",end-start) + print("time cost :", end-start) except Exception as e: print('-'*40) print( - "Failure thread%d, sql: %s \nexception: %s" % - (threadID, str(sql),str(e))) - err_uec='Unable to establish connection' - if err_uec in str(e) and loop >0: - exit(-1) + "Failure thread%d, sql: %s \nexception: %s" % + (threadID, str(sql), str(e))) + err_uec = 'Unable to establish connection' + if err_uec in str(e) and loop > 0: + exit(-1) + loop -= 1 + if loop == 0: + break + fo.close() cl.close() - conn.close() + conn.close() + print("Thread %d: finishing" % threadID) + + def query_thread_nr(self, threadID): # 使用原生python接口进行重放 + host = self.host + user = self.user + password = self.password + conn = taos.connect( + host='%s' % host, + user='%s' % user, + password='%s' % password, + ) + cl = conn.cursor() + cl.execute("use %s;" % self.dbname) + replay_sql = [] + with open('bak_sql_n_%d' % threadID, 'r') as f: + replay_sql = f.readlines() + print("Replay Thread %d: starting" % threadID) + for sql in replay_sql: + try: + print("sql is ", sql) + start = time.time() + cl.execute(sql) + cl.fetchall() + end = time.time() + print("time cost :", end-start) + except Exception as e: + print('-'*40) + print( + "Failure thread%d, sql: %s \nexception: %s" % + (threadID, str(sql), str(e))) + err_uec = 'Unable to establish connection' + if err_uec in str(e) and loop > 0: + exit(-1) + cl.close() + conn.close() print("Replay Thread %d: finishing" % threadID) - - def query_thread_r(self,threadID): #使用rest接口查询 + + def query_thread_r(self, threadID): # 使用rest接口查询 print("Thread %d: starting" % threadID) - fo = open('bak_sql_r_%d'%threadID,'w+') + fo = open('bak_sql_r_%d' % threadID, 'w+') loop = self.loop while loop: try: if self.random_pick(): if self.random_pick(): - sql,temp=self.gen_query_sql() + sql, temp = self.gen_query_sql() else: sql = self.gen_subquery_sql() else: sql = self.gen_query_join() - print("sql is ",sql) + print("sql is ", sql) fo.write(sql+'\n') start = time.time() self.rest_query(sql) end = time.time() - print("time cost :",end-start) + print("time cost :", end-start) except Exception as e: print('-'*40) print( - "Failure thread%d, sql: %s \nexception: %s" % - (threadID, str(sql),str(e))) - err_uec='Unable to establish connection' - if err_uec in str(e) and loop >0: + "Failure thread%d, sql: %s \nexception: %s" % + (threadID, str(sql), str(e))) + err_uec = 'Unable to establish connection' + if err_uec in str(e) and loop > 0: exit(-1) - loop -= 1 - if loop == 0: break - fo.close() - print("Thread %d: finishing" % threadID) + loop -= 1 + if loop == 0: + break + fo.close() + print("Thread %d: finishing" % threadID) - def query_thread_rr(self,threadID): #使用rest接口重放 + def query_thread_rr(self, threadID): # 使用rest接口重放 print("Replay Thread %d: starting" % threadID) replay_sql = [] - with open('bak_sql_r_%d'%threadID,'r') as f: + with open('bak_sql_r_%d' % threadID, 'r') as f: replay_sql = f.readlines() for sql in replay_sql: try: - print("sql is ",sql) + print("sql is ", sql) start = time.time() self.rest_query(sql) end = time.time() - print("time cost :",end-start) + print("time cost :", end-start) except Exception as e: print('-'*40) print( - "Failure thread%d, sql: %s \nexception: %s" % - (threadID, str(sql),str(e))) - err_uec='Unable to establish connection' - if err_uec in str(e) and loop >0: - exit(-1) - print("Replay Thread %d: finishing" % threadID) + "Failure thread%d, sql: %s \nexception: %s" % + (threadID, str(sql), str(e))) + err_uec = 'Unable to establish connection' + if err_uec in str(e) and loop > 0: + exit(-1) + print("Replay Thread %d: finishing" % threadID) def run(self): - print(self.n_numOfTherads,self.r_numOfTherads) + print(self.n_numOfTherads, self.r_numOfTherads) threads = [] - if self.replay: #whether replay + if self.replay: # whether replay for i in range(self.n_numOfTherads): - thread = threading.Thread(target=self.query_thread_nr, args=(i,)) + thread = threading.Thread( + target=self.query_thread_nr, args=(i,)) threads.append(thread) - thread.start() + thread.start() for i in range(self.r_numOfTherads): - thread = threading.Thread(target=self.query_thread_rr, args=(i,)) + thread = threading.Thread( + target=self.query_thread_rr, args=(i,)) threads.append(thread) thread.start() else: for i in range(self.n_numOfTherads): - thread = threading.Thread(target=self.query_thread_n, args=(i,)) - threads.append(thread) - thread.start() - for i in range(self.r_numOfTherads): - thread = threading.Thread(target=self.query_thread_r, args=(i,)) + thread = threading.Thread( + target=self.query_thread_n, args=(i,)) threads.append(thread) thread.start() - + for i in range(self.r_numOfTherads): + thread = threading.Thread( + target=self.query_thread_r, args=(i,)) + threads.append(thread) + thread.start() + + parser = argparse.ArgumentParser() parser.add_argument( '-H', @@ -721,17 +778,17 @@ parser.add_argument( parser.add_argument( '-u', '--user', - action='store', + action='store', default='root', type=str, help='user name') parser.add_argument( '-w', '--password', - action='store', - default='root', + action='store', + default='taosdata', type=str, - help='user name') + help='password') parser.add_argument( '-n', '--number-of-tables', @@ -763,15 +820,14 @@ parser.add_argument( args = parser.parse_args() q = ConcurrentInquiry( - args.ts,args.host_name,args.user,args.password,args.db_name, - args.stb_name_prefix,args.subtb_name_prefix,args.number_of_native_threads,args.number_of_rest_threads, - args.probabilities,args.loop_per_thread,args.number_of_stables,args.number_of_tables ,args.number_of_records, - args.mix_stable_subtable, args.replay ) + args.ts, args.host_name, args.user, args.password, args.db_name, + args.stb_name_prefix, args.subtb_name_prefix, args.number_of_native_threads, args.number_of_rest_threads, + args.probabilities, args.loop_per_thread, args.number_of_stables, args.number_of_tables, args.number_of_records, + args.mix_stable_subtable, args.replay) -if args.create_table: +if args.create_table: q.gen_data() q.get_full() -#q.gen_query_sql() +# q.gen_query_sql() q.run() - From fc41f43c9d2169bf16ad1a13ec950439f1075306 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 4 Nov 2022 16:15:37 +0800 Subject: [PATCH 43/82] fix: deadlock while drop db --- include/util/tworker.h | 2 ++ source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 20 ++++++++++++++------ source/dnode/vnode/src/inc/vnd.h | 1 + source/dnode/vnode/src/vnd/vnodeOpen.c | 7 +------ source/dnode/vnode/src/vnd/vnodeSync.c | 17 +++++++++++++++-- source/util/src/tworker.c | 17 ++++++++++------- 6 files changed, 43 insertions(+), 21 deletions(-) diff --git a/include/util/tworker.h b/include/util/tworker.h index 3545aeed89..6b439d0c29 100644 --- a/include/util/tworker.h +++ b/include/util/tworker.h @@ -28,6 +28,7 @@ typedef struct SWWorkerPool SWWorkerPool; typedef struct SQWorker { int32_t id; // worker ID TdThread thread; // thread + int64_t pid; SQWorkerPool *pool; } SQWorker; @@ -44,6 +45,7 @@ typedef struct SQWorkerPool { typedef struct SWWorker { int32_t id; // worker id TdThread thread; // thread + int64_t pid; STaosQall *qall; STaosQset *qset; // queue set SWWorkerPool *pool; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 16cfa480b1..0d476e30ab 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -79,35 +79,43 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { char path[TSDB_FILENAME_LEN] = {0}; - vnodePreClose(pVnode->pImpl); - taosThreadRwlockWrlock(&pMgmt->lock); taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t)); taosThreadRwlockUnlock(&pMgmt->lock); vmReleaseVnode(pMgmt, pVnode); + dInfo("vgId:%d, pre close", pVnode->vgId); + vnodePreClose(pVnode->pImpl); + dInfo("vgId:%d, wait for vnode ref become 0", pVnode->vgId); while (pVnode->refCount > 0) taosMsleep(10); dInfo("vgId:%d, wait for vnode write queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteQ, pVnode->pWriteQ->threadId); while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10); + dInfo("vgId:%d, wait for vnode sync queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncQ, - pVnode->pWriteQ->threadId); + pVnode->pSyncQ->threadId); while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10); + dInfo("vgId:%d, wait for vnode sync ctrl queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncCtrlQ, - pVnode->pWriteQ->threadId); + pVnode->pSyncCtrlQ->threadId); while (!taosQueueEmpty(pVnode->pSyncCtrlQ)) taosMsleep(10); + dInfo("vgId:%d, wait for vnode apply queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyQ, - pVnode->pWriteQ->threadId); + pVnode->pApplyQ->threadId); while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10); + dInfo("vgId:%d, wait for vnode query queue:%p is empty", pVnode->vgId, pVnode->pQueryQ); while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10); + dInfo("vgId:%d, wait for vnode fetch queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pFetchQ, - pVnode->pWriteQ->threadId); + pVnode->pFetchQ->threadId); while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10); + dInfo("vgId:%d, wait for vnode stream queue:%p is empty", pVnode->vgId, pVnode->pStreamQ); while (!taosQueueEmpty(pVnode->pStreamQ)) taosMsleep(10); + dInfo("vgId:%d, all vnode queues is empty", pVnode->vgId); vmFreeQueue(pMgmt, pVnode); diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index d5ad500fdb..8f8691cfc2 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -97,6 +97,7 @@ bool vnodeShouldRollback(SVnode* pVnode); // vnodeSync.c int32_t vnodeSyncOpen(SVnode* pVnode, char* path); void vnodeSyncStart(SVnode* pVnode); +void vnodeSyncPreClose(SVnode* pVnode); void vnodeSyncClose(SVnode* pVnode); void vnodeRedirectRpcMsg(SVnode* pVnode, SRpcMsg* pMsg); bool vnodeIsLeader(SVnode* pVnode); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index f7164c4ac3..8c2036b97b 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -242,12 +242,7 @@ _err: return NULL; } -void vnodePreClose(SVnode *pVnode) { - if (pVnode) { - syncLeaderTransfer(pVnode->sync); - syncPreStop(pVnode->sync); - } -} +void vnodePreClose(SVnode *pVnode) { vnodeSyncPreClose(pVnode); } void vnodeClose(SVnode *pVnode) { if (pVnode) { diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 7f3b1db7c3..8ce559e232 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -474,12 +474,25 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path) { } void vnodeSyncStart(SVnode *pVnode) { - vDebug("vgId:%d, start sync", pVnode->config.vgId); + vInfo("vgId:%d, start sync", pVnode->config.vgId); syncStart(pVnode->sync); } +void vnodeSyncPreClose(SVnode *pVnode) { + vInfo("vgId:%d, pre close sync", pVnode->config.vgId); + syncLeaderTransfer(pVnode->sync); + syncPreStop(pVnode->sync); + taosThreadMutexLock(&pVnode->lock); + if (pVnode->blocked) { + vInfo("vgId:%d, post block after close sync", pVnode->config.vgId); + pVnode->blocked = false; + tsem_post(&pVnode->syncSem); + } + taosThreadMutexUnlock(&pVnode->lock); +} + void vnodeSyncClose(SVnode *pVnode) { - vDebug("vgId:%d, close sync", pVnode->config.vgId); + vInfo("vgId:%d, close sync", pVnode->config.vgId); syncStop(pVnode->sync); } diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 27c641555f..a5f7da235a 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -73,12 +73,13 @@ static void *tQWorkerThreadFp(SQWorker *worker) { taosBlockSIGPIPE(); setThreadName(pool->name); - uInfo("worker:%s:%d is running, thread:%08" PRId64, pool->name, worker->id, taosGetSelfPthreadId()); + worker->pid = taosGetSelfPthreadId(); + uInfo("worker:%s:%d is running, thread:%08" PRId64, pool->name, worker->id, worker->pid); while (1) { if (taosReadQitemFromQset(pool->qset, (void **)&msg, &qinfo) == 0) { uInfo("worker:%s:%d qset:%p, got no message and exiting, thread:%08" PRId64, pool->name, worker->id, pool->qset, - taosGetSelfPthreadId()); + worker->pid); break; } @@ -125,7 +126,7 @@ STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) { } taosThreadMutexUnlock(&pool->mutex); - uDebug("worker:%s, queue:%p is allocated, ahandle:%p", pool->name, queue, ahandle); + uInfo("worker:%s, queue:%p is allocated, ahandle:%p", pool->name, queue, ahandle); return queue; } @@ -192,13 +193,14 @@ static void *tWWorkerThreadFp(SWWorker *worker) { taosBlockSIGPIPE(); setThreadName(pool->name); - uInfo("worker:%s:%d is running, thread:%08" PRId64, pool->name, worker->id, taosGetSelfPthreadId()); + worker->pid = taosGetSelfPthreadId(); + uInfo("worker:%s:%d is running, thread:%08" PRId64, pool->name, worker->id, worker->pid); while (1) { numOfMsgs = taosReadAllQitemsFromQset(worker->qset, worker->qall, &qinfo); if (numOfMsgs == 0) { uInfo("worker:%s:%d qset:%p, got no message and exiting, thread:%08" PRId64, pool->name, worker->id, worker->qset, - taosGetSelfPthreadId()); + worker->pid); break; } @@ -246,8 +248,9 @@ STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) { pool->nextId = (pool->nextId + 1) % pool->max; } - queue->threadId = taosGetPthreadId(worker->thread); - uDebug("worker:%s, queue:%p is allocated, ahandle:%p thread:%08" PRId64, pool->name, queue, ahandle, queue->threadId); + while (worker->pid <= 0) taosMsleep(10); + queue->threadId = worker->pid; + uInfo("worker:%s, queue:%p is allocated, ahandle:%p thread:%08" PRId64, pool->name, queue, ahandle, queue->threadId); code = 0; _OVER: From 1204faefb67311a1c9e605a19fa05b27df072541 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 4 Nov 2022 16:15:37 +0800 Subject: [PATCH 44/82] fix: fix memory leak --- source/libs/function/src/builtinsimpl.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index c8f0b3d826..72093793a1 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -5451,6 +5451,8 @@ int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { colDataAppendNULL(pCol, currentRow); } + taosHashCleanup(pInfo->pHash); + return pResInfo->numOfRes; } From f995723f206494b3684fe97144e05139520272a0 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 4 Nov 2022 16:17:05 +0800 Subject: [PATCH 45/82] test:add compatibility cases: 4096 columns of table data and tests that need to recover data from wal files --- tests/pytest/util/dnodes.py | 2 +- tests/system-test/0-others/compa4096.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 19f431af6b..b762f8c77f 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -418,7 +418,7 @@ class TDDnode: if i > 50: break with open(logFile) as f: - timeout = time.time() + 60 * 2 + timeout = time.time() + 10 * 2 while True: line = f.readline().encode('utf-8') if bkey in line: diff --git a/tests/system-test/0-others/compa4096.json b/tests/system-test/0-others/compa4096.json index 46f98be8bd..5cc5d2084d 100644 --- a/tests/system-test/0-others/compa4096.json +++ b/tests/system-test/0-others/compa4096.json @@ -23,7 +23,7 @@ "precision": "ms", "keep": 3650, "comp": 2, - "vgroups": 3, + "vgroups": 2, "buffer": 1000 }, "super_tables": [ @@ -45,7 +45,7 @@ "childtable_limit": 0, "childtable_offset": 0, "rows_per_tbl": 0, - "max_sql_len": 1048576, + "max_sql_len": 1048576, "disorder_ratio": 0, "disorder_range": 1000, "timestamp_step": 1000, From 5a893cf75a928e613ffd43c7b7b3ca69883368ed Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 4 Nov 2022 16:18:09 +0800 Subject: [PATCH 46/82] test:add compatibility cases --- tests/system-test/0-others/compatibility.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index e70eab64bf..30513ac020 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -110,7 +110,7 @@ class TDTestCase: print(f"start taosd: nohup taosd -c {cPath} & ") os.system(f" nohup taosd -c {cPath} & " ) - sleep(2) + sleep(10) tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ") os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y") os.system("pkill -9 taosd") From ff1313f9bc46f05935aa9da2baa990343d49a71d Mon Sep 17 00:00:00 2001 From: xleili Date: Fri, 4 Nov 2022 16:21:08 +0800 Subject: [PATCH 47/82] ../../../tests/docs-examples-test/csharp.sh --- tests/docs-examples-test/csharp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docs-examples-test/csharp.sh b/tests/docs-examples-test/csharp.sh index 8d1031ab8f..a1e46394ee 100644 --- a/tests/docs-examples-test/csharp.sh +++ b/tests/docs-examples-test/csharp.sh @@ -27,7 +27,7 @@ dotnet run --project optsTelnet/optstelnet.csproj taos -s "drop database if exists test" dotnet run --project optsJSON/optsJSON.csproj -taos -s "create database if exists test" +taos -s "create database if not exists test" dotnet run --project wsConnect/wsConnect.csproj dotnet run --project wsInsert/wsInsert.csproj dotnet run --project wsStmt/wsStmt.csproj From 4fc0e3bd55ecac34dc05156fbbe38fd55dfc61dc Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 4 Nov 2022 17:21:00 +0800 Subject: [PATCH 48/82] enh: adjust the number of vnode threads so that one vnode has one write thread --- docs/en/14-reference/12-config/index.md | 24 +++--- docs/zh/14-reference/12-config/index.md | 24 +++--- include/common/tglobal.h | 2 - include/util/tworker.h | 12 +-- source/common/src/tglobal.c | 32 -------- source/dnode/mgmt/mgmt_vnode/inc/vmInt.h | 30 +++---- source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 24 +++--- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 87 ++++++--------------- source/util/src/tworker.c | 2 - 9 files changed, 79 insertions(+), 158 deletions(-) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 769d19e7f9..aadf563a63 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -735,19 +735,17 @@ To prevent system resource from being exhausted by multiple concurrent streams, | 44 | numOfVnodeQueryThreads | No | Yes | | 45 | numOfVnodeStreamThreads | No | Yes | | 46 | numOfVnodeFetchThreads | No | Yes | -| 47 | numOfVnodeWriteThreads | No | Yes | -| 48 | numOfVnodeSyncThreads | No | Yes | -| 49 | numOfVnodeRsmaThreads | No | Yes | -| 50 | numOfQnodeQueryThreads | No | Yes | -| 51 | numOfQnodeFetchThreads | No | Yes | -| 52 | numOfSnodeSharedThreads | No | Yes | -| 53 | numOfSnodeUniqueThreads | No | Yes | -| 54 | rpcQueueMemoryAllowed | No | Yes | -| 55 | logDir | Yes | Yes | -| 56 | minimalLogDirGB | Yes | Yes | -| 57 | numOfLogLines | Yes | Yes | -| 58 | asyncLog | Yes | Yes | -| 59 | logKeepDays | Yes | Yes | +| 47 | numOfVnodeRsmaThreads | No | Yes | +| 48 | numOfQnodeQueryThreads | No | Yes | +| 49 | numOfQnodeFetchThreads | No | Yes | +| 50 | numOfSnodeSharedThreads | No | Yes | +| 51 | numOfSnodeUniqueThreads | No | Yes | +| 52 | rpcQueueMemoryAllowed | No | Yes | +| 53 | logDir | Yes | Yes | +| 54 | minimalLogDirGB | Yes | Yes | +| 55 | numOfLogLines | Yes | Yes | +| 56 | asyncLog | Yes | Yes | +| 57 | logKeepDays | Yes | Yes | | 60 | debugFlag | Yes | Yes | | 61 | tmrDebugFlag | Yes | Yes | | 62 | uDebugFlag | Yes | Yes | diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index d587dd2f2c..54c362de95 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -711,19 +711,17 @@ charset 的有效值是 UTF-8。 | 44 | numOfVnodeQueryThreads | 否 | 是 | | | 45 | numOfVnodeStreamThreads | 否 | 是 | | | 46 | numOfVnodeFetchThreads | 否 | 是 | | -| 47 | numOfVnodeWriteThreads | 否 | 是 | | -| 48 | numOfVnodeSyncThreads | 否 | 是 | | -| 49 | numOfVnodeRsmaThreads | 否 | 是 | | -| 50 | numOfQnodeQueryThreads | 否 | 是 | | -| 51 | numOfQnodeFetchThreads | 否 | 是 | | -| 52 | numOfSnodeSharedThreads | 否 | 是 | | -| 53 | numOfSnodeUniqueThreads | 否 | 是 | | -| 54 | rpcQueueMemoryAllowed | 否 | 是 | | -| 55 | logDir | 是 | 是 | | -| 56 | minimalLogDirGB | 是 | 是 | | -| 57 | numOfLogLines | 是 | 是 | | -| 58 | asyncLog | 是 | 是 | | -| 59 | logKeepDays | 是 | 是 | | +| 47 | numOfVnodeRsmaThreads | 否 | 是 | | +| 48 | numOfQnodeQueryThreads | 否 | 是 | | +| 49 | numOfQnodeFetchThreads | 否 | 是 | | +| 50 | numOfSnodeSharedThreads | 否 | 是 | | +| 51 | numOfSnodeUniqueThreads | 否 | 是 | | +| 52 | rpcQueueMemoryAllowed | 否 | 是 | | +| 53 | logDir | 是 | 是 | | +| 54 | minimalLogDirGB | 是 | 是 | | +| 55 | numOfLogLines | 是 | 是 | | +| 56 | asyncLog | 是 | 是 | | +| 57 | logKeepDays | 是 | 是 | | | 60 | debugFlag | 是 | 是 | | | 61 | tmrDebugFlag | 是 | 是 | | | 62 | uDebugFlag | 是 | 是 | | diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 99bbfde3e1..681d1beb79 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -55,8 +55,6 @@ extern int32_t tsNumOfMnodeReadThreads; extern int32_t tsNumOfVnodeQueryThreads; extern int32_t tsNumOfVnodeStreamThreads; extern int32_t tsNumOfVnodeFetchThreads; -extern int32_t tsNumOfVnodeWriteThreads; -extern int32_t tsNumOfVnodeSyncThreads; extern int32_t tsNumOfVnodeRsmaThreads; extern int32_t tsNumOfQnodeQueryThreads; extern int32_t tsNumOfQnodeFetchThreads; diff --git a/include/util/tworker.h b/include/util/tworker.h index 6b439d0c29..8766f87a08 100644 --- a/include/util/tworker.h +++ b/include/util/tworker.h @@ -26,9 +26,9 @@ typedef struct SQWorkerPool SQWorkerPool; typedef struct SWWorkerPool SWWorkerPool; typedef struct SQWorker { - int32_t id; // worker ID - TdThread thread; // thread - int64_t pid; + int32_t id; // worker id + int64_t pid; // thread pid + TdThread thread; // thread id SQWorkerPool *pool; } SQWorker; @@ -44,10 +44,10 @@ typedef struct SQWorkerPool { typedef struct SWWorker { int32_t id; // worker id - TdThread thread; // thread - int64_t pid; + int64_t pid; // thread pid + TdThread thread; // thread id STaosQall *qall; - STaosQset *qset; // queue set + STaosQset *qset; SWWorkerPool *pool; } SWWorker; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 7f4a826c5e..064249d754 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -50,8 +50,6 @@ int32_t tsNumOfMnodeReadThreads = 1; int32_t tsNumOfVnodeQueryThreads = 4; int32_t tsNumOfVnodeStreamThreads = 2; int32_t tsNumOfVnodeFetchThreads = 4; -int32_t tsNumOfVnodeWriteThreads = 2; -int32_t tsNumOfVnodeSyncThreads = 2; int32_t tsNumOfVnodeRsmaThreads = 2; int32_t tsNumOfQnodeQueryThreads = 4; int32_t tsNumOfQnodeFetchThreads = 1; @@ -374,14 +372,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfVnodeFetchThreads = TMAX(tsNumOfVnodeFetchThreads, 4); if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 4, 1024, 0) != 0) return -1; - tsNumOfVnodeWriteThreads = tsNumOfCores; - tsNumOfVnodeWriteThreads = TMAX(tsNumOfVnodeWriteThreads, 1); - if (cfgAddInt32(pCfg, "numOfVnodeWriteThreads", tsNumOfVnodeWriteThreads, 1, 1024, 0) != 0) return -1; - - tsNumOfVnodeSyncThreads = tsNumOfCores * 2; - tsNumOfVnodeSyncThreads = TMAX(tsNumOfVnodeSyncThreads, 16); - if (cfgAddInt32(pCfg, "numOfVnodeSyncThreads", tsNumOfVnodeSyncThreads, 1, 1024, 0) != 0) return -1; - tsNumOfVnodeRsmaThreads = tsNumOfCores; tsNumOfVnodeRsmaThreads = TMAX(tsNumOfVnodeRsmaThreads, 4); if (cfgAddInt32(pCfg, "numOfVnodeRsmaThreads", tsNumOfVnodeRsmaThreads, 1, 1024, 0) != 0) return -1; @@ -506,22 +496,6 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) { pItem->stype = stype; } - pItem = cfgGetItem(tsCfg, "numOfVnodeWriteThreads"); - if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { - tsNumOfVnodeWriteThreads = numOfCores; - tsNumOfVnodeWriteThreads = TMAX(tsNumOfVnodeWriteThreads, 1); - pItem->i32 = tsNumOfVnodeWriteThreads; - pItem->stype = stype; - } - - pItem = cfgGetItem(tsCfg, "numOfVnodeSyncThreads"); - if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { - tsNumOfVnodeSyncThreads = numOfCores * 2; - tsNumOfVnodeSyncThreads = TMAX(tsNumOfVnodeSyncThreads, 16); - pItem->i32 = tsNumOfVnodeSyncThreads; - pItem->stype = stype; - } - pItem = cfgGetItem(tsCfg, "numOfVnodeRsmaThreads"); if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { tsNumOfVnodeRsmaThreads = numOfCores; @@ -699,8 +673,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32; tsNumOfVnodeStreamThreads = cfgGetItem(pCfg, "numOfVnodeStreamThreads")->i32; tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32; - tsNumOfVnodeWriteThreads = cfgGetItem(pCfg, "numOfVnodeWriteThreads")->i32; - tsNumOfVnodeSyncThreads = cfgGetItem(pCfg, "numOfVnodeSyncThreads")->i32; tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32; tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32; // tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32; @@ -943,10 +915,6 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { } else if (strcasecmp("numOfVnodeFetchThreads", name) == 0) { tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32; */ - } else if (strcasecmp("numOfVnodeWriteThreads", name) == 0) { - tsNumOfVnodeWriteThreads = cfgGetItem(pCfg, "numOfVnodeWriteThreads")->i32; - } else if (strcasecmp("numOfVnodeSyncThreads", name) == 0) { - tsNumOfVnodeSyncThreads = cfgGetItem(pCfg, "numOfVnodeSyncThreads")->i32; } else if (strcasecmp("numOfVnodeRsmaThreads", name) == 0) { tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32; } else if (strcasecmp("numOfQnodeQueryThreads", name) == 0) { diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index bf1ccc1a7b..b38dc19361 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -33,10 +33,6 @@ typedef struct SVnodeMgmt { SQWorkerPool queryPool; SQWorkerPool streamPool; SWWorkerPool fetchPool; - SWWorkerPool syncPool; - SWWorkerPool syncCtrlPool; - SWWorkerPool writePool; - SWWorkerPool applyPool; SSingleWorker mgmtWorker; SHashObj *hash; TdThreadRwlock lock; @@ -52,19 +48,19 @@ typedef struct { } SWrapperCfg; typedef struct { - int32_t vgId; - int32_t vgVersion; - int32_t refCount; - int8_t dropped; - char *path; - SVnode *pImpl; - STaosQueue *pWriteQ; - STaosQueue *pSyncQ; - STaosQueue *pSyncCtrlQ; - STaosQueue *pApplyQ; - STaosQueue *pQueryQ; - STaosQueue *pStreamQ; - STaosQueue *pFetchQ; + int32_t vgId; + int32_t vgVersion; + int32_t refCount; + int8_t dropped; + char *path; + SVnode *pImpl; + SMultiWorker pWriteW; + SMultiWorker pSyncW; + SMultiWorker pSyncCtrlW; + SMultiWorker pApplyW; + STaosQueue *pQueryQ; + STaosQueue *pStreamQ; + STaosQueue *pFetchQ; } SVnodeObj; typedef struct { diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 0d476e30ab..07ebd72379 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -90,21 +90,21 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { dInfo("vgId:%d, wait for vnode ref become 0", pVnode->vgId); while (pVnode->refCount > 0) taosMsleep(10); - dInfo("vgId:%d, wait for vnode write queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteQ, - pVnode->pWriteQ->threadId); - while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10); + dInfo("vgId:%d, wait for vnode write queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteW.queue, + pVnode->pWriteW.queue->threadId); + tMultiWorkerCleanup(&pVnode->pWriteW); - dInfo("vgId:%d, wait for vnode sync queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncQ, - pVnode->pSyncQ->threadId); - while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10); + dInfo("vgId:%d, wait for vnode sync queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncW.queue, + pVnode->pSyncW.queue->threadId); + tMultiWorkerCleanup(&pVnode->pSyncW); - dInfo("vgId:%d, wait for vnode sync ctrl queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncCtrlQ, - pVnode->pSyncCtrlQ->threadId); - while (!taosQueueEmpty(pVnode->pSyncCtrlQ)) taosMsleep(10); + dInfo("vgId:%d, wait for vnode sync ctrl queue:%p is empty, thread:%08" PRId64, pVnode->vgId, + pVnode->pSyncCtrlW.queue, pVnode->pSyncCtrlW.queue->threadId); + tMultiWorkerCleanup(&pVnode->pSyncCtrlW); - dInfo("vgId:%d, wait for vnode apply queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyQ, - pVnode->pApplyQ->threadId); - while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10); + dInfo("vgId:%d, wait for vnode apply queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyW.queue, + pVnode->pApplyW.queue->threadId); + tMultiWorkerCleanup(&pVnode->pApplyW); dInfo("vgId:%d, wait for vnode query queue:%p is empty", pVnode->vgId, pVnode->pQueryQ); while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index a5070e09aa..03c242b081 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -188,30 +188,20 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp dDebug("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, terrstr(code)); } else { dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg); - taosWriteQitem(pVnode->pWriteQ, pMsg); -#if 0 // tests for batch writes - if (pMsg->msgType == TDMT_VND_CREATE_TABLE) { - SRpcMsg *pDup = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); - memcpy(pDup, pMsg, sizeof(SRpcMsg)); - pDup->pCont = rpcMallocCont(pMsg->contLen); - memcpy(pDup->pCont, pMsg->pCont, pMsg->contLen); - pDup->info.handle = NULL; - taosWriteQitem(pVnode->pWriteQ, pDup); - } -#endif + taosWriteQitem(pVnode->pWriteW.queue, pMsg); } break; case SYNC_QUEUE: dGTrace("vgId:%d, msg:%p put into vnode-sync queue", pVnode->vgId, pMsg); - taosWriteQitem(pVnode->pSyncQ, pMsg); + taosWriteQitem(pVnode->pSyncW.queue, pMsg); break; case SYNC_CTRL_QUEUE: dGTrace("vgId:%d, msg:%p put into vnode-sync-ctrl queue", pVnode->vgId, pMsg); - taosWriteQitem(pVnode->pSyncCtrlQ, pMsg); + taosWriteQitem(pVnode->pSyncCtrlW.queue, pMsg); break; case APPLY_QUEUE: dGTrace("vgId:%d, msg:%p put into vnode-apply queue", pVnode->vgId, pMsg); - taosWriteQitem(pVnode->pApplyQ, pMsg); + taosWriteQitem(pVnode->pApplyW.queue, pMsg); break; default: code = -1; @@ -276,13 +266,13 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) { if (pVnode != NULL) { switch (qtype) { case WRITE_QUEUE: - size = taosQueueItemSize(pVnode->pWriteQ); + size = taosQueueItemSize(pVnode->pWriteW.queue); break; case SYNC_QUEUE: - size = taosQueueItemSize(pVnode->pSyncQ); + size = taosQueueItemSize(pVnode->pSyncW.queue); break; case APPLY_QUEUE: - size = taosQueueItemSize(pVnode->pApplyQ); + size = taosQueueItemSize(pVnode->pApplyW.queue); break; case QUERY_QUEUE: size = taosQueueItemSize(pVnode->pQueryQ); @@ -306,27 +296,33 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) { } int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { - pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode->pImpl, (FItems)vnodeProposeWriteMsg); - pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)vmProcessSyncQueue); - pVnode->pSyncCtrlQ = tWWorkerAllocQueue(&pMgmt->syncCtrlPool, pVnode, (FItems)vmProcessSyncQueue); - pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->applyPool, pVnode->pImpl, (FItems)vnodeApplyWriteMsg); + SMultiWorkerCfg wcfg = {.max = 1, .name = "vnode-write", .fp = (FItems)vnodeProposeWriteMsg, .param = pVnode->pImpl}; + SMultiWorkerCfg scfg = {.max = 1, .name = "vnode-sync", .fp = (FItems)vmProcessSyncQueue, .param = pVnode}; + SMultiWorkerCfg sccfg = {.max = 1, .name = "vnode-sync-ctrl", .fp = (FItems)vmProcessSyncQueue, .param = pVnode}; + SMultiWorkerCfg acfg = {.max = 1, .name = "vnode-apply", .fp = (FItems)vnodeApplyWriteMsg, .param = pVnode}; + (void)tMultiWorkerInit(&pVnode->pWriteW, &wcfg); + (void)tMultiWorkerInit(&pVnode->pSyncW, &scfg); + (void)tMultiWorkerInit(&pVnode->pSyncCtrlW, &sccfg); + (void)tMultiWorkerInit(&pVnode->pApplyW, &acfg); + pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue); pVnode->pStreamQ = tQWorkerAllocQueue(&pMgmt->streamPool, pVnode, (FItem)vmProcessStreamQueue); pVnode->pFetchQ = tWWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItems)vmProcessFetchQueue); - if (pVnode->pWriteQ == NULL || pVnode->pSyncQ == NULL || pVnode->pApplyQ == NULL || pVnode->pQueryQ == NULL || - pVnode->pStreamQ == NULL || pVnode->pFetchQ == NULL) { + if (pVnode->pWriteW.queue == NULL || pVnode->pSyncW.queue == NULL || pVnode->pSyncCtrlW.queue == NULL || + pVnode->pApplyW.queue == NULL || pVnode->pQueryQ == NULL || pVnode->pStreamQ == NULL || pVnode->pFetchQ == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - dInfo("vgId:%d, write-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteQ, - pVnode->pWriteQ->threadId); - dInfo("vgId:%d, sync-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncQ, pVnode->pSyncQ->threadId); - dInfo("vgId:%d, sync-ctrl-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncCtrlQ, - pVnode->pSyncCtrlQ->threadId); - dInfo("vgId:%d, apply-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyQ, - pVnode->pApplyQ->threadId); + dInfo("vgId:%d, write-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteW.queue, + pVnode->pWriteW.queue->threadId); + dInfo("vgId:%d, sync-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncW.queue, + pVnode->pSyncW.queue->threadId); + dInfo("vgId:%d, sync-ctrl-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncCtrlW.queue, + pVnode->pSyncCtrlW.queue->threadId); + dInfo("vgId:%d, apply-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyW.queue, + pVnode->pApplyW.queue->threadId); dInfo("vgId:%d, query-queue:%p is alloced", pVnode->vgId, pVnode->pQueryQ); dInfo("vgId:%d, fetch-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pFetchQ, pVnode->pFetchQ->threadId); @@ -335,16 +331,9 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { } void vmFreeQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { - tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ); - tWWorkerFreeQueue(&pMgmt->applyPool, pVnode->pApplyQ); - tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ); - tWWorkerFreeQueue(&pMgmt->syncCtrlPool, pVnode->pSyncCtrlQ); tQWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ); tQWorkerFreeQueue(&pMgmt->streamPool, pVnode->pStreamQ); tWWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ); - pVnode->pWriteQ = NULL; - pVnode->pSyncQ = NULL; - pVnode->pApplyQ = NULL; pVnode->pQueryQ = NULL; pVnode->pStreamQ = NULL; pVnode->pFetchQ = NULL; @@ -369,26 +358,6 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) { pFPool->max = tsNumOfVnodeFetchThreads; if (tWWorkerInit(pFPool) != 0) return -1; - SWWorkerPool *pWPool = &pMgmt->writePool; - pWPool->name = "vnode-write"; - pWPool->max = tsNumOfVnodeWriteThreads; - if (tWWorkerInit(pWPool) != 0) return -1; - - SWWorkerPool *pAPool = &pMgmt->applyPool; - pAPool->name = "vnode-apply"; - pAPool->max = tsNumOfVnodeWriteThreads; - if (tWWorkerInit(pAPool) != 0) return -1; - - SWWorkerPool *pSPool = &pMgmt->syncPool; - pSPool->name = "vnode-sync"; - pSPool->max = tsNumOfVnodeSyncThreads; - if (tWWorkerInit(pSPool) != 0) return -1; - - SWWorkerPool *pSCPool = &pMgmt->syncCtrlPool; - pSCPool->name = "vnode-sync-ctrl"; - pSCPool->max = tsNumOfVnodeSyncThreads; - if (tWWorkerInit(pSCPool) != 0) return -1; - SSingleWorkerCfg mgmtCfg = { .min = 1, .max = 1, @@ -403,10 +372,6 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) { } void vmStopWorker(SVnodeMgmt *pMgmt) { - tWWorkerCleanup(&pMgmt->writePool); - tWWorkerCleanup(&pMgmt->applyPool); - tWWorkerCleanup(&pMgmt->syncPool); - tWWorkerCleanup(&pMgmt->syncCtrlPool); tQWorkerCleanup(&pMgmt->queryPool); tQWorkerCleanup(&pMgmt->streamPool); tWWorkerCleanup(&pMgmt->fetchPool); diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index a5f7da235a..5971033bf8 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -43,7 +43,6 @@ int32_t tQWorkerInit(SQWorkerPool *pool) { void tQWorkerCleanup(SQWorkerPool *pool) { for (int32_t i = 0; i < pool->max; ++i) { SQWorker *worker = pool->workers + i; - // if (worker == NULL) continue; if (taosCheckPthreadValid(worker->thread)) { taosQsetThreadResume(pool->qset); } @@ -51,7 +50,6 @@ void tQWorkerCleanup(SQWorkerPool *pool) { for (int32_t i = 0; i < pool->max; ++i) { SQWorker *worker = pool->workers + i; - // if (worker == NULL) continue; if (taosCheckPthreadValid(worker->thread)) { taosThreadJoin(worker->thread, NULL); taosThreadClear(&worker->thread); From cf9e851c2c73071576c8599a1ad0aa458e7fb32f Mon Sep 17 00:00:00 2001 From: xleili Date: Fri, 4 Nov 2022 17:24:32 +0800 Subject: [PATCH 49/82] fix(driver):csharp example ci failed for read from null point --- docs/examples/csharp/wsConnect/Program.cs | 3 ++- docs/examples/csharp/wsInsert/Program.cs | 2 +- docs/examples/csharp/wsQuery/Program.cs | 2 +- docs/examples/csharp/wsStmt/Program.cs | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/examples/csharp/wsConnect/Program.cs b/docs/examples/csharp/wsConnect/Program.cs index 2e89372c3e..f9a56c842f 100644 --- a/docs/examples/csharp/wsConnect/Program.cs +++ b/docs/examples/csharp/wsConnect/Program.cs @@ -9,9 +9,10 @@ namespace Examples { string DSN = "ws://root:taosdata@127.0.0.1:6041/test"; IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN); + if (wsConn == IntPtr.Zero) { - throw new Exception($"get WS connection failed,reason:{LibTaosWS.WSErrorStr(IntPtr.Zero)} code:{LibTaosWS.WSErrorNo(IntPtr.Zero)}"); + throw new Exception("get WS connection failed"); } else { diff --git a/docs/examples/csharp/wsInsert/Program.cs b/docs/examples/csharp/wsInsert/Program.cs index 4cd812cda9..1f2d0a6725 100644 --- a/docs/examples/csharp/wsInsert/Program.cs +++ b/docs/examples/csharp/wsInsert/Program.cs @@ -13,7 +13,7 @@ namespace Examples // Assert if connection is validate if (wsConn == IntPtr.Zero) { - throw new Exception($"get WS connection failed,reason:{LibTaosWS.WSErrorStr(IntPtr.Zero)} code:{LibTaosWS.WSErrorNo(IntPtr.Zero)}"); + throw new Exception("get WS connection failed"); } else { diff --git a/docs/examples/csharp/wsQuery/Program.cs b/docs/examples/csharp/wsQuery/Program.cs index de5591aa53..a220cae903 100644 --- a/docs/examples/csharp/wsQuery/Program.cs +++ b/docs/examples/csharp/wsQuery/Program.cs @@ -13,7 +13,7 @@ namespace Examples IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN); if (wsConn == IntPtr.Zero) { - throw new Exception($"get WS connection failed,reason:{LibTaosWS.WSErrorStr(IntPtr.Zero)} code:{LibTaosWS.WSErrorNo(IntPtr.Zero)}"); + throw new Exception("get WS connection failed"); } else { diff --git a/docs/examples/csharp/wsStmt/Program.cs b/docs/examples/csharp/wsStmt/Program.cs index 54de77ec1f..8af807ec39 100644 --- a/docs/examples/csharp/wsStmt/Program.cs +++ b/docs/examples/csharp/wsStmt/Program.cs @@ -21,7 +21,7 @@ namespace Examples IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN); if (wsConn == IntPtr.Zero) { - throw new Exception($"get WS connection failed,reason:{LibTaosWS.WSErrorStr(IntPtr.Zero)} code:{LibTaosWS.WSErrorNo(IntPtr.Zero)}"); + throw new Exception($"get WS connection failed"); } else { From e297eda2ad529b5e897ef3c0d7f011045c7752a2 Mon Sep 17 00:00:00 2001 From: Alex Duan <51781608+DuanKuanJun@users.noreply.github.com> Date: Fri, 4 Nov 2022 17:28:03 +0800 Subject: [PATCH 50/82] Update 01-deploy.md --- docs/zh/10-deployment/01-deploy.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/10-deployment/01-deploy.md b/docs/zh/10-deployment/01-deploy.md index 304ed7153b..6a03c2f8be 100644 --- a/docs/zh/10-deployment/01-deploy.md +++ b/docs/zh/10-deployment/01-deploy.md @@ -194,13 +194,13 @@ dnodeID 是集群自动分配的，不得人工指定。它在生成时是递增 ## 常见问题 1、建立集群时使用 CREATE DNODE 增加新节点后，新节点始终显示 offline 状态？ - +```sql 1）首先要检查增加的新节点上的 taosd 服务是否已经正常启动 2）如果已经启动，再检查到新节点的网络是否通畅，可以使用 ping fqdn 验证下 3）如果前面两步都没有问题，这一步要检查新节点做为独立集群在运行了，可以使用 taos -h fqdn 连接上后，show dnodes; 命令查看，如果显示的列表与你主节点上显示的不一致，说明此节点自己单独成立了一个集群，解决的方法是停止新节点上的服务，然后清空新节点上 taos.cfg 中配置的 dataDir 目录下的所有文件，重新启动新节点服务即可解决。 - +``` From 2e6f75107d17a55ea2d0ff469449ccc508fe5ce1 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 4 Nov 2022 17:38:35 +0800 Subject: [PATCH 51/82] enh: adjust the number of vnode threads so that one vnode has one write thread --- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 2 +- source/dnode/vnode/src/vnd/vnodeSync.c | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 03c242b081..d4815e4843 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -299,7 +299,7 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { SMultiWorkerCfg wcfg = {.max = 1, .name = "vnode-write", .fp = (FItems)vnodeProposeWriteMsg, .param = pVnode->pImpl}; SMultiWorkerCfg scfg = {.max = 1, .name = "vnode-sync", .fp = (FItems)vmProcessSyncQueue, .param = pVnode}; SMultiWorkerCfg sccfg = {.max = 1, .name = "vnode-sync-ctrl", .fp = (FItems)vmProcessSyncQueue, .param = pVnode}; - SMultiWorkerCfg acfg = {.max = 1, .name = "vnode-apply", .fp = (FItems)vnodeApplyWriteMsg, .param = pVnode}; + SMultiWorkerCfg acfg = {.max = 1, .name = "vnode-apply", .fp = (FItems)vnodeApplyWriteMsg, .param = pVnode->pImpl}; (void)tMultiWorkerInit(&pVnode->pWriteW, &wcfg); (void)tMultiWorkerInit(&pVnode->pSyncW, &scfg); (void)tMultiWorkerInit(&pVnode->pSyncCtrlW, &sccfg); diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 8ce559e232..3913561ae7 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -244,6 +244,16 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } static int32_t vnodeSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { + if (pMsg == NULL || pMsg->pCont == NULL) { + return -1; + } + + if (msgcb == NULL || msgcb->putToQueueFp == NULL) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + return -1; + } + int32_t code = tmsgPutToQueue(msgcb, SYNC_CTRL_QUEUE, pMsg); if (code != 0) { rpcFreeCont(pMsg->pCont); @@ -253,6 +263,16 @@ static int32_t vnodeSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { } static int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { + if (pMsg == NULL || pMsg->pCont == NULL) { + return -1; + } + + if (msgcb == NULL || msgcb->putToQueueFp == NULL) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + return -1; + } + int32_t code = tmsgPutToQueue(msgcb, SYNC_QUEUE, pMsg); if (code != 0) { rpcFreeCont(pMsg->pCont); From 97fd1360eb8652aa4ad68f842bd2a00789214a2b Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 4 Nov 2022 17:53:51 +0800 Subject: [PATCH 52/82] test:modify case --- tests/system-test/7-tmq/tmqSubscribeStb-r3.py | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/tests/system-test/7-tmq/tmqSubscribeStb-r3.py b/tests/system-test/7-tmq/tmqSubscribeStb-r3.py index 5602876893..01f1ca5b15 100644 --- a/tests/system-test/7-tmq/tmqSubscribeStb-r3.py +++ b/tests/system-test/7-tmq/tmqSubscribeStb-r3.py @@ -16,8 +16,11 @@ from util.dnodes import TDDnodes from util.dnodes import TDDnode from util.cluster import * from util.common import * +sys.path.append("./6-cluster") sys.path.append("./7-tmq") from tmqCommon import * +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck class TDTestCase: def __init__(self): @@ -26,6 +29,7 @@ class TDTestCase: self.vgroups = 4 self.ctbNum = 1000 self.rowsPerTbl = 100 + self.dnodeNumbers = 5 def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) @@ -119,15 +123,19 @@ class TDTestCase: tdLog.info("================= restart dnode 2===========================") cluster.dnodes[1].stoptaosd() cluster.dnodes[1].starttaosd() + clusterComCheck.checkDnodes(self.dnodeNumbers) tdLog.info("================= restart dnode 3===========================") cluster.dnodes[2].stoptaosd() cluster.dnodes[2].starttaosd() + clusterComCheck.checkDnodes(self.dnodeNumbers) tdLog.info("================= restart dnode 4===========================") cluster.dnodes[3].stoptaosd() cluster.dnodes[3].starttaosd() + clusterComCheck.checkDnodes(self.dnodeNumbers) tdLog.info("================= restart dnode 5===========================") cluster.dnodes[4].stoptaosd() cluster.dnodes[4].starttaosd() + clusterComCheck.checkDnodes(self.dnodeNumbers) pThread.join() # tdLog.info("restart taosd to ensure that the data falls into the disk") @@ -193,9 +201,9 @@ class TDTestCase: tdSql.query(queryString) totalRowsFromQuery = tdSql.getRows() - tdLog.info("act consume rows: %d, act query rows: %d, expect consume rows: %d, "%(totalConsumeRows, totalRowsFromQuery, expectrowcnt)) + tdLog.info("act consume rows: %d, act query rows: %d "%(totalConsumeRows, totalRowsFromQuery)) - if totalConsumeRows != totalRowsFromQuery: + if totalConsumeRows < totalRowsFromQuery: tdLog.exit("tmq consume rows error!") # tmqCom.checkFileContent(consumerId, queryString) @@ -224,7 +232,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 100, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 15, + 'pollDelay': 30, 'showMsg': 1, 'showRow': 1, 'snapshot': 1} @@ -238,7 +246,10 @@ class TDTestCase: queryString = "select ts, c1, c2 from %s.%s where t4 == 'beijing' or t4 == 'changsha' "%(paraDict['dbName'], paraDict['stbName']) sqlString = "create topic %s as %s" %(topicFromStb1, queryString) tdLog.info("create topic sql: %s"%sqlString) - tdSql.execute(sqlString) + tdSql.execute(sqlString) + + tdSql.query(queryString) + totalRowsFromQuery = tdSql.getRows() consumerId = 0 expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] @@ -257,15 +268,19 @@ class TDTestCase: tdLog.info("================= restart dnode 2===========================") cluster.dnodes[1].stoptaosd() cluster.dnodes[1].starttaosd() + clusterComCheck.checkDnodes(self.dnodeNumbers) tdLog.info("================= restart dnode 3===========================") cluster.dnodes[2].stoptaosd() cluster.dnodes[2].starttaosd() + clusterComCheck.checkDnodes(self.dnodeNumbers) tdLog.info("================= restart dnode 4===========================") cluster.dnodes[3].stoptaosd() cluster.dnodes[3].starttaosd() + clusterComCheck.checkDnodes(self.dnodeNumbers) tdLog.info("================= restart dnode 5===========================") cluster.dnodes[4].stoptaosd() cluster.dnodes[4].starttaosd() + clusterComCheck.checkDnodes(self.dnodeNumbers) tdLog.info("start to check consume result") expectRows = 1 @@ -274,12 +289,9 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - tdSql.query(queryString) - totalRowsFromQuery = tdSql.getRows() + tdLog.info("act consume rows: %d, act query rows: %d "%(totalConsumeRows, totalRowsFromQuery)) - tdLog.info("act consume rows: %d, act query rows: %d, expect consume rows: %d, "%(totalConsumeRows, totalRowsFromQuery, expectrowcnt)) - - if totalConsumeRows != totalRowsFromQuery: + if totalConsumeRows < totalRowsFromQuery: tdLog.exit("tmq consume rows error!") # tmqCom.checkFileContent(consumerId, queryString) @@ -290,8 +302,8 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 2 end ...... ") def run(self): - self.prepareTestEnv() - self.tmqCase1() + #self.prepareTestEnv() + #self.tmqCase1() self.prepareTestEnv() self.tmqCase2() From 1ef5a41c74395cfee91bf96f4bc043586b0f8c12 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 4 Nov 2022 18:12:49 +0800 Subject: [PATCH 53/82] test: Remove the comments on fulltest.sh --- tests/system-test/fulltest.sh | 668 +++++++++++++++++----------------- 1 file changed, 334 insertions(+), 334 deletions(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index cd65b395f8..143201f85e 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -293,351 +293,351 @@ python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_query python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py -N 4 -M 1 -# python3 ./test.py -f 7-tmq/create_wrong_topic.py -# python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 -# python3 ./test.py -f 7-tmq/basic5.py -# python3 ./test.py -f 7-tmq/subscribeDb.py -# python3 ./test.py -f 7-tmq/subscribeDb0.py -# python3 ./test.py -f 7-tmq/subscribeDb1.py -# python3 ./test.py -f 7-tmq/subscribeDb2.py -# python3 ./test.py -f 7-tmq/subscribeDb3.py -# python3 ./test.py -f 7-tmq/subscribeDb4.py -# python3 ./test.py -f 7-tmq/subscribeStb.py -# python3 ./test.py -f 7-tmq/subscribeStb0.py -# python3 ./test.py -f 7-tmq/subscribeStb1.py -# python3 ./test.py -f 7-tmq/subscribeStb2.py -# python3 ./test.py -f 7-tmq/subscribeStb3.py -# python3 ./test.py -f 7-tmq/subscribeStb4.py -# python3 ./test.py -f 7-tmq/db.py -# python3 ./test.py -f 7-tmq/tmqError.py -# python3 ./test.py -f 7-tmq/schema.py -# python3 ./test.py -f 7-tmq/stbFilter.py -# python3 ./test.py -f 7-tmq/tmqCheckData.py -# python3 ./test.py -f 7-tmq/tmqCheckData1.py -# #python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 5 -# python3 ./test.py -f 7-tmq/tmqConsumerGroup.py -# python3 ./test.py -f 7-tmq/tmqShow.py -# python3 ./test.py -f 7-tmq/tmqAlterSchema.py -# python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py -# python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py -# python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg.py -# python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg.py -# python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb.py -# python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb.py -# python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py -# python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py -# python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py -# python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py -# python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py -# python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py -# python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py -# python3 ./test.py -f 7-tmq/tmqDnodeRestart.py -# python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py -# python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py -# python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py -# python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py -# python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py -# python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py -# python3 ./test.py -f 7-tmq/tmqDropStb.py -# python3 ./test.py -f 7-tmq/tmqDropStbCtb.py -# python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py -# python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py -# python3 ./test.py -f 7-tmq/tmqUdf.py -# python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py -# python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py -# python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py -# python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py -# python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py -# python3 ./test.py -f 7-tmq/tmq_taosx.py -# python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py +python3 ./test.py -f 7-tmq/create_wrong_topic.py +python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 +python3 ./test.py -f 7-tmq/basic5.py +python3 ./test.py -f 7-tmq/subscribeDb.py +python3 ./test.py -f 7-tmq/subscribeDb0.py +python3 ./test.py -f 7-tmq/subscribeDb1.py +python3 ./test.py -f 7-tmq/subscribeDb2.py +python3 ./test.py -f 7-tmq/subscribeDb3.py +python3 ./test.py -f 7-tmq/subscribeDb4.py +python3 ./test.py -f 7-tmq/subscribeStb.py +python3 ./test.py -f 7-tmq/subscribeStb0.py +python3 ./test.py -f 7-tmq/subscribeStb1.py +python3 ./test.py -f 7-tmq/subscribeStb2.py +python3 ./test.py -f 7-tmq/subscribeStb3.py +python3 ./test.py -f 7-tmq/subscribeStb4.py +python3 ./test.py -f 7-tmq/db.py +python3 ./test.py -f 7-tmq/tmqError.py +python3 ./test.py -f 7-tmq/schema.py +python3 ./test.py -f 7-tmq/stbFilter.py +python3 ./test.py -f 7-tmq/tmqCheckData.py +python3 ./test.py -f 7-tmq/tmqCheckData1.py +#python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 5 +python3 ./test.py -f 7-tmq/tmqConsumerGroup.py +python3 ./test.py -f 7-tmq/tmqShow.py +python3 ./test.py -f 7-tmq/tmqAlterSchema.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py +python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py +python3 ./test.py -f 7-tmq/tmqDnodeRestart.py +python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py +python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py +python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py +python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py +python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py +python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py +python3 ./test.py -f 7-tmq/tmqDropStb.py +python3 ./test.py -f 7-tmq/tmqDropStbCtb.py +python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py +python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py +python3 ./test.py -f 7-tmq/tmqUdf.py +python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py +python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py +python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py +python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py +python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py +python3 ./test.py -f 7-tmq/tmq_taosx.py +python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py -# python3 ./test.py -f 99-TDcase/TD-19201.py +python3 ./test.py -f 99-TDcase/TD-19201.py -# #------------querPolicy 2----------- +#------------querPolicy 2----------- -# python3 ./test.py -f 2-query/between.py -Q 2 -# python3 ./test.py -f 2-query/distinct.py -Q 2 -# python3 ./test.py -f 2-query/varchar.py -Q 2 -# python3 ./test.py -f 2-query/ltrim.py -Q 2 -# python3 ./test.py -f 2-query/rtrim.py -Q 2 -# python3 ./test.py -f 2-query/length.py -Q 2 -# python3 ./test.py -f 2-query/char_length.py -Q 2 -# python3 ./test.py -f 2-query/upper.py -Q 2 -# python3 ./test.py -f 2-query/lower.py -Q 2 -# python3 ./test.py -f 2-query/join.py -Q 2 -# python3 ./test.py -f 2-query/join2.py -Q 2 -# python3 ./test.py -f 2-query/cast.py -Q 2 -# python3 ./test.py -f 2-query/substr.py -Q 2 -# python3 ./test.py -f 2-query/union.py -Q 2 -# python3 ./test.py -f 2-query/union1.py -Q 2 -# python3 ./test.py -f 2-query/concat.py -Q 2 -# python3 ./test.py -f 2-query/concat2.py -Q 2 -# python3 ./test.py -f 2-query/concat_ws.py -Q 2 -# python3 ./test.py -f 2-query/concat_ws2.py -Q 2 -# python3 ./test.py -f 2-query/check_tsdb.py -Q 2 -# python3 ./test.py -f 2-query/spread.py -Q 2 -# python3 ./test.py -f 2-query/hyperloglog.py -Q 2 -# python3 ./test.py -f 2-query/explain.py -Q 2 -# python3 ./test.py -f 2-query/leastsquares.py -Q 2 -# python3 ./test.py -f 2-query/timezone.py -Q 2 -# python3 ./test.py -f 2-query/Now.py -Q 2 -# python3 ./test.py -f 2-query/Today.py -Q 2 -# python3 ./test.py -f 2-query/max.py -Q 2 -# python3 ./test.py -f 2-query/min.py -Q 2 -# python3 ./test.py -f 2-query/count.py -Q 2 -# python3 ./test.py -f 2-query/last.py -Q 2 -# python3 ./test.py -f 2-query/first.py -Q 2 -# python3 ./test.py -f 2-query/To_iso8601.py -Q 2 -# python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 2 -# python3 ./test.py -f 2-query/timetruncate.py -Q 2 -# python3 ./test.py -f 2-query/diff.py -Q 2 -# python3 ./test.py -f 2-query/Timediff.py -Q 2 -# python3 ./test.py -f 2-query/json_tag.py -Q 2 -# python3 ./test.py -f 2-query/top.py -Q 2 -# python3 ./test.py -f 2-query/bottom.py -Q 2 -# python3 ./test.py -f 2-query/percentile.py -Q 2 -# python3 ./test.py -f 2-query/apercentile.py -Q 2 -# python3 ./test.py -f 2-query/abs.py -Q 2 -# python3 ./test.py -f 2-query/ceil.py -Q 2 -# python3 ./test.py -f 2-query/floor.py -Q 2 -# python3 ./test.py -f 2-query/round.py -Q 2 -# python3 ./test.py -f 2-query/log.py -Q 2 -# python3 ./test.py -f 2-query/pow.py -Q 2 -# python3 ./test.py -f 2-query/sqrt.py -Q 2 -# python3 ./test.py -f 2-query/sin.py -Q 2 -# python3 ./test.py -f 2-query/cos.py -Q 2 -# python3 ./test.py -f 2-query/tan.py -Q 2 -# python3 ./test.py -f 2-query/arcsin.py -Q 2 -# python3 ./test.py -f 2-query/arccos.py -Q 2 -# python3 ./test.py -f 2-query/arctan.py -Q 2 -# python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 2 -# python3 ./test.py -f 2-query/interp.py -Q 2 +python3 ./test.py -f 2-query/between.py -Q 2 +python3 ./test.py -f 2-query/distinct.py -Q 2 +python3 ./test.py -f 2-query/varchar.py -Q 2 +python3 ./test.py -f 2-query/ltrim.py -Q 2 +python3 ./test.py -f 2-query/rtrim.py -Q 2 +python3 ./test.py -f 2-query/length.py -Q 2 +python3 ./test.py -f 2-query/char_length.py -Q 2 +python3 ./test.py -f 2-query/upper.py -Q 2 +python3 ./test.py -f 2-query/lower.py -Q 2 +python3 ./test.py -f 2-query/join.py -Q 2 +python3 ./test.py -f 2-query/join2.py -Q 2 +python3 ./test.py -f 2-query/cast.py -Q 2 +python3 ./test.py -f 2-query/substr.py -Q 2 +python3 ./test.py -f 2-query/union.py -Q 2 +python3 ./test.py -f 2-query/union1.py -Q 2 +python3 ./test.py -f 2-query/concat.py -Q 2 +python3 ./test.py -f 2-query/concat2.py -Q 2 +python3 ./test.py -f 2-query/concat_ws.py -Q 2 +python3 ./test.py -f 2-query/concat_ws2.py -Q 2 +python3 ./test.py -f 2-query/check_tsdb.py -Q 2 +python3 ./test.py -f 2-query/spread.py -Q 2 +python3 ./test.py -f 2-query/hyperloglog.py -Q 2 +python3 ./test.py -f 2-query/explain.py -Q 2 +python3 ./test.py -f 2-query/leastsquares.py -Q 2 +python3 ./test.py -f 2-query/timezone.py -Q 2 +python3 ./test.py -f 2-query/Now.py -Q 2 +python3 ./test.py -f 2-query/Today.py -Q 2 +python3 ./test.py -f 2-query/max.py -Q 2 +python3 ./test.py -f 2-query/min.py -Q 2 +python3 ./test.py -f 2-query/count.py -Q 2 +python3 ./test.py -f 2-query/last.py -Q 2 +python3 ./test.py -f 2-query/first.py -Q 2 +python3 ./test.py -f 2-query/To_iso8601.py -Q 2 +python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 2 +python3 ./test.py -f 2-query/timetruncate.py -Q 2 +python3 ./test.py -f 2-query/diff.py -Q 2 +python3 ./test.py -f 2-query/Timediff.py -Q 2 +python3 ./test.py -f 2-query/json_tag.py -Q 2 +python3 ./test.py -f 2-query/top.py -Q 2 +python3 ./test.py -f 2-query/bottom.py -Q 2 +python3 ./test.py -f 2-query/percentile.py -Q 2 +python3 ./test.py -f 2-query/apercentile.py -Q 2 +python3 ./test.py -f 2-query/abs.py -Q 2 +python3 ./test.py -f 2-query/ceil.py -Q 2 +python3 ./test.py -f 2-query/floor.py -Q 2 +python3 ./test.py -f 2-query/round.py -Q 2 +python3 ./test.py -f 2-query/log.py -Q 2 +python3 ./test.py -f 2-query/pow.py -Q 2 +python3 ./test.py -f 2-query/sqrt.py -Q 2 +python3 ./test.py -f 2-query/sin.py -Q 2 +python3 ./test.py -f 2-query/cos.py -Q 2 +python3 ./test.py -f 2-query/tan.py -Q 2 +python3 ./test.py -f 2-query/arcsin.py -Q 2 +python3 ./test.py -f 2-query/arccos.py -Q 2 +python3 ./test.py -f 2-query/arctan.py -Q 2 +python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 2 +python3 ./test.py -f 2-query/interp.py -Q 2 -# python3 ./test.py -f 2-query/nestedQuery.py -Q 2 -# python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 -# python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 -# python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 -# python3 ./test.py -f 2-query/stablity.py -Q 2 -# python3 ./test.py -f 2-query/stablity_1.py -Q 2 +python3 ./test.py -f 2-query/nestedQuery.py -Q 2 +python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 +python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 +python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 +python3 ./test.py -f 2-query/stablity.py -Q 2 +python3 ./test.py -f 2-query/stablity_1.py -Q 2 -# python3 ./test.py -f 2-query/avg.py -Q 2 -# python3 ./test.py -f 2-query/elapsed.py -Q 2 -# python3 ./test.py -f 2-query/csum.py -Q 2 -# python3 ./test.py -f 2-query/mavg.py -Q 2 -# python3 ./test.py -f 2-query/sample.py -Q 2 -# python3 ./test.py -f 2-query/function_diff.py -Q 2 -# python3 ./test.py -f 2-query/unique.py -Q 2 -# python3 ./test.py -f 2-query/stateduration.py -Q 2 -# python3 ./test.py -f 2-query/function_stateduration.py -Q 2 -# python3 ./test.py -f 2-query/statecount.py -Q 2 -# python3 ./test.py -f 2-query/tail.py -Q 2 -# python3 ./test.py -f 2-query/ttl_comment.py -Q 2 -# python3 ./test.py -f 2-query/distribute_agg_count.py -Q 2 -# python3 ./test.py -f 2-query/distribute_agg_max.py -Q 2 -# python3 ./test.py -f 2-query/distribute_agg_min.py -Q 2 -# python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 2 -# python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 2 -# python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 2 -# python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 2 -# python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 2 -# python3 ./test.py -f 2-query/twa.py -Q 2 -# python3 ./test.py -f 2-query/irate.py -Q 2 -# python3 ./test.py -f 2-query/function_null.py -Q 2 -# python3 ./test.py -f 2-query/count_partition.py -Q 2 -# python3 ./test.py -f 2-query/max_partition.py -Q 2 -# python3 ./test.py -f 2-query/last_row.py -Q 2 -# python3 ./test.py -f 2-query/tsbsQuery.py -Q 2 -# python3 ./test.py -f 2-query/sml.py -Q 2 +python3 ./test.py -f 2-query/avg.py -Q 2 +python3 ./test.py -f 2-query/elapsed.py -Q 2 +python3 ./test.py -f 2-query/csum.py -Q 2 +python3 ./test.py -f 2-query/mavg.py -Q 2 +python3 ./test.py -f 2-query/sample.py -Q 2 +python3 ./test.py -f 2-query/function_diff.py -Q 2 +python3 ./test.py -f 2-query/unique.py -Q 2 +python3 ./test.py -f 2-query/stateduration.py -Q 2 +python3 ./test.py -f 2-query/function_stateduration.py -Q 2 +python3 ./test.py -f 2-query/statecount.py -Q 2 +python3 ./test.py -f 2-query/tail.py -Q 2 +python3 ./test.py -f 2-query/ttl_comment.py -Q 2 +python3 ./test.py -f 2-query/distribute_agg_count.py -Q 2 +python3 ./test.py -f 2-query/distribute_agg_max.py -Q 2 +python3 ./test.py -f 2-query/distribute_agg_min.py -Q 2 +python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 2 +python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 2 +python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 2 +python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 2 +python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 2 +python3 ./test.py -f 2-query/twa.py -Q 2 +python3 ./test.py -f 2-query/irate.py -Q 2 +python3 ./test.py -f 2-query/function_null.py -Q 2 +python3 ./test.py -f 2-query/count_partition.py -Q 2 +python3 ./test.py -f 2-query/max_partition.py -Q 2 +python3 ./test.py -f 2-query/last_row.py -Q 2 +python3 ./test.py -f 2-query/tsbsQuery.py -Q 2 +python3 ./test.py -f 2-query/sml.py -Q 2 -# #------------querPolicy 3----------- -# python3 ./test.py -f 2-query/between.py -Q 3 -# python3 ./test.py -f 2-query/distinct.py -Q 3 -# python3 ./test.py -f 2-query/varchar.py -Q 3 -# python3 ./test.py -f 2-query/ltrim.py -Q 3 -# python3 ./test.py -f 2-query/rtrim.py -Q 3 -# python3 ./test.py -f 2-query/length.py -Q 3 -# python3 ./test.py -f 2-query/char_length.py -Q 3 -# python3 ./test.py -f 2-query/upper.py -Q 3 -# python3 ./test.py -f 2-query/lower.py -Q 3 -# python3 ./test.py -f 2-query/join.py -Q 3 -# python3 ./test.py -f 2-query/join2.py -Q 3 -# python3 ./test.py -f 2-query/cast.py -Q 3 -# python3 ./test.py -f 2-query/substr.py -Q 3 -# python3 ./test.py -f 2-query/union.py -Q 3 -# python3 ./test.py -f 2-query/union1.py -Q 3 -# python3 ./test.py -f 2-query/concat.py -Q 3 -# python3 ./test.py -f 2-query/concat2.py -Q 3 -# python3 ./test.py -f 2-query/concat_ws.py -Q 3 -# python3 ./test.py -f 2-query/concat_ws2.py -Q 3 -# python3 ./test.py -f 2-query/check_tsdb.py -Q 3 -# python3 ./test.py -f 2-query/spread.py -Q 3 -# python3 ./test.py -f 2-query/hyperloglog.py -Q 3 -# python3 ./test.py -f 2-query/explain.py -Q 3 -# python3 ./test.py -f 2-query/leastsquares.py -Q 3 -# python3 ./test.py -f 2-query/timezone.py -Q 3 -# python3 ./test.py -f 2-query/Now.py -Q 3 -# python3 ./test.py -f 2-query/Today.py -Q 3 -# python3 ./test.py -f 2-query/max.py -Q 3 -# python3 ./test.py -f 2-query/min.py -Q 3 -# python3 ./test.py -f 2-query/count.py -Q 3 -# python3 ./test.py -f 2-query/last.py -Q 3 -# python3 ./test.py -f 2-query/first.py -Q 3 -# python3 ./test.py -f 2-query/To_iso8601.py -Q 3 -# python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 3 -# python3 ./test.py -f 2-query/timetruncate.py -Q 3 -# python3 ./test.py -f 2-query/diff.py -Q 3 -# python3 ./test.py -f 2-query/Timediff.py -Q 3 -# python3 ./test.py -f 2-query/json_tag.py -Q 3 -# python3 ./test.py -f 2-query/top.py -Q 3 -# python3 ./test.py -f 2-query/bottom.py -Q 3 -# python3 ./test.py -f 2-query/percentile.py -Q 3 -# python3 ./test.py -f 2-query/apercentile.py -Q 3 -# python3 ./test.py -f 2-query/abs.py -Q 3 -# python3 ./test.py -f 2-query/ceil.py -Q 3 -# python3 ./test.py -f 2-query/floor.py -Q 3 -# python3 ./test.py -f 2-query/round.py -Q 3 -# python3 ./test.py -f 2-query/log.py -Q 3 -# python3 ./test.py -f 2-query/pow.py -Q 3 -# python3 ./test.py -f 2-query/sqrt.py -Q 3 -# python3 ./test.py -f 2-query/sin.py -Q 3 -# python3 ./test.py -f 2-query/cos.py -Q 3 -# python3 ./test.py -f 2-query/tan.py -Q 3 -# python3 ./test.py -f 2-query/arcsin.py -Q 3 -# python3 ./test.py -f 2-query/arccos.py -Q 3 -# python3 ./test.py -f 2-query/arctan.py -Q 3 -# python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 3 +#------------querPolicy 3----------- +python3 ./test.py -f 2-query/between.py -Q 3 +python3 ./test.py -f 2-query/distinct.py -Q 3 +python3 ./test.py -f 2-query/varchar.py -Q 3 +python3 ./test.py -f 2-query/ltrim.py -Q 3 +python3 ./test.py -f 2-query/rtrim.py -Q 3 +python3 ./test.py -f 2-query/length.py -Q 3 +python3 ./test.py -f 2-query/char_length.py -Q 3 +python3 ./test.py -f 2-query/upper.py -Q 3 +python3 ./test.py -f 2-query/lower.py -Q 3 +python3 ./test.py -f 2-query/join.py -Q 3 +python3 ./test.py -f 2-query/join2.py -Q 3 +python3 ./test.py -f 2-query/cast.py -Q 3 +python3 ./test.py -f 2-query/substr.py -Q 3 +python3 ./test.py -f 2-query/union.py -Q 3 +python3 ./test.py -f 2-query/union1.py -Q 3 +python3 ./test.py -f 2-query/concat.py -Q 3 +python3 ./test.py -f 2-query/concat2.py -Q 3 +python3 ./test.py -f 2-query/concat_ws.py -Q 3 +python3 ./test.py -f 2-query/concat_ws2.py -Q 3 +python3 ./test.py -f 2-query/check_tsdb.py -Q 3 +python3 ./test.py -f 2-query/spread.py -Q 3 +python3 ./test.py -f 2-query/hyperloglog.py -Q 3 +python3 ./test.py -f 2-query/explain.py -Q 3 +python3 ./test.py -f 2-query/leastsquares.py -Q 3 +python3 ./test.py -f 2-query/timezone.py -Q 3 +python3 ./test.py -f 2-query/Now.py -Q 3 +python3 ./test.py -f 2-query/Today.py -Q 3 +python3 ./test.py -f 2-query/max.py -Q 3 +python3 ./test.py -f 2-query/min.py -Q 3 +python3 ./test.py -f 2-query/count.py -Q 3 +python3 ./test.py -f 2-query/last.py -Q 3 +python3 ./test.py -f 2-query/first.py -Q 3 +python3 ./test.py -f 2-query/To_iso8601.py -Q 3 +python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 3 +python3 ./test.py -f 2-query/timetruncate.py -Q 3 +python3 ./test.py -f 2-query/diff.py -Q 3 +python3 ./test.py -f 2-query/Timediff.py -Q 3 +python3 ./test.py -f 2-query/json_tag.py -Q 3 +python3 ./test.py -f 2-query/top.py -Q 3 +python3 ./test.py -f 2-query/bottom.py -Q 3 +python3 ./test.py -f 2-query/percentile.py -Q 3 +python3 ./test.py -f 2-query/apercentile.py -Q 3 +python3 ./test.py -f 2-query/abs.py -Q 3 +python3 ./test.py -f 2-query/ceil.py -Q 3 +python3 ./test.py -f 2-query/floor.py -Q 3 +python3 ./test.py -f 2-query/round.py -Q 3 +python3 ./test.py -f 2-query/log.py -Q 3 +python3 ./test.py -f 2-query/pow.py -Q 3 +python3 ./test.py -f 2-query/sqrt.py -Q 3 +python3 ./test.py -f 2-query/sin.py -Q 3 +python3 ./test.py -f 2-query/cos.py -Q 3 +python3 ./test.py -f 2-query/tan.py -Q 3 +python3 ./test.py -f 2-query/arcsin.py -Q 3 +python3 ./test.py -f 2-query/arccos.py -Q 3 +python3 ./test.py -f 2-query/arctan.py -Q 3 +python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 3 -# python3 ./test.py -f 2-query/nestedQuery.py -Q 3 -# python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 -# python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 -# python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 -# python3 ./test.py -f 2-query/stablity.py -Q 3 -# python3 ./test.py -f 2-query/stablity_1.py -Q 3 +python3 ./test.py -f 2-query/nestedQuery.py -Q 3 +python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 +python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 +python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 +python3 ./test.py -f 2-query/stablity.py -Q 3 +python3 ./test.py -f 2-query/stablity_1.py -Q 3 -# python3 ./test.py -f 2-query/avg.py -Q 3 -# python3 ./test.py -f 2-query/elapsed.py -Q 3 -# python3 ./test.py -f 2-query/csum.py -Q 3 -# python3 ./test.py -f 2-query/mavg.py -Q 3 -# python3 ./test.py -f 2-query/sample.py -Q 3 -# python3 ./test.py -f 2-query/function_diff.py -Q 3 -# python3 ./test.py -f 2-query/unique.py -Q 3 -# python3 ./test.py -f 2-query/stateduration.py -Q 3 -# python3 ./test.py -f 2-query/function_stateduration.py -Q 3 -# python3 ./test.py -f 2-query/statecount.py -Q 3 -# python3 ./test.py -f 2-query/tail.py -Q 3 -# python3 ./test.py -f 2-query/ttl_comment.py -Q 3 -# python3 ./test.py -f 2-query/distribute_agg_count.py -Q 3 -# python3 ./test.py -f 2-query/distribute_agg_max.py -Q 3 -# python3 ./test.py -f 2-query/distribute_agg_min.py -Q 3 -# python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 3 -# python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 3 -# python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 3 -# python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 3 -# python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 3 -# python3 ./test.py -f 2-query/twa.py -Q 3 -# python3 ./test.py -f 2-query/irate.py -Q 3 -# python3 ./test.py -f 2-query/function_null.py -Q 3 -# python3 ./test.py -f 2-query/count_partition.py -Q 3 -# python3 ./test.py -f 2-query/max_partition.py -Q 3 -# python3 ./test.py -f 2-query/last_row.py -Q 3 -# python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 -# python3 ./test.py -f 2-query/sml.py -Q 3 -# python3 ./test.py -f 2-query/interp.py -Q 3 +python3 ./test.py -f 2-query/avg.py -Q 3 +python3 ./test.py -f 2-query/elapsed.py -Q 3 +python3 ./test.py -f 2-query/csum.py -Q 3 +python3 ./test.py -f 2-query/mavg.py -Q 3 +python3 ./test.py -f 2-query/sample.py -Q 3 +python3 ./test.py -f 2-query/function_diff.py -Q 3 +python3 ./test.py -f 2-query/unique.py -Q 3 +python3 ./test.py -f 2-query/stateduration.py -Q 3 +python3 ./test.py -f 2-query/function_stateduration.py -Q 3 +python3 ./test.py -f 2-query/statecount.py -Q 3 +python3 ./test.py -f 2-query/tail.py -Q 3 +python3 ./test.py -f 2-query/ttl_comment.py -Q 3 +python3 ./test.py -f 2-query/distribute_agg_count.py -Q 3 +python3 ./test.py -f 2-query/distribute_agg_max.py -Q 3 +python3 ./test.py -f 2-query/distribute_agg_min.py -Q 3 +python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 3 +python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 3 +python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 3 +python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 3 +python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 3 +python3 ./test.py -f 2-query/twa.py -Q 3 +python3 ./test.py -f 2-query/irate.py -Q 3 +python3 ./test.py -f 2-query/function_null.py -Q 3 +python3 ./test.py -f 2-query/count_partition.py -Q 3 +python3 ./test.py -f 2-query/max_partition.py -Q 3 +python3 ./test.py -f 2-query/last_row.py -Q 3 +python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 +python3 ./test.py -f 2-query/sml.py -Q 3 +python3 ./test.py -f 2-query/interp.py -Q 3 -# #------------querPolicy 4----------- +#------------querPolicy 4----------- -# python3 ./test.py -f 2-query/between.py -Q 4 -# python3 ./test.py -f 2-query/distinct.py -Q 4 -# python3 ./test.py -f 2-query/varchar.py -Q 4 -# python3 ./test.py -f 2-query/ltrim.py -Q 4 -# python3 ./test.py -f 2-query/rtrim.py -Q 4 -# python3 ./test.py -f 2-query/length.py -Q 4 -# python3 ./test.py -f 2-query/char_length.py -Q 4 -# python3 ./test.py -f 2-query/upper.py -Q 4 -# python3 ./test.py -f 2-query/lower.py -Q 4 -# python3 ./test.py -f 2-query/join.py -Q 4 -# python3 ./test.py -f 2-query/join2.py -Q 4 -# #python3 ./test.py -f 2-query/cast.py -Q 4 -# python3 ./test.py -f 2-query/substr.py -Q 4 -# python3 ./test.py -f 2-query/union.py -Q 4 -# python3 ./test.py -f 2-query/union1.py -Q 4 -# python3 ./test.py -f 2-query/concat.py -Q 4 -# python3 ./test.py -f 2-query/concat2.py -Q 4 -# python3 ./test.py -f 2-query/concat_ws.py -Q 4 -# python3 ./test.py -f 2-query/concat_ws2.py -Q 4 -# python3 ./test.py -f 2-query/check_tsdb.py -Q 4 -# python3 ./test.py -f 2-query/spread.py -Q 4 -# python3 ./test.py -f 2-query/hyperloglog.py -Q 4 -# python3 ./test.py -f 2-query/explain.py -Q 4 -# python3 ./test.py -f 2-query/leastsquares.py -Q 4 -# python3 ./test.py -f 2-query/timezone.py -Q 4 -# python3 ./test.py -f 2-query/Now.py -Q 4 -# python3 ./test.py -f 2-query/Today.py -Q 4 -# python3 ./test.py -f 2-query/max.py -Q 4 -# python3 ./test.py -f 2-query/min.py -Q 4 -# python3 ./test.py -f 2-query/count.py -Q 4 -# python3 ./test.py -f 2-query/last.py -Q 4 -# python3 ./test.py -f 2-query/first.py -Q 4 -# python3 ./test.py -f 2-query/To_iso8601.py -Q 4 -# python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 4 -# python3 ./test.py -f 2-query/timetruncate.py -Q 4 -# python3 ./test.py -f 2-query/diff.py -Q 4 -# python3 ./test.py -f 2-query/Timediff.py -Q 4 -# python3 ./test.py -f 2-query/json_tag.py -Q 4 -# python3 ./test.py -f 2-query/top.py -Q 4 -# python3 ./test.py -f 2-query/bottom.py -Q 4 -# python3 ./test.py -f 2-query/percentile.py -Q 4 -# python3 ./test.py -f 2-query/apercentile.py -Q 4 -# python3 ./test.py -f 2-query/abs.py -Q 4 -# python3 ./test.py -f 2-query/ceil.py -Q 4 -# python3 ./test.py -f 2-query/floor.py -Q 4 -# python3 ./test.py -f 2-query/round.py -Q 4 -# python3 ./test.py -f 2-query/log.py -Q 4 -# python3 ./test.py -f 2-query/pow.py -Q 4 -# python3 ./test.py -f 2-query/sqrt.py -Q 4 -# python3 ./test.py -f 2-query/sin.py -Q 4 -# python3 ./test.py -f 2-query/cos.py -Q 4 -# python3 ./test.py -f 2-query/tan.py -Q 4 -# python3 ./test.py -f 2-query/arcsin.py -Q 4 -# python3 ./test.py -f 2-query/arccos.py -Q 4 -# python3 ./test.py -f 2-query/arctan.py -Q 4 -# python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 4 +python3 ./test.py -f 2-query/between.py -Q 4 +python3 ./test.py -f 2-query/distinct.py -Q 4 +python3 ./test.py -f 2-query/varchar.py -Q 4 +python3 ./test.py -f 2-query/ltrim.py -Q 4 +python3 ./test.py -f 2-query/rtrim.py -Q 4 +python3 ./test.py -f 2-query/length.py -Q 4 +python3 ./test.py -f 2-query/char_length.py -Q 4 +python3 ./test.py -f 2-query/upper.py -Q 4 +python3 ./test.py -f 2-query/lower.py -Q 4 +python3 ./test.py -f 2-query/join.py -Q 4 +python3 ./test.py -f 2-query/join2.py -Q 4 +#python3 ./test.py -f 2-query/cast.py -Q 4 +python3 ./test.py -f 2-query/substr.py -Q 4 +python3 ./test.py -f 2-query/union.py -Q 4 +python3 ./test.py -f 2-query/union1.py -Q 4 +python3 ./test.py -f 2-query/concat.py -Q 4 +python3 ./test.py -f 2-query/concat2.py -Q 4 +python3 ./test.py -f 2-query/concat_ws.py -Q 4 +python3 ./test.py -f 2-query/concat_ws2.py -Q 4 +python3 ./test.py -f 2-query/check_tsdb.py -Q 4 +python3 ./test.py -f 2-query/spread.py -Q 4 +python3 ./test.py -f 2-query/hyperloglog.py -Q 4 +python3 ./test.py -f 2-query/explain.py -Q 4 +python3 ./test.py -f 2-query/leastsquares.py -Q 4 +python3 ./test.py -f 2-query/timezone.py -Q 4 +python3 ./test.py -f 2-query/Now.py -Q 4 +python3 ./test.py -f 2-query/Today.py -Q 4 +python3 ./test.py -f 2-query/max.py -Q 4 +python3 ./test.py -f 2-query/min.py -Q 4 +python3 ./test.py -f 2-query/count.py -Q 4 +python3 ./test.py -f 2-query/last.py -Q 4 +python3 ./test.py -f 2-query/first.py -Q 4 +python3 ./test.py -f 2-query/To_iso8601.py -Q 4 +python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 4 +python3 ./test.py -f 2-query/timetruncate.py -Q 4 +python3 ./test.py -f 2-query/diff.py -Q 4 +python3 ./test.py -f 2-query/Timediff.py -Q 4 +python3 ./test.py -f 2-query/json_tag.py -Q 4 +python3 ./test.py -f 2-query/top.py -Q 4 +python3 ./test.py -f 2-query/bottom.py -Q 4 +python3 ./test.py -f 2-query/percentile.py -Q 4 +python3 ./test.py -f 2-query/apercentile.py -Q 4 +python3 ./test.py -f 2-query/abs.py -Q 4 +python3 ./test.py -f 2-query/ceil.py -Q 4 +python3 ./test.py -f 2-query/floor.py -Q 4 +python3 ./test.py -f 2-query/round.py -Q 4 +python3 ./test.py -f 2-query/log.py -Q 4 +python3 ./test.py -f 2-query/pow.py -Q 4 +python3 ./test.py -f 2-query/sqrt.py -Q 4 +python3 ./test.py -f 2-query/sin.py -Q 4 +python3 ./test.py -f 2-query/cos.py -Q 4 +python3 ./test.py -f 2-query/tan.py -Q 4 +python3 ./test.py -f 2-query/arcsin.py -Q 4 +python3 ./test.py -f 2-query/arccos.py -Q 4 +python3 ./test.py -f 2-query/arctan.py -Q 4 +python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 4 -# python3 ./test.py -f 2-query/nestedQuery.py -Q 4 -# python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 -# python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 -# python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 -# python3 ./test.py -f 2-query/stablity.py -Q 4 -# python3 ./test.py -f 2-query/stablity_1.py -Q 4 +python3 ./test.py -f 2-query/nestedQuery.py -Q 4 +python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 +python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 +python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 +python3 ./test.py -f 2-query/stablity.py -Q 4 +python3 ./test.py -f 2-query/stablity_1.py -Q 4 -# python3 ./test.py -f 2-query/avg.py -Q 4 -# python3 ./test.py -f 2-query/elapsed.py -Q 4 -# python3 ./test.py -f 2-query/csum.py -Q 4 -# python3 ./test.py -f 2-query/mavg.py -Q 4 -# python3 ./test.py -f 2-query/sample.py -Q 4 -# python3 ./test.py -f 2-query/function_diff.py -Q 4 -# python3 ./test.py -f 2-query/unique.py -Q 4 -# python3 ./test.py -f 2-query/stateduration.py -Q 4 -# python3 ./test.py -f 2-query/function_stateduration.py -Q 4 -# python3 ./test.py -f 2-query/statecount.py -Q 4 -# python3 ./test.py -f 2-query/tail.py -Q 4 -# python3 ./test.py -f 2-query/ttl_comment.py -Q 4 -# python3 ./test.py -f 2-query/distribute_agg_count.py -Q 4 -# python3 ./test.py -f 2-query/distribute_agg_max.py -Q 4 -# python3 ./test.py -f 2-query/distribute_agg_min.py -Q 4 -# python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 4 -# python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 4 -# python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 4 -# python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 4 -# python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 4 -# python3 ./test.py -f 2-query/twa.py -Q 4 -# python3 ./test.py -f 2-query/irate.py -Q 4 -# python3 ./test.py -f 2-query/function_null.py -Q 4 -# python3 ./test.py -f 2-query/count_partition.py -Q 4 -# python3 ./test.py -f 2-query/max_partition.py -Q 4 -# python3 ./test.py -f 2-query/last_row.py -Q 4 -# python3 ./test.py -f 2-query/tsbsQuery.py -Q 4 -# python3 ./test.py -f 2-query/sml.py -Q 4 -# python3 ./test.py -f 2-query/interp.py -Q 4 +python3 ./test.py -f 2-query/avg.py -Q 4 +python3 ./test.py -f 2-query/elapsed.py -Q 4 +python3 ./test.py -f 2-query/csum.py -Q 4 +python3 ./test.py -f 2-query/mavg.py -Q 4 +python3 ./test.py -f 2-query/sample.py -Q 4 +python3 ./test.py -f 2-query/function_diff.py -Q 4 +python3 ./test.py -f 2-query/unique.py -Q 4 +python3 ./test.py -f 2-query/stateduration.py -Q 4 +python3 ./test.py -f 2-query/function_stateduration.py -Q 4 +python3 ./test.py -f 2-query/statecount.py -Q 4 +python3 ./test.py -f 2-query/tail.py -Q 4 +python3 ./test.py -f 2-query/ttl_comment.py -Q 4 +python3 ./test.py -f 2-query/distribute_agg_count.py -Q 4 +python3 ./test.py -f 2-query/distribute_agg_max.py -Q 4 +python3 ./test.py -f 2-query/distribute_agg_min.py -Q 4 +python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 4 +python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 4 +python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 4 +python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 4 +python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 4 +python3 ./test.py -f 2-query/twa.py -Q 4 +python3 ./test.py -f 2-query/irate.py -Q 4 +python3 ./test.py -f 2-query/function_null.py -Q 4 +python3 ./test.py -f 2-query/count_partition.py -Q 4 +python3 ./test.py -f 2-query/max_partition.py -Q 4 +python3 ./test.py -f 2-query/last_row.py -Q 4 +python3 ./test.py -f 2-query/tsbsQuery.py -Q 4 +python3 ./test.py -f 2-query/sml.py -Q 4 +python3 ./test.py -f 2-query/interp.py -Q 4 From 72ea354451c6b3559ed052656c16330c9b65811b Mon Sep 17 00:00:00 2001 From: Alex Duan <51781608+DuanKuanJun@users.noreply.github.com> Date: Fri, 4 Nov 2022 18:19:53 +0800 Subject: [PATCH 54/82] Update 01-deploy.md --- docs/zh/10-deployment/01-deploy.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/zh/10-deployment/01-deploy.md b/docs/zh/10-deployment/01-deploy.md index 6a03c2f8be..b68bf7b743 100644 --- a/docs/zh/10-deployment/01-deploy.md +++ b/docs/zh/10-deployment/01-deploy.md @@ -199,9 +199,7 @@ dnodeID 是集群自动分配的，不得人工指定。它在生成时是递增 2）如果已经启动，再检查到新节点的网络是否通畅，可以使用 ping fqdn 验证下 - 3）如果前面两步都没有问题，这一步要检查新节点做为独立集群在运行了，可以使用 taos -h fqdn 连接上后，show dnodes; 命令查看，如果显示的列表与你主节点上显示的不一致，说明此节点自己单独成立了一个集群，解决的方法是停止新节点上的服务，然后清空新节点上 taos.cfg 中配置的 dataDir 目录下的所有文件，重新启动新节点服务即可解决。 + 3）如果前面两步都没有问题，这一步要检查新节点做为独立集群在运行了，可以使用 taos -h fqdn 连接上后，show dnodes; 命令查看. + 如果显示的列表与你主节点上显示的不一致，说明此节点自己单独成立了一个集群，解决的方法是停止新节点上的服务，然后清空新节点上 + taos.cfg 中配置的 dataDir 目录下的所有文件，重新启动新节点服务即可解决。 ``` - - - - From 4443c7a9eb5ab784dadb1f5c74f0a19dc87f152f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 4 Nov 2022 18:50:59 +0800 Subject: [PATCH 55/82] fix: fix catalog double free issue --- source/libs/scheduler/src/schRemote.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 47de2528fa..7e2d3ccb80 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -185,12 +185,12 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa code = rsp->code; } } - SCH_UNLOCK(SCH_WRITE, &pJob->resLock); if (taosArrayGetSize((SArray*)pJob->execRes.res) <= 0) { taosArrayDestroy((SArray*)pJob->execRes.res); pJob->execRes.res = NULL; } + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); } tDecoderClear(&coder); From 5aef17ab7cf32a63e330ec1a594b0baf871be320 Mon Sep 17 00:00:00 2001 From: xleili Date: Fri, 4 Nov 2022 18:58:26 +0800 Subject: [PATCH 56/82] fix(test):C# CI websocket example failed --- tests/docs-examples-test/csharp.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/docs-examples-test/csharp.sh b/tests/docs-examples-test/csharp.sh index a1e46394ee..21c19b9b3d 100644 --- a/tests/docs-examples-test/csharp.sh +++ b/tests/docs-examples-test/csharp.sh @@ -28,10 +28,10 @@ taos -s "drop database if exists test" dotnet run --project optsJSON/optsJSON.csproj taos -s "create database if not exists test" -dotnet run --project wsConnect/wsConnect.csproj -dotnet run --project wsInsert/wsInsert.csproj -dotnet run --project wsStmt/wsStmt.csproj -dotnet run --project wsQuery/wsQuery.csproj +# dotnet run --project wsConnect/wsConnect.csproj +# dotnet run --project wsInsert/wsInsert.csproj +# dotnet run --project wsStmt/wsStmt.csproj +# dotnet run --project wsQuery/wsQuery.csproj taos -s "drop database if exists test" taos -s "drop database if exists power" \ No newline at end of file From 8a7d34eae72dd8026be23f21d13ad1b55aaaca08 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 4 Nov 2022 19:41:11 +0800 Subject: [PATCH 57/82] fix: adjust log --- source/util/src/tconfig.c | 10 +++++----- source/util/src/tlog.c | 20 ++++++++++---------- source/util/src/tref.c | 2 +- tests/script/sh/deploy.sh | 2 +- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 712ff8feba..c1fee37610 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -722,13 +722,13 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) { const char *filepath = ".env"; if (envFile != NULL && strlen(envFile) > 0) { if (!taosCheckExistFile(envFile)) { - uError("failed to load env file: %s", envFile); + uError("failed to load env file:%s", envFile); return -1; } filepath = envFile; } else { if (!taosCheckExistFile(filepath)) { - uInfo("failed to load env file: %s", filepath); + uInfo("env file:%s not load", filepath); return 0; } } @@ -929,7 +929,7 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { if (strncmp(url, "jsonFile", 8) == 0) { char *filepath = p; if (!taosCheckExistFile(filepath)) { - uError("failed to load json file: %s", filepath); + uError("failed to load json file:%s", filepath); return -1; } @@ -1056,13 +1056,13 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char *apolloUrl const char *filepath = ".env"; if (envFile != NULL && strlen(envFile) > 0) { if (!taosCheckExistFile(envFile)) { - uError("failed to load env file: %s", envFile); + uError("failed to load env file:%s", envFile); return -1; } filepath = envFile; } else { if (!taosCheckExistFile(filepath)) { - uInfo("failed to load env file: %s", filepath); + uInfo("env file:%s not load", filepath); return 0; } } diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 618b80760f..a03c04ed6e 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -81,25 +81,25 @@ int64_t tsNumOfDebugLogs = 0; int64_t tsNumOfTraceLogs = 0; // log -int32_t dDebugFlag = 135; -int32_t vDebugFlag = 135; -int32_t mDebugFlag = 135; +int32_t dDebugFlag = 131; +int32_t vDebugFlag = 131; +int32_t mDebugFlag = 131; int32_t cDebugFlag = 131; int32_t jniDebugFlag = 131; int32_t tmrDebugFlag = 131; int32_t uDebugFlag = 131; int32_t rpcDebugFlag = 131; int32_t qDebugFlag = 131; -int32_t wDebugFlag = 135; -int32_t sDebugFlag = 135; +int32_t wDebugFlag = 131; +int32_t sDebugFlag = 131; int32_t tsdbDebugFlag = 131; int32_t tdbDebugFlag = 131; -int32_t tqDebugFlag = 135; -int32_t fsDebugFlag = 135; -int32_t metaDebugFlag = 135; -int32_t udfDebugFlag = 135; +int32_t tqDebugFlag = 131; +int32_t fsDebugFlag = 131; +int32_t metaDebugFlag = 131; +int32_t udfDebugFlag = 131; int32_t smaDebugFlag = 131; -int32_t idxDebugFlag = 135; +int32_t idxDebugFlag = 131; int64_t dbgEmptyW = 0; int64_t dbgWN = 0; diff --git a/source/util/src/tref.c b/source/util/src/tref.c index aa741b909a..b4322464e2 100644 --- a/source/util/src/tref.c +++ b/source/util/src/tref.c @@ -361,7 +361,7 @@ int32_t taosListRef() { if (pSet->state == TSDB_REF_STATE_EMPTY) continue; - uInfo("rsetId:%d state:%d count::%d", i, pSet->state, pSet->count); + uInfo("rsetId:%d state:%d count:%d", i, pSet->state, pSet->count); for (int32_t j = 0; j < pSet->max; ++j) { pNode = pSet->nodeList[j]; diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index cc707b2ed0..662c4a1a6c 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -139,7 +139,7 @@ echo "fsDebugFlag 143" >> $TAOS_CFG echo "idxDebugFlag 143" >> $TAOS_CFG echo "udfDebugFlag 143" >> $TAOS_CFG echo "smaDebugFlag 143" >> $TAOS_CFG -echo "idxDebugFlag 143" >> $TAOS_CFG +echo "metaDebugFlag 143" >> $TAOS_CFG echo "numOfLogLines 20000000" >> $TAOS_CFG echo "asyncLog 0" >> $TAOS_CFG echo "locale en_US.UTF-8" >> $TAOS_CFG From b911f3576a8e2889e841aec0dd7b6194be0cd6f8 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 4 Nov 2022 20:13:56 +0800 Subject: [PATCH 58/82] test:update verisonFile every time on testpackage.sh --- packaging/testpackage.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh index 20f93ecaec..12fcb309eb 100755 --- a/packaging/testpackage.sh +++ b/packaging/testpackage.sh @@ -67,6 +67,7 @@ fi } + function wgetFile { file=$1 @@ -75,7 +76,10 @@ if [ ! -f ${file} ];then echoColor BD "wget https://www.taosdata.com/assets-download/3.0/${file}" wget https://www.taosdata.com/assets-download/3.0/${file} else - echoColor YD "${file} already exists " + echoColor YD "${file} already exists and use new file " + rm -rf ${file} + echoColor BD "wget https://www.taosdata.com/assets-download/3.0/${file}" + wget https://www.taosdata.com/assets-download/3.0/${file} fi } From 7eb80571ebbbc64514e59fd9680a7f120a3bafb5 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 4 Nov 2022 21:01:49 +0800 Subject: [PATCH 59/82] build: release ver-3.0.1.6 --- packaging/release.bat | 2 +- packaging/tools/make_install.bat | 2 +- packaging/tools/tdengine.iss | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packaging/release.bat b/packaging/release.bat index b87ae68e2b..e3037b378d 100644 --- a/packaging/release.bat +++ b/packaging/release.bat @@ -40,7 +40,7 @@ if not exist %work_dir%\debug\ver-%2-x86 ( ) cd %work_dir%\debug\ver-%2-x64 call vcvarsall.bat x64 -cmake ../../ -G "NMake Makefiles JOM" -DCMAKE_MAKE_PROGRAM=jom -DBUILD_TOOLS=true -DBUILD_HTTP=false -DBUILD_TEST=false -DVERNUMBER=%2 -DCPUTYPE=x64 +cmake ../../ -G "NMake Makefiles JOM" -DCMAKE_MAKE_PROGRAM=jom -DBUILD_TOOLS=true -DWEBSOCKET=true -DBUILD_HTTP=false -DBUILD_TEST=false -DVERNUMBER=%2 -DCPUTYPE=x64 cmake --build . rd /s /Q C:\TDengine cmake --install . diff --git a/packaging/tools/make_install.bat b/packaging/tools/make_install.bat index f777d10918..4e21ddac74 100644 --- a/packaging/tools/make_install.bat +++ b/packaging/tools/make_install.bat @@ -34,7 +34,6 @@ if exist %binary_dir%\\test\\cfg\\taosadapter.toml ( copy %binary_dir%\\test\\cfg\\taosadapter.toml %tagert_dir%\\cfg\\taosadapter.toml > nul ) ) - copy %source_dir%\\include\\client\\taos.h %tagert_dir%\\include > nul copy %source_dir%\\include\\util\\taoserror.h %tagert_dir%\\include > nul copy %source_dir%\\include\\libs\\function\\taosudf.h %tagert_dir%\\include > nul @@ -52,6 +51,7 @@ if exist %binary_dir%\\build\\lib\\taosws.dll.lib ( ) if exist %binary_dir%\\build\\lib\\taosws.dll ( copy %binary_dir%\\build\\lib\\taosws.dll %tagert_dir%\\driver > nul + copy %source_dir%\\tools\\taosws-rs\\target\\release\\taosws.h %tagert_dir%\\include > nul ) if exist %binary_dir%\\build\\bin\\taosdump.exe ( copy %binary_dir%\\build\\bin\\taosdump.exe %tagert_dir% > nul diff --git a/packaging/tools/tdengine.iss b/packaging/tools/tdengine.iss index 272a0dfb5c..ec9c432092 100644 --- a/packaging/tools/tdengine.iss +++ b/packaging/tools/tdengine.iss @@ -15,10 +15,10 @@ #define MyAppExeName "\*.exe" #define MyAppTaosExeName "\taos.bat" #define MyAppTaosdemoExeName "\taosBenchmark.exe" -#define MyAppDLLName "\driver\taos.dll" +#define MyAppDLLName "\driver\*.dll" ;#define MyAppVersion "3.0" ;#define MyAppInstallName "TDengine" - +;#define MyAppInstallName "TDengine" [Setup] VersionInfoVersion={#MyAppVersion} AppId={{A0F7A93C-79C4-485D-B2B8-F0D03DF42FAB} From ddb815ac420bc1bfbebbacc4e3611808bf3db39c Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 5 Nov 2022 10:54:59 +0800 Subject: [PATCH 60/82] refactor(sync): optimize advance commit index when one replica --- source/libs/sync/inc/syncCommit.h | 1 + source/libs/sync/src/syncCommit.c | 50 +++++++++++++++++++++++++++++++ source/libs/sync/src/syncMain.c | 6 +++- 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/inc/syncCommit.h b/source/libs/sync/inc/syncCommit.h index c76236d5bf..7458ce28ab 100644 --- a/source/libs/sync/inc/syncCommit.h +++ b/source/libs/sync/inc/syncCommit.h @@ -49,6 +49,7 @@ extern "C" { // IN commitIndex' = [commitIndex EXCEPT ![i] = newCommitIndex] // /\ UNCHANGED <> // +void syncOneReplicaAdvance(SSyncNode* pSyncNode); void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode); bool syncAgreeIndex(SSyncNode* pSyncNode, SRaftId* pRaftId, SyncIndex index); bool syncAgree(SSyncNode* pSyncNode, SyncIndex index); diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 3aeb2d30b5..95787bbe6c 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -44,6 +44,56 @@ // IN commitIndex' = [commitIndex EXCEPT ![i] = newCommitIndex] // /\ UNCHANGED <> // +void syncOneReplicaAdvance(SSyncNode* pSyncNode) { + if (pSyncNode == NULL) { + sError("pSyncNode is NULL"); + return; + } + + if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { + syncNodeErrorLog(pSyncNode, "not leader, can not advance commit index"); + return; + } + + if (pSyncNode->replicaNum != 1) { + syncNodeErrorLog(pSyncNode, "not one replica, can not advance commit index"); + return; + } + + // advance commit index to snapshot first + SSnapshot snapshot; + pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); + if (snapshot.lastApplyIndex > 0 && snapshot.lastApplyIndex > pSyncNode->commitIndex) { + SyncIndex commitBegin = pSyncNode->commitIndex; + SyncIndex commitEnd = snapshot.lastApplyIndex; + pSyncNode->commitIndex = snapshot.lastApplyIndex; + + char eventLog[128]; + snprintf(eventLog, sizeof(eventLog), "commit by snapshot from index:%" PRId64 " to index:%" PRId64, commitBegin, + commitEnd); + syncNodeEventLog(pSyncNode, eventLog); + } + + // advance commit index as large as possible + SyncIndex lastIndex = syncNodeGetLastIndex(pSyncNode); + if (lastIndex > pSyncNode->commitIndex) { + do { + char eventLog[128]; + snprintf(eventLog, sizeof(eventLog), "commit by wal from index:%" PRId64 " to index:%" PRId64, + pSyncNode->commitIndex + 1, lastIndex); + syncNodeEventLog(pSyncNode, eventLog); + } while (0); + + pSyncNode->commitIndex = lastIndex; + } + + // call back Wal + SyncIndex walCommitVer = logStoreWalCommitVer(pSyncNode->pLogStore); + if (pSyncNode->commitIndex > walCommitVer) { + pSyncNode->pLogStore->syncLogUpdateCommitIndex(pSyncNode->pLogStore, pSyncNode->commitIndex); + } +} + void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { if (pSyncNode == NULL) { sError("pSyncNode is NULL"); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 27f4eeedb2..47b8614717 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -3010,7 +3010,11 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SyncClientRequest* pMsg, SyncInd // if only myself, maybe commit right now if (ths->replicaNum == 1) { - syncMaybeAdvanceCommitIndex(ths); + if (syncNodeIsMnode(ths)) { + syncMaybeAdvanceCommitIndex(ths); + } else { + syncOneReplicaAdvance(ths); + } } } From a61e821a21eb90c5ecced9f54c0e4729e4a9c821 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 5 Nov 2022 13:39:43 +0800 Subject: [PATCH 61/82] test: comment out unstable case --- tests/script/jenkins/basic.txt | 2 +- tests/system-test/fulltest.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 3b00c6bf24..e6fdecf35e 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -231,7 +231,7 @@ # ---- stream ./test.sh -f tsim/stream/basic0.sim -g -./test.sh -f tsim/stream/basic1.sim +# TD-20201 ./test.sh -f tsim/stream/basic1.sim ./test.sh -f tsim/stream/basic2.sim ./test.sh -f tsim/stream/drop_stream.sim ./test.sh -f tsim/stream/fillHistoryBasic1.sim diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 143201f85e..03ac0e0fb6 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -209,7 +209,7 @@ python3 ./test.py -f 2-query/varchar.py -R python3 ./test.py -f 1-insert/update_data.py python3 ./test.py -f 1-insert/tb_100w_data_order.py -python3 ./test.py -f 1-insert/delete_data.py +# TD-20200 python3 ./test.py -f 1-insert/delete_data.py python3 ./test.py -f 1-insert/keep_expired.py python3 ./test.py -f 2-query/join2.py @@ -239,7 +239,7 @@ python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 -i False python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 -n 3 +# TD-20198 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 -n 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 -n 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 From a83c69b8cf3095f607d2ddb0ae2bfa188c676019 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 5 Nov 2022 18:00:46 +0800 Subject: [PATCH 62/82] fix: taosbenchmark query interval unit (#17847) * feat: taosbenchmark kill slow query * fix: taosbenchmark query interval unit * test: disable restful query test due to ci util dnodes.py lack taosadapter support * test: test restful query too * fix: taosbenchmark query interval unit * fix: taosbenchmark query interval --- cmake/taostools_CMakeLists.txt.in | 2 +- .../taosbenchmark/json/queryInsertdata.json | 74 +++ .../json/queryInsertrestdata.json | 73 +++ .../taosbenchmark/json/queryQps.json | 35 ++ .../taosbenchmark/json/queryRestful.json | 38 ++ .../json/querySpeciMutisql100.json | 429 ++++++++++++++++++ .../json/querySuperMutisql100.json | 419 +++++++++++++++++ .../taosbenchmark/json/queryTaosc.json | 37 ++ .../taosdemoTestQueryWithJson.py | 241 ++++++++++ tests/develop-test/fulltest.sh | 1 + tests/develop-test/test.py | 279 +++++++----- 11 files changed, 1514 insertions(+), 114 deletions(-) create mode 100644 tests/develop-test/5-taos-tools/taosbenchmark/json/queryInsertdata.json create mode 100644 tests/develop-test/5-taos-tools/taosbenchmark/json/queryInsertrestdata.json create mode 100644 tests/develop-test/5-taos-tools/taosbenchmark/json/queryQps.json create mode 100644 tests/develop-test/5-taos-tools/taosbenchmark/json/queryRestful.json create mode 100644 tests/develop-test/5-taos-tools/taosbenchmark/json/querySpeciMutisql100.json create mode 100644 tests/develop-test/5-taos-tools/taosbenchmark/json/querySuperMutisql100.json create mode 100644 tests/develop-test/5-taos-tools/taosbenchmark/json/queryTaosc.json create mode 100644 tests/develop-test/5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 83b7ade407..e0d5250d84 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 16eb34f + GIT_TAG 0fb640b SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/queryInsertdata.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/queryInsertdata.json new file mode 100644 index 0000000000..7e3ffe1697 --- /dev/null +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/queryInsertdata.json @@ -0,0 +1,74 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 0, + "num_of_records_per_req": 3000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "precision": "ms" + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 10, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_limit": 0, + "childtable_offset": 0, + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-11-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "BINARY", "len": 1, "count":1}, {"type": "BINARY", "len": 3, "count":1}, {"type": "INT"}, {"type": "DOUBLE", "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 10, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 200, + "childtable_limit": 0, + "childtable_offset": 0, + "interlace_rows": 0 , + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-11-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 1, "count":3}, {"type": "BINARY", "len": 2, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/queryInsertrestdata.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/queryInsertrestdata.json new file mode 100644 index 0000000000..6714497766 --- /dev/null +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/queryInsertrestdata.json @@ -0,0 +1,73 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 0, + "num_of_records_per_req": 3000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "precision": "ms" + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 2, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 10, + "childtable_limit": 0, + "childtable_offset": 0, + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-11-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "BINARY", "len": 1, "count":1}, {"type": "BINARY", "len": 3, "count":1}, {"type": "INT"}, {"type": "DOUBLE", "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 2, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 5, + "childtable_limit": 0, + "childtable_offset": 0, + "interlace_rows": 0 , + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-11-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 1, "count":3}, {"type": "BINARY", "len": 2, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/queryQps.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/queryQps.json new file mode 100644 index 0000000000..9e75d52a6c --- /dev/null +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/queryQps.json @@ -0,0 +1,35 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "db", + "query_times": 1, + "specified_table_query": { + "query_interval": 10, + "threads": 4, + "sqls": [ + { + "sql": "select last_row(*) from stb00_0", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_9 ", + "result": "./query_res1.txt" + }] + }, + "super_table_query": { + "stblname": "stb1", + "query_interval":20, + "threads": 4, + "sqls": [ + { + "sql": "select last_row(ts) from xxxx", + "result": "./query_res2.txt" + } + ] + } +} diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/queryRestful.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/queryRestful.json new file mode 100644 index 0000000000..5de560fd21 --- /dev/null +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/queryRestful.json @@ -0,0 +1,38 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "db", + "query_times": 2, + "query_mode": "rest", + "specified_table_query": { + "query_interval": 1, + "threads": 3, + "sqls": [ + { + "sql": "select last_row(*) from db.stb0 ", + "result": "./query_res0.txt" + }, + { + "sql": "select count(*) from db.stb00_1", + "result": "./query_res1.txt" + } + ] + }, + "super_table_query": { + "stblname": "stb1", + "query_interval": 1, + "threads": 3, + "sqls": [ + { + "sql": "select last_row(ts) from xxxx", + "result": "./query_res2.txt" + } + ] + } + } + diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/querySpeciMutisql100.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/querySpeciMutisql100.json new file mode 100644 index 0000000000..a86d22d69d --- /dev/null +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/querySpeciMutisql100.json @@ -0,0 +1,429 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "db", + "query_times": 2, + "specified_table_query": { + "query_interval": 1, + "threads": 3, + "sqls": [ + { + "sql": "select last_row(*) from stb00_0", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_1", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_2", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_3", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_4", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_5", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_6", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_7", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_8", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_9", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_10 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_11 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_12 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_13 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_14 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_15 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_16 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_17 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_18 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_19 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_20 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_21 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_22 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_23 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_24 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_25 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_26 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_27 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_28 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_29 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_30 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_31 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_32 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_33 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_34 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_35 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_36 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_37 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_38 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_39 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_40 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_41 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_42 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_43 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_44 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_45 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_46 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_47 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_48 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_49 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_50 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_51 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_52 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_53 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_54 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_55 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_56 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_57 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_58 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_59 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_60", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_61", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_62", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_63", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_64", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_65", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_66", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_67", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_68", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_69", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_70 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_71 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_72 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_73 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_74 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_75 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_76 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_77 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_78 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_79 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_80 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_81 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_82 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_83 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_84 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_85 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_86 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_87 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_88 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_89 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_90 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_91 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_92 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_93 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_94 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_95 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_96 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_97 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_98 ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from stb00_99 ", + "result": "./query_res0.txt" + + }] + }, + "super_table_query": { + "stblname": "stb1", + "query_interval": 1, + "threads": 3, + "sqls": [ + { + "sql": "select last_row(ts) from xxxx", + "result": "./query_res2.txt" + } + ] + } +} + \ No newline at end of file diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/querySuperMutisql100.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/querySuperMutisql100.json new file mode 100644 index 0000000000..0f21df47e6 --- /dev/null +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/querySuperMutisql100.json @@ -0,0 +1,419 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "db", + "query_times": 3, + "super_table_query": { + "stblname": "stb0", + "query_interval": 10, + "threads": 9, + "sqls": [ + { + "sql": "select last_row(*) from xxxx", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select last_row(*) from xxxx ", + "result": "./query_res0.txt" + }, + { + "sql": "select * from xxxx ", + "result": "./query_res0.txt" + + }] + } +} + diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/queryTaosc.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/queryTaosc.json new file mode 100644 index 0000000000..9ce4237660 --- /dev/null +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/queryTaosc.json @@ -0,0 +1,37 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "db", + "query_times": 2, + "query_mode": "taosc", + "specified_table_query": { + "query_interval": 1, + "threads": 3, + "sqls": [ + { + "sql": "select last_row(*) from stb0 ", + "result": "./query_res0.txt" + }, + { + "sql": "select count(*) from stb00_1", + "result": "./query_res1.txt" + } + ] + }, + "super_table_query": { + "stblname": "stb1", + "query_interval": 1, + "threads": 3, + "sqls": [ + { + "sql": "select last_row(ts) from xxxx", + "result": "./query_res2.txt" + } + ] + } +} diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py b/tests/develop-test/5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py new file mode 100644 index 0000000000..19500c7dca --- /dev/null +++ b/tests/develop-test/5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py @@ -0,0 +1,241 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import time +from datetime import datetime +import ast +import re + +# from assertpy import assert_that +import subprocess + + +class TDTestCase: + def init(self, conn, logSql, replicaVarl=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getPath(self, tool="taosBenchmark"): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if "community" in selfPath: + projPath = selfPath[: selfPath.find("community")] + elif "src" in selfPath: + projPath = selfPath[: selfPath.find("src")] + elif "/tools/" in selfPath: + projPath = selfPath[: selfPath.find("/tools/")] + elif "/tests/" in selfPath: + projPath = selfPath[: selfPath.find("/tests/")] + else: + tdLog.info("cannot found %s in path: %s, use system's" % (tool, selfPath)) + projPath = "/usr/local/taos/bin/" + + paths = [] + for root, dirs, files in os.walk(projPath): + if (tool) in files: + rootRealPath = os.path.dirname(os.path.realpath(root)) + if "packaging" not in rootRealPath: + paths.append(os.path.join(root, tool)) + break + if len(paths) == 0: + return "" + return paths[0] + + # 获取taosc接口查询的结果文件中的内容,返回每行数据,并断言数据的第一列内容。 + def assertfileDataTaosc(self, filename, expectResult): + self.filename = filename + self.expectResult = expectResult + with open("%s" % filename, "r+") as f1: + for line in f1.readlines(): + queryResultTaosc = line.strip().split()[0] + self.assertCheck(filename, queryResultTaosc, expectResult) + + # 获取restful接口查询的结果文件中的关键内容,目前的关键内容找到第一个key就跳出循，所以就只有一个数据。后续再修改多个结果文件。 + def getfileDataRestful(self, filename): + self.filename = filename + with open("%s" % filename, "r+") as f1: + for line in f1.readlines(): + contents = line.strip() + if contents.find("data") != -1: + pattern = re.compile("{.*}") + contents = pattern.search(contents).group() + contentsDict = ast.literal_eval(contents) # 字符串转换为字典 + queryResultRest = contentsDict["data"][0][0] + break + else: + queryResultRest = "" + return queryResultRest + + # 获取taosc接口查询次数 + def queryTimesTaosc(self, filename): + self.filename = filename + command = "cat %s |wc -l" % filename + times = int(subprocess.getstatusoutput(command)[1]) + return times + + # 获取restful接口查询次数 + def queryTimesRestful(self, filename): + self.filename = filename + command = 'cat %s |grep "200 OK" |wc -l' % filename + times = int(subprocess.getstatusoutput(command)[1]) + return times + + # 定义断言结果是否正确。不正确返回错误结果，正确即通过。 + def assertCheck(self, filename, queryResult, expectResult): + self.filename = filename + self.queryResult = queryResult + self.expectResult = expectResult + args0 = (filename, queryResult, expectResult) + assert queryResult == expectResult, ( + "Queryfile:%s ,result is %s != expect: %s" % args0 + ) + + def run(self): + binPath = self.getPath() + if binPath == "": + tdLog.exit("taosBenchmark not found!") + else: + tdLog.info("taosBenchmark use %s" % binPath) + + # delete useless files + os.system("rm -rf ./query_res*") + os.system("rm -rf ./all_query*") + + # taosc query: query specified table and query super table + os.system("%s -f ./5-taos-tools/taosbenchmark/json/queryInsertdata.json" % binPath) + os.system("%s -f ./5-taos-tools/taosbenchmark/json/queryTaosc.json" % binPath) + os.system("cat query_res0.txt* > all_query_res0_taosc.txt") + os.system("cat query_res1.txt* > all_query_res1_taosc.txt") + os.system("cat query_res2.txt* > all_query_res2_taosc.txt") + + # correct Times testcases + queryTimes0Taosc = self.queryTimesTaosc("all_query_res0_taosc.txt") + self.assertCheck("all_query_res0_taosc.txt", queryTimes0Taosc, 6) + + queryTimes1Taosc = self.queryTimesTaosc("all_query_res1_taosc.txt") + self.assertCheck("all_query_res1_taosc.txt", queryTimes1Taosc, 6) + + queryTimes2Taosc = self.queryTimesTaosc("all_query_res2_taosc.txt") + self.assertCheck("all_query_res2_taosc.txt", queryTimes2Taosc, 20) + + # correct data testcase + self.assertfileDataTaosc("all_query_res0_taosc.txt", "1604160000099") + self.assertfileDataTaosc("all_query_res1_taosc.txt", "100") + self.assertfileDataTaosc("all_query_res2_taosc.txt", "1604160000199") + + # delete useless files + os.system("rm -rf ./query_res*") + os.system("rm -rf ./all_query*") + + # use restful api to query + os.system("%s -f ./5-taos-tools/taosbenchmark/json/queryInsertrestdata.json" % binPath) + os.system("%s -f ./5-taos-tools/taosbenchmark/json/queryRestful.json" % binPath) + os.system("cat query_res0.txt* > all_query_res0_rest.txt") + os.system("cat query_res1.txt* > all_query_res1_rest.txt") + os.system("cat query_res2.txt* > all_query_res2_rest.txt") + + # correct Times testcases + queryTimes0Restful = self.queryTimesRestful("all_query_res0_rest.txt") + self.assertCheck("all_query_res0_rest.txt", queryTimes0Restful, 6) + + queryTimes1Restful = self.queryTimesRestful("all_query_res1_rest.txt") + self.assertCheck("all_query_res1_rest.txt", queryTimes1Restful, 6) + + queryTimes2Restful = self.queryTimesRestful("all_query_res2_rest.txt") + self.assertCheck("all_query_res2_rest.txt", queryTimes2Restful, 4) + + # correct data testcase + data0 = self.getfileDataRestful("all_query_res0_rest.txt") + if data0 != "2020-11-01 00:00:00.009" and data0 != "2020-10-31T16:00:00.009Z": + tdLog.exit( + "data0 is not 2020-11-01 00:00:00.009 and 2020-10-31T16:00:00.009Z" + ) + + data1 = self.getfileDataRestful("all_query_res1_rest.txt") + self.assertCheck("all_query_res1_rest.txt", data1, 10) + + data2 = self.getfileDataRestful("all_query_res2_rest.txt") + if data2 != "2020-11-01 00:00:00.004" and data2 != "2020-10-31T16:00:00.004Z": + tdLog.exit( + "data2 is not 2020-11-01 00:00:00.004 and 2020-10-31T16:00:00.004Z" + ) + + # query times less than or equal to 100 + assert ( + os.system("%s -f ./5-taos-tools/taosbenchmark/json/queryInsertdata.json" % binPath) == 0 + ) + assert ( + os.system("%s -f ./5-taos-tools/taosbenchmark/json/querySpeciMutisql100.json" % binPath) + != 0 + ) + assert ( + os.system("%s -f ./5-taos-tools/taosbenchmark/json/querySuperMutisql100.json" % binPath) + == 0 + ) + + # query result print QPS + os.system("%s -f ./5-taos-tools/taosbenchmark/json/queryInsertdata.json" % binPath) + exceptcode = os.system("%s -f ./5-taos-tools/taosbenchmark/json/queryQps.json" % binPath) + assert exceptcode == 0 + + # 2021.02.09 need modify taosBenchmakr code + # use illegal or out of range parameters query json file + os.system("%s -f ./5-taos-tools/taosbenchmark/json/queryInsertdata.json" % binPath) + # 2021.02.09 need modify taosBenchmakr code + # exceptcode = os.system( + # "%s -f ./taosbenchmark/json/queryTimes0.json" % + # binPath) + # assert exceptcode != 0 + + # 2021.02.09 need modify taosBenchmakr code + # exceptcode0 = os.system( + # "%s -f ./taosbenchmark/json/queryTimesless0.json" % + # binPath) + # assert exceptcode0 != 0 + + # exceptcode1 = os.system( + # "%s -f ./taosbenchmark/json/queryConcurrent0.json" % + # binPath) + # assert exceptcode2 != 0 + + # exceptcode3 = os.system( + # "%s -f ./taosbenchmark/json/querrThreadsless0.json" % + # binPath) + # assert exceptcode3 != 0 + + # exceptcode4 = os.system( + # "%s -f ./taosbenchmark/json/querrThreads0.json" % + # binPath) + # assert exceptcode4 != 0 + + # delete useless files + os.system("rm -rf ./insert_res.txt") + os.system("rm -rf 5-taos-tools/taosbenchmark/*.py.sql") + os.system("rm -rf ./querySystemInfo*") + os.system("rm -rf ./query_res*") + os.system("rm -rf ./all_query*") + os.system("rm -rf ./test_query_res0.txt") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/develop-test/fulltest.sh b/tests/develop-test/fulltest.sh index 69cade3855..e986ed6966 100644 --- a/tests/develop-test/fulltest.sh +++ b/tests/develop-test/fulltest.sh @@ -18,3 +18,4 @@ python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py #python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_telnet_alltypes.py #python3 ./test.py -f 5-taos-tools/taosbenchmark/taosadapter_json.py #python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py +python3 ./test.py -f 5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py -R diff --git a/tests/develop-test/test.py b/tests/develop-test/test.py index b25bda4a3b..062e48b94b 100644 --- a/tests/develop-test/test.py +++ b/tests/develop-test/test.py @@ -24,6 +24,7 @@ import socket import threading import toml + sys.path.append("../pytest") from util.log import * from util.dnodes import * @@ -34,14 +35,16 @@ from util.taosadapter import * import taos import taosrest + def checkRunTimeError(): import win32gui + timeCount = 0 while 1: time.sleep(1) timeCount = timeCount + 1 - print("checkRunTimeError",timeCount) - if (timeCount>600): + print("checkRunTimeError", timeCount) + if timeCount > 600: print("stop the test.") os.system("TASKKILL /F /IM taosd.exe") os.system("TASKKILL /F /IM taos.exe") @@ -53,6 +56,7 @@ def checkRunTimeError(): if hwnd: os.system("TASKKILL /F /IM taosd.exe") + if __name__ == "__main__": fileName = "all" @@ -73,102 +77,124 @@ if __name__ == "__main__": createDnodeNums = 1 restful = False replicaVar = 1 - opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RD:n:', [ - 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','adaptercfgupdate','replicaVar']) + opts, args = getopt.gnu_getopt( + sys.argv[1:], + "f:p:m:l:scghrd:k:e:N:M:Q:C:RD:n:", + [ + "file=", + "path=", + "master", + "logSql", + "stop", + "cluster", + "valgrind", + "help", + "restart", + "updateCfgDict", + "killv", + "execCmd", + "dnodeNums", + "mnodeNums", + "queryPolicy", + "createDnodeNums", + "restful", + "adaptercfgupdate", + "replicaVar", + ], + ) for key, value in opts: - if key in ['-h', '--help']: - tdLog.printNoPrefix( - 'A collection of test cases written using Python') - tdLog.printNoPrefix('-f Name of test case file written by Python') - tdLog.printNoPrefix('-p Deploy Path for Simulator') - tdLog.printNoPrefix('-m Master Ip for Simulator') - tdLog.printNoPrefix('-l logSql Flag') - tdLog.printNoPrefix('-s stop All dnodes') - tdLog.printNoPrefix('-c Test Cluster Flag') - tdLog.printNoPrefix('-g valgrind Test Flag') - tdLog.printNoPrefix('-r taosd restart test') - tdLog.printNoPrefix('-d update cfg dict, base64 json str') - tdLog.printNoPrefix('-k not kill valgrind processer') - tdLog.printNoPrefix('-e eval str to run') - tdLog.printNoPrefix('-N start dnodes numbers in clusters') - tdLog.printNoPrefix('-M create mnode numbers in clusters') - tdLog.printNoPrefix('-Q set queryPolicy in one dnode') - tdLog.printNoPrefix('-C create Dnode Numbers in one cluster') - tdLog.printNoPrefix('-R restful realization form') - tdLog.printNoPrefix('-D taosadapter update cfg dict ') - tdLog.printNoPrefix('-n the number of replicas') + if key in ["-h", "--help"]: + tdLog.printNoPrefix("A collection of test cases written using Python") + tdLog.printNoPrefix("-f Name of test case file written by Python") + tdLog.printNoPrefix("-p Deploy Path for Simulator") + tdLog.printNoPrefix("-m Master Ip for Simulator") + tdLog.printNoPrefix("-l logSql Flag") + tdLog.printNoPrefix("-s stop All dnodes") + tdLog.printNoPrefix("-c Test Cluster Flag") + tdLog.printNoPrefix("-g valgrind Test Flag") + tdLog.printNoPrefix("-r taosd restart test") + tdLog.printNoPrefix("-d update cfg dict, base64 json str") + tdLog.printNoPrefix("-k not kill valgrind processer") + tdLog.printNoPrefix("-e eval str to run") + tdLog.printNoPrefix("-N start dnodes numbers in clusters") + tdLog.printNoPrefix("-M create mnode numbers in clusters") + tdLog.printNoPrefix("-Q set queryPolicy in one dnode") + tdLog.printNoPrefix("-C create Dnode Numbers in one cluster") + tdLog.printNoPrefix("-R restful realization form") + tdLog.printNoPrefix("-D taosadapter update cfg dict ") + tdLog.printNoPrefix("-n the number of replicas") sys.exit(0) - if key in ['-r', '--restart']: + if key in ["-r", "--restart"]: restart = True - if key in ['-f', '--file']: + if key in ["-f", "--file"]: fileName = value - if key in ['-p', '--path']: + if key in ["-p", "--path"]: deployPath = value - if key in ['-m', '--master']: + if key in ["-m", "--master"]: masterIp = value - if key in ['-l', '--logSql']: - if (value.upper() == "TRUE"): + if key in ["-l", "--logSql"]: + if value.upper() == "TRUE": logSql = True - elif (value.upper() == "FALSE"): + elif value.upper() == "FALSE": logSql = False else: tdLog.printNoPrefix("logSql value %s is invalid" % logSql) sys.exit(0) - if key in ['-c', '--cluster']: + if key in ["-c", "--cluster"]: testCluster = True - if key in ['-g', '--valgrind']: + if key in ["-g", "--valgrind"]: valgrind = 1 - if key in ['-s', '--stop']: + if key in ["-s", "--stop"]: stop = 1 - if key in ['-d', '--updateCfgDict']: + if key in ["-d", "--updateCfgDict"]: try: updateCfgDict = eval(base64.b64decode(value.encode()).decode()) except: - print('updateCfgDict convert fail.') + print("updateCfgDict convert fail.") sys.exit(0) - if key in ['-k', '--killValgrind']: + if key in ["-k", "--killValgrind"]: killValgrind = 0 - if key in ['-e', '--execCmd']: + if key in ["-e", "--execCmd"]: try: execCmd = base64.b64decode(value.encode()).decode() except: - print('execCmd run fail.') + print("execCmd run fail.") sys.exit(0) - if key in ['-N', '--dnodeNums']: + if key in ["-N", "--dnodeNums"]: dnodeNums = value - if key in ['-M', '--mnodeNums']: + if key in ["-M", "--mnodeNums"]: mnodeNums = value - if key in ['-Q', '--queryPolicy']: + if key in ["-Q", "--queryPolicy"]: queryPolicy = value - if key in ['-C', '--createDnodeNums']: + if key in ["-C", "--createDnodeNums"]: createDnodeNums = value - if key in ['-R', '--restful']: + if key in ["-R", "--restful"]: restful = True - if key in ['-D', '--adaptercfgupdate']: + if key in ["-D", "--adaptercfgupdate"]: try: adaptercfgupdate = eval(base64.b64decode(value.encode()).decode()) except: - print('adapter cfg update convert fail.') + print("adapter cfg update convert fail.") sys.exit(0) - if key in ['-n', '--replicaVar']: + if key in ["-n", "--replicaVar"]: replicaVar = value if not execCmd == "": @@ -180,18 +206,21 @@ if __name__ == "__main__": exec(execCmd) quit() - if (stop != 0): - if (valgrind == 0): + if stop != 0: + if valgrind == 0: toBeKilled = "taosd" else: toBeKilled = "valgrind.bin" - killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -TERM > /dev/null 2>&1" % toBeKilled + killCmd = ( + "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -TERM > /dev/null 2>&1" + % toBeKilled + ) psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled processID = subprocess.check_output(psCmd, shell=True) - while(processID): + while processID: os.system(killCmd) time.sleep(1) processID = subprocess.check_output(psCmd, shell=True) @@ -218,7 +247,7 @@ if __name__ == "__main__": # psCmd = f"pgrep {toBeKilled}" processID = subprocess.check_output(psCmd, shell=True) - while(processID): + while processID: os.system(killCmd) time.sleep(1) processID = subprocess.check_output(psCmd, shell=True) @@ -233,9 +262,9 @@ if __name__ == "__main__": fuserCmd = f"fuser -k -n tcp {port}" os.system(fuserCmd) - tdLog.info('stop taosadapter') + tdLog.info("stop taosadapter") - tdLog.info('stop All dnodes') + tdLog.info("stop All dnodes") if masterIp == "": host = socket.gethostname() @@ -247,33 +276,40 @@ if __name__ == "__main__": host = masterIp tdLog.info("Procedures for tdengine deployed in %s" % (host)) - if platform.system().lower() == 'windows': + if platform.system().lower() == "windows": fileName = fileName.replace("/", os.sep) - if (masterIp == "" and not fileName == "0-others\\udf_create.py"): - threading.Thread(target=checkRunTimeError,daemon=True).start() + if masterIp == "" and not fileName == "0-others\\udf_create.py": + threading.Thread(target=checkRunTimeError, daemon=True).start() tdLog.info("Procedures for testing self-deployment") tdDnodes.init(deployPath, masterIp) tdDnodes.setTestCluster(testCluster) tdDnodes.setValgrind(valgrind) tdDnodes.stopAll() - key_word = 'tdCases.addWindows' + key_word = "tdCases.addWindows" is_test_framework = 0 try: - if key_word in open(fileName, encoding='UTF-8').read(): + if key_word in open(fileName, encoding="UTF-8").read(): is_test_framework = 1 except Exception as r: print(r) - updateCfgDictStr = '' + updateCfgDictStr = "" # adapter_cfg_dict_str = '' if is_test_framework: moduleName = fileName.replace(".py", "").replace(os.sep, ".") uModule = importlib.import_module(moduleName) try: ucase = uModule.TDTestCase() - if ((json.dumps(updateCfgDict) == '{}') and hasattr(ucase, 'updatecfgDict')): + if (json.dumps(updateCfgDict) == "{}") and hasattr( + ucase, "updatecfgDict" + ): updateCfgDict = ucase.updatecfgDict - updateCfgDictStr = "-d %s"%base64.b64encode(json.dumps(updateCfgDict).encode()).decode() - if ((json.dumps(adapter_cfg_dict) == '{}') and hasattr(ucase, 'taosadapter_cfg_dict')): + updateCfgDictStr = ( + "-d %s" + % base64.b64encode(json.dumps(updateCfgDict).encode()).decode() + ) + if (json.dumps(adapter_cfg_dict) == "{}") and hasattr( + ucase, "taosadapter_cfg_dict" + ): adapter_cfg_dict = ucase.taosadapter_cfg_dict # adapter_cfg_dict_str = f"-D {base64.b64encode(toml.dumps(adapter_cfg_dict).encode()).decode()}" except Exception as r: @@ -284,8 +320,8 @@ if __name__ == "__main__": tAdapter.init(deployPath, masterIp) tAdapter.stop(force_kill=True) - if dnodeNums == 1 : - tdDnodes.deploy(1,updateCfgDict) + if dnodeNums == 1: + tdDnodes.deploy(1, updateCfgDict) tdDnodes.start(1) tdCases.logSql(logSql) if restful: @@ -293,11 +329,11 @@ if __name__ == "__main__": tAdapter.start() if queryPolicy != 1: - queryPolicy=int(queryPolicy) + queryPolicy = int(queryPolicy) if restful: conn = taosrest.connect(url=f"http://{host}:6041") else: - conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) + conn = taos.connect(host, config=tdDnodes.getSimCfgPath()) cursor = conn.cursor() cursor.execute("create qnode on dnode 1") @@ -305,51 +341,58 @@ if __name__ == "__main__": cursor.execute("show local variables") res = cursor.fetchall() for i in range(cursor.rowcount): - if res[i][0] == "queryPolicy" : + if res[i][0] == "queryPolicy": if int(res[i][1]) == int(queryPolicy): - tdLog.success(f'alter queryPolicy to {queryPolicy} successfully') + tdLog.success( + f"alter queryPolicy to {queryPolicy} successfully" + ) else: tdLog.debug(res) tdLog.exit(f"alter queryPolicy to {queryPolicy} failed") - else : - tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums)) - dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums,mnodeNums=mnodeNums) + else: + tdLog.debug( + "create an cluster with %s nodes and make %s dnode as independent mnode" + % (dnodeNums, mnodeNums) + ) + dnodeslist = cluster.configure_cluster( + dnodeNums=dnodeNums, mnodeNums=mnodeNums + ) tdDnodes = ClusterDnodes(dnodeslist) tdDnodes.init(deployPath, masterIp) tdDnodes.setTestCluster(testCluster) tdDnodes.setValgrind(valgrind) tdDnodes.stopAll() for dnode in tdDnodes.dnodes: - tdDnodes.deploy(dnode.index,{}) + tdDnodes.deploy(dnode.index, {}) for dnode in tdDnodes.dnodes: tdDnodes.starttaosd(dnode.index) tdCases.logSql(logSql) - + if restful: tAdapter.deploy(adapter_cfg_dict) tAdapter.start() if not restful: - conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) + conn = taos.connect(host, config=tdDnodes.getSimCfgPath()) else: conn = taosrest.connect(url=f"http://{host}:6041") # tdLog.info(tdDnodes.getSimCfgPath(),host) if createDnodeNums == 1: - createDnodeNums=dnodeNums + createDnodeNums = dnodeNums else: - createDnodeNums=createDnodeNums - cluster.create_dnode(conn,createDnodeNums) + createDnodeNums = createDnodeNums + cluster.create_dnode(conn, createDnodeNums) try: - if cluster.check_dnode(conn) : + if cluster.check_dnode(conn): print("check dnode ready") except Exception as r: print(r) if queryPolicy != 1: - queryPolicy=int(queryPolicy) + queryPolicy = int(queryPolicy) if restful: conn = taosrest.connect(url=f"http://{host}:6041") else: - conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) + conn = taos.connect(host, config=tdDnodes.getSimCfgPath()) cursor = conn.cursor() cursor.execute("create qnode on dnode 1") @@ -357,18 +400,20 @@ if __name__ == "__main__": cursor.execute("show local variables") res = cursor.fetchall() for i in range(cursor.rowcount): - if res[i][0] == "queryPolicy" : + if res[i][0] == "queryPolicy": if int(res[i][1]) == int(queryPolicy): - tdLog.success(f'alter queryPolicy to {queryPolicy} successfully') + tdLog.success( + f"alter queryPolicy to {queryPolicy} successfully" + ) else: tdLog.debug(res) tdLog.exit(f"alter queryPolicy to {queryPolicy} failed") - - if ucase is not None and hasattr(ucase, 'noConn') and ucase.noConn == True: + + if ucase is not None and hasattr(ucase, "noConn") and ucase.noConn == True: conn = None else: if not restful: - conn = taos.connect(host="%s"%(host), config=tdDnodes.sim.getCfgDir()) + conn = taos.connect(host="%s" % (host), config=tdDnodes.sim.getCfgDir()) else: conn = taosrest.connect(url=f"http://{host}:6041") if is_test_framework: @@ -382,7 +427,7 @@ if __name__ == "__main__": tdDnodes.setValgrind(valgrind) tdDnodes.stopAll() is_test_framework = 0 - key_word = 'tdCases.addLinux' + key_word = "tdCases.addLinux" try: if key_word in open(fileName).read(): is_test_framework = 1 @@ -393,9 +438,9 @@ if __name__ == "__main__": uModule = importlib.import_module(moduleName) try: ucase = uModule.TDTestCase() - if (json.dumps(updateCfgDict) == '{}'): + if json.dumps(updateCfgDict) == "{}": updateCfgDict = ucase.updatecfgDict - if (json.dumps(adapter_cfg_dict) == '{}'): + if json.dumps(adapter_cfg_dict) == "{}": adapter_cfg_dict = ucase.taosadapter_cfg_dict except: pass @@ -404,8 +449,8 @@ if __name__ == "__main__": tAdapter.init(deployPath, masterIp) tAdapter.stop(force_kill=True) - if dnodeNums == 1 : - tdDnodes.deploy(1,updateCfgDict) + if dnodeNums == 1: + tdDnodes.deploy(1, updateCfgDict) tdDnodes.start(1) tdCases.logSql(logSql) @@ -414,9 +459,9 @@ if __name__ == "__main__": tAdapter.start() if queryPolicy != 1: - queryPolicy=int(queryPolicy) + queryPolicy = int(queryPolicy) if not restful: - conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) + conn = taos.connect(host, config=tdDnodes.getSimCfgPath()) else: conn = taosrest.connect(url=f"http://{host}:6041") # tdSql.init(conn.cursor()) @@ -437,23 +482,30 @@ if __name__ == "__main__": cursor.execute("show local variables") res = cursor.fetchall() for i in range(cursor.rowcount): - if res[i][0] == "queryPolicy" : + if res[i][0] == "queryPolicy": if int(res[i][1]) == int(queryPolicy): - tdLog.success(f'alter queryPolicy to {queryPolicy} successfully') + tdLog.success( + f"alter queryPolicy to {queryPolicy} successfully" + ) else: tdLog.debug(res) tdLog.exit(f"alter queryPolicy to {queryPolicy} failed") - else : - tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums)) - dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums,mnodeNums=mnodeNums) + else: + tdLog.debug( + "create an cluster with %s nodes and make %s dnode as independent mnode" + % (dnodeNums, mnodeNums) + ) + dnodeslist = cluster.configure_cluster( + dnodeNums=dnodeNums, mnodeNums=mnodeNums + ) tdDnodes = ClusterDnodes(dnodeslist) tdDnodes.init(deployPath, masterIp) tdDnodes.setTestCluster(testCluster) tdDnodes.setValgrind(valgrind) tdDnodes.stopAll() for dnode in tdDnodes.dnodes: - tdDnodes.deploy(dnode.index,{}) + tdDnodes.deploy(dnode.index, {}) for dnode in tdDnodes.dnodes: tdDnodes.starttaosd(dnode.index) tdCases.logSql(logSql) @@ -463,27 +515,27 @@ if __name__ == "__main__": tAdapter.start() if not restful: - conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) + conn = taos.connect(host, config=tdDnodes.getSimCfgPath()) else: conn = taosrest.connect(url=f"http://{host}:6041") - print(tdDnodes.getSimCfgPath(),host) + print(tdDnodes.getSimCfgPath(), host) if createDnodeNums == 1: - createDnodeNums=dnodeNums + createDnodeNums = dnodeNums else: - createDnodeNums=createDnodeNums - cluster.create_dnode(conn,createDnodeNums) + createDnodeNums = createDnodeNums + cluster.create_dnode(conn, createDnodeNums) try: - if cluster.check_dnode(conn) : + if cluster.check_dnode(conn): print("check dnode ready") except Exception as r: print(r) if queryPolicy != 1: - queryPolicy=int(queryPolicy) + queryPolicy = int(queryPolicy) if restful: conn = taosrest.connect(url=f"http://{host}:6041") else: - conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) + conn = taos.connect(host, config=tdDnodes.getSimCfgPath()) cursor = conn.cursor() cursor.execute("create qnode on dnode 1") @@ -491,13 +543,14 @@ if __name__ == "__main__": cursor.execute("show local variables") res = cursor.fetchall() for i in range(cursor.rowcount): - if res[i][0] == "queryPolicy" : + if res[i][0] == "queryPolicy": if int(res[i][1]) == int(queryPolicy): - tdLog.success(f'alter queryPolicy to {queryPolicy} successfully') + tdLog.success( + f"alter queryPolicy to {queryPolicy} successfully" + ) else: tdLog.debug(res) tdLog.exit(f"alter queryPolicy to {queryPolicy} failed") - if testCluster: tdLog.info("Procedures for testing cluster") @@ -508,7 +561,7 @@ if __name__ == "__main__": else: tdLog.info("Procedures for testing self-deployment") if not restful: - conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) + conn = taos.connect(host, config=tdDnodes.getSimCfgPath()) else: conn = taosrest.connect(url=f"http://{host}:6041") @@ -527,7 +580,7 @@ if __name__ == "__main__": tdDnodes.start(1) time.sleep(1) if not restful: - conn = taos.connect( host, config=tdDnodes.getSimCfgPath()) + conn = taos.connect(host, config=tdDnodes.getSimCfgPath()) else: conn = taosrest.connect(url=f"http://{host}:6041") tdLog.info("Procedures for tdengine deployed in %s" % (host)) From adb335467c2327afef57ad821fa9e95c514a846e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 5 Nov 2022 19:59:18 +0800 Subject: [PATCH 63/82] fix: coverity issues --- source/dnode/mgmt/mgmt_snode/src/smWorker.c | 3 +++ source/dnode/mnode/impl/src/mndSync.c | 2 ++ source/dnode/mnode/impl/src/mndTrans.c | 2 +- source/dnode/mnode/impl/src/mndVgroup.c | 8 ++++---- source/libs/sync/src/syncMain.c | 22 ++++++++++----------- source/libs/sync/src/syncRespMgr.c | 18 ++++++++--------- source/util/src/tworker.c | 6 +++--- 7 files changed, 32 insertions(+), 29 deletions(-) diff --git a/source/dnode/mgmt/mgmt_snode/src/smWorker.c b/source/dnode/mgmt/mgmt_snode/src/smWorker.c index dbd081338a..1381d4c391 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smWorker.c +++ b/source/dnode/mgmt/mgmt_snode/src/smWorker.c @@ -140,6 +140,9 @@ int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { if (pSnode == NULL) { dError("snode: msg:%p failed to put into vnode queue since %s, type:%s qtype:%d", pMsg, terrstr(), TMSG_INFO(pMsg->msgType), qtype); + taosFreeQitem(pMsg); + rpcFreeCont(pRpc->pCont); + pRpc->pCont = NULL; return -1; } diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 0f50391ac5..abb23bfb89 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -282,6 +282,8 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { pMgmt->errCode = 0; SRpcMsg req = {.msgType = TDMT_MND_APPLY_MSG, .contLen = sdbGetRawTotalSize(pRaw)}; + if (req.contLen <= 0) return -1; + req.pCont = rpcMallocCont(req.contLen); if (req.pCont == NULL) return -1; memcpy(req.pCont, pRaw, req.contLen); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 6a606a1a7e..ac05598bdc 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -375,7 +375,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto _OVER; action.pCont = NULL; } else { - if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER; + if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto _OVER; } } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 72a4621eae..e00d0d955e 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -325,10 +325,10 @@ static void *mndBuildAlterVnodeConfigReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pV static void *mndBuildAlterVnodeReplicaReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_t dnodeId, int32_t *pContLen) { SAlterVnodeReplicaReq alterReq = { - alterReq.vgId = pVgroup->vgId, - alterReq.strict = pDb->cfg.strict, - alterReq.replica = pVgroup->replica, - alterReq.selfIndex = -1, + .vgId = pVgroup->vgId, + .strict = pDb->cfg.strict, + .replica = pVgroup->replica, + .selfIndex = -1, }; for (int32_t v = 0; v < pVgroup->replica; ++v) { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 47b8614717..8a3047ae32 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -768,7 +768,7 @@ char* sync2SimpleStr(int64_t rid) { sTrace("syncSetRpc get pSyncNode is NULL, rid:%" PRId64, rid); return NULL; } - ASSERT(rid == pSyncNode->rid); + char* s = syncNode2SimpleStr(pSyncNode); syncNodeRelease(pSyncNode); @@ -778,11 +778,9 @@ char* sync2SimpleStr(int64_t rid) { int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak) { SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { - syncNodeRelease(pSyncNode); terrno = TSDB_CODE_SYN_INTERNAL_ERROR; return -1; } - ASSERT(rid == pSyncNode->rid); int32_t ret = syncNodePropose(pSyncNode, pMsg, isWeak); syncNodeRelease(pSyncNode); @@ -3108,15 +3106,15 @@ int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* p if (ths->pFsm->FpLeaderTransferCb != NULL) { SFsmCbMeta cbMeta = { - cbMeta.code = 0, - cbMeta.currentTerm = ths->pRaftStore->currentTerm, - cbMeta.flag = 0, - cbMeta.index = pEntry->index, - cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, pEntry->index), - cbMeta.isWeak = pEntry->isWeak, - cbMeta.seqNum = pEntry->seqNum, - cbMeta.state = ths->state, - cbMeta.term = pEntry->term, + .code = 0, + .currentTerm = ths->pRaftStore->currentTerm, + .flag = 0, + .index = pEntry->index, + .lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, pEntry->index), + .isWeak = pEntry->isWeak, + .seqNum = pEntry->seqNum, + .state = ths->state, + .term = pEntry->term, }; ths->pFsm->FpLeaderTransferCb(ths->pFsm, pRpcMsg, &cbMeta); } diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index 35c831b52f..c31dede0b3 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -146,15 +146,15 @@ void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl, bool rsp) { cnt++; SFsmCbMeta cbMeta = { - cbMeta.index = SYNC_INDEX_INVALID, - cbMeta.lastConfigIndex = SYNC_INDEX_INVALID, - cbMeta.isWeak = false, - cbMeta.code = TSDB_CODE_SYN_TIMEOUT, - cbMeta.state = pSyncNode->state, - cbMeta.seqNum = *pSeqNum, - cbMeta.term = SYNC_TERM_INVALID, - cbMeta.currentTerm = pSyncNode->pRaftStore->currentTerm, - cbMeta.flag = 0, + .index = SYNC_INDEX_INVALID, + .lastConfigIndex = SYNC_INDEX_INVALID, + .isWeak = false, + .code = TSDB_CODE_SYN_TIMEOUT, + .state = pSyncNode->state, + .seqNum = *pSeqNum, + .term = SYNC_TERM_INVALID, + .currentTerm = pSyncNode->pRaftStore->currentTerm, + .flag = 0, }; pStub->rpcMsg.pCont = NULL; diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 5971033bf8..863cee9b08 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -246,9 +246,6 @@ STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) { pool->nextId = (pool->nextId + 1) % pool->max; } - while (worker->pid <= 0) taosMsleep(10); - queue->threadId = worker->pid; - uInfo("worker:%s, queue:%p is allocated, ahandle:%p thread:%08" PRId64, pool->name, queue, ahandle, queue->threadId); code = 0; _OVER: @@ -260,6 +257,9 @@ _OVER: if (worker->qall != NULL) taosFreeQall(worker->qall); return NULL; } else { + while (worker->pid <= 0) taosMsleep(10); + queue->threadId = worker->pid; + uInfo("worker:%s, queue:%p is allocated, ahandle:%p thread:%08" PRId64, pool->name, queue, ahandle, queue->threadId); return queue; } } From 8a1e28dcf4e79ad66eb1f452bc165edf30d7dcb6 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 6 Nov 2022 11:27:21 -0800 Subject: [PATCH 64/82] Update 02-influxdb-line.mdx --- docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx b/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx index 19ccf49906..44d63e0b3a 100644 --- a/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx +++ b/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx @@ -40,7 +40,7 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0 - 为了提高写入的效率，默认假设同一个超级表中 field_set 的顺序是一样的（第一条数据包含所有的 field，后面的数据按照这个顺序），如果顺序不一样，需要配置参数 smlDataFormat 为 false，否则，数据写入按照相同顺序写入，库中数据会异常。（3.0.1.3 之后的版本 smlDataFormat 默认为 false） [TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) - 默认生产的子表名是根据规则生成的唯一 ID 值。为了让用户可以指定生成的表名，可以通过在 taos.cfg 里配置 smlChildTableName 参数来指定。 举例如下：配置 smlChildTableName=tname 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。[TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) - ::: +::: 要了解更多可参考：[InfluxDB Line 协议官方文档](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/) 和 [TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) From b86dac61462eb1ae38ef26a3288a42d2a687b7e5 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 6 Nov 2022 11:33:50 -0800 Subject: [PATCH 65/82] Update 04-opentsdb-json.mdx --- docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx b/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx index 57b00ab310..4118fc3853 100644 --- a/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx +++ b/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx @@ -50,7 +50,7 @@ OpenTSDB JSON 格式协议采用一个 JSON 字符串表示一行或多行数据 - TDengine 只接收 JSON **数组格式**的字符串，即使一行数据也需要转换成数组形式。 - 默认生产的子表名是根据规则生成的唯一 ID 值。为了让用户可以指定生成的表名，可以通过在 taos.cfg 里配置 smlChildTableName 参数来指定。 举例如下：配置 smlChildTableName=tname 插入数据为 `"tags": { "host": "web02","dc": "lga","tname":"cpu1"}` 则创建的表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。 - ::: +::: ## 示例代码 From 93fa00ee367948aad290dd32dfc471bd0d7222fc Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 6 Nov 2022 11:34:34 -0800 Subject: [PATCH 66/82] Update 02-influxdb-line.mdx --- docs/en/07-develop/03-insert-data/02-influxdb-line.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx b/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx index 6f933d52ac..1a23061fd1 100644 --- a/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx +++ b/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx @@ -39,7 +39,7 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0 - Multiple kinds of precision can be used for the `timestamp` field. Time precision can be from nanosecond (ns) to hour (h). - You can configure smlChildTableName in taos.cfg to specify table names, for example, `smlChildTableName=tname`. You can insert `st,tname=cpul,t1=4 c1=3 1626006833639000000` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. - It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat in taos.cfg to false. Otherwise, data will be written out of order and a database error will occur.(smlDataFormat in taos.cfg default to false after version of 3.0.1.3) - ::: +::: For more details please refer to [InfluxDB Line Protocol](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/) and [TDengine Schemaless](/reference/schemaless/#Schemaless-Line-Protocol) From a287efe4d111b541f0b98e851910d895570f1e17 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 6 Nov 2022 11:35:15 -0800 Subject: [PATCH 67/82] Update 04-opentsdb-json.mdx --- docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx b/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx index be8f521cf2..0a92e3b916 100644 --- a/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx +++ b/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx @@ -49,7 +49,7 @@ Please refer to [OpenTSDB HTTP API](http://opentsdb.net/docs/build/html/api_http - In JSON protocol, strings will be converted to NCHAR type and numeric values will be converted to double type. - Only data in array format is accepted and so an array must be used even if there is only one row. - The defult child table name is generated by rules.You can configure smlChildTableName in taos.cfg to specify child table names, for example, `smlChildTableName=tname`. You can insert `"tags": { "host": "web02","dc": "lga","tname":"cpu1"}` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. - ::: +::: ## Examples From da9d42e8fe0e815f782f5e0b6664319bae0b5034 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 6 Nov 2022 11:46:35 -0800 Subject: [PATCH 68/82] Update 02-influxdb-line.mdx --- docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx b/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx index 44d63e0b3a..e2398054b5 100644 --- a/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx +++ b/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx @@ -38,8 +38,7 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0 - field_set 中的每个数据项都需要对自身的数据类型进行描述, 比如 1.2f32 代表 FLOAT 类型的数值 1.2, 如果不带类型后缀会被当作 DOUBLE 处理; - timestamp 支持多种时间精度。写入数据的时候需要用参数指定时间精度，支持从小时到纳秒的 6 种时间精度。 - 为了提高写入的效率，默认假设同一个超级表中 field_set 的顺序是一样的（第一条数据包含所有的 field，后面的数据按照这个顺序），如果顺序不一样，需要配置参数 smlDataFormat 为 false，否则，数据写入按照相同顺序写入，库中数据会异常。（3.0.1.3 之后的版本 smlDataFormat 默认为 false） [TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) -- 默认生产的子表名是根据规则生成的唯一 ID 值。为了让用户可以指定生成的表名，可以通过在 taos.cfg 里配置 smlChildTableName 参数来指定。 - 举例如下：配置 smlChildTableName=tname 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。[TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) +- 默认生产的子表名是根据规则生成的唯一 ID 值。用户也可以通过在 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。[TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) ::: 要了解更多可参考：[InfluxDB Line 协议官方文档](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/) 和 [TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) From 7b3a47a016736f844254f3a9a8b71ab4c389292d Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 6 Nov 2022 11:48:32 -0800 Subject: [PATCH 69/82] Update 03-opentsdb-telnet.mdx --- docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx b/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx index 25be8a0aa4..3b2148ef4a 100644 --- a/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx +++ b/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx @@ -32,8 +32,7 @@ OpenTSDB 行协议同样采用一行字符串来表示一行数据。OpenTSDB meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3 ``` -- 默认生产的子表名是根据规则生成的唯一 ID 值。为了让用户可以指定生成的表名，可以通过在 taos.cfg 里配置 smlChildTableName 参数来指定。 - 举例如下：配置 smlChildTableName=tname 插入数据为 meters.current 1648432611250 11.3 tname=cpu1 location=California.LosAngeles groupid=3 则创建的表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。 +- 默认生产的子表名是根据规则生成的唯一 ID 值。用户也可以通过在 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 meters.current 1648432611250 11.3 tname=cpu1 location=California.LosAngeles groupid=3 则创建的表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。 参考 [OpenTSDB Telnet API 文档](http://opentsdb.net/docs/build/html/api_telnet/put.html)。 ## 示例代码 From 41f121dec84438d28e6d63cf362100c73186a08e Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 6 Nov 2022 11:51:25 -0800 Subject: [PATCH 70/82] Update 04-opentsdb-json.mdx --- docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx b/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx index 4118fc3853..a28320e35e 100644 --- a/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx +++ b/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx @@ -48,8 +48,7 @@ OpenTSDB JSON 格式协议采用一个 JSON 字符串表示一行或多行数据 - 对于 JSON 格式协议，TDengine 并不会自动把所有标签转成 NCHAR 类型， 字符串将将转为 NCHAR 类型， 数值将同样转换为 DOUBLE 类型。 - TDengine 只接收 JSON **数组格式**的字符串，即使一行数据也需要转换成数组形式。 -- 默认生产的子表名是根据规则生成的唯一 ID 值。为了让用户可以指定生成的表名，可以通过在 taos.cfg 里配置 smlChildTableName 参数来指定。 - 举例如下：配置 smlChildTableName=tname 插入数据为 `"tags": { "host": "web02","dc": "lga","tname":"cpu1"}` 则创建的表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。 +- 默认生产的子表名是根据规则生成的唯一 ID 值。用户也可以通过在 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 `"tags": { "host": "web02","dc": "lga","tname":"cpu1"}` 则创建的子表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。 ::: ## 示例代码 From c961cecce09b59d3107e5382cfcba5994ad6dd0e Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 6 Nov 2022 11:59:11 -0800 Subject: [PATCH 71/82] Update 02-influxdb-line.mdx --- docs/en/07-develop/03-insert-data/02-influxdb-line.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx b/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx index 1a23061fd1..8a9dcd536d 100644 --- a/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx +++ b/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx @@ -37,7 +37,7 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0 - All the data in `tag_set` will be converted to NCHAR type automatically . - Each data in `field_set` must be self-descriptive for its data type. For example 1.2f32 means a value 1.2 of float type. Without the "f" type suffix, it will be treated as type double. - Multiple kinds of precision can be used for the `timestamp` field. Time precision can be from nanosecond (ns) to hour (h). -- You can configure smlChildTableName in taos.cfg to specify table names, for example, `smlChildTableName=tname`. You can insert `st,tname=cpul,t1=4 c1=3 1626006833639000000` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. +- The table name is created automatically in a rule to guarantee its uniqueness. But you can configure smlChildTableName in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called 'tname' and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpul,t1=4 c1=3 1626006833639000000`, the table 'cpus1' will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. - It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat in taos.cfg to false. Otherwise, data will be written out of order and a database error will occur.(smlDataFormat in taos.cfg default to false after version of 3.0.1.3) ::: From 34682da76ac6e453b4a11d63ccebcc55dab9e92c Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 6 Nov 2022 12:00:43 -0800 Subject: [PATCH 72/82] Update 03-opentsdb-telnet.mdx --- docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx b/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx index 10482df6b2..43c815382a 100644 --- a/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx +++ b/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx @@ -32,7 +32,7 @@ For example: meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3 ``` -- The defult child table name is generated by rules.You can configure smlChildTableName in taos.cfg to specify child table names, for example, `smlChildTableName=tname`. You can insert `meters.current 1648432611250 11.3 tname=cpu1 location=California.LosAngeles groupid=3` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. +- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure smlChildTableName in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called 'tname' and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpul,t1=4 c1=3 1626006833639000000`, the table 'cpus1' will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. Please refer to [OpenTSDB Telnet API](http://opentsdb.net/docs/build/html/api_telnet/put.html) for more details. ## Examples From 34640d0eb3b471837572fb3835c2a5b54c7c8a60 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 6 Nov 2022 12:02:13 -0800 Subject: [PATCH 73/82] Update 04-opentsdb-json.mdx --- docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx b/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx index 0a92e3b916..1eea5299ac 100644 --- a/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx +++ b/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx @@ -48,7 +48,7 @@ Please refer to [OpenTSDB HTTP API](http://opentsdb.net/docs/build/html/api_http - In JSON protocol, strings will be converted to NCHAR type and numeric values will be converted to double type. - Only data in array format is accepted and so an array must be used even if there is only one row. -- The defult child table name is generated by rules.You can configure smlChildTableName in taos.cfg to specify child table names, for example, `smlChildTableName=tname`. You can insert `"tags": { "host": "web02","dc": "lga","tname":"cpu1"}` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. +- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure smlChildTableName in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called 'tname' and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpul,t1=4 c1=3 1626006833639000000`, the child table 'cpus1' will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. ::: ## Examples From 4078f081129ccfce11f375d1944d39180c80dbfe Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 6 Nov 2022 12:04:08 -0800 Subject: [PATCH 74/82] Update 04-opentsdb-json.mdx --- docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx b/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx index a28320e35e..89818409c5 100644 --- a/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx +++ b/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx @@ -48,7 +48,7 @@ OpenTSDB JSON 格式协议采用一个 JSON 字符串表示一行或多行数据 - 对于 JSON 格式协议，TDengine 并不会自动把所有标签转成 NCHAR 类型， 字符串将将转为 NCHAR 类型， 数值将同样转换为 DOUBLE 类型。 - TDengine 只接收 JSON **数组格式**的字符串，即使一行数据也需要转换成数组形式。 -- 默认生产的子表名是根据规则生成的唯一 ID 值。用户也可以通过在 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 `"tags": { "host": "web02","dc": "lga","tname":"cpu1"}` 则创建的子表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。 +- 默认生成的子表名是根据规则生成的唯一 ID 值。用户也可以通过在 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 `"tags": { "host": "web02","dc": "lga","tname":"cpu1"}` 则创建的子表名为 cpu1。注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。 ::: ## 示例代码 From ff488575a418b039bf8b6ba8c41c6891aebc1774 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 6 Nov 2022 12:05:07 -0800 Subject: [PATCH 75/82] Update 02-influxdb-line.mdx --- docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx b/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx index e2398054b5..afe73af8db 100644 --- a/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx +++ b/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx @@ -38,7 +38,7 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0 - field_set 中的每个数据项都需要对自身的数据类型进行描述, 比如 1.2f32 代表 FLOAT 类型的数值 1.2, 如果不带类型后缀会被当作 DOUBLE 处理; - timestamp 支持多种时间精度。写入数据的时候需要用参数指定时间精度，支持从小时到纳秒的 6 种时间精度。 - 为了提高写入的效率，默认假设同一个超级表中 field_set 的顺序是一样的（第一条数据包含所有的 field，后面的数据按照这个顺序），如果顺序不一样，需要配置参数 smlDataFormat 为 false，否则，数据写入按照相同顺序写入，库中数据会异常。（3.0.1.3 之后的版本 smlDataFormat 默认为 false） [TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) -- 默认生产的子表名是根据规则生成的唯一 ID 值。用户也可以通过在 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。[TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) +- 默认产生的子表名是根据规则生成的唯一 ID 值。用户也可以通过在 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1。注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。[TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) ::: 要了解更多可参考：[InfluxDB Line 协议官方文档](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/) 和 [TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) From 75f69203cec9f94cb1aa0190d3e3e1b756b0e972 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 6 Nov 2022 12:29:54 -0800 Subject: [PATCH 76/82] Update 02-influxdb-line.mdx --- docs/en/07-develop/03-insert-data/02-influxdb-line.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx b/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx index 8a9dcd536d..d299b018d1 100644 --- a/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx +++ b/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx @@ -37,7 +37,7 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0 - All the data in `tag_set` will be converted to NCHAR type automatically . - Each data in `field_set` must be self-descriptive for its data type. For example 1.2f32 means a value 1.2 of float type. Without the "f" type suffix, it will be treated as type double. - Multiple kinds of precision can be used for the `timestamp` field. Time precision can be from nanosecond (ns) to hour (h). -- The table name is created automatically in a rule to guarantee its uniqueness. But you can configure smlChildTableName in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called 'tname' and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpul,t1=4 c1=3 1626006833639000000`, the table 'cpus1' will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. +- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure smlChildTableName in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table 'cpu1' will be created automatically. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. - It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat in taos.cfg to false. Otherwise, data will be written out of order and a database error will occur.(smlDataFormat in taos.cfg default to false after version of 3.0.1.3) ::: From b7b80d3762cfca2b64195f93323672daff558bab Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 6 Nov 2022 12:30:44 -0800 Subject: [PATCH 77/82] Update 03-opentsdb-telnet.mdx --- docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx b/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx index 43c815382a..fb1c67f6fe 100644 --- a/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx +++ b/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx @@ -32,7 +32,7 @@ For example: meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3 ``` -- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure smlChildTableName in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called 'tname' and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpul,t1=4 c1=3 1626006833639000000`, the table 'cpus1' will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. +- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure smlChildTableName in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table 'cpu1' will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. Please refer to [OpenTSDB Telnet API](http://opentsdb.net/docs/build/html/api_telnet/put.html) for more details. ## Examples From 25d13025cd7093a6d20740d11930d1940f79d054 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 6 Nov 2022 12:31:48 -0800 Subject: [PATCH 78/82] Update 04-opentsdb-json.mdx --- docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx b/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx index 1eea5299ac..a00ea02b2a 100644 --- a/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx +++ b/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx @@ -48,7 +48,7 @@ Please refer to [OpenTSDB HTTP API](http://opentsdb.net/docs/build/html/api_http - In JSON protocol, strings will be converted to NCHAR type and numeric values will be converted to double type. - Only data in array format is accepted and so an array must be used even if there is only one row. -- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure smlChildTableName in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called 'tname' and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpul,t1=4 c1=3 1626006833639000000`, the child table 'cpus1' will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. +- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure smlChildTableName in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. ::: ## Examples From b1164a0189e9d9bfc9a69d36c6c5dd81ea7506f7 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 6 Nov 2022 12:32:38 -0800 Subject: [PATCH 79/82] Update 03-opentsdb-telnet.mdx --- docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx b/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx index fb1c67f6fe..09c7b9994d 100644 --- a/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx +++ b/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx @@ -32,7 +32,7 @@ For example: meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3 ``` -- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure smlChildTableName in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table 'cpu1' will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. +- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure smlChildTableName in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. Please refer to [OpenTSDB Telnet API](http://opentsdb.net/docs/build/html/api_telnet/put.html) for more details. ## Examples From 3c30b97f4ed047f165f9f32860afd847a821ccc0 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 6 Nov 2022 12:50:56 -0800 Subject: [PATCH 80/82] Update 02-influxdb-line.mdx --- docs/en/07-develop/03-insert-data/02-influxdb-line.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx b/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx index d299b018d1..3c08860260 100644 --- a/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx +++ b/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx @@ -37,7 +37,7 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0 - All the data in `tag_set` will be converted to NCHAR type automatically . - Each data in `field_set` must be self-descriptive for its data type. For example 1.2f32 means a value 1.2 of float type. Without the "f" type suffix, it will be treated as type double. - Multiple kinds of precision can be used for the `timestamp` field. Time precision can be from nanosecond (ns) to hour (h). -- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure smlChildTableName in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table 'cpu1' will be created automatically. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. +- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be created automatically. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. - It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat in taos.cfg to false. Otherwise, data will be written out of order and a database error will occur.(smlDataFormat in taos.cfg default to false after version of 3.0.1.3) ::: From 341eec70ba728b938a48ae544ceac6a89633112b Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 6 Nov 2022 12:51:46 -0800 Subject: [PATCH 81/82] Update 03-opentsdb-telnet.mdx --- docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx b/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx index 09c7b9994d..5d3f25dca9 100644 --- a/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx +++ b/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx @@ -32,7 +32,7 @@ For example: meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3 ``` -- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure smlChildTableName in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. +- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. Please refer to [OpenTSDB Telnet API](http://opentsdb.net/docs/build/html/api_telnet/put.html) for more details. ## Examples From 658b56ba86b0a5971d53364fd29f003d2bac284d Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 6 Nov 2022 12:52:13 -0800 Subject: [PATCH 82/82] Update 04-opentsdb-json.mdx --- docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx b/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx index a00ea02b2a..7a3ac6bad3 100644 --- a/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx +++ b/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx @@ -48,7 +48,7 @@ Please refer to [OpenTSDB HTTP API](http://opentsdb.net/docs/build/html/api_http - In JSON protocol, strings will be converted to NCHAR type and numeric values will be converted to double type. - Only data in array format is accepted and so an array must be used even if there is only one row. -- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure smlChildTableName in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. +- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. ::: ## Examples