From 25e75f20ad97f22b84b8c212279f5a00663a2572 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 13 Sep 2022 13:53:23 +0800 Subject: [PATCH 01/52] fix:add test cases for sml --- tests/system-test/fulltest.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 8987ba3bbd..138633868a 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -156,8 +156,8 @@ python3 ./test.py -f 2-query/sin.py python3 ./test.py -f 2-query/sin.py -R python3 ./test.py -f 2-query/smaTest.py python3 ./test.py -f 2-query/smaTest.py -R -#python3 ./test.py -f 2-query/sml.py -#python3 ./test.py -f 2-query/sml.py -R +python3 ./test.py -f 2-query/sml.py +python3 ./test.py -f 2-query/sml.py -R python3 ./test.py -f 2-query/spread.py python3 ./test.py -f 2-query/spread.py -R python3 ./test.py -f 2-query/sqrt.py @@ -512,6 +512,6 @@ python3 ./test.py -f 2-query/count_partition.py -Q 3 python3 ./test.py -f 2-query/max_partition.py -Q 3 python3 ./test.py -f 2-query/last_row.py -Q 3 python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 -#python3 ./test.py -f 2-query/sml.py -Q 3 +python3 ./test.py -f 2-query/sml.py -Q 3 python3 ./test.py -f 2-query/interp.py -Q 3 From faeb5297b14984947769bb93c95b3b8b275477a5 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 14 Sep 2022 11:44:54 +0800 Subject: [PATCH 02/52] feat: update taos-tools 9f4c01e for 3.0 --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 0f7a450920..c13aa5f4b1 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 3588b3d + GIT_TAG 9f4c01e SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 42a326fbdd17e664088eb42dc283cb96039d13b9 Mon Sep 17 00:00:00 2001 From: tomchon Date: Thu, 15 Sep 2022 10:45:41 +0800 Subject: [PATCH 03/52] test:update jenkins of ci --- Jenkinsfile2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index d7df07f06a..6a4a2156e5 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -218,12 +218,12 @@ def pre_test_win(){ if (env.CHANGE_URL =~ /\/TDengine\//) { bat ''' cd %WIN_INTERNAL_ROOT% - git pull + git pull ''' + env.CHANGE_TARGET + ''' ''' bat ''' cd %WIN_COMMUNITY_ROOT% git remote prune origin - git pull + git pull ''' + env.CHANGE_TARGET + ''' ''' bat ''' cd %WIN_COMMUNITY_ROOT% @@ -236,7 +236,7 @@ def pre_test_win(){ } else if (env.CHANGE_URL =~ /\/TDinternal\//) { bat ''' cd %WIN_INTERNAL_ROOT% - git pull + git pull ''' + env.CHANGE_TARGET + ''' ''' bat ''' cd %WIN_INTERNAL_ROOT% From 8859a3efa35828c6f8ea21ca01afd0ebfc64fed0 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 15 Sep 2022 14:00:10 +0800 Subject: [PATCH 04/52] feat: update taos-tools e7270c9 for 3.0 --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index c13aa5f4b1..6fb2a69847 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 9f4c01e + GIT_TAG e7270c9 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From c66da3ce908ec97729944c513a71d42c2efe990a Mon Sep 17 00:00:00 2001 From: danielclow <106956386+danielclow@users.noreply.github.com> Date: Fri, 16 Sep 2022 10:26:52 +0800 Subject: [PATCH 05/52] Update README.md Corrected one typo and some links --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a8126d74d4..1034a19f37 100644 --- a/README.md +++ b/README.md @@ -15,19 +15,19 @@ [![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201) -English | [简体中文](README-CN.md) | [Lean more about TSDB](https://tdengine.com/tsdb) +English | [简体中文](README-CN.md) | [Learn more about TSDB](https://tdengine.com/tsdb/) # What is TDengine？ TDengine is an open source, high-performance, cloud native [time-series database](https://tdengine.com/tsdb/) optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. It enables efficient, real-time data ingestion, processing, and monitoring of TB and even PB scale data per day, generated by billions of sensors and data collectors. TDengine differentiates itself from other time-seires databases with the following advantages: -- **[High-Performance](https://tdengine.com/tdengine/high-performance-time-series-database/)**: TDengine is the only time-series database to solve the high cardinality issue to support billions of data collection points while out performing other time-series databases for data ingestion, querying and data compression. +- **[High Performance](https://tdengine.com/tdengine/high-performance-time-series-database/)**: TDengine is the only time-series database to solve the high cardinality issue to support billions of data collection points while out performing other time-series databases for data ingestion, querying and data compression. - **[Simplified Solution](https://tdengine.com/tdengine/simplified-time-series-data-solution/)**: Through built-in caching, stream processing and data subscription features, TDengine provides a simplified solution for time-series data processing. It reduces system design complexity and operation costs significantly. - **[Cloud Native](https://tdengine.com/tdengine/cloud-native-time-series-database/)**: Through native distributed design, sharding and partitioning, separation of compute and storage, RAFT, support for kubernetes deployment and full observability, TDengine is a cloud native Time-Series Database and can be deployed on public, private or hybrid clouds. -- **[Ease of Use](https://docs.tdengine.com/get-started/docker/)**: For administrators, TDengine significantly reduces the effort to deploy and maintain. For developers, it provides a simple interface, simplified solution and seamless integrations for third party tools. For data users, it gives easy data access. +- **[Ease of Use](https://tdengine.com/tdengine/easy-time-series-data-platform/)**: For administrators, TDengine significantly reduces the effort to deploy and maintain. For developers, it provides a simple interface, simplified solution and seamless integrations for third party tools. For data users, it gives easy data access. - **[Easy Data Analytics](https://tdengine.com/tdengine/time-series-data-analytics-made-easy/)**: Through super tables, storage and compute separation, data partitioning by time interval, pre-computation and other means, TDengine makes it easy to explore, format, and get access to data in a highly efficient way. @@ -232,9 +232,9 @@ After building successfully, TDengine can be installed by sudo make install ``` -Users can find more information about directories installed on the system in the [directory and files](https://docs.taosdata.com/reference/directory/) section. +Users can find more information about directories installed on the system in the [directory and files](https://docs.tdengine.com/reference/directory/) section. -Installing from source code will also configure service management for TDengine.Users can also choose to [install from packages](https://docs.taosdata.com/get-started/package/) for it. +Installing from source code will also configure service management for TDengine.Users can also choose to [install from packages](https://docs.tdengine.com/get-started/package/) for it. To start the service after installation, in a terminal, use: From 24615a08da417f56929b64a272431538ef8f8d86 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 16 Sep 2022 18:21:24 +0800 Subject: [PATCH 06/52] feat: update taos-tools 7d7b3ce for 3.0 --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 6fb2a69847..980a15e462 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG e7270c9 + GIT_TAG 7d7b3ce SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 0446d7075aa731be76591a7c73c54c39ee88e8a4 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 18 Sep 2022 02:36:07 +0000 Subject: [PATCH 07/52] feat: update taos-tools 125c77a for 3.0 --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 980a15e462..6a3439ada9 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 7d7b3ce + GIT_TAG 125c77a SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 4e36fe0c1bbdb8ea5829cea8d852feb6b60d5c2e Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 19 Sep 2022 14:54:31 +0800 Subject: [PATCH 08/52] feat: update taos-tools 318246c for 3.0 --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 6a3439ada9..bcf8392052 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 125c77a + GIT_TAG 318246c SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 7b58a94a3591213cec03793f35c596686849435f Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 19 Sep 2022 16:17:27 +0800 Subject: [PATCH 09/52] feat: update taos-tools 509ec72 for 3.0 --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index bcf8392052..5cc580a9c6 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 318246c + GIT_TAG 509ec72 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 9a1f1e1aa4d59faf1b83b2873095b48573ca4231 Mon Sep 17 00:00:00 2001 From: geroge Date: Mon, 19 Sep 2022 02:29:21 -0700 Subject: [PATCH 10/52] Solve arm32, arm-linux-gnueabihf-gcc cross compilation, about JNI function type conflict, github issue number 16903 --- source/client/src/TMQConnector.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/src/TMQConnector.c b/source/client/src/TMQConnector.c index fcf6957df9..26bf55055f 100644 --- a/source/client/src/TMQConnector.c +++ b/source/client/src/TMQConnector.c @@ -212,7 +212,7 @@ JNIEXPORT void JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqCommitAsync(JN tmq_commit_async(tmq, res, commit_cb, consumer); } -JNIEXPORT int JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqUnsubscribeImp(JNIEnv *env, jobject jobj, jlong jtmq) { +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqUnsubscribeImp(JNIEnv *env, jobject jobj, jlong jtmq) { tmq_t *tmq = (tmq_t *)jtmq; if (tmq == NULL) { jniError("jobj:%p, tmq is closed", jobj); @@ -222,7 +222,7 @@ JNIEXPORT int JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqUnsubscribeImp( return tmq_unsubscribe((tmq_t *)tmq); } -JNIEXPORT int JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqConsumerCloseImp(JNIEnv *env, jobject jobj, +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqConsumerCloseImp(JNIEnv *env, jobject jobj, jlong jtmq) { tmq_t *tmq = (tmq_t *)jtmq; if (tmq == NULL) { From b32fc5a44fc36ccc331d37adc1e1970ff6a17829 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 19 Sep 2022 19:46:13 +0800 Subject: [PATCH 11/52] fix config --- source/common/src/tglobal.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 64d376d265..f1c1e79a86 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -58,7 +58,7 @@ int32_t tsNumOfMnodeFetchThreads = 1; int32_t tsNumOfMnodeReadThreads = 1; int32_t tsNumOfVnodeQueryThreads = 4; int32_t tsNumOfVnodeStreamThreads = 2; -int32_t tsNumOfVnodeFetchThreads = 1; +int32_t tsNumOfVnodeFetchThreads = 4; int32_t tsNumOfVnodeWriteThreads = 2; int32_t tsNumOfVnodeSyncThreads = 2; int32_t tsNumOfVnodeRsmaThreads = 2; @@ -365,7 +365,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfVnodeStreamThreads = TMAX(tsNumOfVnodeStreamThreads, 4); if (cfgAddInt32(pCfg, "numOfVnodeStreamThreads", tsNumOfVnodeStreamThreads, 4, 1024, 0) != 0) return -1; - tsNumOfVnodeFetchThreads = 1; + tsNumOfVnodeFetchThreads = tsNumOfCores / 4; + tsNumOfVnodeFetchThreads = TMAX(tsNumOfVnodeFetchThreads, 4); if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 4, 1024, 0) != 0) return -1; tsNumOfVnodeWriteThreads = tsNumOfCores; From 13114a6275eb10da324b521b9ca0d59b07361796 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 20 Sep 2022 09:58:59 +0800 Subject: [PATCH 12/52] feat: update taos-tools 285b5e0 for 3.0 --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 5cc580a9c6..b75d4607b8 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 509ec72 + GIT_TAG 285b5e0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 8a010a58f3e7997c1c68eefb2ec10e5ce5ea5fbf Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 20 Sep 2022 16:13:08 +0800 Subject: [PATCH 13/52] enh: added memory allocators for parser and planner --- include/libs/nodes/nodes.h | 6 ++ source/libs/nodes/src/nodesUtilFuncs.c | 115 +++++++++++++++++++--- source/libs/parser/src/parAstCreater.c | 3 +- source/libs/parser/test/parTestUtil.cpp | 52 +++++----- source/libs/planner/src/planSpliter.c | 1 + source/libs/planner/test/planTestMain.cpp | 8 ++ source/libs/planner/test/planTestUtil.cpp | 49 ++++----- source/libs/planner/test/planTestUtil.h | 11 ++- source/libs/scalar/src/scalar.c | 2 +- 9 files changed, 181 insertions(+), 66 deletions(-) diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 6500d3d183..1e2a53f598 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -275,6 +275,12 @@ typedef struct SNodeList { SListCell* pTail; } SNodeList; +typedef struct SNodeAllocator SNodeAllocator; + +int32_t nodesCreateNodeAllocator(int32_t chunkSize, SNodeAllocator** pAllocator); +void nodesDestroyNodeAllocator(SNodeAllocator* pAllocator); +void nodesResetThreadLevelAllocator(SNodeAllocator* pAllocator); + SNode* nodesMakeNode(ENodeType type); void nodesDestroyNode(SNode* pNode); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 805ddb9e42..be94495856 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -22,8 +22,98 @@ #include "tdatablock.h" #include "thash.h" -static SNode* makeNode(ENodeType type, size_t size) { - SNode* p = taosMemoryCalloc(1, size); +typedef struct SNodeMemChunk { + int32_t availableSize; + int32_t usedSize; + char* pBuf; + struct SNodeMemChunk* pNext; +} SNodeMemChunk; + +typedef struct SNodeAllocator { + int32_t chunkSize; + int32_t chunkNum; + SNodeMemChunk* pCurrChunk; + SNodeMemChunk* pChunks; +} SNodeAllocator; + +static threadlocal SNodeAllocator* pNodeAllocator; + +static SNodeMemChunk* callocNodeChunk(SNodeAllocator* pAllocator) { + SNodeMemChunk* pNewChunk = taosMemoryCalloc(1, sizeof(SNodeMemChunk) + pAllocator->chunkSize); + if (NULL == pNewChunk) { + return NULL; + } + pNewChunk->pBuf = (char*)(pNewChunk + 1); + pNewChunk->availableSize = pAllocator->chunkSize; + pNewChunk->usedSize = 0; + pNewChunk->pNext = NULL; + if (NULL != pAllocator->pCurrChunk) { + pAllocator->pCurrChunk->pNext = pNewChunk; + } + pAllocator->pCurrChunk = pNewChunk; + if (NULL == pAllocator->pChunks) { + pAllocator->pChunks = pNewChunk; + } + ++(pAllocator->chunkNum); + return pNewChunk; +} + +static void* nodesCalloc(int32_t num, int32_t size) { + if (NULL == pNodeAllocator) { + return taosMemoryCalloc(num, size); + } + + if (pNodeAllocator->pCurrChunk->usedSize + size > pNodeAllocator->pCurrChunk->availableSize) { + if (NULL == callocNodeChunk(pNodeAllocator)) { + return NULL; + } + } + void* p = pNodeAllocator->pCurrChunk->pBuf + pNodeAllocator->pCurrChunk->usedSize; + pNodeAllocator->pCurrChunk->usedSize += size; + return p; +} + +static void nodesFree(void* p) { + if (NULL == pNodeAllocator) { + taosMemoryFree(p); + } + return; +} + +int32_t nodesCreateNodeAllocator(int32_t chunkSize, SNodeAllocator** pAllocator) { + *pAllocator = taosMemoryCalloc(1, sizeof(SNodeAllocator)); + if (NULL == *pAllocator) { + return TSDB_CODE_OUT_OF_MEMORY; + } + (*pAllocator)->chunkSize = chunkSize; + if (NULL == callocNodeChunk(*pAllocator)) { + taosMemoryFreeClear(*pAllocator); + return TSDB_CODE_OUT_OF_MEMORY; + } + return TSDB_CODE_SUCCESS; +} + +void nodesDestroyNodeAllocator(SNodeAllocator* pAllocator) { + if (NULL == pAllocator) { + return; + } + + nodesDebug("alloc chunkNum: %d, chunkTotakSize: %d", pAllocator->chunkNum, + pAllocator->chunkNum * pAllocator->chunkSize); + + SNodeMemChunk* pChunk = pAllocator->pChunks; + while (NULL != pChunk) { + SNodeMemChunk* pTemp = pChunk->pNext; + taosMemoryFree(pChunk); + pChunk = pTemp; + } + taosMemoryFree(pAllocator); +} + +void nodesResetThreadLevelAllocator(SNodeAllocator* pAllocator) { pNodeAllocator = pAllocator; } + +static SNode* makeNode(ENodeType type, int32_t size) { + SNode* p = nodesCalloc(1, size); if (NULL == p) { return NULL; } @@ -824,6 +914,7 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pLogicNode->pWStartTs); nodesDestroyNode(pLogicNode->pValues); nodesDestroyList(pLogicNode->pFillExprs); + nodesDestroyList(pLogicNode->pNotFillExprs); break; } case QUERY_NODE_LOGIC_PLAN_SORT: { @@ -1021,12 +1112,12 @@ void nodesDestroyNode(SNode* pNode) { default: break; } - taosMemoryFreeClear(pNode); + nodesFree(pNode); return; } SNodeList* nodesMakeList() { - SNodeList* p = taosMemoryCalloc(1, sizeof(SNodeList)); + SNodeList* p = nodesCalloc(1, sizeof(SNodeList)); if (NULL == p) { return NULL; } @@ -1037,7 +1128,7 @@ int32_t nodesListAppend(SNodeList* pList, SNode* pNode) { if (NULL == pList || NULL == pNode) { return TSDB_CODE_FAILED; } - SListCell* p = taosMemoryCalloc(1, sizeof(SListCell)); + SListCell* p = nodesCalloc(1, sizeof(SListCell)); if (NULL == p) { terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY; @@ -1104,7 +1195,7 @@ int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc) { } pTarget->pTail = pSrc->pTail; pTarget->length += pSrc->length; - taosMemoryFreeClear(pSrc); + nodesFree(pSrc); return TSDB_CODE_SUCCESS; } @@ -1124,7 +1215,7 @@ int32_t nodesListPushFront(SNodeList* pList, SNode* pNode) { if (NULL == pList || NULL == pNode) { return TSDB_CODE_FAILED; } - SListCell* p = taosMemoryCalloc(1, sizeof(SListCell)); + SListCell* p = nodesCalloc(1, sizeof(SListCell)); if (NULL == p) { terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY; @@ -1152,7 +1243,7 @@ SListCell* nodesListErase(SNodeList* pList, SListCell* pCell) { } SListCell* pNext = pCell->pNext; nodesDestroyNode(pCell->pNode); - taosMemoryFreeClear(pCell); + nodesFree(pCell); --(pList->length); return pNext; } @@ -1172,7 +1263,7 @@ void nodesListInsertList(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc) { pPos->pPrev = pSrc->pTail; pTarget->length += pSrc->length; - taosMemoryFreeClear(pSrc); + nodesFree(pSrc); } SNode* nodesListGetNode(SNodeList* pList, int32_t index) { @@ -1204,7 +1295,7 @@ void nodesDestroyList(SNodeList* pList) { while (NULL != pNext) { pNext = nodesListErase(pList, pNext); } - taosMemoryFreeClear(pList); + nodesFree(pList); } void nodesClearList(SNodeList* pList) { @@ -1216,9 +1307,9 @@ void nodesClearList(SNodeList* pList) { while (NULL != pNext) { SListCell* tmp = pNext; pNext = pNext->pNext; - taosMemoryFreeClear(tmp); + nodesFree(tmp); } - taosMemoryFreeClear(pList); + nodesFree(pList); } void* nodesGetValueFromNode(SValueNode* pNode) { diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 6f11c653a4..379bd975b4 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -247,7 +247,8 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { pExpr->userAlias[len] = '\0'; } } - taosMemoryFreeClear(pNode); + pRawExpr->pNode = NULL; + nodesDestroyNode(pNode); return pRealizedExpr; } diff --git a/source/libs/parser/test/parTestUtil.cpp b/source/libs/parser/test/parTestUtil.cpp index 360b904c17..14c991917b 100644 --- a/source/libs/parser/test/parTestUtil.cpp +++ b/source/libs/parser/test/parTestUtil.cpp @@ -119,12 +119,18 @@ class ParserTestBaseImpl { TEST_INTERFACE_ASYNC_API }; - static void _destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request) { + static void destoryParseContext(SParseContext* pCxt) { + taosArrayDestroy(pCxt->pTableMetaPos); + taosArrayDestroy(pCxt->pTableVgroupPos); + delete pCxt; + } + + static void destoryParseMetaCacheWarpper(SParseMetaCache* pMetaCache, bool request) { destoryParseMetaCache(pMetaCache, request); delete pMetaCache; } - static void _destroyQuery(SQuery** pQuery) { + static void destroyQuery(SQuery** pQuery) { if (nullptr == pQuery) { return; } @@ -303,10 +309,10 @@ class ParserTestBaseImpl { setParseContext(sql, &cxt); if (qIsInsertValuesSql(cxt.pSql, cxt.sqlLen)) { - unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), _destroyQuery); + unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery); doParseInsertSql(&cxt, query.get(), nullptr); } else { - unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), _destroyQuery); + unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery); doParse(&cxt, query.get()); SQuery* pQuery = *(query.get()); @@ -335,7 +341,7 @@ class ParserTestBaseImpl { SParseContext cxt = {0}; setParseContext(sql, &cxt); - unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), _destroyQuery); + unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery); doParseSql(&cxt, query.get()); SQuery* pQuery = *(query.get()); @@ -354,26 +360,26 @@ class ParserTestBaseImpl { void runAsyncInternalFuncs(const string& sql, int32_t expect, ParserStage checkStage) { reset(expect, checkStage, TEST_INTERFACE_ASYNC_INTERNAL); try { - SParseContext cxt = {0}; - setParseContext(sql, &cxt, true); + unique_ptr > cxt(new SParseContext(), destoryParseContext); + setParseContext(sql, cxt.get(), true); - unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), _destroyQuery); + unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery); bool request = true; unique_ptr > metaCache( - new SParseMetaCache(), bind(_destoryParseMetaCache, _1, cref(request))); - bool isInsertValues = qIsInsertValuesSql(cxt.pSql, cxt.sqlLen); + new SParseMetaCache(), bind(destoryParseMetaCacheWarpper, _1, cref(request))); + bool isInsertValues = qIsInsertValuesSql(cxt->pSql, cxt->sqlLen); if (isInsertValues) { - doParseInsertSyntax(&cxt, query.get(), metaCache.get()); + doParseInsertSyntax(cxt.get(), query.get(), metaCache.get()); } else { - doParse(&cxt, query.get()); - doCollectMetaKey(&cxt, *(query.get()), metaCache.get()); + doParse(cxt.get(), query.get()); + doCollectMetaKey(cxt.get(), *(query.get()), metaCache.get()); } SQuery* pQuery = *(query.get()); unique_ptr catalogReq(new SCatalogReq(), MockCatalogService::destoryCatalogReq); - doBuildCatalogReq(&cxt, metaCache.get(), catalogReq.get()); + doBuildCatalogReq(cxt.get(), metaCache.get(), catalogReq.get()); string err; thread t1([&]() { @@ -386,13 +392,13 @@ class ParserTestBaseImpl { doPutMetaDataToCache(catalogReq.get(), metaData.get(), metaCache.get(), isInsertValues); if (isInsertValues) { - doParseInsertSql(&cxt, query.get(), metaCache.get()); + doParseInsertSql(cxt.get(), query.get(), metaCache.get()); } else { - doAuthenticate(&cxt, pQuery, metaCache.get()); + doAuthenticate(cxt.get(), pQuery, metaCache.get()); - doTranslate(&cxt, pQuery, metaCache.get()); + doTranslate(cxt.get(), pQuery, metaCache.get()); - doCalculateConstant(&cxt, pQuery); + doCalculateConstant(cxt.get(), pQuery); } } catch (const TerminateFlag& e) { // success and terminate @@ -423,13 +429,13 @@ class ParserTestBaseImpl { void runAsyncApis(const string& sql, int32_t expect, ParserStage checkStage) { reset(expect, checkStage, TEST_INTERFACE_ASYNC_API); try { - SParseContext cxt = {0}; - setParseContext(sql, &cxt); + unique_ptr > cxt(new SParseContext(), destoryParseContext); + setParseContext(sql, cxt.get()); unique_ptr catalogReq(new SCatalogReq(), MockCatalogService::destoryCatalogReq); - unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), _destroyQuery); - doParseSqlSyntax(&cxt, query.get(), catalogReq.get()); + unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery); + doParseSqlSyntax(cxt.get(), query.get(), catalogReq.get()); SQuery* pQuery = *(query.get()); string err; @@ -438,7 +444,7 @@ class ParserTestBaseImpl { unique_ptr metaData(new SMetaData(), MockCatalogService::destoryMetaData); doGetAllMeta(catalogReq.get(), metaData.get()); - doAnalyseSqlSemantic(&cxt, catalogReq.get(), metaData.get(), pQuery); + doAnalyseSqlSemantic(cxt.get(), catalogReq.get(), metaData.get(), pQuery); } catch (const TerminateFlag& e) { // success and terminate } catch (const runtime_error& e) { diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index beb938b161..3af3786363 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -997,6 +997,7 @@ static int32_t stbSplSplitMergeScanNode(SSplitContext* pCxt, SLogicSubplan* pSub code = stbSplCreateMergeNode(pCxt, pSubplan, (SLogicNode*)pScan, pMergeKeys, pMergeScan, groupSort); } if (TSDB_CODE_SUCCESS == code) { + nodesDestroyNode((SNode*)pScan); code = nodesListMakeStrictAppend(&pSubplan->pChildren, (SNode*)splCreateScanSubplan(pCxt, pMergeScan, SPLIT_FLAG_STABLE_SPLIT)); } diff --git a/source/libs/planner/test/planTestMain.cpp b/source/libs/planner/test/planTestMain.cpp index 8f6fc832a2..e86bf90f51 100644 --- a/source/libs/planner/test/planTestMain.cpp +++ b/source/libs/planner/test/planTestMain.cpp @@ -22,6 +22,7 @@ #include "mockCatalog.h" #include "parser.h" #include "planTestUtil.h" +#include "tglobal.h" class PlannerEnv : public testing::Environment { public: @@ -30,6 +31,7 @@ class PlannerEnv : public testing::Environment { initMetaDataEnv(); generateMetaData(); initLog(TD_TMP_DIR_PATH "td"); + initCfg(); } virtual void TearDown() { @@ -67,6 +69,8 @@ class PlannerEnv : public testing::Environment { std::cout << "failed to init log file" << std::endl; } } + + void initCfg() { tsQueryPlannerTrace = true; } }; static void parseArg(int argc, char* argv[]) { @@ -79,6 +83,7 @@ static void parseArg(int argc, char* argv[]) { {"limitSql", required_argument, NULL, 'i'}, {"log", required_argument, NULL, 'l'}, {"queryPolicy", required_argument, NULL, 'q'}, + {"useNodeAllocator", required_argument, NULL, 'a'}, {0, 0, 0, 0} }; // clang-format on @@ -99,6 +104,9 @@ static void parseArg(int argc, char* argv[]) { case 'q': setQueryPolicy(optarg); break; + case 'a': + setUseNodeAllocator(optarg); + break; default: break; } diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index 2b8e3d9864..65aed6cadc 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -41,6 +41,7 @@ using namespace testing; enum DumpModule { DUMP_MODULE_NOTHING = 1, + DUMP_MODULE_SQL, DUMP_MODULE_PARSER, DUMP_MODULE_LOGIC, DUMP_MODULE_OPTIMIZED, @@ -56,10 +57,13 @@ int32_t g_skipSql = 0; int32_t g_limitSql = 0; int32_t g_logLevel = 131; int32_t g_queryPolicy = QUERY_POLICY_VNODE; +bool g_useNodeAllocator = false; void setDumpModule(const char* pModule) { if (NULL == pModule) { g_dumpModule = DUMP_MODULE_ALL; + } else if (0 == strncasecmp(pModule, "sql", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_SQL; } else if (0 == strncasecmp(pModule, "parser", strlen(pModule))) { g_dumpModule = DUMP_MODULE_PARSER; } else if (0 == strncasecmp(pModule, "logic", strlen(pModule))) { @@ -79,10 +83,11 @@ void setDumpModule(const char* pModule) { } } -void setSkipSqlNum(const char* pNum) { g_skipSql = stoi(pNum); } -void setLimitSqlNum(const char* pNum) { g_limitSql = stoi(pNum); } -void setLogLevel(const char* pLogLevel) { g_logLevel = stoi(pLogLevel); } -void setQueryPolicy(const char* pQueryPolicy) { g_queryPolicy = stoi(pQueryPolicy); } +void setSkipSqlNum(const char* pArg) { g_skipSql = stoi(pArg); } +void setLimitSqlNum(const char* pArg) { g_limitSql = stoi(pArg); } +void setLogLevel(const char* pArg) { g_logLevel = stoi(pArg); } +void setQueryPolicy(const char* pArg) { g_queryPolicy = stoi(pArg); } +void setUseNodeAllocator(const char* pArg) { g_useNodeAllocator = stoi(pArg); } int32_t getLogLevel() { return g_logLevel; } @@ -124,6 +129,12 @@ class PlannerTestBaseImpl { } void runImpl(const string& sql, int32_t queryPolicy) { + SNodeAllocator* pAllocator = NULL; + if (g_useNodeAllocator) { + nodesCreateNodeAllocator(32 * 1024, &pAllocator); + nodesResetThreadLevelAllocator(pAllocator); + } + reset(); tsQueryPolicy = queryPolicy; try { @@ -155,8 +166,13 @@ class PlannerTestBaseImpl { dump(g_dumpModule); } catch (...) { dump(DUMP_MODULE_ALL); + nodesDestroyNodeAllocator(pAllocator); + nodesResetThreadLevelAllocator(NULL); throw; } + + nodesDestroyNodeAllocator(pAllocator); + nodesResetThreadLevelAllocator(NULL); } void prepare(const string& sql) { @@ -216,6 +232,8 @@ class PlannerTestBaseImpl { doCreatePhysiPlan(&cxt, pLogicPlan, &pPlan); unique_ptr plan(pPlan, (void (*)(SQueryPlan*))nodesDestroyNode); + checkPlanMsg((SNode*)pPlan); + dump(g_dumpModule); } catch (...) { dump(DUMP_MODULE_ALL); @@ -252,7 +270,6 @@ class PlannerTestBaseImpl { string splitLogicPlan_; string scaledLogicPlan_; string physiPlan_; - string physiPlanMsg_; vector physiSubplans_; }; @@ -276,17 +293,16 @@ class PlannerTestBaseImpl { res_.splitLogicPlan_.clear(); res_.scaledLogicPlan_.clear(); res_.physiPlan_.clear(); - res_.physiPlanMsg_.clear(); res_.physiSubplans_.clear(); } void dump(DumpModule module) { - cout << "========================================== " << sqlNo_ << " sql : [" << stmtEnv_.sql_ << "]" << endl; - if (DUMP_MODULE_NOTHING == module) { return; } + cout << "========================================== " << sqlNo_ << " sql : [" << stmtEnv_.sql_ << "]" << endl; + if (DUMP_MODULE_ALL == module || DUMP_MODULE_PARSER == module) { if (res_.prepareAst_.empty()) { cout << "+++++++++++++++++++++syntax tree : " << endl; @@ -411,8 +427,6 @@ class PlannerTestBaseImpl { SNode* pSubplan; FOREACH(pSubplan, ((SNodeListNode*)pNode)->pNodeList) { res_.physiSubplans_.push_back(toString(pSubplan)); } } - res_.physiPlanMsg_ = toMsg((SNode*)(*pPlan)); - cout << "json len: " << res_.physiPlan_.length() << ", msg len: " << res_.physiPlanMsg_.length() << endl; } void setPlanContext(SQuery* pQuery, SPlanContext* pCxt) { @@ -451,27 +465,16 @@ class PlannerTestBaseImpl { string toString(const SNode* pRoot) { char* pStr = NULL; int32_t len = 0; - - auto start = chrono::steady_clock::now(); DO_WITH_THROW(nodesNodeToString, pRoot, false, &pStr, &len) - if (QUERY_NODE_PHYSICAL_PLAN == nodeType(pRoot)) { - cout << "nodesNodeToString: " - << chrono::duration_cast(chrono::steady_clock::now() - start).count() << "us" << endl; - } - string str(pStr); taosMemoryFreeClear(pStr); return str; } - string toMsg(const SNode* pRoot) { + void checkPlanMsg(const SNode* pRoot) { char* pStr = NULL; int32_t len = 0; - - auto start = chrono::steady_clock::now(); DO_WITH_THROW(nodesNodeToMsg, pRoot, &pStr, &len) - cout << "nodesNodeToMsg: " - << chrono::duration_cast(chrono::steady_clock::now() - start).count() << "us" << endl; string copyStr(pStr, len); SNode* pNode = NULL; @@ -491,9 +494,7 @@ class PlannerTestBaseImpl { nodesDestroyNode(pNode); taosMemoryFreeClear(pNewStr); - string str(pStr, len); taosMemoryFreeClear(pStr); - return str; } caseEnv caseEnv_; diff --git a/source/libs/planner/test/planTestUtil.h b/source/libs/planner/test/planTestUtil.h index b0ddd726a6..be8b51f769 100644 --- a/source/libs/planner/test/planTestUtil.h +++ b/source/libs/planner/test/planTestUtil.h @@ -41,11 +41,12 @@ class PlannerTestBase : public testing::Test { std::unique_ptr impl_; }; -extern void setDumpModule(const char* pModule); -extern void setSkipSqlNum(const char* pNum); -extern void setLimitSqlNum(const char* pNum); -extern void setLogLevel(const char* pLogLevel); -extern void setQueryPolicy(const char* pQueryPolicy); +extern void setDumpModule(const char* pArg); +extern void setSkipSqlNum(const char* pArg); +extern void setLimitSqlNum(const char* pArg); +extern void setLogLevel(const char* pArg); +extern void setQueryPolicy(const char* pArg); +extern void setUseNodeAllocator(const char* pArg); extern int32_t getLogLevel(); #endif // PLAN_TEST_UTIL_H diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index cd1f6624bd..05730c62ac 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -847,7 +847,7 @@ EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) { memcpy(res->datum.p, output.columnData->pData, len); } else if (IS_VAR_DATA_TYPE(type)) { //res->datum.p = taosMemoryCalloc(res->node.resType.bytes + VARSTR_HEADER_SIZE + 1, 1); - res->datum.p = taosMemoryCalloc(varDataTLen(output.columnData->pData), 1); + res->datum.p = taosMemoryCalloc(varDataTLen(output.columnData->pData) + 1, 1); res->node.resType.bytes = varDataTLen(output.columnData->pData); memcpy(res->datum.p, output.columnData->pData, varDataTLen(output.columnData->pData)); } else { From 3f839d683298a891d17234e1191ec7d4ee4faa24 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 20 Sep 2022 16:33:35 +0800 Subject: [PATCH 14/52] fix: wlock for meta update tag value --- source/dnode/vnode/src/meta/metaTable.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 2c11b9bf0f..b2c94369ec 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -361,8 +361,9 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { // update uid index metaUpdateUidIdx(pMeta, &nStbEntry); - if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); metaULock(pMeta); + + if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); tDecoderClear(&dc); tdbTbcClose(pTbDbc); tdbTbcClose(pUidIdxc); @@ -416,20 +417,22 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMe me.ctbEntry.pTags = pReq->ctb.pTag; #ifdef TAG_FILTER_DEBUG - SArray* pTagVals = NULL; - int32_t code = tTagToValArray((STag*)pReq->ctb.pTag, &pTagVals); + SArray *pTagVals = NULL; + int32_t code = tTagToValArray((STag *)pReq->ctb.pTag, &pTagVals); for (int i = 0; i < taosArrayGetSize(pTagVals); i++) { - STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, i); + STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i); if (IS_VAR_DATA_TYPE(pTagVal->type)) { - char* buf = taosMemoryCalloc(pTagVal->nData + 1, 1); + char *buf = taosMemoryCalloc(pTagVal->nData + 1, 1); memcpy(buf, pTagVal->pData, pTagVal->nData); - metaDebug("metaTag table:%s varchar index:%d cid:%d type:%d value:%s", pReq->name, i, pTagVal->cid, pTagVal->type, buf); + metaDebug("metaTag table:%s varchar index:%d cid:%d type:%d value:%s", pReq->name, i, pTagVal->cid, + pTagVal->type, buf); taosMemoryFree(buf); } else { double val = 0; GET_TYPED_DATA(val, double, pTagVal->type, &pTagVal->i64); - metaDebug("metaTag table:%s number index:%d cid:%d type:%d value:%f", pReq->name, i, pTagVal->cid, pTagVal->type, val); + metaDebug("metaTag table:%s number index:%d cid:%d type:%d value:%f", pReq->name, i, pTagVal->cid, + pTagVal->type, val); } } #endif @@ -919,6 +922,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA taosArrayDestroy(pTagArray); } + metaWLock(pMeta); + // save to table.db metaSaveToTbDb(pMeta, &ctbEntry); @@ -933,6 +938,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, ((STag *)(ctbEntry.ctbEntry.pTags))->len, &pMeta->txn); + metaULock(pMeta); + tDecoderClear(&dc1); tDecoderClear(&dc2); if (ctbEntry.ctbEntry.pTags) taosMemoryFree((void *)ctbEntry.ctbEntry.pTags); From 83be0489a391d15fbb80cac4368928f8522f55b8 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 20 Sep 2022 17:12:01 +0800 Subject: [PATCH 15/52] fix(query): fix percentile function crash on large data set --- source/util/src/tpagedbuf.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 2767fed937..e1a8697ad5 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -284,7 +284,6 @@ static char* evacOneDataPage(SDiskbasedBuf* pBuf) { // all pages are referenced by user, try to allocate new space if (pn == NULL) { - assert(0); int32_t prev = pBuf->inMemPages; // increase by 50% of previous mem pages @@ -304,7 +303,6 @@ static char* evacOneDataPage(SDiskbasedBuf* pBuf) { bufPage = flushPageToDisk(pBuf, d); } - ASSERT((bufPage != NULL) || terrno != TSDB_CODE_SUCCESS); return bufPage; } @@ -377,12 +375,6 @@ void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t* pageId) { char* availablePage = NULL; if (NO_IN_MEM_AVAILABLE_PAGES(pBuf)) { availablePage = evacOneDataPage(pBuf); - - // Failed to allocate a new buffer page, and there is an error occurs. - if (availablePage == NULL) { - assert(0); - return NULL; - } } SPageInfo* pi = NULL; @@ -652,4 +644,4 @@ void clearDiskbasedBuf(SDiskbasedBuf* pBuf) { pBuf->totalBufSize = 0; pBuf->allocateId = -1; pBuf->fileSize = 0; -} \ No newline at end of file +} From 0517996b97f20d87091165004b80dfc7f84c6a2f Mon Sep 17 00:00:00 2001 From: tomchon Date: Tue, 20 Sep 2022 17:20:04 +0800 Subject: [PATCH 16/52] test:add testcase of data Compatibility --- tests/parallel_test/run_container.sh | 10 +- tests/system-test/0-others/compatibility.py | 138 ++++++++++++++++++++ 2 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 tests/system-test/0-others/compatibility.py diff --git a/tests/parallel_test/run_container.sh b/tests/parallel_test/run_container.sh index f0ee9be46f..3a7193b39c 100755 --- a/tests/parallel_test/run_container.sh +++ b/tests/parallel_test/run_container.sh @@ -79,9 +79,11 @@ fi ulimit -c unlimited TMP_DIR=$WORKDIR/tmp - +SOURCEDIR=$WORKDIR/src MOUNT_DIR="" +packageName="TDengine-server-3.0.1.0-Linux-x64.tar.gz" rm -rf ${TMP_DIR}/thread_volume/$thread_no/sim +mkdir -p $SOURCEDIR mkdir -p ${TMP_DIR}/thread_volume/$thread_no/sim/tsim mkdir -p ${TMP_DIR}/thread_volume/$thread_no/coredump rm -rf ${TMP_DIR}/thread_volume/$thread_no/coredump/* @@ -90,6 +92,11 @@ if [ ! -d "${TMP_DIR}/thread_volume/$thread_no/$exec_dir" ]; then echo "cp -rf ${REPDIR}/tests/$subdir ${TMP_DIR}/thread_volume/$thread_no/" cp -rf ${REPDIR}/tests/$subdir ${TMP_DIR}/thread_volume/$thread_no/ fi + +if [ ! -d "${SOURCEDIR}/${packageName}" ]; then + wget -P ${SOURCEDIR} https://taosdata.com/assets-download/3.0/${packageName} +fi + MOUNT_DIR="$TMP_DIR/thread_volume/$thread_no/$exec_dir:$CONTAINER_TESTDIR/tests/$exec_dir" echo "$thread_no -> ${exec_dir}:$cmd" coredump_dir=`cat /proc/sys/kernel/core_pattern | xargs dirname` @@ -97,6 +104,7 @@ coredump_dir=`cat /proc/sys/kernel/core_pattern | xargs dirname` docker run \ -v $REP_MOUNT_PARAM \ -v $MOUNT_DIR \ + -v ${SOURCEDIR}:/usr/local/src/ \ -v "$TMP_DIR/thread_volume/$thread_no/sim:${SIM_DIR}" \ -v ${TMP_DIR}/thread_volume/$thread_no/coredump:$coredump_dir \ -v $WORKDIR/taos-connector-python/taos:/usr/local/lib/python3.8/site-packages/taos:ro \ diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py new file mode 100644 index 0000000000..8636274ca3 --- /dev/null +++ b/tests/system-test/0-others/compatibility.py @@ -0,0 +1,138 @@ +from urllib.parse import uses_relative +import taos +import sys +import os +import time + + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * + + +class TDTestCase: + def caseDescription(self): + ''' + 3.0 data compatibility test + case1: basedata version is 3.0.1.0 + ''' + return + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def getCfgPath(self): + buildPath = self.getBuildPath() + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + cfgPath = buildPath + "/../community/sim/dnode1/cfg/" + else: + cfgPath = buildPath + "/../sim/dnode1/cfg/" + + return cfgPath + + def installTaosd(self,bPath,cPath): + # os.system(f"rmtaos && mkdir -p {self.getBuildPath()}/build/lib/temp && mv {self.getBuildPath()}/build/lib/libtaos.so* {self.getBuildPath()}/build/lib/temp/ ") + # os.system(f" mv {bPath}/build {bPath}/build_bak ") + # os.system(f"mv {self.getBuildPath()}/build/lib/libtaos.so {self.getBuildPath()}/build/lib/libtaos.so_bak ") + # os.system(f"mv {self.getBuildPath()}/build/lib/libtaos.so.1 {self.getBuildPath()}/build/lib/libtaos.so.1_bak ") + + packagePath="/usr/local/src/" + packageName="TDengine-server-3.0.1.0-Linux-x64.tar.gz" + os.system(f"cd {packagePath} && tar xvf TDengine-server-3.0.1.0-Linux-x64.tar.gz && cd TDengine-server-3.0.1.0 && ./install.sh -e no " ) + tdDnodes.stop(1) + print(f"start taosd: nohup taosd -c {cPath} & ") + os.system(f" nohup taosd -c {cPath} & " ) + sleep(1) + + + + def buildTaosd(self,bPath): + # os.system(f"mv {bPath}/build_bak {bPath}/build ") + os.system(f" cd {bPath} && make install ") + + + def run(self): + print(f"buildpath:{self.getBuildPath()}") + bPath=self.getBuildPath() + cPath=self.getCfgPath() + dbname = "test" + stb = f"{dbname}.meters" + self.installTaosd(bPath,cPath) + tableNumbers=100 + recordNumbers1=100 + recordNumbers2=1000 + # print(tdSql) + tdsqlF=tdCom.newTdSql() + print(tdsqlF) + tdsqlF.query(f"SELECT SERVER_VERSION();") + print(tdsqlF.query(f"SELECT SERVER_VERSION();")) + oldServerVersion=tdsqlF.queryResult[0][0] + tdLog.info(f"Base server version is {oldServerVersion}") + tdsqlF.query(f"SELECT CLIENT_VERSION();") + # the oldClientVersion can't be updated in the same python process,so the version is new compiled verison + oldClientVersion=tdsqlF.queryResult[0][0] + tdLog.info(f"Base client version is {oldClientVersion}") + + tdLog.printNoPrefix(f"==========step1:prepare and check data in old version-{oldServerVersion}") + tdLog.info(f"taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ") + os.system(f"taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ") + sleep(3) + + # tdsqlF.query(f"select count(*) from {stb}") + # tdsqlF.checkData(0,0,tableNumbers*recordNumbers1) + os.system("pkill taosd") + sleep(1) + + tdLog.printNoPrefix("==========step2:update new version ") + self.buildTaosd(bPath) + tdDnodes.start(1) + sleep(1) + tdsql_new=tdCom.newTdSql() + print(tdsql_new) + + + tdsql_new.query(f"SELECT SERVER_VERSION();") + nowServerVersion=tdsql_new.queryResult[0][0] + tdLog.info(f"New server version is {nowServerVersion}") + tdsql_new.query(f"SELECT CLIENT_VERSION();") + nowClientVersion=tdsql_new.queryResult[0][0] + tdLog.info(f"New client version is {nowClientVersion}") + + tdLog.printNoPrefix(f"==========step3:prepare and check data in new version-{nowServerVersion}") + tdsql_new.query(f"select count(*) from {stb}") + tdsql_new.checkData(0,0,tableNumbers*recordNumbers1) + os.system(f"taosBenchmark -t {tableNumbers} -n {recordNumbers2} -y ") + tdsql_new.query(f"select count(*) from {stb}") + tdsql_new.checkData(0,0,tableNumbers*recordNumbers2) + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 5c7997be4b17a21cb30f887a7c6f33bd88cf31f6 Mon Sep 17 00:00:00 2001 From: tomchon Date: Tue, 20 Sep 2022 18:17:23 +0800 Subject: [PATCH 17/52] test:verify backticks with atuo creating tables --- tests/system-test/0-others/compatibility.py | 30 ++++++++++++--------- tests/system-test/fulltest.sh | 1 + 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index 8636274ca3..4dae6f06cf 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -76,7 +76,6 @@ class TDTestCase: def run(self): - print(f"buildpath:{self.getBuildPath()}") bPath=self.getBuildPath() cPath=self.getCfgPath() dbname = "test" @@ -85,7 +84,6 @@ class TDTestCase: tableNumbers=100 recordNumbers1=100 recordNumbers2=1000 - # print(tdSql) tdsqlF=tdCom.newTdSql() print(tdsqlF) tdsqlF.query(f"SELECT SERVER_VERSION();") @@ -111,24 +109,32 @@ class TDTestCase: self.buildTaosd(bPath) tdDnodes.start(1) sleep(1) - tdsql_new=tdCom.newTdSql() - print(tdsql_new) + tdsql=tdCom.newTdSql() + print(tdsql) - tdsql_new.query(f"SELECT SERVER_VERSION();") - nowServerVersion=tdsql_new.queryResult[0][0] + tdsql.query(f"SELECT SERVER_VERSION();") + nowServerVersion=tdsql.queryResult[0][0] tdLog.info(f"New server version is {nowServerVersion}") - tdsql_new.query(f"SELECT CLIENT_VERSION();") - nowClientVersion=tdsql_new.queryResult[0][0] + tdsql.query(f"SELECT CLIENT_VERSION();") + nowClientVersion=tdsql.queryResult[0][0] tdLog.info(f"New client version is {nowClientVersion}") tdLog.printNoPrefix(f"==========step3:prepare and check data in new version-{nowServerVersion}") - tdsql_new.query(f"select count(*) from {stb}") - tdsql_new.checkData(0,0,tableNumbers*recordNumbers1) + tdsql.query(f"select count(*) from {stb}") + tdsql.checkData(0,0,tableNumbers*recordNumbers1) os.system(f"taosBenchmark -t {tableNumbers} -n {recordNumbers2} -y ") - tdsql_new.query(f"select count(*) from {stb}") - tdsql_new.checkData(0,0,tableNumbers*recordNumbers2) + tdsql.query(f"select count(*) from {stb}") + tdsql.checkData(0,0,tableNumbers*recordNumbers2) + tdsql=tdCom.newTdSql() + tdLog.printNoPrefix(f"==========step4:verify backticks in taos Sql-TD18542") + tdsql.execute("create database db") + tdsql.execute("use db") + tdsql.execute("create stable db.stb1 (ts timestamp, c1 int) tags (t1 int);") + tdsql.execute("insert into db.ct1 using db.stb1 TAGS(9) values(now(),11);") + tdsql.error(" insert into `db.ct2` using db.stb1 TAGS(9) values(now(),11);") + tdsql.error(" insert into db.`db.ct2` using db.stb1 TAGS(9) values(now(),11);") def stop(self): tdSql.close() diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index f4a5c22d39..2f5cf5536e 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -17,6 +17,7 @@ python3 ./test.py -f 0-others/udf_cfg2.py python3 ./test.py -f 0-others/sysinfo.py python3 ./test.py -f 0-others/user_control.py python3 ./test.py -f 0-others/fsync.py +python3 ./test.py -f 0-others/compatibility.py python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py From 78ac5ee1314b0a4b77865826da56c591d6ace7c5 Mon Sep 17 00:00:00 2001 From: xiaolei li <85657333+xleili@users.noreply.github.com> Date: Tue, 20 Sep 2022 18:23:39 +0800 Subject: [PATCH 18/52] docs(diver):C# add docs with WebSocket (#16950) * docs(diver):C# add docs with WebSocket * docs(driver):C# docs update with commnets --- docs/examples/csharp/.gitignore | 10 +- docs/examples/csharp/wsConnect/Program.cs | 25 ++++ .../csharp/wsConnect/wsConnect.csproj | 13 ++ docs/examples/csharp/wsInsert/Program.cs | 58 +++++++++ docs/examples/csharp/wsInsert/wsInsert.csproj | 13 ++ docs/examples/csharp/wsQuery/Program.cs | 74 +++++++++++ docs/examples/csharp/wsQuery/wsQuery.csproj | 13 ++ docs/examples/csharp/wsStmt/Program.cs | 95 ++++++++++++++ docs/examples/csharp/wsStmt/wsStmt.csproj | 13 ++ docs/zh/07-develop/01-connect/_connect_cs.mdx | 6 +- docs/zh/08-connector/40-csharp.mdx | 120 +++++++++++++++++- 11 files changed, 431 insertions(+), 9 deletions(-) create mode 100644 docs/examples/csharp/wsConnect/Program.cs create mode 100644 docs/examples/csharp/wsConnect/wsConnect.csproj create mode 100644 docs/examples/csharp/wsInsert/Program.cs create mode 100644 docs/examples/csharp/wsInsert/wsInsert.csproj create mode 100644 docs/examples/csharp/wsQuery/Program.cs create mode 100644 docs/examples/csharp/wsQuery/wsQuery.csproj create mode 100644 docs/examples/csharp/wsStmt/Program.cs create mode 100644 docs/examples/csharp/wsStmt/wsStmt.csproj diff --git a/docs/examples/csharp/.gitignore b/docs/examples/csharp/.gitignore index b3aff79f37..694da603b9 100644 --- a/docs/examples/csharp/.gitignore +++ b/docs/examples/csharp/.gitignore @@ -1,4 +1,12 @@ bin obj .vs -*.sln \ No newline at end of file +*.sln +wsConnect/obj +wsInsert/obj +wsQuery/obj +wsStmt/obj +wsConnect/bin +wsInsert/bin +wsQuery/bin +wsStmt/bin \ No newline at end of file diff --git a/docs/examples/csharp/wsConnect/Program.cs b/docs/examples/csharp/wsConnect/Program.cs new file mode 100644 index 0000000000..2e89372c3e --- /dev/null +++ b/docs/examples/csharp/wsConnect/Program.cs @@ -0,0 +1,25 @@ +using System; +using TDengineWS.Impl; + +namespace Examples +{ + public class WSConnExample + { + static void Main(string[] args) + { + string DSN = "ws://root:taosdata@127.0.0.1:6041/test"; + IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN); + if (wsConn == IntPtr.Zero) + { + throw new Exception($"get WS connection failed,reason:{LibTaosWS.WSErrorStr(IntPtr.Zero)} code:{LibTaosWS.WSErrorNo(IntPtr.Zero)}"); + } + else + { + Console.WriteLine("Establish connect success."); + } + + // close connection. + LibTaosWS.WSClose(wsConn); + } + } +} \ No newline at end of file diff --git a/docs/examples/csharp/wsConnect/wsConnect.csproj b/docs/examples/csharp/wsConnect/wsConnect.csproj new file mode 100644 index 0000000000..34951dc761 --- /dev/null +++ b/docs/examples/csharp/wsConnect/wsConnect.csproj @@ -0,0 +1,13 @@ + + + + Exe + net5.0 + enable + + + + + + + diff --git a/docs/examples/csharp/wsInsert/Program.cs b/docs/examples/csharp/wsInsert/Program.cs new file mode 100644 index 0000000000..4ff830b437 --- /dev/null +++ b/docs/examples/csharp/wsInsert/Program.cs @@ -0,0 +1,58 @@ +using System; +using TDengineWS.Impl; + +namespace Examples +{ + public class WSInsertExample + { + static void Main(string[] args) + { + string DSN = "ws://root:taosdata@127.0.0.1:6041/test"; + IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN); + + // Assert if connection is validate + if (wsConn == IntPtr.Zero) + { + throw new Exception($"get WS connection failed,reason:{LibTaosWS.WSErrorStr(IntPtr.Zero)} code:{LibTaosWS.WSErrorNo(IntPtr.Zero)}"); + } + else + { + Console.WriteLine("Establish connect success."); + } + + string createTable = "CREATE STABLE test.meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);"; + string insert = "INSERT INTO test.d1001 USING test.meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000)" + + "test.d1002 USING test.meters TAGS('California.SanFrancisco', 3) VALUES('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000)" + + "test.d1003 USING test.meters TAGS('California.LosAngeles', 2) VALUES('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000)('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) " + + "test.d1004 USING test.meters TAGS('California.LosAngeles', 3) VALUES('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000)('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)"; + + IntPtr wsRes = LibTaosWS.WSQuery(wsConn, createTable); + ValidInsert("create table", wsRes); + LibTaosWS.WSFreeResult(wsRes); + + wsRes = LibTaosWS.WSQuery(wsConn, insert); + ValidInsert("insert data", wsRes); + LibTaosWS.WSFreeResult(wsRes); + + // close connection. + LibTaosWS.WSClose(wsConn); + } + + static void ValidInsert(string desc, IntPtr wsRes) + { + int code = LibTaosWS.WSErrorNo(wsRes); + if (code != 0) + { + throw new Exception($"execute SQL failed: reason: {LibTaosWS.WSErrorStr(wsRes)}, code:{code}"); + } + else + { + Console.WriteLine("{0} success affect {2} rows, cost {1} nanoseconds", desc, LibTaosWS.WSTakeTiming(wsRes),LibTaosWS.WSAffectRows(wsRes)); + } + } + } + +} +// Establish connect success. +// create table success affect 0 rows, cost 3717542 nanoseconds +// insert data success affect 8 rows, cost 2613637 nanoseconds \ No newline at end of file diff --git a/docs/examples/csharp/wsInsert/wsInsert.csproj b/docs/examples/csharp/wsInsert/wsInsert.csproj new file mode 100644 index 0000000000..34951dc761 --- /dev/null +++ b/docs/examples/csharp/wsInsert/wsInsert.csproj @@ -0,0 +1,13 @@ + + + + Exe + net5.0 + enable + + + + + + + diff --git a/docs/examples/csharp/wsQuery/Program.cs b/docs/examples/csharp/wsQuery/Program.cs new file mode 100644 index 0000000000..bf3cf2bbe2 --- /dev/null +++ b/docs/examples/csharp/wsQuery/Program.cs @@ -0,0 +1,74 @@ +using System; +using TDengineWS.Impl; +using System.Collections.Generic; +using TDengineDriver; + +namespace Examples +{ + public class WSQueryExample + { + static void Main(string[] args) + { + string DSN = "ws://root:taosdata@127.0.0.1:6041/test"; + IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN); + if (wsConn == IntPtr.Zero) + { + throw new Exception($"get WS connection failed,reason:{LibTaosWS.WSErrorStr(IntPtr.Zero)} code:{LibTaosWS.WSErrorNo(IntPtr.Zero)}"); + } + else + { + Console.WriteLine("Establish connect success."); + } + + string select = "select * from test.meters"; + + // optional:wsRes = LibTaosWS.WSQuery(wsConn, select); + IntPtr wsRes = LibTaosWS.WSQueryTimeout(wsConn, select, 1); + // Assert if query execute success. + int code = LibTaosWS.WSErrorNo(wsRes); + if (code != 0) + { + throw new Exception($"execute SQL failed: reason: {LibTaosWS.WSErrorStr(wsRes)}, code:{code}"); + } + + // get meta data + List metas = LibTaosWS.WSGetFields(wsRes); + // get retrieved data + List dataSet = LibTaosWS.WSGetData(wsRes); + + // do something with result. + foreach (var meta in metas) + { + Console.Write("{0} {1}({2}) \t|\t", meta.name, meta.TypeName(), meta.size); + } + Console.WriteLine(""); + + for (int i = 0; i < dataSet.Count;) + { + for (int j = 0; j < metas.Count; j++) + { + Console.Write("{0}\t|\t", dataSet[i]); + i++; + } + Console.WriteLine(""); + } + + // Free result after use. + LibTaosWS.WSFreeResult(wsRes); + + // close connection. + LibTaosWS.WSClose(wsConn); + } + } +} + +// Establish connect success. +// ts TIMESTAMP(8) | current FLOAT(4) | voltage INT(4) | phase FLOAT(4) | location BINARY(64) | groupid INT(4) | +// 1538548685000 | 10.8 | 223 | 0.29 | California.LosAngeles | 3 | +// 1538548686500 | 11.5 | 221 | 0.35 | California.LosAngeles | 3 | +// 1538548685500 | 11.8 | 221 | 0.28 | California.LosAngeles | 2 | +// 1538548696600 | 13.4 | 223 | 0.29 | California.LosAngeles | 2 | +// 1538548685000 | 10.3 | 219 | 0.31 | California.SanFrancisco | 2 | +// 1538548695000 | 12.6 | 218 | 0.33 | California.SanFrancisco | 2 | +// 1538548696800 | 12.3 | 221 | 0.31 | California.SanFrancisco | 2 | +// 1538548696650 | 10.3 | 218 | 0.25 | California.SanFrancisco | 3 | \ No newline at end of file diff --git a/docs/examples/csharp/wsQuery/wsQuery.csproj b/docs/examples/csharp/wsQuery/wsQuery.csproj new file mode 100644 index 0000000000..34951dc761 --- /dev/null +++ b/docs/examples/csharp/wsQuery/wsQuery.csproj @@ -0,0 +1,13 @@ + + + + Exe + net5.0 + enable + + + + + + + diff --git a/docs/examples/csharp/wsStmt/Program.cs b/docs/examples/csharp/wsStmt/Program.cs new file mode 100644 index 0000000000..54de77ec1f --- /dev/null +++ b/docs/examples/csharp/wsStmt/Program.cs @@ -0,0 +1,95 @@ +using System; +using TDengineWS.Impl; +using TDengineDriver; +using System.Runtime.InteropServices; + +namespace Examples +{ + public class WSStmtExample + { + static void Main(string[] args) + { + const string DSN = "ws://root:taosdata@127.0.0.1:6041/test"; + const string table = "meters"; + const string database = "test"; + const string childTable = "d1005"; + string insert = $"insert into ? using {database}.{table} tags(?,?) values(?,?,?,?)"; + const int numOfTags = 2; + const int numOfColumns = 4; + + // Establish connection + IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN); + if (wsConn == IntPtr.Zero) + { + throw new Exception($"get WS connection failed,reason:{LibTaosWS.WSErrorStr(IntPtr.Zero)} code:{LibTaosWS.WSErrorNo(IntPtr.Zero)}"); + } + else + { + Console.WriteLine("Establish connect success..."); + } + + // init stmt + IntPtr wsStmt = LibTaosWS.WSStmtInit(wsConn); + if (wsStmt != IntPtr.Zero) + { + int code = LibTaosWS.WSStmtPrepare(wsStmt, insert); + ValidStmtStep(code, wsStmt, "WSStmtPrepare"); + + TAOS_MULTI_BIND[] wsTags = new TAOS_MULTI_BIND[] { WSMultiBind.WSBindNchar(new string[] { "California.SanDiego" }), WSMultiBind.WSBindInt(new int?[] { 4 }) }; + code = LibTaosWS.WSStmtSetTbnameTags(wsStmt, $"{database}.{childTable}", wsTags, numOfTags); + ValidStmtStep(code, wsStmt, "WSStmtSetTbnameTags"); + + TAOS_MULTI_BIND[] data = new TAOS_MULTI_BIND[4]; + data[0] = WSMultiBind.WSBindTimestamp(new long[] { 1538548687000, 1538548688000, 1538548689000, 1538548690000, 1538548691000 }); + data[1] = WSMultiBind.WSBindFloat(new float?[] { 10.30F, 10.40F, 10.50F, 10.60F, 10.70F }); + data[2] = WSMultiBind.WSBindInt(new int?[] { 223, 221, 222, 220, 219 }); + data[3] = WSMultiBind.WSBindFloat(new float?[] { 0.31F, 0.32F, 0.33F, 0.35F, 0.28F }); + code = LibTaosWS.WSStmtBindParamBatch(wsStmt, data, numOfColumns); + ValidStmtStep(code, wsStmt, "WSStmtBindParamBatch"); + + code = LibTaosWS.WSStmtAddBatch(wsStmt); + ValidStmtStep(code, wsStmt, "WSStmtAddBatch"); + + IntPtr stmtAffectRowPtr = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(Int32))); + code = LibTaosWS.WSStmtExecute(wsStmt, stmtAffectRowPtr); + ValidStmtStep(code, wsStmt, "WSStmtExecute"); + Console.WriteLine("WS STMT insert {0} rows...", Marshal.ReadInt32(stmtAffectRowPtr)); + Marshal.FreeHGlobal(stmtAffectRowPtr); + + LibTaosWS.WSStmtClose(wsStmt); + + // Free unmanaged memory + WSMultiBind.WSFreeTaosBind(wsTags); + WSMultiBind.WSFreeTaosBind(data); + + //check result with SQL "SELECT * FROM test.d1005;" + } + else + { + throw new Exception("Init STMT failed..."); + } + + // close connection. + LibTaosWS.WSClose(wsConn); + } + + static void ValidStmtStep(int code, IntPtr wsStmt, string desc) + { + if (code != 0) + { + throw new Exception($"{desc} failed,reason: {LibTaosWS.WSErrorStr(wsStmt)}, code: {code}"); + } + else + { + Console.WriteLine("{0} success...", desc); + } + } + } +} + +// WSStmtPrepare success... +// WSStmtSetTbnameTags success... +// WSStmtBindParamBatch success... +// WSStmtAddBatch success... +// WSStmtExecute success... +// WS STMT insert 5 rows... \ No newline at end of file diff --git a/docs/examples/csharp/wsStmt/wsStmt.csproj b/docs/examples/csharp/wsStmt/wsStmt.csproj new file mode 100644 index 0000000000..34951dc761 --- /dev/null +++ b/docs/examples/csharp/wsStmt/wsStmt.csproj @@ -0,0 +1,13 @@ + + + + Exe + net5.0 + enable + + + + + + + diff --git a/docs/zh/07-develop/01-connect/_connect_cs.mdx b/docs/zh/07-develop/01-connect/_connect_cs.mdx index 13b8a5dff2..9d0755fc64 100644 --- a/docs/zh/07-develop/01-connect/_connect_cs.mdx +++ b/docs/zh/07-develop/01-connect/_connect_cs.mdx @@ -2,7 +2,7 @@ {{#include docs/examples/csharp/ConnectExample.cs}} ``` -:::info -C# 连接器目前只支持原生连接。 +```csharp title="WebSocket 连接" +{{#include docs/examples/csharp/wsConnect/Program.cs}} +``` -::: diff --git a/docs/zh/08-connector/40-csharp.mdx b/docs/zh/08-connector/40-csharp.mdx index 4e49d84835..9ba8be2c22 100644 --- a/docs/zh/08-connector/40-csharp.mdx +++ b/docs/zh/08-connector/40-csharp.mdx @@ -17,7 +17,7 @@ import CSAsyncQuery from "../07-develop/04-query-data/_cs_async.mdx" `TDengine.Connector` 是 TDengine 提供的 C# 语言连接器。C# 开发人员可以通过它开发存取 TDengine 集群数据的 C# 应用软件。 -`TDengine.Connector` 连接器支持通过 TDengine 客户端驱动（taosc）建立与 TDengine 运行实例的连接，提供数据写入、查询、订阅、schemaless 数据写入、参数绑定接口数据写入等功能 `TDengine.Connector` 目前暂未提供 REST 连接方式，用户可以参考 [REST API](../rest-api/) 文档自行编写。 +`TDengine.Connector` 连接器支持通过 TDengine 客户端驱动（taosc）建立与 TDengine 运行实例的连接，提供数据写入、查询、数据订阅、schemaless 数据写入、参数绑定接口数据写入等功能。 `TDengine.Connector` 还支持 WebSocket，通过 DSN 建立 WebSocket 连接，提供数据写入、查询、参数绑定接口数据写入等功能。 本文介绍如何在 Linux 或 Windows 环境中安装 `TDengine.Connector`，并通过 `TDengine.Connector` 连接 TDengine 集群，进行数据写入、查询等基本操作。 @@ -35,12 +35,29 @@ import CSAsyncQuery from "../07-develop/04-query-data/_cs_async.mdx" ## 支持的功能特性 + + + + 1. 连接管理 2. 普通查询 3. 连续查询 4. 参数绑定 -5. 订阅功能 +5. 数据订阅（TMQ） 6. Schemaless + + + + + +1. 连接管理 +2. 普通查询 +3. 连续查询 +4. 参数绑定 + + + + ## 安装步骤 @@ -79,7 +96,13 @@ dotnet add exmaple.csproj reference src/TDengine.csproj ## 建立连接 -``` C# + + + + +使用 host、username、password、port 等信息建立连接。 + +``` csharp using TDengineDriver; namespace TDengineExample @@ -109,17 +132,63 @@ namespace TDengineExample } } } - ``` + + + + +使用 DSN 建立 WebSocket 连接 DSN 连接。 描述字符串基本结构如下： + +```text +[]://[[:@]:][/][?=[&=]] +|------------|---|-----------|-----------|------|------|------------|-----------------------| +| protocol | | username | password | host | port | database | params | +``` + +各部分意义见下表： + +* **protocol**: 显示指定以何种方式建立连接，例如：`ws://localhost:6041` 指定以 Websocket 方式建立连接(支持http/ws)。 + +* **username/password**: 用于创建连接的用户名及密码（默认`root/taosdata`）。 + +* **host/port**: 指定创建连接的服务器及端口，WebSocket 连接默认为 `localhost:6041` 。 + +* **database**: 指定默认连接的数据库名。 + +* **params**：其他可选参数。 + +``` csharp +{{#include docs/examples/csharp/wsConnect/Program.cs}} +``` + + + + ## 使用示例 ### 写入数据 #### SQL 写入 + + + + + + + + +```csharp +{{#include docs/examples/csharp/wsInsert/Program.cs}} +``` + + + + + #### InfluxDB 行协议写入 @@ -132,12 +201,50 @@ namespace TDengineExample +#### 参数绑定 + + + + + +``` csharp +{{#include docs/examples/csharp/StmtInsertExample.cs}} +``` + + + + + +```csharp +{{#include docs/examples/csharp/wsStmt/Program.cs}} +``` + + + + + ### 查询数据 #### 同步查询 + + + + + + + + +```csharp +{{#include docs/examples/csharp/wsQuery/Program.cs}} +``` + + + + + #### 异步查询 @@ -151,12 +258,15 @@ namespace TDengineExample | [stmt](https://github.com/taosdata/taos-connector-dotnet/tree/3.0/examples/Stmt) | 使用 TDengine.Connector 实现的参数绑定插入和查询的示例 | | [schemaless](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/schemaless) | 使用 TDengine.Connector 实现的使用 schemaless 写入的示例 | | [async query](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/AsyncQuery/QueryAsync.cs) | 使用 TDengine.Connector 实现的异步查询的示例 | -| [TMQ](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/TMQ/TMQ.cs) | 使用 TDengine.Connector 实现的订阅数据的示例 | +| [数据订阅（TMQ）](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/TMQ/TMQ.cs) | 使用 TDengine.Connector 实现的订阅数据的示例 | +| [Basic WebSocket Usage](https://github.com/taosdata/taos-connector-dotnet/blob/5a4a7cd0dbcda114447cdc6d0c6dedd8e84a52da/examples/WS/WebSocketSample.cs) | 使用 TDengine.Connector 的 WebSocket 基本的示例 | +| [Basic WebSocket STMT](https://github.com/taosdata/taos-connector-dotnet/blob/5a4a7cd0dbcda114447cdc6d0c6dedd8e84a52da/examples/WS/WebSocketSTMT.cs) | 使用 TDengine.Connector 的 WebSocket STMT 基本的示例 | ## 重要更新记录 | TDengine.Connector | 说明 | |--------------------|--------------------------------| +| 3.0.1 | 支持 WebSocket 和 Cloud，查询，插入，参数绑定。 | | 3.0.0 | 支持 TDengine 3.0.0.0，不兼容 2.x。新增接口TDengine.Impl.GetData()，解析查询结果。 | | 1.0.7 | 修复 TDengine.Query()内存泄露。 | | 1.0.6 | 修复 schemaless 在 1.0.4 和 1.0.5 中失效 bug。 | From 4d2fb860bac2478c8ec89bc298dcf5b071040686 Mon Sep 17 00:00:00 2001 From: Linhe Huo Date: Tue, 20 Sep 2022 18:32:06 +0800 Subject: [PATCH 19/52] chore: update taosws (#16956) --- cmake/taosws_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in index 04b1262caf..627cd53c09 100644 --- a/cmake/taosws_CMakeLists.txt.in +++ b/cmake/taosws_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosws-rs ExternalProject_Add(taosws-rs GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git - GIT_TAG e771403 + GIT_TAG 76bc64d SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From dfed091937a4a94f97e7d33afd3e99233a4950f3 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 20 Sep 2022 18:50:23 +0800 Subject: [PATCH 20/52] fix(sync): wait until apply queue is empty when restore --- source/dnode/vnode/src/vnd/vnodeSync.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 65d4e9aaf1..a941b5955c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -676,6 +676,18 @@ static void vnodeLeaderTransfer(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsm static void vnodeRestoreFinish(struct SSyncFSM *pFsm) { SVnode *pVnode = pFsm->data; + + do { + int32_t itemSize = tmsgGetQueueSize(&pVnode->msgCb, pVnode->config.vgId, APPLY_QUEUE); + if (itemSize == 0) { + vInfo("vgId:%d, apply queue is empty, restore finish", pVnode->config.vgId); + break; + } else { + vInfo("vgId:%d, restore not finish since %d items in apply queue", pVnode->config.vgId); + taosMsleep(10); + } + } while (true); + pVnode->restored = true; vDebug("vgId:%d, sync restore finished", pVnode->config.vgId); } From 46b0c2c9c4f597adf0ea811f450e3c8826fcde5b Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 20 Sep 2022 18:52:45 +0800 Subject: [PATCH 21/52] enh: added memory allocators for parser and planner --- include/common/tglobal.h | 2 ++ source/client/inc/clientInt.h | 1 + source/client/src/clientEnv.c | 1 + source/client/src/clientImpl.c | 11 +++++++ source/client/src/clientMain.c | 7 ++++ source/common/src/tglobal.c | 44 ++++++++++++++++---------- source/libs/nodes/src/nodesUtilFuncs.c | 18 ++++++++--- 7 files changed, 63 insertions(+), 21 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 2de4ffdc17..234937ea7e 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -95,6 +95,8 @@ extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in extern int32_t tsQueryPolicy; extern int32_t tsQuerySmaOptimize; extern bool tsQueryPlannerTrace; +extern int32_t tsQueryNodeChunkSize; +extern bool tsQueryUseNodeAllocator; // client extern int32_t tsMinSlidingTime; diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index b8fa9580e7..f042c3ad7c 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -250,6 +250,7 @@ typedef struct SRequestObj { bool inRetry; uint32_t prevCode; // previous error code: todo refactor, add update flag for catalog uint32_t retry; + SNodeAllocator* pNodeAllocator; } SRequestObj; typedef struct SSyncQueryParam { diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index b739aedca0..8fded03472 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -349,6 +349,7 @@ void doDestroyRequest(void *p) { taosArrayDestroy(pRequest->tableList); taosArrayDestroy(pRequest->dbList); taosArrayDestroy(pRequest->targetTableList); + nodesDestroyNodeAllocator(pRequest->pNodeAllocator); destroyQueryExecRes(&pRequest->body.resInfo.execRes); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 0e1d82b273..bc1cdc247e 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -194,6 +194,17 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, return TSDB_CODE_TSC_OUT_OF_MEMORY; } + if (tsQueryUseNodeAllocator && !qIsInsertValuesSql((*pRequest)->sqlstr, (*pRequest)->sqlLen)) { + if (TSDB_CODE_SUCCESS != nodesCreateNodeAllocator(tsQueryNodeChunkSize, &((*pRequest)->pNodeAllocator))) { + tscError("%d failed to create node allocator, reqId:0x%" PRIx64 ", conn:%d, %s", (*pRequest)->self, + (*pRequest)->requestId, pTscObj->id, sql); + + destroyRequest(*pRequest); + *pRequest = NULL; + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } + tscDebugL("0x%" PRIx64 " SQL: %s, reqId:0x%" PRIx64, (*pRequest)->self, (*pRequest)->sqlstr, (*pRequest)->requestId); return TSDB_CODE_SUCCESS; } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 73636e7372..e19d88fcf3 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -700,6 +700,8 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { pRequest->metric.ctgEnd = taosGetTimestampUs(); + nodesResetThreadLevelAllocator(pRequest->pNodeAllocator); + if (code == TSDB_CODE_SUCCESS) { code = qAnalyseSqlSemantic(pWrapper->pCtx, &pWrapper->catalogReq, pResultMeta, pQuery); pRequest->stableQuery = pQuery->stableQuery; @@ -726,9 +728,11 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { pRequest->requestId); launchAsyncQuery(pRequest, pQuery, pResultMeta); qDestroyQuery(pQuery); + nodesResetThreadLevelAllocator(NULL); } else { destorySqlParseWrapper(pWrapper); qDestroyQuery(pQuery); + nodesResetThreadLevelAllocator(NULL); if (NEED_CLIENT_HANDLE_ERROR(code)) { tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); @@ -801,6 +805,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { } SQuery *pQuery = NULL; + nodesResetThreadLevelAllocator(pRequest->pNodeAllocator); pRequest->metric.syntaxStart = taosGetTimestampUs(); @@ -844,6 +849,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { &pRequest->body.queryJob); pCxt = NULL; if (code == TSDB_CODE_SUCCESS) { + nodesResetThreadLevelAllocator(NULL); return; } @@ -851,6 +857,7 @@ _error: tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->requestId); taosMemoryFree(pCxt); + nodesResetThreadLevelAllocator(NULL); terrno = code; pRequest->code = code; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index ddda8f8c9a..e119dae743 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -92,6 +92,8 @@ bool tsSmlDataFormat = int32_t tsQueryPolicy = 1; int32_t tsQuerySmaOptimize = 0; bool tsQueryPlannerTrace = false; +int32_t tsQueryNodeChunkSize = 32 * 1024; +bool tsQueryUseNodeAllocator = true; /* * denote if the server needs to compress response message at the application layer to client, including query rsp, @@ -284,6 +286,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1; if (cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, 1) != 0) return -1; if (cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, true) != 0) return -1; + if (cfgAddInt32(pCfg, "queryNodeChunkSize", tsQueryNodeChunkSize, 1024, 128 * 1024, true) != 0) return -1; + if (cfgAddBool(pCfg, "queryUseNodeAllocator", tsQueryUseNodeAllocator, true) != 0) return -1; if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1; if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1; if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1; @@ -385,9 +389,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfQnodeQueryThreads = TMAX(tsNumOfQnodeQueryThreads, 4); if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 1, 1024, 0) != 0) return -1; -// tsNumOfQnodeFetchThreads = tsNumOfCores / 2; -// tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4); -// if (cfgAddInt32(pCfg, "numOfQnodeFetchThreads", tsNumOfQnodeFetchThreads, 1, 1024, 0) != 0) return -1; + // tsNumOfQnodeFetchThreads = tsNumOfCores / 2; + // tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4); + // if (cfgAddInt32(pCfg, "numOfQnodeFetchThreads", tsNumOfQnodeFetchThreads, 1, 1024, 0) != 0) return -1; tsNumOfSnodeSharedThreads = tsNumOfCores / 4; tsNumOfSnodeSharedThreads = TRANGE(tsNumOfSnodeSharedThreads, 2, 4); @@ -527,15 +531,15 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) { pItem->stype = stype; } -/* - pItem = cfgGetItem(tsCfg, "numOfQnodeFetchThreads"); - if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { - tsNumOfQnodeFetchThreads = numOfCores / 2; - tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4); - pItem->i32 = tsNumOfQnodeFetchThreads; - pItem->stype = stype; - } -*/ + /* + pItem = cfgGetItem(tsCfg, "numOfQnodeFetchThreads"); + if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { + tsNumOfQnodeFetchThreads = numOfCores / 2; + tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4); + pItem->i32 = tsNumOfQnodeFetchThreads; + pItem->stype = stype; + } + */ pItem = cfgGetItem(tsCfg, "numOfSnodeSharedThreads"); if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { @@ -643,6 +647,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32; tsQuerySmaOptimize = cfgGetItem(pCfg, "querySmaOptimize")->i32; tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval; + tsQueryNodeChunkSize = cfgGetItem(pCfg, "queryNodeChunkSize")->i32; + tsQueryUseNodeAllocator = cfgGetItem(pCfg, "queryUseNodeAllocator")->bval; return 0; } @@ -693,7 +699,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsNumOfVnodeSyncThreads = cfgGetItem(pCfg, "numOfVnodeSyncThreads")->i32; tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32; tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32; -// tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32; + // tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32; tsNumOfSnodeSharedThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32; tsNumOfSnodeUniqueThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32; tsRpcQueueMemoryAllowed = cfgGetItem(pCfg, "rpcQueueMemoryAllowed")->i64; @@ -941,10 +947,10 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32; } else if (strcasecmp("numOfQnodeQueryThreads", name) == 0) { tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32; -/* - } else if (strcasecmp("numOfQnodeFetchThreads", name) == 0) { - tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32; -*/ + /* + } else if (strcasecmp("numOfQnodeFetchThreads", name) == 0) { + tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32; + */ } else if (strcasecmp("numOfSnodeSharedThreads", name) == 0) { tsNumOfSnodeSharedThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32; } else if (strcasecmp("numOfSnodeUniqueThreads", name) == 0) { @@ -976,6 +982,10 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32; } else if (strcasecmp("queryPlannerTrace", name) == 0) { tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval; + } else if (strcasecmp("queryNodeChunkSize", name) == 0) { + tsQueryNodeChunkSize = cfgGetItem(pCfg, "queryNodeChunkSize")->i32; + } else if (strcasecmp("queryUseNodeAllocator", name) == 0) { + tsQueryUseNodeAllocator = cfgGetItem(pCfg, "queryUseNodeAllocator")->bval; } break; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index be94495856..8b9b723a2a 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -58,9 +58,9 @@ static SNodeMemChunk* callocNodeChunk(SNodeAllocator* pAllocator) { return pNewChunk; } -static void* nodesCalloc(int32_t num, int32_t size) { +static void* nodesCallocImpl(int32_t size) { if (NULL == pNodeAllocator) { - return taosMemoryCalloc(num, size); + return taosMemoryCalloc(1, size); } if (pNodeAllocator->pCurrChunk->usedSize + size > pNodeAllocator->pCurrChunk->availableSize) { @@ -73,9 +73,19 @@ static void* nodesCalloc(int32_t num, int32_t size) { return p; } +static void* nodesCalloc(int32_t num, int32_t size) { + void* p = nodesCallocImpl(num * size + 1); + if (NULL == p) { + return NULL; + } + *(char*)p = (NULL != pNodeAllocator) ? 1 : 0; + return (char*)p + 1; +} + static void nodesFree(void* p) { - if (NULL == pNodeAllocator) { - taosMemoryFree(p); + char* ptr = (char*)p - 1; + if (0 == *ptr) { + taosMemoryFree(ptr); } return; } From 9c84d59297a5e5835bdf0ee3e5b90b2e3578dfe5 Mon Sep 17 00:00:00 2001 From: tomchon Date: Tue, 20 Sep 2022 19:52:10 +0800 Subject: [PATCH 22/52] test:add testcase of data Compatibility --- tests/parallel_test/run_container.sh | 2 +- tests/system-test/0-others/compatibility.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/parallel_test/run_container.sh b/tests/parallel_test/run_container.sh index 3a7193b39c..bb57f238f0 100755 --- a/tests/parallel_test/run_container.sh +++ b/tests/parallel_test/run_container.sh @@ -93,7 +93,7 @@ if [ ! -d "${TMP_DIR}/thread_volume/$thread_no/$exec_dir" ]; then cp -rf ${REPDIR}/tests/$subdir ${TMP_DIR}/thread_volume/$thread_no/ fi -if [ ! -d "${SOURCEDIR}/${packageName}" ]; then +if [ ! -f "${SOURCEDIR}/${packageName}" ]; then wget -P ${SOURCEDIR} https://taosdata.com/assets-download/3.0/${packageName} fi diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index 4dae6f06cf..2b059615e9 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -129,12 +129,22 @@ class TDTestCase: tdsql=tdCom.newTdSql() tdLog.printNoPrefix(f"==========step4:verify backticks in taos Sql-TD18542") + tdsql.execute("drop database if exists db") tdsql.execute("create database db") tdsql.execute("use db") tdsql.execute("create stable db.stb1 (ts timestamp, c1 int) tags (t1 int);") - tdsql.execute("insert into db.ct1 using db.stb1 TAGS(9) values(now(),11);") + tdsql.execute("insert into db.ct1 using db.stb1 TAGS(1) values(now(),11);") tdsql.error(" insert into `db.ct2` using db.stb1 TAGS(9) values(now(),11);") tdsql.error(" insert into db.`db.ct2` using db.stb1 TAGS(9) values(now(),11);") + tdsql.execute("insert into `db`.ct3 using db.stb1 TAGS(3) values(now(),13);") + tdsql.query("select * from db.ct3") + tdsql.checkData(0,1,13) + tdsql.execute("insert into db.`ct4` using db.stb1 TAGS(4) values(now(),14);") + tdsql.query("select * from db.ct4") + tdsql.checkData(0,1,14) + tdsql.query("describe information_schema.ins_databases;") + # for i in tdsql.queryResult[i][0]: + # tdsql.checkData(0,1,13) def stop(self): tdSql.close() From 8bc6a6297f73564060488bb8af2fd33819759d42 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 20 Sep 2022 19:57:02 +0800 Subject: [PATCH 23/52] enh(sync): log entry cache for sync --- include/libs/sync/sync.h | 4 ++- source/libs/sync/src/syncCommit.c | 25 +++++++++++--- source/libs/sync/src/syncMain.c | 52 ++++++++++++++++++++++++++---- source/libs/sync/src/syncRaftLog.c | 17 +++++++++- 4 files changed, 85 insertions(+), 13 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index e6a4dd1d49..285e079b3e 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -22,6 +22,7 @@ extern "C" { #include "cJSON.h" #include "tdef.h" +#include "tlrucache.h" #include "tmsgcb.h" extern bool gRaftDetailLog; @@ -153,7 +154,8 @@ typedef struct SSyncFSM { // abstract definition of log store in raft // SWal implements it typedef struct SSyncLogStore { - void* data; + SLRUCache* pCache; + void* data; // append one log entry int32_t (*appendEntry)(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry); diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 1e68fe346c..b604d25816 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -69,15 +69,26 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { if (agree) { // term - SSyncRaftEntry* pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, index); - ASSERT(pEntry != NULL); - + SSyncRaftEntry* pEntry = NULL; + SLRUCache* pCache = pSyncNode->pLogStore->pCache; + LRUHandle* h = taosLRUCacheLookup(pCache, &index, sizeof(index)); + if (h) { + pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h); + } else { + pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, index); + ASSERT(pEntry != NULL); + } // cannot commit, even if quorum agree. need check term! if (pEntry->term <= pSyncNode->pRaftStore->currentTerm) { // update commit index newCommitIndex = index; - syncEntryDestory(pEntry); + if (h) { + taosLRUCacheRelease(pCache, h, false); + } else { + syncEntryDestory(pEntry); + } + break; } else { do { @@ -88,7 +99,11 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { } while (0); } - syncEntryDestory(pEntry); + if (h) { + taosLRUCacheRelease(pCache, h, false); + } else { + syncEntryDestory(pEntry); + } } } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 6f29b54f80..17157fbd23 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2581,6 +2581,20 @@ static int32_t syncNodeEqNoop(SSyncNode* ths) { return ret; } +static void deleteCacheEntry(const void* key, size_t keyLen, void* value) { taosMemoryFree(value); } + +static int32_t syncCacheEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, LRUHandle** h) { + int code = 0; + int entryLen = sizeof(*pEntry) + pEntry->dataLen; + LRUStatus status = taosLRUCacheInsert(pLogStore->pCache, &pEntry->index, sizeof(pEntry->index), pEntry, entryLen, + deleteCacheEntry, h, TAOS_LRU_PRIORITY_LOW); + if (status != TAOS_LRU_STATUS_OK) { + code = -1; + } + + return code; +} + static int32_t syncNodeAppendNoop(SSyncNode* ths) { int32_t ret = 0; @@ -2589,13 +2603,21 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) { SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index, ths->vgId); ASSERT(pEntry != NULL); + LRUHandle* h = NULL; + syncCacheEntry(ths->pLogStore, pEntry, &h); + if (ths->state == TAOS_SYNC_STATE_LEADER) { int32_t code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); ASSERT(code == 0); syncNodeReplicate(ths, false); } - syncEntryDestory(pEntry); + if (h) { + taosLRUCacheRelease(ths->pLogStore->pCache, h, false); + } else { + syncEntryDestory(pEntry); + } + return ret; } @@ -2654,6 +2676,9 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncI SSyncRaftEntry* pEntry = syncEntryBuild2((SyncClientRequest*)pMsg, term, index); ASSERT(pEntry != NULL); + LRUHandle* h = NULL; + syncCacheEntry(ths->pLogStore, pEntry, &h); + if (ths->state == TAOS_SYNC_STATE_LEADER) { // append entry code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); @@ -2685,7 +2710,12 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncI } } - syncEntryDestory(pEntry); + if (h) { + taosLRUCacheRelease(ths->pLogStore->pCache, h, false); + } else { + syncEntryDestory(pEntry); + } + return ret; } @@ -2973,9 +3003,15 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, for (SyncIndex i = beginIndex; i <= endIndex; ++i) { if (i != SYNC_INDEX_INVALID) { SSyncRaftEntry* pEntry; - code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, i, &pEntry); - ASSERT(code == 0); - ASSERT(pEntry != NULL); + SLRUCache* pCache = ths->pLogStore->pCache; + LRUHandle* h = taosLRUCacheLookup(pCache, &i, sizeof(i)); + if (h) { + pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h); + } else { + code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, i, &pEntry); + ASSERT(code == 0); + ASSERT(pEntry != NULL); + } SRpcMsg rpcMsg; syncEntry2OriginalRpc(pEntry, &rpcMsg); @@ -3058,7 +3094,11 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, } rpcFreeCont(rpcMsg.pCont); - syncEntryDestory(pEntry); + if (h) { + taosLRUCacheRelease(pCache, h, false); + } else { + syncEntryDestory(pEntry); + } } } } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 0649e064e4..496c8419de 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -53,6 +53,15 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { SSyncLogStore* pLogStore = taosMemoryMalloc(sizeof(SSyncLogStore)); ASSERT(pLogStore != NULL); + pLogStore->pCache = taosLRUCacheInit(10 * 1024 * 1024, 1, .5); + if (pLogStore->pCache == NULL) { + terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + taosMemoryFree(pLogStore); + return NULL; + } + + taosLRUCacheSetStrictCapacity(pLogStore->pCache, false); + pLogStore->data = taosMemoryMalloc(sizeof(SSyncLogStoreData)); ASSERT(pLogStore->data != NULL); @@ -102,6 +111,10 @@ void logStoreDestory(SSyncLogStore* pLogStore) { taosThreadMutexDestroy(&(pData->mutex)); taosMemoryFree(pLogStore->data); + + taosLRUCacheEraseUnrefEntries(pLogStore->pCache); + taosLRUCacheCleanup(pLogStore->pCache); + taosMemoryFree(pLogStore); } } @@ -243,7 +256,7 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - int32_t code; + int32_t code = 0; *ppEntry = NULL; @@ -257,6 +270,7 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, taosThreadMutexLock(&(pData->mutex)); code = walReadVer(pWalHandle, index); + // code = walReadVerCached(pWalHandle, index); if (code != 0) { int32_t err = terrno; const char* errStr = tstrerror(err); @@ -412,6 +426,7 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { ASSERT(pWalHandle != NULL); int32_t code = walReadVer(pWalHandle, index); + // int32_t code = walReadVerCached(pWalHandle, index); if (code != 0) { int32_t err = terrno; const char* errStr = tstrerror(err); From 5d497d4db75a7de0b4a897ad237d547a3370c3b1 Mon Sep 17 00:00:00 2001 From: xleili Date: Tue, 20 Sep 2022 21:55:53 +0800 Subject: [PATCH 24/52] fix(test):uncomment C# docs-example temporily --- tests/docs-examples-test/csharp.sh | 31 +++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/tests/docs-examples-test/csharp.sh b/tests/docs-examples-test/csharp.sh index a8f1ce4119..d7f2670478 100644 --- a/tests/docs-examples-test/csharp.sh +++ b/tests/docs-examples-test/csharp.sh @@ -6,23 +6,24 @@ pgrep taosd || taosd >> /dev/null 2>&1 & pgrep taosadapter || taosadapter >> /dev/null 2>&1 & cd ../../docs/examples/csharp -dotnet run --project connect.csproj +#dotnet run --project connect.csproj -taos -s "drop database if exists power" -dotnet run --project sqlinsert.csproj -dotnet run --project query.csproj -dotnet run --project asyncquery.csproj -dotnet run --project subscribe.csproj +#taos -s "drop database if exists power" +#dotnet run --project sqlinsert.csproj +#dotnet run --project query.csproj +#dotnet run --project asyncquery.csproj +#dotnet run --project subscribe.csproj -taos -s "drop topic if exists topic_example" -taos -s "drop database if exists power" -dotnet run --project stmtinsert.csproj +#taos -s "drop topic if exists topic_example" +#taos -s "drop database if exists power" +#dotnet run --project stmtinsert.csproj -taos -s "drop database if exists test" -dotnet run --project influxdbline.csproj +#taos -s "drop database if exists test" +#dotnet run --project influxdbline.csproj -taos -s "drop database if exists test" -dotnet run --project optstelnet.csproj +#taos -s "drop database if exists test" +#dotnet run --project optstelnet.csproj -taos -s "drop database if exists test" -dotnet run --project optsjson.csproj \ No newline at end of file +#taos -s "drop database if exists test" +#dotnet run --project optsjson.csproji +echo "uncomment temporily" From 96eec9f47a9b89eb14949a51602ffc1b151bd819 Mon Sep 17 00:00:00 2001 From: tomchon Date: Tue, 20 Sep 2022 22:24:33 +0800 Subject: [PATCH 25/52] test:update jenkins of ci --- Jenkinsfile2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 6a4a2156e5..b95b3ff86b 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -218,12 +218,12 @@ def pre_test_win(){ if (env.CHANGE_URL =~ /\/TDengine\//) { bat ''' cd %WIN_INTERNAL_ROOT% - git pull ''' + env.CHANGE_TARGET + ''' + git pull origin ''' + env.CHANGE_TARGET + ''' ''' bat ''' cd %WIN_COMMUNITY_ROOT% git remote prune origin - git pull ''' + env.CHANGE_TARGET + ''' + git pull origin ''' + env.CHANGE_TARGET + ''' ''' bat ''' cd %WIN_COMMUNITY_ROOT% @@ -236,7 +236,7 @@ def pre_test_win(){ } else if (env.CHANGE_URL =~ /\/TDinternal\//) { bat ''' cd %WIN_INTERNAL_ROOT% - git pull ''' + env.CHANGE_TARGET + ''' + git pull origin ''' + env.CHANGE_TARGET + ''' ''' bat ''' cd %WIN_INTERNAL_ROOT% From c3edf0b18bdb93ddc83e3111cf50a0671f0cefde Mon Sep 17 00:00:00 2001 From: baolinking <38206840+baolinking@users.noreply.github.com> Date: Wed, 21 Sep 2022 09:06:52 +0800 Subject: [PATCH 26/52] =?UTF-8?q?fix:=E4=BF=AE=E6=94=B9SERVER=5FSTATUS?= =?UTF-8?q?=E7=9A=84=E8=AF=AD=E6=B3=95=E5=AE=9E=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将SELECT SERVER_VERSION()改为SELECT SERVER_STATUS() --- docs/zh/12-taos-sql/10-function.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index 86e9aaa80f..9726406b4d 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -1233,7 +1233,7 @@ SELECT SERVER_VERSION(); ### SERVER_STATUS ```sql -SELECT SERVER_VERSION(); +SELECT SERVER_STATUS(); ``` **说明**：返回服务端当前的状态。 From adf48d545898d766f4084ce215753fbbd67f344a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Wed, 21 Sep 2022 09:50:56 +0800 Subject: [PATCH 27/52] test: add pre_suf cases --- .../system-test/1-insert/database_pre_suf.py | 377 ++++++++++++++++++ 1 file changed, 377 insertions(+) create mode 100755 tests/system-test/1-insert/database_pre_suf.py diff --git a/tests/system-test/1-insert/database_pre_suf.py b/tests/system-test/1-insert/database_pre_suf.py new file mode 100755 index 0000000000..fe788af0db --- /dev/null +++ b/tests/system-test/1-insert/database_pre_suf.py @@ -0,0 +1,377 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import os +import time +import taos +import subprocess +from faker import Faker +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes +from util.dnodes import * + +class TDTestCase: + updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.testcasePath = os.path.split(__file__)[0] + self.testcaseFilename = os.path.split(__file__)[-1] + os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) + + self.db = "pre_suf" + + def dropandcreateDB_random(self,database,n,vgroups,table_prefix,table_suffix,check_result): + ts = 1630000000000 + num_random = 100 + fake = Faker('zh_CN') + tdSql.execute('''drop database if exists %s ;''' %database) + tdSql.execute('''create database %s keep 36500 vgroups %d table_prefix %d table_suffix %d;'''%(database,vgroups,table_prefix,table_suffix)) + tdSql.execute('''use %s;'''%database) + + tdSql.execute('''create stable stable_1 (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , \ + q_binary1 binary(100) , q_nchar1 nchar(100) ,q_binary2 binary(100) , q_nchar2 nchar(100) ,q_binary3 binary(100) , q_nchar3 nchar(100) ,q_binary4 binary(100) , q_nchar4 nchar(100) ,\ + q_binary5 binary(100) , q_nchar5 nchar(100) ,q_binary6 binary(100) , q_nchar6 nchar(100) ,q_binary7 binary(100) , q_nchar7 nchar(100) ,q_binary8 binary(100) , q_nchar8 nchar(100) ,\ + q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) \ + tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);''') + tdSql.execute('''create stable stable_2 (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , \ + q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) \ + tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);''') + + for i in range(10*n): + tdSql.execute('''create table bj_%d (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp ) ;'''%i) + tdSql.execute('''create table sh_%d (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp ) ;'''%i) + tdSql.execute('''create table bj_table_%d_r (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp ) ;'''%i) + tdSql.execute('''create table sh_table_%d_r (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp ) ;'''%i) + tdSql.execute('''create table hn_table_%d_r \ + (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , \ + q_binary1 binary(100) , q_nchar1 nchar(100) ,q_binary2 binary(100) , q_nchar2 nchar(100) ,q_binary3 binary(100) , q_nchar3 nchar(100) ,q_binary4 binary(100) , q_nchar4 nchar(100) ,\ + q_binary5 binary(100) , q_nchar5 nchar(100) ,q_binary6 binary(100) , q_nchar6 nchar(100) ,q_binary7 binary(100) , q_nchar7 nchar(100) ,q_binary8 binary(100) , q_nchar8 nchar(100) ,\ + q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) ;'''%i) + tdSql.execute('''create table bj_stable_1_%d using stable_1 tags('bj_stable_1_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;''' + %(i,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + tdSql.execute('''create table sh_table_%d_a using stable_1 tags('sh_a_table_1_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;''' + %(i,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + tdSql.execute('''create table sh_table_%d_b using stable_1 tags('sh_b_table_1_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;''' + %(i,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + tdSql.execute('''create table sh_table_%d_c using stable_1 tags('sh_c_table_1_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;''' + %(i,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + + tdSql.execute('''create table bj_table_%d_a using stable_1 tags('bj_a_table_1_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;''' + %(i,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + tdSql.execute('''create table bj_table_%d_b using stable_1 tags('bj_b_table_1_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;''' + %(i,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + tdSql.execute('''create table bj_table_%d_c using stable_1 tags('bj_c_table_1_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;''' + %(i,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + + + tdSql.execute('''create table tj_table_%d_a using stable_2 tags('tj_a_table_2_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;''' + %(i,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + tdSql.execute('''create table tj_table_%d_b using stable_2 tags('tj_b_table_2_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;''' + %(i,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + + + # create stream + tdSql.execute('''create stream current_stream into stream_max_stable_1 as select _wstart as start, _wend as end, max(q_int) as max_int, min(q_bigint) as min_int from stable_1 where ts is not null interval (5s);''') + + # insert data + for i in range(num_random*n): + tdSql.execute('''insert into bj_stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts,\ + q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8) \ + values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \ + 'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' + % (ts + i*1000, fake.random_int(min=-2147483647, max=2147483647, step=1), + fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , + fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr())) + tdSql.execute('''insert into hn_table_1_r (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts,\ + q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8) \ + values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \ + 'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' + % (ts + i*1000, fake.random_int(min=-2147483647, max=2147483647, step=1) , + fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1) , + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i, fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , + fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr())) + + tdSql.execute('''insert into bj_stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts,\ + q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8)\ + values(%d, %d, %d, %d, %d, %f, %f, 1, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \ + 'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' + % (ts + i*1000, fake.random_int(min=0, max=2147483647, step=1), + fake.random_int(min=0, max=9223372036854775807, step=1), + fake.random_int(min=0, max=32767, step=1) , fake.random_int(min=0, max=127, step=1) , + fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i, fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , + fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr())) + tdSql.execute('''insert into hn_table_2_r (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts,\ + q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8) \ + values(%d, %d, %d, %d, %d, %f, %f, 1, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \ + 'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' + % (ts + i*1000, fake.random_int(min=0, max=2147483647, step=1), + fake.random_int(min=0, max=9223372036854775807, step=1), + fake.random_int(min=0, max=32767, step=1) , fake.random_int(min=0, max=127, step=1) , + fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i, fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , + fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr())) + + tdSql.execute('''insert into bj_stable_1_3 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts,\ + q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8) \ + values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \ + 'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' + % (ts + i*1000, fake.random_int(min=-0, max=2147483647, step=1), + fake.random_int(min=-0, max=9223372036854775807, step=1), + fake.random_int(min=-0, max=32767, step=1) , fake.random_int(min=-0, max=127, step=1) , + fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i, fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , + fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr())) + + tdSql.execute('''insert into bj_stable_1_4 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts,\ + q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8) \ + values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \ + 'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' + % (ts + i*1000 +1, fake.random_int(min=-0, max=2147483647, step=1), + fake.random_int(min=-0, max=9223372036854775807, step=1), + fake.random_int(min=-0, max=32767, step=1) , fake.random_int(min=-0, max=127, step=1) , + fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i, fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , + fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr())) + + tdSql.execute('''insert into bj_stable_1_5 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts,\ + q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8) \ + values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \ + 'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' + % (ts + i*1000 +10, fake.random_int(min=-0, max=2147483647, step=1), + fake.random_int(min=-0, max=9223372036854775807, step=1), + fake.random_int(min=-0, max=32767, step=1) , fake.random_int(min=-0, max=127, step=1) , + fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i, fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , + fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr())) + + tdSql.query("select count(*) from stable_1;") + tdSql.checkData(0,0,5*num_random*n) + tdSql.query("select count(*) from hn_table_1_r;") + tdSql.checkData(0,0,num_random*n) + + # stream data check + tdSql.query("select start,end,max_int from stream_max_stable_1 ;") + tdSql.checkRows(20) + tdSql.query("select sum(max_int) from stream_max_stable_1 ;") + stream_data_1 = tdSql.queryResult[0][0] + tdSql.query("select sum(min_int) from stream_max_stable_1 ;") + stream_data_2 = tdSql.queryResult[0][0] + tdSql.query("select sum(max_int),sum(min_int) from (select _wstart as start, _wend as end, max(q_int) as max_int, min(q_bigint) as min_int from stable_1 where ts is not null interval (5s));") + sql_data_1 = tdSql.queryResult[0][0] + sql_data_2 = tdSql.queryResult[0][1] + + self.stream_value_check(stream_data_1,sql_data_1) + self.stream_value_check(stream_data_2,sql_data_2) + + tdSql.query("select sum(max_int),sum(min_int) from (select _wstart as start, _wend as end, max(q_int) as max_int, min(q_bigint) as min_int from stable_1 interval (5s));") + sql_data_1 = tdSql.queryResult[0][0] + sql_data_2 = tdSql.queryResult[0][1] + + self.stream_value_check(stream_data_1,sql_data_1) + self.stream_value_check(stream_data_2,sql_data_2) + + tdSql.query("select max(max_int) from stream_max_stable_1 ;") + stream_data_1 = tdSql.queryResult[0][0] + tdSql.query("select min(min_int) from stream_max_stable_1 ;") + stream_data_2 = tdSql.queryResult[0][0] + tdSql.query("select max(q_int) as max_int, min(q_bigint) as min_int from stable_1;") + sql_data_1 = tdSql.queryResult[0][0] + sql_data_2 = tdSql.queryResult[0][1] + + self.stream_value_check(stream_data_1,sql_data_1) + self.stream_value_check(stream_data_2,sql_data_2) + + + tdSql.query(" select * from information_schema.ins_databases where name = '%s';" %database) + print(tdSql.queryResult) + + # tdSql.query(" select table_prefix,table_suffix from information_schema.ins_databases where name = '%s';" %database) + # print(tdSql.queryResult) + #TD-19082 + + #tdSql.query(" select * from information_schema.ins_tables where db_name = '%s';" %database) + #print(tdSql.queryResult) + + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s';" %database) + queryRows = len(tdSql.queryResult) + for i in range(queryRows): + print("row=%d, vgroup_id=%s, tbname=%s " %(i,tdSql.queryResult[i][1],tdSql.queryResult[i][0])) + + tdLog.info("\n=============flush database ====================\n") + + tdSql.execute(" flush database %s;" %database) + + tdSql.query(" select * from information_schema.ins_databases where name = '%s';" %database) + print(tdSql.queryResult) + + # tdSql.query(" select table_prefix,table_suffix from information_schema.ins_databases where name = '%s';" %database) + # print(tdSql.queryResult) + #TD-19082 + + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s';" %database) + queryRows = len(tdSql.queryResult) + for i in range(queryRows): + print("row=%d, vgroup_id=%s, tbname=%s " %(i,tdSql.queryResult[i][1],tdSql.queryResult[i][0])) + + + # check in one vgroup + if check_result == 'Y': + #base table : sh_table_0_a + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='sh_table_0_a';" %(database)) + base_value_table_name = tdSql.queryResult[0][0] + base_value_table_vgroup = tdSql.queryResult[0][1] + + #check table :sh_table_i_a + check_rows = tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name like 'sh_table_%%_a';" %(database)) + for i in range(check_rows): + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='sh_table_%d_a';" %(database,i)) + self.value_check(base_value_table_name,base_value_table_vgroup) + + #check table :sh_table_i_b + check_rows = tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name like 'sh_table_%%_b';" %(database)) + for i in range(check_rows): + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='sh_table_%d_b';" %(database,i)) + self.value_check(base_value_table_name,base_value_table_vgroup) + + #check table :sh_table_i_c + check_rows = tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name like 'sh_table_%%_c';" %(database)) + for i in range(check_rows): + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='sh_table_%d_c';" %(database,i)) + self.value_check(base_value_table_name,base_value_table_vgroup) + + #check table :sh_table_i_r + check_rows = tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name like 'sh_table_%%_r';" %(database)) + for i in range(check_rows): + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='sh_table_%d_r';" %(database,i)) + self.value_check(base_value_table_name,base_value_table_vgroup) + + #check table :bj_table_i_a + check_rows = tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name like 'bj_table_%%_a';" %(database)) + for i in range(check_rows): + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='bj_table_%d_a';" %(database,i)) + self.value_check(base_value_table_name,base_value_table_vgroup) + + #check table :bj_table_i_b + check_rows = tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name like 'bj_table_%%_b';" %(database)) + for i in range(check_rows): + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='bj_table_%d_b';" %(database,i)) + self.value_check(base_value_table_name,base_value_table_vgroup) + + #check table :bj_table_i_c + check_rows = tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name like 'bj_table_%%_c';" %(database)) + for i in range(check_rows): + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='bj_table_%d_c';" %(database,i)) + self.value_check(base_value_table_name,base_value_table_vgroup) + + #check table :bj_table_i_r + check_rows = tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name like 'bj_table_%%_r';" %(database)) + for i in range(check_rows): + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='bj_table_%d_r';" %(database,i)) + self.value_check(base_value_table_name,base_value_table_vgroup) + + #check table :hn_table_i_r + check_rows = tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name like 'hn_table_%%_r';" %(database)) + for i in range(check_rows): + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='hn_table_%d_r';" %(database,i)) + self.value_check(base_value_table_name,base_value_table_vgroup) + + #check table :tj_table_i_a + check_rows = tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name like 'tj_table_%%_a';" %(database)) + for i in range(check_rows): + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='tj_table_%d_a';" %(database,i)) + self.value_check(base_value_table_name,base_value_table_vgroup) + + #check table :tj_table_i_b + check_rows = tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name like 'tj_table_%%_b';" %(database)) + for i in range(check_rows): + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='tj_table_%d_b';" %(database,i)) + self.value_check(base_value_table_name,base_value_table_vgroup) + + else: + pass + + def value_check(self,base_value_table_name,base_value_table_vgroup): + check_value_table_name = tdSql.queryResult[0][0] + check_value_table_vgroup = tdSql.queryResult[0][1] + #tdLog.info(f"{base_value_table_name},{base_value_table_vgroup},{check_value_table_name},{check_value_table_vgroup}") + + if base_value_table_vgroup==check_value_table_vgroup: + tdLog.info(f"checkEqual success, base_table_name={base_value_table_name},base_table_host={base_value_table_vgroup} ,check_table_name={check_value_table_name},check_table_host={check_value_table_vgroup}") + else : + tdLog.exit(f"checkEqual error, base_table_name=={base_value_table_name},base_table_host={base_value_table_vgroup} ,check_table_name={check_value_table_name},check_table_host={check_value_table_vgroup}") + + def stream_value_check(self,stream_data,sql_data): + if stream_data==sql_data: + tdLog.info(f"checkEqual success, stream_data={stream_data},sql_data={sql_data}") + else : + tdLog.exit(f"checkEqual error, stream_data=={stream_data},sql_data={sql_data}") + + def run(self): + startTime = time.time() + + os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) + + #(self,database,n,vgroups,table_prefix,table_suffix) + self.dropandcreateDB_random("%s" %self.db, 1,2,0,0,'N') + self.dropandcreateDB_random("%s" %self.db, 1,2,0,2,'N') + self.dropandcreateDB_random("%s" %self.db, 1,2,2,0,'N') + self.dropandcreateDB_random("%s" %self.db, 1,2,3,3,'Y') + self.dropandcreateDB_random("%s" %self.db, 1,3,3,3,'Y') + self.dropandcreateDB_random("%s" %self.db, 1,4,4,4,'Y') + self.dropandcreateDB_random("%s" %self.db, 1,5,5,5,'Y') + + + #taos -f sql + print("taos -f sql start!") + taos_cmd1 = "taos -f %s/%s.sql" % (self.testcasePath,self.testcaseFilename) + _ = subprocess.check_output(taos_cmd1, shell=True) + print("taos -f sql over!") + + + endTime = time.time() + print("total time %ds" % (endTime - startTime)) + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 29042f74eb882e694d435b906ba39532bcbaef27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Wed, 21 Sep 2022 09:52:06 +0800 Subject: [PATCH 28/52] test: add pre_suf cases --- tests/system-test/fulltest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index f4a5c22d39..847bc300b3 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -35,6 +35,7 @@ python3 ./test.py -f 1-insert/mutil_stage.py python3 ./test.py -f 1-insert/table_param_ttl.py -R python3 ./test.py -f 1-insert/update_data_muti_rows.py python3 ./test.py -f 1-insert/db_tb_name_check.py +python3 ./test.py -f 1-insert/database_pre_suf.py python3 ./test.py -f 2-query/abs.py python3 ./test.py -f 2-query/abs.py -R From 196e28f322e55772394acb78338cc30f3daa8ad6 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 21 Sep 2022 10:08:19 +0800 Subject: [PATCH 29/52] build: release ver-3.0.1.2 --- cmake/cmake.version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cmake.version b/cmake/cmake.version index d5721f7671..fcb31adc39 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.0.1.1") + SET(TD_VER_NUMBER "3.0.1.2") ENDIF () IF (DEFINED VERCOMPATIBLE) From be122addf4dddc76e73aaac73736667dcd70418e Mon Sep 17 00:00:00 2001 From: Pan YANG Date: Wed, 21 Sep 2022 10:10:05 +0800 Subject: [PATCH 30/52] docs: remove case sensitive description of quotes Remove case sensitive description of quotes and other minor content. --- docs/en/12-taos-sql/01-data-type.md | 76 +++++++++++++-------------- docs/zh/12-taos-sql/01-data-type.md | 81 +++++++++++++++-------------- 2 files changed, 79 insertions(+), 78 deletions(-) diff --git a/docs/en/12-taos-sql/01-data-type.md b/docs/en/12-taos-sql/01-data-type.md index 876de50f35..60046629a4 100644 --- a/docs/en/12-taos-sql/01-data-type.md +++ b/docs/en/12-taos-sql/01-data-type.md @@ -1,70 +1,70 @@ --- sidebar_label: Data Types title: Data Types -description: "TDengine supports a variety of data types including timestamp, float, JSON and many others." +description: 'TDengine supports a variety of data types including timestamp, float, JSON and many others.' --- ## Timestamp When using TDengine to store and query data, the most important part of the data is timestamp. Timestamp must be specified when creating and inserting data rows. Timestamp must follow the rules below: -- The format must be `YYYY-MM-DD HH:mm:ss.MS`, the default time precision is millisecond (ms), for example `2017-08-12 18:25:58.128` -- Internal function `now` can be used to get the current timestamp on the client side -- The current timestamp of the client side is applied when `now` is used to insert data +- The format must be `YYYY-MM-DD HH:mm:ss.MS`, the default time precision is millisecond (ms), for example `2017-08-12 18:25:58.128`. +- Internal function `NOW` can be used to get the current timestamp on the client side. +- The current timestamp of the client side is applied when `NOW` is used to insert data. - Epoch Time：timestamp can also be a long integer number, which means the number of seconds, milliseconds or nanoseconds, depending on the time precision, from UTC 1970-01-01 00:00:00. -- Add/subtract operations can be carried out on timestamps. For example `now-2h` means 2 hours prior to the time at which query is executed. The units of time in operations can be b(nanosecond), u(microsecond), a(millisecond), s(second), m(minute), h(hour), d(day), or w(week). So `select * from t1 where ts > now-2w and ts <= now-1w` means the data between two weeks ago and one week ago. The time unit can also be n (calendar month) or y (calendar year) when specifying the time window for down sampling operations. +- Add/subtract operations can be carried out on timestamps. For example `NOW-2h` means 2 hours prior to the time at which query is executed. The units of time in operations can be b(nanosecond), u(microsecond), a(millisecond), s(second), m(minute), h(hour), d(day), or w(week). So `SELECT * FROM t1 WHERE ts > NOW-2w AND ts <= NOW-1w` means the data between two weeks ago and one week ago. The time unit can also be n (calendar month) or y (calendar year) when specifying the time window for down sampling operations. Time precision in TDengine can be set by the `PRECISION` parameter when executing `CREATE DATABASE`. The default time precision is millisecond. In the statement below, the precision is set to nanonseconds. ```sql CREATE DATABASE db_name PRECISION 'ns'; ``` + ## Data Types In TDengine, the data types below can be used when specifying a column or tag. -| # | **type** | **Bytes** | **Description** | -| --- | :-------: | --------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1 | TIMESTAMP | 8 | Default precision is millisecond, microsecond and nanosecond are also supported | -| 2 | INT | 4 | Integer, the value range is [-2^31, 2^31-1] | -| 3 | INT UNSIGNED| 4| unsigned integer, the value range is [0, 2^32-1] -| 4 | BIGINT | 8 | Long integer, the value range is [-2^63, 2^63-1] | -| 5 | BIGINT UNSIGNED | 8 | unsigned long integer, the value range is [0, 2^64-1] | -| 6 | FLOAT | 4 | Floating point number, the effective number of digits is 6-7, the value range is [-3.4E38, 3.4E38] | -| 7 | DOUBLE | 8 | Double precision floating point number, the effective number of digits is 15-16, the value range is [-1.7E308, 1.7E308] | -| 8 | BINARY | User Defined | Single-byte string for ASCII visible characters. Length must be specified when defining a column or tag of binary type. | -| 9 | SMALLINT | 2 | Short integer, the value range is [-32768, 32767] | -| 10 | INT UNSIGNED| 2| unsigned integer, the value range is [0, 65535]| -| 11 | TINYINT | 1 | Single-byte integer, the value range is [-128, 127] | -| 12 | TINYINT UNSIGNED | 1 | unsigned single-byte integer, the value range is [0, 255] | -| 13 | BOOL | 1 | Bool, the value range is {true, false} | -| 14 | NCHAR | User Defined| Multi-Byte string that can include multi byte characters like Chinese characters. Each character of NCHAR type consumes 4 bytes storage. The string value should be quoted with single quotes. Literal single quote inside the string must be preceded with backslash, like `\’`. The length must be specified when defining a column or tag of NCHAR type, for example nchar(10) means it can store at most 10 characters of nchar type and will consume fixed storage of 40 bytes. An error will be reported if the string value exceeds the length defined. | -| 15 | JSON | | JSON type can only be used on tags. A tag of json type is excluded with any other tags of any other type | -| 16 | VARCHAR | User-defined | Alias of BINARY | - +| # | **type** | **Bytes** | **Description** | +| --- | :--------------: | ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | TIMESTAMP | 8 | Default precision is millisecond, microsecond and nanosecond are also supported. | +| 2 | INT | 4 | Integer, the value range is [-2^31, 2^31-1]. | +| 3 | INT UNSIGNED | 4 | Unsigned integer, the value range is [0, 2^32-1]. | +| 4 | BIGINT | 8 | Long integer, the value range is [-2^63, 2^63-1]. | +| 5 | BIGINT UNSIGNED | 8 | unsigned long integer, the value range is [0, 2^64-1]. | +| 6 | FLOAT | 4 | Floating point number, the effective number of digits is 6-7, the value range is [-3.4E38, 3.4E38]. | +| 7 | DOUBLE | 8 | Double precision floating point number, the effective number of digits is 15-16, the value range is [-1.7E308, 1.7E308]. | +| 8 | BINARY | User Defined | Single-byte string for ASCII visible characters. Length must be specified when defining a column or tag of binary type. | +| 9 | SMALLINT | 2 | Short integer, the value range is [-32768, 32767]. | +| 10 | INT UNSIGNED | 2 | unsigned integer, the value range is [0, 65535]. | +| 11 | TINYINT | 1 | Single-byte integer, the value range is [-128, 127]. | +| 12 | TINYINT UNSIGNED | 1 | unsigned single-byte integer, the value range is [0, 255]. | +| 13 | BOOL | 1 | Bool, the value range is {true, false}. | +| 14 | NCHAR | User Defined | Multi-byte string that can include multi byte characters like Chinese characters. Each character of NCHAR type consumes 4 bytes storage. The string value should be quoted with single quotes. Literal single quote inside the string must be preceded with backslash, like `\'`. The length must be specified when defining a column or tag of NCHAR type, for example nchar(10) means it can store at most 10 characters of nchar type and will consume fixed storage of 40 bytes. An error will be reported if the string value exceeds the length defined. | +| 15 | JSON | | JSON type can only be used on tags. A tag of json type is excluded with any other tags of any other type. | +| 16 | VARCHAR | User-defined | Alias of BINARY | :::note -- TDengine is case insensitive and treats any characters in the sql command as lower case by default, case sensitive strings must be quoted with single quotes. -- Only ASCII visible characters are suggested to be used in a column or tag of BINARY type. Multi-byte characters must be stored in NCHAR type. -- The length of BINARY can be up to 16374 bytes. The string value must be quoted with single quotes. You must specify a length in bytes for a BINARY value, for example binary(20) for up to twenty single-byte characters. If the data exceeds the specified length, an error will occur. The literal single quote inside the string must be preceded with back slash like `\'` + +- Only ASCII visible characters are suggested to be used in a column or tag of BINARY type. Multi-byte characters must be stored in NCHAR type. +- The length of BINARY can be up to 16,374 bytes. The string value must be quoted with single quotes. You must specify a length in bytes for a BINARY value, for example binary(20) for up to twenty single-byte characters. If the data exceeds the specified length, an error will occur. The literal single quote inside the string must be preceded with back slash like `\'` - Numeric values in SQL statements will be determined as integer or float type according to whether there is decimal point or whether scientific notation is used, so attention must be paid to avoid overflow. For example, 9999999999999999999 will be considered as overflow because it exceeds the upper limit of long integer, but 9999999999999999999.0 will be considered as a legal float number. ::: - ## Constants + TDengine supports a variety of constants: -| # | **Syntax** | **Type** | **Description** | -| --- | :-------: | --------- | -------------------------------------- | -| 1 | [{+ \| -}]123 | BIGINT | Integer literals are of type BIGINT. Data that exceeds the length of the BIGINT type is truncated. | -| 2 | 123.45 | DOUBLE | Floating-point literals are of type DOUBLE. Numeric values will be determined as integer or float type according to whether there is decimal point or whether scientific notation is used. | -| 3 | 1.2E3 | DOUBLE | Literals in scientific notation are of type DOUBLE. | -| 4 | 'abc' | BINARY | Content enclosed in single quotation marks is of type BINARY. The size of a BINARY is the size of the string in bytes. A literal single quote inside the string must be escaped with a backslash (\'). | -| 5 | 'abc' | BINARY | Content enclosed in double quotation marks is of type BINARY. The size of a BINARY is the size of the string in bytes. A literal double quote inside the string must be escaped with a backslash (\"). | -| 6 | TIMESTAMP {'literal' \| "literal"} | TIMESTAMP | The TIMESTAMP keyword indicates that the following string literal is interpreted as a timestamp. The string must be in YYYY-MM-DD HH:mm:ss.MS format. The precision is inherited from the database configuration. | -| 7 | {TRUE \| FALSE} | BOOL | Boolean literals are of type BOOL. | -| 8 | {'' \| "" \| '\t' \| "\t" \| ' ' \| " " \| NULL } | -- | The preceding characters indicate null literals. These can be used with any data type. | +| # | **Syntax** | **Type** | **Description** | +| --- | :-----------------------------------------------: | --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | [{+ \| -}]123 | BIGINT | Integer literals are of type BIGINT. Data that exceeds the length of the BIGINT type is truncated. | +| 2 | 123.45 | DOUBLE | Floating-point literals are of type DOUBLE. Numeric values will be determined as integer or float type according to whether there is decimal point or whether scientific notation is used. | +| 3 | 1.2E3 | DOUBLE | Literals in scientific notation are of type DOUBLE. | +| 4 | 'abc' | BINARY | Content enclosed in single quotation marks is of type BINARY. The size of a BINARY is the size of the string in bytes. A literal single quote inside the string must be escaped with a backslash `\'`. | +| 5 | 'abc' | BINARY | Content enclosed in double quotation marks is of type BINARY. The size of a BINARY is the size of the string in bytes. A literal double quote inside the string must be escaped with a backslash `\"`. | +| 6 | TIMESTAMP {'literal' \| "literal"} | TIMESTAMP | The TIMESTAMP keyword indicates that the following string literal is interpreted as a timestamp. The string must be in YYYY-MM-DD HH:mm:ss.MS format. The precision is inherited from the database configuration. | +| 7 | {TRUE \| FALSE} | BOOL | Boolean literals are of type BOOL. | +| 8 | {'' \| "" \| '\t' \| "\t" \| ' ' \| " " \| NULL } | -- | The preceding characters indicate null literals. These can be used with any data type. | :::note Numeric values will be determined as integer or float type according to whether there is decimal point or whether scientific notation is used, so attention must be paid to avoid overflow. For example, 9999999999999999999 will be considered as overflow because it exceeds the upper limit of long integer, but 9999999999999999999.0 will be considered as a legal float number. diff --git a/docs/zh/12-taos-sql/01-data-type.md b/docs/zh/12-taos-sql/01-data-type.md index ee7b3a4715..128fa20930 100644 --- a/docs/zh/12-taos-sql/01-data-type.md +++ b/docs/zh/12-taos-sql/01-data-type.md @@ -1,7 +1,7 @@ --- sidebar_label: 数据类型 title: 数据类型 -description: "TDengine 支持的数据类型: 时间戳、浮点型、JSON 类型等" +description: 'TDengine 支持的数据类型: 时间戳、浮点型、JSON 类型等' --- ## 时间戳 @@ -9,64 +9,65 @@ description: "TDengine 支持的数据类型: 时间戳、浮点型、JSON 类 使用 TDengine，最重要的是时间戳。创建并插入记录、查询历史记录的时候，均需要指定时间戳。时间戳有如下规则： - 时间格式为 `YYYY-MM-DD HH:mm:ss.MS`，默认时间分辨率为毫秒。比如：`2017-08-12 18:25:58.128` -- 内部函数 now 是客户端的当前时间 -- 插入记录时，如果时间戳为 now，插入数据时使用提交这条记录的客户端的当前时间 -- Epoch Time：时间戳也可以是一个长整数，表示从 UTC 时间 1970-01-01 00:00:00 开始的毫秒数。相应地，如果所在 Database 的时间精度设置为“微秒”，则长整型格式的时间戳含义也就对应于从 UTC 时间 1970-01-01 00:00:00 开始的微秒数；纳秒精度逻辑类似。 -- 时间可以加减，比如 now-2h，表明查询时刻向前推 2 个小时（最近 2 小时）。数字后面的时间单位可以是 b(纳秒)、u(微秒)、a(毫秒)、s(秒)、m(分)、h(小时)、d(天)、w(周)。 比如 `select * from t1 where ts > now-2w and ts <= now-1w`，表示查询两周前整整一周的数据。在指定降采样操作（down sampling）的时间窗口（interval）时，时间单位还可以使用 n (自然月) 和 y (自然年)。 +- 内部函数 NOW 是客户端的当前时间 +- 插入记录时，如果时间戳为 NOW，插入数据时使用提交这条记录的客户端的当前时间 +- Epoch Time：时间戳也可以是一个长整数，表示从 UTC 时间 1970-01-01 00:00:00 开始的毫秒数。相应地，如果所在 Database 的时间精度设置为“微秒”，则长整型格式的时间戳含义也就对应于从 UTC 时间 1970-01-01 00:00:00 开始的微秒数；纳秒精度逻辑相同。 +- 时间可以加减，比如 NOW-2h，表明查询时刻向前推 2 个小时（最近 2 小时）。数字后面的时间单位可以是 b（纳秒）、u（微秒）、a（毫秒）、s（秒）、m（分）、h（小时）、d（天）、w（周）。 比如 `SELECT * FROM t1 WHERE ts > NOW-2w AND ts <= NOW-1w`，表示查询两周前整整一周的数据。在指定降采样操作（Down Sampling）的时间窗口（Interval）时，时间单位还可以使用 n（自然月）和 y（自然年）。 -TDengine 缺省的时间戳精度是毫秒，但通过在 `CREATE DATABASE` 时传递的 PRECISION 参数也可以支持微秒和纳秒。 +TDengine 缺省的时间戳精度是毫秒，但通过在 `CREATE DATABASE` 时传递的 `PRECISION` 参数也可以支持微秒和纳秒。 ```sql CREATE DATABASE db_name PRECISION 'ns'; ``` + ## 数据类型 在 TDengine 中，普通表的数据模型中可使用以下数据类型。 -| # | **类型** | **Bytes** | **说明** | -| --- | :-------: | --------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1 | TIMESTAMP | 8 | 时间戳。缺省精度毫秒，可支持微秒和纳秒，详细说明见上节。 | -| 2 | INT | 4 | 整型，范围 [-2^31, 2^31-1] | -| 3 | INT UNSIGNED| 4| 无符号整数，[0, 2^32-1] -| 4 | BIGINT | 8 | 长整型，范围 [-2^63, 2^63-1] | -| 5 | BIGINT UNSIGNED | 8 | 长整型，范围 [0, 2^64-1] | -| 6 | FLOAT | 4 | 浮点型，有效位数 6-7，范围 [-3.4E38, 3.4E38] | -| 7 | DOUBLE | 8 | 双精度浮点型，有效位数 15-16，范围 [-1.7E308, 1.7E308] | -| 8 | BINARY | 自定义 | 记录单字节字符串，建议只用于处理 ASCII 可见字符，中文等多字节字符需使用 nchar。 | -| 9 | SMALLINT | 2 | 短整型， 范围 [-32768, 32767] | -| 10 | SMALLINT UNSIGNED | 2| 无符号短整型，范围 [0, 65535] | -| 11 | TINYINT | 1 | 单字节整型，范围 [-128, 127] | -| 12 | TINYINT UNSIGNED | 1 | 无符号单字节整型，范围 [0, 255] | -| 13 | BOOL | 1 | 布尔型，{true, false} | -| 14 | NCHAR | 自定义 | 记录包含多字节字符在内的字符串，如中文字符。每个 nchar 字符占用 4 bytes 的存储空间。字符串两端使用单引号引用，字符串内的单引号需用转义字符 `\’`。nchar 使用时须指定字符串大小，类型为 nchar(10) 的列表示此列的字符串最多存储 10 个 nchar 字符，会固定占用 40 bytes 的空间。如果用户字符串长度超出声明长度，将会报错。 | -| 15 | JSON | | json 数据类型， 只有 tag 可以是 json 格式 | -| 16 | VARCHAR | 自定义 | BINARY类型的别名 | - +| # | **类型** | **Bytes** | **说明** | +| --- | :---------------: | --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | TIMESTAMP | 8 | 时间戳。缺省精度毫秒，可支持微秒和纳秒，详细说明见上节。 | +| 2 | INT | 4 | 整型，范围 [-2^31, 2^31-1] | +| 3 | INT UNSIGNED | 4 | 无符号整数，[0, 2^32-1] | +| 4 | BIGINT | 8 | 长整型，范围 [-2^63, 2^63-1] | +| 5 | BIGINT UNSIGNED | 8 | 长整型，范围 [0, 2^64-1] | +| 6 | FLOAT | 4 | 浮点型，有效位数 6-7，范围 [-3.4E38, 3.4E38] | +| 7 | DOUBLE | 8 | 双精度浮点型，有效位数 15-16，范围 [-1.7E308, 1.7E308] | +| 8 | BINARY | 自定义 | 记录单字节字符串，建议只用于处理 ASCII 可见字符，中文等多字节字符需使用 NCHAR | +| 9 | SMALLINT | 2 | 短整型， 范围 [-32768, 32767] | +| 10 | SMALLINT UNSIGNED | 2 | 无符号短整型，范围 [0, 65535] | +| 11 | TINYINT | 1 | 单字节整型，范围 [-128, 127] | +| 12 | TINYINT UNSIGNED | 1 | 无符号单字节整型，范围 [0, 255] | +| 13 | BOOL | 1 | 布尔型，{true, false} | +| 14 | NCHAR | 自定义 | 记录包含多字节字符在内的字符串，如中文字符。每个 NCHAR 字符占用 4 字节的存储空间。字符串两端使用单引号引用，字符串内的单引号需用转义字符 `\'`。NCHAR 使用时须指定字符串大小，类型为 NCHAR(10) 的列表示此列的字符串最多存储 10 个 NCHAR 字符，会固定占用 40 字节的空间。如果用户字符串长度超出声明长度，将会报错。 | +| 15 | JSON | | JSON 数据类型， 只有 Tag 可以是 JSON 格式 | +| 16 | VARCHAR | 自定义 | BINARY 类型的别名 | :::note -- TDengine 对 SQL 语句中的英文字符不区分大小写，自动转化为小写执行。因此用户大小写敏感的字符串及密码，需要使用单引号将字符串引起来。 + - 虽然 BINARY 类型在底层存储上支持字节型的二进制字符，但不同编程语言对二进制数据的处理方式并不保证一致，因此建议在 BINARY 类型中只存储 ASCII 可见字符，而避免存储不可见字符。多字节的数据，例如中文字符，则需要使用 NCHAR 类型进行保存。如果强行使用 BINARY 类型保存中文字符，虽然有时也能正常读写，但并不带有字符集信息，很容易出现数据乱码甚至数据损坏等情况。 -- BINARY 类型理论上最长可以有 16374 字节。binary 仅支持字符串输入，字符串两端需使用单引号引用。使用时须指定大小，如 binary(20) 定义了最长为 20 个单字节字符的字符串，每个字符占 1 byte 的存储空间，总共固定占用 20 bytes 的空间，此时如果用户字符串超出 20 字节将会报错。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示，即 `\’`。 +- BINARY 类型理论上最长可以有 16,374 字节。BINARY 仅支持字符串输入，字符串两端需使用单引号引用。使用时须指定大小，如 BINARY(20) 定义了最长为 20 个单字节字符的字符串，每个字符占 1 字节的存储空间，总共固定占用 20 字节的空间，此时如果用户字符串超出 20 字节将会报错。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示，即 `\'`。 - SQL 语句中的数值类型将依据是否存在小数点，或使用科学计数法表示，来判断数值类型是否为整型或者浮点型，因此在使用时要注意相应类型越界的情况。例如，9999999999999999999 会认为超过长整型的上边界而溢出，而 9999999999999999999.0 会被认为是有效的浮点数。 ::: - ## 常量 -TDengine支持多个类型的常量，细节如下表： -| # | **语法** | **类型** | **说明** | -| --- | :-------: | --------- | -------------------------------------- | -| 1 | [{+ \| -}]123 | BIGINT | 整型数值的字面量的类型均为BIGINT。如果用户输入超过了BIGINT的表示范围，TDengine 按BIGINT对数值进行截断。| -| 2 | 123.45 | DOUBLE | 浮点数值的字面量的类型均为DOUBLE。TDengine依据是否存在小数点，或使用科学计数法表示，来判断数值类型是否为整型或者浮点型。| -| 3 | 1.2E3 | DOUBLE | 科学计数法的字面量的类型为DOUBLE。| -| 4 | 'abc' | BINARY | 单引号括住的内容为字符串字面值，其类型为BINARY，BINARY的size为实际的字符个数。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示，即 \'。| -| 5 | "abc" | BINARY | 双引号括住的内容为字符串字面值，其类型为BINARY，BINARY的size为实际的字符个数。对于字符串内的双引号，可以用转义字符反斜线加单引号来表示，即 \"。 | -| 6 | TIMESTAMP {'literal' \| "literal"} | TIMESTAMP | TIMESTAMP关键字表示后面的字符串字面量需要被解释为TIMESTAMP类型。字符串需要满足YYYY-MM-DD HH:mm:ss.MS格式，其时间分辨率为当前数据库的时间分辨率。 | -| 7 | {TRUE \| FALSE} | BOOL | 布尔类型字面量。 | -| 8 | {'' \| "" \| '\t' \| "\t" \| ' ' \| " " \| NULL } | -- | 空值字面量。可以用于任意类型。| +TDengine 支持多个类型的常量，细节如下表： + +| # | **语法** | **类型** | **说明** | +| --- | :-----------------------------------------------: | --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | [{+ \| -}]123 | BIGINT | 整型数值的字面量的类型均为 BIGINT。如果用户输入超过了 BIGINT 的表示范围，TDengine 按 BIGINT 对数值进行截断。 | +| 2 | 123.45 | DOUBLE | 浮点数值的字面量的类型均为 DOUBLE。TDengine 依据是否存在小数点，或使用科学计数法表示，来判断数值类型是否为整型或者浮点型。 | +| 3 | 1.2E3 | DOUBLE | 科学计数法的字面量的类型为 DOUBLE。 | +| 4 | 'abc' | BINARY | 单引号括住的内容为字符串字面值，其类型为 BINARY，BINARY 的 Size 为实际的字符个数。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示，即 `\'`。 | +| 5 | "abc" | BINARY | 双引号括住的内容为字符串字面值，其类型为 BINARY，BINARY 的 Size 为实际的字符个数。对于字符串内的双引号，可以用转义字符反斜线加单引号来表示，即 `\"`。 | +| 6 | TIMESTAMP {'literal' \| "literal"} | TIMESTAMP | TIMESTAMP 关键字表示后面的字符串字面量需要被解释为 TIMESTAMP 类型。字符串需要满足 YYYY-MM-DD HH:mm:ss.MS 格式，其时间分辨率为当前数据库的时间分辨率。 | +| 7 | {TRUE \| FALSE} | BOOL | 布尔类型字面量。 | +| 8 | {'' \| "" \| '\t' \| "\t" \| ' ' \| " " \| NULL } | -- | 空值字面量。可以用于任意类型。 | :::note -- TDengine依据是否存在小数点，或使用科学计数法表示，来判断数值类型是否为整型或者浮点型，因此在使用时要注意相应类型越界的情况。例如，9999999999999999999会认为超过长整型的上边界而溢出，而9999999999999999999.0会被认为是有效的浮点数。 + +- TDengine 依据是否存在小数点，或使用科学计数法表示，来判断数值类型是否为整型或者浮点型，因此在使用时要注意相应类型越界的情况。例如，9999999999999999999 会认为超过长整型的上边界而溢出，而 9999999999999999999.0 会被认为是有效的浮点数。 ::: From 2f475399a698301987f90f3ceea3bafddde2417c Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 21 Sep 2022 10:57:41 +0800 Subject: [PATCH 31/52] enh: added memory allocators for parser and planner --- include/libs/nodes/nodes.h | 8 +++- include/libs/scheduler/scheduler.h | 1 + source/client/inc/clientInt.h | 3 +- source/client/src/clientEnv.c | 3 +- source/client/src/clientImpl.c | 4 +- source/client/src/clientMain.c | 12 +++--- source/libs/nodes/src/nodesUtilFuncs.c | 58 +++++++++++++++++++++++++- source/libs/scheduler/inc/schInt.h | 3 +- source/libs/scheduler/src/schJob.c | 2 + 9 files changed, 81 insertions(+), 13 deletions(-) diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 1e2a53f598..3d5b04ddf4 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -278,9 +278,15 @@ typedef struct SNodeList { typedef struct SNodeAllocator SNodeAllocator; int32_t nodesCreateNodeAllocator(int32_t chunkSize, SNodeAllocator** pAllocator); -void nodesDestroyNodeAllocator(SNodeAllocator* pAllocator); +void nodesDestroyNodeAllocator(void* pAllocator); void nodesResetThreadLevelAllocator(SNodeAllocator* pAllocator); +int32_t nodesAllocatorInit(); +int32_t nodesCreateAllocator(int32_t chunkSize, int64_t* pRefId); +void nodesDestroyAllocator(int64_t refId); +void nodesResetAllocator(int64_t refId); +int64_t nodesIncAllocatorRefCount(int64_t refId); + SNode* nodesMakeNode(ENodeType type); void nodesDestroyNode(SNode* pNode); diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index e6973cd390..738d057e6a 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -67,6 +67,7 @@ typedef struct SSchedulerReq { SRequestConnInfo *pConn; SArray *pNodeList; SQueryPlan *pDag; + int64_t allocatorRefId; const char *sql; int64_t startTs; schedulerExecFp execFp; diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index f042c3ad7c..eca1a0ebbe 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -250,7 +250,8 @@ typedef struct SRequestObj { bool inRetry; uint32_t prevCode; // previous error code: todo refactor, add update flag for catalog uint32_t retry; - SNodeAllocator* pNodeAllocator; + // SNodeAllocator* pNodeAllocator; + int64_t allocatorRefId; } SRequestObj; typedef struct SSyncQueryParam { diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 8fded03472..e95a2d2871 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -349,7 +349,7 @@ void doDestroyRequest(void *p) { taosArrayDestroy(pRequest->tableList); taosArrayDestroy(pRequest->dbList); taosArrayDestroy(pRequest->targetTableList); - nodesDestroyNodeAllocator(pRequest->pNodeAllocator); + nodesDestroyAllocator(pRequest->allocatorRefId); destroyQueryExecRes(&pRequest->body.resInfo.execRes); @@ -412,6 +412,7 @@ void taos_init_imp(void) { initTaskQueue(); fmFuncMgtInit(); + nodesAllocatorInit(); clientConnRefPool = taosOpenRef(200, destroyTscObj); clientReqRefPool = taosOpenRef(40960, doDestroyRequest); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index bc1cdc247e..75e0966243 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -194,8 +194,9 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, return TSDB_CODE_TSC_OUT_OF_MEMORY; } + (*pRequest)->allocatorRefId = -1; if (tsQueryUseNodeAllocator && !qIsInsertValuesSql((*pRequest)->sqlstr, (*pRequest)->sqlLen)) { - if (TSDB_CODE_SUCCESS != nodesCreateNodeAllocator(tsQueryNodeChunkSize, &((*pRequest)->pNodeAllocator))) { + if (TSDB_CODE_SUCCESS != nodesCreateAllocator(tsQueryNodeChunkSize, &((*pRequest)->allocatorRefId))) { tscError("%d failed to create node allocator, reqId:0x%" PRIx64 ", conn:%d, %s", (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql); @@ -1058,6 +1059,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM .pConn = &conn, .pNodeList = pNodeList, .pDag = pDag, + .allocatorRefId = pRequest->allocatorRefId, .sql = pRequest->sqlstr, .startTs = pRequest->metric.start, .execFp = schedulerExecCb, diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index e19d88fcf3..c3f8ca32b8 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -700,7 +700,7 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { pRequest->metric.ctgEnd = taosGetTimestampUs(); - nodesResetThreadLevelAllocator(pRequest->pNodeAllocator); + nodesResetAllocator(pRequest->allocatorRefId); if (code == TSDB_CODE_SUCCESS) { code = qAnalyseSqlSemantic(pWrapper->pCtx, &pWrapper->catalogReq, pResultMeta, pQuery); @@ -728,11 +728,11 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { pRequest->requestId); launchAsyncQuery(pRequest, pQuery, pResultMeta); qDestroyQuery(pQuery); - nodesResetThreadLevelAllocator(NULL); + nodesResetAllocator(-1); } else { destorySqlParseWrapper(pWrapper); qDestroyQuery(pQuery); - nodesResetThreadLevelAllocator(NULL); + nodesResetAllocator(-1); if (NEED_CLIENT_HANDLE_ERROR(code)) { tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); @@ -805,7 +805,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { } SQuery *pQuery = NULL; - nodesResetThreadLevelAllocator(pRequest->pNodeAllocator); + nodesResetAllocator(pRequest->allocatorRefId); pRequest->metric.syntaxStart = taosGetTimestampUs(); @@ -849,7 +849,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { &pRequest->body.queryJob); pCxt = NULL; if (code == TSDB_CODE_SUCCESS) { - nodesResetThreadLevelAllocator(NULL); + nodesResetAllocator(-1); return; } @@ -857,7 +857,7 @@ _error: tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->requestId); taosMemoryFree(pCxt); - nodesResetThreadLevelAllocator(NULL); + nodesResetAllocator(-1); terrno = code; pRequest->code = code; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 8b9b723a2a..b7caead3e5 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -21,6 +21,7 @@ #include "taoserror.h" #include "tdatablock.h" #include "thash.h" +#include "tref.h" typedef struct SNodeMemChunk { int32_t availableSize; @@ -30,6 +31,7 @@ typedef struct SNodeMemChunk { } SNodeMemChunk; typedef struct SNodeAllocator { + int64_t self; int32_t chunkSize; int32_t chunkNum; SNodeMemChunk* pCurrChunk; @@ -37,6 +39,22 @@ typedef struct SNodeAllocator { } SNodeAllocator; static threadlocal SNodeAllocator* pNodeAllocator; +static int32_t allocatorReqRefPool = -1; + +int32_t nodesAllocatorInit() { + if (allocatorReqRefPool >= 0) { + nodesWarn("nodes already initialized"); + return TSDB_CODE_SUCCESS; + } + + allocatorReqRefPool = taosOpenRef(40960, nodesDestroyNodeAllocator); + if (allocatorReqRefPool < 0) { + nodesError("init nodes failed"); + return TSDB_CODE_OUT_OF_MEMORY; + } + + return TSDB_CODE_SUCCESS; +} static SNodeMemChunk* callocNodeChunk(SNodeAllocator* pAllocator) { SNodeMemChunk* pNewChunk = taosMemoryCalloc(1, sizeof(SNodeMemChunk) + pAllocator->chunkSize); @@ -103,11 +121,13 @@ int32_t nodesCreateNodeAllocator(int32_t chunkSize, SNodeAllocator** pAllocator) return TSDB_CODE_SUCCESS; } -void nodesDestroyNodeAllocator(SNodeAllocator* pAllocator) { - if (NULL == pAllocator) { +void nodesDestroyNodeAllocator(void* p) { + if (NULL == p) { return; } + SNodeAllocator* pAllocator = p; + nodesDebug("alloc chunkNum: %d, chunkTotakSize: %d", pAllocator->chunkNum, pAllocator->chunkNum * pAllocator->chunkSize); @@ -122,6 +142,40 @@ void nodesDestroyNodeAllocator(SNodeAllocator* pAllocator) { void nodesResetThreadLevelAllocator(SNodeAllocator* pAllocator) { pNodeAllocator = pAllocator; } +int32_t nodesCreateAllocator(int32_t chunkSize, int64_t* pRefId) { + SNodeAllocator* pAllocator = NULL; + int32_t code = nodesCreateNodeAllocator(chunkSize, &pAllocator); + if (TSDB_CODE_SUCCESS == code) { + pAllocator->self = taosAddRef(allocatorReqRefPool, pAllocator); + *pRefId = pAllocator->self; + } + return code; +} + +void nodesDestroyAllocator(int64_t refId) { + if (refId < 0) { + return; + } + taosReleaseRef(allocatorReqRefPool, refId); +} + +void nodesResetAllocator(int64_t refId) { + if (refId < 0) { + pNodeAllocator = NULL; + } else { + pNodeAllocator = taosAcquireRef(allocatorReqRefPool, refId); + taosReleaseRef(allocatorReqRefPool, refId); + } +} + +int64_t nodesIncAllocatorRefCount(int64_t refId) { + if (refId < 0) { + return -1; + } + SNodeAllocator* pAllocator = taosAcquireRef(allocatorReqRefPool, refId); + return pAllocator->self; +} + static SNode* makeNode(ENodeType type, int32_t size) { SNode* p = nodesCalloc(1, size); if (NULL == p) { diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 7fea286732..a62531a875 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -254,7 +254,8 @@ typedef struct SSchJob { SRequestConnInfo conn; SArray *nodeList; // qnode/vnode list, SArray SArray *levels; // starting from 0. SArray - SQueryPlan *pDag; + SQueryPlan *pDag; + int64_t allocatorRefId; SArray *dataSrcTasks; // SArray int32_t levelIdx; diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 98501427ab..345f4680b0 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -673,6 +673,7 @@ void schFreeJobImpl(void *job) { destroyQueryExecRes(&pJob->execRes); qDestroyQueryPlan(pJob->pDag); + nodesDestroyAllocator(pJob->allocatorRefId); taosMemoryFreeClear(pJob->userRes.execRes); taosMemoryFreeClear(pJob->fetchRes); @@ -724,6 +725,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { pJob->sql = strdup(pReq->sql); } pJob->pDag = pReq->pDag; + pJob->allocatorRefId = nodesIncAllocatorRefCount(pReq->allocatorRefId); pJob->chkKillFp = pReq->chkKillFp; pJob->chkKillParam = pReq->chkKillParam; pJob->userRes.execFp = pReq->execFp; From a939ec47b91dded11672e7117139cd893040fa5d Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Wed, 21 Sep 2022 11:25:33 +0800 Subject: [PATCH 32/52] Update 06-select.md --- docs/zh/12-taos-sql/06-select.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md index 5841904e61..394e4a99d5 100644 --- a/docs/zh/12-taos-sql/06-select.md +++ b/docs/zh/12-taos-sql/06-select.md @@ -69,7 +69,7 @@ order_expr: ### 通配符 -通配符 \* 可以用于代指全部列。对于普通表，结果中只有普通列。对于超级表和子表，还包含了 TAG 列。 +通配符 \* 可以用于代指全部列。对于普通表和子表，结果中只有普通列。对于超级表，还包含了 TAG 列。 ```sql SELECT * FROM d1001; From 4ebdab3540a90fe747fa90ad49d1c6d1abba5902 Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Wed, 21 Sep 2022 11:26:30 +0800 Subject: [PATCH 33/52] Update 06-select.md --- docs/en/12-taos-sql/06-select.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md index e409973173..786444ad77 100644 --- a/docs/en/12-taos-sql/06-select.md +++ b/docs/en/12-taos-sql/06-select.md @@ -68,7 +68,7 @@ A query can be performed on some or all columns. Data and tag columns can all be ### Wildcards -You can use an asterisk (\*) as a wildcard character to indicate all columns. For standard tables, the asterisk indicates only data columns. For supertables and subtables, tag columns are also included. +You can use an asterisk (\*) as a wildcard character to indicate all columns. For normal tables or sub-tables, the asterisk indicates only data columns. For supertables, tag columns are also included when using asterisk (\*). ```sql SELECT * FROM d1001; From a914816ce75111b15d75dbaf4a93255da9dba975 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 21 Sep 2022 11:28:34 +0800 Subject: [PATCH 34/52] enh: added memory allocators for parser and planner --- source/client/inc/clientInt.h | 3 +-- source/client/src/clientEnv.c | 1 + source/libs/nodes/src/nodesUtilFuncs.c | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index eca1a0ebbe..7a9c665556 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -250,8 +250,7 @@ typedef struct SRequestObj { bool inRetry; uint32_t prevCode; // previous error code: todo refactor, add update flag for catalog uint32_t retry; - // SNodeAllocator* pNodeAllocator; - int64_t allocatorRefId; + int64_t allocatorRefId; } SRequestObj; typedef struct SSyncQueryParam { diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index e95a2d2871..5019314fed 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -288,6 +288,7 @@ void *createRequest(uint64_t connId, int32_t type) { pRequest->body.resInfo.convertUcs4 = true; // convert ucs4 by default pRequest->type = type; + pRequest->allocatorRefId = -1; pRequest->pDb = getDbOfConnection(pTscObj); pRequest->pTscObj = pTscObj; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index b7caead3e5..480d2cce72 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -153,14 +153,14 @@ int32_t nodesCreateAllocator(int32_t chunkSize, int64_t* pRefId) { } void nodesDestroyAllocator(int64_t refId) { - if (refId < 0) { + if (refId <= 0) { return; } taosReleaseRef(allocatorReqRefPool, refId); } void nodesResetAllocator(int64_t refId) { - if (refId < 0) { + if (refId <= 0) { pNodeAllocator = NULL; } else { pNodeAllocator = taosAcquireRef(allocatorReqRefPool, refId); @@ -169,7 +169,7 @@ void nodesResetAllocator(int64_t refId) { } int64_t nodesIncAllocatorRefCount(int64_t refId) { - if (refId < 0) { + if (refId <= 0) { return -1; } SNodeAllocator* pAllocator = taosAcquireRef(allocatorReqRefPool, refId); From d3a21816ccefd85a616454c18d413b6d5c936708 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 21 Sep 2022 11:43:36 +0800 Subject: [PATCH 35/52] shell: taos shell websocket reconnect error --- tools/shell/src/shellUtil.c | 2 +- tools/shell/src/shellWebsocket.c | 39 ++++++++++++++++++-------------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/tools/shell/src/shellUtil.c b/tools/shell/src/shellUtil.c index 0430428c38..8c47d16555 100644 --- a/tools/shell/src/shellUtil.c +++ b/tools/shell/src/shellUtil.c @@ -143,7 +143,7 @@ void shellCheckConnectMode() { shell.args.port = 6041; } shell.args.dsn = taosMemoryCalloc(1, 1024); - snprintf(shell.args.dsn, 1024, "ws://%s:%d/rest/ws", + snprintf(shell.args.dsn, 1024, "ws://%s:%d", shell.args.host, shell.args.port); } shell.args.cloud = false; diff --git a/tools/shell/src/shellWebsocket.c b/tools/shell/src/shellWebsocket.c index b8b8392b96..94bb909e29 100644 --- a/tools/shell/src/shellWebsocket.c +++ b/tools/shell/src/shellWebsocket.c @@ -206,26 +206,31 @@ void shellRunSingleCommandWebsocketImp(char *command) { printMode = true; // When output to a file, the switch does not work. } - if (!shell.ws_conn && shell_conn_ws_server(0)) { - return; - } - shell.stop_query = false; - st = taosGetTimestampUs(); + WS_RES* res; - WS_RES* res = ws_query_timeout(shell.ws_conn, command, shell.args.timeout); - int code = ws_errno(res); - if (code != 0) { - et = taosGetTimestampUs(); - fprintf(stderr, "\nDB: error: %s (%.6fs)\n", ws_errstr(res), (et - st)/1E6); - if (code == TSDB_CODE_WS_SEND_TIMEOUT || code == TSDB_CODE_WS_RECV_TIMEOUT) { - fprintf(stderr, "Hint: use -t to increase the timeout in seconds\n"); - } else if (code == TSDB_CODE_WS_INTERNAL_ERRO || code == TSDB_CODE_WS_CLOSED) { - fprintf(stderr, "TDengine server is down, will try to reconnect\n"); - shell.ws_conn = NULL; + for (int reconnectNum = 0; reconnectNum < 2; reconnectNum++) { + if (!shell.ws_conn && shell_conn_ws_server(0)) { + return; } - ws_free_result(res); - return; + st = taosGetTimestampUs(); + + res = ws_query_timeout(shell.ws_conn, command, shell.args.timeout); + int code = ws_errno(res); + if (code != 0 && !shell.stop_query) { + et = taosGetTimestampUs(); + fprintf(stderr, "\nDB: error: %s (%.6fs)\n", ws_errstr(res), (et - st)/1E6); + if (code == TSDB_CODE_WS_SEND_TIMEOUT || code == TSDB_CODE_WS_RECV_TIMEOUT) { + fprintf(stderr, "Hint: use -t to increase the timeout in seconds\n"); + } else if (code == TSDB_CODE_WS_INTERNAL_ERRO || code == TSDB_CODE_WS_CLOSED) { + fprintf(stderr, "TDengine server is down, will try to reconnect\n"); + shell.ws_conn = NULL; + } + ws_free_result(res); + if (reconnectNum == 0) continue; + return; + } + break; } double execute_time = ws_take_timing(res)/1E6; From bc9655745709862990fac16da5757d59e88b1b25 Mon Sep 17 00:00:00 2001 From: Huo Linhe Date: Wed, 21 Sep 2022 11:47:27 +0800 Subject: [PATCH 36/52] fix: core in case of null args with taos -c dir/ -R --- cmake/taosws_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in index 627cd53c09..ca8fff8da5 100644 --- a/cmake/taosws_CMakeLists.txt.in +++ b/cmake/taosws_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosws-rs ExternalProject_Add(taosws-rs GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git - GIT_TAG 76bc64d + GIT_TAG 1bdfca3 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 72fba5a8ce28671f619d9d831a1cc6f73ef3d1da Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 21 Sep 2022 11:54:44 +0800 Subject: [PATCH 37/52] fix(stream): check delete uid --- include/common/tcommon.h | 1 + include/libs/stream/tstream.h | 15 ++++++++-- source/dnode/vnode/src/tq/tq.c | 40 +++++++++++++++++++++++-- source/libs/executor/src/scanoperator.c | 40 +++++++++++++++++++++++++ source/libs/stream/src/stream.c | 2 +- source/libs/stream/src/streamData.c | 10 +++++++ source/libs/stream/src/streamDispatch.c | 21 ++++++++----- source/libs/stream/src/streamExec.c | 3 ++ 8 files changed, 119 insertions(+), 13 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index ba4baa0130..2544cedda7 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -116,6 +116,7 @@ enum { STREAM_INPUT__DATA_RETRIEVE, STREAM_INPUT__GET_RES, STREAM_INPUT__CHECKPOINT, + STREAM_INPUT__REF_DATA_BLOCK, STREAM_INPUT__DESTROY, }; diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 554d66d621..bdc12f7e3f 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -125,6 +125,14 @@ typedef struct { SArray* blocks; // SArray } SStreamDataBlock; +// ref data block, for delete +typedef struct { + int8_t type; + int64_t ver; + int32_t* dataRef; + SSDataBlock* pBlock; +} SStreamRefDataBlock; + typedef struct { int8_t type; } SStreamCheckpoint; @@ -339,7 +347,8 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem qDebug("task %d %p submit enqueue %p %p %p", pTask->taskId, pTask, pItem, pSubmitClone, pSubmitClone->data); taosWriteQitem(pTask->inputQueue->queue, pSubmitClone); // qStreamInput(pTask->exec.executor, pSubmitClone); - } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { + } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE || + pItem->type == STREAM_INPUT__REF_DATA_BLOCK) { taosWriteQitem(pTask->inputQueue->queue, pItem); // qStreamInput(pTask->exec.executor, pItem); } else if (pItem->type == STREAM_INPUT__CHECKPOINT) { @@ -492,7 +501,9 @@ typedef struct { int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq); int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq); -void tFreeStreamDispatchReq(SStreamDispatchReq* pReq); +void tDeleteStreamRetrieveReq(SStreamRetrieveReq* pReq); + +void tDeleteStreamDispatchReq(SStreamDispatchReq* pReq); int32_t streamSetupTrigger(SStreamTask* pTask); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 98b7dd7163..29217e29a4 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -59,7 +59,7 @@ static void destroySTqHandle(void* data) { tqCloseReader(pData->execHandle.pExecReader); walCloseReader(pData->pWalReader); taosHashCleanup(pData->execHandle.execDb.pFilterOutTbUid); - } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__TABLE){ + } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { walCloseReader(pData->pWalReader); tqCloseReader(pData->execHandle.pExecReader); } @@ -664,7 +664,10 @@ int32_t tqProcessVgChangeReq(STQ* pTq, int64_t version, char* msg, int32_t msgLe tqError("vgId:%d, build new consumer handle %s for consumer %d, but old consumerId is %ld", req.vgId, req.subKey, req.newConsumerId, req.oldConsumerId); } - ASSERT(req.newConsumerId != -1); + if (req.newConsumerId == -1) { + tqError("vgId:%d, tq invalid rebalance request, new consumerId %ld", req.vgId, req.newConsumerId); + return 0; + } STqHandle tqHandle = {0}; pHandle = &tqHandle; /*taosInitRWLatch(&pExec->lock);*/ @@ -876,6 +879,9 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { taosArrayDestroy(pRes->uidList); + int32_t* pRef = taosMemoryMalloc(sizeof(int32_t)); + *pRef = 1; + void* pIter = NULL; while (1) { pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); @@ -885,6 +891,33 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { qDebug("delete req enqueue stream task: %d, ver: %" PRId64, pTask->taskId, ver); + if (!failed) { + SStreamRefDataBlock* pRefBlock = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM); + pRefBlock->type = STREAM_INPUT__REF_DATA_BLOCK; + pRefBlock->pBlock = pDelBlock; + pRefBlock->dataRef = pRef; + atomic_add_fetch_32(pRefBlock->dataRef, 1); + + if (streamTaskInput(pTask, (SStreamQueueItem*)pRefBlock) < 0) { + qError("stream task input del failed, task id %d", pTask->taskId); + continue; + } + if (streamSchedExec(pTask) < 0) { + qError("stream task launch failed, task id %d", pTask->taskId); + continue; + } + } else { + streamTaskInputFail(pTask); + } + } + int32_t ref = atomic_sub_fetch_32(pRef, 1); + ASSERT(ref >= 0); + if (ref == 0) { + taosMemoryFree(pDelBlock); + taosMemoryFree(pRef); + } + +#if 0 SStreamDataBlock* pStreamBlock = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); pStreamBlock->type = STREAM_INPUT__DATA_BLOCK; pStreamBlock->blocks = taosArrayInit(0, sizeof(SSDataBlock)); @@ -908,6 +941,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { } } blockDataDestroy(pDelBlock); +#endif return 0; } @@ -1045,6 +1079,7 @@ int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) { SDecoder decoder; tDecoderInit(&decoder, msgBody, msgLen); tDecodeStreamRetrieveReq(&decoder, &req); + tDecoderClear(&decoder); int32_t taskId = req.dstTaskId; SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); if (pTask) { @@ -1053,6 +1088,7 @@ int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) { .code = 0, }; streamProcessRetrieveReq(pTask, &req, &rsp); + tDeleteStreamRetrieveReq(&req); return 0; } else { return -1; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8f9a78db83..4d16b2e170 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1480,6 +1480,40 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { } } +static int32_t filterDelBlockByUid(SSDataBlock* pDst, const SSDataBlock* pSrc, SStreamScanInfo* pInfo) { + STqReader* pReader = pInfo->tqReader; + int32_t rows = pSrc->info.rows; + blockDataEnsureCapacity(pDst, rows); + + SColumnInfoData* pSrcStartCol = taosArrayGet(pSrc->pDataBlock, START_TS_COLUMN_INDEX); + uint64_t* startCol = (uint64_t*)pSrcStartCol->pData; + SColumnInfoData* pSrcEndCol = taosArrayGet(pSrc->pDataBlock, END_TS_COLUMN_INDEX); + uint64_t* endCol = (uint64_t*)pSrcEndCol->pData; + SColumnInfoData* pSrcUidCol = taosArrayGet(pSrc->pDataBlock, UID_COLUMN_INDEX); + uint64_t* uidCol = (uint64_t*)pSrcUidCol->pData; + + SColumnInfoData* pDstStartCol = taosArrayGet(pDst->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pDstEndCol = taosArrayGet(pDst->pDataBlock, END_TS_COLUMN_INDEX); + SColumnInfoData* pDstUidCol = taosArrayGet(pDst->pDataBlock, UID_COLUMN_INDEX); + int32_t j = 0; + for (int32_t i = 0; i < rows; i++) { + if (taosHashGet(pReader->tbIdHash, &uidCol[i], sizeof(uint64_t))) { + colDataAppend(pDstStartCol, j, (const char*)&startCol[i], false); + colDataAppend(pDstEndCol, j, (const char*)&endCol[i], false); + colDataAppend(pDstUidCol, j, (const char*)&uidCol[i], false); + + colDataAppendNULL(taosArrayGet(pDst->pDataBlock, GROUPID_COLUMN_INDEX), j); + colDataAppendNULL(taosArrayGet(pDst->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX), j); + colDataAppendNULL(taosArrayGet(pDst->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX), j); + j++; + } + } + pDst->info = pSrc->info; + pDst->info.rows = j; + + return 0; +} + static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -1568,6 +1602,12 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { } break; case STREAM_DELETE_DATA: { printDataBlock(pBlock, "stream scan delete recv"); + if (pInfo->tqReader) { + SSDataBlock* pDelBlock = createSpecialDataBlock(STREAM_DELETE_DATA); + filterDelBlockByUid(pDelBlock, pBlock, pInfo); + pBlock = pDelBlock; + } + printDataBlock(pBlock, "stream scan delete recv filtered"); if (!isIntervalWindow(pInfo) && !isSessionWindow(pInfo) && !isStateWindow(pInfo)) { generateDeleteResultBlock(pInfo, pBlock, pInfo->pDeleteDataRes); pInfo->pDeleteDataRes->info.type = STREAM_DELETE_RESULT; diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index d6e87c2736..4a63cd3bb2 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -182,7 +182,7 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S pReq->upstreamTaskId); streamTaskEnqueue(pTask, pReq, pRsp); - tFreeStreamDispatchReq(pReq); + tDeleteStreamDispatchReq(pReq); if (exec) { if (streamTryExec(pTask) < 0) { diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index cd5f499c34..e6705a77b2 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -179,5 +179,15 @@ void streamFreeQitem(SStreamQueueItem* data) { taosArrayDestroy(pMerge->reqs); taosArrayDestroy(pMerge->dataRefs); taosFreeQitem(pMerge); + } else if (type == STREAM_INPUT__REF_DATA_BLOCK) { + SStreamRefDataBlock* pRefBlock = (SStreamRefDataBlock*)data; + + int32_t ref = atomic_sub_fetch_32(pRefBlock->dataRef, 1); + ASSERT(ref >= 0); + if (ref == 0) { + blockDataDestroy(pRefBlock->pBlock); + taosMemoryFree(pRefBlock->dataRef); + } + taosFreeQitem(pRefBlock); } } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 7cdb7c0db9..e6960ae350 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -62,7 +62,7 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) { return 0; } -void tFreeStreamDispatchReq(SStreamDispatchReq* pReq) { +void tDeleteStreamDispatchReq(SStreamDispatchReq* pReq) { taosArrayDestroyP(pReq->data, taosMemoryFree); taosArrayDestroy(pReq->dataLen); } @@ -95,7 +95,10 @@ int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq) { return 0; } +void tDeleteStreamRetrieveReq(SStreamRetrieveReq* pReq) { taosMemoryFree(pReq->pRetrieve); } + int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) { + int32_t code = -1; SRetrieveTableRsp* pRetrieve = NULL; void* buf = NULL; int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); @@ -143,7 +146,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) buf = rpcMallocCont(sizeof(SMsgHead) + len); if (buf == NULL) { - goto FAIL; + goto CLEAR; } ((SMsgHead*)buf)->vgId = htonl(pEpInfo->nodeId); @@ -151,6 +154,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) SEncoder encoder; tEncoderInit(&encoder, abuf, len); tEncodeStreamRetrieveReq(&encoder, &req); + tEncoderClear(&encoder); SRpcMsg rpcMsg = { .code = 0, @@ -161,17 +165,18 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) if (tmsgSendReq(&pEpInfo->epSet, &rpcMsg) < 0) { ASSERT(0); - return -1; + goto CLEAR; } + buf = NULL; qDebug("task %d(child %d) send retrieve req to task %d at node %d, reqId %" PRId64, pTask->taskId, pTask->selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req.reqId); } - return 0; -FAIL: - if (pRetrieve) taosMemoryFree(pRetrieve); - if (buf) taosMemoryFree(buf); - return -1; + code = 0; +CLEAR: + taosMemoryFree(pRetrieve); + rpcFreeCont(buf); + return code; } static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDispatchReq* pReq) { diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 102bad7426..5ad5aa549d 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -38,6 +38,9 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* SArray* blocks = pMerged->reqs; qDebug("task %d %p set submit input (merged), batch num: %d", pTask->taskId, pTask, (int32_t)blocks->size); qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__MERGED_SUBMIT); + } else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) { + const SStreamRefDataBlock* pRefBlock = (const SStreamRefDataBlock*)data; + qSetMultiStreamInput(exec, pRefBlock->pBlock, 1, STREAM_INPUT__DATA_BLOCK); } else { ASSERT(0); } From a1eca821765f30bc1364f1b3828affe9230cf207 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 21 Sep 2022 13:23:54 +0800 Subject: [PATCH 38/52] feat: update taos-tools 8207c74 for 3.0 (#16976) --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index b75d4607b8..52e2c0944a 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 285b5e0 + GIT_TAG 8207c74 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 90c300e3202f73d879eb5f66bc2b92769638a371 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 21 Sep 2022 13:53:40 +0800 Subject: [PATCH 39/52] enh: added memory allocators for parser and planner --- source/client/src/clientImpl.c | 2 ++ source/client/src/clientMain.c | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 75e0966243..b74dca3cfe 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1046,6 +1046,8 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM pRequest->body.subplanNum = pDag->numOfSubplans; } + nodesResetAllocator(-1); + pRequest->metric.planEnd = taosGetTimestampUs(); if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index c3f8ca32b8..77bd71c756 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -728,7 +728,6 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { pRequest->requestId); launchAsyncQuery(pRequest, pQuery, pResultMeta); qDestroyQuery(pQuery); - nodesResetAllocator(-1); } else { destorySqlParseWrapper(pWrapper); qDestroyQuery(pQuery); @@ -811,6 +810,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { SCatalogReq catalogReq = {.forceUpdate = updateMetaForce, .qNodeRequired = qnodeRequired(pRequest)}; code = qParseSqlSyntax(pCxt, &pQuery, &catalogReq); + nodesResetAllocator(-1); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -849,7 +849,6 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { &pRequest->body.queryJob); pCxt = NULL; if (code == TSDB_CODE_SUCCESS) { - nodesResetAllocator(-1); return; } @@ -857,7 +856,6 @@ _error: tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->requestId); taosMemoryFree(pCxt); - nodesResetAllocator(-1); terrno = code; pRequest->code = code; From 5a7ad1344b38c21d1467a84dbbab047987270850 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 21 Sep 2022 14:36:44 +0800 Subject: [PATCH 40/52] fix: remove stream api and use tmq syntax in lua example (#16979) --- examples/lua/lua51/lua_connector51.c | 71 ++++--------------- examples/lua/lua_connector.c | 4 +- examples/lua/test.lua | 100 ++++++++++++++++++--------- 3 files changed, 87 insertions(+), 88 deletions(-) diff --git a/examples/lua/lua51/lua_connector51.c b/examples/lua/lua51/lua_connector51.c index 578622bf1f..ac840913f9 100644 --- a/examples/lua/lua51/lua_connector51.c +++ b/examples/lua/lua51/lua_connector51.c @@ -1,11 +1,11 @@ +#include #include #include -#include #include -#include "../../../../include/client/taos.h" -#include "lauxlib.h" #include "lua.h" +#include "lauxlib.h" #include "lualib.h" +#include struct cb_param{ lua_State* state; @@ -35,7 +35,7 @@ static int l_connect(lua_State *L){ } lua_getfield(L, 1, "port"); - if (lua_isnumber(L,-1)){ + if (lua_isnumber(L, -1)){ port = lua_tonumber(L, -1); //printf("port = %d\n", port); } @@ -102,7 +102,7 @@ static int l_query(lua_State *L){ printf("failed, reason:%s\n", taos_errstr(result)); lua_pushinteger(L, -1); lua_setfield(L, table_index, "code"); - lua_pushstring(L, taos_errstr(taos)); + lua_pushstring(L, taos_errstr(result)); lua_setfield(L, table_index, "error"); return 1; @@ -113,7 +113,6 @@ static int l_query(lua_State *L){ int rows = 0; int num_fields = taos_field_count(result); const TAOS_FIELD *fields = taos_fetch_fields(result); - //char temp[256]; const int affectRows = taos_affected_rows(result); // printf(" affect rows:%d\r\n", affectRows); @@ -122,7 +121,7 @@ static int l_query(lua_State *L){ lua_pushinteger(L, affectRows); lua_setfield(L, table_index, "affected"); lua_newtable(L); - + while ((row = taos_fetch_row(result))) { //printf("row index:%d\n",rows); rows++; @@ -136,17 +135,21 @@ static int l_query(lua_State *L){ } lua_pushstring(L,fields[i].name); - + int32_t* length = taos_fetch_lengths(result); switch (fields[i].type) { + case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_TINYINT: lua_pushinteger(L,*((char *)row[i])); break; + case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_SMALLINT: lua_pushinteger(L,*((short *)row[i])); break; + case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_INT: lua_pushinteger(L,*((int *)row[i])); break; + case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_BIGINT: lua_pushinteger(L,*((int64_t *)row[i])); break; @@ -156,9 +159,11 @@ static int l_query(lua_State *L){ case TSDB_DATA_TYPE_DOUBLE: lua_pushnumber(L,*((double *)row[i])); break; + case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: - lua_pushstring(L,(char *)row[i]); + //printf("type:%d, max len:%d, current len:%d\n",fields[i].type, fields[i].bytes, length[i]); + lua_pushlstring(L,(char *)row[i], length[i]); break; case TSDB_DATA_TYPE_TIMESTAMP: lua_pushinteger(L,*((int64_t *)row[i])); @@ -166,6 +171,7 @@ static int l_query(lua_State *L){ case TSDB_DATA_TYPE_BOOL: lua_pushinteger(L,*((char *)row[i])); break; + case TSDB_DATA_TYPE_NULL: default: lua_pushnil(L); break; @@ -297,51 +303,6 @@ void stream_cb(void *param, TAOS_RES *result, TAOS_ROW row){ // printf("-----------------------------------------------------------------------------------\n\r"); } -static int l_open_stream(lua_State *L){ - int r = luaL_ref(L, LUA_REGISTRYINDEX); - TAOS * taos = (TAOS*)lua_topointer(L,1); - const char * sqlstr = lua_tostring(L,2); - int stime = luaL_checknumber(L,3); - - lua_newtable(L); - int table_index = lua_gettop(L); - - struct cb_param *p = malloc(sizeof(struct cb_param)); - p->state = L; - p->callback=r; - // printf("r:%d, L:%d\n",r,L); - void * s = taos_open_stream(taos,sqlstr,stream_cb,stime,p,NULL); - if (s == NULL) { - printf("failed to open stream, reason:%s\n", taos_errstr(taos)); - free(p); - lua_pushnumber(L, -1); - lua_setfield(L, table_index, "code"); - lua_pushstring(L, taos_errstr(taos)); - lua_setfield(L, table_index, "error"); - lua_pushlightuserdata(L,NULL); - lua_setfield(L, table_index, "stream"); - }else{ - // printf("success to open stream\n"); - lua_pushnumber(L, 0); - lua_setfield(L, table_index, "code"); - lua_pushstring(L, taos_errstr(taos)); - lua_setfield(L, table_index, "error"); - p->stream = s; - lua_pushlightuserdata(L,p); - lua_setfield(L, table_index, "stream");//stream has different content in lua and c. - } - - return 1; -} - -static int l_close_stream(lua_State *L){ - //TODO:get stream and free cb_param - struct cb_param *p = lua_touserdata(L,1); - taos_close_stream(p->stream); - free(p); - return 0; -} - static int l_close(lua_State *L){ TAOS *taos= (TAOS*)lua_topointer(L,1); lua_newtable(L); @@ -367,8 +328,6 @@ static const struct luaL_Reg lib[] = { {"query", l_query}, {"query_a",l_async_query}, {"close", l_close}, - {"open_stream", l_open_stream}, - {"close_stream", l_close_stream}, {NULL, NULL} }; diff --git a/examples/lua/lua_connector.c b/examples/lua/lua_connector.c index 3c13b196b9..ce13ab3829 100644 --- a/examples/lua/lua_connector.c +++ b/examples/lua/lua_connector.c @@ -5,7 +5,7 @@ #include #include #include -#include "taos.h" +#include struct cb_param{ lua_State* state; @@ -60,6 +60,8 @@ static int l_connect(lua_State *L){ lua_settop(L,0); + taos_init(); + lua_newtable(L); int table_index = lua_gettop(L); diff --git a/examples/lua/test.lua b/examples/lua/test.lua index 89c0904c6a..ff49b8408d 100644 --- a/examples/lua/test.lua +++ b/examples/lua/test.lua @@ -9,6 +9,50 @@ local config = { max_packet_size = 1024 * 1024 } +function dump(obj) + local getIndent, quoteStr, wrapKey, wrapVal, dumpObj + getIndent = function(level) + return string.rep("\t", level) + end + quoteStr = function(str) + return '"' .. string.gsub(str, '"', '\\"') .. '"' + end + wrapKey = function(val) + if type(val) == "number" then + return "[" .. val .. "]" + elseif type(val) == "string" then + return "[" .. quoteStr(val) .. "]" + else + return "[" .. tostring(val) .. "]" + end + end + wrapVal = function(val, level) + if type(val) == "table" then + return dumpObj(val, level) + elseif type(val) == "number" then + return val + elseif type(val) == "string" then + return quoteStr(val) + else + return tostring(val) + end + end + dumpObj = function(obj, level) + if type(obj) ~= "table" then + return wrapVal(obj) + end + level = level + 1 + local tokens = {} + tokens[#tokens + 1] = "{" + for k, v in pairs(obj) do + tokens[#tokens + 1] = getIndent(level) .. wrapKey(k) .. " = " .. wrapVal(v, level) .. "," + end + tokens[#tokens + 1] = getIndent(level - 1) .. "}" + return table.concat(tokens, "\n") + end + return dumpObj(obj, 0) +end + local conn local res = driver.connect(config) if res.code ~=0 then @@ -37,7 +81,7 @@ else print("select db--- pass.") end -res = driver.query(conn,"create table m1 (ts timestamp, speed int,owner binary(20))") +res = driver.query(conn,"create table m1 (ts timestamp, speed int, owner binary(20), mark nchar(30))") if res.code ~=0 then print("create table---failed: "..res.error) return @@ -45,7 +89,7 @@ else print("create table--- pass.") end -res = driver.query(conn,"insert into m1 values ('2019-09-01 00:00:00.001',0,'robotspace'), ('2019-09-01 00:00:00.002',1,'Hilink'),('2019-09-01 00:00:00.003',2,'Harmony')") +res = driver.query(conn,"insert into m1 values ('2019-09-01 00:00:00.001', 0, 'robotspace', '世界人民大团结万岁'), ('2019-09-01 00:00:00.002', 1, 'Hilink', '⾾⾿⿀⿁⿂⿃⿄⿅⿆⿇⿈⿉⿊⿋⿌⿍⿎⿏⿐⿑⿒⿓⿔⿕'),('2019-09-01 00:00:00.003', 2, 'Harmony', '₠₡₢₣₤₥₦₧₨₩₪₫€₭₮₯₰₱₲₳₴₵')") if res.code ~=0 then print("insert records failed: "..res.error) return @@ -64,21 +108,25 @@ if res.code ~=0 then return else if (#(res.item) == 3) then - print("select--- pass") + print("select--- pass") + print(res.item[1].mark) + print(res.item[2].mark) + print(res.item[3].mark) + else print("select--- failed: expect 3 affected records, actually received "..#(res.item)) end end -res = driver.query(conn,"CREATE TABLE thermometer (ts timestamp, degree double) TAGS(location binary(20), type int)") +res = driver.query(conn,"create table thermometer (ts timestamp, degree double) tags(location binary(20), type int)") if res.code ~=0 then print(res.error) return else print("create super table--- pass") end -res = driver.query(conn,"CREATE TABLE therm1 USING thermometer TAGS ('beijing', 1)") +res = driver.query(conn,"create table therm1 using thermometer tags ('beijing', 1)") if res.code ~=0 then print(res.error) return @@ -86,7 +134,7 @@ else print("create table--- pass") end -res = driver.query(conn,"INSERT INTO therm1 VALUES ('2019-09-01 00:00:00.001', 20),('2019-09-01 00:00:00.002', 21)") +res = driver.query(conn,"insert into therm1 values ('2019-09-01 00:00:00.001', 20),('2019-09-01 00:00:00.002', 21)") if res.code ~=0 then print(res.error) @@ -99,14 +147,14 @@ else end end -res = driver.query(conn,"SELECT COUNT(*) count, AVG(degree) AS av, MAX(degree), MIN(degree) FROM thermometer WHERE location='beijing' or location='tianjin' GROUP BY location, type") +res = driver.query(conn,"select count(*) as cnt, avg(degree) as av, max(degree), min(degree) from thermometer where location='beijing' or location='tianjin' group by location, type") if res.code ~=0 then print("select from super table--- failed:"..res.error) return else print("select from super table--- pass") for i = 1, #(res.item) do - print("res:"..res.item[i].count) + print("res:"..res.item[i].cnt) end end @@ -125,33 +173,16 @@ function async_query_callback(res) end end -driver.query_a(conn,"INSERT INTO therm1 VALUES ('2019-09-01 00:00:00.005', 100),('2019-09-01 00:00:00.006', 101),('2019-09-01 00:00:00.007', 102)", async_query_callback) +driver.query_a(conn,"insert into therm1 values ('2019-09-01 00:00:00.005', 100),('2019-09-01 00:00:00.006', 101),('2019-09-01 00:00:00.007', 102)", async_query_callback) +res = driver.query(conn, "create stream stream_avg_degree into avg_degree as select avg(degree) from thermometer interval(5s) sliding(1s)") -function stream_callback(t) - print("------------------------") - print("continuous query result:") - for key, value in pairs(t) do - print("key:"..key..", value:"..value) - end -end - -local stream -res = driver.open_stream(conn,"SELECT COUNT(*) as count, AVG(degree) as avg, MAX(degree) as max, MIN(degree) as min FROM thermometer interval(2s) sliding(2s);)",0, stream_callback) -if res.code ~=0 then - print("open stream--- failed:"..res.error) - return -else - print("open stream--- pass") - stream = res.stream -end - -print("From now on we start continous insert in an definite (infinite if you want) loop.") +print("From now on we start continous insertion in an definite (infinite if you want) loop.") local loop_index = 0 while loop_index < 30 do local t = os.time()*1000 local v = loop_index - res = driver.query(conn,string.format("INSERT INTO therm1 VALUES (%d, %d)",t,v)) + res = driver.query(conn,string.format("insert into therm1 values (%d, %d)",t,v)) if res.code ~=0 then print("continous insertion--- failed:" .. res.error) @@ -159,10 +190,17 @@ while loop_index < 30 do else --print("insert successfully, affected:"..res.affected) end + local res1 = driver.query(conn, string.format("select last(*) from avg_degree")) + if res1.code ~=0 then + print("select failed: "..res1.error) + return + else +-- print(dump(res1)) + if(#res1.item > 0) then print("avg_degree: " .. res1.item[1]["last(avg(degree))"]) end + end + os.execute("sleep " .. 1) loop_index = loop_index + 1 end -driver.close_stream(stream) - driver.close(conn) From 47b7aeabbc4cff145058c53238aa8c9f2ac119c0 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 21 Sep 2022 15:21:56 +0800 Subject: [PATCH 41/52] enh(tdb): more debug output for page's overflow cells --- source/libs/tdb/src/db/tdbPage.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c index 1e2eefabf4..f4878ea861 100644 --- a/source/libs/tdb/src/db/tdbPage.c +++ b/source/libs/tdb/src/db/tdbPage.c @@ -80,6 +80,7 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) ASSERT(xFree); for (int iOvfl = 0; iOvfl < pPage->nOverflow; iOvfl++) { + tdbDebug("tdbPage/destroy/free ovfl cell: %p/%p", pPage->apOvfl[iOvfl], pPage); tdbOsFree(pPage->apOvfl[iOvfl]); } @@ -152,7 +153,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl pNewCell = (SCell *)tdbOsMalloc(szCell); memcpy(pNewCell, pCell, szCell); - tdbDebug("tdbPage/new ovfl cell: %p", pNewCell); + tdbDebug("tdbPage/insert/new ovfl cell: %p/%p", pNewCell, pPage); pPage->apOvfl[iOvfl] = pNewCell; pPage->aiOvfl[iOvfl] = idx; @@ -202,7 +203,7 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) { if (pPage->aiOvfl[iOvfl] == idx) { // remove the over flow cell tdbOsFree(pPage->apOvfl[iOvfl]); - tdbDebug("tdbPage/free ovfl cell: %p", pPage->apOvfl[iOvfl]); + tdbDebug("tdbPage/drop/free ovfl cell: %p", pPage->apOvfl[iOvfl]); for (; (++iOvfl) < pPage->nOverflow;) { pPage->aiOvfl[iOvfl - 1] = pPage->aiOvfl[iOvfl] - 1; pPage->apOvfl[iOvfl - 1] = pPage->apOvfl[iOvfl]; @@ -255,6 +256,7 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) { int szCell = (*pFromPage->xCellSize)(pFromPage, pFromPage->apOvfl[iOvfl], 0, NULL, NULL); pNewCell = (SCell *)tdbOsMalloc(szCell); memcpy(pNewCell, pFromPage->apOvfl[iOvfl], szCell); + tdbDebug("tdbPage/copy/new ovfl cell: %p/%p/%p", pNewCell, pToPage, pFromPage); } pToPage->apOvfl[iOvfl] = pNewCell; From e7526a081b5ac5139b9d5d33bd34900b2c4da14b Mon Sep 17 00:00:00 2001 From: tomchon Date: Wed, 21 Sep 2022 16:26:43 +0800 Subject: [PATCH 42/52] test:modify testcase of data Compatibility --- tests/system-test/0-others/compatibility.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index 2b059615e9..25b023bb76 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -48,7 +48,7 @@ class TDTestCase: selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): - cfgPath = buildPath + "/../community/sim/dnode1/cfg/" + cfgPath = buildPath + "/../sim/dnode1/cfg/" else: cfgPath = buildPath + "/../sim/dnode1/cfg/" @@ -143,9 +143,12 @@ class TDTestCase: tdsql.query("select * from db.ct4") tdsql.checkData(0,1,14) tdsql.query("describe information_schema.ins_databases;") - # for i in tdsql.queryResult[i][0]: - # tdsql.checkData(0,1,13) - + qRows=tdsql.queryRows + for i in range(qRows) : + if tdsql.queryResult[i][0]=="retentions" : + return True + else: + return False def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") From 5ca1dc10056801ca776e26602aede1b8231a50e7 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Wed, 21 Sep 2022 14:24:26 +0800 Subject: [PATCH 43/52] fix(stream): free array --- source/libs/executor/src/scanoperator.c | 1 + source/libs/executor/src/timewindowoperator.c | 40 +++---------------- 2 files changed, 6 insertions(+), 35 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 4d16b2e170..18a75f0d04 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1687,6 +1687,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { while (1) { if (pInfo->tqReader->pMsg == NULL) { if (pInfo->validBlockIndex >= totBlockNum) { + updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); return NULL; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index aed4ae9b26..971f3618a0 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1695,6 +1695,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) { } nodesDestroyNode((SNode*)pInfo->pPhyNode); colDataDestroy(&pInfo->twAggSup.timeWindowData); + cleanupGroupResInfo(&pInfo->groupResInfo); taosMemoryFreeClear(param); } @@ -3073,6 +3074,7 @@ void processPullOver(SSDataBlock* pBlock, SHashObj* pMap) { taosArrayRemove(chArray, index); if (taosArrayGetSize(chArray) == 0) { // pull data is over + taosArrayDestroy(chArray); taosHashRemove(pMap, &winRes, sizeof(SWinKey)); } } @@ -3109,9 +3111,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info; SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); TSKEY maxTs = INT64_MIN; TSKEY minTs = INT64_MAX; @@ -3175,6 +3174,9 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } } + SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -5755,8 +5757,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); - SStreamState* pState = pTaskInfo->streamInfo.pState; - while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -5805,36 +5805,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { } pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, minTs); - -#if 0 - if (pState) { - printf(">>>>>>>> stream read backend\n"); - SWinKey key = { - .ts = 1, - .groupId = 2, - }; - char* val = NULL; - int32_t sz; - if (streamStateGet(pState, &key, (void**)&val, &sz) < 0) { - ASSERT(0); - } - printf("stream read %s %d\n", val, sz); - streamFreeVal(val); - - SStreamStateCur* pCur = streamStateGetCur(pState, &key); - ASSERT(pCur); - while (streamStateCurNext(pState, pCur) == 0) { - SWinKey key1; - const void* val1; - if (streamStateGetKVByCur(pCur, &key1, &val1, &sz) < 0) { - break; - } - printf("stream iter key groupId:%d ts:%d, value %s %d\n", key1.groupId, key1.ts, val1, sz); - } - streamStateFreeCur(pCur); - } -#endif - pOperator->status = OP_RES_TO_RETURN; closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdatedMap, pOperator); From 5116e0a01b5f619e129a92808d6192bea9c1632b Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 21 Sep 2022 17:04:12 +0800 Subject: [PATCH 44/52] enh: added memory allocators for parser and planner --- include/libs/nodes/nodes.h | 17 ++- include/libs/parser/parser.h | 1 + include/libs/planner/planner.h | 1 + source/client/src/clientEnv.c | 2 +- source/client/src/clientImpl.c | 8 +- source/client/src/clientMain.c | 8 +- source/libs/nodes/src/nodesUtilFuncs.c | 148 ++++++++++++++-------- source/libs/parser/src/parser.c | 19 ++- source/libs/planner/src/planner.c | 6 +- source/libs/planner/test/planTestMain.cpp | 2 + source/libs/planner/test/planTestUtil.cpp | 14 +- source/libs/scheduler/src/schJob.c | 4 +- 12 files changed, 143 insertions(+), 87 deletions(-) diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 3d5b04ddf4..634dae9ec5 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -277,15 +277,14 @@ typedef struct SNodeList { typedef struct SNodeAllocator SNodeAllocator; -int32_t nodesCreateNodeAllocator(int32_t chunkSize, SNodeAllocator** pAllocator); -void nodesDestroyNodeAllocator(void* pAllocator); -void nodesResetThreadLevelAllocator(SNodeAllocator* pAllocator); - -int32_t nodesAllocatorInit(); -int32_t nodesCreateAllocator(int32_t chunkSize, int64_t* pRefId); -void nodesDestroyAllocator(int64_t refId); -void nodesResetAllocator(int64_t refId); -int64_t nodesIncAllocatorRefCount(int64_t refId); +int32_t nodesInitAllocatorSet(); +void nodesDestroyAllocatorSet(); +int32_t nodesCreateAllocator(int64_t queryId, int32_t chunkSize, int64_t* pAllocatorId); +int32_t nodesAcquireAllocator(int64_t allocatorId); +int32_t nodesReleaseAllocator(int64_t allocatorId); +int64_t nodesMakeAllocatorWeakRef(int64_t allocatorId); +int64_t nodesReleaseAllocatorWeakRef(int64_t allocatorId); +void nodesDestroyAllocator(int64_t allocatorId); SNode* nodesMakeNode(ENodeType type); void nodesDestroyNode(SNode* pNode); diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 95bde85864..b1a937910d 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -56,6 +56,7 @@ typedef struct SParseContext { bool nodeOffline; SArray* pTableMetaPos; // sql table pos => catalog data pos SArray* pTableVgroupPos; // sql table pos => catalog data pos + int64_t allocatorId; } SParseContext; int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery); diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index e03ac3811a..e52fe39527 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -39,6 +39,7 @@ typedef struct SPlanContext { int32_t msgLen; const char* pUser; bool sysInfo; + int64_t allocatorId; } SPlanContext; // Create the physical plan for the query, according to the AST. diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 5019314fed..2faf268880 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -413,7 +413,7 @@ void taos_init_imp(void) { initTaskQueue(); fmFuncMgtInit(); - nodesAllocatorInit(); + nodesInitAllocatorSet(); clientConnRefPool = taosOpenRef(200, destroyTscObj); clientReqRefPool = taosOpenRef(40960, doDestroyRequest); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index c0809ca822..ef19eba7fe 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -197,7 +197,8 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, (*pRequest)->allocatorRefId = -1; if (tsQueryUseNodeAllocator && !qIsInsertValuesSql((*pRequest)->sqlstr, (*pRequest)->sqlLen)) { - if (TSDB_CODE_SUCCESS != nodesCreateAllocator(tsQueryNodeChunkSize, &((*pRequest)->allocatorRefId))) { + if (TSDB_CODE_SUCCESS != + nodesCreateAllocator((*pRequest)->requestId, tsQueryNodeChunkSize, &((*pRequest)->allocatorRefId))) { tscError("%d failed to create node allocator, reqId:0x%" PRIx64 ", conn:%d, %s", (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql); @@ -1035,7 +1036,8 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM .pMsg = pRequest->msgBuf, .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, .pUser = pRequest->pTscObj->user, - .sysInfo = pRequest->pTscObj->sysInfo}; + .sysInfo = pRequest->pTscObj->sysInfo, + .allocatorId = pRequest->allocatorRefId}; SAppInstInfo* pAppInfo = getAppInfo(pRequest); SQueryPlan* pDag = NULL; @@ -1047,8 +1049,6 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM pRequest->body.subplanNum = pDag->numOfSubplans; } - nodesResetAllocator(-1); - pRequest->metric.planEnd = taosGetTimestampUs(); if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index a03b6cd5a5..ae386b819c 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -700,8 +700,6 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { pRequest->metric.ctgEnd = taosGetTimestampUs(); - nodesResetAllocator(pRequest->allocatorRefId); - if (code == TSDB_CODE_SUCCESS) { code = qAnalyseSqlSemantic(pWrapper->pCtx, &pWrapper->catalogReq, pResultMeta, pQuery); pRequest->stableQuery = pQuery->stableQuery; @@ -731,7 +729,6 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { } else { destorySqlParseWrapper(pWrapper); qDestroyQuery(pQuery); - nodesResetAllocator(-1); if (NEED_CLIENT_HANDLE_ERROR(code)) { tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); @@ -778,7 +775,8 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) { .enableSysInfo = pTscObj->sysInfo, .async = true, .svrVer = pTscObj->sVer, - .nodeOffline = (pTscObj->pAppInfo->onlineDnodes < pTscObj->pAppInfo->totalDnodes)}; + .nodeOffline = (pTscObj->pAppInfo->onlineDnodes < pTscObj->pAppInfo->totalDnodes), + .allocatorId = pRequest->allocatorRefId}; return TSDB_CODE_SUCCESS; } @@ -804,13 +802,11 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { } SQuery *pQuery = NULL; - nodesResetAllocator(pRequest->allocatorRefId); pRequest->metric.syntaxStart = taosGetTimestampUs(); SCatalogReq catalogReq = {.forceUpdate = updateMetaForce, .qNodeRequired = qnodeRequired(pRequest)}; code = qParseSqlSyntax(pCxt, &pQuery, &catalogReq); - nodesResetAllocator(-1); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 480d2cce72..2e5a0d935b 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -32,29 +32,16 @@ typedef struct SNodeMemChunk { typedef struct SNodeAllocator { int64_t self; + int64_t queryId; int32_t chunkSize; int32_t chunkNum; SNodeMemChunk* pCurrChunk; SNodeMemChunk* pChunks; + TdThreadMutex mutex; } SNodeAllocator; -static threadlocal SNodeAllocator* pNodeAllocator; -static int32_t allocatorReqRefPool = -1; - -int32_t nodesAllocatorInit() { - if (allocatorReqRefPool >= 0) { - nodesWarn("nodes already initialized"); - return TSDB_CODE_SUCCESS; - } - - allocatorReqRefPool = taosOpenRef(40960, nodesDestroyNodeAllocator); - if (allocatorReqRefPool < 0) { - nodesError("init nodes failed"); - return TSDB_CODE_OUT_OF_MEMORY; - } - - return TSDB_CODE_SUCCESS; -} +static threadlocal SNodeAllocator* g_pNodeAllocator; +static int32_t g_allocatorReqRefPool = -1; static SNodeMemChunk* callocNodeChunk(SNodeAllocator* pAllocator) { SNodeMemChunk* pNewChunk = taosMemoryCalloc(1, sizeof(SNodeMemChunk) + pAllocator->chunkSize); @@ -77,17 +64,17 @@ static SNodeMemChunk* callocNodeChunk(SNodeAllocator* pAllocator) { } static void* nodesCallocImpl(int32_t size) { - if (NULL == pNodeAllocator) { + if (NULL == g_pNodeAllocator) { return taosMemoryCalloc(1, size); } - if (pNodeAllocator->pCurrChunk->usedSize + size > pNodeAllocator->pCurrChunk->availableSize) { - if (NULL == callocNodeChunk(pNodeAllocator)) { + if (g_pNodeAllocator->pCurrChunk->usedSize + size > g_pNodeAllocator->pCurrChunk->availableSize) { + if (NULL == callocNodeChunk(g_pNodeAllocator)) { return NULL; } } - void* p = pNodeAllocator->pCurrChunk->pBuf + pNodeAllocator->pCurrChunk->usedSize; - pNodeAllocator->pCurrChunk->usedSize += size; + void* p = g_pNodeAllocator->pCurrChunk->pBuf + g_pNodeAllocator->pCurrChunk->usedSize; + g_pNodeAllocator->pCurrChunk->usedSize += size; return p; } @@ -96,7 +83,7 @@ static void* nodesCalloc(int32_t num, int32_t size) { if (NULL == p) { return NULL; } - *(char*)p = (NULL != pNodeAllocator) ? 1 : 0; + *(char*)p = (NULL != g_pNodeAllocator) ? 1 : 0; return (char*)p + 1; } @@ -108,7 +95,7 @@ static void nodesFree(void* p) { return; } -int32_t nodesCreateNodeAllocator(int32_t chunkSize, SNodeAllocator** pAllocator) { +static int32_t createNodeAllocator(int32_t chunkSize, SNodeAllocator** pAllocator) { *pAllocator = taosMemoryCalloc(1, sizeof(SNodeAllocator)); if (NULL == *pAllocator) { return TSDB_CODE_OUT_OF_MEMORY; @@ -118,18 +105,19 @@ int32_t nodesCreateNodeAllocator(int32_t chunkSize, SNodeAllocator** pAllocator) taosMemoryFreeClear(*pAllocator); return TSDB_CODE_OUT_OF_MEMORY; } + taosThreadMutexInit(&(*pAllocator)->mutex, NULL); return TSDB_CODE_SUCCESS; } -void nodesDestroyNodeAllocator(void* p) { +static void destroyNodeAllocator(void* p) { if (NULL == p) { return; } SNodeAllocator* pAllocator = p; - nodesDebug("alloc chunkNum: %d, chunkTotakSize: %d", pAllocator->chunkNum, - pAllocator->chunkNum * pAllocator->chunkSize); + nodesDebug("query id %" PRIx64 " allocator id %" PRIx64 " alloc chunkNum: %d, chunkTotakSize: %d", + pAllocator->queryId, pAllocator->self, pAllocator->chunkNum, pAllocator->chunkNum * pAllocator->chunkSize); SNodeMemChunk* pChunk = pAllocator->pChunks; while (NULL != pChunk) { @@ -137,43 +125,101 @@ void nodesDestroyNodeAllocator(void* p) { taosMemoryFree(pChunk); pChunk = pTemp; } + taosThreadMutexDestroy(&pAllocator->mutex); taosMemoryFree(pAllocator); } -void nodesResetThreadLevelAllocator(SNodeAllocator* pAllocator) { pNodeAllocator = pAllocator; } +int32_t nodesInitAllocatorSet() { + if (g_allocatorReqRefPool >= 0) { + nodesWarn("nodes already initialized"); + return TSDB_CODE_SUCCESS; + } -int32_t nodesCreateAllocator(int32_t chunkSize, int64_t* pRefId) { + g_allocatorReqRefPool = taosOpenRef(1024, destroyNodeAllocator); + if (g_allocatorReqRefPool < 0) { + nodesError("init nodes failed"); + return TSDB_CODE_OUT_OF_MEMORY; + } + + return TSDB_CODE_SUCCESS; +} + +void nodesDestroyAllocatorSet() { + if (g_allocatorReqRefPool >= 0) { + SNodeAllocator* pAllocator = taosIterateRef(g_allocatorReqRefPool, 0); + int64_t refId = 0; + while (NULL != pAllocator) { + refId = pAllocator->self; + taosRemoveRef(g_allocatorReqRefPool, refId); + pAllocator = taosIterateRef(g_allocatorReqRefPool, refId); + } + taosCloseRef(g_allocatorReqRefPool); + } +} + +int32_t nodesCreateAllocator(int64_t queryId, int32_t chunkSize, int64_t* pAllocatorId) { SNodeAllocator* pAllocator = NULL; - int32_t code = nodesCreateNodeAllocator(chunkSize, &pAllocator); + int32_t code = createNodeAllocator(chunkSize, &pAllocator); if (TSDB_CODE_SUCCESS == code) { - pAllocator->self = taosAddRef(allocatorReqRefPool, pAllocator); - *pRefId = pAllocator->self; + pAllocator->self = taosAddRef(g_allocatorReqRefPool, pAllocator); + if (pAllocator->self <= 0) { + return terrno; + } + pAllocator->queryId = queryId; + *pAllocatorId = pAllocator->self; } return code; } -void nodesDestroyAllocator(int64_t refId) { - if (refId <= 0) { +int32_t nodesAcquireAllocator(int64_t allocatorId) { + if (allocatorId <= 0) { + return TSDB_CODE_SUCCESS; + } + + SNodeAllocator* pAllocator = taosAcquireRef(g_allocatorReqRefPool, allocatorId); + if (NULL == pAllocator) { + return terrno; + } + taosThreadMutexLock(&pAllocator->mutex); + g_pNodeAllocator = pAllocator; + return TSDB_CODE_SUCCESS; +} + +int32_t nodesReleaseAllocator(int64_t allocatorId) { + if (allocatorId <= 0) { + return TSDB_CODE_SUCCESS; + } + + if (NULL == g_pNodeAllocator) { + nodesError("allocator id %" PRIx64 + " release failed: The nodesReleaseAllocator function needs to be called after the nodesAcquireAllocator " + "function is called!", + allocatorId); + return TSDB_CODE_FAILED; + } + SNodeAllocator* pAllocator = g_pNodeAllocator; + g_pNodeAllocator = NULL; + taosThreadMutexUnlock(&pAllocator->mutex); + return taosReleaseRef(g_allocatorReqRefPool, allocatorId); +} + +int64_t nodesMakeAllocatorWeakRef(int64_t allocatorId) { + if (allocatorId <= 0) { + return 0; + } + + SNodeAllocator* pAllocator = taosAcquireRef(g_allocatorReqRefPool, allocatorId); + return pAllocator->self; +} + +int64_t nodesReleaseAllocatorWeakRef(int64_t allocatorId) { return taosReleaseRef(g_allocatorReqRefPool, allocatorId); } + +void nodesDestroyAllocator(int64_t allocatorId) { + if (allocatorId <= 0) { return; } - taosReleaseRef(allocatorReqRefPool, refId); -} -void nodesResetAllocator(int64_t refId) { - if (refId <= 0) { - pNodeAllocator = NULL; - } else { - pNodeAllocator = taosAcquireRef(allocatorReqRefPool, refId); - taosReleaseRef(allocatorReqRefPool, refId); - } -} - -int64_t nodesIncAllocatorRefCount(int64_t refId) { - if (refId <= 0) { - return -1; - } - SNodeAllocator* pAllocator = taosAcquireRef(allocatorReqRefPool, refId); - return pAllocator->self; + taosRemoveRef(g_allocatorReqRefPool, allocatorId); } static SNode* makeNode(ENodeType type, int32_t size) { diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 7ee6a5b223..940c676689 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -177,15 +177,18 @@ int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery) { int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq* pCatalogReq) { SParseMetaCache metaCache = {0}; - int32_t code = TSDB_CODE_SUCCESS; - if (qIsInsertValuesSql(pCxt->pSql, pCxt->sqlLen)) { - code = parseInsertSyntax(pCxt, pQuery, &metaCache); - } else { - code = parseSqlSyntax(pCxt, pQuery, &metaCache); + int32_t code = nodesAcquireAllocator(pCxt->allocatorId); + if (TSDB_CODE_SUCCESS == code) { + if (qIsInsertValuesSql(pCxt->pSql, pCxt->sqlLen)) { + code = parseInsertSyntax(pCxt, pQuery, &metaCache); + } else { + code = parseSqlSyntax(pCxt, pQuery, &metaCache); + } } if (TSDB_CODE_SUCCESS == code) { code = buildCatalogReq(pCxt, &metaCache, pCatalogReq); } + code = nodesReleaseAllocator(pCxt->allocatorId); destoryParseMetaCache(&metaCache, true); terrno = code; return code; @@ -194,7 +197,10 @@ int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCatalogReq, const struct SMetaData* pMetaData, SQuery* pQuery) { SParseMetaCache metaCache = {0}; - int32_t code = putMetaDataToCache(pCatalogReq, pMetaData, &metaCache, NULL == pQuery->pRoot); + int32_t code = nodesAcquireAllocator(pCxt->allocatorId); + if (TSDB_CODE_SUCCESS == code) { + code = putMetaDataToCache(pCatalogReq, pMetaData, &metaCache, NULL == pQuery->pRoot); + } if (TSDB_CODE_SUCCESS == code) { if (NULL == pQuery->pRoot) { code = parseInsertSql(pCxt, &pQuery, &metaCache); @@ -202,6 +208,7 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata code = analyseSemantic(pCxt, pQuery, &metaCache); } } + code = nodesReleaseAllocator(pCxt->allocatorId); destoryParseMetaCache(&metaCache, false); terrno = code; return code; diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 35903d45b1..0b6128ddec 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -33,7 +33,10 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo SLogicSubplan* pLogicSubplan = NULL; SQueryLogicPlan* pLogicPlan = NULL; - int32_t code = createLogicPlan(pCxt, &pLogicSubplan); + int32_t code = nodesAcquireAllocator(pCxt->allocatorId); + if (TSDB_CODE_SUCCESS == code) { + code = createLogicPlan(pCxt, &pLogicSubplan); + } if (TSDB_CODE_SUCCESS == code) { code = optimizeLogicPlan(pCxt, pLogicSubplan); } @@ -49,6 +52,7 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo if (TSDB_CODE_SUCCESS == code) { dumpQueryPlan(*pPlan); } + code = nodesReleaseAllocator(pCxt->allocatorId); nodesDestroyNode((SNode*)pLogicSubplan); nodesDestroyNode((SNode*)pLogicPlan); diff --git a/source/libs/planner/test/planTestMain.cpp b/source/libs/planner/test/planTestMain.cpp index e86bf90f51..df6e72ce46 100644 --- a/source/libs/planner/test/planTestMain.cpp +++ b/source/libs/planner/test/planTestMain.cpp @@ -32,6 +32,7 @@ class PlannerEnv : public testing::Environment { generateMetaData(); initLog(TD_TMP_DIR_PATH "td"); initCfg(); + nodesInitAllocatorSet(); } virtual void TearDown() { @@ -39,6 +40,7 @@ class PlannerEnv : public testing::Environment { qCleanupKeywordsTable(); fmFuncMgtDestroy(); taosCloseLog(); + nodesDestroyAllocatorSet(); } PlannerEnv() {} diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index 65aed6cadc..73d695195c 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -129,10 +129,10 @@ class PlannerTestBaseImpl { } void runImpl(const string& sql, int32_t queryPolicy) { - SNodeAllocator* pAllocator = NULL; + int64_t allocatorId = 0; if (g_useNodeAllocator) { - nodesCreateNodeAllocator(32 * 1024, &pAllocator); - nodesResetThreadLevelAllocator(pAllocator); + nodesCreateAllocator(sqlNo_, 32 * 1024, &allocatorId); + nodesAcquireAllocator(allocatorId); } reset(); @@ -166,13 +166,13 @@ class PlannerTestBaseImpl { dump(g_dumpModule); } catch (...) { dump(DUMP_MODULE_ALL); - nodesDestroyNodeAllocator(pAllocator); - nodesResetThreadLevelAllocator(NULL); + nodesReleaseAllocator(allocatorId); + nodesDestroyAllocator(allocatorId); throw; } - nodesDestroyNodeAllocator(pAllocator); - nodesResetThreadLevelAllocator(NULL); + nodesReleaseAllocator(allocatorId); + nodesDestroyAllocator(allocatorId); } void prepare(const string& sql) { diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 345f4680b0..9880490594 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -673,7 +673,7 @@ void schFreeJobImpl(void *job) { destroyQueryExecRes(&pJob->execRes); qDestroyQueryPlan(pJob->pDag); - nodesDestroyAllocator(pJob->allocatorRefId); + nodesReleaseAllocatorWeakRef(pJob->allocatorRefId); taosMemoryFreeClear(pJob->userRes.execRes); taosMemoryFreeClear(pJob->fetchRes); @@ -725,7 +725,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { pJob->sql = strdup(pReq->sql); } pJob->pDag = pReq->pDag; - pJob->allocatorRefId = nodesIncAllocatorRefCount(pReq->allocatorRefId); + pJob->allocatorRefId = nodesMakeAllocatorWeakRef(pReq->allocatorRefId); pJob->chkKillFp = pReq->chkKillFp; pJob->chkKillParam = pReq->chkKillParam; pJob->userRes.execFp = pReq->execFp; From 5680c03c1783db19eb313e471558972c444f7230 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 21 Sep 2022 17:07:22 +0800 Subject: [PATCH 45/52] enh: added memory allocators for parser and planner --- source/client/src/clientMain.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index ae386b819c..6b707bf7a0 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -65,6 +65,7 @@ void taos_cleanup(void) { fmFuncMgtDestroy(); qCleanupKeywordsTable(); + nodesDestroyAllocatorSet(); id = clientConnRefPool; clientConnRefPool = -1; From 62f5c1e465217c7fb8e83f8d1598d0e49326b3a0 Mon Sep 17 00:00:00 2001 From: dingbo Date: Wed, 21 Sep 2022 17:31:19 +0800 Subject: [PATCH 46/52] docs: release 3.0.1.2 --- docs/en/28-releases/01-tdengine.md | 3 +++ docs/en/28-releases/02-tools.md | 4 ++++ docs/zh/28-releases/01-tdengine.md | 3 +++ docs/zh/28-releases/02-tools.md | 4 ++++ 4 files changed, 14 insertions(+) diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index e3901114d3..414986d107 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -6,6 +6,9 @@ description: TDengine release history, Release Notes and download links. import Release from "/components/ReleaseV3"; +## 3.0.1.2 + + ## 3.0.1.1 diff --git a/docs/en/28-releases/02-tools.md b/docs/en/28-releases/02-tools.md index 228990df3b..086d3adea2 100644 --- a/docs/en/28-releases/02-tools.md +++ b/docs/en/28-releases/02-tools.md @@ -6,6 +6,10 @@ description: taosTools release history, Release Notes, download links. import Release from "/components/ReleaseV3"; +## 2.2.2 + + + ## 2.2.0 diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index 59a241b6c8..b05cf7a942 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -6,6 +6,9 @@ description: TDengine 发布历史、Release Notes 及下载链接 import Release from "/components/ReleaseV3"; +## 3.0.1.2 + + ## 3.0.1.1 diff --git a/docs/zh/28-releases/02-tools.md b/docs/zh/28-releases/02-tools.md index 9ff8f42224..f793981d06 100644 --- a/docs/zh/28-releases/02-tools.md +++ b/docs/zh/28-releases/02-tools.md @@ -6,6 +6,10 @@ description: taosTools 的发布历史、Release Notes 和下载链接 import Release from "/components/ReleaseV3"; +## 2.2.2 + + + ## 2.2.0 From 876303be5621c6b84b41bf5f8c24a851035ece40 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 21 Sep 2022 17:42:03 +0800 Subject: [PATCH 47/52] enh: added memory allocators for parser and planner --- source/libs/parser/src/parser.c | 4 ++-- source/libs/planner/src/planner.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 940c676689..2fe6ebfb79 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -188,7 +188,7 @@ int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq if (TSDB_CODE_SUCCESS == code) { code = buildCatalogReq(pCxt, &metaCache, pCatalogReq); } - code = nodesReleaseAllocator(pCxt->allocatorId); + nodesReleaseAllocator(pCxt->allocatorId); destoryParseMetaCache(&metaCache, true); terrno = code; return code; @@ -208,7 +208,7 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata code = analyseSemantic(pCxt, pQuery, &metaCache); } } - code = nodesReleaseAllocator(pCxt->allocatorId); + nodesReleaseAllocator(pCxt->allocatorId); destoryParseMetaCache(&metaCache, false); terrno = code; return code; diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 0b6128ddec..e4f02f12e6 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -52,7 +52,7 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo if (TSDB_CODE_SUCCESS == code) { dumpQueryPlan(*pPlan); } - code = nodesReleaseAllocator(pCxt->allocatorId); + nodesReleaseAllocator(pCxt->allocatorId); nodesDestroyNode((SNode*)pLogicSubplan); nodesDestroyNode((SNode*)pLogicPlan); From c5d659286b3fb740d4ca671c02532307e9bed808 Mon Sep 17 00:00:00 2001 From: tomchon Date: Wed, 21 Sep 2022 18:01:03 +0800 Subject: [PATCH 48/52] test: modify checkpackages scritps --- packaging/testpackage.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh index 794b3968fe..f604def9c4 100755 --- a/packaging/testpackage.sh +++ b/packaging/testpackage.sh @@ -202,8 +202,8 @@ elif [[ ${packgeName} =~ "tar" ]];then cd ${oriInstallPath}/${originTdpPath} && tar xf ${subFile} fi - cd ${oriInstallPath}/${originTdpPath} && tree > ${installPath}/base_${originversion}_checkfile - cd ${installPath}/${tdPath} && tree > ${installPath}/now_${version}_checkfile + cd ${oriInstallPath}/${originTdpPath} && tree -I "driver" > ${installPath}/base_${originversion}_checkfile + cd ${installPath}/${tdPath} && tree -I "driver" > ${installPath}/now_${version}_checkfile cd ${installPath} diff ${installPath}/base_${originversion}_checkfile ${installPath}/now_${version}_checkfile > ${installPath}/diffFile.log From e8a4f50c0a65559f7055b048693cb0754769316e Mon Sep 17 00:00:00 2001 From: tomchon Date: Wed, 21 Sep 2022 18:50:25 +0800 Subject: [PATCH 49/52] modify checkpackages scritps of muti-platform tes --- packaging/testpackage.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh index f604def9c4..20f93ecaec 100755 --- a/packaging/testpackage.sh +++ b/packaging/testpackage.sh @@ -215,6 +215,7 @@ elif [[ ${packgeName} =~ "tar" ]];then exit -1 else echoColor G "The number and names of files are the same as previous installation packages" + rm -rf ${installPath}/diffFile.log fi echoColor YD "===== install Package of tar =====" cd ${installPath}/${tdPath} @@ -251,6 +252,9 @@ if [[ ${packgeName} =~ "server" ]] ;then systemctl restart taosd fi +rm -rf ${installPath}/${packgeName} +rm -rf ${installPath}/${tdPath}/ + # if ([[ ${packgeName} =~ "Lite" ]] && [[ ${packgeName} =~ "tar" ]]) || [[ ${packgeName} =~ "client" ]] ;then # echoColor G "===== install taos-tools when package is lite or client =====" # cd ${installPath} From 16be3ca310777b9159c640b77370af0defba0201 Mon Sep 17 00:00:00 2001 From: robotspace Date: Wed, 21 Sep 2022 23:04:48 +0800 Subject: [PATCH 50/52] Remove error about stream in lua test case. (#16990) --- examples/lua/OpenResty/so/luaconnector51.so | Bin 22472 -> 17672 bytes examples/lua/lua51/lua_connector51.c | 73 +------------------- examples/lua/test.lua | 21 ++---- 3 files changed, 9 insertions(+), 85 deletions(-) diff --git a/examples/lua/OpenResty/so/luaconnector51.so b/examples/lua/OpenResty/so/luaconnector51.so index 442de6e39f909e1aeb869988722b84795c048855..168d3a9d2406680ceec3c12f29a8157d19aca2ff 100755 GIT binary patch literal 17672 zcmeHPeQ;dWb$=_{AdK+}upGw*yf}Oq1B+~8Vm@3}w$@&#O@y(Slq9oS?LJA1SG!{O z3EOl45rbi)D5H{;q?u{sbW&G^HWf4wL5mSzwji`42zV5lx~egZwwOfN1QcSd{?5JU zyuG{LM>?7Qk?H7a_MLlv_ndRjJ@>x*-n;Kz-5uVruFB`LB+an?$SQJ{72<;7)gnRI zD(h4n&$iA|vNMZCC*}6FS1d5?ZazC?Cnse)gxF*0G$>a=lU3E2rnN zo=_DFU^l((Bz~Xz8dPrxpY@Yi=E{ngUQ*B zlj&4=JZr}s+fh!DJs77VCzi=Ya_wO3Hm(9zi+)j| z;R~^;WLvw_m9w*nm=j~CIx}P;tE;%J%g%0hYfEQzR@sCvU1|2D)pp|Th!8@M(;~6f zR`_WrqS?&Woa-AmST#lKr0jH?(_WM?qeUL3lPRT5@lg>mbYYtP$=Y@_Yv;OBjup#o zPsgLm?`S-hO0~q|Ta<-tY%3y}RCy|(%m!$+S{>jracxQ~#xxU8Wpb*6Go5xi3V9BH zCcARER;K`WrQ`5F*+QNLbD1EDk}Yw{U16=?aNU}<(G|fJ!Id;5aoyv;Vonf~etM}X zsZRhxwi@SB*TN(&wN=(Vz$nRmliat|ovHC{e19bP1W%IPWw`LW%JBUGZKuBszmn|_ zl;H&mEnh2UhZeb&jV*^`H?dD!z@2qhEJZ&_OI2H?j6+qtY)oY1E0$n zkr4x@`wmH?25w$g#tghxgh3aCG;=ZtPgx2G7&v#8!j~Gj+~QI}oq^N+l%!Q5og@3y zHc6q7&Jj*ONs*Aw5hgJuq%lNCEi3nG#D6J7M~%M48`YM1FtslB?ew&;GZ+_1qObefd>ryd;?!<;1?Kpoq=Cy;HwOL zse%80%O3Tg^KoRy;p#}wtVb`itjK*sPSw;%WXB8D&#R_Rtvm>L>ik1E`WJ_gBfg!= z3S(19=f6q3kN6SAzeao&@k5G#nRuFV3ImFNj(D1C3cD2l6!A257Wx$b81Xda6uK4v zF!3~83Z07oYvO5&DKsnod&JY!QfO5CgT&7!KBV}s6Hik~p-%B%Bc7&^LO}6%5KmJ_ zp+@mp;%Uk*Sc<=uct7zI#{rz*LOe|!g)zn7LOe|wg%QPXBK{2GhZKK3@icW51{A-B zc$zW_yA*#F@ibKw`V_x{c$y*#-HKmEJWUOSPQ@=Fo~DFCv*H&LPg6mmQSoOGPg6i4 zr1;sy)2UymQ+yTibjlY3ivKtWo=){bjpC0GPp5dnQv81tPp5Wa;uC8BLE;0%k1765 z;?E_1MDec?Pp5EUNbxTdPp58SK=IEJPp52Q7x+l;>L&q2dS^XG7DF+J)fO6x zr%a9c7k5L}dY&od^Gbf@*N9wnJd&S?Jp0RQBhOCGi1_wKUOn#2gMm-*{6_|=r^Z_S ziyMmiG~T;choNhAUECDevHDF&T9N$Q&K#IdFSD%ii_q-x1e&^c*4dEreHC@P^^<;K zD+-&SgA=DQlD~5>vM)SBPDXmdgHO>=-b3d@B)@O`Y1kq((p&o=#+-}4cueDaU}=g3 zUjP7q7b+8E%PpBrCi3A)Hq&UDxvSjFC)&&c5_T>013cotWGEm01ska`ja*r7WR*5j z!I!(J$MWF`HuD-zE4N8Uep}Y0_W@LH(igOuaW>Ount7z$%)_2$Cdl!8cuX~xVn%^{ zgj@5J3ZkHovsIG#J4ociuk?g};r|Jux)H5uf{iY6XRteYR%w^#bK8f`Z5d^wSMGk9ponLkv82}V!p=}d=hx!>)PW%SNdp)*-fr=k2b5ESxou7>CEa9 zt#e(iCZ*LhV)*@BiRdxR$hx)5AUa6r!%NTeIn#`DrXQO9?%3I>DpupFUw0f`Kqwk> zbP4qhmq_yu{ze77pQQGq?tJ(VI+6q@B07?dT6YxCC`pYesn>Vh`MTx5_nSIEs^CHt z=q|k;ZbMXeX^qrS?9zV#)Uy#4h5K2e_b@W^LvZCjvB6kL-PHGybfwlcmPpU4PirbjE={MuGSu_$?*}i z@7=afuaHlx0IGZU8V}JE`{w;$L;N&d)xPnYP){E*d+4r&hVFk+xvy_Wc+v-(SBFRZ z-`GqSDpTfZ$N*4hr;xuvkEIHxN0bV*6hACHV)s-ICCCo!=LY(Yrr1(tinHbf6UZ;JSD80h{>;vT)=DvO7`yi1IAKr22E7tbA zpo=qR-}o9*Q`apUci%wtBzLjs6(_l2Ks_4|Q>8cwE`+q6u9~@GfVU|VYI0jxB7GcF zgqB_b>340Z{a)qLxN3+x)8J8aiRLq|<|N_aaXs$QAXUxfCBl!p!aoI0R&_Km(y3|V zVW55P1&4x+VHO^CV`wXPjwwuZeAGN0^WoBq(EU_L9v;C&Y1F@ZRMkFAb>0XmT!fyX zXM^#1RFFr1QlIuTL4Z!n(VnJbdFsw4fa-<=&3iS{Qyaz&GuikLvDZ_33!3PjS+&6R zfEq^hzl|wU2l-z`zPI*$fE=s)kC@V1TWt-b zTQk{&c-tCFrtCz($pm=CIsmEL?d&B1yj#d+(pR0In{CB23ERRugiIFMblTENB_gtl zZzUbO1IBN|ZOj_{cH<6V6u;TH`?w0fcKmkZw;#XHpqywf(_uS!8_^a><^o%@8N3k5 zY_ZZ^sZ@YpF$Apy-o6A??etPZzif~dc)j>8Ei&oEGrmPLu5JKC3k?f?H8s@?x*VPP z04PlsbX6l?he7w_zH%ItotvB~mntMM(Gb8>Yb%FaU>J#-KJ z61FjRt+_Sp<}SG2f9lriZtL2GS1-H#yd^-D4EgZ`jIqyS2C(Yq*6gfWd)i4eE}H>U zs+>CIaZEOIa9t^upFB&-?gwN3)~C~S1g0Y}9f9cxOh;fk0@D$gj=*#TrXyfRK)#bH|1QRqG4t|0 zjC}ucAfQX-`<6q@%l9jXnV0WVv~=;vk15uz^bCR|Jbmj!2``3p;*TTf#VDn(@XLMq z=0Lvxk#7p*-hZJMx8yq@dgh@d<-a~Yl~Dja+aS>+45d1H@IsPrO_p*w-Z-e#%l^qX zAqyC1G0O^RfW@qBTp-V5Y~GS@EC$(-;H$VI@w1x)gB5<2q&__ADgPI+Jk|~sFL9h; zeh(|+F-(<<-)!y=hD@xk?8+6~|Ld6E%(Rv1?M!zveTeC9rcW^4%XB}}cbI<2^d$b+ zWId3oc!L5$d7wrJ9Zb zv%da3-{rC8RyW%cT1iGerk&4mt0W`mXoAnxN-|Q1%qNv(Zee|)zpsq=y697U7GING zC-p_8MCaE`>;0+nGMb;NFl$M>q}^W@vG#|0pLCiwnhq6S*j#|SjqS;**798)KOLVB2UJo z;Ij}|pM}p`LV{?zh;fO(lwPRtGcEaERPfI$e4bT4f2{^yWnpsWF(>kCftL*hRdh4( zGEs%o{)b9^!LK7=h>PUjbioB5XbBAwTSazu|!&1YRi) z??IBJn&_}D~+qGJ@9oNb~by+w|U@nUscKfogVm~d)V3KAwS@O z|BDCyqKADeo^@KVDIXGEQjC|(1k?Ho38Br?&qRHh}CiYA;)HW!U`ZNr;` zj?R>gZ3e-*$`Z6|BN>flv$5?_JMCaELu)qHVMh~P9Ua@DVsKGNIwhjC`NH_(KNz=C z!Xu~OR-~fFh65_sTLzLX)FeBdpaw;w>o(PI3`fH^G|(QDDDFR^;Ru6~hE108(u%I% z@WnOt8=_xax9+C!=IG}7H5lb`WL1;#MCrs5XbF2Ji-(XvzK`uei9oQFzFd?k5qq z^&TIO{Up?|VsG)jAJQ!DBGQ4SjW*`~8#s!+Igs3-OsE|xN>X0n>1{Y#+1Mo)+3cmXB<1&mhK96W)X(0 zv)u~1P)k=bmAEvSu#`Z1EZ1%Y6Wh};rYUwisS>=(%q27F5+;hWteuLH0_QrhA1Fwt zwH0*iZ8*|wfU=C*?i94KWl7Bj?Fr}#r#19Mpfw~18$)c4xAEb=oY z!?Ye0AIH#TUnk7#DluKktmx4*iJATw3_W9uzN{Yzt!719Z@{gN>%TnL-vo@R5`9^Z z5Gv~r5K)P2nCZ)4Xe~hW<@sOe%`7bbi=5DQl+)UR@Up%jG{E}h{xfXRa|hWLeOb>C z+RYA2{SrTE|2?d~h7HR4h)`KiA$|Hdy;F5kyoHKZM_jjHw@B zJ)zCgFjkOz5TSoorr*ssM?&9p7q+wk(PMMgcgpnTxkad~gNb~(|9{E)uA2dtWJ=!& zm`UO%*xw*SBBC$PQ$ia=kZF1S|5T>mc&-){I`enb|1`ws;wPt_tnbM_{c?YW7y1%( z>6u&fW&Lm4r2vscPIEDdwnme)mw47H-zyBJoadw&iWvul-Y_COy literal 22472 zcmeHPe{@vUoxe!{BOntHs0Dva5nMo=4R}D5ViRD(pb17K9_iLjCdq`%I+=+x69p@p zU6eXTD&p4K{iBENww86fyU{A5v>4DO(b{daT^c=ho4Vkck#3EGY-%a9pYOf*`*PpQ zdwO=y?&Z>Iw~bxg~(m|~=DxsJz4=W0Esq)Vm#c&3H+>laIQ z46RkLV9Itk!cNAi<)214?HhD`UU}K2<7cXyE88|5l~B6Yu5S=O_}-0I{py#GPCvJG z`qA44=3eyNO?zLa7|8qzY-A_ndq9YC;RmPiQxA;n{PCmJ&r87N=P6*iE7+tz2FGnM z1uIaYoBk6X`u8F*H~ol*AF9)DR;c%@pzjvXb`QM9qu!(N=N8XBz}@_x>*4=^hyFAV z{r5fUJ%u>k>ZJqghEMR&KkX6ELmoKX2tPJw`6d9jdUt!&`wt%Yau2*6xLf{@dEl2K z4{rJm9&yg`sQ2HY@8;)5k9uG5z!!SN^EV##e#yhnBE;jz<}9yx=%4T5zXlHGIGuoR z!RB@xZ}5o!b`SlxWIU5ZX11ziqMpCw($6F@RaaiE#6pC_o7 z+!9SiH^Uc5{-56<#N0rQ75^mlS3Ae@)k$CK`sEBt)YQxQK(dOGSTRS8h z-W<)e#-i~Sl7(aGbS4!`Y?e$W88^6CI?>tQ6isQ7!K9;^OtM2{BFS_(mWXAfMsq3} z$wV_UJo1ccET!gTA`xxQD2r$+g<3=hs?D@Y(~iz`8<^;3T^an3Mo2;^ zbTrix$wYK`9ZB+#QdQjE8BJ}qYD*;5uJjphPIe}AB(2d*a~pDnWTcf!q_q`sMq9$E z|Rp!(P( zQ%e^UTpOnBcP5(A{8$s21k%X>M6sr3+FK^p)~;K%I=m#XB(PNXTy$uU^52)RMTtBw zlUv&R7GNHeowH)+(%E~jR^{s@zP{-=Uj?&8Uq>-ssqroc&ioz+K0{yU>~-K#Tg@F$lxNee^<_!EYLx0%77Z~^j^@^c7F-2O%Z{XYuNI2iX`4X7$N(1L> zO~RKOxV(-h5T(k%sm)wM2F`tjWc3D)4ri5H41A)63ejlblMKAgz$Y7chk>7G;9CrQ zih*|-xX-{JFz^Zk-(}!a4g7HfH_z*P3|x0{X|>nD>AIRrkAcr%5OlwRf5yOj4g3NF z?>F$z8u);L&ob~q1J|ckDIGHK3l068fnQ|c!v?O0kd%%X_$7wExKT0GZ~X>dZs2<8 zN@<0G&oT5bFz`za+;8A>4Sc?V&ol5!1OF2PUvA*@4ZO<0&G8sA@P&qcy@6kD;I|k! zU5ou+m6N`UKMrmGO?jw${Oey3B6NRmrnE2++WvC+OR~zs(suwC<{iVY?~*F)5#L6# z{7?a9-dn`e6q6s2{2Rp6)ROO&{Hw&%l#=g}{9)p0D#`DW{0qd>6q4U1`KO7esUzPd z`6r2|?vU@0{I`jxDJ0)0`M)NfrjC5QzR?rxHIP`3=O=kk9u@{yO4msONhm zzlwMo;`u$2zlL}k+WB3QUqU=h{rN7*FC?CZc)mmObBL#*oo|%iUnQP~V7^E4hl!`5m)|4#7l@}Jm)|A%r-`RiT)s>4PZCc< zFW({gZxc^LF5d_~v~%S*0fcsr-%aG|k<1Kq<!xFKr^S z;B4s7*|Jb}ICLoYg;2?n(CcS2)8XJJ+Ch0?sMUALno)ZiFI_9=!B%u$-VoZp@=0il zQ1*CcGF%6)5hAw$S<1EG=g9a!;AvCIt*G0opX~E@Lf8Ns47~bKcH90?UvPke40Q+h zzd%FbAdRO`wlB95hk($~&KVnU%nrh^K={9E+yagPz}TH7U63zJ^4=LT365wt737B9 zf6eKp4k?A33rN}W5CZt2Z&7bH__lU3fc|7TS?6?e3jp_6zN}&y)^2v2Zc3eQrh2;h zjB=CHZt6`p2dGn+(Z8yi)!7l!`7>1srqpc%!gv2ak(=(|0ExO}GmfC&d<)}M zReCqO?>oZB`tJaB*S$uepc_06)$G5J`Clki>1wHTby(_TkCOCJAfr~BEh}mtsaXwr z&*pxOr8oz8{y`P)sP=_6?MasQDWrniwU?^Mn!!x41@jhWE7gR9(yC#=h@0{?8uzo7 z>Z4HIPyP0lm-NtS#L(Fb%e_`l{3*H12B~}PeF3WCAf1OO8^?bqC3Il4gv2B{K@y;} z|EJi`29Mz=lHzbkWiqR52LTO|)|;p$8~pk9Z9f;j`?|@E7WByK>3EhQY;`=p&+bQ$ z=bM1K>re|S-8rKlQ@KjUQ%V)p_T{3|6{Qe?$>aGFkWs5|SXN(wmCQ^2Z*h|C3!ae; z(-9?PsFYqA`HNITHrUU6a)es)GSy;rhZVNy7FzY3PdId&$r}eX#is1Hl;4G8Qd0+k zQ{4qL|2z)K_TZV44}A-RXOyeH+-X<+dZ2y55i-w5VTM5!dOWmq`akO??hln97`kTh z-B+dM2roMjIy5wnWQT@IRX^+}E^DSUBbzfq&GFFp?W?Nx?d#n>GST-zAH6-WJG$ z?Z9-B{%5`QL){g|VO&evC(tyHcux8%#7W;5d--rl{R7_mA3~pXXOr)TJ$216hM;<4 zvquc}h`)Xss-O2z9h>l0Qa_JF3Zb4|#2mz_0Ry5hw}a#*yS4{MO5k%vaKQJ_!|FUT zs?!7=7|DV3F70PAA8+5bACvkbr7J~~;CytP&cubC8r?tp$T@%2WB%$7WjlLA*@pe_ zjD0zEf$z&rS2fT)WbtxpA{9(rhyycJcYQg9bZ~f3wPQUhA;3QF%7N;ML64&|pE8cl zc#MmjHIC3$IV~+hoZT46?&FB`KXDu_z=bf6ql>Vw$B}D%qlc9%fA|#omKsOy7S=do zi(AP;jU!hJ9!IQy1?j7CLR*a^S8Zb)!35V2q_4)2tG*sbQuk-1tHzO= zE{!7&<8cH{BbAlAOlT8H}SdD}u+Y zaiqH>W+ZgpzTC4P!qhsGSILop5ny%XAmmwje(1N(?jHctL!{q+c0Wf+Xt4Ajq+3m@ zVh%uc)k>#YZ2_4Se)NGHC>U~mxvR0C4Gz7CvzN&IP`Oek`|buP$aHl#oXOUgcjMZC zrrA1(oX!0)OVJNJ|8*66f2;5PUaIGMoAN|U`6S}zHhzz4$eOW?x5e@%Zl_fXex$6B zPh13M>yhgQNBl)9=gfG6E1=RRsnAj}#d3*xvxT zOuy9)$NIR3!c+H7Is2UY40%U&p_s&24U zmqFE@Po!&`Or5>iKT>s%NHnx_>8+Qm$Qyub2LhK0`k9jT6p4Cyu@o~<%s?>%#S9cP zP|QFv1H}v!Gf>PxF$2X6{4q1omP}_vM>3TW^qOB&Bpns>a+~Ogq|(&_`<9e z3$L6z2dLD+-`C=iUl$51=+Rj9l!_gttIwNIwyX@Uq?``&_m33{z2i*zfpILq4{$B= zFW;@T?8jU7-+_D?^1qtmu;ka4vi-||H{k)n8c<7qFG*E=$3VA1KAGBV*`HIw_TzxZ zkeh0VE&1Kbo}gkWW}ujXVg`yCC}yCTfno-V87OAp|7!;LyBX(qGG6iW_c8oE41fQ^ z-@EYlE&M$Tf4{=ttMKwop?pdg?{G+plB=L>9K6?9=%CP=;gufwqK+E}CkY_YMNvR9^ z^PURb;>c33h*lGWTEO>H^g=lm{&r$eJ7Rpku88C8(TU-2RJcC8qblPs*ZO#&Nb;OV z+{#J*JK7Mhok^basG|5I{;qegdzn7oHJaY6X{)ApXu3mFdMSa*6PoVR^oXWMHGNmp z)0$48H|J4iYI>Qb%QUUg^kz+4HN8XA9hyF-=@Xjn)AWd@M>Ty{)6<$x&>!c_)buh< zmuXs~>CKw9YI=vJJ2ZVv(*Be_ z&CdpH&-8z)!?Di(gqS(jh^|vb?)_Va*^Ad%Fty4=sWrdY@iM{liX9(k%|CX0yx?(c z$0rCL|91Qw!Sje6FBd%D*l|n}Iv;j?qVT)oldS$>*Pkr-c-wKzI+Z4(_RGW+Yn=&G zS|)sA&tel{{R(ToH>G7_sx{Bs@o9p;|Fq-yEXHO2DZ^7Q{%+E)kGCo7O+@XNiJ7AD zMmL;ice|_%XN$Jm?HJA1j$0gf+PUC!jXZ$Mclo)aH z`JqPQWAjGOM#(>4FR-7jQh#jz9|!Ih|C8Dek7@Y~5ZE&s=j#ah><<{dsz&wl^@Dt- z2kcdiAM#@dMLx>|eG?n$^Yw>(h6n7|*h;XX0e|n?2%d^po{vkJPxLO4YPKJ@cAbPq zDXvbO{rq>*&)EJ(>r#}6vHi^_55(C1PwO<4h#6!0RY2p;ei#B?idAu(<97{kM;DQ8 zTMyi2NQ+G#`nP%L-{GO3^}zoMxL-zK|Do#}w|HLkz<&bVtzCm2_;C+ECq3{p9{6Nj z(7DBV5pcJ77JKNg^uTL8@S8mFW)Hjr_%w_Q=eiVk0e8!P*2B+Z9{6`X@TWcSUf^!| zdBX#L#{)m@;eR}?uicKzY~Yj!=Qv?YQjm z(0|Iqe=qPkW1A)3_R!CJ`2W3!zGzNm(wWZI)Z;jLT^9@o*RP=^S|~0G zIh_o*MG`HvGRvAx>#OV5tpu@ln4MJ-+0Ky_JY`V7ghE>(I;TvjdZVGM;Z>(Nb z8w{(JJ**{F^b#txx<^%&x$Fit$y|hm|GHGmmKf`Ne9{^+P1G3Wn!$AO2Uo>VC*ZGsMTJ^w!&@k6S)G3wL*<^aTse^n$cBN91G{DhB+5)8C}c9 zTDL_h+E$JkyCjUUT8mnKWlSS5`#DU{= zXr!6Bu#G(8t}6p6RbyEiYCicvB54Jnux;g_F|8b3H%L0Nj}UmWRuU3{^wxI#VGo)~ zDN0qyy+kq-4Qx(y2AVo!@s=xMEkbf_k#w5~v}{elQc)}rBqjJCKOIXZY)lxkR5TtT z1HIRQRdxb$+ypYwE!dX>GmuKk6@mg$tnwo#jJ6ioGN&xmT%jy@pRA|~IFR8)+GEWK zAqmI$3CP--(rFREK&%@wH~H*S?IpmEX%>rBPh1#`%{%^z4f5`97EFm$fp-&w5N7Ag5>|Y4^4Z& zwqv@78>Wr;+Jxyn4*OnxvBq?_&#Z0iwatAFd%iDX>U`eljQ>GxZ-rT@>(jK}(P23M z9Mx`zy>tCadMCy#&ieO2NL6tC{M>=*+)t^W?sMs$ocjkqkLGnPo$)fybRTRf#ca>d ztG8eU2P&+mxKW<*!w^u2Y|qa-y0m>48*1uz@IM7leUI&}=OWiC#R1)Z&KuXu{bvwr zX8rv9WB{vuP?@iY^~?55e+h|cZ{6QkDU(KBKRaMMrtd;P`y4+%k7=w{7Ocni4tL@N z1cY(7V?DNC#DJ!3&-4^ #include #include +#include #include -#include "lua.h" +#include "taos.h" #include "lauxlib.h" +#include "lua.h" #include "lualib.h" -#include struct cb_param{ lua_State* state; @@ -60,7 +60,6 @@ static int l_connect(lua_State *L){ lua_settop(L,0); - taos_init(); lua_newtable(L); int table_index = lua_gettop(L); @@ -137,19 +136,15 @@ static int l_query(lua_State *L){ lua_pushstring(L,fields[i].name); int32_t* length = taos_fetch_lengths(result); switch (fields[i].type) { - case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_TINYINT: lua_pushinteger(L,*((char *)row[i])); break; - case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_SMALLINT: lua_pushinteger(L,*((short *)row[i])); break; - case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_INT: lua_pushinteger(L,*((int *)row[i])); break; - case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_BIGINT: lua_pushinteger(L,*((int64_t *)row[i])); break; @@ -159,7 +154,6 @@ static int l_query(lua_State *L){ case TSDB_DATA_TYPE_DOUBLE: lua_pushnumber(L,*((double *)row[i])); break; - case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: //printf("type:%d, max len:%d, current len:%d\n",fields[i].type, fields[i].bytes, length[i]); @@ -241,67 +235,6 @@ static int l_async_query(lua_State *L){ return 1; } -void stream_cb(void *param, TAOS_RES *result, TAOS_ROW row){ - struct cb_param* p = (struct cb_param*) param; - TAOS_FIELD *fields = taos_fetch_fields(result); - int numFields = taos_num_fields(result); - - // printf("\nnumfields:%d\n", numFields); - //printf("\n\r-----------------------------------------------------------------------------------\n"); - - lua_State *L = p->state; - lua_rawgeti(L, LUA_REGISTRYINDEX, p->callback); - - lua_newtable(L); - - for (int i = 0; i < numFields; ++i) { - if (row[i] == NULL) { - continue; - } - - lua_pushstring(L,fields[i].name); - - switch (fields[i].type) { - case TSDB_DATA_TYPE_TINYINT: - lua_pushinteger(L,*((char *)row[i])); - break; - case TSDB_DATA_TYPE_SMALLINT: - lua_pushinteger(L,*((short *)row[i])); - break; - case TSDB_DATA_TYPE_INT: - lua_pushinteger(L,*((int *)row[i])); - break; - case TSDB_DATA_TYPE_BIGINT: - lua_pushinteger(L,*((int64_t *)row[i])); - break; - case TSDB_DATA_TYPE_FLOAT: - lua_pushnumber(L,*((float *)row[i])); - break; - case TSDB_DATA_TYPE_DOUBLE: - lua_pushnumber(L,*((double *)row[i])); - break; - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - lua_pushstring(L,(char *)row[i]); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - lua_pushinteger(L,*((int64_t *)row[i])); - break; - case TSDB_DATA_TYPE_BOOL: - lua_pushinteger(L,*((char *)row[i])); - break; - default: - lua_pushnil(L); - break; - } - - lua_settable(L, -3); - } - - lua_call(L, 1, 0); - - // printf("-----------------------------------------------------------------------------------\n\r"); -} static int l_close(lua_State *L){ TAOS *taos= (TAOS*)lua_topointer(L,1); diff --git a/examples/lua/test.lua b/examples/lua/test.lua index ff49b8408d..3d725cc6a3 100644 --- a/examples/lua/test.lua +++ b/examples/lua/test.lua @@ -173,16 +173,16 @@ function async_query_callback(res) end end -driver.query_a(conn,"insert into therm1 values ('2019-09-01 00:00:00.005', 100),('2019-09-01 00:00:00.006', 101),('2019-09-01 00:00:00.007', 102)", async_query_callback) +driver.query_a(conn,"INSERT INTO therm1 VALUES ('2019-09-01 00:00:00.005', 100),('2019-09-01 00:00:00.006', 101),('2019-09-01 00:00:00.007', 102)", async_query_callback) res = driver.query(conn, "create stream stream_avg_degree into avg_degree as select avg(degree) from thermometer interval(5s) sliding(1s)") -print("From now on we start continous insertion in an definite (infinite if you want) loop.") +print("From now on we start continous insert in an definite loop, pls wait for about 10 seconds and check stream table for result.") local loop_index = 0 -while loop_index < 30 do +while loop_index < 10 do local t = os.time()*1000 - local v = loop_index - res = driver.query(conn,string.format("insert into therm1 values (%d, %d)",t,v)) + local v = math.random(20) + res = driver.query(conn,string.format("INSERT INTO therm1 VALUES (%d, %d)",t,v)) if res.code ~=0 then print("continous insertion--- failed:" .. res.error) @@ -190,17 +190,8 @@ while loop_index < 30 do else --print("insert successfully, affected:"..res.affected) end - local res1 = driver.query(conn, string.format("select last(*) from avg_degree")) - if res1.code ~=0 then - print("select failed: "..res1.error) - return - else --- print(dump(res1)) - if(#res1.item > 0) then print("avg_degree: " .. res1.item[1]["last(avg(degree))"]) end - end - os.execute("sleep " .. 1) loop_index = loop_index + 1 end - +driver.query(conn,"DROP STREAM IF EXISTS avg_therm_s") driver.close(conn) From a6733dbbd1d68f13d5d64f603ecb4ffc5f75d096 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 21 Sep 2022 22:53:35 +0800 Subject: [PATCH 51/52] feat(wal): auto fix corrupt file --- source/libs/wal/src/walMeta.c | 96 ++++++++++++++++++++++++++++++++--- source/libs/wal/src/walMgmt.c | 8 ++- 2 files changed, 96 insertions(+), 8 deletions(-) diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 5284aeff77..a01869fbe0 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -116,7 +116,6 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { } #endif } - // TODO truncate file if (found == NULL) { // file corrupted, no complete log @@ -125,8 +124,20 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } + + // truncate file SWalCkHead* lastEntry = (SWalCkHead*)found; int64_t retVer = lastEntry->head.version; + int64_t lastEntryBeginOffset = offset + (int64_t)((char*)found - (char*)buf); + int64_t lastEntryEndOffset = lastEntryBeginOffset + sizeof(SWalCkHead) + lastEntry->head.bodyLen; + if (lastEntryEndOffset != fileSize) { + wWarn("vgId:%d repair meta truncate file %s to %ld, orig size %ld", pWal->cfg.vgId, fnameStr, lastEntryEndOffset, + fileSize); + taosFtruncateFile(pFile, lastEntryEndOffset); + ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = lastEntryEndOffset; + pWal->totSize -= (fileSize - lastEntryEndOffset); + } + taosCloseFile(&pFile); taosMemoryFree(buf); @@ -226,16 +237,87 @@ int walCheckAndRepairMeta(SWal* pWal) { } } - // TODO: set fileSize and lastVer if necessary - return 0; } int walCheckAndRepairIdx(SWal* pWal) { - // TODO: iterate all log files - // if idx not found, scan log and write idx - // if found, check complete by first and last entry of each idx file - // if idx incomplete, binary search last valid entry, and then build other part + int32_t sz = taosArrayGetSize(pWal->fileInfoSet); + for (int32_t i = 0; i < sz; i++) { + SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, i); + + char fnameStr[WAL_FILE_LEN]; + walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); + int64_t fsize; + TdFilePtr pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE); + if (pIdxFile == NULL) { + ASSERT(0); + terrno = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fnameStr, terrstr()); + return -1; + } + + taosFStatFile(pIdxFile, &fsize, NULL); + + int32_t left = fsize % sizeof(SWalIdxEntry); + int64_t offset = taosLSeekFile(pIdxFile, -left, SEEK_END); + if (left != 0) { + taosFtruncateFile(pIdxFile, offset); + wWarn("vgId:%d wal truncate file %s to offset %ld since size invalid, file size %ld", pWal->cfg.vgId, fnameStr, + offset, fsize); + } + offset -= sizeof(SWalIdxEntry); + + SWalIdxEntry idxEntry = {0}; + while (1) { + if (offset < 0) { + taosLSeekFile(pIdxFile, 0, SEEK_SET); + taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)); + break; + } + taosLSeekFile(pIdxFile, offset, SEEK_SET); + int64_t contLen = taosReadFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)); + if (contLen < 0 || contLen != sizeof(SWalIdxEntry)) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + if ((idxEntry.ver - pFileInfo->firstVer) * sizeof(SWalIdxEntry) != offset) { + taosFtruncateFile(pIdxFile, offset); + wWarn("vgId:%d wal truncate file %s to offset %ld since entry invalid, entry ver %ld, entry offset %ld", + pWal->cfg.vgId, fnameStr, offset, idxEntry.ver, idxEntry.offset); + offset -= sizeof(SWalIdxEntry); + } else { + break; + } + } + + if (idxEntry.ver < pFileInfo->lastVer) { + char fLogNameStr[WAL_FILE_LEN]; + walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr); + TdFilePtr pLogFile = taosOpenFile(fLogNameStr, TD_FILE_READ); + if (pLogFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fLogNameStr, terrstr()); + return -1; + } + while (idxEntry.ver < pFileInfo->lastVer) { + taosLSeekFile(pLogFile, idxEntry.offset, SEEK_SET); + SWalCkHead ckHead; + taosReadFile(pLogFile, &ckHead, sizeof(SWalCkHead)); + if (idxEntry.ver != ckHead.head.version) { + // todo truncate this idx also + taosCloseFile(&pLogFile); + wError("vgId:%d, invalid repair case", pWal->cfg.vgId); + return -1; + } + idxEntry.ver = ckHead.head.version + 1; + idxEntry.offset = idxEntry.offset + sizeof(SWalCkHead) + ckHead.head.bodyLen; + wWarn("vgId:%d wal idx append new entry %ld %ld", pWal->cfg.vgId, idxEntry.ver, idxEntry.offset); + taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)); + } + taosCloseFile(&pLogFile); + } + taosCloseFile(&pIdxFile); + } return 0; } diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index c939c8c436..a55f00d277 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -149,15 +149,21 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { walLoadMeta(pWal); if (walCheckAndRepairMeta(pWal) < 0) { + wError("vgId:%d cannot open wal since repair meta file failed", pWal->cfg.vgId); taosHashCleanup(pWal->pRefHash); taosRemoveRef(tsWal.refSetId, pWal->refId); taosThreadMutexDestroy(&pWal->mutex); taosArrayDestroy(pWal->fileInfoSet); - taosMemoryFree(pWal); return NULL; } if (walCheckAndRepairIdx(pWal) < 0) { + wError("vgId:%d cannot open wal since repair idx file failed", pWal->cfg.vgId); + taosHashCleanup(pWal->pRefHash); + taosRemoveRef(tsWal.refSetId, pWal->refId); + taosThreadMutexDestroy(&pWal->mutex); + taosArrayDestroy(pWal->fileInfoSet); + return NULL; } wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level, From 78b4d1024934d7db380a0d26cf27936700f753dd Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 22 Sep 2022 00:36:04 +0800 Subject: [PATCH 52/52] optimize fix condition --- source/libs/wal/src/walMeta.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index a01869fbe0..c69046f707 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -257,6 +257,10 @@ int walCheckAndRepairIdx(SWal* pWal) { } taosFStatFile(pIdxFile, &fsize, NULL); + if (fsize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) { + taosCloseFile(&pIdxFile); + continue; + } int32_t left = fsize % sizeof(SWalIdxEntry); int64_t offset = taosLSeekFile(pIdxFile, -left, SEEK_END); @@ -267,7 +271,7 @@ int walCheckAndRepairIdx(SWal* pWal) { } offset -= sizeof(SWalIdxEntry); - SWalIdxEntry idxEntry = {0}; + SWalIdxEntry idxEntry = {.ver = pFileInfo->firstVer}; while (1) { if (offset < 0) { taosLSeekFile(pIdxFile, 0, SEEK_SET); @@ -306,7 +310,8 @@ int walCheckAndRepairIdx(SWal* pWal) { if (idxEntry.ver != ckHead.head.version) { // todo truncate this idx also taosCloseFile(&pLogFile); - wError("vgId:%d, invalid repair case", pWal->cfg.vgId); + wError("vgId:%d, invalid repair case, log seek to %ld to find ver %ld, actual ver %ld", pWal->cfg.vgId, + idxEntry.offset, idxEntry.ver, ckHead.head.version); return -1; } idxEntry.ver = ckHead.head.version + 1;