From c1b77953c5549f7e585a484b119be36e052a8edb Mon Sep 17 00:00:00 2001 From: EdgeGo <80612608+edgego@users.noreply.github.com> Date: Thu, 19 Oct 2023 08:46:32 +0800 Subject: [PATCH 01/14] Update libuv_CMakeLists.txt.in compiling error from alpine, need upgrade from 1.44.2 to 1.46.0 --- cmake/libuv_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/libuv_CMakeLists.txt.in b/cmake/libuv_CMakeLists.txt.in index 9c48ddefef..3e259024b2 100644 --- a/cmake/libuv_CMakeLists.txt.in +++ b/cmake/libuv_CMakeLists.txt.in @@ -2,7 +2,7 @@ # libuv ExternalProject_Add(libuv GIT_REPOSITORY https://github.com/libuv/libuv.git - GIT_TAG v1.44.2 + GIT_TAG v1.46.0 SOURCE_DIR "${TD_CONTRIB_DIR}/libuv" BINARY_DIR "${TD_CONTRIB_DIR}/libuv" CONFIGURE_COMMAND "" From e55961068fc9bd0dd2fa5f8b1a59282ec8112783 Mon Sep 17 00:00:00 2001 From: Yann Diorcet Date: Fri, 27 Oct 2023 14:16:36 +0200 Subject: [PATCH 02/14] CMake: support build without COS --- source/dnode/vnode/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 84d54f3350..4eef8b9d1b 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -163,6 +163,7 @@ target_link_libraries( if(${TD_LINUX}) set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") +if(${BUILD_WITH_COS}) find_library(APR_LIBRARY apr-1 PATHS /usr/local/apr/lib/) find_library(APR_UTIL_LIBRARY aprutil-1 PATHS /usr/local/apr/lib/) find_library(MINIXML_LIBRARY mxml) @@ -194,7 +195,6 @@ target_include_directories( PUBLIC "$ENV{HOME}/.cos-local.1/include" ) -if(${BUILD_WITH_COS}) add_definitions(-DUSE_COS) endif(${BUILD_WITH_COS}) From aaf3de42ef2aee3d646035d96405227e170f2bfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=A4=E8=83=86=E6=9E=AA=E6=89=8B?= <37200724+alona789@users.noreply.github.com> Date: Tue, 5 Dec 2023 09:22:49 +0800 Subject: [PATCH 03/14] Update 02-docker.md --- docs/zh/10-deployment/02-docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/10-deployment/02-docker.md b/docs/zh/10-deployment/02-docker.md index 3dbfabca7d..83e624224d 100644 --- a/docs/zh/10-deployment/02-docker.md +++ b/docs/zh/10-deployment/02-docker.md @@ -457,7 +457,7 @@ services: ## 使用 docker swarm 部署 -如果要想将基于容器的 TDengine 集群部署在多台主机上，可以使用 docker swarm。首先要在这些主机上建立 docke swarm 集群，请参考 docker 官方文档。 +如果要想将基于容器的 TDengine 集群部署在多台主机上，可以使用 docker swarm。首先要在这些主机上建立 docker swarm 集群，请参考 docker 官方文档。 docker-compose 文件可以参考上节。下面是使用 docker swarm 启动 TDengine 的命令： From 2106a59872b1795c7c2cb7411cfe280912e18812 Mon Sep 17 00:00:00 2001 From: Jing Sima Date: Tue, 29 Oct 2024 13:27:09 +0800 Subject: [PATCH 04/14] fix:[TD-32727] fix bug when select interp with two more group key. --- source/libs/executor/src/timesliceoperator.c | 62 ++++++++++++++------ 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index 6803f40da4..71851fe931 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -42,7 +42,7 @@ typedef struct STimeSliceOperatorInfo { SRowKey prevKey; bool prevTsSet; uint64_t groupId; - SGroupKeys* pPrevGroupKey; + SArray* pPrevGroupKeys; SSDataBlock* pNextGroupRes; SSDataBlock* pRemainRes; // save block unfinished processing int32_t remainIndex; // the remaining index in the block to be processed @@ -288,6 +288,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp // output the result int32_t fillColIndex = 0; + int32_t groupKeyIndex = 0; bool hasInterp = true; for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) { SExprInfo* pExprInfo = &pExprSup->pExprInfo[j]; @@ -320,7 +321,9 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp QUERY_CHECK_CODE(code, lino, _end); } else if (!isSelectGroupConstValueFunc(pExprInfo)) { // use stored group key - SGroupKeys* pkey = pSliceInfo->pPrevGroupKey; + SGroupKeys *pkey = taosArrayGet(pSliceInfo->pPrevGroupKeys, groupKeyIndex); + QUERY_CHECK_NULL(pkey, code, lino, _end, terrno); + groupKeyIndex++; if (pkey->isNull == false) { code = colDataSetVal(pDst, rows, pkey->pData, false); QUERY_CHECK_CODE(code, lino, _end); @@ -645,13 +648,20 @@ _end: return code; } +static void destroyGroupKey(void* pKey) { + SGroupKeys* key = (SGroupKeys*)pKey; + if (key->pData != NULL) { + taosMemoryFreeClear(key->pData); + } +} + static int32_t initGroupKeyKeeper(STimeSliceOperatorInfo* pInfo, SExprSupp* pExprSup) { - if (pInfo->pPrevGroupKey != NULL) { + if (pInfo->pPrevGroupKeys != NULL) { return TSDB_CODE_SUCCESS; } - pInfo->pPrevGroupKey = taosMemoryCalloc(1, sizeof(SGroupKeys)); - if (pInfo->pPrevGroupKey == NULL) { + pInfo->pPrevGroupKeys = taosArrayInit(pExprSup->numOfExprs, sizeof(SGroupKeys)); + if (pInfo->pPrevGroupKeys == NULL) { return terrno; } @@ -659,11 +669,17 @@ static int32_t initGroupKeyKeeper(STimeSliceOperatorInfo* pInfo, SExprSupp* pExp SExprInfo* pExprInfo = &pExprSup->pExprInfo[i]; if (isGroupKeyFunc(pExprInfo)) { - pInfo->pPrevGroupKey->bytes = pExprInfo->base.resSchema.bytes; - pInfo->pPrevGroupKey->type = pExprInfo->base.resSchema.type; - pInfo->pPrevGroupKey->isNull = false; - pInfo->pPrevGroupKey->pData = taosMemoryCalloc(1, pInfo->pPrevGroupKey->bytes); - if (!pInfo->pPrevGroupKey->pData) { + SGroupKeys key = {.bytes = pExprInfo->base.resSchema.bytes, + .type = pExprInfo->base.resSchema.type, + .isNull = false, + .pData = taosMemoryCalloc(1, pExprInfo->base.resSchema.bytes)}; + if (!key.pData) { + taosArrayDestroyEx(pInfo->pPrevGroupKeys, destroyGroupKey); + return terrno; + } + if (NULL == taosArrayPush(pInfo->pPrevGroupKeys, &key)) { + taosMemoryFree(key.pData); + taosArrayDestroyEx(pInfo->pPrevGroupKeys, destroyGroupKey); return terrno; } } @@ -910,7 +926,7 @@ static void genInterpAfterDataBlock(STimeSliceOperatorInfo* pSliceInfo, SOperato SInterval* pInterval = &pSliceInfo->interval; if (pSliceInfo->fillType == TSDB_FILL_NEXT || pSliceInfo->fillType == TSDB_FILL_LINEAR || - pSliceInfo->pPrevGroupKey == NULL) { + pSliceInfo->pPrevGroupKeys == NULL) { return; } @@ -921,12 +937,18 @@ static void genInterpAfterDataBlock(STimeSliceOperatorInfo* pSliceInfo, SOperato } } -static void copyPrevGroupKey(SExprSupp* pExprSup, SGroupKeys* pGroupKey, SSDataBlock* pSrcBlock) { +static int32_t copyPrevGroupKey(SExprSupp* pExprSup, SArray * pGroupKeys, SSDataBlock* pSrcBlock) { + int32_t groupKeyIdx = 0; for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) { SExprInfo* pExprInfo = &pExprSup->pExprInfo[j]; if (isGroupKeyFunc(pExprInfo)) { int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; + SGroupKeys *pGroupKey = taosArrayGet(pGroupKeys, groupKeyIdx); + if (pGroupKey == NULL) { + return terrno; + } + groupKeyIdx++; SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot); if (colDataIsNull_s(pSrc, 0)) { @@ -942,9 +964,9 @@ static void copyPrevGroupKey(SExprSupp* pExprSup, SGroupKeys* pGroupKey, SSDataB } pGroupKey->isNull = false; - break; } } + return TSDB_CODE_SUCCESS; } static void resetTimesliceInfo(STimeSliceOperatorInfo* pSliceInfo) { @@ -986,7 +1008,11 @@ static void doHandleTimeslice(SOperatorInfo* pOperator, SSDataBlock* pBlock) { T_LONG_JMP(pTaskInfo->env, code); } doTimesliceImpl(pOperator, pSliceInfo, pBlock, pTaskInfo, ignoreNull); - copyPrevGroupKey(&pOperator->exprSupp, pSliceInfo->pPrevGroupKey, pBlock); + code = copyPrevGroupKey(&pOperator->exprSupp, pSliceInfo->pPrevGroupKeys, pBlock); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + T_LONG_JMP(pTaskInfo->env, code); + } } static int32_t doTimesliceNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { @@ -1160,7 +1186,7 @@ int32_t createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyN pInfo->prevTsSet = false; pInfo->prevKey.ts = INT64_MIN; pInfo->groupId = 0; - pInfo->pPrevGroupKey = NULL; + pInfo->pPrevGroupKeys = NULL; pInfo->pNextGroupRes = NULL; pInfo->pRemainRes = NULL; pInfo->remainIndex = 0; @@ -1233,9 +1259,9 @@ void destroyTimeSliceOperatorInfo(void* param) { } taosArrayDestroy(pInfo->pLinearInfo); - if (pInfo->pPrevGroupKey) { - taosMemoryFree(pInfo->pPrevGroupKey->pData); - taosMemoryFree(pInfo->pPrevGroupKey); + if (pInfo->pPrevGroupKeys) { + taosArrayDestroyEx(pInfo->pPrevGroupKeys, destroyGroupKey); + pInfo->pPrevGroupKeys = NULL; } if (pInfo->hasPk && IS_VAR_DATA_TYPE(pInfo->pkCol.type)) { taosMemoryFreeClear(pInfo->prevKey.pks[0].pData); From 412c20968e7dcdc7395aaeb27f50d1637acd4595 Mon Sep 17 00:00:00 2001 From: Jing Sima Date: Wed, 30 Oct 2024 16:28:12 +0800 Subject: [PATCH 05/14] fix:[TD-32727] Add test case for TD-32727. --- tests/army/query/function/ans/interp.csv | 368 +++++++++++++++++++++++ tests/army/query/function/in/interp.in | 15 + tests/army/query/function/test_interp.py | 72 +++++ tests/parallel_test/cases.task | 1 + 4 files changed, 456 insertions(+) create mode 100644 tests/army/query/function/ans/interp.csv create mode 100644 tests/army/query/function/in/interp.in create mode 100644 tests/army/query/function/test_interp.py diff --git a/tests/army/query/function/ans/interp.csv b/tests/army/query/function/ans/interp.csv new file mode 100644 index 0000000000..e1ba236aa1 --- /dev/null +++ b/tests/army/query/function/ans/interp.csv @@ -0,0 +1,368 @@ + +taos> select _irowts as irowts ,tbname as table_name, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (null) order by irowts; + irowts | table_name | isfilled | intp_c1 | +================================================================================ + 2020-02-01 00:00:04.000 | td32727 | true | NULL | + 2020-02-01 00:00:05.000 | td32727 | false | 5 | + 2020-02-01 00:00:06.000 | td32727 | true | NULL | + 2020-02-01 00:00:07.000 | td32727 | true | NULL | + 2020-02-01 00:00:08.000 | td32727 | true | NULL | + 2020-02-01 00:00:09.000 | td32727 | true | NULL | + 2020-02-01 00:00:10.000 | td32727 | false | 10 | + 2020-02-01 00:00:11.000 | td32727 | true | NULL | + 2020-02-01 00:00:12.000 | td32727 | true | NULL | + 2020-02-01 00:00:13.000 | td32727 | true | NULL | + 2020-02-01 00:00:14.000 | td32727 | true | NULL | + 2020-02-01 00:00:15.000 | td32727 | false | 15 | + 2020-02-01 00:00:16.000 | td32727 | true | NULL | + +taos> select _irowts as irowts ,tbname as table_name, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (next) order by irowts; + irowts | table_name | isfilled | intp_c1 | +================================================================================ + 2020-02-01 00:00:04.000 | td32727 | true | 5 | + 2020-02-01 00:00:05.000 | td32727 | false | 5 | + 2020-02-01 00:00:06.000 | td32727 | true | 10 | + 2020-02-01 00:00:07.000 | td32727 | true | 10 | + 2020-02-01 00:00:08.000 | td32727 | true | 10 | + 2020-02-01 00:00:09.000 | td32727 | true | 10 | + 2020-02-01 00:00:10.000 | td32727 | false | 10 | + 2020-02-01 00:00:11.000 | td32727 | true | 15 | + 2020-02-01 00:00:12.000 | td32727 | true | 15 | + 2020-02-01 00:00:13.000 | td32727 | true | 15 | + 2020-02-01 00:00:14.000 | td32727 | true | 15 | + 2020-02-01 00:00:15.000 | td32727 | false | 15 | + +taos> select _irowts as irowts ,tbname as table_name, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (prev) order by irowts; + irowts | table_name | isfilled | intp_c1 | +================================================================================ + 2020-02-01 00:00:05.000 | td32727 | false | 5 | + 2020-02-01 00:00:06.000 | td32727 | true | 5 | + 2020-02-01 00:00:07.000 | td32727 | true | 5 | + 2020-02-01 00:00:08.000 | td32727 | true | 5 | + 2020-02-01 00:00:09.000 | td32727 | true | 5 | + 2020-02-01 00:00:10.000 | td32727 | false | 10 | + 2020-02-01 00:00:11.000 | td32727 | true | 10 | + 2020-02-01 00:00:12.000 | td32727 | true | 10 | + 2020-02-01 00:00:13.000 | td32727 | true | 10 | + 2020-02-01 00:00:14.000 | td32727 | true | 10 | + 2020-02-01 00:00:15.000 | td32727 | false | 15 | + 2020-02-01 00:00:16.000 | td32727 | true | 15 | + +taos> select _irowts as irowts ,tbname as table_name, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (linear) order by irowts; + irowts | table_name | isfilled | intp_c1 | +================================================================================ + 2020-02-01 00:00:05.000 | td32727 | false | 5 | + 2020-02-01 00:00:06.000 | td32727 | true | 6 | + 2020-02-01 00:00:07.000 | td32727 | true | 7 | + 2020-02-01 00:00:08.000 | td32727 | true | 8 | + 2020-02-01 00:00:09.000 | td32727 | true | 9 | + 2020-02-01 00:00:10.000 | td32727 | false | 10 | + 2020-02-01 00:00:11.000 | td32727 | true | 11 | + 2020-02-01 00:00:12.000 | td32727 | true | 12 | + 2020-02-01 00:00:13.000 | td32727 | true | 13 | + 2020-02-01 00:00:14.000 | td32727 | true | 14 | + 2020-02-01 00:00:15.000 | td32727 | false | 15 | + +taos> select _irowts as irowts ,tbname as table_name, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (value, 1) order by irowts; + irowts | table_name | isfilled | intp_c1 | +================================================================================ + 2020-02-01 00:00:04.000 | td32727 | true | 1 | + 2020-02-01 00:00:05.000 | td32727 | false | 5 | + 2020-02-01 00:00:06.000 | td32727 | true | 1 | + 2020-02-01 00:00:07.000 | td32727 | true | 1 | + 2020-02-01 00:00:08.000 | td32727 | true | 1 | + 2020-02-01 00:00:09.000 | td32727 | true | 1 | + 2020-02-01 00:00:10.000 | td32727 | false | 10 | + 2020-02-01 00:00:11.000 | td32727 | true | 1 | + 2020-02-01 00:00:12.000 | td32727 | true | 1 | + 2020-02-01 00:00:13.000 | td32727 | true | 1 | + 2020-02-01 00:00:14.000 | td32727 | true | 1 | + 2020-02-01 00:00:15.000 | td32727 | false | 15 | + 2020-02-01 00:00:16.000 | td32727 | true | 1 | + +taos> select _irowts as irowts ,tbname as table_name, c2 as c_c2, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname,c2 range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (null) order by irowts, c2; + irowts | table_name | c_c2 | isfilled | intp_c1 | +============================================================================================== + 2020-02-01 00:00:04.000 | td32727 | 5 | true | NULL | + 2020-02-01 00:00:04.000 | td32727 | 10 | true | NULL | + 2020-02-01 00:00:04.000 | td32727 | 15 | true | NULL | + 2020-02-01 00:00:05.000 | td32727 | 5 | false | 5 | + 2020-02-01 00:00:05.000 | td32727 | 10 | true | NULL | + 2020-02-01 00:00:05.000 | td32727 | 15 | true | NULL | + 2020-02-01 00:00:06.000 | td32727 | 5 | true | NULL | + 2020-02-01 00:00:06.000 | td32727 | 10 | true | NULL | + 2020-02-01 00:00:06.000 | td32727 | 15 | true | NULL | + 2020-02-01 00:00:07.000 | td32727 | 5 | true | NULL | + 2020-02-01 00:00:07.000 | td32727 | 10 | true | NULL | + 2020-02-01 00:00:07.000 | td32727 | 15 | true | NULL | + 2020-02-01 00:00:08.000 | td32727 | 5 | true | NULL | + 2020-02-01 00:00:08.000 | td32727 | 10 | true | NULL | + 2020-02-01 00:00:08.000 | td32727 | 15 | true | NULL | + 2020-02-01 00:00:09.000 | td32727 | 5 | true | NULL | + 2020-02-01 00:00:09.000 | td32727 | 10 | true | NULL | + 2020-02-01 00:00:09.000 | td32727 | 15 | true | NULL | + 2020-02-01 00:00:10.000 | td32727 | 5 | true | NULL | + 2020-02-01 00:00:10.000 | td32727 | 10 | false | 10 | + 2020-02-01 00:00:10.000 | td32727 | 15 | true | NULL | + 2020-02-01 00:00:11.000 | td32727 | 5 | true | NULL | + 2020-02-01 00:00:11.000 | td32727 | 10 | true | NULL | + 2020-02-01 00:00:11.000 | td32727 | 15 | true | NULL | + 2020-02-01 00:00:12.000 | td32727 | 5 | true | NULL | + 2020-02-01 00:00:12.000 | td32727 | 10 | true | NULL | + 2020-02-01 00:00:12.000 | td32727 | 15 | true | NULL | + 2020-02-01 00:00:13.000 | td32727 | 5 | true | NULL | + 2020-02-01 00:00:13.000 | td32727 | 10 | true | NULL | + 2020-02-01 00:00:13.000 | td32727 | 15 | true | NULL | + 2020-02-01 00:00:14.000 | td32727 | 5 | true | NULL | + 2020-02-01 00:00:14.000 | td32727 | 10 | true | NULL | + 2020-02-01 00:00:14.000 | td32727 | 15 | true | NULL | + 2020-02-01 00:00:15.000 | td32727 | 5 | true | NULL | + 2020-02-01 00:00:15.000 | td32727 | 10 | true | NULL | + 2020-02-01 00:00:15.000 | td32727 | 15 | false | 15 | + 2020-02-01 00:00:16.000 | td32727 | 5 | true | NULL | + 2020-02-01 00:00:16.000 | td32727 | 10 | true | NULL | + 2020-02-01 00:00:16.000 | td32727 | 15 | true | NULL | + +taos> select _irowts as irowts ,tbname as table_name, c2 as c_c2, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname,c2 range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (next) order by irowts, c2; + irowts | table_name | c_c2 | isfilled | intp_c1 | +============================================================================================== + 2020-02-01 00:00:04.000 | td32727 | 5 | true | 5 | + 2020-02-01 00:00:04.000 | td32727 | 10 | true | 10 | + 2020-02-01 00:00:04.000 | td32727 | 15 | true | 15 | + 2020-02-01 00:00:05.000 | td32727 | 5 | false | 5 | + 2020-02-01 00:00:05.000 | td32727 | 10 | true | 10 | + 2020-02-01 00:00:05.000 | td32727 | 15 | true | 15 | + 2020-02-01 00:00:06.000 | td32727 | 10 | true | 10 | + 2020-02-01 00:00:06.000 | td32727 | 15 | true | 15 | + 2020-02-01 00:00:07.000 | td32727 | 10 | true | 10 | + 2020-02-01 00:00:07.000 | td32727 | 15 | true | 15 | + 2020-02-01 00:00:08.000 | td32727 | 10 | true | 10 | + 2020-02-01 00:00:08.000 | td32727 | 15 | true | 15 | + 2020-02-01 00:00:09.000 | td32727 | 10 | true | 10 | + 2020-02-01 00:00:09.000 | td32727 | 15 | true | 15 | + 2020-02-01 00:00:10.000 | td32727 | 10 | false | 10 | + 2020-02-01 00:00:10.000 | td32727 | 15 | true | 15 | + 2020-02-01 00:00:11.000 | td32727 | 15 | true | 15 | + 2020-02-01 00:00:12.000 | td32727 | 15 | true | 15 | + 2020-02-01 00:00:13.000 | td32727 | 15 | true | 15 | + 2020-02-01 00:00:14.000 | td32727 | 15 | true | 15 | + 2020-02-01 00:00:15.000 | td32727 | 15 | false | 15 | + +taos> select _irowts as irowts ,tbname as table_name, c2 as c_c2, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname,c2 range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (prev) order by irowts, c2; + irowts | table_name | c_c2 | isfilled | intp_c1 | +============================================================================================== + 2020-02-01 00:00:05.000 | td32727 | 5 | false | 5 | + 2020-02-01 00:00:06.000 | td32727 | 5 | true | 5 | + 2020-02-01 00:00:07.000 | td32727 | 5 | true | 5 | + 2020-02-01 00:00:08.000 | td32727 | 5 | true | 5 | + 2020-02-01 00:00:09.000 | td32727 | 5 | true | 5 | + 2020-02-01 00:00:10.000 | td32727 | 5 | true | 5 | + 2020-02-01 00:00:10.000 | td32727 | 10 | false | 10 | + 2020-02-01 00:00:11.000 | td32727 | 5 | true | 5 | + 2020-02-01 00:00:11.000 | td32727 | 10 | true | 10 | + 2020-02-01 00:00:12.000 | td32727 | 5 | true | 5 | + 2020-02-01 00:00:12.000 | td32727 | 10 | true | 10 | + 2020-02-01 00:00:13.000 | td32727 | 5 | true | 5 | + 2020-02-01 00:00:13.000 | td32727 | 10 | true | 10 | + 2020-02-01 00:00:14.000 | td32727 | 5 | true | 5 | + 2020-02-01 00:00:14.000 | td32727 | 10 | true | 10 | + 2020-02-01 00:00:15.000 | td32727 | 5 | true | 5 | + 2020-02-01 00:00:15.000 | td32727 | 10 | true | 10 | + 2020-02-01 00:00:15.000 | td32727 | 15 | false | 15 | + 2020-02-01 00:00:16.000 | td32727 | 5 | true | 5 | + 2020-02-01 00:00:16.000 | td32727 | 10 | true | 10 | + 2020-02-01 00:00:16.000 | td32727 | 15 | true | 15 | + +taos> select _irowts as irowts ,tbname as table_name, c2 as c_c2, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname,c2 range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (linear) order by irowts, c2; + irowts | table_name | c_c2 | isfilled | intp_c1 | +============================================================================================== + 2020-02-01 00:00:05.000 | td32727 | 5 | false | 5 | + 2020-02-01 00:00:10.000 | td32727 | 10 | false | 10 | + 2020-02-01 00:00:15.000 | td32727 | 15 | false | 15 | + +taos> select _irowts as irowts ,tbname as table_name, c2 as c_c2, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname,c2 range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (value, 1) order by irowts, c2; + irowts | table_name | c_c2 | isfilled | intp_c1 | +============================================================================================== + 2020-02-01 00:00:04.000 | td32727 | 5 | true | 1 | + 2020-02-01 00:00:04.000 | td32727 | 10 | true | 1 | + 2020-02-01 00:00:04.000 | td32727 | 15 | true | 1 | + 2020-02-01 00:00:05.000 | td32727 | 5 | false | 5 | + 2020-02-01 00:00:05.000 | td32727 | 10 | true | 1 | + 2020-02-01 00:00:05.000 | td32727 | 15 | true | 1 | + 2020-02-01 00:00:06.000 | td32727 | 5 | true | 1 | + 2020-02-01 00:00:06.000 | td32727 | 10 | true | 1 | + 2020-02-01 00:00:06.000 | td32727 | 15 | true | 1 | + 2020-02-01 00:00:07.000 | td32727 | 5 | true | 1 | + 2020-02-01 00:00:07.000 | td32727 | 10 | true | 1 | + 2020-02-01 00:00:07.000 | td32727 | 15 | true | 1 | + 2020-02-01 00:00:08.000 | td32727 | 5 | true | 1 | + 2020-02-01 00:00:08.000 | td32727 | 10 | true | 1 | + 2020-02-01 00:00:08.000 | td32727 | 15 | true | 1 | + 2020-02-01 00:00:09.000 | td32727 | 5 | true | 1 | + 2020-02-01 00:00:09.000 | td32727 | 10 | true | 1 | + 2020-02-01 00:00:09.000 | td32727 | 15 | true | 1 | + 2020-02-01 00:00:10.000 | td32727 | 5 | true | 1 | + 2020-02-01 00:00:10.000 | td32727 | 10 | false | 10 | + 2020-02-01 00:00:10.000 | td32727 | 15 | true | 1 | + 2020-02-01 00:00:11.000 | td32727 | 5 | true | 1 | + 2020-02-01 00:00:11.000 | td32727 | 10 | true | 1 | + 2020-02-01 00:00:11.000 | td32727 | 15 | true | 1 | + 2020-02-01 00:00:12.000 | td32727 | 5 | true | 1 | + 2020-02-01 00:00:12.000 | td32727 | 10 | true | 1 | + 2020-02-01 00:00:12.000 | td32727 | 15 | true | 1 | + 2020-02-01 00:00:13.000 | td32727 | 5 | true | 1 | + 2020-02-01 00:00:13.000 | td32727 | 10 | true | 1 | + 2020-02-01 00:00:13.000 | td32727 | 15 | true | 1 | + 2020-02-01 00:00:14.000 | td32727 | 5 | true | 1 | + 2020-02-01 00:00:14.000 | td32727 | 10 | true | 1 | + 2020-02-01 00:00:14.000 | td32727 | 15 | true | 1 | + 2020-02-01 00:00:15.000 | td32727 | 5 | true | 1 | + 2020-02-01 00:00:15.000 | td32727 | 10 | true | 1 | + 2020-02-01 00:00:15.000 | td32727 | 15 | false | 15 | + 2020-02-01 00:00:16.000 | td32727 | 5 | true | 1 | + 2020-02-01 00:00:16.000 | td32727 | 10 | true | 1 | + 2020-02-01 00:00:16.000 | td32727 | 15 | true | 1 | + +taos> select _irowts as irowts ,tbname as table_name, c2 as c_c2, c3 as c_c3, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname,c2,c3 range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (null) order by irowts, c2, c3; + irowts | table_name | c_c2 | c_c3 | isfilled | intp_c1 | +====================================================================================================================== + 2020-02-01 00:00:04.000 | td32727 | 5 | 5 | true | NULL | + 2020-02-01 00:00:04.000 | td32727 | 10 | 10 | true | NULL | + 2020-02-01 00:00:04.000 | td32727 | 15 | 15 | true | NULL | + 2020-02-01 00:00:05.000 | td32727 | 5 | 5 | false | 5 | + 2020-02-01 00:00:05.000 | td32727 | 10 | 10 | true | NULL | + 2020-02-01 00:00:05.000 | td32727 | 15 | 15 | true | NULL | + 2020-02-01 00:00:06.000 | td32727 | 5 | 5 | true | NULL | + 2020-02-01 00:00:06.000 | td32727 | 10 | 10 | true | NULL | + 2020-02-01 00:00:06.000 | td32727 | 15 | 15 | true | NULL | + 2020-02-01 00:00:07.000 | td32727 | 5 | 5 | true | NULL | + 2020-02-01 00:00:07.000 | td32727 | 10 | 10 | true | NULL | + 2020-02-01 00:00:07.000 | td32727 | 15 | 15 | true | NULL | + 2020-02-01 00:00:08.000 | td32727 | 5 | 5 | true | NULL | + 2020-02-01 00:00:08.000 | td32727 | 10 | 10 | true | NULL | + 2020-02-01 00:00:08.000 | td32727 | 15 | 15 | true | NULL | + 2020-02-01 00:00:09.000 | td32727 | 5 | 5 | true | NULL | + 2020-02-01 00:00:09.000 | td32727 | 10 | 10 | true | NULL | + 2020-02-01 00:00:09.000 | td32727 | 15 | 15 | true | NULL | + 2020-02-01 00:00:10.000 | td32727 | 5 | 5 | true | NULL | + 2020-02-01 00:00:10.000 | td32727 | 10 | 10 | false | 10 | + 2020-02-01 00:00:10.000 | td32727 | 15 | 15 | true | NULL | + 2020-02-01 00:00:11.000 | td32727 | 5 | 5 | true | NULL | + 2020-02-01 00:00:11.000 | td32727 | 10 | 10 | true | NULL | + 2020-02-01 00:00:11.000 | td32727 | 15 | 15 | true | NULL | + 2020-02-01 00:00:12.000 | td32727 | 5 | 5 | true | NULL | + 2020-02-01 00:00:12.000 | td32727 | 10 | 10 | true | NULL | + 2020-02-01 00:00:12.000 | td32727 | 15 | 15 | true | NULL | + 2020-02-01 00:00:13.000 | td32727 | 5 | 5 | true | NULL | + 2020-02-01 00:00:13.000 | td32727 | 10 | 10 | true | NULL | + 2020-02-01 00:00:13.000 | td32727 | 15 | 15 | true | NULL | + 2020-02-01 00:00:14.000 | td32727 | 5 | 5 | true | NULL | + 2020-02-01 00:00:14.000 | td32727 | 10 | 10 | true | NULL | + 2020-02-01 00:00:14.000 | td32727 | 15 | 15 | true | NULL | + 2020-02-01 00:00:15.000 | td32727 | 5 | 5 | true | NULL | + 2020-02-01 00:00:15.000 | td32727 | 10 | 10 | true | NULL | + 2020-02-01 00:00:15.000 | td32727 | 15 | 15 | false | 15 | + 2020-02-01 00:00:16.000 | td32727 | 5 | 5 | true | NULL | + 2020-02-01 00:00:16.000 | td32727 | 10 | 10 | true | NULL | + 2020-02-01 00:00:16.000 | td32727 | 15 | 15 | true | NULL | + +taos> select _irowts as irowts ,tbname as table_name, c2 as c_c2, c3 as c_c3, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname,c2,c3 range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (next) order by irowts, c2, c3; + irowts | table_name | c_c2 | c_c3 | isfilled | intp_c1 | +====================================================================================================================== + 2020-02-01 00:00:04.000 | td32727 | 5 | 5 | true | 5 | + 2020-02-01 00:00:04.000 | td32727 | 10 | 10 | true | 10 | + 2020-02-01 00:00:04.000 | td32727 | 15 | 15 | true | 15 | + 2020-02-01 00:00:05.000 | td32727 | 5 | 5 | false | 5 | + 2020-02-01 00:00:05.000 | td32727 | 10 | 10 | true | 10 | + 2020-02-01 00:00:05.000 | td32727 | 15 | 15 | true | 15 | + 2020-02-01 00:00:06.000 | td32727 | 10 | 10 | true | 10 | + 2020-02-01 00:00:06.000 | td32727 | 15 | 15 | true | 15 | + 2020-02-01 00:00:07.000 | td32727 | 10 | 10 | true | 10 | + 2020-02-01 00:00:07.000 | td32727 | 15 | 15 | true | 15 | + 2020-02-01 00:00:08.000 | td32727 | 10 | 10 | true | 10 | + 2020-02-01 00:00:08.000 | td32727 | 15 | 15 | true | 15 | + 2020-02-01 00:00:09.000 | td32727 | 10 | 10 | true | 10 | + 2020-02-01 00:00:09.000 | td32727 | 15 | 15 | true | 15 | + 2020-02-01 00:00:10.000 | td32727 | 10 | 10 | false | 10 | + 2020-02-01 00:00:10.000 | td32727 | 15 | 15 | true | 15 | + 2020-02-01 00:00:11.000 | td32727 | 15 | 15 | true | 15 | + 2020-02-01 00:00:12.000 | td32727 | 15 | 15 | true | 15 | + 2020-02-01 00:00:13.000 | td32727 | 15 | 15 | true | 15 | + 2020-02-01 00:00:14.000 | td32727 | 15 | 15 | true | 15 | + 2020-02-01 00:00:15.000 | td32727 | 15 | 15 | false | 15 | + +taos> select _irowts as irowts ,tbname as table_name, c2 as c_c2, c3 as c_c3, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname,c2,c3 range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (prev) order by irowts, c2, c3; + irowts | table_name | c_c2 | c_c3 | isfilled | intp_c1 | +====================================================================================================================== + 2020-02-01 00:00:05.000 | td32727 | 5 | 5 | false | 5 | + 2020-02-01 00:00:06.000 | td32727 | 5 | 5 | true | 5 | + 2020-02-01 00:00:07.000 | td32727 | 5 | 5 | true | 5 | + 2020-02-01 00:00:08.000 | td32727 | 5 | 5 | true | 5 | + 2020-02-01 00:00:09.000 | td32727 | 5 | 5 | true | 5 | + 2020-02-01 00:00:10.000 | td32727 | 5 | 5 | true | 5 | + 2020-02-01 00:00:10.000 | td32727 | 10 | 10 | false | 10 | + 2020-02-01 00:00:11.000 | td32727 | 5 | 5 | true | 5 | + 2020-02-01 00:00:11.000 | td32727 | 10 | 10 | true | 10 | + 2020-02-01 00:00:12.000 | td32727 | 5 | 5 | true | 5 | + 2020-02-01 00:00:12.000 | td32727 | 10 | 10 | true | 10 | + 2020-02-01 00:00:13.000 | td32727 | 5 | 5 | true | 5 | + 2020-02-01 00:00:13.000 | td32727 | 10 | 10 | true | 10 | + 2020-02-01 00:00:14.000 | td32727 | 5 | 5 | true | 5 | + 2020-02-01 00:00:14.000 | td32727 | 10 | 10 | true | 10 | + 2020-02-01 00:00:15.000 | td32727 | 5 | 5 | true | 5 | + 2020-02-01 00:00:15.000 | td32727 | 10 | 10 | true | 10 | + 2020-02-01 00:00:15.000 | td32727 | 15 | 15 | false | 15 | + 2020-02-01 00:00:16.000 | td32727 | 5 | 5 | true | 5 | + 2020-02-01 00:00:16.000 | td32727 | 10 | 10 | true | 10 | + 2020-02-01 00:00:16.000 | td32727 | 15 | 15 | true | 15 | + +taos> select _irowts as irowts ,tbname as table_name, c2 as c_c2, c3 as c_c3, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname,c2,c3 range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (linear) order by irowts, c2, c3; + irowts | table_name | c_c2 | c_c3 | isfilled | intp_c1 | +====================================================================================================================== + 2020-02-01 00:00:05.000 | td32727 | 5 | 5 | false | 5 | + 2020-02-01 00:00:10.000 | td32727 | 10 | 10 | false | 10 | + 2020-02-01 00:00:15.000 | td32727 | 15 | 15 | false | 15 | + +taos> select _irowts as irowts ,tbname as table_name, c2 as c_c2, c3 as c_c3, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname,c2,c3 range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (value, 1) order by irowts, c2, c3; + irowts | table_name | c_c2 | c_c3 | isfilled | intp_c1 | +====================================================================================================================== + 2020-02-01 00:00:04.000 | td32727 | 5 | 5 | true | 1 | + 2020-02-01 00:00:04.000 | td32727 | 10 | 10 | true | 1 | + 2020-02-01 00:00:04.000 | td32727 | 15 | 15 | true | 1 | + 2020-02-01 00:00:05.000 | td32727 | 5 | 5 | false | 5 | + 2020-02-01 00:00:05.000 | td32727 | 10 | 10 | true | 1 | + 2020-02-01 00:00:05.000 | td32727 | 15 | 15 | true | 1 | + 2020-02-01 00:00:06.000 | td32727 | 5 | 5 | true | 1 | + 2020-02-01 00:00:06.000 | td32727 | 10 | 10 | true | 1 | + 2020-02-01 00:00:06.000 | td32727 | 15 | 15 | true | 1 | + 2020-02-01 00:00:07.000 | td32727 | 5 | 5 | true | 1 | + 2020-02-01 00:00:07.000 | td32727 | 10 | 10 | true | 1 | + 2020-02-01 00:00:07.000 | td32727 | 15 | 15 | true | 1 | + 2020-02-01 00:00:08.000 | td32727 | 5 | 5 | true | 1 | + 2020-02-01 00:00:08.000 | td32727 | 10 | 10 | true | 1 | + 2020-02-01 00:00:08.000 | td32727 | 15 | 15 | true | 1 | + 2020-02-01 00:00:09.000 | td32727 | 5 | 5 | true | 1 | + 2020-02-01 00:00:09.000 | td32727 | 10 | 10 | true | 1 | + 2020-02-01 00:00:09.000 | td32727 | 15 | 15 | true | 1 | + 2020-02-01 00:00:10.000 | td32727 | 5 | 5 | true | 1 | + 2020-02-01 00:00:10.000 | td32727 | 10 | 10 | false | 10 | + 2020-02-01 00:00:10.000 | td32727 | 15 | 15 | true | 1 | + 2020-02-01 00:00:11.000 | td32727 | 5 | 5 | true | 1 | + 2020-02-01 00:00:11.000 | td32727 | 10 | 10 | true | 1 | + 2020-02-01 00:00:11.000 | td32727 | 15 | 15 | true | 1 | + 2020-02-01 00:00:12.000 | td32727 | 5 | 5 | true | 1 | + 2020-02-01 00:00:12.000 | td32727 | 10 | 10 | true | 1 | + 2020-02-01 00:00:12.000 | td32727 | 15 | 15 | true | 1 | + 2020-02-01 00:00:13.000 | td32727 | 5 | 5 | true | 1 | + 2020-02-01 00:00:13.000 | td32727 | 10 | 10 | true | 1 | + 2020-02-01 00:00:13.000 | td32727 | 15 | 15 | true | 1 | + 2020-02-01 00:00:14.000 | td32727 | 5 | 5 | true | 1 | + 2020-02-01 00:00:14.000 | td32727 | 10 | 10 | true | 1 | + 2020-02-01 00:00:14.000 | td32727 | 15 | 15 | true | 1 | + 2020-02-01 00:00:15.000 | td32727 | 5 | 5 | true | 1 | + 2020-02-01 00:00:15.000 | td32727 | 10 | 10 | true | 1 | + 2020-02-01 00:00:15.000 | td32727 | 15 | 15 | false | 15 | + 2020-02-01 00:00:16.000 | td32727 | 5 | 5 | true | 1 | + 2020-02-01 00:00:16.000 | td32727 | 10 | 10 | true | 1 | + 2020-02-01 00:00:16.000 | td32727 | 15 | 15 | true | 1 | + diff --git a/tests/army/query/function/in/interp.in b/tests/army/query/function/in/interp.in new file mode 100644 index 0000000000..4825ab46b1 --- /dev/null +++ b/tests/army/query/function/in/interp.in @@ -0,0 +1,15 @@ +select _irowts as irowts ,tbname as table_name, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (null) order by irowts; +select _irowts as irowts ,tbname as table_name, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (next) order by irowts; +select _irowts as irowts ,tbname as table_name, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (prev) order by irowts; +select _irowts as irowts ,tbname as table_name, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (linear) order by irowts; +select _irowts as irowts ,tbname as table_name, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (value, 1) order by irowts; +select _irowts as irowts ,tbname as table_name, c2 as c_c2, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname,c2 range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (null) order by irowts, c2; +select _irowts as irowts ,tbname as table_name, c2 as c_c2, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname,c2 range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (next) order by irowts, c2; +select _irowts as irowts ,tbname as table_name, c2 as c_c2, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname,c2 range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (prev) order by irowts, c2; +select _irowts as irowts ,tbname as table_name, c2 as c_c2, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname,c2 range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (linear) order by irowts, c2; +select _irowts as irowts ,tbname as table_name, c2 as c_c2, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname,c2 range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (value, 1) order by irowts, c2; +select _irowts as irowts ,tbname as table_name, c2 as c_c2, c3 as c_c3, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname,c2,c3 range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (null) order by irowts, c2, c3; +select _irowts as irowts ,tbname as table_name, c2 as c_c2, c3 as c_c3, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname,c2,c3 range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (next) order by irowts, c2, c3; +select _irowts as irowts ,tbname as table_name, c2 as c_c2, c3 as c_c3, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname,c2,c3 range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (prev) order by irowts, c2, c3; +select _irowts as irowts ,tbname as table_name, c2 as c_c2, c3 as c_c3, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname,c2,c3 range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (linear) order by irowts, c2, c3; +select _irowts as irowts ,tbname as table_name, c2 as c_c2, c3 as c_c3, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname,c2,c3 range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (value, 1) order by irowts, c2, c3; diff --git a/tests/army/query/function/test_interp.py b/tests/army/query/function/test_interp.py new file mode 100644 index 0000000000..f903e7be73 --- /dev/null +++ b/tests/army/query/function/test_interp.py @@ -0,0 +1,72 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +from frame import etool +from frame.etool import * +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame.common import * + +class TDTestCase(TBase): + updatecfgDict = { + "keepColumnName": "1", + "ttlChangeOnWrite": "1", + "querySmaOptimize": "1", + "slowLogScope": "none", + "queryBufferSize": 10240 + } + + def insert_data(self): + tdLog.printNoPrefix("==========step1:create table") + + tdSql.execute("create database test keep 36500") + tdSql.execute("use test") + tdSql.execute( + f'''create table if not exists test.td32727 + (ts timestamp, c0 tinyint, c1 smallint, c2 int, c3 bigint, c4 double, c5 float, c6 bool, c7 varchar(10), c8 nchar(10), c9 tinyint unsigned, c10 smallint unsigned, c11 int unsigned, c12 bigint unsigned) + ''' + ) + + tdLog.printNoPrefix("==========step2:insert data") + + tdSql.execute(f"insert into test.td32727 values ('2020-02-01 00:00:05', 5, 5, 5, 5, 5.0, 5.0, true, 'varchar', 'nchar', 5, 5, 5, 5)") + tdSql.execute(f"insert into test.td32727 values ('2020-02-01 00:00:10', 10, 10, 10, 10, 10.0, 10.0, true, 'varchar', 'nchar', 10, 10, 10, 10)") + tdSql.execute(f"insert into test.td32727 values ('2020-02-01 00:00:15', 15, 15, 15, 15, 15.0, 15.0, true, 'varchar', 'nchar', 15, 15, 15, 15)") + + + def test_normal_query_new(self, testCase): + # read sql from .sql file and execute + tdLog.info("test normal query.") + self.sqlFile = etool.curFile(__file__, f"in/{testCase}.in") + self.ansFile = etool.curFile(__file__, f"ans/{testCase}.csv") + + tdCom.compare_testcase_result(self.sqlFile, self.ansFile, testCase) + + def test_interp(self): + self.test_normal_query_new("interp") + + def run(self): + tdLog.debug(f"start to excute {__file__}") + + self.insert_data() + + # math function + self.test_interp() + + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 09216add82..216be82868 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -16,6 +16,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f query/function/test_func_elapsed.py ,,y,army,./pytest.sh python3 ./test.py -f query/function/test_function.py ,,y,army,./pytest.sh python3 ./test.py -f query/function/test_resinfo.py +,,y,army,./pytest.sh python3 ./test.py -f query/function/test_interp.py ,,y,army,./pytest.sh python3 ./test.py -f query/function/concat.py ,,y,army,./pytest.sh python3 ./test.py -f query/function/cast.py ,,y,army,./pytest.sh python3 ./test.py -f query/test_join.py From 4d420effb826f84059425c8a38b00e191fbb4940 Mon Sep 17 00:00:00 2001 From: Yu Chen <74105241+yu285@users.noreply.github.com> Date: Thu, 31 Oct 2024 14:40:09 +0800 Subject: [PATCH 06/14] docs:Update description about "stt_trigger" --- docs/zh/14-reference/03-taos-sql/02-database.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/03-taos-sql/02-database.md b/docs/zh/14-reference/03-taos-sql/02-database.md index 7d040a2c44..cfbe8423d9 100644 --- a/docs/zh/14-reference/03-taos-sql/02-database.md +++ b/docs/zh/14-reference/03-taos-sql/02-database.md @@ -63,7 +63,7 @@ database_option: { - MAXROWS：文件块中记录的最大条数，默认为 4096 条。 - MINROWS：文件块中记录的最小条数，默认为 100 条。 - KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于3倍的 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 \<= keep 1 \<= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。 -- STT_TRIGGER：表示落盘文件触发文件合并的个数。默认为 1，范围 1 到 16。对于少表高频场景，此参数建议使用默认配置，或较小的值；而对于多表低频场景，此参数建议配置较大的值。 +- STT_TRIGGER：表示落盘文件触发文件合并的个数。开源版本固定为 1，企业版本可设置范围为 1 到 16。对于少表高频写入场景，此参数建议使用默认配置；而对于多表低频写入场景，此参数建议配置较大的值。 - SINGLE_STABLE：表示此数据库中是否只可以创建一个超级表，用于超级表列非常多的情况。 - 0：表示可以创建多张超级表。 - 1：表示只可以创建一张超级表。 From 135adabd98241980255f0d1c5139e029e74a8894 Mon Sep 17 00:00:00 2001 From: Jing Sima Date: Fri, 1 Nov 2024 09:11:35 +0800 Subject: [PATCH 07/14] fix:[TD-32727] set SArray pointer to NULL after destroy. --- source/libs/executor/src/timesliceoperator.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index 71851fe931..88c1642654 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -675,11 +675,13 @@ static int32_t initGroupKeyKeeper(STimeSliceOperatorInfo* pInfo, SExprSupp* pExp .pData = taosMemoryCalloc(1, pExprInfo->base.resSchema.bytes)}; if (!key.pData) { taosArrayDestroyEx(pInfo->pPrevGroupKeys, destroyGroupKey); + pInfo->pPrevGroupKeys = NULL; return terrno; } if (NULL == taosArrayPush(pInfo->pPrevGroupKeys, &key)) { taosMemoryFree(key.pData); taosArrayDestroyEx(pInfo->pPrevGroupKeys, destroyGroupKey); + pInfo->pPrevGroupKeys = NULL; return terrno; } } From 31f4cb5eb4506e2d37119447d95a5de9321d3ab7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 1 Nov 2024 09:46:09 +0800 Subject: [PATCH 08/14] fix possible crash --- source/util/src/tcompression.c | 5 ++++- tests/script/tsim/compress/commitlog.sim | 9 ++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 9c9ded693e..7ef35bebaf 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1617,7 +1617,10 @@ int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int uTrace("encode:%s, compress:%s, level:%d, type:%s, l1:%d", compressL1Dict[l1].name, compressL2Dict[l2].name, \ lvl, tDataTypes[type].name, l1); \ int32_t len = compressL1Dict[l1].comprFn(pIn, nEle, pBuf, type); \ - int8_t alvl = tsGetCompressL2Level(l2, lvl); \ + if (len < 0) { \ + return len; \ + } \ + int8_t alvl = tsGetCompressL2Level(l2, lvl); \ return compressL2Dict[l2].comprFn(pBuf, len, pOut, nOut, type, alvl); \ } else { \ uTrace("dencode:%s, decompress:%s, level:%d, type:%s", compressL1Dict[l1].name, compressL2Dict[l2].name, lvl, \ diff --git a/tests/script/tsim/compress/commitlog.sim b/tests/script/tsim/compress/commitlog.sim index 38899b95ba..c936d8be46 100644 --- a/tests/script/tsim/compress/commitlog.sim +++ b/tests/script/tsim/compress/commitlog.sim @@ -20,7 +20,7 @@ sql create table $tb (ts timestamp, b bool, t tinyint, s smallint, i int, big bi $count = 0 while $count < $N $ms = 1591200000000 + $count - sql insert into $tb values( $ms , 1, 0, $count , $count , $count ,'it is a string') + sql insert into $tb values( $ms , $count/256, 0, $count , $count , $count ,'it is a string') $count = $count + 1 endw @@ -29,6 +29,13 @@ if $rows != $N then return -1 endi +sql flush database $db + +sql select * from $tb +if $rows != $N then + return -1 +endi + print =============== step2 $i = 1 $db = $dbPrefix . $i From d6f1d0853c39e3355d6e4a7d176526445156c7dc Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 1 Nov 2024 09:50:35 +0800 Subject: [PATCH 09/14] change test case --- tests/script/tsim/compress/commitlog.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/tsim/compress/commitlog.sim b/tests/script/tsim/compress/commitlog.sim index c936d8be46..d0380dd559 100644 --- a/tests/script/tsim/compress/commitlog.sim +++ b/tests/script/tsim/compress/commitlog.sim @@ -20,7 +20,7 @@ sql create table $tb (ts timestamp, b bool, t tinyint, s smallint, i int, big bi $count = 0 while $count < $N $ms = 1591200000000 + $count - sql insert into $tb values( $ms , $count/256, 0, $count , $count , $count ,'it is a string') + sql insert into $tb values( $ms , 10, 0, $count , $count , $count ,'it is a string') $count = $count + 1 endw From 53aa105da9df0a7ce08be9cde95ebc40e817106b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 1 Nov 2024 11:23:52 +0800 Subject: [PATCH 10/14] add disable support and add test case --- source/common/src/tcol.c | 3 + source/util/src/tcompression.c | 3 +- .../script/tsim/compress/compressDisable.sim | 146 ++++++++++++++++++ tests/script/win-test-file | 1 + 4 files changed, 151 insertions(+), 2 deletions(-) create mode 100644 tests/script/tsim/compress/compressDisable.sim diff --git a/source/common/src/tcol.c b/source/common/src/tcol.c index 84027c25b6..923aab12ca 100644 --- a/source/common/src/tcol.c +++ b/source/common/src/tcol.c @@ -363,6 +363,9 @@ int8_t validColEncode(uint8_t type, uint8_t l1) { if (l1 == TSDB_COLVAL_ENCODE_NOCHANGE) { return 1; } + if (l1 == TSDB_COLVAL_ENCODE_DISABLED) { + return 1; + } if (type == TSDB_DATA_TYPE_BOOL) { return TSDB_COLVAL_ENCODE_RLE == l1 ? 1 : 0; } else if (type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_INT) { diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 9c9ded693e..b954a24744 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1628,8 +1628,7 @@ int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int } \ } else if (l1 == L1_DISABLED && l2 != L2_DISABLED) { \ if (compress) { \ - uTrace("encode:%s, compress:%s, level:%d, type:%s", "disabled", compressL2Dict[l1].name, lvl, \ - tDataTypes[type].name); \ + uTrace("encode:%s, compress:%s, level:%d, type:%s", "disabled", "disable", lvl, tDataTypes[type].name); \ int8_t alvl = tsGetCompressL2Level(l2, lvl); \ return compressL2Dict[l2].comprFn(pIn, nIn, pOut, nOut, type, alvl); \ } else { \ diff --git a/tests/script/tsim/compress/compressDisable.sim b/tests/script/tsim/compress/compressDisable.sim new file mode 100644 index 0000000000..dac7ef712a --- /dev/null +++ b/tests/script/tsim/compress/compressDisable.sim @@ -0,0 +1,146 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ============================ dnode1 start +$i = 0 +$dbPrefix = db +$tbPrefix = tb +$db = $dbPrefix . $i +$tb = $tbPrefix . $i + +$N = 2000 + +print =============== step1 +sql create database $db +sql use $db +sql create table $tb (ts timestamp, b bool encode 'disabled', t tinyint encode 'disabled', s smallint encode 'disabled', i int encode 'disabled', big bigint encode 'disabled', str binary(256)) + +$count = 0 +while $count < $N + $ms = 1591200000000 + $count + sql insert into $tb values( $ms , 1, 0, $count , $count , $count ,'it is a string') + $count = $count + 1 +endw + +sql select * from $tb +if $rows != $N then + return -1 +endi + +sql flush database $db + +sql select * from $tb +if $rows != $N then + return -1 +endi + +sql alter table $tb modify column ts encode 'disabled' + +$count = 0 +while $count < $N + $ms = 1591200030000 + $count + sql insert into $tb values( $ms , 1, 0, $count , $count , $count ,'it is a string') + $count = $count + 1 +endw + +$M = 4000 +sql select * from $tb +if $rows != $M then + return -1 +endi + +sql flush database $db + +sql select * from $tb +if $rows != $M then + return -1 +endi + + +$stb1 = txx1 +sql create table txx1 (ts timestamp encode 'disabled' compress 'disabled' level 'h', f int compress 'lz4') tags(t int) + +$count = 0 +$subTb1 = txx1_sub1 +$subTb2 = txx1_sub2 + +sql create table $subTb1 using $stb1 tags(1) +sql create table $subTb2 using $stb1 tags(2) + +while $count < $N + $ms = 1591200030000 + $count + sql insert into $subTb1 values( $ms , 1) + + $ms2 = 1591200040000 + $count + sql insert into $subTb2 values( $ms2 , 1) + $count = $count + 1 +endw + +$count = 0 +sql select * from $stb1 +if $rows != $M then + return -1 +endi + +sql flush database $db + +sql select * from $stb1 +if $rows != $M then + return -1 +endi + +$L = 8000 +sql alter table $stb1 modify column ts encode 'delta-i' +sql alter table $stb1 modify column f encode 'disabled' + +while $count < $N + $ms = 1591200050000 + $count + sql insert into $subTb1 values( $ms , 1) + + $ms2 = 1591200060000 + $count + sql insert into $subTb2 values( $ms2 , 1) + $count = $count + 1 +endw + + +sql select * from $stb1 +if $rows != $L then + return -1 +endi + +sql flush database $db + +sql select * from $stb1 +if $rows != $L then + return -1 +endi + +sql alter table $stb1 modify column ts encode 'disabled' + + +$count = 0 +$I = 12000 +while $count < $N + $ms = 1591200070000 + $count + sql insert into $subTb1 values( $ms , 1) + + $ms2 = 1591200080000 + $count + sql insert into $subTb2 values( $ms2 , 1) + $count = $count + 1 +endw + + +sql select * from $stb1 +if $rows != $I then + return -1 +endi + +sql flush database $db + +sql select * from $stb1 +if $rows != $I then + return -1 +endi + diff --git a/tests/script/win-test-file b/tests/script/win-test-file index ff69e91967..ef99442214 100644 --- a/tests/script/win-test-file +++ b/tests/script/win-test-file @@ -325,6 +325,7 @@ ./test.sh -f tsim/compress/compress.sim ./test.sh -f tsim/compress/compress_col.sim ./test.sh -f tsim/compress/uncompress.sim +./test.sh -f tsim/compress/compressDisable.sim ./test.sh -f tsim/compute/avg.sim ./test.sh -f tsim/compute/block_dist.sim ./test.sh -f tsim/compute/bottom.sim From 2a248056466945bbd227f48dc71c076adcdcfbb3 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Fri, 1 Nov 2024 11:34:18 +0800 Subject: [PATCH 11/14] add block data capacity --- source/libs/executor/src/groupoperator.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 9cf2a3ea17..0f85c3346d 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -1263,7 +1263,10 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { QUERY_CHECK_CONDITION((hasRemainPartion(pInfo)), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->parIte; blockDataCleanup(pDest); - int32_t rows = taosArrayGetSize(pParInfo->rowIds); + int32_t rows = taosArrayGetSize(pParInfo->rowIds); + code = blockDataEnsureCapacity(pDest, rows); + QUERY_CHECK_CODE(code, lino, _end); + SSDataBlock* pSrc = pInfo->pInputDataBlock; for (int32_t i = 0; i < rows; i++) { int32_t rowIndex = *(int32_t*)taosArrayGet(pParInfo->rowIds, i); From 647066c4f9221c4e77cf2291b483a2f51ff3ba7f Mon Sep 17 00:00:00 2001 From: Jinqing Kuang Date: Wed, 30 Oct 2024 15:12:49 +0800 Subject: [PATCH 12/14] enh(query)[TD-32652]: enable AVX implementation with CPU dispatching Extract AVX implementation of specific functions into separate files. The CPU dispatching mechanism will now check the machine's instruction set at runtime to determine whether to use the AVX implementation or fallback to the naive version. This enhancement improves performance on most machines while ensuring compatibility with older hardwares. --- cmake/cmake.define | 41 +-- include/util/tcompression.h | 9 +- include/util/tdef.h | 1 + source/libs/function/CMakeLists.txt | 4 + source/libs/function/inc/builtinsimpl.h | 5 + source/libs/function/src/detail/tminmax.c | 343 +++++------------- source/libs/function/src/detail/tminmaxavx.c | 227 ++++++++++++ source/os/src/osEnv.c | 1 - source/util/CMakeLists.txt | 4 + source/util/src/tcompression.c | 53 ++- .../src/{tdecompress.c => tdecompressavx.c} | 120 +++--- 11 files changed, 434 insertions(+), 374 deletions(-) create mode 100644 source/libs/function/src/detail/tminmaxavx.c rename source/util/src/{tdecompress.c => tdecompressavx.c} (95%) diff --git a/cmake/cmake.define b/cmake/cmake.define index 8b762011a4..91e35a1e7b 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -177,48 +177,11 @@ ELSE() SET(COMPILER_SUPPORT_AVX512VL false) ELSE() CHECK_C_COMPILER_FLAG("-mfma" COMPILER_SUPPORT_FMA) + CHECK_C_COMPILER_FLAG("-mavx" COMPILER_SUPPORT_AVX) + CHECK_C_COMPILER_FLAG("-mavx2" COMPILER_SUPPORT_AVX2) CHECK_C_COMPILER_FLAG("-mavx512f" COMPILER_SUPPORT_AVX512F) CHECK_C_COMPILER_FLAG("-mavx512vbmi" COMPILER_SUPPORT_AVX512BMI) CHECK_C_COMPILER_FLAG("-mavx512vl" COMPILER_SUPPORT_AVX512VL) - - INCLUDE(CheckCSourceRuns) - SET(CMAKE_REQUIRED_FLAGS "-mavx") - check_c_source_runs(" - #include - int main() { - __m256d a, b, c; - double buf[4] = {0}; - a = _mm256_loadu_pd(buf); - b = _mm256_loadu_pd(buf); - c = _mm256_add_pd(a, b); - _mm256_storeu_pd(buf, c); - for (int i = 0; i < sizeof(buf) / sizeof(buf[0]); ++i) { - IF (buf[i] != 0) { - return 1; - } - } - return 0; - } - " COMPILER_SUPPORT_AVX) - - SET(CMAKE_REQUIRED_FLAGS "-mavx2") - check_c_source_runs(" - #include - int main() { - __m256i a, b, c; - int buf[8] = {0}; - a = _mm256_loadu_si256((__m256i *)buf); - b = _mm256_loadu_si256((__m256i *)buf); - c = _mm256_and_si256(a, b); - _mm256_storeu_si256((__m256i *)buf, c); - for (int i = 0; i < sizeof(buf) / sizeof(buf[0]); ++i) { - IF (buf[i] != 0) { - return 1; - } - } - return 0; - } - " COMPILER_SUPPORT_AVX2) ENDIF() IF(COMPILER_SUPPORT_SSE42) diff --git a/include/util/tcompression.h b/include/util/tcompression.h index 1f09b750cb..140b7fe392 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -152,15 +152,12 @@ int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int // for internal usage int32_t getWordLength(char type); -#ifdef __AVX2__ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, char *const output, const char type); int32_t tsDecompressFloatImpAvx2(const char *input, int32_t nelements, char *output); int32_t tsDecompressDoubleImpAvx2(const char *input, int32_t nelements, char *output); -#endif -#ifdef __AVX512VL__ -void tsDecompressTimestampAvx2(const char *input, int32_t nelements, char *output, bool bigEndian); -void tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output, bool bigEndian); -#endif +int32_t tsDecompressTimestampAvx2(const char *input, int32_t nelements, char *output, bool bigEndian); +int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output, + bool bigEndian); /************************************************************************* * REGULAR COMPRESSION 2 diff --git a/include/util/tdef.h b/include/util/tdef.h index b4cb1bdd1c..2ed22dc09a 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -41,6 +41,7 @@ extern const int32_t TYPE_BYTES[21]; #define FLOAT_BYTES sizeof(float) #define DOUBLE_BYTES sizeof(double) #define POINTER_BYTES sizeof(void *) +#define M256_BYTES 32 #define TSDB_KEYSIZE sizeof(TSKEY) #define TSDB_NCHAR_SIZE sizeof(TdUcs4) diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index 4164852111..d5c9cccc0e 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -1,6 +1,10 @@ aux_source_directory(src FUNCTION_SRC) aux_source_directory(src/detail FUNCTION_SRC_DETAIL) list(REMOVE_ITEM FUNCTION_SRC src/udfd.c) +IF(COMPILER_SUPPORT_AVX2) + MESSAGE(STATUS "AVX2 instructions is ACTIVATED") + set_source_files_properties(src/detail/tminmaxavx.c PROPERTIES COMPILE_FLAGS -mavx2) +ENDIF() add_library(function STATIC ${FUNCTION_SRC} ${FUNCTION_SRC_DETAIL}) target_include_directories( function diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 36e53d0a80..a1c82dc58b 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -25,6 +25,11 @@ extern "C" { #include "functionResInfoInt.h" int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems); +int32_t i8VectorCmpAVX2(const void* pData, int32_t numOfRows, bool isMinFunc, bool signVal, int64_t* res); +int32_t i16VectorCmpAVX2(const void* pData, int32_t numOfRows, bool isMinFunc, bool signVal, int64_t* res); +int32_t i32VectorCmpAVX2(const void* pData, int32_t numOfRows, bool isMinFunc, bool signVal, int64_t* res); +int32_t floatVectorCmpAVX2(const float* pData, int32_t numOfRows, bool isMinFunc, float* res); +int32_t doubleVectorCmpAVX2(const double* pData, int32_t numOfRows, bool isMinFunc, double* res); int32_t saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); diff --git a/source/libs/function/src/detail/tminmax.c b/source/libs/function/src/detail/tminmax.c index 69c1a8a6dd..8712096033 100644 --- a/source/libs/function/src/detail/tminmax.c +++ b/source/libs/function/src/detail/tminmax.c @@ -72,173 +72,6 @@ #define GET_INVOKE_INTRINSIC_THRESHOLD(_bits, _bytes) ((_bits) / ((_bytes) << 3u)) -#ifdef __AVX2__ -static void calculateRounds(int32_t numOfRows, int32_t bytes, int32_t* remainder, int32_t* rounds, int32_t* width) { - const int32_t bitWidth = 256; - - *width = (bitWidth >> 3u) / bytes; - *remainder = numOfRows % (*width); - *rounds = numOfRows / (*width); -} - -#define EXTRACT_MAX_VAL(_first, _sec, _width, _remain, _v) \ - __COMPARE_EXTRACT_MAX(0, (_width), (_v), (_first)) \ - __COMPARE_EXTRACT_MAX(0, (_remain), (_v), (_sec)) - -#define EXTRACT_MIN_VAL(_first, _sec, _width, _remain, _v) \ - __COMPARE_EXTRACT_MIN(0, (_width), (_v), (_first)) \ - __COMPARE_EXTRACT_MIN(0, (_remain), (_v), (_sec)) - -#define CMP_TYPE_MIN_MAX(type, cmp) \ - const type* p = pData; \ - __m256i initVal = _mm256_lddqu_si256((__m256i*)p); \ - p += width; \ - for (int32_t i = 1; i < (rounds); ++i) { \ - __m256i next = _mm256_lddqu_si256((__m256i*)p); \ - initVal = CMP_FUNC_##cmp##_##type(initVal, next); \ - p += width; \ - } \ - const type* q = (const type*)&initVal; \ - type* v = (type*)res; \ - EXTRACT_##cmp##_VAL(q, p, width, remain, *v) - -static void i8VectorCmpAVX2(const void* pData, int32_t numOfRows, bool isMinFunc, bool signVal, int64_t* res) { - const int8_t* p = pData; - - int32_t width, remain, rounds; - calculateRounds(numOfRows, sizeof(int8_t), &remain, &rounds, &width); - -#define CMP_FUNC_MIN_int8_t _mm256_min_epi8 -#define CMP_FUNC_MAX_int8_t _mm256_max_epi8 -#define CMP_FUNC_MIN_uint8_t _mm256_min_epu8 -#define CMP_FUNC_MAX_uint8_t _mm256_max_epu8 - - if (!isMinFunc) { // max function - if (signVal) { - CMP_TYPE_MIN_MAX(int8_t, MAX); - } else { - CMP_TYPE_MIN_MAX(uint8_t, MAX); - } - } else { // min function - if (signVal) { - CMP_TYPE_MIN_MAX(int8_t, MIN); - } else { - CMP_TYPE_MIN_MAX(uint8_t, MIN); - } - } -} - -static void i16VectorCmpAVX2(const void* pData, int32_t numOfRows, bool isMinFunc, bool signVal, int64_t* res) { - int32_t width, remain, rounds; - calculateRounds(numOfRows, sizeof(int16_t), &remain, &rounds, &width); - -#define CMP_FUNC_MIN_int16_t _mm256_min_epi16 -#define CMP_FUNC_MAX_int16_t _mm256_max_epi16 -#define CMP_FUNC_MIN_uint16_t _mm256_min_epu16 -#define CMP_FUNC_MAX_uint16_t _mm256_max_epu16 - if (!isMinFunc) { // max function - if (signVal) { - CMP_TYPE_MIN_MAX(int16_t, MAX); - } else { - CMP_TYPE_MIN_MAX(uint16_t, MAX); - } - } else { // min function - if (signVal) { - CMP_TYPE_MIN_MAX(int16_t, MIN); - } else { - CMP_TYPE_MIN_MAX(uint16_t, MIN); - } - } -} - -static void i32VectorCmpAVX2(const void* pData, int32_t numOfRows, bool isMinFunc, bool signVal, int64_t* res) { - int32_t width, remain, rounds; - calculateRounds(numOfRows, sizeof(int32_t), &remain, &rounds, &width); - -#define CMP_FUNC_MIN_int32_t _mm256_min_epi32 -#define CMP_FUNC_MAX_int32_t _mm256_max_epi32 -#define CMP_FUNC_MIN_uint32_t _mm256_min_epu32 -#define CMP_FUNC_MAX_uint32_t _mm256_max_epu32 - if (!isMinFunc) { // max function - if (signVal) { - CMP_TYPE_MIN_MAX(int32_t, MAX); - } else { - CMP_TYPE_MIN_MAX(uint32_t, MAX); - } - } else { // min function - if (signVal) { - CMP_TYPE_MIN_MAX(int32_t, MIN); - } else { - CMP_TYPE_MIN_MAX(uint32_t, MIN); - } - } -} - -static void floatVectorCmpAVX2(const float* pData, int32_t numOfRows, bool isMinFunc, float* res) { - const float* p = pData; - - int32_t width, remain, rounds; - calculateRounds(numOfRows, sizeof(float), &remain, &rounds, &width); - - __m256 next; - __m256 initVal = _mm256_loadu_ps(p); - p += width; - - if (!isMinFunc) { // max function - for (int32_t i = 1; i < rounds; ++i) { - next = _mm256_loadu_ps(p); - initVal = _mm256_max_ps(initVal, next); - p += width; - } - - const float* q = (const float*)&initVal; - EXTRACT_MAX_VAL(q, p, width, remain, *res) - } else { // min function - for (int32_t i = 1; i < rounds; ++i) { - next = _mm256_loadu_ps(p); - initVal = _mm256_min_ps(initVal, next); - p += width; - } - - const float* q = (const float*)&initVal; - EXTRACT_MIN_VAL(q, p, width, remain, *res) - } -} - -static void doubleVectorCmpAVX2(const double* pData, int32_t numOfRows, bool isMinFunc, double* res) { - const double* p = pData; - - int32_t width, remain, rounds; - calculateRounds(numOfRows, sizeof(double), &remain, &rounds, &width); - - __m256d next; - __m256d initVal = _mm256_loadu_pd(p); - p += width; - - if (!isMinFunc) { // max function - for (int32_t i = 1; i < rounds; ++i) { - next = _mm256_loadu_pd(p); - initVal = _mm256_max_pd(initVal, next); - p += width; - } - - // let sum up the final results - const double* q = (const double*)&initVal; - EXTRACT_MAX_VAL(q, p, width, remain, *res) - } else { // min function - for (int32_t i = 1; i < rounds; ++i) { - next = _mm256_loadu_pd(p); - initVal = _mm256_min_pd(initVal, next); - p += width; - } - - // let sum up the final results - const double* q = (const double*)&initVal; - EXTRACT_MIN_VAL(q, p, width, remain, *res) - } -} -#endif - static int32_t findFirstValPosition(const SColumnInfoData* pCol, int32_t start, int32_t numOfRows, bool isStr) { int32_t i = start; @@ -255,31 +88,31 @@ static void handleInt8Col(const void* data, int32_t start, int32_t numOfRows, SM pBuf->v = ((const int8_t*)data)[start]; } -#ifdef __AVX2__ - if (tsAVX2Supported && tsSIMDEnable && numOfRows * sizeof(int8_t) >= sizeof(__m256i)) { - i8VectorCmpAVX2(data + start * sizeof(int8_t), numOfRows, isMinFunc, signVal, &pBuf->v); - } else { -#else - if (true) { -#endif - if (signVal) { - const int8_t* p = (const int8_t*)data; - int8_t* v = (int8_t*)&pBuf->v; + if (tsAVX2Supported && tsSIMDEnable && numOfRows * sizeof(int8_t) >= M256_BYTES) { + int32_t code = i8VectorCmpAVX2(((char*)data) + start * sizeof(int8_t), numOfRows, isMinFunc, signVal, &pBuf->v); + if (code == TSDB_CODE_SUCCESS) { + pBuf->assign = true; + return; + } + } - if (isMinFunc) { - __COMPARE_EXTRACT_MIN(start, start + numOfRows, *v, p); - } else { - __COMPARE_EXTRACT_MAX(start, start + numOfRows, *v, p); - } + if (signVal) { + const int8_t* p = (const int8_t*)data; + int8_t* v = (int8_t*)&pBuf->v; + + if (isMinFunc) { + __COMPARE_EXTRACT_MIN(start, start + numOfRows, *v, p); } else { - const uint8_t* p = (const uint8_t*)data; - uint8_t* v = (uint8_t*)&pBuf->v; + __COMPARE_EXTRACT_MAX(start, start + numOfRows, *v, p); + } + } else { + const uint8_t* p = (const uint8_t*)data; + uint8_t* v = (uint8_t*)&pBuf->v; - if (isMinFunc) { - __COMPARE_EXTRACT_MIN(start, start + numOfRows, *v, p); - } else { - __COMPARE_EXTRACT_MAX(start, start + numOfRows, *v, p); - } + if (isMinFunc) { + __COMPARE_EXTRACT_MIN(start, start + numOfRows, *v, p); + } else { + __COMPARE_EXTRACT_MAX(start, start + numOfRows, *v, p); } } @@ -292,31 +125,31 @@ static void handleInt16Col(const void* data, int32_t start, int32_t numOfRows, S pBuf->v = ((const int16_t*)data)[start]; } -#ifdef __AVX2__ - if (tsAVX2Supported && tsSIMDEnable && numOfRows * sizeof(int16_t) >= sizeof(__m256i)) { - i16VectorCmpAVX2(data + start * sizeof(int16_t), numOfRows, isMinFunc, signVal, &pBuf->v); - } else { -#else - if (true) { -#endif - if (signVal) { - const int16_t* p = (const int16_t*)data; - int16_t* v = (int16_t*)&pBuf->v; + if (tsAVX2Supported && tsSIMDEnable && numOfRows * sizeof(int16_t) >= M256_BYTES) { + int32_t code = i16VectorCmpAVX2(((char*)data) + start * sizeof(int16_t), numOfRows, isMinFunc, signVal, &pBuf->v); + if (code == TSDB_CODE_SUCCESS) { + pBuf->assign = true; + return; + } + } - if (isMinFunc) { - __COMPARE_EXTRACT_MIN(start, start + numOfRows, *v, p); - } else { - __COMPARE_EXTRACT_MAX(start, start + numOfRows, *v, p); - } + if (signVal) { + const int16_t* p = (const int16_t*)data; + int16_t* v = (int16_t*)&pBuf->v; + + if (isMinFunc) { + __COMPARE_EXTRACT_MIN(start, start + numOfRows, *v, p); } else { - const uint16_t* p = (const uint16_t*)data; - uint16_t* v = (uint16_t*)&pBuf->v; + __COMPARE_EXTRACT_MAX(start, start + numOfRows, *v, p); + } + } else { + const uint16_t* p = (const uint16_t*)data; + uint16_t* v = (uint16_t*)&pBuf->v; - if (isMinFunc) { - __COMPARE_EXTRACT_MIN(start, start + numOfRows, *v, p); - } else { - __COMPARE_EXTRACT_MAX(start, start + numOfRows, *v, p); - } + if (isMinFunc) { + __COMPARE_EXTRACT_MIN(start, start + numOfRows, *v, p); + } else { + __COMPARE_EXTRACT_MAX(start, start + numOfRows, *v, p); } } @@ -329,31 +162,31 @@ static void handleInt32Col(const void* data, int32_t start, int32_t numOfRows, S pBuf->v = ((const int32_t*)data)[start]; } -#ifdef __AVX2__ - if (tsAVX2Supported && tsSIMDEnable && numOfRows * sizeof(int32_t) >= sizeof(__m256i)) { - i32VectorCmpAVX2(data + start * sizeof(int32_t), numOfRows, isMinFunc, signVal, &pBuf->v); - } else { -#else - if (true) { -#endif - if (signVal) { - const int32_t* p = (const int32_t*)data; - int32_t* v = (int32_t*)&pBuf->v; + if (tsAVX2Supported && tsSIMDEnable && numOfRows * sizeof(int32_t) >= M256_BYTES) { + int32_t code = i32VectorCmpAVX2(((char*)data) + start * sizeof(int32_t), numOfRows, isMinFunc, signVal, &pBuf->v); + if (code == TSDB_CODE_SUCCESS) { + pBuf->assign = true; + return; + } + } - if (isMinFunc) { - __COMPARE_EXTRACT_MIN(start, start + numOfRows, *v, p); - } else { - __COMPARE_EXTRACT_MAX(start, start + numOfRows, *v, p); - } + if (signVal) { + const int32_t* p = (const int32_t*)data; + int32_t* v = (int32_t*)&pBuf->v; + + if (isMinFunc) { + __COMPARE_EXTRACT_MIN(start, start + numOfRows, *v, p); } else { - const uint32_t* p = (const uint32_t*)data; - uint32_t* v = (uint32_t*)&pBuf->v; + __COMPARE_EXTRACT_MAX(start, start + numOfRows, *v, p); + } + } else { + const uint32_t* p = (const uint32_t*)data; + uint32_t* v = (uint32_t*)&pBuf->v; - if (isMinFunc) { - __COMPARE_EXTRACT_MIN(start, start + numOfRows, *v, p); - } else { - __COMPARE_EXTRACT_MAX(start, start + numOfRows, *v, p); - } + if (isMinFunc) { + __COMPARE_EXTRACT_MIN(start, start + numOfRows, *v, p); + } else { + __COMPARE_EXTRACT_MAX(start, start + numOfRows, *v, p); } } @@ -397,20 +230,20 @@ static void handleFloatCol(SColumnInfoData* pCol, int32_t start, int32_t numOfRo *val = pData[start]; } -#ifdef __AVX2__ - if (tsAVXSupported && tsSIMDEnable && numOfRows * sizeof(float) >= sizeof(__m256i)) { - floatVectorCmpAVX2(pData + start, numOfRows, isMinFunc, val); - } else { -#else - if (true) { -#endif - if (isMinFunc) { // min - __COMPARE_EXTRACT_MIN(start, start + numOfRows, *val, pData); - } else { // max - __COMPARE_EXTRACT_MAX(start, start + numOfRows, *val, pData); + if (tsAVX2Supported && tsSIMDEnable && numOfRows * sizeof(float) >= M256_BYTES) { + int32_t code = floatVectorCmpAVX2(pData + start, numOfRows, isMinFunc, val); + if (code == TSDB_CODE_SUCCESS) { + pBuf->assign = true; + return; } } + if (isMinFunc) { // min + __COMPARE_EXTRACT_MIN(start, start + numOfRows, *val, pData); + } else { // max + __COMPARE_EXTRACT_MAX(start, start + numOfRows, *val, pData); + } + pBuf->assign = true; } @@ -422,20 +255,20 @@ static void handleDoubleCol(SColumnInfoData* pCol, int32_t start, int32_t numOfR *val = pData[start]; } -#ifdef __AVX2__ - if (tsAVXSupported && tsSIMDEnable && numOfRows * sizeof(double) >= sizeof(__m256i)) { - doubleVectorCmpAVX2(pData + start, numOfRows, isMinFunc, val); - } else { -#else - if (true) { -#endif - if (isMinFunc) { // min - __COMPARE_EXTRACT_MIN(start, start + numOfRows, *val, pData); - } else { // max - __COMPARE_EXTRACT_MAX(start, start + numOfRows, *val, pData); + if (tsAVX2Supported && tsSIMDEnable && numOfRows * sizeof(double) >= M256_BYTES) { + int32_t code = doubleVectorCmpAVX2(pData + start, numOfRows, isMinFunc, val); + if (code == TSDB_CODE_SUCCESS) { + pBuf->assign = true; + return; } } + if (isMinFunc) { // min + __COMPARE_EXTRACT_MIN(start, start + numOfRows, *val, pData); + } else { // max + __COMPARE_EXTRACT_MAX(start, start + numOfRows, *val, pData); + } + pBuf->assign = true; } diff --git a/source/libs/function/src/detail/tminmaxavx.c b/source/libs/function/src/detail/tminmaxavx.c new file mode 100644 index 0000000000..8fe6cc5448 --- /dev/null +++ b/source/libs/function/src/detail/tminmaxavx.c @@ -0,0 +1,227 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "builtinsimpl.h" + +#ifdef __AVX2__ +static void calculateRounds(int32_t numOfRows, int32_t bytes, int32_t* remainder, int32_t* rounds, int32_t* width) { + const int32_t bitWidth = 256; + + *width = (bitWidth >> 3u) / bytes; + *remainder = numOfRows % (*width); + *rounds = numOfRows / (*width); +} + +#define __COMPARE_EXTRACT_MIN(start, end, val, _data) \ + for (int32_t i = (start); i < (end); ++i) { \ + if ((val) > (_data)[i]) { \ + (val) = (_data)[i]; \ + } \ + } + +#define __COMPARE_EXTRACT_MAX(start, end, val, _data) \ + for (int32_t i = (start); i < (end); ++i) { \ + if ((val) < (_data)[i]) { \ + (val) = (_data)[i]; \ + } \ + } + +#define EXTRACT_MAX_VAL(_first, _sec, _width, _remain, _v) \ + __COMPARE_EXTRACT_MAX(0, (_width), (_v), (_first)) \ + __COMPARE_EXTRACT_MAX(0, (_remain), (_v), (_sec)) + +#define EXTRACT_MIN_VAL(_first, _sec, _width, _remain, _v) \ + __COMPARE_EXTRACT_MIN(0, (_width), (_v), (_first)) \ + __COMPARE_EXTRACT_MIN(0, (_remain), (_v), (_sec)) + +#define CMP_TYPE_MIN_MAX(type, cmp) \ + const type* p = pData; \ + __m256i initVal = _mm256_lddqu_si256((__m256i*)p); \ + p += width; \ + for (int32_t i = 1; i < (rounds); ++i) { \ + __m256i next = _mm256_lddqu_si256((__m256i*)p); \ + initVal = CMP_FUNC_##cmp##_##type(initVal, next); \ + p += width; \ + } \ + const type* q = (const type*)&initVal; \ + type* v = (type*)res; \ + EXTRACT_##cmp##_VAL(q, p, width, remain, *v) +#endif + +int32_t i8VectorCmpAVX2(const void* pData, int32_t numOfRows, bool isMinFunc, bool signVal, int64_t* res) { +#ifdef __AVX2__ + const int8_t* p = pData; + + int32_t width, remain, rounds; + calculateRounds(numOfRows, sizeof(int8_t), &remain, &rounds, &width); + +#define CMP_FUNC_MIN_int8_t _mm256_min_epi8 +#define CMP_FUNC_MAX_int8_t _mm256_max_epi8 +#define CMP_FUNC_MIN_uint8_t _mm256_min_epu8 +#define CMP_FUNC_MAX_uint8_t _mm256_max_epu8 + + if (!isMinFunc) { // max function + if (signVal) { + CMP_TYPE_MIN_MAX(int8_t, MAX); + } else { + CMP_TYPE_MIN_MAX(uint8_t, MAX); + } + } else { // min function + if (signVal) { + CMP_TYPE_MIN_MAX(int8_t, MIN); + } else { + CMP_TYPE_MIN_MAX(uint8_t, MIN); + } + } + return TSDB_CODE_SUCCESS; +#else + uError("unable run %s without avx2 instructions", __func__); + return TSDB_CODE_OPS_NOT_SUPPORT; +#endif +} + +int32_t i16VectorCmpAVX2(const void* pData, int32_t numOfRows, bool isMinFunc, bool signVal, int64_t* res) { +#ifdef __AVX2__ + int32_t width, remain, rounds; + calculateRounds(numOfRows, sizeof(int16_t), &remain, &rounds, &width); + +#define CMP_FUNC_MIN_int16_t _mm256_min_epi16 +#define CMP_FUNC_MAX_int16_t _mm256_max_epi16 +#define CMP_FUNC_MIN_uint16_t _mm256_min_epu16 +#define CMP_FUNC_MAX_uint16_t _mm256_max_epu16 + if (!isMinFunc) { // max function + if (signVal) { + CMP_TYPE_MIN_MAX(int16_t, MAX); + } else { + CMP_TYPE_MIN_MAX(uint16_t, MAX); + } + } else { // min function + if (signVal) { + CMP_TYPE_MIN_MAX(int16_t, MIN); + } else { + CMP_TYPE_MIN_MAX(uint16_t, MIN); + } + } + return TSDB_CODE_SUCCESS; +#else + uError("unable run %s without avx2 instructions", __func__); + return TSDB_CODE_OPS_NOT_SUPPORT; +#endif +} + +int32_t i32VectorCmpAVX2(const void* pData, int32_t numOfRows, bool isMinFunc, bool signVal, int64_t* res) { +#ifdef __AVX2__ + int32_t width, remain, rounds; + calculateRounds(numOfRows, sizeof(int32_t), &remain, &rounds, &width); + +#define CMP_FUNC_MIN_int32_t _mm256_min_epi32 +#define CMP_FUNC_MAX_int32_t _mm256_max_epi32 +#define CMP_FUNC_MIN_uint32_t _mm256_min_epu32 +#define CMP_FUNC_MAX_uint32_t _mm256_max_epu32 + if (!isMinFunc) { // max function + if (signVal) { + CMP_TYPE_MIN_MAX(int32_t, MAX); + } else { + CMP_TYPE_MIN_MAX(uint32_t, MAX); + } + } else { // min function + if (signVal) { + CMP_TYPE_MIN_MAX(int32_t, MIN); + } else { + CMP_TYPE_MIN_MAX(uint32_t, MIN); + } + } + return TSDB_CODE_SUCCESS; +#else + uError("unable run %s without avx2 instructions", __func__); + return TSDB_CODE_OPS_NOT_SUPPORT; +#endif +} + +int32_t floatVectorCmpAVX2(const float* pData, int32_t numOfRows, bool isMinFunc, float* res) { +#ifdef __AVX2__ + const float* p = pData; + + int32_t width, remain, rounds; + calculateRounds(numOfRows, sizeof(float), &remain, &rounds, &width); + + __m256 next; + __m256 initVal = _mm256_loadu_ps(p); + p += width; + + if (!isMinFunc) { // max function + for (int32_t i = 1; i < rounds; ++i) { + next = _mm256_loadu_ps(p); + initVal = _mm256_max_ps(initVal, next); + p += width; + } + + const float* q = (const float*)&initVal; + EXTRACT_MAX_VAL(q, p, width, remain, *res) + } else { // min function + for (int32_t i = 1; i < rounds; ++i) { + next = _mm256_loadu_ps(p); + initVal = _mm256_min_ps(initVal, next); + p += width; + } + + const float* q = (const float*)&initVal; + EXTRACT_MIN_VAL(q, p, width, remain, *res) + } + return TSDB_CODE_SUCCESS; +#else + uError("unable run %s without avx2 instructions", __func__); + return TSDB_CODE_OPS_NOT_SUPPORT; +#endif +} + +int32_t doubleVectorCmpAVX2(const double* pData, int32_t numOfRows, bool isMinFunc, double* res) { +#ifdef __AVX2__ + const double* p = pData; + + int32_t width, remain, rounds; + calculateRounds(numOfRows, sizeof(double), &remain, &rounds, &width); + + __m256d next; + __m256d initVal = _mm256_loadu_pd(p); + p += width; + + if (!isMinFunc) { // max function + for (int32_t i = 1; i < rounds; ++i) { + next = _mm256_loadu_pd(p); + initVal = _mm256_max_pd(initVal, next); + p += width; + } + + // let sum up the final results + const double* q = (const double*)&initVal; + EXTRACT_MAX_VAL(q, p, width, remain, *res) + } else { // min function + for (int32_t i = 1; i < rounds; ++i) { + next = _mm256_loadu_pd(p); + initVal = _mm256_min_pd(initVal, next); + p += width; + } + + // let sum up the final results + const double* q = (const double*)&initVal; + EXTRACT_MIN_VAL(q, p, width, remain, *res) + } + return TSDB_CODE_SUCCESS; +#else + uError("unable run %s without avx2 instructions", __func__); + return TSDB_CODE_OPS_NOT_SUPPORT; +#endif +} diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index a3791eb026..05c9936c2e 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -37,7 +37,6 @@ float tsNumOfCores = 0; int64_t tsTotalMemoryKB = 0; char *tsProcPath = NULL; -char tsSIMDEnable = 1; char tsAVX512Enable = 0; char tsSSE42Supported = 0; char tsAVXSupported = 0; diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index 063988ea00..7f5955f3dd 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -1,5 +1,9 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/src/version.c.in" "${CMAKE_CURRENT_SOURCE_DIR}/src/version.c") aux_source_directory(src UTIL_SRC) +IF(COMPILER_SUPPORT_AVX2) + MESSAGE(STATUS "AVX2 instructions is ACTIVATED") + set_source_files_properties(src/tdecompressavx.c PROPERTIES COMPILE_FLAGS -mavx2) +ENDIF() add_library(util STATIC ${UTIL_SRC}) if(DEFINED GRANT_CFG_INCLUDE_DIR) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 9c9ded693e..525ee71126 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -471,12 +471,12 @@ int32_t tsDecompressINTImp(const char *const input, const int32_t nelements, cha return nelements * word_length; } -#ifdef __AVX512F__ if (tsSIMDEnable && tsAVX512Enable && tsAVX512Supported) { - tsDecompressIntImpl_Hw(input, nelements, output, type); - return nelements * word_length; + int32_t cnt = tsDecompressIntImpl_Hw(input, nelements, output, type); + if (cnt >= 0) { + return cnt; + } } -#endif // Selector value: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 char bit_per_integer[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60}; @@ -867,12 +867,12 @@ int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelement memcpy(output, input + 1, nelements * longBytes); return nelements * longBytes; } else if (input[0] == 1) { // Decompress -#ifdef __AVX512VL__ if (tsSIMDEnable && tsAVX512Enable && tsAVX512Supported) { - tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output, bool bigEndian); - return nelements * longBytes; + int32_t cnt = tsDecompressTimestampAvx512(input, nelements, output, false); + if (cnt >= 0) { + return cnt; + } } -#endif int64_t *ostream = (int64_t *)output; @@ -1103,13 +1103,14 @@ int32_t tsDecompressDoubleImp(const char *const input, int32_t ninput, const int return nelements * DOUBLE_BYTES; } -#ifdef __AVX2__ // use AVX2 implementation when allowed and the compression ratio is not high double compressRatio = 1.0 * nelements * DOUBLE_BYTES / ninput; if (tsSIMDEnable && tsAVX2Supported && compressRatio < 2) { - return tsDecompressDoubleImpAvx2(input + 1, nelements, output); + int32_t cnt = tsDecompressDoubleImpAvx2(input + 1, nelements, output); + if (cnt >= 0) { + return cnt; + } } -#endif // use implementation without SIMD instructions by default return tsDecompressDoubleImpHelper(input + 1, nelements, output); @@ -1257,13 +1258,14 @@ int32_t tsDecompressFloatImp(const char *const input, int32_t ninput, const int3 return nelements * FLOAT_BYTES; } -#ifdef __AVX2__ // use AVX2 implementation when allowed and the compression ratio is not high double compressRatio = 1.0 * nelements * FLOAT_BYTES / ninput; if (tsSIMDEnable && tsAVX2Supported && compressRatio < 2) { - return tsDecompressFloatImpAvx2(input + 1, nelements, output); + int32_t cnt = tsDecompressFloatImpAvx2(input + 1, nelements, output); + if (cnt >= 0) { + return cnt; + } } -#endif // use implementation without SIMD instructions by default return tsDecompressFloatImpHelper(input + 1, nelements, output); @@ -1883,3 +1885,26 @@ int8_t tUpdateCompress(uint32_t oldCmpr, uint32_t newCmpr, uint8_t l2Disabled, u return update; } + +int32_t getWordLength(char type) { + int32_t wordLength = 0; + switch (type) { + case TSDB_DATA_TYPE_BIGINT: + wordLength = LONG_BYTES; + break; + case TSDB_DATA_TYPE_INT: + wordLength = INT_BYTES; + break; + case TSDB_DATA_TYPE_SMALLINT: + wordLength = SHORT_BYTES; + break; + case TSDB_DATA_TYPE_TINYINT: + wordLength = CHAR_BYTES; + break; + default: + uError("Invalid decompress integer type:%d", type); + return TSDB_CODE_INVALID_PARA; + } + + return wordLength; +} diff --git a/source/util/src/tdecompress.c b/source/util/src/tdecompressavx.c similarity index 95% rename from source/util/src/tdecompress.c rename to source/util/src/tdecompressavx.c index 81223d7311..143867b783 100644 --- a/source/util/src/tdecompress.c +++ b/source/util/src/tdecompressavx.c @@ -13,35 +13,16 @@ * along with this program. If not, see . */ -#include "os.h" #include "tcompression.h" -#include "ttypes.h" - -int32_t getWordLength(char type) { - int32_t wordLength = 0; - switch (type) { - case TSDB_DATA_TYPE_BIGINT: - wordLength = LONG_BYTES; - break; - case TSDB_DATA_TYPE_INT: - wordLength = INT_BYTES; - break; - case TSDB_DATA_TYPE_SMALLINT: - wordLength = SHORT_BYTES; - break; - case TSDB_DATA_TYPE_TINYINT: - wordLength = CHAR_BYTES; - break; - default: - uError("Invalid decompress integer type:%d", type); - return TSDB_CODE_INVALID_PARA; - } - - return wordLength; -} #ifdef __AVX2__ +char tsSIMDEnable = 1; +#else +char tsSIMDEnable = 0; +#endif + int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, char *const output, const char type) { +#ifdef __AVX2__ int32_t word_length = getWordLength(type); // Selector value: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 @@ -75,12 +56,12 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, int32_t batch = 0; int32_t remain = 0; if (tsSIMDEnable && tsAVX512Supported && tsAVX512Enable) { -#if __AVX512F__ +#ifdef __AVX512F__ batch = num >> 3; remain = num & 0x07; #endif } else if (tsSIMDEnable && tsAVX2Supported) { -#if __AVX2__ +#ifdef __AVX2__ batch = num >> 2; remain = num & 0x03; #endif @@ -88,7 +69,7 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, if (selector == 0 || selector == 1) { if (tsSIMDEnable && tsAVX512Supported && tsAVX512Enable) { -#if __AVX512F__ +#ifdef __AVX512F__ for (int32_t i = 0; i < batch; ++i) { __m512i prev = _mm512_set1_epi64(prevValue); _mm512_storeu_si512((__m512i *)&p[_pos], prev); @@ -117,7 +98,7 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, } } else { if (tsSIMDEnable && tsAVX512Supported && tsAVX512Enable) { -#if __AVX512F__ +#ifdef __AVX512F__ __m512i sum_mask1 = _mm512_set_epi64(6, 6, 4, 4, 2, 2, 0, 0); __m512i sum_mask2 = _mm512_set_epi64(5, 5, 5, 5, 1, 1, 1, 1); __m512i sum_mask3 = _mm512_set_epi64(3, 3, 3, 3, 3, 3, 3, 3); @@ -310,10 +291,13 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, } return nelements * word_length; +#else + uError("unable run %s without avx2 instructions", __func__); + return -1; +#endif } -#define M256_BYTES sizeof(__m256i) - +#ifdef __AVX2__ FORCE_INLINE __m256i decodeFloatAvx2(const char *data, const char *flag) { __m256i dataVec = _mm256_load_si256((__m256i *)data); __m256i flagVec = _mm256_load_si256((__m256i *)flag); @@ -332,7 +316,27 @@ FORCE_INLINE __m256i decodeFloatAvx2(const char *data, const char *flag) { return diffVec; } +FORCE_INLINE __m256i decodeDoubleAvx2(const char *data, const char *flag) { + __m256i dataVec = _mm256_load_si256((__m256i *)data); + __m256i flagVec = _mm256_load_si256((__m256i *)flag); + __m256i k7 = _mm256_set1_epi64x(7); + __m256i lopart = _mm256_set_epi64x(0, -1, 0, -1); + __m256i hipart = _mm256_set_epi64x(-1, 0, -1, 0); + __m256i trTail = _mm256_cmpgt_epi64(flagVec, k7); + __m256i trHead = _mm256_andnot_si256(trTail, _mm256_set1_epi64x(-1)); + __m256i shiftVec = _mm256_slli_epi64(_mm256_sub_epi64(k7, _mm256_and_si256(flagVec, k7)), 3); + __m256i maskVec = hipart; + __m256i diffVec = _mm256_sllv_epi64(dataVec, _mm256_and_si256(shiftVec, maskVec)); + maskVec = _mm256_or_si256(trHead, lopart); + diffVec = _mm256_srlv_epi64(diffVec, _mm256_and_si256(shiftVec, maskVec)); + maskVec = _mm256_and_si256(trTail, lopart); + diffVec = _mm256_sllv_epi64(diffVec, _mm256_and_si256(shiftVec, maskVec)); + return diffVec; +} +#endif + int32_t tsDecompressFloatImpAvx2(const char *input, int32_t nelements, char *output) { +#ifdef __AVX2__ // Allocate memory-aligned buffer char buf[M256_BYTES * 3]; memset(buf, 0, sizeof(buf)); @@ -343,7 +347,7 @@ int32_t tsDecompressFloatImpAvx2(const char *input, int32_t nelements, char *out // Load data into the buffer for batch processing int32_t batchSize = M256_BYTES / FLOAT_BYTES; - int32_t idx = 0; + int32_t idx = 0; uint32_t cur = 0; for (int32_t i = 0; i < nelements; i += 2) { if (idx == batchSize) { @@ -380,27 +384,14 @@ int32_t tsDecompressFloatImpAvx2(const char *input, int32_t nelements, char *out out += idx * FLOAT_BYTES; } return (int32_t)(out - output); -} - -FORCE_INLINE __m256i decodeDoubleAvx2(const char *data, const char *flag) { - __m256i dataVec = _mm256_load_si256((__m256i *)data); - __m256i flagVec = _mm256_load_si256((__m256i *)flag); - __m256i k7 = _mm256_set1_epi64x(7); - __m256i lopart = _mm256_set_epi64x(0, -1, 0, -1); - __m256i hipart = _mm256_set_epi64x(-1, 0, -1, 0); - __m256i trTail = _mm256_cmpgt_epi64(flagVec, k7); - __m256i trHead = _mm256_andnot_si256(trTail, _mm256_set1_epi64x(-1)); - __m256i shiftVec = _mm256_slli_epi64(_mm256_sub_epi64(k7, _mm256_and_si256(flagVec, k7)), 3); - __m256i maskVec = hipart; - __m256i diffVec = _mm256_sllv_epi64(dataVec, _mm256_and_si256(shiftVec, maskVec)); - maskVec = _mm256_or_si256(trHead, lopart); - diffVec = _mm256_srlv_epi64(diffVec, _mm256_and_si256(shiftVec, maskVec)); - maskVec = _mm256_and_si256(trTail, lopart); - diffVec = _mm256_sllv_epi64(diffVec, _mm256_and_si256(shiftVec, maskVec)); - return diffVec; +#else + uError("unable run %s without avx2 instructions", __func__); + return -1; +#endif } int32_t tsDecompressDoubleImpAvx2(const char *input, const int32_t nelements, char *const output) { +#ifdef __AVX2__ // Allocate memory-aligned buffer char buf[M256_BYTES * 3]; memset(buf, 0, sizeof(buf)); @@ -448,12 +439,15 @@ int32_t tsDecompressDoubleImpAvx2(const char *input, const int32_t nelements, ch out += idx * DOUBLE_BYTES; } return (int32_t)(out - output); -} +#else + uError("unable run %s without avx2 instructions", __func__); + return -1; #endif +} -#if __AVX512VL__ -// decode two timestamps in one loop. -void tsDecompressTimestampAvx2(const char *const input, const int32_t nelements, char *const output, bool bigEndian) { +int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelements, char *const output, + bool bigEndian) { +#ifdef __AVX512VL__ int64_t *ostream = (int64_t *)output; int32_t ipos = 1, opos = 0; @@ -588,11 +582,16 @@ void tsDecompressTimestampAvx2(const char *const input, const int32_t nelements, ostream[opos++] = prevVal[1] + prevDeltaX; } } - return; + return opos; +#else + uError("unable run %s without avx512 instructions", __func__); + return -1; +#endif } -void tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output, - bool UNUSED_PARAM(bigEndian)) { +int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output, + bool UNUSED_PARAM(bigEndian)) { +#ifdef __AVX512VL__ int64_t *ostream = (int64_t *)output; int32_t ipos = 1, opos = 0; @@ -700,6 +699,9 @@ void tsDecompressTimestampAvx512(const char *const input, const int32_t nelement } } - return; -} + return opos; +#else + uError("unable run %s without avx512 instructions", __func__); + return -1; #endif +} From e7b974fd6a41a5dd9b54f85028e202b684c8ce90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=B9=E9=87=91=E6=A0=87?= <34178510+JinbiaoYin@users.noreply.github.com> Date: Thu, 13 Jun 2024 17:19:37 +0800 Subject: [PATCH 13/14] Update 07-tag-index.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文括号错误 --- docs/zh/14-reference/03-taos-sql/07-tag-index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/03-taos-sql/07-tag-index.md b/docs/zh/14-reference/03-taos-sql/07-tag-index.md index 383c5b2a1f..364d465ba3 100644 --- a/docs/zh/14-reference/03-taos-sql/07-tag-index.md +++ b/docs/zh/14-reference/03-taos-sql/07-tag-index.md @@ -11,7 +11,7 @@ description: 使用标签索引提升查询性能 创建索引的语法如下 ```sql -CREATE INDEX index_name ON tbl_name (tagColName） +CREATE INDEX index_name ON tbl_name (tagColName) ``` 其中 `index_name` 为索引名称， `tbl_name` 为超级表名称，`tagColName` 为要在其上建立索引的 tag 列的名称。`tagColName` 的类型不受限制，即任何类型的 tag 列都可以建立索引。 From 74969b0fd4a6d5341092a3733e51bf08e76540ad Mon Sep 17 00:00:00 2001 From: yinheli Date: Mon, 13 May 2024 17:37:08 +0800 Subject: [PATCH 14/14] chore(typo): fix typo --- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 742db8f450..d2d9b2e8eb 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -70,7 +70,7 @@ typedef enum { MND_OPER_WRITE_DB, MND_OPER_READ_DB, MND_OPER_READ_OR_WRITE_DB, - MND_OPER_SHOW_VARIBALES, + MND_OPER_SHOW_VARIABLES, MND_OPER_SUBSCRIBE, MND_OPER_CREATE_TOPIC, MND_OPER_DROP_TOPIC, diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 5e10583a0a..2a1689854b 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -1068,7 +1068,7 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { SShowVariablesRsp rsp = {0}; int32_t code = -1; - if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_SHOW_VARIBALES) != 0) { + if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_SHOW_VARIABLES) != 0) { goto _OVER; }