From 5fffeba2f3759f74c78fe79ede05048d4f585d7c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 3 Nov 2022 10:39:53 +0800 Subject: [PATCH 1/7] fix(query): fix countAlwaysReturnValue param not taking effect for count and hyperloglog function TD-20140 --- source/libs/function/src/builtinsimpl.c | 2 +- source/libs/function/src/functionMgt.c | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index c8f0b3d826..9ae169bcf4 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -551,7 +551,7 @@ int32_t countFunction(SqlFunctionCtx* pCtx) { if (tsCountAlwaysReturnValue) { pResInfo->numOfRes = 1; } else { - SET_VAL(pResInfo, 1, 1); + SET_VAL(pResInfo, *((int64_t*)buf), 1); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 0de914271c..6ab91a4483 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -241,7 +241,11 @@ bool fmIsNotNullOutputFunc(int32_t funcId) { FUNCTION_TYPE_LAST_MERGE == funcMgtBuiltins[funcId].type || FUNCTION_TYPE_FIRST == funcMgtBuiltins[funcId].type || FUNCTION_TYPE_FIRST_PARTIAL == funcMgtBuiltins[funcId].type || - FUNCTION_TYPE_FIRST_MERGE == funcMgtBuiltins[funcId].type; + FUNCTION_TYPE_FIRST_MERGE == funcMgtBuiltins[funcId].type || + FUNCTION_TYPE_COUNT == funcMgtBuiltins[funcId].type || + FUNCTION_TYPE_HYPERLOGLOG == funcMgtBuiltins[funcId].type || + FUNCTION_TYPE_HYPERLOGLOG_PARTIAL == funcMgtBuiltins[funcId].type || + FUNCTION_TYPE_HYPERLOGLOG_MERGE == funcMgtBuiltins[funcId].type; } bool fmIsSelectValueFunc(int32_t funcId) { From e20d0038b42e5873c38aca20c7cba563486c3c1f Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 3 Nov 2022 11:12:27 +0800 Subject: [PATCH 2/7] add client side config for select count(NULL) --- source/common/src/tglobal.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 7f4a826c5e..a04c3dab03 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -286,6 +286,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddDir(pCfg, "tempDir", tsTempDir, 1) != 0) return -1; if (cfgAddFloat(pCfg, "minimalTmpDirGB", 1.0f, 0.001f, 10000000, 1) != 0) return -1; if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, 0) != 0) return -1; if (cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, 1) != 0) return -1; if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, 1) != 0) return -1; if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 4, 1) != 0) return -1; @@ -652,6 +653,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32; tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32; + tsCountAlwaysReturnValue = cfgGetItem(pCfg, "countAlwaysReturnValue")->i32; tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32; tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32; tsEnableQueryHb = cfgGetItem(pCfg, "enableQueryHb")->bval; From 49715a2898149e0e8e79aa64dd53c0f5fb03dafc Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 3 Nov 2022 12:52:04 +0800 Subject: [PATCH 3/7] Revert "add client side config for select count(NULL)" This reverts commit e20d0038b42e5873c38aca20c7cba563486c3c1f. --- source/common/src/tglobal.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index a04c3dab03..7f4a826c5e 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -286,7 +286,6 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddDir(pCfg, "tempDir", tsTempDir, 1) != 0) return -1; if (cfgAddFloat(pCfg, "minimalTmpDirGB", 1.0f, 0.001f, 10000000, 1) != 0) return -1; if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, 0) != 0) return -1; if (cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, 1) != 0) return -1; if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, 1) != 0) return -1; if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 4, 1) != 0) return -1; @@ -653,7 +652,6 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32; tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32; - tsCountAlwaysReturnValue = cfgGetItem(pCfg, "countAlwaysReturnValue")->i32; tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32; tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32; tsEnableQueryHb = cfgGetItem(pCfg, "enableQueryHb")->bval; From 0b3a3aff214fc27d78fe51230c98d99b1d3737ed Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 3 Nov 2022 15:01:52 +0800 Subject: [PATCH 4/7] fix hll tsCountAlwaysReturnValue not working --- source/libs/function/src/builtinsimpl.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 9ae169bcf4..889d3aeda9 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -181,6 +181,7 @@ typedef enum { UNKNOWN_BIN = 0, USER_INPUT_BIN, LINEAR_BIN, LOG_BIN } EHistoBinT typedef struct SHLLFuncInfo { uint64_t result; + uint64_t totalCount; uint8_t buckets[HLL_BUCKETS]; } SHLLInfo; @@ -4605,7 +4606,14 @@ int32_t hllFunction(SqlFunctionCtx* pCtx) { } } - SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); + pInfo->totalCount += numOfElems; + + if (pInfo->totalCount == 0 && !tsCountAlwaysReturnValue) { + SET_VAL(GET_RES_INFO(pCtx), 0, 1); + } else { + SET_VAL(GET_RES_INFO(pCtx), 1, 1); + } + return TSDB_CODE_SUCCESS; } @@ -4615,6 +4623,7 @@ static void hllTransferInfo(SHLLInfo* pInput, SHLLInfo* pOutput) { pOutput->buckets[k] = pInput->buckets[k]; } } + pOutput->totalCount += pInput->totalCount; } int32_t hllFunctionMerge(SqlFunctionCtx* pCtx) { @@ -4632,7 +4641,12 @@ int32_t hllFunctionMerge(SqlFunctionCtx* pCtx) { hllTransferInfo(pInputInfo, pInfo); } - SET_VAL(GET_RES_INFO(pCtx), 1, 1); + if (pInfo->totalCount == 0 && !tsCountAlwaysReturnValue) { + SET_VAL(GET_RES_INFO(pCtx), 0, 1); + } else { + SET_VAL(GET_RES_INFO(pCtx), 1, 1); + } + return TSDB_CODE_SUCCESS; } From 3a08d3bea4617004bb5a3922b50a655451fd5df7 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 3 Nov 2022 16:22:21 +0800 Subject: [PATCH 5/7] add test cases --- .../2-query/countAlwaysReturnValue.py | 125 ++++++++++++++++++ tests/system-test/2-query/hyperloglog.py | 2 +- tests/system-test/fulltest.sh | 9 +- 3 files changed, 133 insertions(+), 3 deletions(-) create mode 100644 tests/system-test/2-query/countAlwaysReturnValue.py diff --git a/tests/system-test/2-query/countAlwaysReturnValue.py b/tests/system-test/2-query/countAlwaysReturnValue.py new file mode 100644 index 0000000000..25eae49516 --- /dev/null +++ b/tests/system-test/2-query/countAlwaysReturnValue.py @@ -0,0 +1,125 @@ +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * +import random + + +class TDTestCase: + updatecfgDict = {"countAlwaysReturnValue":0} + + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def prepare_data(self, dbname="db"): + tdSql.execute( + f"create database if not exists {dbname} keep 3650 duration 1000") + tdSql.execute(f"use {dbname} ") + tdSql.execute( + f"create table {dbname}.tb (ts timestamp, c0 int)" + ) + tdSql.execute( + f"create table {dbname}.stb (ts timestamp, c0 int) tags (t0 int)" + ) + tdSql.execute( + f"create table {dbname}.ctb1 using {dbname}.stb tags (1)" + ) + tdSql.execute( + f"create table {dbname}.ctb2 using {dbname}.stb tags (2)" + ) + + tdSql.execute( + f"insert into {dbname}.tb values (now(), NULL)") + + tdSql.execute( + f"insert into {dbname}.ctb1 values (now(), NULL)") + + tdSql.execute( + f"insert into {dbname}.ctb2 values (now() + 1s, NULL)") + + def test_results(self, dbname="db"): + + # count + tdSql.query(f"select count(c0) from {dbname}.tb") + tdSql.checkRows(0) + + tdSql.query(f"select count(NULL) from {dbname}.tb") + tdSql.checkRows(0) + + tdSql.query(f"select c0,count(c0) from {dbname}.tb group by c0") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query(f"select count(c0) from {dbname}.stb") + tdSql.checkRows(0) + + tdSql.query(f"select count(NULL) from {dbname}.stb") + tdSql.checkRows(0) + + tdSql.query(f"select c0,count(c0) from {dbname}.stb group by c0") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query(f"select tbname,count(c0) from {dbname}.stb partition by tbname") + tdSql.checkRows(2) + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + + tdSql.query(f"select count(NULL)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + # hyperloglog + tdSql.query(f"select hyperloglog(c0) from {dbname}.tb") + tdSql.checkRows(0) + + tdSql.query(f"select hyperloglog(NULL) from {dbname}.tb") + tdSql.checkRows(0) + + tdSql.query(f"select c0,hyperloglog(c0) from {dbname}.tb group by c0") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query(f"select hyperloglog(c0) from {dbname}.stb") + tdSql.checkRows(0) + + tdSql.query(f"select hyperloglog(NULL) from {dbname}.stb") + tdSql.checkRows(0) + + tdSql.query(f"select c0,hyperloglog(c0) from {dbname}.stb group by c0") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query(f"select tbname,hyperloglog(c0) from {dbname}.stb partition by tbname") + tdSql.checkRows(2) + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + + tdSql.query(f"select hyperloglog(NULL)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + def run(self): + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:prepare data ==============") + + self.prepare_data() + + tdLog.printNoPrefix("==========step2:test results ==============") + + self.test_results() + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/hyperloglog.py b/tests/system-test/2-query/hyperloglog.py index eff687cba7..a139cc19af 100644 --- a/tests/system-test/2-query/hyperloglog.py +++ b/tests/system-test/2-query/hyperloglog.py @@ -28,7 +28,7 @@ ALL_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BOOL_C DBNAME = "db" class TDTestCase: - + updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } def init(self, conn, logSql, replicaVar=1): diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index c65c574e7c..3bbf3096a4 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -17,7 +17,7 @@ python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3 python3 ./test.py -f 0-others/sysinfo.py python3 ./test.py -f 0-others/user_control.py python3 ./test.py -f 0-others/fsync.py -python3 ./test.py -f 0-others/compatibility.py +python3 ./test.py -f 0-others/compatibility.py python3 ./test.py -f 1-insert/alter_database.py python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py @@ -76,6 +76,8 @@ python3 ./test.py -f 2-query/count_partition.py python3 ./test.py -f 2-query/count_partition.py -R python3 ./test.py -f 2-query/count.py python3 ./test.py -f 2-query/count.py -R +python3 ./test.py -f 2-query/countAlwaysReturnValue.py +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -R python3 ./test.py -f 2-query/db.py python3 ./test.py -f 2-query/db.py -R python3 ./test.py -f 2-query/diff.py @@ -383,6 +385,7 @@ python3 ./test.py -f 2-query/Today.py -Q 2 python3 ./test.py -f 2-query/max.py -Q 2 python3 ./test.py -f 2-query/min.py -Q 2 python3 ./test.py -f 2-query/count.py -Q 2 +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 2 python3 ./test.py -f 2-query/last.py -Q 2 python3 ./test.py -f 2-query/first.py -Q 2 python3 ./test.py -f 2-query/To_iso8601.py -Q 2 @@ -478,6 +481,7 @@ python3 ./test.py -f 2-query/Today.py -Q 3 python3 ./test.py -f 2-query/max.py -Q 3 python3 ./test.py -f 2-query/min.py -Q 3 python3 ./test.py -f 2-query/count.py -Q 3 +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 3 python3 ./test.py -f 2-query/last.py -Q 3 python3 ./test.py -f 2-query/first.py -Q 3 python3 ./test.py -f 2-query/To_iso8601.py -Q 3 @@ -575,6 +579,7 @@ python3 ./test.py -f 2-query/Today.py -Q 4 python3 ./test.py -f 2-query/max.py -Q 4 python3 ./test.py -f 2-query/min.py -Q 4 python3 ./test.py -f 2-query/count.py -Q 4 +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 4 python3 ./test.py -f 2-query/last.py -Q 4 python3 ./test.py -f 2-query/first.py -Q 4 python3 ./test.py -f 2-query/To_iso8601.py -Q 4 @@ -590,7 +595,7 @@ python3 ./test.py -f 2-query/apercentile.py -Q 4 python3 ./test.py -f 2-query/abs.py -Q 4 python3 ./test.py -f 2-query/ceil.py -Q 4 python3 ./test.py -f 2-query/floor.py -Q 4 -python3 ./test.py -f 2-query/round.py -Q 4 +python3 ./test.py -f 2-query/round.py -Q 4 python3 ./test.py -f 2-query/log.py -Q 4 python3 ./test.py -f 2-query/pow.py -Q 4 python3 ./test.py -f 2-query/sqrt.py -Q 4 From 694253b7d319feae334110ddd90d59924bf6c855 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 4 Nov 2022 10:11:24 +0800 Subject: [PATCH 6/7] add test case --- tests/system-test/2-query/countAlwaysReturnValue.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/system-test/2-query/countAlwaysReturnValue.py b/tests/system-test/2-query/countAlwaysReturnValue.py index 25eae49516..08c5ae104f 100644 --- a/tests/system-test/2-query/countAlwaysReturnValue.py +++ b/tests/system-test/2-query/countAlwaysReturnValue.py @@ -65,11 +65,6 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, None) - tdSql.query(f"select tbname,count(c0) from {dbname}.stb partition by tbname") - tdSql.checkRows(2) - tdSql.checkData(0, 1, None) - tdSql.checkData(1, 1, None) - tdSql.query(f"select count(NULL)") tdSql.checkRows(1) tdSql.checkData(0, 0, 0) @@ -95,11 +90,6 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, None) - tdSql.query(f"select tbname,hyperloglog(c0) from {dbname}.stb partition by tbname") - tdSql.checkRows(2) - tdSql.checkData(0, 1, None) - tdSql.checkData(1, 1, None) - tdSql.query(f"select hyperloglog(NULL)") tdSql.checkRows(1) tdSql.checkData(0, 0, 0) From 534550f468831f993af0e77b45ec6260d6d03835 Mon Sep 17 00:00:00 2001 From: Huo Linhe Date: Sun, 6 Nov 2022 17:23:03 +0800 Subject: [PATCH 7/7] chore: update taosws to v0.3.8 --- cmake/taosws_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in index 67a23b32b0..33ca6c659c 100644 --- a/cmake/taosws_CMakeLists.txt.in +++ b/cmake/taosws_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosws-rs ExternalProject_Add(taosws-rs GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git - GIT_TAG 0373a70 + GIT_TAG 38c4599 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" BINARY_DIR "" #BUILD_IN_SOURCE TRUE