From 5341a6eeac510289bd1d9e0bacfed15cbc84f388 Mon Sep 17 00:00:00 2001 From: jtcheng Date: Fri, 2 Jul 2021 17:01:49 +0800 Subject: [PATCH 01/27] [TD-6536]: Fix memory leak during alter database --- src/query/src/qSqlParser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qSqlParser.c b/src/query/src/qSqlParser.c index a2cb7aee00..2021bc41c0 100644 --- a/src/query/src/qSqlParser.c +++ b/src/query/src/qSqlParser.c @@ -938,7 +938,7 @@ void SqlInfoDestroy(SSqlInfo *pInfo) { taosArrayDestroy(pInfo->pMiscInfo->a); } - if (pInfo->pMiscInfo != NULL && pInfo->type == TSDB_SQL_CREATE_DB) { + if (pInfo->pMiscInfo != NULL && (pInfo->type == TSDB_SQL_CREATE_DB || pInfo->type == TSDB_SQL_ALTER_DB)) { taosArrayDestroyEx(pInfo->pMiscInfo->dbOpt.keep, freeVariant); } From 95bc9aa96f10ff86ad3b96703220960060a883fe Mon Sep 17 00:00:00 2001 From: jtcheng Date: Mon, 5 Jul 2021 10:10:40 +0800 Subject: [PATCH 02/27] [self-assign]: Fix self-assignment --- src/client/src/tscServer.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 029fa853ac..fdd748b693 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2005,7 +2005,6 @@ static SVgroupsInfo* createVgroupInfoFromMsg(char* pMsg, int32_t* size, uint64_t SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; vmsg->vgId = htonl(vmsg->vgId); - vmsg->numOfEps = vmsg->numOfEps; for (int32_t k = 0; k < vmsg->numOfEps; ++k) { vmsg->epAddr[k].port = htons(vmsg->epAddr[k].port); } From 1caaab48eef593f5df7dcd68a6af65ff350bc3e4 Mon Sep 17 00:00:00 2001 From: wpan Date: Wed, 7 Jul 2021 10:06:16 +0800 Subject: [PATCH 03/27] fix nchar filter issue --- src/client/src/tscUtil.c | 36 ++++++++++++++++++++++++++++++++++++ src/query/inc/qExecutor.h | 1 + 2 files changed, 37 insertions(+) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 838416fa2e..7b010677c3 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -746,6 +746,37 @@ typedef struct SJoinOperatorInfo { SRspResultInfo resultInfo; // todo refactor, add this info for each operator } SJoinOperatorInfo; +static void converNcharFilterColumn(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t rows, bool *gotNchar) { + for (int32_t i = 0; i < numOfFilterCols; ++i) { + if (pFilterInfo[i].info.type == TSDB_DATA_TYPE_NCHAR) { + pFilterInfo[i].pData2 = pFilterInfo[i].pData; + pFilterInfo[i].pData = malloc(rows * pFilterInfo[i].info.bytes); + int32_t bufSize = pFilterInfo[i].info.bytes - VARSTR_HEADER_SIZE; + for (int32_t j = 0; j < rows; ++j) { + char* dst = pFilterInfo[i].pData + j * pFilterInfo[i].info.bytes; + char* src = pFilterInfo[i].pData2 + j * pFilterInfo[i].info.bytes; + int32_t len = 0; + taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len); + varDataLen(dst) = len; + } + *gotNchar = true; + } + } +} + +static void freeNcharFilterColumn(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols) { + for (int32_t i = 0; i < numOfFilterCols; ++i) { + if (pFilterInfo[i].info.type == TSDB_DATA_TYPE_NCHAR) { + if (pFilterInfo[i].pData2) { + tfree(pFilterInfo[i].pData); + pFilterInfo[i].pData = pFilterInfo[i].pData2; + pFilterInfo[i].pData2 = NULL; + } + } + } +} + + static void doSetupSDataBlock(SSqlRes* pRes, SSDataBlock* pBlock, SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols) { int32_t offset = 0; char* pData = pRes->data; @@ -764,8 +795,13 @@ static void doSetupSDataBlock(SSqlRes* pRes, SSDataBlock* pBlock, SSingleColumnF // filter data if needed if (numOfFilterCols > 0) { doSetFilterColumnInfo(pFilterInfo, numOfFilterCols, pBlock); + bool gotNchar = false; + converNcharFilterColumn(pFilterInfo, numOfFilterCols, pBlock->info.rows, &gotNchar); int8_t* p = calloc(pBlock->info.rows, sizeof(int8_t)); bool all = doFilterDataBlock(pFilterInfo, numOfFilterCols, pBlock->info.rows, p); + if (gotNchar) { + freeNcharFilterColumn(pFilterInfo, numOfFilterCols); + } if (!all) { doCompactSDataBlock(pBlock, pBlock->info.rows, p); } diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index bc934647ec..1d26d292fa 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -118,6 +118,7 @@ typedef struct SColumnFilterElem { typedef struct SSingleColumnFilterInfo { void* pData; + void* pData2; //used for nchar column int32_t numOfFilters; SColumnInfo info; SColumnFilterElem* pFilters; From 7f25afc8311989fd9de2085e1e427037202d6421 Mon Sep 17 00:00:00 2001 From: wpan Date: Wed, 7 Jul 2021 11:12:37 +0800 Subject: [PATCH 04/27] fix windows compile error --- src/client/src/tscUtil.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 7b010677c3..adb7e33902 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -753,8 +753,8 @@ static void converNcharFilterColumn(SSingleColumnFilterInfo* pFilterInfo, int32_ pFilterInfo[i].pData = malloc(rows * pFilterInfo[i].info.bytes); int32_t bufSize = pFilterInfo[i].info.bytes - VARSTR_HEADER_SIZE; for (int32_t j = 0; j < rows; ++j) { - char* dst = pFilterInfo[i].pData + j * pFilterInfo[i].info.bytes; - char* src = pFilterInfo[i].pData2 + j * pFilterInfo[i].info.bytes; + char* dst = (char *)pFilterInfo[i].pData + j * pFilterInfo[i].info.bytes; + char* src = (char *)pFilterInfo[i].pData2 + j * pFilterInfo[i].info.bytes; int32_t len = 0; taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len); varDataLen(dst) = len; From 7f3654bdfc60a15f9a93666bb3b2fe9b36c5e5eb Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 7 Jul 2021 16:41:04 +0800 Subject: [PATCH 05/27] [TD-5056] fix input exception --- src/kit/shell/inc/shellCommand.h | 2 +- src/kit/shell/src/shellCommand.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kit/shell/inc/shellCommand.h b/src/kit/shell/inc/shellCommand.h index 3094bdb9dd..a08c1f48d1 100644 --- a/src/kit/shell/inc/shellCommand.h +++ b/src/kit/shell/inc/shellCommand.h @@ -45,7 +45,7 @@ extern void updateBuffer(Command *cmd); extern int isReadyGo(Command *cmd); extern void resetCommand(Command *cmd, const char s[]); -int countPrefixOnes(char c); +int countPrefixOnes(unsigned char c); void clearScreen(int ecmd_pos, int cursor_pos); void printChar(char c, int times); void positionCursor(int step, int direction); diff --git a/src/kit/shell/src/shellCommand.c b/src/kit/shell/src/shellCommand.c index 9173ab0efd..e1a3dfe102 100644 --- a/src/kit/shell/src/shellCommand.c +++ b/src/kit/shell/src/shellCommand.c @@ -26,7 +26,7 @@ typedef struct { char widthOnScreen; } UTFCodeInfo; -int countPrefixOnes(char c) { +int countPrefixOnes(unsigned char c) { unsigned char mask = 127; mask = ~mask; int ret = 0; @@ -48,7 +48,7 @@ void getPrevCharSize(const char *str, int pos, int *size, int *width) { while (--pos >= 0) { *size += 1; - if (str[pos] > 0 || countPrefixOnes(str[pos]) > 1) break; + if (str[pos] > 0 || countPrefixOnes((unsigned char )str[pos]) > 1) break; } int rc = mbtowc(&wc, str + pos, MB_CUR_MAX); From c5ee2fd58d1a5bac94194214f4c8a3c0e747ed7f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 7 Jul 2021 17:02:33 +0800 Subject: [PATCH 06/27] [td-225] refactor. --- src/query/src/qExecutor.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 6bd5e03377..e98c7830f3 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1265,8 +1265,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul for (int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. SResultRow* pRes = pResultRowInfo->pResult[j]; if (pRes->closed) { - assert(resultRowInterpolated(pRes, RESULT_ROW_START_INTERP) && - resultRowInterpolated(pRes, RESULT_ROW_END_INTERP)); + assert(resultRowInterpolated(pRes, RESULT_ROW_START_INTERP) && resultRowInterpolated(pRes, RESULT_ROW_END_INTERP)); continue; } @@ -5327,9 +5326,16 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { SOperatorInfo* upstream = pOperator->upstream[0]; + int64_t ss = taosGetTimestampMs(); + printf("=-=============%ld\n", ss); while(1) { publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + int64_t s = taosGetTimestampMs(); + printf("start: %ld\n", s); SSDataBlock* pBlock = upstream->exec(upstream, newgroup); + + int64_t end = taosGetTimestampMs(); + printf("end: %ld, el:%ld \n", end, end - s); publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -5338,9 +5344,16 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); + + int64_t x1 = taosGetTimestampMs(); hashIntervalAgg(pOperator, &pIntervalInfo->resultRowInfo, pBlock, 0); + int64_t x2 = taosGetTimestampMs(); + printf("-------------------------inter:%ld\n", x2-x1); } + int64_t s1 = taosGetTimestampMs(); + printf("=-=============%ld, el:%ld\n", s1, s1-ss); + // restore the value pQueryAttr->order.order = order; pQueryAttr->window = win; From ce9119b332bae9abce1cba238f40ae3314f3cd24 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 7 Jul 2021 17:24:51 +0800 Subject: [PATCH 07/27] Feature/sangshuduo/td 3973 use jemalloc (#6788) * [TD-3973]: add jemalloc as submodule. * add macro definitions in cmake. * [TD-3973]: use jemalloc. build works as following instructions: cmake .. -DJEMALLOC_ENABLED=true make * fix jemalloc at tag 5.2.1 * link jemalloc works. * make install works. * support jemalloc in release.sh. * release script works. * fix a typo. * [TD-3937]: support jemalloc add install funtion to all scripts. * adjust install_jemalloc() position for update check compatiblity. * fix position bug. * add ldconfig for jemalloc library cache refresh. * add /etc/ld.so.conf.d/jemalloc.conf for centos * check ver comp by file. Co-authored-by: Shuduo Sang --- packaging/release.sh | 8 +++--- packaging/tools/install.sh | 8 ++++-- packaging/tools/install_power.sh | 8 ++++-- packaging/tools/install_tq.sh | 8 ++++-- packaging/tools/makepkg.sh | 4 +-- packaging/tools/makepkg_power.sh | 42 ++++++++++++++++---------------- packaging/tools/makepkg_tq.sh | 30 +++++++++++------------ 7 files changed, 60 insertions(+), 48 deletions(-) diff --git a/packaging/release.sh b/packaging/release.sh index 34fda44243..5ba6c01a0b 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -204,7 +204,7 @@ else exit 1 fi -make -j8 +make cd ${curr_dir} @@ -246,15 +246,15 @@ if [ "$osType" != "Darwin" ]; then cd ${script_dir}/tools if [[ "$dbName" == "taos" ]]; then - ${csudo} ./makepkg.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} + ${csudo} ./makepkg.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${verNumberComp} ${csudo} ./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${csudo} ./makearbi.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} elif [[ "$dbName" == "tq" ]]; then - ${csudo} ./makepkg_tq.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} + ${csudo} ./makepkg_tq.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} ${verNumberComp} ${csudo} ./makeclient_tq.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} ${csudo} ./makearbi_tq.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} else - ${csudo} ./makepkg_power.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} + ${csudo} ./makepkg_power.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} ${verNumberComp} ${csudo} ./makeclient_power.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} ${csudo} ./makearbi_power.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} fi diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index c8970ed821..57467e4b72 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -772,9 +772,13 @@ vercomp () { function is_version_compatible() { - curr_version=$(${bin_dir}/taosd -V | head -1 | cut -d ' ' -f 3) + curr_version=`ls ${script_dir}/driver/libtaos.so* |cut -d '.' -f 3-6` - min_compatible_version=$(${script_dir}/bin/taosd -V | head -1 | cut -d ' ' -f 5) + if [ -f ${script_dir}/driver/vercomp.txt ]; then + min_compatible_version=`cat ${script_dir}/driver/vercomp.txt` + else + min_compatible_version=$(${script_dir}/bin/tqd -V | head -1 | cut -d ' ' -f 5) + fi vercomp $curr_version $min_compatible_version case $? in diff --git a/packaging/tools/install_power.sh b/packaging/tools/install_power.sh index b73b0b70c0..d0220cca25 100755 --- a/packaging/tools/install_power.sh +++ b/packaging/tools/install_power.sh @@ -741,9 +741,13 @@ vercomp () { function is_version_compatible() { - curr_version=$(${bin_dir}/powerd -V | head -1 | cut -d ' ' -f 3) + curr_version=`ls ${script_dir}/driver/libtaos.so* |cut -d '.' -f 3-6` - min_compatible_version=$(${script_dir}/bin/powerd -V | head -1 | cut -d ' ' -f 5) + if [ -f ${script_dir}/driver/vercomp.txt ]; then + min_compatible_version=`cat ${script_dir}/driver/vercomp.txt` + else + min_compatible_version=$(${script_dir}/bin/tqd -V | head -1 | cut -d ' ' -f 5) + fi vercomp $curr_version $min_compatible_version case $? in diff --git a/packaging/tools/install_tq.sh b/packaging/tools/install_tq.sh index 4bdb39838a..52e08cb6b0 100755 --- a/packaging/tools/install_tq.sh +++ b/packaging/tools/install_tq.sh @@ -741,9 +741,13 @@ vercomp () { function is_version_compatible() { - curr_version=$(${bin_dir}/tqd -V | head -1 | cut -d ' ' -f 3) + curr_version=`ls ${script_dir}/driver/libtaos.so* |cut -d '.' -f 3-6` - min_compatible_version=$(${script_dir}/bin/tqd -V | head -1 | cut -d ' ' -f 5) + if [ -f ${script_dir}/driver/vercomp.txt ]; then + min_compatible_version=`cat ${script_dir}/driver/vercomp.txt` + else + min_compatible_version=$(${script_dir}/bin/tqd -V | head -1 | cut -d ' ' -f 5) + fi vercomp $curr_version $min_compatible_version case $? in diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 624f72278a..81061416a2 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -14,6 +14,7 @@ osType=$5 verMode=$6 verType=$7 pagMode=$8 +versionComp=$9 script_dir="$(dirname $(readlink -f $0))" top_dir="$(readlink -f ${script_dir}/../..)" @@ -175,8 +176,7 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then cp -r ${examples_dir}/C# ${install_dir}/examples fi # Copy driver -mkdir -p ${install_dir}/driver -cp ${lib_files} ${install_dir}/driver +mkdir -p ${install_dir}/driver && cp ${lib_files} ${install_dir}/driver && echo "${versionComp}" > ${install_dir}/driver/vercomp.txt # Copy connector connector_dir="${code_dir}/connector" diff --git a/packaging/tools/makepkg_power.sh b/packaging/tools/makepkg_power.sh index 633a135c14..a2643b7486 100755 --- a/packaging/tools/makepkg_power.sh +++ b/packaging/tools/makepkg_power.sh @@ -14,6 +14,7 @@ osType=$5 verMode=$6 verType=$7 pagMode=$8 +versionComp=$9 script_dir="$(dirname $(readlink -f $0))" top_dir="$(readlink -f ${script_dir}/../..)" @@ -32,10 +33,10 @@ fi # Directories and files. #if [ "$pagMode" == "lite" ]; then -# strip ${build_dir}/bin/taosd +# strip ${build_dir}/bin/taosd # strip ${build_dir}/bin/taos # bin_files="${build_dir}/bin/powerd ${build_dir}/bin/power ${script_dir}/remove_power.sh" -#else +#else # bin_files="${build_dir}/bin/powerd ${build_dir}/bin/power ${build_dir}/bin/powerdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove_power.sh\ # ${script_dir}/set_core.sh ${script_dir}/startPre.sh ${script_dir}/taosd-dump-cfg.gdb" #fi @@ -70,19 +71,19 @@ mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/taos.cf #mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : mkdir -p ${install_dir}/bin if [ "$pagMode" == "lite" ]; then - strip ${build_dir}/bin/taosd + strip ${build_dir}/bin/taosd strip ${build_dir}/bin/taos # bin_files="${build_dir}/bin/powerd ${build_dir}/bin/power ${script_dir}/remove_power.sh" cp ${build_dir}/bin/taos ${install_dir}/bin/power cp ${build_dir}/bin/taosd ${install_dir}/bin/powerd cp ${script_dir}/remove_power.sh ${install_dir}/bin -else +else # bin_files="${build_dir}/bin/powerd ${build_dir}/bin/power ${build_dir}/bin/powerdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove_power.sh ${script_dir}/set_core.sh" cp ${build_dir}/bin/taos ${install_dir}/bin/power cp ${build_dir}/bin/taosd ${install_dir}/bin/powerd cp ${script_dir}/remove_power.sh ${install_dir}/bin - cp ${build_dir}/bin/taosdemo ${install_dir}/bin/powerdemo - cp ${build_dir}/bin/taosdump ${install_dir}/bin/powerdump + cp ${build_dir}/bin/taosdemo ${install_dir}/bin/powerdemo + cp ${build_dir}/bin/taosdump ${install_dir}/bin/powerdump cp ${build_dir}/bin/tarbitrator ${install_dir}/bin cp ${script_dir}/set_core.sh ${install_dir}/bin cp ${script_dir}/get_client.sh ${install_dir}/bin @@ -99,14 +100,14 @@ mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir} if [ "$verMode" == "cluster" ]; then sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove_power.sh >> remove_power_temp.sh mv remove_power_temp.sh ${install_dir}/bin/remove_power.sh - + mkdir -p ${install_dir}/nginxd && cp -r ${nginx_dir}/* ${install_dir}/nginxd cp ${nginx_dir}/png/taos.png ${install_dir}/nginxd/admin/images/taos.png rm -rf ${install_dir}/nginxd/png - sed -i "s/TDengine/PowerDB/g" ${install_dir}/nginxd/admin/*.html + sed -i "s/TDengine/PowerDB/g" ${install_dir}/nginxd/admin/*.html sed -i "s/TDengine/PowerDB/g" ${install_dir}/nginxd/admin/js/*.js - + sed -i '/dataDir/ {s/taos/power/g}' ${install_dir}/cfg/taos.cfg sed -i '/logDir/ {s/taos/power/g}' ${install_dir}/cfg/taos.cfg sed -i "s/TDengine/PowerDB/g" ${install_dir}/cfg/taos.cfg @@ -149,17 +150,16 @@ sed -i '/root/ {s/taosdata/powerdb/g}' ${install_dir}/examples/c/*.c if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then cp -r ${examples_dir}/JDBC ${install_dir}/examples cp -r ${examples_dir}/matlab ${install_dir}/examples - sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/examples/matlab/TDengineDemo.m + sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/examples/matlab/TDengineDemo.m cp -r ${examples_dir}/python ${install_dir}/examples - sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/examples/python/read_example.py + sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/examples/python/read_example.py cp -r ${examples_dir}/R ${install_dir}/examples - sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/examples/R/command.txt - cp -r ${examples_dir}/go ${install_dir}/examples + sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/examples/R/command.txt + cp -r ${examples_dir}/go ${install_dir}/examples sed -i '/root/ {s/taosdata/powerdb/g}' ${install_dir}/examples/go/taosdemo.go fi # Copy driver -mkdir -p ${install_dir}/driver -cp ${lib_files} ${install_dir}/driver +mkdir -p ${install_dir}/driver && cp ${lib_files} ${install_dir}/driver && echo "${versionComp}" > ${install_dir}/driver/vercomp.txt # Copy connector connector_dir="${code_dir}/connector" @@ -178,11 +178,11 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then echo "WARNING: go connector not found, please check if want to use it!" fi cp -r ${connector_dir}/python ${install_dir}/connector/ - + sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/taos/cinterface.py - + sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/taos/subscription.py - + sed -i '/self._password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/taos/connection.py fi # Copy release note @@ -190,7 +190,7 @@ fi # exit 1 -cd ${release_dir} +cd ${release_dir} if [ "$verMode" == "cluster" ]; then pkg_name=${install_dir}-${osType}-${cpuType} @@ -207,8 +207,8 @@ fi if [ "$verType" == "beta" ]; then pkg_name=${pkg_name}-${verType} -elif [ "$verType" == "stable" ]; then - pkg_name=${pkg_name} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} else echo "unknow verType, nor stabel or beta" exit 1 diff --git a/packaging/tools/makepkg_tq.sh b/packaging/tools/makepkg_tq.sh index 086092ea33..6f897de0ce 100755 --- a/packaging/tools/makepkg_tq.sh +++ b/packaging/tools/makepkg_tq.sh @@ -14,6 +14,7 @@ osType=$5 verMode=$6 verType=$7 pagMode=$8 +versionComp=$9 script_dir="$(dirname $(readlink -f $0))" top_dir="$(readlink -f ${script_dir}/../..)" @@ -32,10 +33,10 @@ fi # Directories and files. #if [ "$pagMode" == "lite" ]; then -# strip ${build_dir}/bin/taosd +# strip ${build_dir}/bin/taosd # strip ${build_dir}/bin/taos # bin_files="${build_dir}/bin/tqd ${build_dir}/bin/tq ${script_dir}/remove_tq.sh" -#else +#else # bin_files="${build_dir}/bin/tqd ${build_dir}/bin/tq ${build_dir}/bin/tqdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove_tq.sh\ # ${script_dir}/set_core.sh ${script_dir}/startPre.sh ${script_dir}/taosd-dump-cfg.gdb" #fi @@ -70,13 +71,13 @@ mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/taos.cf #mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : mkdir -p ${install_dir}/bin if [ "$pagMode" == "lite" ]; then - strip ${build_dir}/bin/taosd + strip ${build_dir}/bin/taosd strip ${build_dir}/bin/taos # bin_files="${build_dir}/bin/tqd ${build_dir}/bin/tq ${script_dir}/remove_tq.sh" cp ${build_dir}/bin/taos ${install_dir}/bin/tq cp ${build_dir}/bin/taosd ${install_dir}/bin/tqd cp ${script_dir}/remove_tq.sh ${install_dir}/bin -else +else # bin_files="${build_dir}/bin/tqd ${build_dir}/bin/tq ${build_dir}/bin/tqdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove_tq.sh ${script_dir}/set_core.sh" cp ${build_dir}/bin/taos ${install_dir}/bin/tq cp ${build_dir}/bin/taosd ${install_dir}/bin/tqd @@ -99,14 +100,14 @@ chmod a+x ${install_dir}/bin/* || : if [ "$verMode" == "cluster" ]; then sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove_tq.sh >> remove_tq_temp.sh mv remove_tq_temp.sh ${install_dir}/bin/remove_tq.sh - + mkdir -p ${install_dir}/nginxd && cp -r ${nginx_dir}/* ${install_dir}/nginxd cp ${nginx_dir}/png/taos.png ${install_dir}/nginxd/admin/images/taos.png rm -rf ${install_dir}/nginxd/png sed -i "s/TDengine/TQ/g" ${install_dir}/nginxd/admin/*.html sed -i "s/TDengine/TQ/g" ${install_dir}/nginxd/admin/js/*.js - + sed -i '/dataDir/ {s/taos/tq/g}' ${install_dir}/cfg/taos.cfg sed -i '/logDir/ {s/taos/tq/g}' ${install_dir}/cfg/taos.cfg sed -i "s/TDengine/TQ/g" ${install_dir}/cfg/taos.cfg @@ -154,12 +155,11 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then sed -i '/password/ {s/taosdata/tqueue/g}' ${install_dir}/examples/python/read_example.py cp -r ${examples_dir}/R ${install_dir}/examples sed -i '/password/ {s/taosdata/tqueue/g}' ${install_dir}/examples/R/command.txt - cp -r ${examples_dir}/go ${install_dir}/examples + cp -r ${examples_dir}/go ${install_dir}/examples sed -i '/root/ {s/taosdata/tqueue/g}' ${install_dir}/examples/go/taosdemo.go fi # Copy driver -mkdir -p ${install_dir}/driver -cp ${lib_files} ${install_dir}/driver +mkdir -p ${install_dir}/driver && cp ${lib_files} ${install_dir}/driver && echo "${versionComp}" > ${install_dir}/driver/vercomp.txt # Copy connector connector_dir="${code_dir}/connector" @@ -178,11 +178,11 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then echo "WARNING: go connector not found, please check if want to use it!" fi cp -r ${connector_dir}/python ${install_dir}/connector/ - + sed -i '/password/ {s/taosdata/tqueue/g}' ${install_dir}/connector/python/taos/cinterface.py - + sed -i '/password/ {s/taosdata/tqueue/g}' ${install_dir}/connector/python/taos/subscription.py - + sed -i '/self._password/ {s/taosdata/tqueue/g}' ${install_dir}/connector/python/taos/connection.py fi # Copy release note @@ -190,7 +190,7 @@ fi # exit 1 -cd ${release_dir} +cd ${release_dir} if [ "$verMode" == "cluster" ]; then pkg_name=${install_dir}-${osType}-${cpuType} @@ -207,8 +207,8 @@ fi if [ "$verType" == "beta" ]; then pkg_name=${pkg_name}-${verType} -elif [ "$verType" == "stable" ]; then - pkg_name=${pkg_name} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} else echo "unknow verType, nor stabel or beta" exit 1 From 3c61f6ce8f148b74eaabc73892d9c7dd7f995864 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 7 Jul 2021 17:38:47 +0800 Subject: [PATCH 08/27] Feature/sangshuduo/td 5053 taosdump support nanosecond (#6779) * [TD-5053]: taosdump support nanosecond huge code refactoring * fix arm32 compile issue. * [TD-5053]: taosdump supports nanosecond. * fix precision parsing order issue. * [TD-5053]: taosdump support nanosecond. pre-cherry-pick. * turn nanosecond support on on the develop branch. --- src/kit/taosdump/taosdump.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 94a89bcba6..b03d557309 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -27,6 +27,8 @@ #include "tutil.h" #include +#define TSDB_SUPPORT_NANOSECOND 1 + #define MAX_FILE_NAME_LEN 256 // max file name length on linux is 255 #define COMMAND_SIZE 65536 #define MAX_RECORDS_PER_REQ 32766 @@ -228,7 +230,11 @@ static struct argp_option options[] = { {"avro", 'V', 0, 0, "Dump apache avro format data file. By default, dump sql command sequence.", 2}, {"start-time", 'S', "START_TIME", 0, "Start time to dump. Either epoch or ISO8601/RFC3339 format is acceptable. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 4}, {"end-time", 'E', "END_TIME", 0, "End time to dump. Either epoch or ISO8601/RFC3339 format is acceptable. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 5}, +#if TSDB_SUPPORT_NANOSECOND == 1 {"precision", 'C', "PRECISION", 0, "Epoch precision. Valid value is one of ms, us, and ns. Default is ms.", 6}, +#else + {"precision", 'C', "PRECISION", 0, "Epoch precision. Valid value is one of ms and us. Default is ms.", 6}, +#endif {"data-batch", 'B', "DATA_BATCH", 0, "Number of data point per insert statement. Max value is 32766. Default is 1.", 3}, {"max-sql-len", 'L', "SQL_LEN", 0, "Max length of one sql. Default is 65480.", 3}, {"table-batch", 't', "TABLE_BATCH", 0, "Number of table dumpout into one output file. Default is 1.", 3}, @@ -527,7 +533,10 @@ static void parse_precision_first( } if ((0 != strncasecmp(tmp, "ms", strlen("ms"))) && (0 != strncasecmp(tmp, "us", strlen("us"))) - && (0 != strncasecmp(tmp, "ns", strlen("ns")))) { +#if TSDB_SUPPORT_NANOSECOND == 1 + && (0 != strncasecmp(tmp, "ns", strlen("ns"))) +#endif + ) { // errorPrint("input precision: %s is invalid value\n", tmp); free(tmp); @@ -564,9 +573,11 @@ static void parse_timestamp( } else if (0 == strncasecmp(arguments->precision, "us", strlen("us"))) { timePrec = TSDB_TIME_PRECISION_MICRO; +#if TSDB_SUPPORT_NANOSECOND == 1 } else if (0 == strncasecmp(arguments->precision, "ns", strlen("ns"))) { timePrec = TSDB_TIME_PRECISION_NANO; +#endif } else { errorPrint("Invalid time precision: %s", arguments->precision); From 2d78785573edc0890c1fa954073a4573a6b8165a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 7 Jul 2021 18:23:57 +0800 Subject: [PATCH 09/27] Hotfix/sangshuduo/td 5100 gtest centos (#6780) * [TD-5100]: gtest existance detection failed on CentOS. * detect both shared library and static library. --- src/client/tests/CMakeLists.txt | 7 ++++--- src/os/tests/CMakeLists.txt | 7 ++++--- src/query/tests/CMakeLists.txt | 5 +++-- src/util/tests/CMakeLists.txt | 7 ++++--- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/client/tests/CMakeLists.txt b/src/client/tests/CMakeLists.txt index 1a6c45aade..c4cd2f1dba 100644 --- a/src/client/tests/CMakeLists.txt +++ b/src/client/tests/CMakeLists.txt @@ -2,9 +2,10 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) -FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib) +FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib /usr/lib64) +FIND_LIBRARY(LIB_GTEST_SHARED_DIR libgtest.so /usr/lib/ /usr/local/lib /usr/lib64) -IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) +IF (HEADER_GTEST_INCLUDE_DIR AND (LIB_GTEST_STATIC_DIR OR LIB_GTEST_SHARED_DIR)) MESSAGE(STATUS "gTest library found, build unit test") # GoogleTest requires at least C++11 @@ -17,4 +18,4 @@ IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) ADD_EXECUTABLE(cliTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES(cliTest taos tutil common gtest pthread) -ENDIF() \ No newline at end of file +ENDIF() diff --git a/src/os/tests/CMakeLists.txt b/src/os/tests/CMakeLists.txt index b00f0ebdc8..17bb908096 100644 --- a/src/os/tests/CMakeLists.txt +++ b/src/os/tests/CMakeLists.txt @@ -2,9 +2,10 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) -FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib) +FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib /usr/lib64) +FIND_LIBRARY(LIB_GTEST_SHARED_DIR libgtest.so /usr/lib/ /usr/local/lib /usr/lib64) -IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) +IF (HEADER_GTEST_INCLUDE_DIR AND (LIB_GTEST_STATIC_DIR OR LIB_GTEST_SHARED_DIR)) MESSAGE(STATUS "gTest library found, build unit test") # GoogleTest requires at least C++11 @@ -17,4 +18,4 @@ IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) ADD_EXECUTABLE(osTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES(osTest taos os tutil common gtest pthread) -ENDIF() \ No newline at end of file +ENDIF() diff --git a/src/query/tests/CMakeLists.txt b/src/query/tests/CMakeLists.txt index 0cfe2ff165..6169c5666e 100644 --- a/src/query/tests/CMakeLists.txt +++ b/src/query/tests/CMakeLists.txt @@ -2,9 +2,10 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) -FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib) +FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib /usr/lib64) +FIND_LIBRARY(LIB_GTEST_SHARED_DIR libgtest.so /usr/lib/ /usr/local/lib /usr/lib64) -IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) +IF (HEADER_GTEST_INCLUDE_DIR AND (LIB_GTEST_STATIC_DIR OR LIB_GTEST_SHARED_DIR)) MESSAGE(STATUS "gTest library found, build unit test") # GoogleTest requires at least C++11 diff --git a/src/util/tests/CMakeLists.txt b/src/util/tests/CMakeLists.txt index ee99348cd9..69108f6fa6 100644 --- a/src/util/tests/CMakeLists.txt +++ b/src/util/tests/CMakeLists.txt @@ -2,14 +2,15 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) -FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib) +FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib /usr/lib64) +FIND_LIBRARY(LIB_GTEST_SHARED_DIR libgtest.so /usr/lib/ /usr/local/lib /usr/lib64) -IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) +IF (HEADER_GTEST_INCLUDE_DIR AND (LIB_GTEST_STATIC_DIR OR LIB_GTEST_SHARED_DIR)) MESSAGE(STATUS "gTest library found, build unit test") INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR}) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) - + LIST(REMOVE_ITEM SOURCE_LIST ${CMAKE_CURRENT_SOURCE_DIR}/trefTest.c) ADD_EXECUTABLE(utilTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES(utilTest tutil common os gtest pthread gcov) From ef075ea963c13eeede75497c18690eda68cdabe4 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 7 Jul 2021 21:37:42 +0800 Subject: [PATCH 10/27] Hotfix/sangshuduo/td 4987 detected by mem sanitizer for develop (#6792) * [TD-4987]: issues memory sanitizier detected. * fix mask width issue. * fix issue memory sanitizer detected. --- src/kit/taosdemo/taosdemo.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 6513f3e214..9f300102df 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -5122,13 +5122,13 @@ static int32_t generateStbDataTail( } else { lenOfRow = getRowDataFromSample( data, - remainderBufLen < MAX_DATA_SIZE ? remainderBufLen : MAX_DATA_SIZE, + (remainderBufLen < MAX_DATA_SIZE)?remainderBufLen:MAX_DATA_SIZE, startTime + superTblInfo->timeStampStep * k, superTblInfo, pSamplePos); } - if (lenOfRow > remainderBufLen) { + if ((lenOfRow + 1) > remainderBufLen) { break; } From cb802a0c42261c7f53a49f3327220f9b327633a2 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 8 Jul 2021 07:01:49 +0800 Subject: [PATCH 11/27] Hotfix/sangshuduo/m2d 20210707 (#6795) * [TD-4132]: add taosdump-testcase of boundary value testing * [TD-3952]: add taodemo-testcase that query with restful connector * merge master * modify taosdemo-testcase * add testcase of async subscirbe and max sql num * testcase-subscribe of super table * testcase-modify sub * modify subscribe testcase * [TD-4238]: add taodemo-testcase that records of number in request sql less than 32767 * add taosdemo-subscribe-testcase * update taosdemo testcase * temp commit * temp commit * temp commit * temp1 * temp * [TD-4835] add lua lib * [TD-4369]: add testcase of resubscibe * [TD-4826]: new error code 0x0406 (Dnode is exiting) for tsc * [TD-4369]: add testcase of resubscibe and json file * temp * update master to develop's version * Update boundary2.py * [TD-4912]: fix coredump when drop vnode * Hotfix/sangshuduo/td 3801 taosdump coverity scan issue (#6642) * [TD-3801]: taosdump coverity scan issue. * merge from commit: 069169e7c1 and fix coverity scan issue. Co-authored-by: Shuduo Sang * [TD-4932]fix jenkins errors on master branch (#6645) * [TD-4932]fix jenkins errors on master branch * fix * fix * fix * fix * [TD-4132]:modify taosdump-testcase of column length * [TD-4918]: taosdemo subscribe endAfterResume. (#6653) * [TD-4915] fix show-create-database except * Hotfix/sangshuduo/td 4892 taosdemo sub fetch (#6669) * [TD-4892]: taosdemo subscribe fetch result. * fix stbname length. * restrict prefix length. * submit empty * fix minor code. * fix crash if no result file. * decrease few debug print level. * fix offset issue * [TD-4987]: issues memory sanitizier detected. (#6675) * [TD-4826]: no extra sleep if no message in vwqueue * [TD-4593]: fix vnode cannnot close while syncing * Hotfix/sangshuduo/td 4918 taosdemo sub afterconsume (#6685) * [TD-4918]: taosdemo subscribe endAfterResume. * fix mask length. * fix heap use-after-free. * Hotfix/sangshuduo/td 4987 detected by mem sanitizer (#6684) * [TD-4987]: issues memory sanitizier detected. * fix mask width issue. * [TD-4828] handle dnode exit code * Hotfix/sangshuduo/td 3973 use jemalloc for master (#6702) * [TD-3973]: add jemalloc as submodule. * [TD-3973]: add jemalloc as submodule. * [TD-3973]: use jemalloc. build works as following instructions: cmake .. -DJEMALLOC_ENABLED=true make * fix jemalloc at tag 5.2.1 * fix conflicts * make install works. * fix conflicts. * release script works. * fix typo Co-authored-by: Shuduo Sang * TD-4835 * only compile in lua * [td-4748]:fix the temp file generate buf on windows platform. * change cmake options * [TD-5007]: use build directory's taosdump to test. (#6710) btw, solve two taosdemo test cases too. * Hotfix/sangshuduo/td 4838 taosdump binary len bug (#6713) * [TD-4838]: taosdump binary length bug. * taosdump code refactoring. * fix arm32 compile issue. * Hotfix/sangshuduo/td 4918 taosdemo sub afterconsume (#6723) * [TD-4918]: taosdemo subscribe endAfterResume. * fix mask length. * fix heap use-after-free. * fix illegal input value of consume. * Hotfix/sangshuduo/td 3197 taosdemo coverity scan for master (#6725) * [TD-3197]: taosdemo and taosdump coverity scan issues. * exit if read sample file failed. Co-authored-by: Shuduo Sang * [TD-4985]:test query limit and offset function * [TD-4985]:test query limit and offset function * [TD-4985]:test query limit and offset function * [TD-4985]:test query limit and offset function * [TD-5018]: taosdemo prompt if the batch is too large to insert data. (#6736) * fix dead loop issue * fix taosdemoPerformance.py runtime error * Hotfix/sangshuduo/td 3973 use jemalloc for master (#6753) * [TD-3973]: add jemalloc as submodule. * [TD-3973]: add jemalloc as submodule. * [TD-3973]: use jemalloc. build works as following instructions: cmake .. -DJEMALLOC_ENABLED=true make * fix jemalloc at tag 5.2.1 * fix conflicts * make install works. * fix conflicts. * release script works. * fix typo * [TD-3937]: support jemalloc add install funtion to all scripts. Co-authored-by: Shuduo Sang * Hotfix/sangshuduo/td 3973 use jemalloc for master (#6763) * [TD-3973]: add jemalloc as submodule. * [TD-3973]: add jemalloc as submodule. * [TD-3973]: use jemalloc. build works as following instructions: cmake .. -DJEMALLOC_ENABLED=true make * fix jemalloc at tag 5.2.1 * fix conflicts * make install works. * fix conflicts. * release script works. * fix typo * [TD-3937]: support jemalloc add install funtion to all scripts. * adjust install_jemalloc() position for check compatibility. Co-authored-by: Shuduo Sang * Hotfix/sangshuduo/td 3973 use jemalloc for master (#6765) * [TD-3973]: add jemalloc as submodule. * [TD-3973]: add jemalloc as submodule. * [TD-3973]: use jemalloc. build works as following instructions: cmake .. -DJEMALLOC_ENABLED=true make * fix jemalloc at tag 5.2.1 * fix conflicts * make install works. * fix conflicts. * release script works. * fix typo * [TD-3937]: support jemalloc add install funtion to all scripts. * adjust install_jemalloc() position for check compatibility. * fix position bug. Co-authored-by: Shuduo Sang * [TD-5067]: taosdemo stmt use sample data (#6760) * Hotfix/sangshuduo/td 3973 use jemalloc for master (#6768) * [TD-3973]: add jemalloc as submodule. * [TD-3973]: add jemalloc as submodule. * [TD-3973]: use jemalloc. build works as following instructions: cmake .. -DJEMALLOC_ENABLED=true make * fix jemalloc at tag 5.2.1 * fix conflicts * make install works. * fix conflicts. * release script works. * fix typo * [TD-3937]: support jemalloc add install funtion to all scripts. * adjust install_jemalloc() position for check compatibility. * fix position bug. * add ldconfig for jemalloc library cache refresh. Co-authored-by: Shuduo Sang * [TD-5063]:test insert locking bug * [TD-5063]:test insert locking bug * [TD-4705]: disallow file sync starting when vnode closing * Hotfix/sangshuduo/td 5053 taosdump support nanosecond for master (#6778) * cherrypick from develop branch * fix arm32 compile issue. * [TD-5053]: taosdump supports nanosecond. * fix precision parsing order issue. * [TD-5053]: taosdump support nanosecond. pre-cherry-pick. * Hotfix/sangshuduo/td 5100 gtest centos for master (#6781) * cherry pick from develop branch. * detect both shared library and static library. * Hotfix/sangshuduo/td 3973 use jemalloc for master (#6789) * [TD-3973]: add jemalloc as submodule. * [TD-3973]: add jemalloc as submodule. * [TD-3973]: use jemalloc. build works as following instructions: cmake .. -DJEMALLOC_ENABLED=true make * fix jemalloc at tag 5.2.1 * fix conflicts * make install works. * fix conflicts. * release script works. * fix typo * [TD-3937]: support jemalloc add install funtion to all scripts. * adjust install_jemalloc() position for check compatibility. * fix position bug. * add ldconfig for jemalloc library cache refresh. Co-authored-by: Shuduo Sang * Hotfix/sangshuduo/td 3973 use jemalloc for master (#6794) * [TD-3973]: add jemalloc as submodule. * [TD-3973]: add jemalloc as submodule. * [TD-3973]: use jemalloc. build works as following instructions: cmake .. -DJEMALLOC_ENABLED=true make * fix jemalloc at tag 5.2.1 * fix conflicts * make install works. * fix conflicts. * release script works. * fix typo * [TD-3937]: support jemalloc add install funtion to all scripts. * adjust install_jemalloc() position for check compatibility. * fix position bug. * add ldconfig for jemalloc library cache refresh. * cherry pick from develop branch. * cherry pick from develop branch. Co-authored-by: Shuduo Sang * fix taosdumpTest case as -N will lead no property dumped. Co-authored-by: tomchon Co-authored-by: Shengliang Guan Co-authored-by: Minglei Jin Co-authored-by: Baosheng Chang Co-authored-by: Hongze Cheng Co-authored-by: Shuduo Sang Co-authored-by: Yiqing Liu Co-authored-by: Hui Li <52318143+plum-lihui@users.noreply.github.com> Co-authored-by: yihaoDeng Co-authored-by: wpan Co-authored-by: Haojun Liao Co-authored-by: Haojun Liao Co-authored-by: happyguoxy Co-authored-by: Ping Xiao --- src/client/src/tscParseInsert.c | 4 + src/kit/taosdemo/taosdemo.c | 159 +++++++++++---- src/vnode/src/vnodeSync.c | 4 +- tests/pytest/fulltest.sh | 2 + tests/pytest/insert/insert_locking.py | 178 ++++++++++++++++ .../TD-4985/query-limit-offset.csv | 10 + .../TD-4985/query-limit-offset.json | 62 ++++++ .../TD-4985/query-limit-offset.py | 191 ++++++++++++++++++ tests/pytest/tools/taosdemoPerformance.py | 2 + tests/pytest/tools/taosdumpTest.py | 6 +- 10 files changed, 575 insertions(+), 43 deletions(-) create mode 100644 tests/pytest/insert/insert_locking.py create mode 100644 tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.csv create mode 100644 tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.json create mode 100644 tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.py diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 137a7be7c7..cd56153bc4 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -961,6 +961,10 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC break; } + if (sToken.n == 0 || sToken.type == TK_SEMI || index == 0) { + return tscSQLSyntaxErrMsg(pCmd->payload, "unexpected token", sql); + } + sql += index; ++numOfColsAfterTags; } diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 9f300102df..e5945f4eac 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -3216,13 +3216,6 @@ static int readTagFromCsvFileToMem(SSuperTable * superTblInfo) { return 0; } -#if 0 -int readSampleFromJsonFileToMem(SSuperTable * superTblInfo) { - // TODO - return 0; -} -#endif - /* Read 10000 lines at most. If more than 10000 lines, continue to read after using */ @@ -5338,7 +5331,7 @@ static int64_t generateInterlaceDataWithoutStb( #if STMT_IFACE_ENABLED == 1 static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, - char *dataType, int32_t dataLen, char **ptr) + char *dataType, int32_t dataLen, char **ptr, char *value) { if (0 == strncasecmp(dataType, "BINARY", strlen("BINARY"))) { @@ -5348,12 +5341,18 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, return -1; } char *bind_binary = (char *)*ptr; - rand_string(bind_binary, dataLen); bind->buffer_type = TSDB_DATA_TYPE_BINARY; - bind->buffer_length = dataLen; - bind->buffer = bind_binary; + if (value) { + strncpy(bind_binary, value, strlen(value)); + bind->buffer_length = strlen(bind_binary); + } else { + rand_string(bind_binary, dataLen); + bind->buffer_length = dataLen; + } + bind->length = &bind->buffer_length; + bind->buffer = bind_binary; bind->is_null = NULL; *ptr += bind->buffer_length; @@ -5365,9 +5364,14 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, return -1; } char *bind_nchar = (char *)*ptr; - rand_string(bind_nchar, dataLen); bind->buffer_type = TSDB_DATA_TYPE_NCHAR; + if (value) { + strncpy(bind_nchar, value, strlen(value)); + } else { + rand_string(bind_nchar, dataLen); + } + bind->buffer_length = strlen(bind_nchar); bind->buffer = bind_nchar; bind->length = &bind->buffer_length; @@ -5378,7 +5382,11 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "INT", strlen("INT"))) { int32_t *bind_int = (int32_t *)*ptr; - *bind_int = rand_int(); + if (value) { + *bind_int = atoi(value); + } else { + *bind_int = rand_int(); + } bind->buffer_type = TSDB_DATA_TYPE_INT; bind->buffer_length = sizeof(int32_t); bind->buffer = bind_int; @@ -5390,7 +5398,11 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "BIGINT", strlen("BIGINT"))) { int64_t *bind_bigint = (int64_t *)*ptr; - *bind_bigint = rand_bigint(); + if (value) { + *bind_bigint = atoll(value); + } else { + *bind_bigint = rand_bigint(); + } bind->buffer_type = TSDB_DATA_TYPE_BIGINT; bind->buffer_length = sizeof(int64_t); bind->buffer = bind_bigint; @@ -5402,7 +5414,11 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "FLOAT", strlen("FLOAT"))) { float *bind_float = (float *) *ptr; - *bind_float = rand_float(); + if (value) { + *bind_float = (float)atof(value); + } else { + *bind_float = rand_float(); + } bind->buffer_type = TSDB_DATA_TYPE_FLOAT; bind->buffer_length = sizeof(float); bind->buffer = bind_float; @@ -5414,7 +5430,11 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "DOUBLE", strlen("DOUBLE"))) { double *bind_double = (double *)*ptr; - *bind_double = rand_double(); + if (value) { + *bind_double = atof(value); + } else { + *bind_double = rand_double(); + } bind->buffer_type = TSDB_DATA_TYPE_DOUBLE; bind->buffer_length = sizeof(double); bind->buffer = bind_double; @@ -5426,7 +5446,11 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "SMALLINT", strlen("SMALLINT"))) { int16_t *bind_smallint = (int16_t *)*ptr; - *bind_smallint = rand_smallint(); + if (value) { + *bind_smallint = (int16_t)atoi(value); + } else { + *bind_smallint = rand_smallint(); + } bind->buffer_type = TSDB_DATA_TYPE_SMALLINT; bind->buffer_length = sizeof(int16_t); bind->buffer = bind_smallint; @@ -5438,7 +5462,11 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "TINYINT", strlen("TINYINT"))) { int8_t *bind_tinyint = (int8_t *)*ptr; - *bind_tinyint = rand_tinyint(); + if (value) { + *bind_tinyint = (int8_t)atoi(value); + } else { + *bind_tinyint = rand_tinyint(); + } bind->buffer_type = TSDB_DATA_TYPE_TINYINT; bind->buffer_length = sizeof(int8_t); bind->buffer = bind_tinyint; @@ -5461,7 +5489,11 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "TIMESTAMP", strlen("TIMESTAMP"))) { int64_t *bind_ts2 = (int64_t *) *ptr; - *bind_ts2 = rand_bigint(); + if (value) { + *bind_ts2 = atoll(value); + } else { + *bind_ts2 = rand_bigint(); + } bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; bind->buffer_length = sizeof(int64_t); bind->buffer = bind_ts2; @@ -5527,12 +5559,13 @@ static int32_t prepareStmtWithoutStb( ptr += bind->buffer_length; for (int i = 0; i < g_args.num_of_CPR; i ++) { - bind = (TAOS_BIND *)((char *)bindArray + (sizeof(TAOS_BIND) * (i + 1))); + bind = (TAOS_BIND *)((char *)bindArray + + (sizeof(TAOS_BIND) * (i + 1))); if ( -1 == prepareStmtBindArrayByType( bind, data_type[i], g_args.len_of_binary, - &ptr)) { + &ptr, NULL)) { return -1; } } @@ -5551,12 +5584,14 @@ static int32_t prepareStmtWithoutStb( return k; } -static int32_t prepareStbStmt(SSuperTable *stbInfo, +static int32_t prepareStbStmt( + SSuperTable *stbInfo, TAOS_STMT *stmt, char *tableName, uint32_t batch, uint64_t insertRows, uint64_t recordFrom, - int64_t startTime, char *buffer) + int64_t startTime, + int64_t *pSamplePos) { int ret = taos_stmt_set_tbname(stmt, tableName); if (ret != 0) { @@ -5567,16 +5602,24 @@ static int32_t prepareStbStmt(SSuperTable *stbInfo, char *bindArray = malloc(sizeof(TAOS_BIND) * (stbInfo->columnCount + 1)); if (bindArray == NULL) { - errorPrint("Failed to allocate %d bind params\n", - (stbInfo->columnCount + 1)); + errorPrint("%s() LN%d, Failed to allocate %d bind params\n", + __func__, __LINE__, (stbInfo->columnCount + 1)); return -1; } - bool tsRand; + bool sourceRand; if (0 == strncasecmp(stbInfo->dataSource, "rand", strlen("rand"))) { - tsRand = true; + sourceRand = true; } else { - tsRand = false; + sourceRand = false; // from sample data file + } + + char *bindBuffer = malloc(g_args.len_of_binary); + if (bindBuffer == NULL) { + errorPrint("%s() LN%d, Failed to allocate %d bind buffer\n", + __func__, __LINE__, g_args.len_of_binary); + free(bindArray); + return -1; } uint32_t k; @@ -5592,7 +5635,7 @@ static int32_t prepareStbStmt(SSuperTable *stbInfo, bind_ts = (int64_t *)ptr; bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - if (tsRand) { + if (sourceRand) { *bind_ts = startTime + getTSRandTail( stbInfo->timeStampStep, k, stbInfo->disorderRatio, @@ -5607,14 +5650,46 @@ static int32_t prepareStbStmt(SSuperTable *stbInfo, ptr += bind->buffer_length; + int cursor = 0; for (int i = 0; i < stbInfo->columnCount; i ++) { bind = (TAOS_BIND *)((char *)bindArray + (sizeof(TAOS_BIND) * (i + 1))); - if ( -1 == prepareStmtBindArrayByType( - bind, - stbInfo->columns[i].dataType, - stbInfo->columns[i].dataLen, - &ptr)) { - return -1; + + if (sourceRand) { + if ( -1 == prepareStmtBindArrayByType( + bind, + stbInfo->columns[i].dataType, + stbInfo->columns[i].dataLen, + &ptr, + NULL)) { + free(bindArray); + free(bindBuffer); + return -1; + } + } else { + char *restStr = stbInfo->sampleDataBuf + cursor; + int lengthOfRest = strlen(restStr); + + int index = 0; + for (index = 0; index < lengthOfRest; index ++) { + if (restStr[index] == ',') { + break; + } + } + + memset(bindBuffer, 0, g_args.len_of_binary); + strncpy(bindBuffer, restStr, index); + cursor += index + 1; // skip ',' too + + if ( -1 == prepareStmtBindArrayByType( + bind, + stbInfo->columns[i].dataType, + stbInfo->columns[i].dataLen, + &ptr, + bindBuffer)) { + free(bindArray); + free(bindBuffer); + return -1; + } } } taos_stmt_bind_param(stmt, (TAOS_BIND *)bindArray); @@ -5623,11 +5698,16 @@ static int32_t prepareStbStmt(SSuperTable *stbInfo, k++; recordFrom ++; + + if (!sourceRand) { + (*pSamplePos) ++; + } if (recordFrom >= insertRows) { break; } } + free(bindBuffer); free(bindArray); return k; } @@ -5820,13 +5900,14 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { if (superTblInfo) { if (superTblInfo->iface == STMT_IFACE) { #if STMT_IFACE_ENABLED == 1 - generated = prepareStbStmt(superTblInfo, + generated = prepareStbStmt( + superTblInfo, pThreadInfo->stmt, tableName, batchPerTbl, insertRows, i, startTime, - pThreadInfo->buffer); + &(pThreadInfo->samplePos)); #else generated = -1; #endif @@ -6051,7 +6132,8 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { pThreadInfo->stmt, tableName, g_args.num_of_RPR, - insertRows, i, start_time, pstr); + insertRows, i, start_time, + &(pThreadInfo->samplePos)); #else generated = -1; #endif @@ -7332,6 +7414,7 @@ static void *superSubscribe(void *sarg) { TAOS_RES* res = NULL; uint64_t st = 0, et = 0; + while ((g_queryInfo.superQueryInfo.endAfterConsume == -1) || (g_queryInfo.superQueryInfo.endAfterConsume > consumed[pThreadInfo->end_table_to diff --git a/src/vnode/src/vnodeSync.c b/src/vnode/src/vnodeSync.c index 4197428fec..2bdfd2ead3 100644 --- a/src/vnode/src/vnodeSync.c +++ b/src/vnode/src/vnodeSync.c @@ -95,7 +95,7 @@ void vnodeCtrlFlow(int32_t vgId, int32_t level) { } void vnodeStartSyncFile(int32_t vgId) { - SVnodeObj *pVnode = vnodeAcquire(vgId); + SVnodeObj *pVnode = vnodeAcquireNotClose(vgId); if (pVnode == NULL) { vError("vgId:%d, vnode not found while start filesync", vgId); return; @@ -155,7 +155,7 @@ int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rpara } int32_t vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver) { - SVnodeObj *pVnode = vnodeAcquire(vgId); + SVnodeObj *pVnode = vnodeAcquireNotClose(vgId); if (pVnode == NULL) { vError("vgId:%d, vnode not found while write to cache", vgId); return -1; diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 2cbc3747f6..c9a91e1688 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -345,12 +345,14 @@ python3 ./test.py -f tag_lite/unsignedTinyint.py python3 ./test.py -f functions/function_percentile2.py python3 ./test.py -f insert/boundary2.py +python3 ./test.py -f insert/insert_locking.py python3 ./test.py -f alter/alter_debugFlag.py python3 ./test.py -f query/queryBetweenAnd.py python3 ./test.py -f tag_lite/alter_tag.py python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertWithJson.py python3 test.py -f tools/taosdemoAllTest/taosdemoTestQueryWithJson.py +python3 test.py -f tools/taosdemoAllTest/TD-4985/query-limit-offset.py python3 ./test.py -f tag_lite/drop_auto_create.py python3 test.py -f insert/insert_before_use_db.py python3 test.py -f alter/alter_keep.py diff --git a/tests/pytest/insert/insert_locking.py b/tests/pytest/insert/insert_locking.py new file mode 100644 index 0000000000..0d780a7132 --- /dev/null +++ b/tests/pytest/insert/insert_locking.py @@ -0,0 +1,178 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +import random + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + + + def run(self): + tdSql.prepare() + # test case for https://jira.taosdata.com:18080/browse/TD-5021 + + tdLog.info("\n\n----------step1 : drop db and create db----------\n") + tdSql.execute('''drop database if exists db ;''') + tdSql.execute('''create database db ;''') + sql = '''show databases;''' + tdSql.query(sql) + tdSql.checkRows(1) + + tdLog.info("\n\n----------step2 : create stable----------\n") + tdSql.execute('''create stable + db.stable_1 (ts timestamp, payload binary(256)) + tags(t1 binary(16),t2 int);''') + sql = '''show db.stables;''' + tdSql.query(sql) + tdSql.checkRows(1) + + tdLog.info("\n\n----------step3 : create table and insert----------\n") + sql = '''insert into db.table1 using db.stable_1 (t1 , t2) tags ("table_1" , 111) ( values (now, ;''' + tdLog.info(sql) + tdSql.error(sql) + try: + tdSql.execute(sql) + tdLog.exit(" unexpected token") + except Exception as e: + tdLog.info(repr(e)) + tdLog.info("DB error: syntax error near ', ;' (unexpected token)") + + sql = '''insert into db.table1(ts , payload) using db.stable_1 (t1 , t2) tags ("table_1" , 111) ( values (now, ;''' + tdLog.info(sql) + tdSql.error(sql) + try: + tdSql.execute(sql) + tdLog.exit(" bind columns again") + except Exception as e: + tdLog.info(repr(e)) + tdLog.info("DB error: syntax error near ', ;' (bind columns again)") + + sql = '''insert into db.table1 using db.stable_1 (t1 , t2) tags ("table_1",111) (ts , payload) ( values (now, ;''' + tdLog.info(sql) + tdSql.error(sql) + try: + tdSql.execute(sql) + tdLog.exit(" keyword VALUES or FILE required ") + except Exception as e: + tdLog.info(repr(e)) + tdLog.info("DB error: invalid SQL: (keyword VALUES or FILE required)") + + tdSql.execute('''insert into db.table1 using db.stable_1 (t1 , t2) + tags ("table_1" , 111) values ( now , 1) ''') + sql = '''select * from db.stable_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + tdSql.checkData(0,1,1) + tdSql.checkData(0,2,'table_1') + + tdLog.info("\n\n----------step4 : create table and insert again----------\n") + sql = '''insert into db.table2 using db.stable_1 (t1) tags ("table_2") ( values (now, ;''' + tdLog.info(sql) + tdSql.error(sql) + try: + tdSql.execute(sql) + tdLog.exit(" unexpected token") + except Exception as e: + tdLog.info(repr(e)) + tdLog.info("DB error: syntax error near ', ;' (unexpected token)") + + tdSql.execute('''insert into db.table2 using db.stable_1 (t1) + tags ("table_2") values ( now , 2) ''') + sql = '''select * from db.stable_1;''' + tdSql.query(sql) + tdSql.checkRows(2) + tdSql.checkData(1,1,2) + tdSql.checkData(1,2,'table_2') + + tdLog.info("\n\n----------step5 : create table and insert without db----------\n") + tdSql.execute('''use db''') + sql = '''insert into table3 using stable_1 (t1) tags ("table_3") ( values (now, ;''' + tdLog.info(sql) + tdSql.error(sql) + try: + tdSql.execute(sql) + tdLog.exit(" unexpected token") + except Exception as e: + tdLog.info(repr(e)) + tdLog.info("DB error: syntax error near ', ;' (unexpected token)") + + tdSql.execute('''insert into table3 using stable_1 (t1 , t2) + tags ("table_3" , 333) values ( now , 3) ''') + sql = '''select * from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(3) + tdSql.checkData(2,1,3) + tdSql.checkData(2,2,'table_3') + + tdLog.info("\n\n----------step6 : create tables in one sql ----------\n") + sql = '''insert into table4 using stable_1 (t1) tags ("table_4") values (now, 4) + table5 using stable_1 (t1) tags ("table_5") ( values (now, ;''' + tdLog.info(sql) + tdSql.error(sql) + try: + tdSql.execute(sql) + tdLog.exit(" unexpected token") + except Exception as e: + tdLog.info(repr(e)) + tdLog.info("DB error: syntax error near ', ;' (unexpected token)") + + tdSql.execute('''insert into table4 using stable_1 (t1) tags ("table_4") values (now, 4) + table5 using stable_1 (t1) tags ("table_5") values (now, 5) ''') + sql = '''select * from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(5) + tdSql.checkData(3,1,4) + tdSql.checkData(3,2,'table_4') + tdSql.checkData(4,1,5) + tdSql.checkData(4,2,'table_5') + + + sql = '''insert into table6 using stable_1 (t1) tags ("table_6") ( values (now, + table7 using stable_1 (t1) tags ("table_7") values (now, 7);''' + tdLog.info(sql) + tdSql.error(sql) + try: + tdSql.execute(sql) + tdLog.exit(" invalid SQL") + except Exception as e: + tdLog.info(repr(e)) + tdLog.info("invalid SQL") + + tdSql.execute('''insert into table6 using stable_1 (t1 , t2) tags ("table_6" , 666) values (now, 6) + table7 using stable_1 (t1) tags ("table_7") values (now, 7) ''') + sql = '''select * from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(7) + tdSql.checkData(5,1,6) + tdSql.checkData(5,2,'table_6') + tdSql.checkData(6,1,7) + tdSql.checkData(6,2,'table_7') + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.csv b/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.csv new file mode 100644 index 0000000000..d4138798e3 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.csv @@ -0,0 +1,10 @@ +0,0,'TAOSdata-0' +1,1,'TAOSdata-1' +2,22,'TAOSdata-2' +3,333,'TAOSdata-3' +4,4444,'TAOSdata-4' +5,55555,'TAOSdata-5' +6,666666,'TAOSdata-6' +7,7777777,'TAOSdata-7' +8,88888888,'TAOSdata-8' +9,999999999,'TAOSdata-9' \ No newline at end of file diff --git a/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.json b/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.json new file mode 100644 index 0000000000..265f42036b --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.json @@ -0,0 +1,62 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 10, + "thread_count_create_tbl": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 10, + "num_of_records_per_req": 1, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 10000, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 1, + "data_source": "sample", + "insert_mode": "taosc", + "insert_rows": 10, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./tools/taosdemoAllTest/TD-4985/query-limit-offset.csv", + "tags_file": "./tools/taosdemoAllTest/TD-4985/query-limit-offset.csv", + "columns": [{"type": "INT","count":2}, {"type": "BINARY", "len": 16, "count":1}], + "tags": [{"type": "INT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.py b/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.py new file mode 100644 index 0000000000..081057f180 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.py @@ -0,0 +1,191 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + # insert: create one or mutiple tables per sql and insert multiple rows per sql + # test case for https://jira.taosdata.com:18080/browse/TD-4985 + os.system("%staosdemo -f tools/taosdemoAllTest/TD-4985/query-limit-offset.json -y " % binPath) + tdSql.execute("use db") + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 10000) + + for i in range(1000): + tdSql.execute('''insert into stb00_9999 values(%d, %d, %d,'test99.%s')''' + % (1600000000000 + i, i, -10000+i, i)) + tdSql.execute('''insert into stb00_8888 values(%d, %d, %d,'test98.%s')''' + % (1600000000000 + i, i, -10000+i, i)) + tdSql.execute('''insert into stb00_7777 values(%d, %d, %d,'test97.%s')''' + % (1600000000000 + i, i, -10000+i, i)) + tdSql.execute('''insert into stb00_6666 values(%d, %d, %d,'test96.%s')''' + % (1600000000000 + i, i, -10000+i, i)) + tdSql.execute('''insert into stb00_5555 values(%d, %d, %d,'test95.%s')''' + % (1600000000000 + i, i, -10000+i, i)) + tdSql.execute('''insert into stb00_4444 values(%d, %d, %d,'test94.%s')''' + % (1600000000000 + i, i, -10000+i, i)) + tdSql.execute('''insert into stb00_3333 values(%d, %d, %d,'test93.%s')''' + % (1600000000000 + i, i, -10000+i, i)) + tdSql.execute('''insert into stb00_2222 values(%d, %d, %d,'test92.%s')''' + % (1600000000000 + i, i, -10000+i, i)) + tdSql.execute('''insert into stb00_1111 values(%d, %d, %d,'test91.%s')''' + % (1600000000000 + i, i, -10000+i, i)) + tdSql.execute('''insert into stb00_100 values(%d, %d, %d,'test90.%s')''' + % (1600000000000 + i, i, -10000+i, i)) + tdSql.query("select * from stb0 where col2 like 'test99%' ") + tdSql.checkRows(1000) + tdSql.query("select * from stb0 where tbname like 'stb00_9999' limit 10" ) + tdSql.checkData(0, 1, 0) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 1, 2) + tdSql.query("select * from stb0 where tbname like 'stb00_9999' limit 10 offset 5" ) + tdSql.checkData(0, 1, 5) + tdSql.checkData(1, 1, 6) + tdSql.checkData(2, 1, 7) + tdSql.query("select * from stb0 where col2 like 'test98%' ") + tdSql.checkRows(1000) + tdSql.query("select * from stb0 where tbname like 'stb00_8888' limit 10" ) + tdSql.checkData(0, 1, 0) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 1, 2) + tdSql.query("select * from stb0 where tbname like 'stb00_8888' limit 10 offset 5" ) + tdSql.checkData(0, 1, 5) + tdSql.checkData(1, 1, 6) + tdSql.checkData(2, 1, 7) + tdSql.query("select * from stb0 where col2 like 'test97%' ") + tdSql.checkRows(1000) + tdSql.query("select * from stb0 where tbname like 'stb00_7777' limit 10" ) + tdSql.checkData(0, 1, 0) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 1, 2) + tdSql.query("select * from stb0 where tbname like 'stb00_7777' limit 10 offset 5" ) + tdSql.checkData(0, 1, 5) + tdSql.checkData(1, 1, 6) + tdSql.checkData(2, 1, 7) + tdSql.query("select * from stb0 where col2 like 'test96%' ") + tdSql.checkRows(1000) + tdSql.query("select * from stb0 where tbname like 'stb00_6666' limit 10" ) + tdSql.checkData(0, 1, 0) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 1, 2) + tdSql.query("select * from stb0 where tbname like 'stb00_6666' limit 10 offset 5" ) + tdSql.checkData(0, 1, 5) + tdSql.checkData(1, 1, 6) + tdSql.checkData(2, 1, 7) + tdSql.query("select * from stb0 where col2 like 'test95%' ") + tdSql.checkRows(1000) + tdSql.query("select * from stb0 where tbname like 'stb00_5555' limit 10" ) + tdSql.checkData(0, 1, 0) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 1, 2) + tdSql.query("select * from stb0 where tbname like 'stb00_5555' limit 10 offset 5" ) + tdSql.checkData(0, 1, 5) + tdSql.checkData(1, 1, 6) + tdSql.checkData(2, 1, 7) + tdSql.query("select * from stb0 where col2 like 'test94%' ") + tdSql.checkRows(1000) + tdSql.query("select * from stb0 where tbname like 'stb00_4444' limit 10" ) + tdSql.checkData(0, 1, 0) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 1, 2) + tdSql.query("select * from stb0 where tbname like 'stb00_4444' limit 10 offset 5" ) + tdSql.checkData(0, 1, 5) + tdSql.checkData(1, 1, 6) + tdSql.checkData(2, 1, 7) + tdSql.query("select * from stb0 where col2 like 'test93%' ") + tdSql.checkRows(1000) + tdSql.query("select * from stb0 where tbname like 'stb00_3333' limit 100" ) + tdSql.checkData(0, 1, 0) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 1, 2) + tdSql.query("select * from stb0 where tbname like 'stb00_3333' limit 100 offset 5" ) + tdSql.checkData(0, 1, 5) + tdSql.checkData(1, 1, 6) + tdSql.checkData(2, 1, 7) + tdSql.query("select * from stb0 where col2 like 'test92%' ") + tdSql.checkRows(1000) + tdSql.query("select * from stb0 where tbname like 'stb00_2222' limit 100" ) + tdSql.checkData(0, 1, 0) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 1, 2) + tdSql.query("select * from stb0 where tbname like 'stb00_2222' limit 100 offset 5" ) + tdSql.checkData(0, 1, 5) + tdSql.checkData(1, 1, 6) + tdSql.checkData(2, 1, 7) + tdSql.query("select * from stb0 where col2 like 'test91%' ") + tdSql.checkRows(1000) + tdSql.query("select * from stb0 where tbname like 'stb00_1111' limit 100" ) + tdSql.checkData(0, 1, 0) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 1, 2) + tdSql.query("select * from stb0 where tbname like 'stb00_1111' limit 100 offset 5" ) + tdSql.checkData(0, 1, 5) + tdSql.checkData(1, 1, 6) + tdSql.checkData(2, 1, 7) + tdSql.query("select * from stb0 where col2 like 'test90%' ") + tdSql.checkRows(1000) + tdSql.query("select * from stb0 where tbname like 'stb00_100' limit 100" ) + tdSql.checkData(0, 1, 0) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 1, 2) + tdSql.query("select * from stb0 where tbname like 'stb00_100' limit 100 offset 5" ) + tdSql.checkData(0, 1, 5) + tdSql.checkData(1, 1, 6) + tdSql.checkData(2, 1, 7) + + + os.system("rm -rf tools/taosdemoAllTest/TD-4985/query-limit-offset.py.sql") + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py index c4b125969a..90d2c52c15 100644 --- a/tests/pytest/tools/taosdemoPerformance.py +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -16,6 +16,8 @@ import pandas as pd import argparse import os.path import json +from util.log import tdLog +from util.sql import tdSql class taosdemoPerformace: diff --git a/tests/pytest/tools/taosdumpTest.py b/tests/pytest/tools/taosdumpTest.py index beac22a2af..b2c9eb3ec1 100644 --- a/tests/pytest/tools/taosdumpTest.py +++ b/tests/pytest/tools/taosdumpTest.py @@ -63,7 +63,6 @@ class TDTestCase: tdSql.execute("create database db days 11 keep 3649 blocks 8 ") tdSql.execute("create database db1 days 12 keep 3640 blocks 7 ") tdSql.execute("use db") - tdSql.execute( "create table st(ts timestamp, c1 int, c2 nchar(10)) tags(t1 int, t2 binary(10))") tdSql.execute("create table t1 using st tags(1, 'beijing')") @@ -86,9 +85,10 @@ class TDTestCase: tdLog.info("taosdump found in %s" % buildPath) binPath = buildPath + "/build/bin/" - os.system("rm ./taosdumptest/tmp1/*.sql") os.system("%staosdump --databases db -o ./taosdumptest/tmp1" % binPath) - os.system("%staosdump --databases db1 -o ./taosdumptest/tmp2" % binPath) + os.system( + "%staosdump --databases db1 -o ./taosdumptest/tmp2" % + binPath) tdSql.execute("drop database db") tdSql.execute("drop database db1") From 52cef5f97a6b76dc048b36a397ca91358349d7f0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 8 Jul 2021 10:58:46 +0800 Subject: [PATCH 12/27] [td-5126]: optimize the outer query performance when handling the time window query. --- src/query/inc/qExecutor.h | 8 ++--- src/query/src/qExecutor.c | 70 +++++++++++++++++++++++---------------- src/query/src/qUtil.c | 8 ++--- 3 files changed, 50 insertions(+), 36 deletions(-) diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index bc934647ec..83214d293e 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -73,14 +73,14 @@ typedef struct SResultRowPool { typedef struct SResultRow { int32_t pageId; // pageId & rowId is the position of current result in disk-based output buffer - int32_t offset:29; // row index in buffer page + int32_t offset:29; // row index in buffer page bool startInterp; // the time window start timestamp has done the interpolation already. bool endInterp; // the time window end timestamp has done the interpolation already. bool closed; // this result status: closed or opened uint32_t numOfRows; // number of rows of current time window SResultRowCellInfo* pCellInfo; // For each result column, there is a resultInfo - STimeWindow win; - char* key; // start key of current result row + STimeWindow win; + char *key; // start key of current result row } SResultRow; typedef struct SGroupResInfo { @@ -105,7 +105,7 @@ typedef struct SResultRowInfo { int16_t type:8; // data type for hash key int32_t size:24; // number of result set int32_t capacity; // max capacity - int32_t curIndex; // current start active index + SResultRow* current; // current start active index int64_t prevSKey; // previous (not completed) sliding window start key } SResultRowInfo; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index e98c7830f3..7089760e35 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -426,13 +426,8 @@ static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SRes } if (p1 != NULL) { - for(int32_t i = pResultRowInfo->size - 1; i >= 0; --i) { - if (pResultRowInfo->pResult[i] == (*p1)) { - pResultRowInfo->curIndex = i; - existed = true; - break; - } - } + pResultRowInfo->current = (*p1); + existed = true; } } else { if (p1 != NULL) { // group by column query @@ -457,8 +452,8 @@ static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SRes pResult = *p1; } - pResultRowInfo->pResult[pResultRowInfo->size] = pResult; - pResultRowInfo->curIndex = pResultRowInfo->size++; + pResultRowInfo->pResult[pResultRowInfo->size++] = pResult; + pResultRowInfo->current = pResult; } // too many time window in query @@ -466,7 +461,7 @@ static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SRes longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); } - return getResultRow(pResultRowInfo, pResultRowInfo->curIndex); + return pResultRowInfo->current; } static void getInitialStartTimeWindow(SQueryAttr* pQueryAttr, TSKEY ts, STimeWindow* w) { @@ -497,7 +492,7 @@ static void getInitialStartTimeWindow(SQueryAttr* pQueryAttr, TSKEY ts, STimeWin static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, SQueryAttr *pQueryAttr) { STimeWindow w = {0}; - if (pResultRowInfo->curIndex == -1) { // the first window, from the previous stored value + if (pResultRowInfo->current == NULL) { // the first window, from the previous stored value if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { getInitialStartTimeWindow(pQueryAttr, ts, &w); pResultRowInfo->prevSKey = w.skey; @@ -511,8 +506,9 @@ static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t w.ekey = w.skey + pQueryAttr->interval.interval - 1; } } else { - int32_t slot = curTimeWindowIndex(pResultRowInfo); - SResultRow* pWindowRes = getResultRow(pResultRowInfo, slot); +// int32_t slot = curTimeWindowIndex(pResultRowInfo); +// SResultRow* pWindowRes = getResultRow(pResultRowInfo, slot); + SResultRow* pWindowRes = pResultRowInfo->current; w = pWindowRes->win; } @@ -698,7 +694,12 @@ static void doUpdateResultRowIndex(SResultRowInfo*pResultRowInfo, TSKEY lastKey, // all result rows are closed, set the last one to be the skey if (skey == TSKEY_INITIAL_VAL) { - pResultRowInfo->curIndex = pResultRowInfo->size - 1; + if (pResultRowInfo->size == 0) { +// assert(pResultRowInfo->current == NULL); + pResultRowInfo->current = NULL; + } else { + pResultRowInfo->current = pResultRowInfo->pResult[pResultRowInfo->size - 1]; + } } else { for (i = pResultRowInfo->size - 1; i >= 0; --i) { @@ -709,12 +710,12 @@ static void doUpdateResultRowIndex(SResultRowInfo*pResultRowInfo, TSKEY lastKey, } if (i == pResultRowInfo->size - 1) { - pResultRowInfo->curIndex = i; + pResultRowInfo->current = pResultRowInfo->pResult[i]; } else { - pResultRowInfo->curIndex = i + 1; // current not closed result object + pResultRowInfo->current = pResultRowInfo->pResult[i + 1]; // current not closed result object } - pResultRowInfo->prevSKey = pResultRowInfo->pResult[pResultRowInfo->curIndex]->win.skey; + pResultRowInfo->prevSKey = pResultRowInfo->current->win.skey; } } @@ -722,7 +723,7 @@ static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, SQuer bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); if ((lastKey > pQueryAttr->window.ekey && ascQuery) || (lastKey < pQueryAttr->window.ekey && (!ascQuery))) { closeAllResultRows(pResultRowInfo); - pResultRowInfo->curIndex = pResultRowInfo->size - 1; + pResultRowInfo->current = pResultRowInfo->pResult[pResultRowInfo->size - 1]; } else { int32_t step = ascQuery ? 1 : -1; doUpdateResultRowIndex(pResultRowInfo, lastKey - step, ascQuery, pQueryAttr->timeWindowInterpo); @@ -1231,7 +1232,8 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); - int32_t prevIndex = curTimeWindowIndex(pResultRowInfo); + SResultRow* prevRow = pResultRowInfo->current; +// int32_t prevIndex = curTimeWindowIndex(pResultRowInfo); TSKEY* tsCols = NULL; if (pSDataBlock->pDataBlock != NULL) { @@ -1260,9 +1262,16 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); // prev time window not interpolation yet. - int32_t curIndex = curTimeWindowIndex(pResultRowInfo); - if (prevIndex != -1 && prevIndex < curIndex && pQueryAttr->timeWindowInterpo) { - for (int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. +// int32_t curIndex = curTimeWindowIndex(pResultRowInfo); +// if (prevIndex != -1 && prevIndex < curIndex && pQueryAttr->timeWindowInterpo) { +// for (int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. + if (prevRow != NULL && prevRow != pResultRowInfo->current && pQueryAttr->timeWindowInterpo) { + int32_t j = 0; + while(pResultRowInfo->pResult[j] != prevRow) { + j++; + } + + for(; pResultRowInfo->pResult[j] != pResultRowInfo->current; ++j) { SResultRow* pRes = pResultRowInfo->pResult[j]; if (pRes->closed) { assert(resultRowInterpolated(pRes, RESULT_ROW_START_INTERP) && resultRowInterpolated(pRes, RESULT_ROW_END_INTERP)); @@ -3146,7 +3155,7 @@ void copyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, int32_t threshold, SSDataBl } -static void updateTableQueryInfoForReverseScan(SQueryAttr *pQueryAttr, STableQueryInfo *pTableQueryInfo) { +static void updateTableQueryInfoForReverseScan(STableQueryInfo *pTableQueryInfo) { if (pTableQueryInfo == NULL) { return; } @@ -3158,7 +3167,12 @@ static void updateTableQueryInfoForReverseScan(SQueryAttr *pQueryAttr, STableQue pTableQueryInfo->cur.vgroupIndex = -1; // set the index to be the end slot of result rows array - pTableQueryInfo->resInfo.curIndex = pTableQueryInfo->resInfo.size - 1; + SResultRowInfo* pResRowInfo = &pTableQueryInfo->resInfo; + if (pResRowInfo->size > 0) { + pResRowInfo->current = pResRowInfo->pResult[pResRowInfo->size - 1]; + } else { + pResRowInfo->current = NULL; + } } static void setupQueryRangeForReverseScan(SQueryRuntimeEnv* pRuntimeEnv) { @@ -3172,7 +3186,7 @@ static void setupQueryRangeForReverseScan(SQueryRuntimeEnv* pRuntimeEnv) { size_t t = taosArrayGetSize(group); for (int32_t j = 0; j < t; ++j) { STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); - updateTableQueryInfoForReverseScan(pQueryAttr, pCheckInfo); + updateTableQueryInfoForReverseScan(pCheckInfo); // update the last key in tableKeyInfo list, the tableKeyInfo is used to build the tsdbQueryHandle and decide // the start check timestamp of tsdbQueryHandle @@ -4571,7 +4585,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { } if (pResultRowInfo->size > 0) { - pResultRowInfo->curIndex = 0; + pResultRowInfo->current = pResultRowInfo->pResult[0]; pResultRowInfo->prevSKey = pResultRowInfo->pResult[0]->win.skey; } @@ -4597,8 +4611,8 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { pTableScanInfo->order = cond.order; if (pResultRowInfo->size > 0) { - pResultRowInfo->curIndex = pResultRowInfo->size-1; - pResultRowInfo->prevSKey = pResultRowInfo->pResult[pResultRowInfo->size-1]->win.skey; + pResultRowInfo->current = pResultRowInfo->pResult[pResultRowInfo->size - 1]; + pResultRowInfo->prevSKey = pResultRowInfo->current->win.skey; } p = doTableScanImpl(pOperator, newgroup); diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 7b08450d3b..7c5a95312e 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -45,7 +45,7 @@ int32_t initResultRowInfo(SResultRowInfo *pResultRowInfo, int32_t size, int16_t pResultRowInfo->type = type; pResultRowInfo->size = 0; pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL; - pResultRowInfo->curIndex = -1; + pResultRowInfo->current = NULL; pResultRowInfo->capacity = size; pResultRowInfo->pResult = calloc(pResultRowInfo->capacity, POINTER_BYTES); @@ -90,9 +90,9 @@ void resetResultRowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRo SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, &groupIndex, sizeof(groupIndex), uid); taosHashRemove(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(sizeof(groupIndex))); } - - pResultRowInfo->curIndex = -1; - pResultRowInfo->size = 0; + + pResultRowInfo->size = 0; + pResultRowInfo->current = NULL; pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL; } From ad23a6979fe11ea76eadf85f197c51d41d1f38d3 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 8 Jul 2021 11:15:36 +0800 Subject: [PATCH 13/27] [TD-5108]: CI support lua (#6798) * [TD-5108]: CI support lua * [TD-5108]: CI support lua on Windows and Mac --- .appveyor.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index e7802b3d0d..83ef67c352 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -22,6 +22,7 @@ for: - call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %ARCH% before_build: + - choco install lua - cd c:\dev\TDengine - md build @@ -35,6 +36,9 @@ for: - image: macos clone_depth: 1 + before_build: + - brew install lua + build_script: - mkdir debug - cd debug From 82998b12ba0a122925cc590a7e31a4f7d4773e07 Mon Sep 17 00:00:00 2001 From: wpan Date: Thu, 8 Jul 2021 13:26:58 +0800 Subject: [PATCH 14/27] fix bool hash issue --- src/util/src/thashutil.c | 41 +++++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/src/util/src/thashutil.c b/src/util/src/thashutil.c index ffe167977b..4a0208a3d0 100644 --- a/src/util/src/thashutil.c +++ b/src/util/src/thashutil.c @@ -126,15 +126,38 @@ _hash_fn_t taosGetDefaultHashFunction(int32_t type) { _hash_fn_t fn = NULL; switch(type) { case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: fn = taosIntHash_64;break; - case TSDB_DATA_TYPE_BINARY: fn = MurmurHash3_32;break; - case TSDB_DATA_TYPE_NCHAR: fn = MurmurHash3_32;break; - case TSDB_DATA_TYPE_INT: fn = taosIntHash_32; break; - case TSDB_DATA_TYPE_SMALLINT: fn = taosIntHash_16; break; - case TSDB_DATA_TYPE_TINYINT: fn = taosIntHash_8; break; - case TSDB_DATA_TYPE_FLOAT: fn = taosFloatHash; break; - case TSDB_DATA_TYPE_DOUBLE: fn = taosDoubleHash; break; - default: fn = taosIntHash_32;break; + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_BIGINT: + fn = taosIntHash_64; + break; + case TSDB_DATA_TYPE_BINARY: + fn = MurmurHash3_32; + break; + case TSDB_DATA_TYPE_NCHAR: + fn = MurmurHash3_32; + break; + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_INT: + fn = taosIntHash_32; + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + fn = taosIntHash_16; + break; + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_TINYINT: + fn = taosIntHash_8; + break; + case TSDB_DATA_TYPE_FLOAT: + fn = taosFloatHash; + break; + case TSDB_DATA_TYPE_DOUBLE: + fn = taosDoubleHash; + break; + default: + fn = taosIntHash_32; + break; } return fn; From f40ebe0e407f70e4919a37eacf2e1a8fd8b2c84f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 8 Jul 2021 14:05:56 +0800 Subject: [PATCH 15/27] [td-225] fix bug caused by code refactor. --- src/query/src/qExecutor.c | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 7089760e35..76ee5c3860 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -411,6 +411,21 @@ static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, SQueryRuntim pResultRowInfo->capacity = (int32_t)newCapacity; } +static int32_t ascResultRowCompareFn(const void* p1, const void* p2) { + SResultRow* pRow1 = *(SResultRow**)p1; + SResultRow* pRow2 = *(SResultRow**)p2; + + if (pRow1 == pRow2) { + return 0; + } else { + return pRow1->win.skey < pRow2->win.skey? -1:1; + } +} + +static int32_t descResultRowCompareFn(const void* p1, const void* p2) { + return -ascResultRowCompareFn(p1, p2); +} + static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, char *pData, int16_t bytes, bool masterscan, uint64_t uid) { bool existed = false; @@ -427,7 +442,18 @@ static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SRes if (p1 != NULL) { pResultRowInfo->current = (*p1); - existed = true; + + if (pResultRowInfo->size == 0) { + existed = false; + } else if (pResultRowInfo->size == 1) { + existed = (pResultRowInfo->pResult[0] == (*p1)); + } else { + __compar_fn_t fn = QUERY_IS_ASC_QUERY(pRuntimeEnv->pQueryAttr)? ascResultRowCompareFn:descResultRowCompareFn; + void* ptr = taosbsearch(p1, pResultRowInfo->pResult, pResultRowInfo->size, POINTER_BYTES, fn, TD_EQ); + if (ptr != NULL) { + existed = true; + } + } } } else { if (p1 != NULL) { // group by column query From 7fdffa08d2ff37f9682006956e7f9d46feadfc43 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 8 Jul 2021 15:21:20 +0800 Subject: [PATCH 16/27] [TD-5128]: odbc compile failed. (#6804) --- src/connector/odbc/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/connector/odbc/CMakeLists.txt b/src/connector/odbc/CMakeLists.txt index 5a93ac3f7e..dd10fff1b7 100644 --- a/src/connector/odbc/CMakeLists.txt +++ b/src/connector/odbc/CMakeLists.txt @@ -20,8 +20,8 @@ IF (TD_LINUX_64) if (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS 5.0.0) message(WARNING "gcc 4.8.0 will complain too much about flex-generated code, we just bypass building ODBC driver in such case") else () - SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -Wconversion") - SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -Wconversion") + SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ") + SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ") ADD_SUBDIRECTORY(src) ADD_SUBDIRECTORY(tools) ADD_SUBDIRECTORY(examples) From 3d7b4e431efa081a7c729cfb0b81a45ed3c1b908 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 8 Jul 2021 15:32:17 +0800 Subject: [PATCH 17/27] [td-225]fix compiler error. --- src/query/src/qExecutor.c | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 76ee5c3860..c8834a1574 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5366,16 +5366,9 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { SOperatorInfo* upstream = pOperator->upstream[0]; - int64_t ss = taosGetTimestampMs(); - printf("=-=============%ld\n", ss); while(1) { publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - int64_t s = taosGetTimestampMs(); - printf("start: %ld\n", s); SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - - int64_t end = taosGetTimestampMs(); - printf("end: %ld, el:%ld \n", end, end - s); publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -5384,16 +5377,9 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); - - int64_t x1 = taosGetTimestampMs(); hashIntervalAgg(pOperator, &pIntervalInfo->resultRowInfo, pBlock, 0); - int64_t x2 = taosGetTimestampMs(); - printf("-------------------------inter:%ld\n", x2-x1); } - int64_t s1 = taosGetTimestampMs(); - printf("=-=============%ld, el:%ld\n", s1, s1-ss); - // restore the value pQueryAttr->order.order = order; pQueryAttr->window = win; From 1e63299aeb9e6bc77b1caaf9c9eed860cc871b86 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 8 Jul 2021 15:59:26 +0800 Subject: [PATCH 18/27] Revert "[TD-5108]: CI support lua (#6798)" (#6807) This reverts commit ad23a6979fe11ea76eadf85f197c51d41d1f38d3. --- .appveyor.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 83ef67c352..e7802b3d0d 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -22,7 +22,6 @@ for: - call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %ARCH% before_build: - - choco install lua - cd c:\dev\TDengine - md build @@ -36,9 +35,6 @@ for: - image: macos clone_depth: 1 - before_build: - - brew install lua - build_script: - mkdir debug - cd debug From 00a02452b2fa3e48d22c87810ffd61887b409538 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 8 Jul 2021 21:57:17 +0800 Subject: [PATCH 19/27] Feature/sangshuduo/td 5108 lua support on win and mac (#6812) * [TD-5108]: support lua on Windows and Mac * fix compile warning on mac --- deps/CMakeLists.txt | 4 +--- deps/lua/CMakeLists.txt | 3 +++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index 62c3b643c0..8a43b4e56e 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -11,9 +11,7 @@ ADD_SUBDIRECTORY(wepoll) ADD_SUBDIRECTORY(MsvcLibX) ADD_SUBDIRECTORY(rmonotonic) -IF (TD_LINUX_64) - ADD_SUBDIRECTORY(lua) -ENDIF () +ADD_SUBDIRECTORY(lua) IF (TD_LINUX AND TD_MQTT) ADD_SUBDIRECTORY(MQTT-C) diff --git a/deps/lua/CMakeLists.txt b/deps/lua/CMakeLists.txt index 8981350c3b..3e34d44775 100644 --- a/deps/lua/CMakeLists.txt +++ b/deps/lua/CMakeLists.txt @@ -2,3 +2,6 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SOURCE_LIST) ADD_LIBRARY(lua ${SOURCE_LIST}) TARGET_INCLUDE_DIRECTORIES(lua PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/inc) +SET_SOURCE_FILES_PROPERTIES(./src/lgc.c PROPERTIES COMPILE_FLAGS -w) +SET_SOURCE_FILES_PROPERTIES(./src/ltable.c PROPERTIES COMPILE_FLAGS -w) +SET_SOURCE_FILES_PROPERTIES(./src/loslib.c PROPERTIES COMPILE_FLAGS -w) From 8d1e7a804f4951df26640f7c7dff44397f5d2f4b Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 8 Jul 2021 21:58:30 +0800 Subject: [PATCH 20/27] Hotfix/sangshuduo/td 5128 odbc compile failure (#6811) * [TD-5128]: odbc compile failed. * cherry pick from master branch. * fix visual studio 2012 build issue. --- CMakeLists.txt | 7 ++++++- cmake/define.inc | 2 +- cmake/env.inc | 2 +- cmake/input.inc | 2 +- cmake/platform.inc | 2 +- cmake/version.inc | 2 +- deps/CMakeLists.txt | 7 ++++++- deps/MQTT-C/CMakeLists.txt | 2 +- deps/MsvcLibX/CMakeLists.txt | 7 ++++++- deps/iconv/CMakeLists.txt | 2 +- deps/pthread/CMakeLists.txt | 2 +- deps/regex/CMakeLists.txt | 2 +- deps/wepoll/CMakeLists.txt | 2 +- deps/zlib-1.2.11/CMakeLists.txt | 2 +- src/CMakeLists.txt | 2 +- src/balance/CMakeLists.txt | 2 +- src/client/CMakeLists.txt | 2 +- src/client/tests/CMakeLists.txt | 2 +- src/common/CMakeLists.txt | 2 +- src/connector/jdbc/CMakeLists.txt | 2 +- src/connector/odbc/CMakeLists.txt | 2 +- src/connector/odbc/src/CMakeLists.txt | 2 +- src/connector/odbc/src/base/CMakeLists.txt | 2 +- src/cq/CMakeLists.txt | 2 +- src/cq/test/CMakeLists.txt | 2 +- src/dnode/CMakeLists.txt | 2 +- src/kit/CMakeLists.txt | 2 +- src/kit/shell/CMakeLists.txt | 2 +- src/kit/taosdemo/CMakeLists.txt | 2 +- src/kit/taosdump/CMakeLists.txt | 2 +- src/mnode/CMakeLists.txt | 2 +- src/os/CMakeLists.txt | 2 +- src/os/src/darwin/CMakeLists.txt | 2 +- src/os/src/detail/CMakeLists.txt | 2 +- src/os/src/linux/CMakeLists.txt | 2 +- src/os/src/windows/CMakeLists.txt | 2 +- src/os/tests/CMakeLists.txt | 2 +- src/plugins/CMakeLists.txt | 2 +- src/plugins/http/CMakeLists.txt | 2 +- src/plugins/monitor/CMakeLists.txt | 2 +- src/plugins/mqtt/CMakeLists.txt | 2 +- src/query/CMakeLists.txt | 2 +- src/query/tests/CMakeLists.txt | 2 +- src/rpc/CMakeLists.txt | 2 +- src/rpc/test/CMakeLists.txt | 2 +- src/sync/CMakeLists.txt | 2 +- src/sync/test/CMakeLists.txt | 2 +- src/tfs/CMakeLists.txt | 2 +- src/tsdb/CMakeLists.txt | 2 +- src/util/CMakeLists.txt | 2 +- src/util/tests/CMakeLists.txt | 2 +- src/vnode/CMakeLists.txt | 2 +- src/wal/CMakeLists.txt | 2 +- src/wal/test/CMakeLists.txt | 2 +- tests/CMakeLists.txt | 2 +- tests/comparisonTest/tdengine/CMakeLists.txt | 2 +- tests/test/c/CMakeLists.txt | 2 +- 57 files changed, 72 insertions(+), 57 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fe2cc6b42e..093731f190 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,3 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) IF (CMAKE_VERSION VERSION_LESS 3.0) PROJECT(TDengine CXX) SET(PROJECT_VERSION_MAJOR "${LIB_MAJOR_VERSION}") @@ -10,6 +9,12 @@ ELSE () PROJECT(TDengine VERSION "${LIB_VERSION_STRING}" LANGUAGES CXX) ENDIF () +IF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) +ELSE () + CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +ENDIF () + SET(TD_ACCOUNT FALSE) SET(TD_ADMIN FALSE) SET(TD_GRANT FALSE) diff --git a/cmake/define.inc b/cmake/define.inc index c4f9396cf1..96a406cfb4 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) IF (TD_ACCOUNT) diff --git a/cmake/env.inc b/cmake/env.inc index fa15ec6aee..2ceaecc2d9 100755 --- a/cmake/env.inc +++ b/cmake/env.inc @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) SET(CMAKE_C_STANDARD 11) diff --git a/cmake/input.inc b/cmake/input.inc index 04160b4775..06a23ecb14 100755 --- a/cmake/input.inc +++ b/cmake/input.inc @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) IF (${ACCOUNT} MATCHES "true") diff --git a/cmake/platform.inc b/cmake/platform.inc index bdc54c2384..a78082a1fc 100755 --- a/cmake/platform.inc +++ b/cmake/platform.inc @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) # diff --git a/cmake/version.inc b/cmake/version.inc index 93bad45fe8..eb0237c301 100755 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) IF (DEFINED VERNUMBER) diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index 8a43b4e56e..eb22459d34 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -1,6 +1,11 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) +IF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) +ELSE () + CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +ENDIF () + ADD_SUBDIRECTORY(zlib-1.2.11) ADD_SUBDIRECTORY(pthread) ADD_SUBDIRECTORY(regex) diff --git a/deps/MQTT-C/CMakeLists.txt b/deps/MQTT-C/CMakeLists.txt index 15b3552521..37959140e7 100644 --- a/deps/MQTT-C/CMakeLists.txt +++ b/deps/MQTT-C/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) # MQTT-C build options option(MQTT_C_OpenSSL_SUPPORT "Build MQTT-C with OpenSSL support?" OFF) diff --git a/deps/MsvcLibX/CMakeLists.txt b/deps/MsvcLibX/CMakeLists.txt index 4428579e1c..4197f502b1 100644 --- a/deps/MsvcLibX/CMakeLists.txt +++ b/deps/MsvcLibX/CMakeLists.txt @@ -1,6 +1,11 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) +IF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) +ELSE () + CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +ENDIF () + IF (TD_WINDOWS) INCLUDE_DIRECTORIES(include) AUX_SOURCE_DIRECTORY(src SRC) diff --git a/deps/iconv/CMakeLists.txt b/deps/iconv/CMakeLists.txt index 286070fa90..ab5fa1a5d1 100644 --- a/deps/iconv/CMakeLists.txt +++ b/deps/iconv/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) IF (TD_WINDOWS) diff --git a/deps/pthread/CMakeLists.txt b/deps/pthread/CMakeLists.txt index 04e5be7472..16d03f3590 100644 --- a/deps/pthread/CMakeLists.txt +++ b/deps/pthread/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) IF (TD_WINDOWS) diff --git a/deps/regex/CMakeLists.txt b/deps/regex/CMakeLists.txt index 054b093d07..05d01f02ef 100644 --- a/deps/regex/CMakeLists.txt +++ b/deps/regex/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) IF (TD_WINDOWS) diff --git a/deps/wepoll/CMakeLists.txt b/deps/wepoll/CMakeLists.txt index a81fd782bb..e9b7749d82 100644 --- a/deps/wepoll/CMakeLists.txt +++ b/deps/wepoll/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) IF (TD_WINDOWS) diff --git a/deps/zlib-1.2.11/CMakeLists.txt b/deps/zlib-1.2.11/CMakeLists.txt index f83aa70085..1220cc4246 100644 --- a/deps/zlib-1.2.11/CMakeLists.txt +++ b/deps/zlib-1.2.11/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) IF (TD_WINDOWS) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8cc5cee3b5..f480996cef 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) # Base compile diff --git a/src/balance/CMakeLists.txt b/src/balance/CMakeLists.txt index 967635e52c..bffa415deb 100644 --- a/src/balance/CMakeLists.txt +++ b/src/balance/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/mnode/inc) diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 6ff36c33c4..bdd42cb457 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) diff --git a/src/client/tests/CMakeLists.txt b/src/client/tests/CMakeLists.txt index c4cd2f1dba..24bfb44ac9 100644 --- a/src/client/tests/CMakeLists.txt +++ b/src/client/tests/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 0da7bda994..4dce63e54f 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) diff --git a/src/connector/jdbc/CMakeLists.txt b/src/connector/jdbc/CMakeLists.txt index 9642b0490f..81af0ec144 100644 --- a/src/connector/jdbc/CMakeLists.txt +++ b/src/connector/jdbc/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) diff --git a/src/connector/odbc/CMakeLists.txt b/src/connector/odbc/CMakeLists.txt index dd10fff1b7..87746f23ae 100644 --- a/src/connector/odbc/CMakeLists.txt +++ b/src/connector/odbc/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) IF (TD_LINUX_64) diff --git a/src/connector/odbc/src/CMakeLists.txt b/src/connector/odbc/src/CMakeLists.txt index f0e50415e2..e990647e1a 100644 --- a/src/connector/odbc/src/CMakeLists.txt +++ b/src/connector/odbc/src/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) add_subdirectory(base) diff --git a/src/connector/odbc/src/base/CMakeLists.txt b/src/connector/odbc/src/base/CMakeLists.txt index fa13f3e077..e340913609 100644 --- a/src/connector/odbc/src/base/CMakeLists.txt +++ b/src/connector/odbc/src/base/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) aux_source_directory(. SRC) diff --git a/src/cq/CMakeLists.txt b/src/cq/CMakeLists.txt index e9ed2996c7..f01ccb8728 100644 --- a/src/cq/CMakeLists.txt +++ b/src/cq/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) diff --git a/src/cq/test/CMakeLists.txt b/src/cq/test/CMakeLists.txt index cd124567af..d713dd7401 100644 --- a/src/cq/test/CMakeLists.txt +++ b/src/cq/test/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) LIST(APPEND CQTEST_SRC ./cqtest.c) diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt index f8d8f88438..e7ac1be5b1 100644 --- a/src/dnode/CMakeLists.txt +++ b/src/dnode/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) diff --git a/src/kit/CMakeLists.txt b/src/kit/CMakeLists.txt index 66e8cf7398..7053052007 100644 --- a/src/kit/CMakeLists.txt +++ b/src/kit/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) ADD_SUBDIRECTORY(shell) diff --git a/src/kit/shell/CMakeLists.txt b/src/kit/shell/CMakeLists.txt index d904945435..794ca5e2de 100644 --- a/src/kit/shell/CMakeLists.txt +++ b/src/kit/shell/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) diff --git a/src/kit/taosdemo/CMakeLists.txt b/src/kit/taosdemo/CMakeLists.txt index 091eecfe27..584de34094 100644 --- a/src/kit/taosdemo/CMakeLists.txt +++ b/src/kit/taosdemo/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) diff --git a/src/kit/taosdump/CMakeLists.txt b/src/kit/taosdump/CMakeLists.txt index 58897b89e9..51f4748eab 100644 --- a/src/kit/taosdump/CMakeLists.txt +++ b/src/kit/taosdump/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) diff --git a/src/mnode/CMakeLists.txt b/src/mnode/CMakeLists.txt index 2df4708c23..a7fc54d877 100644 --- a/src/mnode/CMakeLists.txt +++ b/src/mnode/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) diff --git a/src/os/CMakeLists.txt b/src/os/CMakeLists.txt index 4472c683c7..a64c9d79dd 100644 --- a/src/os/CMakeLists.txt +++ b/src/os/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) IF (TD_LINUX) diff --git a/src/os/src/darwin/CMakeLists.txt b/src/os/src/darwin/CMakeLists.txt index 259e1a7a0b..ed75cac03d 100644 --- a/src/os/src/darwin/CMakeLists.txt +++ b/src/os/src/darwin/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) AUX_SOURCE_DIRECTORY(. SRC) diff --git a/src/os/src/detail/CMakeLists.txt b/src/os/src/detail/CMakeLists.txt index 5c49df24c1..ac68cf4cd8 100644 --- a/src/os/src/detail/CMakeLists.txt +++ b/src/os/src/detail/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) INCLUDE_DIRECTORIES(.) diff --git a/src/os/src/linux/CMakeLists.txt b/src/os/src/linux/CMakeLists.txt index 08b696ba1a..d1d95b0096 100644 --- a/src/os/src/linux/CMakeLists.txt +++ b/src/os/src/linux/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) AUX_SOURCE_DIRECTORY(. SRC) diff --git a/src/os/src/windows/CMakeLists.txt b/src/os/src/windows/CMakeLists.txt index e5472e1abd..83012d6e3e 100644 --- a/src/os/src/windows/CMakeLists.txt +++ b/src/os/src/windows/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) AUX_SOURCE_DIRECTORY(. SRC) diff --git a/src/os/tests/CMakeLists.txt b/src/os/tests/CMakeLists.txt index 17bb908096..3c47764189 100644 --- a/src/os/tests/CMakeLists.txt +++ b/src/os/tests/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) diff --git a/src/plugins/CMakeLists.txt b/src/plugins/CMakeLists.txt index 7dcaaf27e6..320445f7f7 100644 --- a/src/plugins/CMakeLists.txt +++ b/src/plugins/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) ADD_SUBDIRECTORY(monitor) diff --git a/src/plugins/http/CMakeLists.txt b/src/plugins/http/CMakeLists.txt index bfb47ad12e..57fc2ee3a2 100644 --- a/src/plugins/http/CMakeLists.txt +++ b/src/plugins/http/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc) diff --git a/src/plugins/monitor/CMakeLists.txt b/src/plugins/monitor/CMakeLists.txt index 28c62a099c..8a05d63e14 100644 --- a/src/plugins/monitor/CMakeLists.txt +++ b/src/plugins/monitor/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) diff --git a/src/plugins/mqtt/CMakeLists.txt b/src/plugins/mqtt/CMakeLists.txt index 50b0bbe8af..0815121385 100644 --- a/src/plugins/mqtt/CMakeLists.txt +++ b/src/plugins/mqtt/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) diff --git a/src/query/CMakeLists.txt b/src/query/CMakeLists.txt index f23ac7dd86..d2411d47b5 100644 --- a/src/query/CMakeLists.txt +++ b/src/query/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc) diff --git a/src/query/tests/CMakeLists.txt b/src/query/tests/CMakeLists.txt index 6169c5666e..cc4b607bb4 100644 --- a/src/query/tests/CMakeLists.txt +++ b/src/query/tests/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index f94b4aeb6d..14b77356ba 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) diff --git a/src/rpc/test/CMakeLists.txt b/src/rpc/test/CMakeLists.txt index c10cea6c9d..a32ac9943d 100644 --- a/src/rpc/test/CMakeLists.txt +++ b/src/rpc/test/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/rpc/inc) diff --git a/src/sync/CMakeLists.txt b/src/sync/CMakeLists.txt index 521f51ceb7..2cd84c7c3f 100644 --- a/src/sync/CMakeLists.txt +++ b/src/sync/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) diff --git a/src/sync/test/CMakeLists.txt b/src/sync/test/CMakeLists.txt index f2b05ab226..a5ab819137 100644 --- a/src/sync/test/CMakeLists.txt +++ b/src/sync/test/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) IF (TD_LINUX) diff --git a/src/tfs/CMakeLists.txt b/src/tfs/CMakeLists.txt index b435c84366..7f956f07a2 100644 --- a/src/tfs/CMakeLists.txt +++ b/src/tfs/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) diff --git a/src/tsdb/CMakeLists.txt b/src/tsdb/CMakeLists.txt index 8080a61a6c..c5b77df5a2 100644 --- a/src/tsdb/CMakeLists.txt +++ b/src/tsdb/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index e8a1d61ee5..85b15c0a4f 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/rpc/inc) diff --git a/src/util/tests/CMakeLists.txt b/src/util/tests/CMakeLists.txt index 69108f6fa6..a60c6cff28 100644 --- a/src/util/tests/CMakeLists.txt +++ b/src/util/tests/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) diff --git a/src/vnode/CMakeLists.txt b/src/vnode/CMakeLists.txt index 3fefbea05b..6238f43d32 100644 --- a/src/vnode/CMakeLists.txt +++ b/src/vnode/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc) diff --git a/src/wal/CMakeLists.txt b/src/wal/CMakeLists.txt index a89024dab5..0d9be42bd5 100644 --- a/src/wal/CMakeLists.txt +++ b/src/wal/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) ADD_DEFINITIONS(-DWAL_CHECKSUM_WHOLE) diff --git a/src/wal/test/CMakeLists.txt b/src/wal/test/CMakeLists.txt index 071ff6fdba..c5bc4198f1 100644 --- a/src/wal/test/CMakeLists.txt +++ b/src/wal/test/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) IF (TD_LINUX) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4e7e9a87ea..e21905af3b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -3,7 +3,7 @@ # generate release version: # mkdir release; cd release; cmake -DCMAKE_BUILD_TYPE=Release .. -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) SET(CMAKE_C_STANDARD 11) diff --git a/tests/comparisonTest/tdengine/CMakeLists.txt b/tests/comparisonTest/tdengine/CMakeLists.txt index 36ed3efe19..0f389c4c0c 100644 --- a/tests/comparisonTest/tdengine/CMakeLists.txt +++ b/tests/comparisonTest/tdengine/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) IF (TD_LINUX) diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index 2eb8ee1614..2702d192d3 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) From 1d61e72be3fb54c450cbefef2cb659f96dbbcad8 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Fri, 9 Jul 2021 11:35:05 +0800 Subject: [PATCH 21/27] [TD-4808] : update Windows specific commands in building & installation. --- README-CN.md | 9 +++++---- README.md | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/README-CN.md b/README-CN.md index d5586c78b7..a9bc814e8d 100644 --- a/README-CN.md +++ b/README-CN.md @@ -185,9 +185,10 @@ cmake .. && cmake --build . # 安装 -如果你不想安装，可以直接在shell中运行。生成完成后，安装 TDengine： +生成完成后，安装 TDengine（下文给出的指令以 Linux 为例，如果是在 Windows 下，那么对应的指令会是 `nmake install`）： + ```bash -make install +sudo make install ``` 用户可以在[文件目录结构](https://www.taosdata.com/cn/documentation/administrator#directories)中了解更多在操作系统中生成的目录或文件。 @@ -195,7 +196,7 @@ make install 安装成功后，在终端中启动 TDengine 服务： ```bash -taosd +sudo systemctl start taosd ``` 用户可以使用 TDengine Shell 来连接 TDengine 服务，在终端中，输入： @@ -208,7 +209,7 @@ taos ## 快速运行 -TDengine 生成后，在终端执行以下命令： +如果不希望以服务方式运行 TDengine，也可以在终端中直接运行它。也即在生成完成后，执行以下命令（在 Windows 下，生成的可执行文件会带有 .exe 后缀，例如会名为 taosd.exe ）： ```bash ./build/bin/taosd -c test/cfg diff --git a/README.md b/README.md index 89e35f6e63..2dea05f09d 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,7 @@ cmake .. && cmake --build . # Installing -After building successfully, TDengine can be installed by: +After building successfully, TDengine can be installed by: (On Windows platform, the following command should be `nmake install`) ```bash sudo make install ``` @@ -197,7 +197,7 @@ If TDengine shell connects the server successfully, welcome messages and version ## Quick Run -If you don't want to run TDengine as a service, you can run it in current shell. For example, to quickly start a TDengine server after building, run the command below in terminal: +If you don't want to run TDengine as a service, you can run it in current shell. For example, to quickly start a TDengine server after building, run the command below in terminal: (We take Linux as an example, command on Windows will be `taosd.exe`) ```bash ./build/bin/taosd -c test/cfg ``` From 03455e6fe973ace409de47109b94774cfd619569 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 9 Jul 2021 12:19:59 +0800 Subject: [PATCH 22/27] [td-5126]:improve the nest query performance. --- src/query/inc/qExecutor.h | 1 + src/query/src/qExecutor.c | 112 ++++++++++------- tests/script/general/parser/subInfrom.sim | 147 ---------------------- 3 files changed, 65 insertions(+), 195 deletions(-) delete mode 100644 tests/script/general/parser/subInfrom.sim diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 83214d293e..1bde994058 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -276,6 +276,7 @@ typedef struct SQueryRuntimeEnv { bool enableGroupData; SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file SHashObj* pResultRowHashTable; // quick locate the window object for each result + SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not char* keyBuf; // window key buffer SResultRowPool* pool; // window result object pool char** prevRow; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index c8834a1574..6b9eaecf53 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -411,25 +411,25 @@ static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, SQueryRuntim pResultRowInfo->capacity = (int32_t)newCapacity; } -static int32_t ascResultRowCompareFn(const void* p1, const void* p2) { - SResultRow* pRow1 = *(SResultRow**)p1; - SResultRow* pRow2 = *(SResultRow**)p2; +//static int32_t ascResultRowCompareFn(const void* p1, const void* p2) { +// SResultRow* pRow1 = *(SResultRow**)p1; +// SResultRow* pRow2 = *(SResultRow**)p2; +// +// if (pRow1 == pRow2) { +// return 0; +// } else { +// return pRow1->win.skey < pRow2->win.skey? -1:1; +// } +//} - if (pRow1 == pRow2) { - return 0; - } else { - return pRow1->win.skey < pRow2->win.skey? -1:1; - } -} +//static int32_t descResultRowCompareFn(const void* p1, const void* p2) { +// return -ascResultRowCompareFn(p1, p2); +//} -static int32_t descResultRowCompareFn(const void* p1, const void* p2) { - return -ascResultRowCompareFn(p1, p2); -} - -static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, char *pData, - int16_t bytes, bool masterscan, uint64_t uid) { +static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, int64_t tid, char *pData, + int16_t bytes, bool masterscan, uint64_t tableGroupId) { bool existed = false; - SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid); + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tableGroupId); SResultRow **p1 = (SResultRow **)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); @@ -447,16 +447,20 @@ static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SRes existed = false; } else if (pResultRowInfo->size == 1) { existed = (pResultRowInfo->pResult[0] == (*p1)); - } else { - __compar_fn_t fn = QUERY_IS_ASC_QUERY(pRuntimeEnv->pQueryAttr)? ascResultRowCompareFn:descResultRowCompareFn; - void* ptr = taosbsearch(p1, pResultRowInfo->pResult, pResultRowInfo->size, POINTER_BYTES, fn, TD_EQ); - if (ptr != NULL) { - existed = true; - } + } else { // check if current pResultRowInfo contains the existed pResultRow + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid); + void* ptr = taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + existed = (ptr != NULL); +// __compar_fn_t fn = QUERY_IS_ASC_QUERY(pRuntimeEnv->pQueryAttr)? ascResultRowCompareFn:descResultRowCompareFn; +// void* ptr = taosbsearch(p1, pResultRowInfo->pResult, pResultRowInfo->size, POINTER_BYTES, fn, TD_EQ); +// if (ptr != NULL) { +// existed = true; +// } } } } else { - if (p1 != NULL) { // group by column query + // In case of group by column query, the required SResultRow object must be existed in the pResultRowInfo object. + if (p1 != NULL) { return *p1; } } @@ -480,6 +484,10 @@ static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SRes pResultRowInfo->pResult[pResultRowInfo->size++] = pResult; pResultRowInfo->current = pResult; + + int64_t dummyVal = 0; + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid); + taosHashPut(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &dummyVal, POINTER_BYTES); } // too many time window in query @@ -615,13 +623,13 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf return 0; } -static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, STimeWindow *win, - bool masterscan, SResultRow **pResult, int64_t groupId, SQLFunctionCtx* pCtx, +static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, int64_t tid, STimeWindow *win, + bool masterscan, SResultRow **pResult, int64_t tableGroupId, SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { assert(win->skey <= win->ekey); SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, groupId); + SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pResultRowInfo, tid, (char *)&win->skey, TSDB_KEYSIZE, masterscan, tableGroupId); if (pResultRow == NULL) { *pResult = NULL; return TSDB_CODE_SUCCESS; @@ -629,7 +637,7 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRow // not assign result buffer yet, add new result buffer if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, (int32_t) groupId, pRuntimeEnv->pQueryAttr->intermediateResultRowSize); + int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, (int32_t) tableGroupId, pRuntimeEnv->pQueryAttr->intermediateResultRowSize); if (ret != TSDB_CODE_SUCCESS) { return -1; } @@ -1248,7 +1256,7 @@ static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBloc } } -static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, int32_t groupId) { +static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, int32_t tableGroupId) { STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*) pOperatorInfo->info; SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; @@ -1276,7 +1284,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); SResultRow* pResult = NULL; - int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pResultRowInfo, &win, masterScan, &pResult, groupId, pInfo->pCtx, + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.tid, &win, masterScan, &pResult, tableGroupId, pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1305,7 +1313,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul } STimeWindow w = pRes->win; - ret = setWindowOutputBufByKey(pRuntimeEnv, pResultRowInfo, &w, masterScan, &pResult, groupId, pInfo->pCtx, + ret = setWindowOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.tid, &w, masterScan, &pResult, tableGroupId, pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); if (ret != TSDB_CODE_SUCCESS) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1323,7 +1331,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul } // restore current time window - ret = setWindowOutputBufByKey(pRuntimeEnv, pResultRowInfo, &win, masterScan, &pResult, groupId, pInfo->pCtx, + ret = setWindowOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.tid, &win, masterScan, &pResult, tableGroupId, pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); if (ret != TSDB_CODE_SUCCESS) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1343,7 +1351,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul } // null data, failed to allocate more memory buffer - int32_t code = setWindowOutputBufByKey(pRuntimeEnv, pResultRowInfo, &nextWin, masterScan, &pResult, groupId, + int32_t code = setWindowOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.tid, &nextWin, masterScan, &pResult, tableGroupId, pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1483,7 +1491,7 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSWindowOperatorInf SResultRow* pResult = NULL; pInfo->curWindow.ekey = pInfo->curWindow.skey; - int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, &pInfo->curWindow, masterScan, + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, pSDataBlock->info.tid, &pInfo->curWindow, masterScan, &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput, pBInfo->rowCellInfoOffset); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code @@ -1504,7 +1512,7 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSWindowOperatorInf SResultRow* pResult = NULL; pInfo->curWindow.ekey = pInfo->curWindow.skey; - int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, &pInfo->curWindow, masterScan, + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, pSDataBlock->info.tid, &pInfo->curWindow, masterScan, &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput, pBInfo->rowCellInfoOffset); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code @@ -1547,7 +1555,8 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasic len = varDataLen(pData); } - SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pResultRowInfo, d, len, true, groupIndex); + int64_t tid = 0; + SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pResultRowInfo, tid, d, len, true, groupIndex); assert (pResultRow != NULL); setResultRowKey(pResultRow, pData, type); @@ -1812,6 +1821,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf pRuntimeEnv->pQueryAttr = pQueryAttr; pRuntimeEnv->pResultRowHashTable = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + pRuntimeEnv->pResultRowListSet = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); pRuntimeEnv->keyBuf = malloc(pQueryAttr->maxTableColumnWidth + sizeof(int64_t)); pRuntimeEnv->pool = initResultRowPool(getResultRowSize(pRuntimeEnv)); @@ -2061,6 +2071,9 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { taosHashCleanup(pRuntimeEnv->pTableRetrieveTsMap); pRuntimeEnv->pTableRetrieveTsMap = NULL; + taosHashCleanup(pRuntimeEnv->pResultRowListSet); + pRuntimeEnv->pResultRowListSet = NULL; + destroyOperatorInfo(pRuntimeEnv->proot); pRuntimeEnv->pool = destroyResultRowPool(pRuntimeEnv->pool); @@ -2791,7 +2804,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey; STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQueryAttr); - if (setWindowOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, &win, masterScan, &pResult, groupId, + if (setWindowOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.tid, &win, masterScan, &pResult, groupId, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, pTableScanInfo->rowCellInfoOffset) != TSDB_CODE_SUCCESS) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -2837,7 +2850,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey; STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQueryAttr); - if (setWindowOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, &win, masterScan, &pResult, groupId, + if (setWindowOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.tid, &win, masterScan, &pResult, groupId, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, pTableScanInfo->rowCellInfoOffset) != TSDB_CODE_SUCCESS) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -3251,8 +3264,8 @@ void setDefaultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, i int32_t* rowCellInfoOffset = pInfo->rowCellInfoOffset; SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo; - int32_t tid = 0; - SResultRow* pRow = doPrepareResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char *)&tid, sizeof(tid), true, uid); + int64_t tid = 0; + SResultRow* pRow = doPrepareResultRowFromKey(pRuntimeEnv, pResultRowInfo, tid, (char *)&tid, sizeof(tid), true, uid); for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { SColumnInfoData* pData = taosArrayGet(pDataBlock->pDataBlock, i); @@ -3483,10 +3496,13 @@ void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pRe } void doSetTableGroupOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, SQLFunctionCtx* pCtx, - int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t groupIndex) { + int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t tableGroupId) { + // for simple group by query without interval, all the tables belong to one group result. int64_t uid = 0; + int64_t tid = 0; + SResultRow* pResultRow = - doPrepareResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char*)&groupIndex, sizeof(groupIndex), true, uid); + doPrepareResultRowFromKey(pRuntimeEnv, pResultRowInfo, tid, (char*)&tableGroupId, sizeof(tableGroupId), true, uid); assert (pResultRow != NULL); /* @@ -3494,7 +3510,7 @@ void doSetTableGroupOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pRe * all group belong to one result set, and each group result has different group id so set the id to be one */ if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pRuntimeEnv->pResultBuf, groupIndex, pRuntimeEnv->pQueryAttr->resultRowSize); + int32_t ret = addNewWindowResultBuf(pResultRow, pRuntimeEnv->pResultBuf, tableGroupId, pRuntimeEnv->pQueryAttr->resultRowSize); if (ret != TSDB_CODE_SUCCESS) { return; } @@ -3503,20 +3519,20 @@ void doSetTableGroupOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pRe setResultRowOutputBufInitCtx(pRuntimeEnv, pResultRow, pCtx, numOfOutput, rowCellInfoOffset); } -void setExecutionContext(SQueryRuntimeEnv* pRuntimeEnv, SOptrBasicInfo* pInfo, int32_t numOfOutput, int32_t groupIndex, +void setExecutionContext(SQueryRuntimeEnv* pRuntimeEnv, SOptrBasicInfo* pInfo, int32_t numOfOutput, int32_t tableGroupId, TSKEY nextKey) { STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; // lastKey needs to be updated pTableQueryInfo->lastKey = nextKey; - if (pRuntimeEnv->prevGroupId != INT32_MIN && pRuntimeEnv->prevGroupId == groupIndex) { + if (pRuntimeEnv->prevGroupId != INT32_MIN && pRuntimeEnv->prevGroupId == tableGroupId) { return; } - doSetTableGroupOutputBuf(pRuntimeEnv, &pInfo->resultRowInfo, pInfo->pCtx, pInfo->rowCellInfoOffset, numOfOutput, groupIndex); + doSetTableGroupOutputBuf(pRuntimeEnv, &pInfo->resultRowInfo, pInfo->pCtx, pInfo->rowCellInfoOffset, numOfOutput, tableGroupId); // record the current active group id - pRuntimeEnv->prevGroupId = groupIndex; + pRuntimeEnv->prevGroupId = tableGroupId; } void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, @@ -5498,7 +5514,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI } else { SResultRow* pResult = NULL; pInfo->curWindow.ekey = pInfo->curWindow.skey; - int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, &pInfo->curWindow, masterScan, + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, pSDataBlock->info.tid, &pInfo->curWindow, masterScan, &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput, pBInfo->rowCellInfoOffset); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code @@ -5518,7 +5534,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI SResultRow* pResult = NULL; pInfo->curWindow.ekey = pInfo->curWindow.skey; - int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, &pInfo->curWindow, masterScan, + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, pSDataBlock->info.tid, &pInfo->curWindow, masterScan, &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput, pBInfo->rowCellInfoOffset); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code diff --git a/tests/script/general/parser/subInfrom.sim b/tests/script/general/parser/subInfrom.sim deleted file mode 100644 index e47831ee87..0000000000 --- a/tests/script/general/parser/subInfrom.sim +++ /dev/null @@ -1,147 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/exec.sh -n dnode1 -s start -sleep 100 -sql connect -sleep 100 - -print ========== sub_in_from.sim -$i = 0 - -$dbPrefix = subdb -$tbPrefix = sub_tb -$stbPrefix = sub_stb -$tbNum = 10 -$rowNum = 1000 -$totalNum = $tbNum * $rowNum -$loops = 200000 -$log = 10000 -$ts0 = 1537146000000 -$delta = 600000 -$i = 0 -$db = $dbPrefix . $i -$stb = $stbPrefix . $i - -sql drop database $db -x step1 -step1: -sql create database $db cache 16 maxrows 4096 keep 36500 -print ====== create tables -sql use $db -sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 int) - -$i = 0 -$ts = $ts0 -$halfNum = $tbNum / 2 -while $i < $halfNum - $tbId = $i + $halfNum - $tb = $tbPrefix . $i - $tb1 = $tbPrefix . $tbId - sql create table $tb using $stb tags( $i ) - sql create table $tb1 using $stb tags( $tbId ) - - $x = 0 - while $x < $rowNum - $xs = $x * $delta - $ts = $ts0 + $xs - $c = $x / 10 - $c = $c * 10 - $c = $x - $c - $binary = 'binary . $c - $binary = $binary . ' - $nchar = 'nchar . $c - $nchar = $nchar . ' - sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) - sql insert into $tb1 values ( $ts , $c , NULL , $c , NULL , $c , $c , true, $binary , $nchar ) - $x = $x + 1 - endw - - $i = $i + 1 -endw -print ====== tables created - -sql_error select count(*) from (select count(*) from abc.sub_stb0) -sql_error select val + 20 from (select count(*) from sub_stb0 interval(10h)) -sql_error select abc+20 from (select count(*) from sub_stb0 interval(1s)) - -sql select count(*) from (select count(*) from sub_stb0 interval(10h)) -if $rows != 1 then - return -1 -endi - -if $data00 != 18 then - print expect 18, actual: $data00 - return -1 -endi - -sql select ts from (select count(*) from sub_stb0 interval(10h)) -if $rows != 18 then - return -1 -endi - -if $data00 != @18-09-17 04:00:00.000@ then - return -1 -endi - -if $data01 != @18-09-17 14:00:00.000@ then - return -1 -endi - -sql select val + 20, val from (select count(*) as val from sub_stb0 interval(10h)) -if $rows != 18 then - return -1 -endi - -if $data00 != 320.000000 then - return -1 -endi - -if $data01 != 300 then - return -1 -endi - -if $data10 != 620 then - return -1 -endi - -if $data11 != 600 then - return -1 -endi - -if $data20 != 620 then - return -1 -endi - -if $data21 != 600 then - return -1 -endi - -sql select max(val), min(val), max(val) - min(val) from (select count(*) val from sub_stb0 interval(10h)) -if $rows != 1 then - return -1 -endi - -if $data00 != 600 then - return -1 -endi - -if $data01 != 100 then - return -1 -endi - -if $data02 != 500.000000 then - return -1 -endi - -sql select first(ts,val),last(ts,val) from (select count(*) val from sub_stb0 interval(10h)) -sql select top(val, 5) from (select count(*) val from sub_stb0 interval(10h)) -sql select diff(val) from (select count(*) val from sub_stb0 interval(10h)) -sql select apercentile(val, 50) from (select count(*) val from sub_stb0 interval(10h)) - -# not support yet -sql select percentile(val, 50) from (select count(*) val from sub_stb0 interval(10h)) -sql select stddev(val) from (select count(*) val from sub_stb0 interval(10h)) - -print ====================>complex query - From 4b9190ab38c93122fdebdc7083c9c00f848c5c43 Mon Sep 17 00:00:00 2001 From: wpan Date: Fri, 9 Jul 2021 14:06:49 +0800 Subject: [PATCH 23/27] fix interval issue --- src/query/src/qExecutor.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 557d38d7f1..b04db7ca1d 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1304,7 +1304,8 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul j++; } - for(; pResultRowInfo->pResult[j] != pResultRowInfo->current; ++j) { + SResultRow* current = pResultRowInfo->current; + for(; pResultRowInfo->pResult[j] != current && j < pResultRowInfo->size; ++j) { SResultRow* pRes = pResultRowInfo->pResult[j]; if (pRes->closed) { assert(resultRowInterpolated(pRes, RESULT_ROW_START_INTERP) && resultRowInterpolated(pRes, RESULT_ROW_END_INTERP)); From e811cb0ad85499214c5cac7fd802439222eca0f1 Mon Sep 17 00:00:00 2001 From: jtcheng Date: Fri, 9 Jul 2021 14:12:23 +0800 Subject: [PATCH 24/27] [TD-6809]: Fix jdbc NPE (#6814) --- .gitignore | 2 ++ .../main/java/com/taosdata/jdbc/TSDBPreparedStatement.java | 2 +- .../java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java | 4 ++-- .../com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java | 6 ++++++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index da47590a2f..50f4251320 100644 --- a/.gitignore +++ b/.gitignore @@ -68,6 +68,8 @@ CMakeError.log *.o version.c taos.rc +src/connector/jdbc/.classpath +src/connector/jdbc/.project src/connector/jdbc/.settings/ tests/comparisonTest/cassandra/cassandratest/.classpath tests/comparisonTest/cassandra/cassandratest/.project diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java index d9042fb729..22fb0c4ae4 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java @@ -57,8 +57,8 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat parameterCnt++; } } - parameters = new Object[parameterCnt]; } + parameters = new Object[parameterCnt]; if (parameterCnt > 1) { // the table name is also a parameter, so ignore it. diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java index 1eaddce1d6..f2abbd2445 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java @@ -22,16 +22,16 @@ public class RestfulPreparedStatement extends RestfulStatement implements Prepar super(conn, database); this.rawSql = sql; + int parameterCnt = 0; if (sql.contains("?")) { - int parameterCnt = 0; for (int i = 0; i < sql.length(); i++) { if ('?' == sql.charAt(i)) { parameterCnt++; } } - parameters = new Object[parameterCnt]; this.isPrepared = true; } + parameters = new Object[parameterCnt]; // build parameterMetaData this.parameterMetaData = new RestfulParameterMetaData(parameters); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java index 244e8afd11..4760a723e4 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java @@ -15,6 +15,8 @@ public class RestfulPreparedStatementTest { private static PreparedStatement pstmt_insert; private static final String sql_select = "select * from t1 where ts > ? and ts <= ? and f1 >= ?"; private static PreparedStatement pstmt_select; + private static final String sql_without_parameters = "select count(*) from t1"; + private static PreparedStatement pstmt_without_parameters; @Test public void executeQuery() throws SQLException { @@ -237,6 +239,7 @@ public class RestfulPreparedStatementTest { @Test public void clearParameters() throws SQLException { pstmt_insert.clearParameters(); + pstmt_without_parameters.clearParameters(); } @Test @@ -382,6 +385,7 @@ public class RestfulPreparedStatementTest { pstmt_insert = conn.prepareStatement(sql_insert); pstmt_select = conn.prepareStatement(sql_select); + pstmt_without_parameters = conn.prepareStatement(sql_without_parameters); } catch (SQLException e) { e.printStackTrace(); } @@ -394,6 +398,8 @@ public class RestfulPreparedStatementTest { pstmt_insert.close(); if (pstmt_select != null) pstmt_select.close(); + if (pstmt_without_parameters != null) + pstmt_without_parameters.close(); if (conn != null) conn.close(); } catch (SQLException e) { From 66ab17dd58a3110b15a89c3a3a9a5b4edfa74738 Mon Sep 17 00:00:00 2001 From: Linhe Huo Date: Fri, 9 Jul 2021 16:48:32 +0800 Subject: [PATCH 25/27] [TD-4946]: Node.js connector support nanoseconds (#6817) * [TD-4946]: Node.js connector support nanoseconds * [TD-5166]: Node.js connector support no host settings --- src/connector/nodejs/nodetaos/cinterface.js | 57 +++----- src/connector/nodejs/nodetaos/taosobjects.js | 133 ++++++++++++++---- src/connector/nodejs/package.json | 4 +- src/connector/nodejs/tdengine.js | 2 +- src/connector/nodejs/test/test.js | 2 +- src/connector/nodejs/test/testMicroseconds.js | 49 +++++++ src/connector/nodejs/test/testNanoseconds.js | 49 +++++++ 7 files changed, 223 insertions(+), 73 deletions(-) create mode 100644 src/connector/nodejs/test/testMicroseconds.js create mode 100644 src/connector/nodejs/test/testNanoseconds.js diff --git a/src/connector/nodejs/nodetaos/cinterface.js b/src/connector/nodejs/nodetaos/cinterface.js index 1c2685b8cb..03d27e5593 100644 --- a/src/connector/nodejs/nodetaos/cinterface.js +++ b/src/connector/nodejs/nodetaos/cinterface.js @@ -15,36 +15,18 @@ const { NULL_POINTER } = require('ref-napi'); module.exports = CTaosInterface; -function convertMillisecondsToDatetime(time) { - return new TaosObjects.TaosTimestamp(time); -} -function convertMicrosecondsToDatetime(time) { - return new TaosObjects.TaosTimestamp(time * 0.001, true); -} - -function convertTimestamp(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { - timestampConverter = convertMillisecondsToDatetime; - if (micro == true) { - timestampConverter = convertMicrosecondsToDatetime; - } +function convertTimestamp(data, num_of_rows, nbytes = 0, offset = 0, precision = 0) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; while (currOffset < data.length) { - let queue = []; - let time = 0; - for (let i = currOffset; i < currOffset + nbytes; i++) { - queue.push(data[i]); - } - for (let i = queue.length - 1; i >= 0; i--) { - time += queue[i] * Math.pow(16, i * 2); - } + let time = data.readInt64LE(currOffset); currOffset += nbytes; - res.push(timestampConverter(time)); + res.push(new TaosObjects.TaosTimestamp(time, precision)); } return res; } -function convertBool(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { +function convertBool(data, num_of_rows, nbytes = 0, offset = 0, precision = 0) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = new Array(data.length); for (let i = 0; i < data.length; i++) { @@ -60,7 +42,7 @@ function convertBool(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { } return res; } -function convertTinyint(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { +function convertTinyint(data, num_of_rows, nbytes = 0, offset = 0, precision = 0) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -71,7 +53,7 @@ function convertTinyint(data, num_of_rows, nbytes = 0, offset = 0, micro = false } return res; } -function convertSmallint(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { +function convertSmallint(data, num_of_rows, nbytes = 0, offset = 0, precision = 0) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -82,7 +64,7 @@ function convertSmallint(data, num_of_rows, nbytes = 0, offset = 0, micro = fals } return res; } -function convertInt(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { +function convertInt(data, num_of_rows, nbytes = 0, offset = 0, precision = 0) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -93,7 +75,7 @@ function convertInt(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { } return res; } -function convertBigint(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { +function convertBigint(data, num_of_rows, nbytes = 0, offset = 0, precision = 0) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -104,7 +86,7 @@ function convertBigint(data, num_of_rows, nbytes = 0, offset = 0, micro = false) } return res; } -function convertFloat(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { +function convertFloat(data, num_of_rows, nbytes = 0, offset = 0, precision = 0) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -115,7 +97,7 @@ function convertFloat(data, num_of_rows, nbytes = 0, offset = 0, micro = false) } return res; } -function convertDouble(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { +function convertDouble(data, num_of_rows, nbytes = 0, offset = 0, precision = 0) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -127,7 +109,7 @@ function convertDouble(data, num_of_rows, nbytes = 0, offset = 0, micro = false) return res; } -function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { +function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, precision = 0) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; @@ -272,7 +254,7 @@ CTaosInterface.prototype.config = function config() { CTaosInterface.prototype.connect = function connect(host = null, user = "root", password = "taosdata", db = null, port = 0) { let _host, _user, _password, _db, _port; try { - _host = host != null ? ref.allocCString(host) : ref.alloc(ref.types.char_ptr, ref.NULL); + _host = host != null ? ref.allocCString(host) : ref.NULL; } catch (err) { throw "Attribute Error: host is expected as a str"; @@ -290,7 +272,7 @@ CTaosInterface.prototype.connect = function connect(host = null, user = "root", throw "Attribute Error: password is expected as a str"; } try { - _db = db != null ? ref.allocCString(db) : ref.alloc(ref.types.char_ptr, ref.NULL); + _db = db != null ? ref.allocCString(db) : ref.NULL; } catch (err) { throw "Attribute Error: db is expected as a str"; @@ -345,8 +327,7 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { } var fieldL = this.libtaos.taos_fetch_lengths(result); - - let isMicro = (this.libtaos.taos_result_precision(result) == FieldTypes.C_TIMESTAMP_MICRO); + let precision = this.libtaos.taos_result_precision(result); var fieldlens = []; @@ -373,7 +354,7 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { if (!convertFunctions[fields[i]['type']]) { throw new errors.DatabaseError("Invalid data type returned from database"); } - blocks[i] = convertFunctions[fields[i]['type']](pdata, num_of_rows, fieldlens[i], offset, isMicro); + blocks[i] = convertFunctions[fields[i]['type']](pdata, num_of_rows, fieldlens[i], offset, precision); } } return { blocks: blocks, num_of_rows } @@ -423,7 +404,7 @@ CTaosInterface.prototype.fetch_rows_a = function fetch_rows_a(result, callback, let row = cti.libtaos.taos_fetch_row(result2); let fields = cti.fetchFields_a(result2); - let isMicro = (cti.libtaos.taos_result_precision(result2) == FieldTypes.C_TIMESTAMP_MICRO); + let precision = cti.libtaos.taos_result_precision(result2); let blocks = new Array(fields.length); blocks.fill(null); numOfRows2 = Math.abs(numOfRows2); @@ -449,7 +430,7 @@ CTaosInterface.prototype.fetch_rows_a = function fetch_rows_a(result, callback, let prow = ref.reinterpret(row, 8, i * 8); prow = prow.readPointer(); prow = ref.ref(prow); - blocks[i] = convertFunctions[fields[i]['type']](prow, 1, fieldlens[i], offset, isMicro); + blocks[i] = convertFunctions[fields[i]['type']](prow, 1, fieldlens[i], offset, precision); //offset += fields[i]['bytes'] * numOfRows2; } } @@ -572,7 +553,7 @@ CTaosInterface.prototype.openStream = function openStream(connection, sql, callb var cti = this; let asyncCallbackWrapper = function (param2, result2, row) { let fields = cti.fetchFields_a(result2); - let isMicro = (cti.libtaos.taos_result_precision(result2) == FieldTypes.C_TIMESTAMP_MICRO); + let precision = cti.libtaos.taos_result_precision(result2); let blocks = new Array(fields.length); blocks.fill(null); let numOfRows2 = 1; @@ -582,7 +563,7 @@ CTaosInterface.prototype.openStream = function openStream(connection, sql, callb if (!convertFunctions[fields[i]['type']]) { throw new errors.DatabaseError("Invalid data type returned from database"); } - blocks[i] = convertFunctions[fields[i]['type']](row, numOfRows2, fields[i]['bytes'], offset, isMicro); + blocks[i] = convertFunctions[fields[i]['type']](row, numOfRows2, fields[i]['bytes'], offset, precision); offset += fields[i]['bytes'] * numOfRows2; } } diff --git a/src/connector/nodejs/nodetaos/taosobjects.js b/src/connector/nodejs/nodetaos/taosobjects.js index 809d17a016..0fc8dc8ef1 100644 --- a/src/connector/nodejs/nodetaos/taosobjects.js +++ b/src/connector/nodejs/nodetaos/taosobjects.js @@ -1,5 +1,5 @@ const FieldTypes = require('./constants'); - +const util = require('util'); /** * Various objects such as TaosRow and TaosColumn that help make parsing data easier * @module TaosObjects @@ -14,7 +14,7 @@ const FieldTypes = require('./constants'); * var trow = new TaosRow(row); * console.log(trow.data); */ -function TaosRow (row) { +function TaosRow(row) { this.data = row; this.length = row.length; return this; @@ -29,10 +29,10 @@ function TaosRow (row) { */ function TaosField(field) { - this._field = field; - this.name = field.name; - this.type = FieldTypes.getType(field.type); - return this; + this._field = field; + this.name = field.name; + this.type = FieldTypes.getType(field.type); + return this; } /** @@ -42,39 +42,110 @@ function TaosField(field) { * @param {Date} date - A Javascript date time object or the time in milliseconds past 1970-1-1 00:00:00.000 */ class TaosTimestamp extends Date { - constructor(date, micro = false) { - super(date); - this._type = 'TaosTimestamp'; - if (micro) { - this.microTime = date - Math.floor(date); + constructor(date, precision = 0) { + if (precision === 1) { + super(Math.floor(date / 1000)); + this.precisionExtras = date % 1000; + } else if (precision === 2) { + super(parseInt(date / 1000000)); + // use BigInt to fix: 1625801548423914405 % 1000000 = 914496 which not expected (914405) + this.precisionExtras = parseInt(BigInt(date) % 1000000n); + } else { + super(parseInt(date)); + } + this.precision = precision; + } + + /** + * TDengine raw timestamp. + * @returns raw taos timestamp (int64) + */ + taosTimestamp() { + if (this.precision == 1) { + return (this * 1000 + this.precisionExtras); + } else if (this.precision == 2) { + return (this * 1000000 + this.precisionExtras); + } else { + return Math.floor(this); + } + } + + /** + * Gets the microseconds of a Date. + * @return {Int} A microseconds integer + */ + getMicroseconds() { + if (this.precision == 1) { + return this.getMilliseconds() * 1000 + this.precisionExtras; + } else if (this.precision == 2) { + return this.getMilliseconds() * 1000 + this.precisionExtras / 1000; + } else { + return 0; + } + } + /** + * Gets the nanoseconds of a TaosTimestamp. + * @return {Int} A nanoseconds integer + */ + getNanoseconds() { + if (this.precision == 1) { + return this.getMilliseconds() * 1000000 + this.precisionExtras * 1000; + } else if (this.precision == 2) { + return this.getMilliseconds() * 1000000 + this.precisionExtras; + } else { + return 0; + } + } + + /** + * @returns {String} a string for timestamp string format + */ + _precisionExtra() { + if (this.precision == 1) { + return String(this.precisionExtras).padStart(3, '0'); + } else if (this.precision == 2) { + return String(this.precisionExtras).padStart(6, '0'); + } else { + return ''; } } /** * @function Returns the date into a string usable by TDengine * @return {string} A Taos Timestamp String */ - toTaosString(){ + toTaosString() { var tzo = -this.getTimezoneOffset(), - dif = tzo >= 0 ? '+' : '-', - pad = function(num) { - var norm = Math.floor(Math.abs(num)); - return (norm < 10 ? '0' : '') + norm; - }, - pad2 = function(num) { - var norm = Math.floor(Math.abs(num)); - if (norm < 10) return '00' + norm; - if (norm < 100) return '0' + norm; - if (norm < 1000) return norm; - }; + dif = tzo >= 0 ? '+' : '-', + pad = function (num) { + var norm = Math.floor(Math.abs(num)); + return (norm < 10 ? '0' : '') + norm; + }, + pad2 = function (num) { + var norm = Math.floor(Math.abs(num)); + if (norm < 10) return '00' + norm; + if (norm < 100) return '0' + norm; + if (norm < 1000) return norm; + }; return this.getFullYear() + - '-' + pad(this.getMonth() + 1) + - '-' + pad(this.getDate()) + - ' ' + pad(this.getHours()) + - ':' + pad(this.getMinutes()) + - ':' + pad(this.getSeconds()) + - '.' + pad2(this.getMilliseconds()) + - '' + (this.microTime ? pad2(Math.round(this.microTime * 1000)) : ''); + '-' + pad(this.getMonth() + 1) + + '-' + pad(this.getDate()) + + ' ' + pad(this.getHours()) + + ':' + pad(this.getMinutes()) + + ':' + pad(this.getSeconds()) + + '.' + pad2(this.getMilliseconds()) + + '' + this._precisionExtra(); + } + + /** + * Custom console.log + * @returns {String} string format for debug + */ + [util.inspect.custom](depth, opts) { + return this.toTaosString() + JSON.stringify({ precision: this.precision, precisionExtras: this.precisionExtras }, opts); + } + toString() { + return this.toTaosString(); } } -module.exports = {TaosRow, TaosField, TaosTimestamp} +module.exports = { TaosRow, TaosField, TaosTimestamp } diff --git a/src/connector/nodejs/package.json b/src/connector/nodejs/package.json index b57d4c635c..db37318a16 100644 --- a/src/connector/nodejs/package.json +++ b/src/connector/nodejs/package.json @@ -1,13 +1,13 @@ { "name": "td2.0-connector", - "version": "2.0.8", + "version": "2.0.9", "description": "A Node.js connector for TDengine.", "main": "tdengine.js", "directories": { "test": "test" }, "scripts": { - "test": "node test/test.js" + "test": "node test/test.js && node test/testMicroseconds.js && node test/testNanoseconds.js" }, "repository": { "type": "git", diff --git a/src/connector/nodejs/tdengine.js b/src/connector/nodejs/tdengine.js index aa296279d5..047c744a4f 100644 --- a/src/connector/nodejs/tdengine.js +++ b/src/connector/nodejs/tdengine.js @@ -1,4 +1,4 @@ var TDengineConnection = require('./nodetaos/connection.js') -module.exports.connect = function (connection=null) { +module.exports.connect = function (connection={}) { return new TDengineConnection(connection); } diff --git a/src/connector/nodejs/test/test.js b/src/connector/nodejs/test/test.js index bf4bb2c541..caf05955da 100644 --- a/src/connector/nodejs/test/test.js +++ b/src/connector/nodejs/test/test.js @@ -1,5 +1,5 @@ const taos = require('../tdengine'); -var conn = taos.connect({host:"127.0.0.1", user:"root", password:"taosdata", config:"/etc/taos",port:10}); +var conn = taos.connect(); var c1 = conn.cursor(); let stime = new Date(); let interval = 1000; diff --git a/src/connector/nodejs/test/testMicroseconds.js b/src/connector/nodejs/test/testMicroseconds.js new file mode 100644 index 0000000000..cc65b3d919 --- /dev/null +++ b/src/connector/nodejs/test/testMicroseconds.js @@ -0,0 +1,49 @@ +const taos = require('../tdengine'); +var conn = taos.connect(); +var c1 = conn.cursor(); +let stime = new Date(); +let interval = 1000; + +function convertDateToTS(date) { + let tsArr = date.toISOString().split("T") + return "\"" + tsArr[0] + " " + tsArr[1].substring(0, tsArr[1].length - 1) + "\""; +} +function R(l, r) { + return Math.random() * (r - l) - r; +} +function randomBool() { + if (Math.random() < 0.5) { + return true; + } + return false; +} + +// Initialize +//c1.execute('drop database td_connector_test;'); +const dbname = 'nodejs_test_us'; +c1.execute('create database if not exists ' + dbname + ' precision "us"'); +c1.execute('use ' + dbname) +c1.execute('create table if not exists tstest (ts timestamp, _int int);'); +c1.execute('insert into tstest values(1625801548423914, 0)'); +// Select +console.log('select * from tstest'); +c1.execute('select * from tstest'); + +var d = c1.fetchall(); +console.log(c1.fields); +let ts = d[0][0]; +console.log(ts); + +if (ts.taosTimestamp() != 1625801548423914) { + throw "microseconds not match!"; +} +if (ts.getMicroseconds() % 1000 !== 914) { + throw "micronsecond precision error"; +} +setTimeout(function () { + c1.query('drop database nodejs_us_test;'); +}, 200); + +setTimeout(function () { + conn.close(); +}, 2000); diff --git a/src/connector/nodejs/test/testNanoseconds.js b/src/connector/nodejs/test/testNanoseconds.js new file mode 100644 index 0000000000..85a7600b01 --- /dev/null +++ b/src/connector/nodejs/test/testNanoseconds.js @@ -0,0 +1,49 @@ +const taos = require('../tdengine'); +var conn = taos.connect(); +var c1 = conn.cursor(); +let stime = new Date(); +let interval = 1000; + +function convertDateToTS(date) { + let tsArr = date.toISOString().split("T") + return "\"" + tsArr[0] + " " + tsArr[1].substring(0, tsArr[1].length - 1) + "\""; +} +function R(l, r) { + return Math.random() * (r - l) - r; +} +function randomBool() { + if (Math.random() < 0.5) { + return true; + } + return false; +} + +// Initialize +//c1.execute('drop database td_connector_test;'); +const dbname = 'nodejs_test_ns'; +c1.execute('create database if not exists ' + dbname + ' precision "ns"'); +c1.execute('use ' + dbname) +c1.execute('create table if not exists tstest (ts timestamp, _int int);'); +c1.execute('insert into tstest values(1625801548423914405, 0)'); +// Select +console.log('select * from tstest'); +c1.execute('select * from tstest'); + +var d = c1.fetchall(); +console.log(c1.fields); +let ts = d[0][0]; +console.log(ts); + +if (ts.taosTimestamp() != 1625801548423914405) { + throw "nanosecond not match!"; +} +if (ts.getNanoseconds() % 1000000 !== 914405) { + throw "nanosecond precision error"; +} +setTimeout(function () { + c1.query('drop database nodejs_ns_test;'); +}, 200); + +setTimeout(function () { + conn.close(); +}, 2000); From a13206ddbf35d91e4e4e1bbe2f74fccff0cf2788 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 9 Jul 2021 17:58:08 +0800 Subject: [PATCH 26/27] Feature/sangshuduo/td 3973 use jemalloc (#6822) * [TD-3973]: add jemalloc as submodule. * add macro definitions in cmake. * [TD-3973]: use jemalloc. build works as following instructions: cmake .. -DJEMALLOC_ENABLED=true make * fix jemalloc at tag 5.2.1 * link jemalloc works. * make install works. * support jemalloc in release.sh. * release script works. * fix a typo. * [TD-3937]: support jemalloc add install funtion to all scripts. * adjust install_jemalloc() position for update check compatiblity. * fix position bug. * add ldconfig for jemalloc library cache refresh. * add /etc/ld.so.conf.d/jemalloc.conf for centos * check ver comp by file. * fix makeclient.sh Co-authored-by: Shuduo Sang --- packaging/tools/makeclient.sh | 33 +++++++++++++++++++++++++++++ packaging/tools/makeclient_power.sh | 33 +++++++++++++++++++++++++++++ packaging/tools/makeclient_tq.sh | 33 +++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+) diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index d0eeffc86a..0ae5733599 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -69,6 +69,39 @@ mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/taos.cfg mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* +if [ -f ${build_dir}/bin/jemalloc-config ]; then + mkdir -p ${install_dir}/jemalloc/{bin,lib,lib/pkgconfig,include/jemalloc,share/doc/jemalloc,share/man/man3} + cp ${build_dir}/bin/jemalloc-config ${install_dir}/jemalloc/bin + if [ -f ${build_dir}/bin/jemalloc.sh ]; then + cp ${build_dir}/bin/jemalloc.sh ${install_dir}/jemalloc/bin + fi + if [ -f ${build_dir}/bin/jeprof ]; then + cp ${build_dir}/bin/jeprof ${install_dir}/jemalloc/bin + fi + if [ -f ${build_dir}/include/jemalloc/jemalloc.h ]; then + cp ${build_dir}/include/jemalloc/jemalloc.h ${install_dir}/jemalloc/include/jemalloc + fi + if [ -f ${build_dir}/lib/libjemalloc.so.2 ]; then + cp ${build_dir}/lib/libjemalloc.so.2 ${install_dir}/jemalloc/lib + ln -sf libjemalloc.so.2 ${install_dir}/jemalloc/lib/libjemalloc.so + fi + if [ -f ${build_dir}/lib/libjemalloc.a ]; then + cp ${build_dir}/lib/libjemalloc.a ${install_dir}/jemalloc/lib + fi + if [ -f ${build_dir}/lib/libjemalloc_pic.a ]; then + cp ${build_dir}/lib/libjemalloc_pic.a ${install_dir}/jemalloc/lib + fi + if [ -f ${build_dir}/lib/pkgconfig/jemalloc.pc ]; then + cp ${build_dir}/lib/pkgconfig/jemalloc.pc ${install_dir}/jemalloc/lib/pkgconfig + fi + if [ -f ${build_dir}/share/doc/jemalloc/jemalloc.html ]; then + cp ${build_dir}/share/doc/jemalloc/jemalloc.html ${install_dir}/jemalloc/share/doc/jemalloc + fi + if [ -f ${build_dir}/share/man/man3/jemalloc.3 ]; then + cp ${build_dir}/share/man/man3/jemalloc.3 ${install_dir}/jemalloc/share/man/man3 + fi +fi + cd ${install_dir} if [ "$osType" != "Darwin" ]; then diff --git a/packaging/tools/makeclient_power.sh b/packaging/tools/makeclient_power.sh index 8241319e4f..6d10245b4f 100755 --- a/packaging/tools/makeclient_power.sh +++ b/packaging/tools/makeclient_power.sh @@ -91,6 +91,39 @@ else fi chmod a+x ${install_dir}/bin/* || : +if [ -f ${build_dir}/bin/jemalloc-config ]; then + mkdir -p ${install_dir}/jemalloc/{bin,lib,lib/pkgconfig,include/jemalloc,share/doc/jemalloc,share/man/man3} + cp ${build_dir}/bin/jemalloc-config ${install_dir}/jemalloc/bin + if [ -f ${build_dir}/bin/jemalloc.sh ]; then + cp ${build_dir}/bin/jemalloc.sh ${install_dir}/jemalloc/bin + fi + if [ -f ${build_dir}/bin/jeprof ]; then + cp ${build_dir}/bin/jeprof ${install_dir}/jemalloc/bin + fi + if [ -f ${build_dir}/include/jemalloc/jemalloc.h ]; then + cp ${build_dir}/include/jemalloc/jemalloc.h ${install_dir}/jemalloc/include/jemalloc + fi + if [ -f ${build_dir}/lib/libjemalloc.so.2 ]; then + cp ${build_dir}/lib/libjemalloc.so.2 ${install_dir}/jemalloc/lib + ln -sf libjemalloc.so.2 ${install_dir}/jemalloc/lib/libjemalloc.so + fi + if [ -f ${build_dir}/lib/libjemalloc.a ]; then + cp ${build_dir}/lib/libjemalloc.a ${install_dir}/jemalloc/lib + fi + if [ -f ${build_dir}/lib/libjemalloc_pic.a ]; then + cp ${build_dir}/lib/libjemalloc_pic.a ${install_dir}/jemalloc/lib + fi + if [ -f ${build_dir}/lib/pkgconfig/jemalloc.pc ]; then + cp ${build_dir}/lib/pkgconfig/jemalloc.pc ${install_dir}/jemalloc/lib/pkgconfig + fi + if [ -f ${build_dir}/share/doc/jemalloc/jemalloc.html ]; then + cp ${build_dir}/share/doc/jemalloc/jemalloc.html ${install_dir}/jemalloc/share/doc/jemalloc + fi + if [ -f ${build_dir}/share/man/man3/jemalloc.3 ]; then + cp ${build_dir}/share/man/man3/jemalloc.3 ${install_dir}/jemalloc/share/man/man3 + fi +fi + cd ${install_dir} if [ "$osType" != "Darwin" ]; then diff --git a/packaging/tools/makeclient_tq.sh b/packaging/tools/makeclient_tq.sh index 51fd064e1b..03d9b13059 100755 --- a/packaging/tools/makeclient_tq.sh +++ b/packaging/tools/makeclient_tq.sh @@ -91,6 +91,39 @@ else fi chmod a+x ${install_dir}/bin/* || : +if [ -f ${build_dir}/bin/jemalloc-config ]; then + mkdir -p ${install_dir}/jemalloc/{bin,lib,lib/pkgconfig,include/jemalloc,share/doc/jemalloc,share/man/man3} + cp ${build_dir}/bin/jemalloc-config ${install_dir}/jemalloc/bin + if [ -f ${build_dir}/bin/jemalloc.sh ]; then + cp ${build_dir}/bin/jemalloc.sh ${install_dir}/jemalloc/bin + fi + if [ -f ${build_dir}/bin/jeprof ]; then + cp ${build_dir}/bin/jeprof ${install_dir}/jemalloc/bin + fi + if [ -f ${build_dir}/include/jemalloc/jemalloc.h ]; then + cp ${build_dir}/include/jemalloc/jemalloc.h ${install_dir}/jemalloc/include/jemalloc + fi + if [ -f ${build_dir}/lib/libjemalloc.so.2 ]; then + cp ${build_dir}/lib/libjemalloc.so.2 ${install_dir}/jemalloc/lib + ln -sf libjemalloc.so.2 ${install_dir}/jemalloc/lib/libjemalloc.so + fi + if [ -f ${build_dir}/lib/libjemalloc.a ]; then + cp ${build_dir}/lib/libjemalloc.a ${install_dir}/jemalloc/lib + fi + if [ -f ${build_dir}/lib/libjemalloc_pic.a ]; then + cp ${build_dir}/lib/libjemalloc_pic.a ${install_dir}/jemalloc/lib + fi + if [ -f ${build_dir}/lib/pkgconfig/jemalloc.pc ]; then + cp ${build_dir}/lib/pkgconfig/jemalloc.pc ${install_dir}/jemalloc/lib/pkgconfig + fi + if [ -f ${build_dir}/share/doc/jemalloc/jemalloc.html ]; then + cp ${build_dir}/share/doc/jemalloc/jemalloc.html ${install_dir}/jemalloc/share/doc/jemalloc + fi + if [ -f ${build_dir}/share/man/man3/jemalloc.3 ]; then + cp ${build_dir}/share/man/man3/jemalloc.3 ${install_dir}/jemalloc/share/man/man3 + fi +fi + cd ${install_dir} if [ "$osType" != "Darwin" ]; then From c9933fd333594b2e2c69a1490a59ab600ca1b55d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 9 Jul 2021 19:16:25 +0800 Subject: [PATCH 27/27] [td-225] --- src/client/src/tscSQLParser.c | 8 +- src/query/inc/qExecutor.h | 3 +- src/query/src/qExecutor.c | 103 +++++++++------------- src/query/src/qUtil.c | 4 +- tests/script/general/parser/nestquery.sim | 40 +++++++++ 5 files changed, 89 insertions(+), 69 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index c2542fb8c6..d3651efef4 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -7802,14 +7802,16 @@ static int32_t doLoadAllTableMeta(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNod } static STableMeta* extractTempTableMetaFromSubquery(SQueryInfo* pUpstream) { - int32_t numOfColumns = pUpstream->fieldsInfo.numOfOutput; + STableMetaInfo* pUpstreamTableMetaInfo = tscGetMetaInfo(pUpstream, 0); - STableMeta* meta = calloc(1, sizeof(STableMeta) + sizeof(SSchema) * numOfColumns); + int32_t numOfColumns = pUpstream->fieldsInfo.numOfOutput; + STableMeta *meta = calloc(1, sizeof(STableMeta) + sizeof(SSchema) * numOfColumns); meta->tableType = TSDB_TEMP_TABLE; STableComInfo *info = &meta->tableInfo; info->numOfColumns = numOfColumns; - info->numOfTags = 0; + info->precision = pUpstreamTableMetaInfo->pTableMeta->tableInfo.precision; + info->numOfTags = 0; int32_t n = 0; for(int32_t i = 0; i < numOfColumns; ++i) { diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index ed9494a443..7c82aa659d 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -105,8 +105,7 @@ typedef struct SResultRowInfo { int16_t type:8; // data type for hash key int32_t size:24; // number of result set int32_t capacity; // max capacity - SResultRow* current; // current start active index - int64_t prevSKey; // previous (not completed) sliding window start key + SResultRow* current; // current active result row } SResultRowInfo; typedef struct SColumnFilterElem { diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index b04db7ca1d..ea0806e541 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -242,6 +242,7 @@ static void sortGroupResByOrderList(SGroupResInfo *pGroupResInfo, SQueryRuntimeE if (size <= 0) { return; } + int32_t orderId = pRuntimeEnv->pQueryAttr->order.orderColId; if (orderId <= 0) { return; @@ -410,21 +411,6 @@ static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, SQueryRuntim pResultRowInfo->capacity = (int32_t)newCapacity; } -//static int32_t ascResultRowCompareFn(const void* p1, const void* p2) { -// SResultRow* pRow1 = *(SResultRow**)p1; -// SResultRow* pRow2 = *(SResultRow**)p2; -// -// if (pRow1 == pRow2) { -// return 0; -// } else { -// return pRow1->win.skey < pRow2->win.skey? -1:1; -// } -//} - -//static int32_t descResultRowCompareFn(const void* p1, const void* p2) { -// return -ascResultRowCompareFn(p1, p2); -//} - static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, int64_t tid, char *pData, int16_t bytes, bool masterscan, uint64_t tableGroupId) { bool existed = false; @@ -450,11 +436,6 @@ static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SRes SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid); void* ptr = taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); existed = (ptr != NULL); -// __compar_fn_t fn = QUERY_IS_ASC_QUERY(pRuntimeEnv->pQueryAttr)? ascResultRowCompareFn:descResultRowCompareFn; -// void* ptr = taosbsearch(p1, pResultRowInfo->pResult, pResultRowInfo->size, POINTER_BYTES, fn, TD_EQ); -// if (ptr != NULL) { -// existed = true; -// } } } } else { @@ -526,12 +507,12 @@ static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t STimeWindow w = {0}; if (pResultRowInfo->current == NULL) { // the first window, from the previous stored value - if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { +// if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { getInitialStartTimeWindow(pQueryAttr, ts, &w); - pResultRowInfo->prevSKey = w.skey; - } else { - w.skey = pResultRowInfo->prevSKey; - } +// pResultRowInfo->prevSKey = w.skey; +// } else { +// w.skey = pResultRowInfo->prevSKey; +// } if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { w.ekey = taosTimeAdd(w.skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; @@ -539,10 +520,7 @@ static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t w.ekey = w.skey + pQueryAttr->interval.interval - 1; } } else { -// int32_t slot = curTimeWindowIndex(pResultRowInfo); -// SResultRow* pWindowRes = getResultRow(pResultRowInfo, slot); - SResultRow* pWindowRes = pResultRowInfo->current; - w = pWindowRes->win; + w = pResultRowInfo->current->win; } if (w.skey > ts || w.ekey < ts) { @@ -747,8 +725,6 @@ static void doUpdateResultRowIndex(SResultRowInfo*pResultRowInfo, TSKEY lastKey, } else { pResultRowInfo->current = pResultRowInfo->pResult[i + 1]; // current not closed result object } - - pResultRowInfo->prevSKey = pResultRowInfo->current->win.skey; } } @@ -1266,7 +1242,6 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); SResultRow* prevRow = pResultRowInfo->current; -// int32_t prevIndex = curTimeWindowIndex(pResultRowInfo); TSKEY* tsCols = NULL; if (pSDataBlock->pDataBlock != NULL) { @@ -1312,24 +1287,24 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul continue; } - STimeWindow w = pRes->win; - ret = setWindowOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.tid, &w, masterScan, &pResult, tableGroupId, pInfo->pCtx, - numOfOutput, pInfo->rowCellInfoOffset); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + STimeWindow w = pRes->win; + ret = setWindowOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.tid, &w, masterScan, &pResult, + tableGroupId, pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + assert(!resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); + + doTimeWindowInterpolation(pOperatorInfo, pInfo, pSDataBlock->pDataBlock, *(TSKEY*)pRuntimeEnv->prevRow[0], -1, + tsCols[startPos], startPos, w.ekey, RESULT_ROW_END_INTERP); + + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pInfo->pCtx, pQueryAttr->numOfOutput, RESULT_ROW_START_INTERP); + + doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &w, startPos, 0, tsCols, pSDataBlock->info.rows, numOfOutput); } - assert(!resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - - doTimeWindowInterpolation(pOperatorInfo, pInfo, pSDataBlock->pDataBlock, *(TSKEY *)pRuntimeEnv->prevRow[0], - -1, tsCols[startPos], startPos, w.ekey, RESULT_ROW_END_INTERP); - - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - setNotInterpoWindowKey(pInfo->pCtx, pQueryAttr->numOfOutput, RESULT_ROW_START_INTERP); - - doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &w, startPos, 0, tsCols, pSDataBlock->info.rows, numOfOutput); - } - // restore current time window ret = setWindowOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.tid, &win, masterScan, &pResult, tableGroupId, pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); @@ -3703,12 +3678,16 @@ void setParamForStableStddevByColData(SQueryRuntimeEnv* pRuntimeEnv, SQLFunction void setIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key) { SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; - SResultRowInfo *pWindowResInfo = &pTableQueryInfo->resInfo; + SResultRowInfo *pResultRowInfo = &pTableQueryInfo->resInfo; - if (pWindowResInfo->prevSKey != TSKEY_INITIAL_VAL) { + if (pResultRowInfo->current != NULL) { return; } +// if (pWindowResInfo->prevSKey != TSKEY_INITIAL_VAL) { +// return; +// } + pTableQueryInfo->win.skey = key; STimeWindow win = {.skey = key, .ekey = pQueryAttr->window.ekey}; @@ -3724,13 +3703,13 @@ void setIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key) { TSKEY ek = MAX(win.skey, win.ekey); getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w); - if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { - if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { - assert(win.ekey == pQueryAttr->window.ekey); - } - - pWindowResInfo->prevSKey = w.skey; - } +// if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { +// if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { +// assert(win.ekey == pQueryAttr->window.ekey); +// } +// +// pResultRowInfo->prevSKey = w.skey; +// } pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; } @@ -3773,8 +3752,8 @@ static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* } int32_t numOfRowsToCopy = pRow->numOfRows; - if (numOfResult + numOfRowsToCopy >= pRuntimeEnv->resultInfo.capacity) { - break; + if (numOfResult + numOfRowsToCopy >= pRuntimeEnv->resultInfo.capacity) { + break; } pGroupResInfo->index += 1; @@ -4631,7 +4610,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { if (pResultRowInfo->size > 0) { pResultRowInfo->current = pResultRowInfo->pResult[0]; - pResultRowInfo->prevSKey = pResultRowInfo->pResult[0]->win.skey; +// pResultRowInfo->prevSKey = pResultRowInfo->pResult[0]->win.skey; } qDebug("QInfo:0x%"PRIx64" start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, @@ -4657,7 +4636,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { if (pResultRowInfo->size > 0) { pResultRowInfo->current = pResultRowInfo->pResult[pResultRowInfo->size - 1]; - pResultRowInfo->prevSKey = pResultRowInfo->current->win.skey; +// pResultRowInfo->prevSKey = pResultRowInfo->current->win.skey; } p = doTableScanImpl(pOperator, newgroup); @@ -6166,7 +6145,7 @@ SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperato SGroupbyOperatorInfo* pInfo = calloc(1, sizeof(SGroupbyOperatorInfo)); pInfo->colIndex = -1; // group by column index - + pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 7c5a95312e..c14f46523d 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -44,7 +44,7 @@ int32_t getOutputInterResultBufSize(SQueryAttr* pQueryAttr) { int32_t initResultRowInfo(SResultRowInfo *pResultRowInfo, int32_t size, int16_t type) { pResultRowInfo->type = type; pResultRowInfo->size = 0; - pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL; +// pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL; pResultRowInfo->current = NULL; pResultRowInfo->capacity = size; @@ -93,7 +93,7 @@ void resetResultRowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRo pResultRowInfo->size = 0; pResultRowInfo->current = NULL; - pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL; +// pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL; } int32_t numOfClosedResultRows(SResultRowInfo *pResultRowInfo) { diff --git a/tests/script/general/parser/nestquery.sim b/tests/script/general/parser/nestquery.sim index fe12972bf6..6035992d30 100644 --- a/tests/script/general/parser/nestquery.sim +++ b/tests/script/general/parser/nestquery.sim @@ -450,4 +450,44 @@ if $data11 != 1 then return -1 endi +print =====================>TD-5157 +sql select twa(c1) from nest_tb1 interval(19a); +if $rows != 10000 then + return -1 +endi + +if $data00 != @20-09-14 23:59:59.992@ then + return -1 +endi + +if $data01 != 0.000083333 then + return -1 +endi + +print =================>us database interval query, TD-5039 +sql create database test precision 'us'; +sql use test; +sql create table t1(ts timestamp, k int); +sql insert into t1 values('2020-01-01 01:01:01.000', 1) ('2020-01-01 01:02:00.000', 2); +sql select avg(k) from (select avg(k) k from t1 interval(1s)) interval(1m); +if $rows != 2 then + return -1 +endi + +if $data00 != @20-01-01 01:01:00.000000@ then + return -1 +endi + +if $data01 != 1.000000000 then + return -1 +endi + +if $data10 != @20-01-01 01:02:00.000000@ then + return -1 +endi + +if $data11 != 2.000000000 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file