From 9be09492f0ff2deff9cd8dea5c0d889bd81ae2c8 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 24 Mar 2023 09:05:06 +0800 Subject: [PATCH 01/17] enh: disable timely vnodeCommit --- include/libs/sync/sync.h | 2 +- source/dnode/vnode/src/vnd/vnodeCommit.c | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 08a6be8015..1cd32f05ad 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -48,7 +48,7 @@ extern "C" { #define SYNC_HEARTBEAT_REPLY_SLOW_MS 1500 #define SYNC_SNAP_RESEND_MS 1000 * 60 -#define SYNC_VND_COMMIT_MIN_MS 1000 +#define SYNC_VND_COMMIT_MIN_MS 3000 #define SYNC_MAX_BATCH_SIZE 1 #define SYNC_INDEX_BEGIN 0 diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 0779388ba9..09f1ca7877 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -156,13 +156,10 @@ int vnodeShouldCommit(SVnode *pVnode) { bool needCommit = false; taosThreadMutexLock(&pVnode->mutex); - if (!pVnode->inUse || !diskAvail) { - goto _out; + if (pVnode->inUse && diskAvail) { + needCommit = + ((pVnode->inUse->size > pVnode->inUse->node.size) && (pSched->commitMs + SYNC_VND_COMMIT_MIN_MS < nowMs)); } - needCommit = - (((pVnode->inUse->size > pVnode->inUse->node.size) && (pSched->commitMs + SYNC_VND_COMMIT_MIN_MS < nowMs)) || - (pVnode->inUse->size > 0 && pSched->commitMs + pSched->maxWaitMs < nowMs)); -_out: taosThreadMutexUnlock(&pVnode->mutex); return needCommit; } From 357223d0653b06dfaf7e5d4074e81560bce47b88 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Sat, 25 Mar 2023 16:17:21 +0800 Subject: [PATCH 02/17] fix: recorect ci buildPath --- tests/parallel_test/run.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index f0880bdb04..21499a7ca8 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -167,8 +167,10 @@ function run_thread() { local case_build_san=`echo "$line"|cut -d, -f3` if [ "${case_build_san}" == "y" ]; then case_build_san="y" + DEBUGPATH="debugSan" elif [[ "${case_build_san}" == "n" ]] || [[ "${case_build_san}" == "" ]]; then case_build_san="n" + DEBUGPATH="debugNoSan" else usage exit 1 @@ -301,9 +303,9 @@ function run_thread() { if [ ! -z "$corefile" ]; then echo -e "\e[34m corefiles: $corefile \e[0m" local build_dir=$log_dir/build_${hosts[index]} - local remote_build_dir="${workdirs[index]}/TDengine/debug/build" + local remote_build_dir="${workdirs[index]}/TDengine/{DEBUGPATH}/build" if [ $ent -ne 0 ]; then - remote_build_dir="${workdirs[index]}/TDinternal/debug/build" + remote_build_dir="${workdirs[index]}/TDinternal/{DEBUGPATH}/build" fi mkdir $build_dir 2>/dev/null if [ $? -eq 0 ]; then From 0c00b4e0b54285ace0292ebd4c5b8cf2a1913f4a Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Sat, 25 Mar 2023 16:19:29 +0800 Subject: [PATCH 03/17] fix: recorect ci buildPath --- tests/parallel_test/run.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index 21499a7ca8..de343752c6 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -303,10 +303,10 @@ function run_thread() { if [ ! -z "$corefile" ]; then echo -e "\e[34m corefiles: $corefile \e[0m" local build_dir=$log_dir/build_${hosts[index]} - local remote_build_dir="${workdirs[index]}/TDengine/{DEBUGPATH}/build" - if [ $ent -ne 0 ]; then - remote_build_dir="${workdirs[index]}/TDinternal/{DEBUGPATH}/build" - fi + local remote_build_dir="${workdirs[index]}/{DEBUGPATH}/build" + # if [ $ent -ne 0 ]; then + # remote_build_dir="${workdirs[index]}/{DEBUGPATH}/build" + # fi mkdir $build_dir 2>/dev/null if [ $? -eq 0 ]; then # scp build binary From 5a2e3c1603800d263bd886c9d190df509c1c8258 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 27 Mar 2023 10:32:30 +0800 Subject: [PATCH 04/17] fix:add test case for raw_block interface --- source/libs/parser/src/parInsertUtil.c | 6 +- tests/parallel_test/cases.task | 1 + .../7-tmq/raw_block_interface_test.py | 54 ++++++++ utils/test/c/CMakeLists.txt | 9 ++ utils/test/c/write_raw_block_test.c | 126 ++++++++++++++++++ 5 files changed, 193 insertions(+), 3 deletions(-) create mode 100644 tests/system-test/7-tmq/raw_block_interface_test.py create mode 100644 utils/test/c/write_raw_block_test.c diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 4768231fb5..0ed32afbbc 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -621,12 +621,12 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate SSchema* pSchema = getTableColumnSchema(pTableCxt->pMeta); SBoundColInfo* boundInfo = &pTableCxt->boundColsInfo; - if (numFields != numOfCols) { + if (tFields != NULL && numFields != numOfCols) { uError("numFields:%d != numOfCols:%d", numFields, numOfCols); ret = TSDB_CODE_INVALID_PARA; goto end; } - if (numFields > boundInfo->numOfBound) { + if (tFields != NULL && numFields > boundInfo->numOfBound) { uError("numFields:%d > boundInfo->numOfBound:%d", numFields, boundInfo->numOfBound); ret = TSDB_CODE_INVALID_PARA; goto end; @@ -634,7 +634,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate for (int c = 0; c < boundInfo->numOfBound; ++c) { SSchema* pColSchema = &pSchema[c]; SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, c); - if(findFileds(pColSchema, tFields, numFields)){ + if(tFields == NULL || findFileds(pColSchema, tFields, numFields)){ if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { uError("type or bytes not equal"); ret = TSDB_CODE_INVALID_PARA; diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 58fa0cbfab..c2f8e75e79 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -93,6 +93,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/raw_block_interface_test.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 diff --git a/tests/system-test/7-tmq/raw_block_interface_test.py b/tests/system-test/7-tmq/raw_block_interface_test.py new file mode 100644 index 0000000000..1e89de1cce --- /dev/null +++ b/tests/system-test/7-tmq/raw_block_interface_test.py @@ -0,0 +1,54 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def checkData(self): + tdSql.execute('use db_raw') + tdSql.query("select * from d1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 120) + + tdSql.query("select * from d2") + tdSql.checkRows(1) + tdSql.checkData(0, 1, None) + + return + + def check(self): + buildPath = tdCom.getBuildPath() + cmdStr = '%s/build/bin/write_raw_block_test'%(buildPath) + tdLog.info(cmdStr) + os.system(cmdStr) + + self.checkData() + + return + + def run(self): + tdSql.prepare() + self.check() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/utils/test/c/CMakeLists.txt b/utils/test/c/CMakeLists.txt index 6ca266c555..eda9c70f15 100644 --- a/utils/test/c/CMakeLists.txt +++ b/utils/test/c/CMakeLists.txt @@ -3,6 +3,7 @@ add_dependencies(tmq_demo taos) add_executable(tmq_sim tmqSim.c) add_executable(create_table createTable.c) add_executable(tmq_taosx_ci tmq_taosx_ci.c) +add_executable(write_raw_block_test write_raw_block_test.c) add_executable(sml_test sml_test.c) add_executable(get_db_name_test get_db_name_test.c) target_link_libraries( @@ -34,6 +35,14 @@ target_link_libraries( PUBLIC os ) +target_link_libraries( + write_raw_block_test + PUBLIC taos_static + PUBLIC util + PUBLIC common + PUBLIC os +) + target_link_libraries( sml_test PUBLIC taos_static diff --git a/utils/test/c/write_raw_block_test.c b/utils/test/c/write_raw_block_test.c new file mode 100644 index 0000000000..998b47a66e --- /dev/null +++ b/utils/test/c/write_raw_block_test.c @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include "taos.h" +#include "types.h" + +int buildStable(TAOS* pConn, TAOS_RES* pRes) { + pRes = taos_query(pConn, + "CREATE STABLE `meters` (`ts` TIMESTAMP, `current` INT, `voltage` INT, `phase` FLOAT) TAGS " + "(`groupid` INT, `location` VARCHAR(16))"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table meters, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table d0 using meters tags(1, 'San Francisco')"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table d0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into d0 (ts, current) values (now, 120)"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into table d0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table d1 using meters tags(2, 'San Francisco')"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table d1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table d2 using meters tags(3, 'San Francisco')"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table d1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + return 0; +} + +int32_t init_env() { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + TAOS_RES* pRes = taos_query(pConn, "drop database if exists db_raw"); + if (taos_errno(pRes) != 0) { + printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database if not exists db_raw vgroups 2"); + if (taos_errno(pRes) != 0) { + printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "use db_raw"); + if (taos_errno(pRes) != 0) { + printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + buildStable(pConn, pRes); + + pRes = taos_query(pConn, "select * from d0"); + if (taos_errno(pRes) != 0) { + printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + void *data = NULL; + int32_t numOfRows = 0; + int error_code = taos_fetch_raw_block(pRes, &numOfRows, &data); + ASSERT(error_code == 0); + ASSERT(numOfRows == 1); + + taos_write_raw_block(pConn, numOfRows, data, "d1"); + + pRes = taos_query(pConn, "select ts,phase from d0"); + if (taos_errno(pRes) != 0) { + printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + error_code = taos_fetch_raw_block(pRes, &numOfRows, &data); + ASSERT(error_code == 0); + ASSERT(numOfRows == 1); + + int numFields = taos_num_fields(pRes); + TAOS_FIELD *fields = taos_fetch_fields(pRes); + taos_write_raw_block_with_fields(pConn, numOfRows, data, "d2", fields, numFields); + + taos_close(pConn); + return 0; +} + +int main(int argc, char* argv[]) { + if (init_env() < 0) { + return -1; + } +} From e1b9aac4c4f00eed05d6bf6ffe44ad29cb289a42 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 27 Mar 2023 10:41:25 +0800 Subject: [PATCH 05/17] fix:add test case for raw_block interface --- utils/test/c/write_raw_block_test.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/test/c/write_raw_block_test.c b/utils/test/c/write_raw_block_test.c index 998b47a66e..8e5dd62752 100644 --- a/utils/test/c/write_raw_block_test.c +++ b/utils/test/c/write_raw_block_test.c @@ -101,6 +101,7 @@ int32_t init_env() { ASSERT(numOfRows == 1); taos_write_raw_block(pConn, numOfRows, data, "d1"); + taos_free_result(pRes); pRes = taos_query(pConn, "select ts,phase from d0"); if (taos_errno(pRes) != 0) { @@ -114,6 +115,7 @@ int32_t init_env() { int numFields = taos_num_fields(pRes); TAOS_FIELD *fields = taos_fetch_fields(pRes); taos_write_raw_block_with_fields(pConn, numOfRows, data, "d2", fields, numFields); + taos_free_result(pRes); taos_close(pConn); return 0; From 5f9f5be6f4e5e49bc3c6d1a1c7f55566c6b91722 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 27 Mar 2023 11:06:10 +0800 Subject: [PATCH 06/17] fix: enterprise version displays the associated community version gitinfo --- cmake/cmake.version | 19 +++++++++++++++- include/util/version.h | 1 + source/dnode/mgmt/exe/dmMain.c | 22 ++++++++++-------- source/util/src/version.c.in | 1 + tools/shell/src/shellArguments.c | 39 +++++++++++++++++--------------- 5 files changed, 54 insertions(+), 28 deletions(-) diff --git a/cmake/cmake.version b/cmake/cmake.version index de85025a8c..42de285c8e 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -16,7 +16,7 @@ find_program(HAVE_GIT NAMES git) IF (DEFINED GITINFO) SET(TD_VER_GIT ${GITINFO}) ELSEIF (HAVE_GIT) - execute_process(COMMAND git log -1 --format=%H WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE GIT_COMMITID) + execute_process(COMMAND git log -1 --format=%H WORKING_DIRECTORY ${TD_COMMUNITY_DIR} OUTPUT_VARIABLE GIT_COMMITID) #message(STATUS "git log result:${GIT_COMMITID}") IF (GIT_COMMITID) string (REGEX REPLACE "[\n\t\r]" "" GIT_COMMITID ${GIT_COMMITID}) @@ -30,6 +30,23 @@ ELSE () SET(TD_VER_GIT "no git commit id") ENDIF () +IF (DEFINED GITINFOI) + SET(TD_VER_GIT_INTERNAL ${GITINFOI}) +ELSEIF (HAVE_GIT) + execute_process(COMMAND git log -1 --format=%H WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_VARIABLE GIT_COMMITID) + message(STATUS "git log result:${GIT_COMMITID}") + IF (GIT_COMMITID) + string (REGEX REPLACE "[\n\t\r]" "" GIT_COMMITID ${GIT_COMMITID}) + SET(TD_VER_GIT_INTERNAL ${GIT_COMMITID}) + ELSE () + message(STATUS "not a git repository") + SET(TD_VER_GIT "no git commit id") + ENDIF () +ELSE () + message(STATUS "no git cmd") + SET(TD_VER_GIT_INTERNAL "no git commit id") +ENDIF () + IF (DEFINED VERDATE) SET(TD_VER_DATE ${VERDATE}) ELSE () diff --git a/include/util/version.h b/include/util/version.h index bac29e5baf..b241dd248b 100644 --- a/include/util/version.h +++ b/include/util/version.h @@ -23,6 +23,7 @@ extern "C" { extern char version[]; extern char compatible_version[]; extern char gitinfo[]; +extern char gitinfoOfInternal[]; extern char buildinfo[]; #ifdef __cplusplus diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index d3cffaa185..9a092e2df5 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -18,6 +18,7 @@ #include "mnode.h" #include "tconfig.h" #include "tglobal.h" +#include "version.h" // clang-format off #define DM_APOLLO_URL "The apollo string to use when configuring the server, such as: -a 'jsonFile:./tests/cfg.json', cfg.json text can be '{\"fqdn\":\"td1\"}'." @@ -76,28 +77,28 @@ void dmLogCrash(int signum, void *sigInfo, void *context) { taosIgnSignal(SIGINT); taosIgnSignal(SIGBREAK); -#ifndef WINDOWS +#ifndef WINDOWS taosIgnSignal(SIGBUS); #endif taosIgnSignal(SIGABRT); taosIgnSignal(SIGFPE); taosIgnSignal(SIGSEGV); - char *pMsg = NULL; + char *pMsg = NULL; const char *flags = "UTL FATAL "; ELogLevel level = DEBUG_FATAL; int32_t dflag = 255; - int64_t msgLen= -1; - + int64_t msgLen = -1; + if (tsEnableCrashReport) { if (taosGenCrashJsonMsg(signum, &pMsg, dmGetClusterId(), global.startTime)) { taosPrintLog(flags, level, dflag, "failed to generate crash json msg"); goto _return; } else { - msgLen = strlen(pMsg); + msgLen = strlen(pMsg); } } - + _return: taosLogCrashInfo("taosd", pMsg, msgLen, signum, sigInfo); @@ -123,7 +124,7 @@ static void dmSetSignalHandle() { #ifndef WINDOWS taosSetSignal(SIGBUS, dmLogCrash); -#endif +#endif taosSetSignal(SIGABRT, dmLogCrash); taosSetSignal(SIGFPE, dmLogCrash); taosSetSignal(SIGSEGV, dmLogCrash); @@ -134,7 +135,7 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { int32_t cmdEnvIndex = 0; if (argc < 2) return 0; - + global.envCmd = taosMemoryMalloc((argc - 1) * sizeof(char *)); memset(global.envCmd, 0, (argc - 1) * sizeof(char *)); for (int32_t i = 1; i < argc; ++i) { @@ -203,6 +204,9 @@ static void dmPrintVersion() { #endif printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version); printf("gitinfo: %s\n", gitinfo); +#ifdef TD_ENTERPRISE + printf("gitinfoOfInternal: %s\n", gitinfoOfInternal); +#endif printf("buildInfo: %s\n", buildinfo); } @@ -284,7 +288,7 @@ int mainWindows(int argc, char **argv) { printf("failed to init memory dbg, error:%s\n", tstrerror(code)); return code; } - tsAsyncLog = false; + tsAsyncLog = false; printf("memory dbg enabled\n"); } #endif diff --git a/source/util/src/version.c.in b/source/util/src/version.c.in index cb307b57fc..71998e3321 100644 --- a/source/util/src/version.c.in +++ b/source/util/src/version.c.in @@ -1,6 +1,7 @@ char version[64] = "${TD_VER_NUMBER}"; char compatible_version[12] = "${TD_VER_COMPATIBLE}"; char gitinfo[48] = "${TD_VER_GIT}"; +char gitinfoOfInternal[48] = "${TD_VER_GIT_INTERNAL}"; char buildinfo[64] = "Built at ${TD_VER_DATE}"; void libtaos_${TD_LIB_VER_NUMBER}_${TD_VER_OSTYPE}_${TD_VER_CPUTYPE}_${TD_VER_VERTYPE}() {}; diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index c16769ccba..8111ccc0b1 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -18,17 +18,18 @@ #endif #include "shellInt.h" +#include "version.h" #ifndef CUS_NAME - char cusName[] = "TDengine"; +char cusName[] = "TDengine"; #endif #ifndef CUS_PROMPT - char cusPrompt[] = "taos"; +char cusPrompt[] = "taos"; #endif #ifndef CUS_EMAIL - char cusEmail[] = ""; +char cusEmail[] = ""; #endif #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) @@ -58,9 +59,9 @@ #define SHELL_VERSION "Print program version." #ifdef WEBSOCKET -#define SHELL_DSN "The dsn to use when connecting to cloud server." -#define SHELL_REST "Use restful mode when connecting." -#define SHELL_TIMEOUT "Set the timeout for websocket query in seconds, default is 30." +#define SHELL_DSN "The dsn to use when connecting to cloud server." +#define SHELL_REST "Use restful mode when connecting." +#define SHELL_TIMEOUT "Set the timeout for websocket query in seconds, default is 30." #endif static int32_t shellParseSingleOpt(int32_t key, char *arg); @@ -145,7 +146,7 @@ static void shellParseArgsUseArgp(int argc, char *argv[]) { #endif #ifndef ARGP_ERR_UNKNOWN - #define ARGP_ERR_UNKNOWN E2BIG +#define ARGP_ERR_UNKNOWN E2BIG #endif static int32_t shellParseSingleOpt(int32_t key, char *arg) { @@ -246,8 +247,8 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) { SShellArgs *pArgs = &shell.args; for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 - || strcmp(argv[i], "-?") == 0 || strcmp(argv[i], "/?") == 0) { + if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 || strcmp(argv[i], "-?") == 0 || + strcmp(argv[i], "/?") == 0) { shellParseSingleOpt('?', NULL); return 0; } @@ -263,10 +264,8 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) { return -1; } - if (key[1] == 'h' || key[1] == 'P' || key[1] == 'u' - || key[1] == 'a' || key[1] == 'c' || key[1] == 's' - || key[1] == 'f' || key[1] == 'd' || key[1] == 'w' - || key[1] == 'n' || key[1] == 'l' || key[1] == 'N' + if (key[1] == 'h' || key[1] == 'P' || key[1] == 'u' || key[1] == 'a' || key[1] == 'c' || key[1] == 's' || + key[1] == 'f' || key[1] == 'd' || key[1] == 'w' || key[1] == 'n' || key[1] == 'l' || key[1] == 'N' #ifdef WEBSOCKET || key[1] == 'E' || key[1] == 'T' #endif @@ -282,12 +281,10 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) { } shellParseSingleOpt(key[1], val); i++; - } else if (key[1] == 'p' || key[1] == 'A' || key[1] == 'C' - || key[1] == 'r' || key[1] == 'k' - || key[1] == 't' || key[1] == 'V' - || key[1] == '?' || key[1] == 1 + } else if (key[1] == 'p' || key[1] == 'A' || key[1] == 'C' || key[1] == 'r' || key[1] == 'k' || key[1] == 't' || + key[1] == 'V' || key[1] == '?' || key[1] == 1 #ifdef WEBSOCKET - ||key[1] == 'R' + || key[1] == 'R' #endif ) { shellParseSingleOpt(key[1], NULL); @@ -420,9 +417,15 @@ int32_t shellParseArgs(int32_t argc, char *argv[]) { sprintf(promptContinueFormat, "%%%zus> ", strlen(cusPrompt)); sprintf(shell.info.promptContinue, promptContinueFormat, " "); shell.info.promptSize = strlen(shell.info.promptHeader); +#ifdef TD_ENTERPRISE + snprintf(shell.info.programVersion, sizeof(shell.info.programVersion), + "version: %s compatible_version: %s\ngitinfo: %s\ngitinfoOfInternal: %s\nbuildInfo: %s", version, + compatible_version, gitinfo, gitinfoOfInternal, buildinfo); +#else snprintf(shell.info.programVersion, sizeof(shell.info.programVersion), "version: %s compatible_version: %s\ngitinfo: %s\nbuildInfo: %s", version, compatible_version, gitinfo, buildinfo); +#endif #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) shell.info.osname = "Windows"; From d0b6f23dc20afba8888f1e7fbc94fa3d64d38d01 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 27 Mar 2023 11:24:06 +0800 Subject: [PATCH 07/17] fix: enterprise version displays the associated community version gitinfo --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 566d4ad29d..fb2b306f65 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,8 @@ if (NOT DEFINED TD_SOURCE_DIR) set( TD_SOURCE_DIR ${PROJECT_SOURCE_DIR} ) endif() +SET(TD_COMMUNITY_DIR ${PROJECT_SOURCE_DIR}) + set(TD_SUPPORT_DIR "${TD_SOURCE_DIR}/cmake") set(TD_CONTRIB_DIR "${TD_SOURCE_DIR}/contrib") From 3c70008b4b888f0e42d5898d8f801147b22da0f3 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 27 Mar 2023 15:41:50 +0800 Subject: [PATCH 08/17] fix: fix cfg load enviroment variables crash when environ is NULL --- source/util/src/tconfig.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index c15bd96903..f811d2f203 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -637,6 +637,8 @@ int32_t cfgLoadFromEnvVar(SConfig *pConfig) { int32_t code = 0; char **pEnv = environ; line[1023] = 0; + + if (pEnv == NULL) return 0; while (*pEnv != NULL) { name = value = value2 = value3 = NULL; olen = vlen = vlen2 = vlen3 = 0; From c4791d58a65f5dcae375d25ebb1f3c8f76a8df4f Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 27 Mar 2023 16:09:38 +0800 Subject: [PATCH 09/17] fix: create stream does not support event_window --- source/libs/parser/src/parCalcConst.c | 23 +++++++++++++---------- source/libs/parser/src/parTranslater.c | 12 ++++++++---- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index 33b8a686ae..a7f6d7cb3c 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -183,16 +183,18 @@ static int32_t calcConstProject(SNode* pProject, bool dual, SNode** pNew) { } else { code = scalarCalculateConstants(pProject, pNew); } - if (TSDB_CODE_SUCCESS == code && QUERY_NODE_VALUE == nodeType(*pNew) && NULL != pAssociation) { + if (TSDB_CODE_SUCCESS == code) { strcpy(((SExprNode*)*pNew)->aliasName, aliasName); - int32_t size = taosArrayGetSize(pAssociation); - for (int32_t i = 0; i < size; ++i) { - SNode** pCol = taosArrayGetP(pAssociation, i); - nodesDestroyNode(*pCol); - *pCol = nodesCloneNode(*pNew); - if (NULL == *pCol) { - code = TSDB_CODE_OUT_OF_MEMORY; - break; + if (QUERY_NODE_VALUE == nodeType(*pNew) && NULL != pAssociation) { + int32_t size = taosArrayGetSize(pAssociation); + for (int32_t i = 0; i < size; ++i) { + SNode** pCol = taosArrayGetP(pAssociation, i); + nodesDestroyNode(*pCol); + *pCol = nodesCloneNode(*pNew); + if (NULL == *pCol) { + code = TSDB_CODE_OUT_OF_MEMORY; + break; + } } } } @@ -383,7 +385,8 @@ static int32_t calcConstSetOpProjections(SCalcConstContext* pCxt, SSetOperator* int32_t index = 0; SNode* pProj = NULL; WHERE_EACH(pProj, pSetOp->pProjectionList) { - if (subquery && notRefByOrderBy((SColumnNode*)pProj, pSetOp->pOrderByList) && isSetUselessCol(pSetOp, index, (SExprNode*)pProj)) { + if (subquery && notRefByOrderBy((SColumnNode*)pProj, pSetOp->pOrderByList) && + isSetUselessCol(pSetOp, index, (SExprNode*)pProj)) { ERASE_NODE(pSetOp->pProjectionList); eraseSetOpChildProjection(pSetOp, index); continue; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 68c93f8ca9..726790443e 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1342,8 +1342,8 @@ static bool isCountNotNullValue(SFunctionNode* pFunc) { // count(1) is rewritten as count(ts) for scannning optimization static int32_t rewriteCountNotNullValue(STranslateContext* pCxt, SFunctionNode* pCount) { SValueNode* pValue = (SValueNode*)nodesListGetNode(pCount->pParameterList, 0); - STableNode* pTable = NULL; - int32_t code = findTable(pCxt, NULL, &pTable); + STableNode* pTable = NULL; + int32_t code = findTable(pCxt, NULL, &pTable); if (TSDB_CODE_SUCCESS == code && QUERY_NODE_REAL_TABLE == nodeType(pTable)) { SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pCol) { @@ -1424,7 +1424,7 @@ static int32_t translateAggFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { } if (isCountNotNullValue(pFunc)) { return rewriteCountNotNullValue(pCxt, pFunc); - } + } if (isCountTbname(pFunc)) { return rewriteCountTbname(pCxt, pFunc); } @@ -5923,11 +5923,15 @@ static int32_t addSubtableInfoToCreateStreamQuery(STranslateContext* pCxt, STabl return code; } +static bool isEventWindowQuery(SSelectStmt* pSelect) { + return NULL != pSelect->pWindow && QUERY_NODE_EVENT_WINDOW == nodeType(pSelect->pWindow); +} + static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt) { SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; if (TSDB_DATA_TYPE_TIMESTAMP != ((SExprNode*)nodesListGetNode(pSelect->pProjectionList, 0))->resType.type || !pSelect->isTimeLineResult || crossTableWithoutAggOper(pSelect) || NULL != pSelect->pOrderByList || - crossTableWithUdaf(pSelect)) { + crossTableWithUdaf(pSelect) || isEventWindowQuery(pSelect)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query"); } if (NULL != pSelect->pSubtable && TSDB_DATA_TYPE_VARCHAR != ((SExprNode*)pSelect->pSubtable)->resType.type) { From 9d207b3de2620b6927c092be4a2f119ec0751d42 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 27 Mar 2023 17:25:23 +0800 Subject: [PATCH 10/17] fix: taosbenchmark codacy complain for main (#20647) --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 897ccdd158..40fa48b815 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 04296a5 + GIT_TAG 2864326 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 5c128e2292c2c76c7f377797bcb53f83c0dcbd84 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 28 Mar 2023 09:27:48 +0800 Subject: [PATCH 11/17] refactor: do some internal refactor. --- include/util/tarray.h | 16 ----- source/dnode/mnode/impl/src/mndConsumer.c | 9 ++- source/dnode/vnode/src/tq/tq.c | 13 +++- source/dnode/vnode/src/tq/tqExec.c | 19 +++--- source/libs/executor/src/executorimpl.c | 2 +- source/util/src/tarray.c | 75 +---------------------- 6 files changed, 34 insertions(+), 100 deletions(-) diff --git a/include/util/tarray.h b/include/util/tarray.h index a8510e4bc8..278f9f6bab 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -69,14 +69,6 @@ void* taosArrayAddBatch(SArray* pArray, const void* pData, int32_t nEles); */ void taosArrayRemoveDuplicate(SArray* pArray, __compar_fn_t comparFn, void (*fp)(void*)); -/** - * - * @param pArray - * @param comparFn - * @param fp - */ -void taosArrayRemoveDuplicateP(SArray* pArray, __compar_fn_t comparFn, void (*fp)(void*)); - /** * add all element from the source array list into the destination * @param pArray @@ -252,14 +244,6 @@ void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* par int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode); void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz); -/** - * swap array - * @param a - * @param b - * @return - */ -void taosArraySwap(SArray* a, SArray* b); - #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 2f560910d4..616f69cd5b 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -601,6 +601,13 @@ static void* topicNameDup(void* p){ return taosStrdup((char*) p); } +static void freeItem(void* param) { + void* pItem = *(void**)param; + if (pItem != NULL) { + taosMemoryFree(pItem); + } +} + int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; char *msgStr = pMsg->pCont; @@ -616,7 +623,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { int32_t code = -1; SArray *pTopicList = subscribe.topicNames; taosArraySort(pTopicList, taosArrayCompareString); - taosArrayRemoveDuplicateP(pTopicList, taosArrayCompareString, taosMemoryFree); + taosArrayRemoveDuplicate(pTopicList, taosArrayCompareString, freeItem); int32_t newTopicNum = taosArrayGetSize(pTopicList); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index de2732fcb5..5210a8cc66 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -885,6 +885,12 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId, req.newConsumerId); + // kill executing task + qTaskInfo_t pTaskInfo = pHandle->execHandle.task; + if (pTaskInfo != NULL) { +// qAsyncKillTask(pTaskInfo); + } + taosWLockLatch(&pTq->lock); atomic_store_32(&pHandle->epoch, -1); @@ -895,7 +901,12 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg atomic_add_fetch_32(&pHandle->epoch, 1); if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - qStreamCloseTsdbReader(pHandle->execHandle.task); + qStreamCloseTsdbReader(pTaskInfo); + } + + // reset the error code. + if (pHandle->execHandle.task != NULL) { + } taosWUnLockLatch(&pTq->lock); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index a62101eb47..ce86a6757e 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -65,16 +65,17 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs const STqExecHandle* pExec = &pHandle->execHandle; qTaskInfo_t task = pExec->task; + int32_t vgId = TD_VID(pTq->pVnode); if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) { - tqDebug("prepare scan failed, return"); + tqDebug("prepare scan failed, return, consumer:0x%"PRIx64, pHandle->consumerId); if (pOffset->type == TMQ_OFFSET__LOG) { pRsp->rspOffset = *pOffset; return 0; } else { tqOffsetResetToLog(pOffset, pHandle->snapshotVer); if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) { - tqDebug("prepare scan failed, return"); + tqDebug("prepare scan failed, return, consumer:0x%"PRIx64, pHandle->consumerId); pRsp->rspOffset = *pOffset; return 0; } @@ -86,13 +87,14 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; - tqDebug("vgId:%d, tmq task start to execute", pTq->pVnode->config.vgId); + tqDebug("vgId:%d, tmq task start to execute, consumer:0x%"PRIx64, vgId, pHandle->consumerId); if (qExecTask(task, &pDataBlock, &ts) < 0) { - tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, terrstr()); + tqError("vgId:%d, task exec error since %s, consumer:0x%" PRIx64, vgId, terrstr(), + pHandle->consumerId); return -1; } - tqDebug("consumer:0x%"PRIx64" vgId:%d, tmq task executed, get %p", pHandle->consumerId, pTq->pVnode->config.vgId, pDataBlock); + tqDebug("consumer:0x%"PRIx64" vgId:%d, tmq task executed, get %p", pHandle->consumerId, vgId, pDataBlock); // current scan should be stopped asap, since the rebalance occurs. if (pDataBlock == NULL) { @@ -115,15 +117,16 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs } if (pRsp->rspOffset.type == 0) { - tqError("expected rsp offset: type %d %" PRId64 " %" PRId64 " %" PRId64, pRsp->rspOffset.type, pRsp->rspOffset.ts, - pRsp->rspOffset.uid, pRsp->rspOffset.version); + tqError("vgId:%d, expected rsp offset: type %d %" PRId64 " %" PRId64 " %" PRId64, vgId, pRsp->rspOffset.type, + pRsp->rspOffset.ts, pRsp->rspOffset.uid, pRsp->rspOffset.version); return -1; } if (pRsp->withTbName || pRsp->withSchema) { - tqError("get column should not with meta:%d,%d", pRsp->withTbName, pRsp->withSchema); + tqError("vgId:%d, get column should not with meta:%d,%d", vgId, pRsp->withTbName, pRsp->withSchema); return -1; } + return 0; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 67174c3267..24a26d575a 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -633,7 +633,7 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB } } -bool isTaskKilled(SExecTaskInfo* pTaskInfo) { return (0 != pTaskInfo->code) ? true : false; } +bool isTaskKilled(SExecTaskInfo* pTaskInfo) { return (0 != pTaskInfo->code);} void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) { pTaskInfo->code = rspCode; } diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 7467fa2948..6c1d3e17bb 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -140,7 +140,7 @@ void taosArrayRemoveDuplicate(SArray* pArray, __compar_fn_t comparFn, void (*fp) taosArraySet(pArray, pos + 1, p2); memset(TARRAY_GET_ELEM(pArray, i), 0, pArray->elemSize); - pos += 1; + pos += 1; } else { pos += 1; } @@ -157,45 +157,6 @@ void taosArrayRemoveDuplicate(SArray* pArray, __compar_fn_t comparFn, void (*fp) pArray->size = pos + 1; } -void taosArrayRemoveDuplicateP(SArray* pArray, __compar_fn_t comparFn, void (*fp)(void*)) { - size_t size = pArray->size; - if (size <= 1) { - return; - } - - int32_t pos = 0; - for (int32_t i = 1; i < size; ++i) { - char* p1 = taosArrayGet(pArray, pos); - char* p2 = taosArrayGet(pArray, i); - - if (comparFn(p1, p2) == 0) { - // do nothing - } else { - if (pos + 1 != i) { - void* p = taosArrayGetP(pArray, pos + 1); - if (fp != NULL) { - fp(p); - } - - taosArraySet(pArray, pos + 1, p2); - memset(TARRAY_GET_ELEM(pArray, i), 0, pArray->elemSize); - pos += 1; - } else { - pos += 1; - } - } - } - - if (fp != NULL) { - for (int32_t i = pos + 1; i < pArray->size; ++i) { - void* p = taosArrayGetP(pArray, i); - fp(p); - } - } - - pArray->size = pos + 1; -} - void* taosArrayAddAll(SArray* pArray, const SArray* pInput) { if (pInput) { return taosArrayAddBatch(pArray, pInput->pData, (int32_t)taosArrayGetSize(pInput)); @@ -392,20 +353,6 @@ void taosArrayClearEx(SArray* pArray, void (*fp)(void*)) { pArray->size = 0; } -void taosArrayClearP(SArray* pArray, FDelete fp) { - if (pArray == NULL) return; - if (fp == NULL) { - pArray->size = 0; - return; - } - - for (int32_t i = 0; i < pArray->size; ++i) { - fp(*(void**)TARRAY_GET_ELEM(pArray, i)); - } - - pArray->size = 0; -} - void* taosArrayDestroy(SArray* pArray) { if (pArray) { taosMemoryFree(pArray->pData); @@ -495,6 +442,7 @@ static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void if (pArray->size <= 1) { return; } + for (int32_t i = 1; i <= pArray->size - 1; ++i) { for (int32_t j = i; j > 0; --j) { if (fn(taosArrayGetP(pArray, j), taosArrayGetP(pArray, j - 1), param) == -1) { @@ -507,7 +455,6 @@ static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void } } } - return; } int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode) { @@ -539,21 +486,3 @@ void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* par taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param); } -void taosArraySwap(SArray* a, SArray* b) { - if (a == NULL || b == NULL) return; - size_t t = a->size; - a->size = b->size; - b->size = t; - - uint32_t cap = a->capacity; - a->capacity = b->capacity; - b->capacity = cap; - - uint32_t elem = a->elemSize; - a->elemSize = b->elemSize; - b->elemSize = elem; - - void* data = a->pData; - a->pData = b->pData; - b->pData = data; -} From c158087d66c5d4e7c19cfebc5504ad2a8d8e60a3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 28 Mar 2023 09:53:49 +0800 Subject: [PATCH 12/17] fix(tmq): kill the ongoing tsdb scans while transferring the ownership of vnode to other consumers. --- include/libs/executor/executor.h | 10 +--------- source/dnode/vnode/src/tq/tq.c | 7 +------ source/libs/executor/src/executor.c | 17 +++++++++++++++++ 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index e76422ee34..19a6407b77 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -149,7 +149,6 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table * @param handle * @return */ - int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bool* hasMore, SLocalFetch* pLocal); int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pBlock, uint64_t* useconds); @@ -162,6 +161,7 @@ void qCleanExecTaskBlockBuf(qTaskInfo_t tinfo); * @return */ int32_t qAsyncKillTask(qTaskInfo_t tinfo, int32_t rspCode); +int32_t qKillTask(qTaskInfo_t tinfo, int32_t rspCode); bool qTaskIsExecuting(qTaskInfo_t qinfo); @@ -171,14 +171,6 @@ bool qTaskIsExecuting(qTaskInfo_t qinfo); */ void qDestroyTask(qTaskInfo_t tinfo); -/** - * Extract the qualified table id list, and than pass them to the TSDB driver to load the required table data blocks. - * - * @param iter the table iterator to traverse all tables belongs to a super table, or an invert index - * @return - */ -int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t tagCondLen, SArray* pTableIdList); - void qProcessRspMsg(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet); int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, SArray* pExecInfoList /*,int32_t* resNum, SExplainExecInfo** pRes*/); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 5210a8cc66..ccd8b885a0 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -888,7 +888,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg // kill executing task qTaskInfo_t pTaskInfo = pHandle->execHandle.task; if (pTaskInfo != NULL) { -// qAsyncKillTask(pTaskInfo); + qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); } taosWLockLatch(&pTq->lock); @@ -904,11 +904,6 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg qStreamCloseTsdbReader(pTaskInfo); } - // reset the error code. - if (pHandle->execHandle.task != NULL) { - - } - taosWUnLockLatch(&pTq->lock); if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) { taosMemoryFree(req.qmsg); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 86fcd319c5..1ea2101aff 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -749,6 +749,23 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo, int32_t rspCode) { return TSDB_CODE_SUCCESS; } +int32_t qKillTask(qTaskInfo_t tinfo, int32_t rspCode) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + if (pTaskInfo == NULL) { + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + + qDebug("%s execTask async killed", GET_TASKID(pTaskInfo)); + setTaskKilled(pTaskInfo, rspCode); + + while(qTaskIsExecuting(pTaskInfo)) { + taosMsleep(10); + } + + pTaskInfo->code = rspCode; + return TSDB_CODE_SUCCESS; +} + bool qTaskIsExecuting(qTaskInfo_t qinfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo; if (NULL == pTaskInfo) { From 2198ae2060a856bdb32e305b2d85b8da296045cd Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 28 Mar 2023 10:49:49 +0800 Subject: [PATCH 13/17] fix(query): spread/elapsed using sma result for computation --- source/libs/function/src/builtins.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 4fcc9ac1ad..84de55653c 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2375,7 +2375,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "spread", .type = FUNCTION_TYPE_SPREAD, - .classification = FUNC_MGT_AGG_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED, .translateFunc = translateSpread, .dataRequiredFunc = statisDataRequired, .getEnvFunc = getSpreadFuncEnv, @@ -2417,7 +2417,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "elapsed", .type = FUNCTION_TYPE_ELAPSED, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_INTERVAL_INTERPO_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_INTERVAL_INTERPO_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED, .dataRequiredFunc = statisDataRequired, .translateFunc = translateElapsed, .getEnvFunc = getElapsedFuncEnv, From 6c1515624a1e14549d9c1392399f0039e03ce050 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 28 Mar 2023 11:24:12 +0800 Subject: [PATCH 14/17] refactor: do some internal refactor. --- include/util/tarray.h | 2 ++ source/util/src/tarray.c | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/include/util/tarray.h b/include/util/tarray.h index 278f9f6bab..4bf24b46b9 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -208,6 +208,8 @@ void taosArrayDestroyP(SArray* pArray, FDelete fp); void taosArrayDestroyEx(SArray* pArray, FDelete fp); +void taosArraySwap(SArray* a, SArray* b); + /** * sort the array * @param pArray diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 6c1d3e17bb..b15bb519e7 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -486,3 +486,21 @@ void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* par taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param); } +void taosArraySwap(SArray* a, SArray* b) { + if (a == NULL || b == NULL) return; + size_t t = a->size; + a->size = b->size; + b->size = t; + + uint32_t cap = a->capacity; + a->capacity = b->capacity; + b->capacity = cap; + + uint32_t elem = a->elemSize; + a->elemSize = b->elemSize; + b->elemSize = elem; + + void* data = a->pData; + a->pData = b->pData; + b->pData = data; +} \ No newline at end of file From c36c5253c59b83235b33d2b7e0fc39b92aa84039 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 28 Mar 2023 13:31:40 +0800 Subject: [PATCH 15/17] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5ad9276c6c..8953e4b907 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3929,8 +3929,13 @@ int32_t tsdbSetTableList(STsdbReader* pReader, const void* pTableList, int32_t n if (code) { return code; } - pReader->status.uidList.tableUidList = - (uint64_t*)taosMemoryRealloc(pReader->status.uidList.tableUidList, sizeof(uint64_t) * num); + + char* p1 = taosMemoryRealloc(pReader->status.uidList.tableUidList, sizeof(uint64_t) * num); + if (p1 == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pReader->status.uidList.tableUidList = (uint64_t*)p1; } taosHashClear(pReader->status.pTableMap); From 38b5f4ae1b545c5f55dd3640c82421d0881332e5 Mon Sep 17 00:00:00 2001 From: xiaolei li <85657333+xleili@users.noreply.github.com> Date: Tue, 28 Mar 2023 15:34:48 +0800 Subject: [PATCH 16/17] fix: taosToolsInstallDir if condition order (#20665) --- packaging/tools/makepkg.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 29160238ce..a2dec155e8 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -51,9 +51,9 @@ fi if [ -d ${top_dir}/tools/taos-tools/packaging/deb ]; then cd ${top_dir}/tools/taos-tools/packaging/deb - [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0" - + taostools_ver=$(git for-each-ref --sort=taggerdate --format '%(tag)' refs/tags|grep -v taos | tail -1) + [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0" taostools_install_dir="${release_dir}/${clientName2}Tools-${taostools_ver}" cd ${curr_dir} From b5a1246fea538de8b87e89f7de73bc5e98a2be2f Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 28 Mar 2023 16:52:54 +0800 Subject: [PATCH 17/17] fix: post script add jemalloc (#20672) * fix: tools/post.sh to ldconfig for jemalloc * fix: rpm spec for jemalloc --- packaging/rpm/tdengine.spec | 38 +++++++++----------------- packaging/tools/post.sh | 54 +++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 25 deletions(-) diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index e9d86219cb..c21063e6a4 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -90,45 +90,33 @@ cp %{_compiledir}/../include/libs/function/taosudf.h %{buildroot}%{homepat cp -r %{_compiledir}/../examples/* %{buildroot}%{homepath}/examples if [ -f %{_compiledir}/build/bin/jemalloc-config ]; then - mkdir -p %{buildroot}%{userlocalpath}/bin - mkdir -p %{buildroot}%{userlocalpath}/lib - mkdir -p %{buildroot}%{userlocalpath}/lib/pkgconfig - mkdir -p %{buildroot}%{userlocalpath}/include - mkdir -p %{buildroot}%{userlocalpath}/include/jemalloc - mkdir -p %{buildroot}%{userlocalpath}/share - mkdir -p %{buildroot}%{userlocalpath}/share/doc - mkdir -p %{buildroot}%{userlocalpath}/share/doc/jemalloc - mkdir -p %{buildroot}%{userlocalpath}/share/man - mkdir -p %{buildroot}%{userlocalpath}/share/man/man3 + mkdir -p %{buildroot}%{homepath}/jemalloc/ ||: + mkdir -p %{buildroot}%{homepath}/jemalloc/include/jemalloc/ ||: + mkdir -p %{buildroot}%{homepath}/jemalloc/lib/ ||: + mkdir -p %{buildroot}%{homepath}/jemalloc/lib/pkgconfig ||: - cp %{_compiledir}/build/bin/jemalloc-config %{buildroot}%{userlocalpath}/bin/ + cp %{_compiledir}/build/bin/jemalloc-config %{buildroot}%{homepath}/jemalloc/bin if [ -f %{_compiledir}/build/bin/jemalloc.sh ]; then - cp %{_compiledir}/build/bin/jemalloc.sh %{buildroot}%{userlocalpath}/bin/ + cp %{_compiledir}/build/bin/jemalloc.sh %{buildroot}%{homepath}/jemalloc/bin fi if [ -f %{_compiledir}/build/bin/jeprof ]; then - cp %{_compiledir}/build/bin/jeprof %{buildroot}%{userlocalpath}/bin/ + cp %{_compiledir}/build/bin/jeprof %{buildroot}%{homepath}/jemalloc/bin fi if [ -f %{_compiledir}/build/include/jemalloc/jemalloc.h ]; then - cp %{_compiledir}/build/include/jemalloc/jemalloc.h %{buildroot}%{userlocalpath}/include/jemalloc/ + cp %{_compiledir}/build/include/jemalloc/jemalloc.h %{buildroot}%{homepath}/jemalloc/include/jemalloc/ fi if [ -f %{_compiledir}/build/lib/libjemalloc.so.2 ]; then - cp %{_compiledir}/build/lib/libjemalloc.so.2 %{buildroot}%{userlocalpath}/lib/ - ln -sf libjemalloc.so.2 %{buildroot}%{userlocalpath}/lib/libjemalloc.so + cp %{_compiledir}/build/lib/libjemalloc.so.2 %{buildroot}%{homepath}/jemalloc/lib + ln -sf libjemalloc.so.2 %{buildroot}%{homepath}/jemalloc/lib/libjemalloc.so fi if [ -f %{_compiledir}/build/lib/libjemalloc.a ]; then - cp %{_compiledir}/build/lib/libjemalloc.a %{buildroot}%{userlocalpath}/lib/ + cp %{_compiledir}/build/lib/libjemalloc.a %{buildroot}%{homepath}/jemalloc/lib fi if [ -f %{_compiledir}/build/lib/libjemalloc_pic.a ]; then - cp %{_compiledir}/build/lib/libjemalloc_pic.a %{buildroot}%{userlocalpath}/lib/ + cp %{_compiledir}/build/lib/libjemalloc_pic.a %{buildroot}%{homepath}/jemalloc/lib fi if [ -f %{_compiledir}/build/lib/pkgconfig/jemalloc.pc ]; then - cp %{_compiledir}/build/lib/pkgconfig/jemalloc.pc %{buildroot}%{userlocalpath}/lib/pkgconfig/ - fi - if [ -f %{_compiledir}/build/share/doc/jemalloc/jemalloc.html ]; then - cp %{_compiledir}/build/share/doc/jemalloc/jemalloc.html %{buildroot}%{userlocalpath}/share/doc/jemalloc/ - fi - if [ -f %{_compiledir}/build/share/man/man3/jemalloc.3 ]; then - cp %{_compiledir}/build/share/man/man3/jemalloc.3 %{buildroot}%{userlocalpath}/share/man/man3/ + cp %{_compiledir}/build/lib/pkgconfig/jemalloc.pc %{buildroot}%{homepath}/jemalloc/lib/pkgconfig fi fi diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index c9fab51b28..35c28feb6a 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -145,6 +145,59 @@ function install_include() { log_print "install include success" } +function install_jemalloc() { + jemalloc_dir=${script_dir}/../jemalloc + + if [ -d ${jemalloc_dir} ]; then + ${csudo}/usr/bin/install -c -d /usr/local/bin + + if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jeprof ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin + fi + if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then + ${csudo}/usr/bin/install -c -d /usr/local/include/jemalloc + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then + ${csudo}/usr/bin/install -c -d /usr/local/lib + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib + ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo}/usr/bin/install -c -d /usr/local/lib + if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo}/usr/bin/install -c -d /usr/local/lib/pkgconfig + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig + fi + fi + if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then + ${csudo}/usr/bin/install -c -d /usr/local/share/doc/jemalloc + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc + fi + if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then + ${csudo}/usr/bin/install -c -d /usr/local/share/man/man3 + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 + fi + + if [ -d /etc/ld.so.conf.d ]; then + echo "/usr/local/lib" | ${csudo}tee /etc/ld.so.conf.d/jemalloc.conf >/dev/null || echo -e "failed to write /etc/ld.so.conf.d/jemalloc.conf" + ${csudo}ldconfig + else + echo "/etc/ld.so.conf.d not found!" + fi + fi +} + function install_lib() { log_print "start install lib from ${lib_dir} to ${lib_link_dir}" ${csudo}rm -f ${lib_link_dir}/libtaos* || : @@ -663,6 +716,7 @@ function install_TDengine() { # Install include, lib, binary and service install_include && install_lib && + install_jemalloc install_bin if [[ "$?" != 0 ]];then