From ccffe94579d3495b6d4e825b308c4358f8f98058 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 26 Apr 2022 18:46:08 +0800 Subject: [PATCH 1/5] test: add client config for test cases --- tests/pytest/util/dnodes.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index c3c2424397..2c50a62117 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -63,7 +63,7 @@ class TDSimClient: if os.system(cmd) != 0: tdLog.exit(cmd) - def deploy(self): + def deploy(self, *updatecfgDict): self.logDir = "%s/sim/psim/log" % (self.path) self.cfgDir = "%s/sim/psim/cfg" % (self.path) self.cfgPath = "%s/sim/psim/cfg/taos.cfg" % (self.path) @@ -95,6 +95,12 @@ class TDSimClient: for key, value in self.cfgDict.items(): self.cfg(key, value) + + if bool(updatecfgDict) and updatecfgDict[0] and updatecfgDict[0][0]: + print(updatecfgDict[0][0]) + clientCfg = dict (updatecfgDict[0][0].get('clientCfg')) + for key, value in clientCfg.items(): + self.cfg(key, value) tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath)) @@ -214,9 +220,11 @@ class TDDnode: # self.cfg("logDir",self.logDir) # print(updatecfgDict) isFirstDir = 1 - if updatecfgDict[0] and updatecfgDict[0][0]: + if bool(updatecfgDict) and updatecfgDict[0] and updatecfgDict[0][0]: print(updatecfgDict[0][0]) for key, value in updatecfgDict[0][0].items(): + if key == "clientCfg": + continue if value == 'dataDir': if isFirstDir: self.cfgDict.pop('dataDir') @@ -491,7 +499,7 @@ class TDDnodes: self.sim.setTestCluster(self.testCluster) if (self.simDeployed == False): - self.sim.deploy() + self.sim.deploy(updatecfgDict) self.simDeployed = True self.check(index) From 48e379f32048488cc3c46564c610e4467ebe2ac0 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 27 Apr 2022 09:48:33 +0800 Subject: [PATCH 2/5] fix(shell): memory init error. --- contrib/CMakeLists.txt | 26 +++++++++++++------------- source/dnode/mgmt/exe/dmMain.c | 4 ++-- source/libs/function/src/tudf.c | 2 +- source/util/src/tconfig.c | 12 ++++++------ 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 19923a5ad6..6a290dda15 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -110,7 +110,7 @@ execute_process(COMMAND "${CMAKE_COMMAND}" --build . # ================================================================================================ # googletest if(${BUILD_TEST}) - add_subdirectory(googletest) + add_subdirectory(googletest EXCLUDE_FROM_ALL) target_include_directories( gtest PUBLIC $ @@ -143,7 +143,7 @@ set(CMAKE_PROJECT_INCLUDE_BEFORE "${TD_SUPPORT_DIR}/EnableCMP0048.txt.in") option(ENABLE_CJSON_TEST "Enable building cJSON test" OFF) option(CJSON_OVERRIDE_BUILD_SHARED_LIBS "Override BUILD_SHARED_LIBS with CJSON_BUILD_SHARED_LIBS" ON) option(CJSON_BUILD_SHARED_LIBS "Overrides BUILD_SHARED_LIBS if CJSON_OVERRIDE_BUILD_SHARED_LIBS is enabled" OFF) -add_subdirectory(cJson) +add_subdirectory(cJson EXCLUDE_FROM_ALL) target_include_directories( cjson # see https://stackoverflow.com/questions/25676277/cmake-target-include-directories-prints-an-error-when-i-try-to-add-the-source @@ -152,7 +152,7 @@ target_include_directories( unset(CMAKE_PROJECT_INCLUDE_BEFORE) # lz4 -add_subdirectory(lz4/build/cmake) +add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL) target_include_directories( lz4_static PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/lz4/lib @@ -160,7 +160,7 @@ target_include_directories( # zlib set(CMAKE_PROJECT_INCLUDE_BEFORE "${TD_SUPPORT_DIR}/EnableCMP0048.txt.in") -add_subdirectory(zlib) +add_subdirectory(zlib EXCLUDE_FROM_ALL) target_include_directories( zlibstatic PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib @@ -176,7 +176,7 @@ unset(CMAKE_PROJECT_INCLUDE_BEFORE) # leveldb if(${BUILD_WITH_LEVELDB}) option(LEVELDB_BUILD_TESTS "" OFF) - add_subdirectory(leveldb) + add_subdirectory(leveldb EXCLUDE_FROM_ALL) target_include_directories( leveldb PUBLIC $ @@ -192,7 +192,7 @@ if(${BUILD_WITH_ROCKSDB}) option(WITH_TOOLS "" OFF) option(WITH_LIBURING "" OFF) option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF) - add_subdirectory(rocksdb) + add_subdirectory(rocksdb EXCLUDE_FROM_ALL) target_include_directories( rocksdb PUBLIC $ @@ -203,7 +203,7 @@ endif(${BUILD_WITH_ROCKSDB}) # To support build on ubuntu: sudo apt-get install libboost-all-dev if(${BUILD_WITH_LUCENE}) option(ENABLE_TEST "Enable the tests" OFF) - add_subdirectory(lucene) + add_subdirectory(lucene EXCLUDE_FROM_ALL) target_include_directories( lucene++ PUBLIC $ @@ -213,14 +213,14 @@ endif(${BUILD_WITH_LUCENE}) # NuRaft if(${BUILD_WITH_NURAFT}) - add_subdirectory(nuraft) + add_subdirectory(nuraft EXCLUDE_FROM_ALL) endif(${BUILD_WITH_NURAFT}) # pthread if(${BUILD_PTHREAD}) set(CMAKE_BUILD_TYPE release) add_definitions(-DPTW32_STATIC_LIB) - add_subdirectory(pthread) + add_subdirectory(pthread EXCLUDE_FROM_ALL) set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread) add_library(pthread STATIC IMPORTED GLOBAL) SET_PROPERTY(TARGET pthread PROPERTY IMPORTED_LOCATION ${LIBRARY_OUTPUT_PATH}/pthread.lib) @@ -228,12 +228,12 @@ endif() # iconv if(${BUILD_WITH_ICONV}) - add_subdirectory(iconv) + add_subdirectory(iconv EXCLUDE_FROM_ALL) endif(${BUILD_WITH_ICONV}) # wingetopt if(${BUILD_WINGETOPT}) - add_subdirectory(wingetopt) + add_subdirectory(wingetopt EXCLUDE_FROM_ALL) endif(${BUILD_WINGETOPT}) # msvcregex @@ -293,7 +293,7 @@ if(${BUILD_WITH_UV}) MESSAGE("Windows need set no-sign-compare") add_compile_options(-Wno-sign-compare) endif () - add_subdirectory(libuv) + add_subdirectory(libuv EXCLUDE_FROM_ALL) endif(${BUILD_WITH_UV}) # BDB @@ -334,5 +334,5 @@ endif(${BUILD_WITH_SQLITE}) # Build test # ================================================================================================ if(${BUILD_DEPENDENCY_TESTS}) - add_subdirectory(test) + add_subdirectory(test EXCLUDE_FROM_ALL) endif(${BUILD_DEPENDENCY_TESTS}) diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 6d9ead4c87..4a2d02d25d 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -69,8 +69,8 @@ static void dmSetSignalHandle() { static int32_t dmParseArgs(int32_t argc, char const *argv[]) { int32_t cmdEnvIndex = 0; if (argc < 2) return 0; - global.envCmd = taosMemoryMalloc(argc-1); - memset(global.envCmd, 0, argc-1); + global.envCmd = taosMemoryMalloc((argc-1)*sizeof(char*)); + memset(global.envCmd, 0, (argc-1)*sizeof(char*)); for (int32_t i = 1; i < argc; ++i) { if (strcmp(argv[i], "-c") == 0) { if (i < argc - 1) { diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index f8a7e77814..f4a724afa3 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -213,7 +213,7 @@ enum { int32_t getUdfdPipeName(char* pipeName, int32_t size) { char dnodeId[8] = {0}; - size_t dnodeIdSize; + size_t dnodeIdSize = sizeof(dnodeId); int32_t err = uv_os_getenv(UDF_DNODE_ID_ENV_NAME, dnodeId, &dnodeIdSize); if (err != 0) { dnodeId[0] = '1'; diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index b864f48522..cb9ef87701 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -823,7 +823,7 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { } p++; - if (memcmp(url, "jsonFile", 8) == 0) { + if (strncmp(url, "jsonFile", 8) == 0) { char *filepath = p; if (!taosCheckExistFile(filepath)) { uError("fial to load json file: %s", filepath); @@ -893,8 +893,8 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { } tjsonDelete(pJson); - // } else if (memcmp(url, "jsonUrl", 7) == 0) { - // } else if (memcmp(url, "etcdUrl", 7) == 0) { + // } else if (strncmp(url, "jsonUrl", 7) == 0) { + // } else if (strncmp(url, "etcdUrl", 7) == 0) { } else { uError("Unsupported url: %s", url); return -1; @@ -908,7 +908,7 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char* apolloUrl int32_t index = 0; if (envCmd == NULL) return 0; while (envCmd[index]!=NULL) { - if (memcmp(envCmd[index], "TAOS_APOLLO_URL", 14) == 0) { + if (strncmp(envCmd[index], "TAOS_APOLLO_URL", 14) == 0) { char *p = strchr(envCmd[index], '='); if (p != NULL) { p++; @@ -934,7 +934,7 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char* apolloUrl break; } if(line[_bytes - 1] == '\n') line[_bytes - 1] = 0; - if (memcmp(line, "TAOS_APOLLO_URL", 14) == 0) { + if (strncmp(line, "TAOS_APOLLO_URL", 14) == 0) { char *p = strchr(line, '='); if (p != NULL) { p++; @@ -975,7 +975,7 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char* apolloUrl break; } if(line[_bytes - 1] == '\n') line[_bytes - 1] = 0; - if (memcmp(line, "TAOS_APOLLO_URL", 14) == 0) { + if (strncmp(line, "TAOS_APOLLO_URL", 14) == 0) { char *p = strchr(line, '='); if (p != NULL) { p++; From d4b87d903978318e15d610ccc5bcc03b83884a3a Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 27 Apr 2022 09:54:50 +0800 Subject: [PATCH 3/5] fix(shell): memory init error. --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 72ff6179db..494b302165 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -87,6 +87,6 @@ # ./test.sh -f tsim/sma/tsmaCreateInsertData.sim # --- valgrind -#./test.sh -f tsim/valgrind/checkError.sim -v +./test.sh -f tsim/valgrind/checkError.sim -v #======================b1-end=============== From c6cc7eff8609c07d7cd79009610ecc36ce2aee97 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 27 Apr 2022 10:02:22 +0800 Subject: [PATCH 4/5] test: add client config for test cases --- tests/pytest/util/dnodes.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 2c50a62117..8a29a0a7a6 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -96,11 +96,13 @@ class TDSimClient: for key, value in self.cfgDict.items(): self.cfg(key, value) - if bool(updatecfgDict) and updatecfgDict[0] and updatecfgDict[0][0]: - print(updatecfgDict[0][0]) - clientCfg = dict (updatecfgDict[0][0].get('clientCfg')) - for key, value in clientCfg.items(): - self.cfg(key, value) + try: + if bool(updatecfgDict) and updatecfgDict[0] and updatecfgDict[0][0]: + clientCfg = dict (updatecfgDict[0][0].get('clientCfg')) + for key, value in clientCfg.items(): + self.cfg(key, value) + except Exception: + pass tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath)) From 02cee2ba62a0fe9f4e35f12f12f5864b106649a4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 27 Apr 2022 10:11:32 +0800 Subject: [PATCH 5/5] refactor: do some internal refactor. --- source/common/src/tdatablock.c | 15 ++++---- source/libs/executor/src/executorimpl.c | 50 +++++++++++++++++-------- 2 files changed, 43 insertions(+), 22 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index d6a6ba5b13..63e009ed0a 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -117,22 +117,23 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con int32_t type = pColumnInfoData->info.type; if (IS_VAR_DATA_TYPE(type)) { int32_t dataLen = varDataTLen(pData); - if(type == TSDB_DATA_TYPE_JSON) { - if(*pData == TSDB_DATA_TYPE_NULL) { + if (type == TSDB_DATA_TYPE_JSON) { + if (*pData == TSDB_DATA_TYPE_NULL) { dataLen = 0; - }else if(*pData == TSDB_DATA_TYPE_NCHAR) { - dataLen = varDataTLen(pData+CHAR_BYTES); - }else if(*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { + } else if (*pData == TSDB_DATA_TYPE_NCHAR) { + dataLen = varDataTLen(pData + CHAR_BYTES); + } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { dataLen = LONG_BYTES; - }else if(*pData == TSDB_DATA_TYPE_BOOL) { + } else if (*pData == TSDB_DATA_TYPE_BOOL) { dataLen = CHAR_BYTES; } dataLen += CHAR_BYTES; } + SVarColAttr* pAttr = &pColumnInfoData->varmeta; if (pAttr->allocLen < pAttr->length + dataLen) { uint32_t newSize = pAttr->allocLen; - if (newSize == 0) { + if (newSize <= 1) { newSize = 8; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 318d87e100..44c46785cc 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1034,7 +1034,7 @@ void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlo } } -static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunctParam* pFuncParam, int32_t type, +static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunctParam* pFuncParam, int32_t paramIndex, int32_t numOfRows) { SColumnInfoData* pColInfo = NULL; if (pInput->pData[paramIndex] == NULL) { @@ -1044,17 +1044,17 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc } // Set the correct column info (data type and bytes) - pColInfo->info.type = type; - pColInfo->info.bytes = tDataTypes[type].bytes; + pColInfo->info.type = pFuncParam->param.nType; + pColInfo->info.bytes = pFuncParam->param.nLen; pInput->pData[paramIndex] = pColInfo; } else { pColInfo = pInput->pData[paramIndex]; } - ASSERT(!IS_VAR_DATA_TYPE(type)); colInfoDataEnsureCapacity(pColInfo, 0, numOfRows); + int8_t type = pFuncParam->param.nType; if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_UBIGINT) { int64_t v = pFuncParam->param.i; for (int32_t i = 0; i < numOfRows; ++i) { @@ -1065,6 +1065,12 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc for (int32_t i = 0; i < numOfRows; ++i) { colDataAppendDouble(pColInfo, i, &v); } + } else if (type == TSDB_DATA_TYPE_VARCHAR) { + char *tmp = taosMemoryMalloc(pFuncParam->param.nLen + VARSTR_HEADER_SIZE); + STR_WITH_SIZE_TO_VARSTR(tmp, pFuncParam->param.pz, pFuncParam->param.nLen); + for(int32_t i = 0; i < numOfRows; ++i) { + colDataAppend(pColInfo, i, tmp, false); + } } return TSDB_CODE_SUCCESS; @@ -1104,7 +1110,7 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt pInput->numOfRows = pBlock->info.rows; pInput->startRowIndex = 0; - code = doCreateConstantValColumnInfo(pInput, pFuncParam, pFuncParam->param.nType, j, pBlock->info.rows); + code = doCreateConstantValColumnInfo(pInput, pFuncParam, j, pBlock->info.rows); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -6684,18 +6690,32 @@ SArray* extractColumnInfo(SNodeList* pNodeList) { for (int32_t i = 0; i < numOfCols; ++i) { STargetNode* pNode = (STargetNode*)nodesListGetNode(pNodeList, i); - SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; - // todo extract method - SColumn c = {0}; - c.slotId = pColNode->slotId; - c.colId = pColNode->colId; - c.type = pColNode->node.resType.type; - c.bytes = pColNode->node.resType.bytes; - c.precision = pColNode->node.resType.precision; - c.scale = pColNode->node.resType.scale; + if (nodeType(pNode->pExpr) == QUERY_NODE_COLUMN) { + SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; - taosArrayPush(pList, &c); + // todo extract method + SColumn c = {0}; + c.slotId = pColNode->slotId; + c.colId = pColNode->colId; + c.type = pColNode->node.resType.type; + c.bytes = pColNode->node.resType.bytes; + c.scale = pColNode->node.resType.scale; + c.precision = pColNode->node.resType.precision; + + taosArrayPush(pList, &c); + } else if (nodeType(pNode->pExpr) == QUERY_NODE_VALUE) { + SValueNode* pValNode = (SValueNode*) pNode->pExpr; + SColumn c = {0}; + c.slotId = pNode->slotId; + c.colId = pNode->slotId; + c.type = pValNode->node.type; + c.bytes = pValNode->node.resType.bytes; + c.scale = pValNode->node.resType.scale; + c.precision = pValNode->node.resType.precision; + + taosArrayPush(pList, &c); + } } return pList;