From 30bfd495e6356f1a5311bdb5113c24b21b2e4353 Mon Sep 17 00:00:00 2001 From: tomchon Date: Wed, 7 Jul 2021 20:25:10 +0800 Subject: [PATCH 001/106] modify release scripts and change version number --- cmake/version.inc | 2 +- packaging/docker/dockerManifest.sh | 43 ++++++++++++++----- packaging/docker/dockerbuild.sh | 67 +++++++++++++++++++++++------- snap/snapcraft.yaml | 4 +- 4 files changed, 88 insertions(+), 28 deletions(-) diff --git a/cmake/version.inc b/cmake/version.inc index 7e556ec225..8ef5640a2a 100755 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.0.20.8") + SET(TD_VER_NUMBER "2.0.20.9") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/packaging/docker/dockerManifest.sh b/packaging/docker/dockerManifest.sh index c846e8345d..598bd46f13 100755 --- a/packaging/docker/dockerManifest.sh +++ b/packaging/docker/dockerManifest.sh @@ -5,22 +5,28 @@ set -e # dockerbuild.sh # -n [version number] # -p [xxxx] +# -V [stable | beta] # set parameters by default value -verNumber="" +version="" passWord="" +verType="" -while getopts "hn:p:" arg +while getopts "hn:p:V:" arg do case $arg in n) - #echo "verNumber=$OPTARG" - verNumber=$(echo $OPTARG) + #echo "version=$OPTARG" + version=$(echo $OPTARG) ;; p) #echo "passWord=$OPTARG" passWord=$(echo $OPTARG) ;; + V) + #echo "verType=$OPTARG" + verType=$(echo $OPTARG) + ;; h) echo "Usage: `basename $0` -n [version number] " echo " -p [password for docker hub] " @@ -33,13 +39,30 @@ do esac done -echo "verNumber=${verNumber}" +echo "version=${version}" -#docker manifest create -a tdengine/tdengine:${verNumber} tdengine/tdengine-amd64:${verNumber} tdengine/tdengine-aarch64:${verNumber} tdengine/tdengine-aarch32:${verNumber} -docker manifest create -a tdengine/tdengine:latest tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest +#docker manifest rm tdengine/tdengine +#docker manifest rm tdengine/tdengine:${version} +if [ "$verType" == "beta" ]; then + docker manifest create -a tdengine/tdengine-beta:${version} tdengine/tdengine-amd64-beta:${version} tdengine/tdengine-aarch64-beta:${version} tdengine/tdengine-aarch32-beta:${version} + docker manifest create -a tdengine/tdengine-beta:latest tdengine/tdengine-amd64-beta:latest tdengine/tdengine-aarch64-beta:latest tdengine/tdengine-aarch32-beta:latest + docker login -u tdengine -p ${passWord} #replace the docker registry username and password + docker manifest push tdengine/tdengine-beta:latest +elif [ "$verType" == "stable" ]; then + docker manifest create -a tdengine/tdengine:${version} tdengine/tdengine-amd64:${version} tdengine/tdengine-aarch64:${version} tdengine/tdengine-aarch32:${version} + docker manifest create -a tdengine/tdengine:latest tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest + docker login -u tdengine -p ${passWord} #replace the docker registry username and password + docker manifest push tdengine/tdengine:latest +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi -docker login -u tdengine -p ${passWord} #replace the docker registry username and password +# docker manifest create -a tdengine/${dockername}:${version} tdengine/tdengine-amd64:${version} tdengine/tdengine-aarch64:${version} tdengine/tdengine-aarch32:${version} +# docker manifest create -a tdengine/${dockername}:latest tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest -docker manifest push tdengine/tdengine:latest +# docker login -u tdengine -p ${passWord} #replace the docker registry username and password -# how set latest version ??? +# docker manifest push tdengine/tdengine:latest + +# # how set latest version ??? diff --git a/packaging/docker/dockerbuild.sh b/packaging/docker/dockerbuild.sh index b7991465b0..76717d5892 100755 --- a/packaging/docker/dockerbuild.sh +++ b/packaging/docker/dockerbuild.sh @@ -1,20 +1,24 @@ #!/bin/bash +# + set -e #set -x # dockerbuild.sh # -c [aarch32 | aarch64 | amd64 | x86 | mips64 ...] -# -f [pkg file] # -n [version number] # -p [password for docker hub] +# -V [stable | beta] +# -f [pkg file] # set parameters by default value -cpuType=amd64 -verNumber="" +cpuType="" +version="" passWord="" pkgFile="" +verType="" -while getopts "hc:n:p:f:" arg +while getopts "hc:n:p:f:V:" arg do case $arg in c) @@ -22,8 +26,8 @@ do cpuType=$(echo $OPTARG) ;; n) - #echo "verNumber=$OPTARG" - verNumber=$(echo $OPTARG) + #echo "version=$OPTARG" + version=$(echo $OPTARG) ;; p) #echo "passWord=$OPTARG" @@ -33,11 +37,17 @@ do #echo "pkgFile=$OPTARG" pkgFile=$(echo $OPTARG) ;; + V) + #echo "verType=$OPTARG" + verType=$(echo $OPTARG) + ;; h) - echo "Usage: `basename $0` -c [aarch32 | aarch64 | amd64 | x86 | mips64 ...] " - echo " -f [pkg file] " + echo "Usage: `basename $0` -c [aarch32 | aarch64 | amd64 | x86 | mips64 ...] " echo " -n [version number] " echo " -p [password for docker hub] " + echo " -V [stable | beta] " + echo " -f [pkg file] " + exit 0 ;; ?) #unknow option @@ -47,17 +57,44 @@ do esac done -echo "cpuType=${cpuType} verNumber=${verNumber} pkgFile=${pkgFile} " +# if [ "$verType" == "beta" ]; then +# pkgFile=TDengine-server-${version}-Linux-${cpuType}-${verType}.tar.gz +# elif [ "$verType" == "stable" ]; then +# pkgFile=TDengine-server-${version}-Linux-${cpuType}.tar.gz +# else +# echo "unknow verType, nor stabel or beta" +# exit 1 + +if [ "$verType" == "beta" ]; then + dockername=${cpuType}-${verType} +elif [ "$verType" == "stable" ]; then + dockername=${cpuType} +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + + +echo "cpuType=${cpuType} version=${version} pkgFile=${pkgFile} verType=${verType} " echo "$(pwd)" echo "====NOTES: ${pkgFile} must be in the same directory as dockerbuild.sh====" -dirName=${pkgFile%-Linux*} -#echo "dirName=${dirName}" +scriptDir=$(dirname $(readlink -f $0)) +comunityArchiveDir=/nas/TDengine/v$version/community # community version’package directory +cd ${scriptDir} +cp -f ${comunityArchiveDir}/${pkgFile} . -docker build --rm -f "Dockerfile" -t tdengine/tdengine-${cpuType}:${verNumber} "." --build-arg pkgFile=${pkgFile} --build-arg dirName=${dirName} +dirName=${pkgFile%-Linux*} +echo "dirName=${dirName}" + + +docker build --rm -f "Dockerfile" -t tdengine/tdengine-${dockername}:${version} "." --build-arg pkgFile=${pkgFile} --build-arg dirName=${dirName} docker login -u tdengine -p ${passWord} #replace the docker registry username and password -docker push tdengine/tdengine-${cpuType}:${verNumber} +docker push tdengine/tdengine-${dockername}:${version} # set this version to latest version -docker tag tdengine/tdengine-${cpuType}:${verNumber} tdengine/tdengine-${cpuType}:latest -docker push tdengine/tdengine-${cpuType}:latest +docker tag tdengine/tdengine-${dockername}:${version} tdengine/tdengine-${dockername}:latest +docker push tdengine/tdengine-${dockername}:latest + + +rm -f ${pkgFile} \ No newline at end of file diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 39538e39ab..16fd9145fa 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: tdengine base: core18 -version: '2.0.20.8' +version: '2.0.20.9' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | @@ -72,7 +72,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.0.20.5 + - usr/lib/libtaos.so.2.0.20.9 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so From 1e021fd2542f283955d763b5941cbec8e38e1528 Mon Sep 17 00:00:00 2001 From: tomchon Date: Wed, 7 Jul 2021 20:27:18 +0800 Subject: [PATCH 002/106] modify release scripts and change version number --- alert/release.sh | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/alert/release.sh b/alert/release.sh index 35eb4d677f..77f6798480 100755 --- a/alert/release.sh +++ b/alert/release.sh @@ -7,8 +7,9 @@ set -e cpuType=amd64 # [armv6l | arm64 | amd64 | 386] osType=linux # [linux | darwin | windows] version="" +verType=stable # [stable, beta] declare -A archMap=(["armv6l"]="arm" ["arm64"]="arm64" ["amd64"]="x64" ["386"]="x86") -while getopts "h:c:o:n:" arg +while getopts "h:c:o:n:V:" arg do case $arg in c) @@ -23,6 +24,10 @@ do #echo "version=$OPTARG" version=$(echo $OPTARG) ;; + V) + #echo "verType=$OPTARG" + verType=$(echo $OPTARG) + ;; h) echo "Usage: `basename $0` -c [armv6l | arm64 | amd64 | 386] -o [linux | darwin | windows]" exit 0 @@ -55,6 +60,22 @@ cp alert alert.cfg install_driver.sh ./TDengine-alert/. cp ../../../debug/build/lib/libtaos.so.${version} ./TDengine-alert/driver/. chmod 777 ./TDengine-alert/install_driver.sh -tar -I 'gzip -9' -cf ${startdir}/TDengine-alert-${version}-${osType^}-${archMap[${cpuType}]}.tar.gz TDengine-alert/ +tar -I 'gzip -9' -cf ${scriptdir}/TDengine-alert-${version}-${osType^}-${archMap[${cpuType}]}.tar.gz TDengine-alert/ rm -rf ./TDengine-alert + + +# mv package to comminuty/release/ +pkg_name=TDengine-alert-${version}-${osType^}-${archMap[${cpuType}]} + +if [ "$verType" == "beta" ]; then + pkg_name=${pkg_name}-${verType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + +cd ${scriptdir}/../release/ +mv ${scriptdir}/TDengine-alert-${version}-${osType^}-${archMap[${cpuType}]}.tar.gz ${pkg_name}.tar.gz From 76985feccbdad754d0e1e3edfbfb4dd8b91ed37d Mon Sep 17 00:00:00 2001 From: tomchon Date: Sun, 11 Jul 2021 19:39:16 +0800 Subject: [PATCH 003/106] modify docker_manifest.sh --- packaging/docker/dockerManifest.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packaging/docker/dockerManifest.sh b/packaging/docker/dockerManifest.sh index 598bd46f13..8bf449c8f2 100755 --- a/packaging/docker/dockerManifest.sh +++ b/packaging/docker/dockerManifest.sh @@ -48,11 +48,13 @@ if [ "$verType" == "beta" ]; then docker manifest create -a tdengine/tdengine-beta:latest tdengine/tdengine-amd64-beta:latest tdengine/tdengine-aarch64-beta:latest tdengine/tdengine-aarch32-beta:latest docker login -u tdengine -p ${passWord} #replace the docker registry username and password docker manifest push tdengine/tdengine-beta:latest + docker manifest push tdengine/tdengine-beta:${version} elif [ "$verType" == "stable" ]; then docker manifest create -a tdengine/tdengine:${version} tdengine/tdengine-amd64:${version} tdengine/tdengine-aarch64:${version} tdengine/tdengine-aarch32:${version} docker manifest create -a tdengine/tdengine:latest tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest docker login -u tdengine -p ${passWord} #replace the docker registry username and password docker manifest push tdengine/tdengine:latest + docker manifest push tdengine/tdengine:${version} else echo "unknow verType, nor stabel or beta" exit 1 From e6842e22569eab4213c6e93f2476669b5ffeddad Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 16 Jul 2021 18:55:56 +0800 Subject: [PATCH 004/106] [td-5278]: enable the show queries display the projection query on super tables. --- src/client/src/tscUtil.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 89749202ef..720327e327 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2437,7 +2437,9 @@ void tscDoQuery(SSqlObj* pSql) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); uint16_t type = pQueryInfo->type; - if ((pCmd->command == TSDB_SQL_SELECT) && (!TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_SUBQUERY)) && (!TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_STABLE_SUBQUERY))) { + if ((pCmd->command == TSDB_SQL_SELECT) && (((!TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_SUBQUERY)) && + (!TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_STABLE_SUBQUERY))) || + (tscIsProjectionQuery(pQueryInfo)))) { tscAddIntoSqlList(pSql); } From 98c026fd7c1e521c50c235f49d0ad1c5ba37c369 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 17 Jul 2021 17:53:48 +0800 Subject: [PATCH 005/106] Hotfix/sangshuduo/td 5302 taosdemo stmt interlace for master (#6900) * [TD-5302]:taosdemo stmt interlace. * fix the logic interlace rows compare insert rows. --- src/kit/taosdemo/taosdemo.c | 160 +++++++++++++++++++++++++----------- 1 file changed, 111 insertions(+), 49 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 8b3f5743de..e35e54751f 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -106,9 +106,9 @@ enum TEST_MODE { typedef enum CREATE_SUB_TALBE_MOD_EN { - PRE_CREATE_SUBTBL, - AUTO_CREATE_SUBTBL, - NO_CREATE_SUBTBL + PRE_CREATE_SUBTBL, + AUTO_CREATE_SUBTBL, + NO_CREATE_SUBTBL } CREATE_SUB_TALBE_MOD_EN; typedef enum TALBE_EXISTS_EN { @@ -637,7 +637,7 @@ static FILE * g_fpOfInsertResult = NULL; #define performancePrint(fmt, ...) \ do { if (g_args.performance_print) \ - fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0) + fprintf(stderr, "PERF: "fmt, __VA_ARGS__); } while(0) #define errorPrint(fmt, ...) \ do { fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); } while(0) @@ -3184,8 +3184,10 @@ static void createChildTables() { if (g_Dbs.db[i].superTblCount > 0) { // with super table for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { - if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) - || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) { + if ((AUTO_CREATE_SUBTBL + == g_Dbs.db[i].superTbls[j].autoCreateTable) + || (TBL_ALREADY_EXISTS + == g_Dbs.db[i].superTbls[j].childTblExists)) { continue; } verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, @@ -4157,6 +4159,22 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } */ + cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows"); + if (insertRows && insertRows->type == cJSON_Number) { + if (insertRows->valueint < 0) { + errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint; + } else if (!insertRows) { + g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF; + } else { + errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + cJSON* stbInterlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows"); if (stbInterlaceRows && stbInterlaceRows->type == cJSON_Number) { if (stbInterlaceRows->valueint < 0) { @@ -4165,15 +4183,15 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } g_Dbs.db[i].superTbls[j].interlaceRows = stbInterlaceRows->valueint; - // rows per table need be less than insert batch - if (g_Dbs.db[i].superTbls[j].interlaceRows > g_args.num_of_RPR) { - printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %u > num_of_records_per_req %u\n\n", - i, j, g_Dbs.db[i].superTbls[j].interlaceRows, - g_args.num_of_RPR); - printf(" interlace rows value will be set to num_of_records_per_req %u\n\n", - g_args.num_of_RPR); - prompt(); - g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR; + + if (g_Dbs.db[i].superTbls[j].interlaceRows > g_Dbs.db[i].superTbls[j].insertRows) { + printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %u > insert_rows %"PRId64"\n\n", + i, j, g_Dbs.db[i].superTbls[j].interlaceRows, + g_Dbs.db[i].superTbls[j].insertRows); + printf(" interlace rows value will be set to insert_rows %"PRId64"\n\n", + g_Dbs.db[i].superTbls[j].insertRows); + prompt(); + g_Dbs.db[i].superTbls[j].interlaceRows = g_Dbs.db[i].superTbls[j].insertRows; } } else if (!stbInterlaceRows) { g_Dbs.db[i].superTbls[j].interlaceRows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req @@ -4210,22 +4228,6 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } - cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows"); - if (insertRows && insertRows->type == cJSON_Number) { - if (insertRows->valueint < 0) { - errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint; - } else if (!insertRows) { - g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF; - } else { - errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - cJSON* insertInterval = cJSON_GetObjectItem(stbInfo, "insert_interval"); if (insertInterval && insertInterval->type == cJSON_Number) { g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint; @@ -5152,12 +5154,18 @@ static int32_t generateDataTailWithoutStb( char **data_type = g_args.datatype; int lenOfBinary = g_args.len_of_binary; - retLen = generateData(data, data_type, - startTime + getTSRandTail( - (int64_t) DEFAULT_TIMESTAMP_STEP, k, - g_args.disorderRatio, - g_args.disorderRange), - lenOfBinary); + if (g_args.disorderRatio) { + retLen = generateData(data, data_type, + startTime + getTSRandTail( + (int64_t) DEFAULT_TIMESTAMP_STEP, k, + g_args.disorderRatio, + g_args.disorderRange), + lenOfBinary); + } else { + retLen = generateData(data, data_type, + startTime + (int64_t) (DEFAULT_TIMESTAMP_STEP* k), + lenOfBinary); + } if (len > remainderBufLen) break; @@ -5209,14 +5217,14 @@ static int32_t generateStbDataTail( bool tsRand; if (0 == strncasecmp(superTblInfo->dataSource, "rand", strlen("rand"))) { - tsRand = true; + tsRand = true; } else { - tsRand = false; + tsRand = false; } verbosePrint("%s() LN%d batch=%u buflen=%"PRId64"\n", __func__, __LINE__, batch, remainderBufLen); - int32_t k = 0; + int32_t k; for (k = 0; k < batch;) { char data[MAX_DATA_SIZE]; memset(data, 0, MAX_DATA_SIZE); @@ -5665,10 +5673,15 @@ static int32_t prepareStmtWithoutStb( bind_ts = (int64_t *)ptr; bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - *bind_ts = startTime + getTSRandTail( - (int64_t)DEFAULT_TIMESTAMP_STEP, k, - g_args.disorderRatio, - g_args.disorderRange); + + if (g_args.disorderRatio) { + *bind_ts = startTime + getTSRandTail( + (int64_t)DEFAULT_TIMESTAMP_STEP, k, + g_args.disorderRatio, + g_args.disorderRange); + } else { + *bind_ts = startTime + (int64_t)(DEFAULT_TIMESTAMP_STEP * k); + } bind->buffer_length = sizeof(int64_t); bind->buffer = bind_ts; bind->length = &bind->buffer_length; @@ -5753,7 +5766,7 @@ static int32_t prepareStbStmt( bind_ts = (int64_t *)ptr; bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - if (sourceRand) { + if (stbInfo->disorderRatio) { *bind_ts = startTime + getTSRandTail( stbInfo->timeStampStep, k, stbInfo->disorderRatio, @@ -5820,6 +5833,7 @@ static int32_t prepareStbStmt( if (!sourceRand) { (*pSamplePos) ++; } + if (recordFrom >= insertRows) { break; } @@ -5829,6 +5843,43 @@ static int32_t prepareStbStmt( free(bindArray); return k; } + +static int32_t prepareStbStmtInterlace( + SSuperTable *stbInfo, + TAOS_STMT *stmt, + char *tableName, uint32_t batch, + uint64_t insertRows, + uint64_t recordFrom, + int64_t startTime, + int64_t *pSamplePos) +{ + return prepareStbStmt( + stbInfo, + stmt, + tableName, + g_args.num_of_RPR, + insertRows, 0, startTime, + pSamplePos); +} + +static int32_t prepareStbStmtProgressive( + SSuperTable *stbInfo, + TAOS_STMT *stmt, + char *tableName, uint32_t batch, + uint64_t insertRows, + uint64_t recordFrom, + int64_t startTime, + int64_t *pSamplePos) +{ + return prepareStbStmt( + stbInfo, + stmt, + tableName, + g_args.num_of_RPR, + insertRows, recordFrom, startTime, + pSamplePos); +} + #endif static int32_t generateStbProgressiveData( @@ -6003,7 +6054,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { uint32_t recOfBatch = 0; - for (uint32_t i = 0; i < batchPerTblTimes; i ++) { + for (uint64_t i = 0; i < batchPerTblTimes; i ++) { char tableName[TSDB_TABLE_NAME_LEN]; getTableName(tableName, pThreadInfo, tableSeq); @@ -6020,7 +6071,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { if (superTblInfo) { if (superTblInfo->iface == STMT_IFACE) { #if STMT_IFACE_ENABLED == 1 - generated = prepareStbStmt( + generated = prepareStbStmtInterlace( superTblInfo, pThreadInfo->stmt, tableName, @@ -6249,7 +6300,7 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { if (superTblInfo) { if (superTblInfo->iface == STMT_IFACE) { #if STMT_IFACE_ENABLED == 1 - generated = prepareStbStmt( + generated = prepareStbStmtProgressive( superTblInfo, pThreadInfo->stmt, tableName, @@ -6677,13 +6728,24 @@ static void startMultiThreadInsertData(int threads, char* db_name, char buffer[3000]; char *pstr = buffer; - pstr += sprintf(pstr, "INSERT INTO ? values(?"); + + if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { + pstr += sprintf(pstr, "INSERT INTO ? USING %s TAGS(?", + superTblInfo->sTblName); + for (int tag = 0; tag < (superTblInfo->tagCount - 1); tag ++ ) { + pstr += sprintf(pstr, ",?"); + } + pstr += sprintf(pstr, ") VALUES(?"); + } else { + pstr += sprintf(pstr, "INSERT INTO ? VALUES(?"); + } for (int col = 0; col < columnCount; col ++) { pstr += sprintf(pstr, ",?"); } pstr += sprintf(pstr, ")"); + debugPrint("%s() LN%d, buffer: %s", __func__, __LINE__, buffer); int ret = taos_stmt_prepare(pThreadInfo->stmt, buffer, 0); if (ret != 0){ errorPrint("failed to execute taos_stmt_prepare. return 0x%x. reason: %s\n", From c06b23c58eaee7fa8e077ae23884d4080745f47a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 17 Jul 2021 19:19:38 +0800 Subject: [PATCH 006/106] cherry pick from develop branch. (#6903) --- src/kit/taosdemo/taosdemo.c | 3055 ++++++++++++++++++----------------- 1 file changed, 1537 insertions(+), 1518 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index e35e54751f..2e0d8538b3 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -54,6 +54,7 @@ #include "tutil.h" #define STMT_IFACE_ENABLED 0 +#define NANO_SECOND_ENABLED 0 #define REQ_EXTRA_BUF_LEN 1024 #define RESP_BUF_LEN 4096 @@ -66,13 +67,6 @@ extern char configDir[]; #define STR_INSERT_INTO "INSERT INTO " -enum TEST_MODE { - INSERT_TEST, // 0 - QUERY_TEST, // 1 - SUBSCRIBE_TEST, // 2 - INVAID_TEST -}; - #define MAX_RECORDS_PER_REQ 32766 #define HEAD_BUFF_LEN TSDB_MAX_COLUMNS*24 // 16*MAX_COLUMNS + (192+32)*2 + insert into .. @@ -105,6 +99,13 @@ enum TEST_MODE { #define DEFAULT_TIMESTAMP_STEP 1 +enum TEST_MODE { + INSERT_TEST, // 0 + QUERY_TEST, // 1 + SUBSCRIBE_TEST, // 2 + INVAID_TEST +}; + typedef enum CREATE_SUB_TALBE_MOD_EN { PRE_CREATE_SUBTBL, AUTO_CREATE_SUBTBL, @@ -112,15 +113,15 @@ typedef enum CREATE_SUB_TALBE_MOD_EN { } CREATE_SUB_TALBE_MOD_EN; typedef enum TALBE_EXISTS_EN { - TBL_NO_EXISTS, - TBL_ALREADY_EXISTS, - TBL_EXISTS_BUTT + TBL_NO_EXISTS, + TBL_ALREADY_EXISTS, + TBL_EXISTS_BUTT } TALBE_EXISTS_EN; enum enumSYNC_MODE { - SYNC_MODE, - ASYNC_MODE, - MODE_BUT + SYNC_MODE, + ASYNC_MODE, + MODE_BUT }; enum enum_TAOS_INTERFACE { @@ -143,52 +144,52 @@ typedef enum enum_PROGRESSIVE_OR_INTERLACE { } PROG_OR_INTERLACE_MODE; typedef enum enumQUERY_TYPE { - NO_INSERT_TYPE, - INSERT_TYPE, - QUERY_TYPE_BUT + NO_INSERT_TYPE, + INSERT_TYPE, + QUERY_TYPE_BUT } QUERY_TYPE; enum _show_db_index { - TSDB_SHOW_DB_NAME_INDEX, - TSDB_SHOW_DB_CREATED_TIME_INDEX, - TSDB_SHOW_DB_NTABLES_INDEX, - TSDB_SHOW_DB_VGROUPS_INDEX, - TSDB_SHOW_DB_REPLICA_INDEX, - TSDB_SHOW_DB_QUORUM_INDEX, - TSDB_SHOW_DB_DAYS_INDEX, - TSDB_SHOW_DB_KEEP_INDEX, - TSDB_SHOW_DB_CACHE_INDEX, - TSDB_SHOW_DB_BLOCKS_INDEX, - TSDB_SHOW_DB_MINROWS_INDEX, - TSDB_SHOW_DB_MAXROWS_INDEX, - TSDB_SHOW_DB_WALLEVEL_INDEX, - TSDB_SHOW_DB_FSYNC_INDEX, - TSDB_SHOW_DB_COMP_INDEX, - TSDB_SHOW_DB_CACHELAST_INDEX, - TSDB_SHOW_DB_PRECISION_INDEX, - TSDB_SHOW_DB_UPDATE_INDEX, - TSDB_SHOW_DB_STATUS_INDEX, - TSDB_MAX_SHOW_DB + TSDB_SHOW_DB_NAME_INDEX, + TSDB_SHOW_DB_CREATED_TIME_INDEX, + TSDB_SHOW_DB_NTABLES_INDEX, + TSDB_SHOW_DB_VGROUPS_INDEX, + TSDB_SHOW_DB_REPLICA_INDEX, + TSDB_SHOW_DB_QUORUM_INDEX, + TSDB_SHOW_DB_DAYS_INDEX, + TSDB_SHOW_DB_KEEP_INDEX, + TSDB_SHOW_DB_CACHE_INDEX, + TSDB_SHOW_DB_BLOCKS_INDEX, + TSDB_SHOW_DB_MINROWS_INDEX, + TSDB_SHOW_DB_MAXROWS_INDEX, + TSDB_SHOW_DB_WALLEVEL_INDEX, + TSDB_SHOW_DB_FSYNC_INDEX, + TSDB_SHOW_DB_COMP_INDEX, + TSDB_SHOW_DB_CACHELAST_INDEX, + TSDB_SHOW_DB_PRECISION_INDEX, + TSDB_SHOW_DB_UPDATE_INDEX, + TSDB_SHOW_DB_STATUS_INDEX, + TSDB_MAX_SHOW_DB }; // -----------------------------------------SHOW TABLES CONFIGURE ------------------------------------- enum _show_stables_index { - TSDB_SHOW_STABLES_NAME_INDEX, - TSDB_SHOW_STABLES_CREATED_TIME_INDEX, - TSDB_SHOW_STABLES_COLUMNS_INDEX, - TSDB_SHOW_STABLES_METRIC_INDEX, - TSDB_SHOW_STABLES_UID_INDEX, - TSDB_SHOW_STABLES_TID_INDEX, - TSDB_SHOW_STABLES_VGID_INDEX, - TSDB_MAX_SHOW_STABLES + TSDB_SHOW_STABLES_NAME_INDEX, + TSDB_SHOW_STABLES_CREATED_TIME_INDEX, + TSDB_SHOW_STABLES_COLUMNS_INDEX, + TSDB_SHOW_STABLES_METRIC_INDEX, + TSDB_SHOW_STABLES_UID_INDEX, + TSDB_SHOW_STABLES_TID_INDEX, + TSDB_SHOW_STABLES_VGID_INDEX, + TSDB_MAX_SHOW_STABLES }; enum _describe_table_index { - TSDB_DESCRIBE_METRIC_FIELD_INDEX, - TSDB_DESCRIBE_METRIC_TYPE_INDEX, - TSDB_DESCRIBE_METRIC_LENGTH_INDEX, - TSDB_DESCRIBE_METRIC_NOTE_INDEX, - TSDB_MAX_DESCRIBE_METRIC + TSDB_DESCRIBE_METRIC_FIELD_INDEX, + TSDB_DESCRIBE_METRIC_TYPE_INDEX, + TSDB_DESCRIBE_METRIC_LENGTH_INDEX, + TSDB_DESCRIBE_METRIC_NOTE_INDEX, + TSDB_MAX_DESCRIBE_METRIC }; /* Used by main to communicate with parse_opt. */ @@ -228,7 +229,7 @@ typedef struct SArguments_S { int64_t num_of_DPT; int abort; uint32_t disorderRatio; // 0: no disorder, >0: x% - int disorderRange; // ms or us by database precision + int disorderRange; // ms, us or ns. accordig to database precision uint32_t method_of_delete; char ** arg_list; uint64_t totalInsertRows; @@ -237,191 +238,191 @@ typedef struct SArguments_S { } SArguments; typedef struct SColumn_S { - char field[TSDB_COL_NAME_LEN]; - char dataType[16]; - uint32_t dataLen; - char note[128]; + char field[TSDB_COL_NAME_LEN]; + char dataType[16]; + uint32_t dataLen; + char note[128]; } StrColumn; typedef struct SSuperTable_S { - char sTblName[TSDB_TABLE_NAME_LEN]; - char dataSource[MAX_TB_NAME_SIZE]; // rand_gen or sample - char childTblPrefix[TSDB_TABLE_NAME_LEN - 20]; // 20 characters reserved for seq - char insertMode[MAX_TB_NAME_SIZE]; // taosc, rest - uint16_t childTblExists; - int64_t childTblCount; - uint64_t batchCreateTableNum; // 0: no batch, > 0: batch table number in one sql - uint8_t autoCreateTable; // 0: create sub table, 1: auto create sub table - uint16_t iface; // 0: taosc, 1: rest, 2: stmt - int64_t childTblLimit; - uint64_t childTblOffset; + char sTblName[TSDB_TABLE_NAME_LEN]; + char dataSource[MAX_TB_NAME_SIZE]; // rand_gen or sample + char childTblPrefix[TSDB_TABLE_NAME_LEN - 20]; // 20 characters reserved for seq + char insertMode[MAX_TB_NAME_SIZE]; // taosc, rest + uint16_t childTblExists; + int64_t childTblCount; + uint64_t batchCreateTableNum; // 0: no batch, > 0: batch table number in one sql + uint8_t autoCreateTable; // 0: create sub table, 1: auto create sub table + uint16_t iface; // 0: taosc, 1: rest, 2: stmt + int64_t childTblLimit; + uint64_t childTblOffset; -// int multiThreadWriteOneTbl; // 0: no, 1: yes - uint32_t interlaceRows; // - int disorderRatio; // 0: no disorder, >0: x% - int disorderRange; // ms or us by database precision - uint64_t maxSqlLen; // + // int multiThreadWriteOneTbl; // 0: no, 1: yes + uint32_t interlaceRows; // + int disorderRatio; // 0: no disorder, >0: x% + int disorderRange; // ms, us or ns. according to database precision + uint64_t maxSqlLen; // - uint64_t insertInterval; // insert interval, will override global insert interval - int64_t insertRows; - int64_t timeStampStep; - char startTimestamp[MAX_TB_NAME_SIZE]; - char sampleFormat[MAX_TB_NAME_SIZE]; // csv, json - char sampleFile[MAX_FILE_NAME_LEN]; - char tagsFile[MAX_FILE_NAME_LEN]; + uint64_t insertInterval; // insert interval, will override global insert interval + int64_t insertRows; + int64_t timeStampStep; + char startTimestamp[MAX_TB_NAME_SIZE]; + char sampleFormat[MAX_TB_NAME_SIZE]; // csv, json + char sampleFile[MAX_FILE_NAME_LEN]; + char tagsFile[MAX_FILE_NAME_LEN]; - uint32_t columnCount; - StrColumn columns[TSDB_MAX_COLUMNS]; - uint32_t tagCount; - StrColumn tags[TSDB_MAX_TAGS]; + uint32_t columnCount; + StrColumn columns[TSDB_MAX_COLUMNS]; + uint32_t tagCount; + StrColumn tags[TSDB_MAX_TAGS]; - char* childTblName; - char* colsOfCreateChildTable; - uint64_t lenOfOneRow; - uint64_t lenOfTagOfOneRow; + char* childTblName; + char* colsOfCreateChildTable; + uint64_t lenOfOneRow; + uint64_t lenOfTagOfOneRow; - char* sampleDataBuf; - //int sampleRowCount; - //int sampleUsePos; + char* sampleDataBuf; + //int sampleRowCount; + //int sampleUsePos; - uint32_t tagSource; // 0: rand, 1: tag sample - char* tagDataBuf; - uint32_t tagSampleCount; - uint32_t tagUsePos; + uint32_t tagSource; // 0: rand, 1: tag sample + char* tagDataBuf; + uint32_t tagSampleCount; + uint32_t tagUsePos; - // statistics - uint64_t totalInsertRows; - uint64_t totalAffectedRows; + // statistics + uint64_t totalInsertRows; + uint64_t totalAffectedRows; } SSuperTable; typedef struct { - char name[TSDB_DB_NAME_LEN]; - char create_time[32]; - int64_t ntables; - int32_t vgroups; - int16_t replica; - int16_t quorum; - int16_t days; - char keeplist[32]; - int32_t cache; //MB - int32_t blocks; - int32_t minrows; - int32_t maxrows; - int8_t wallevel; - int32_t fsync; - int8_t comp; - int8_t cachelast; - char precision[8]; // time resolution - int8_t update; - char status[16]; + char name[TSDB_DB_NAME_LEN]; + char create_time[32]; + int64_t ntables; + int32_t vgroups; + int16_t replica; + int16_t quorum; + int16_t days; + char keeplist[32]; + int32_t cache; //MB + int32_t blocks; + int32_t minrows; + int32_t maxrows; + int8_t wallevel; + int32_t fsync; + int8_t comp; + int8_t cachelast; + char precision[8]; // time resolution + int8_t update; + char status[16]; } SDbInfo; typedef struct SDbCfg_S { -// int maxtablesPerVnode; - uint32_t minRows; // 0 means default - uint32_t maxRows; // 0 means default - int comp; - int walLevel; - int cacheLast; - int fsync; - int replica; - int update; - int keep; - int days; - int cache; - int blocks; - int quorum; - char precision[8]; + // int maxtablesPerVnode; + uint32_t minRows; // 0 means default + uint32_t maxRows; // 0 means default + int comp; + int walLevel; + int cacheLast; + int fsync; + int replica; + int update; + int keep; + int days; + int cache; + int blocks; + int quorum; + char precision[8]; } SDbCfg; typedef struct SDataBase_S { - char dbName[TSDB_DB_NAME_LEN]; - bool drop; // 0: use exists, 1: if exists, drop then new create - SDbCfg dbCfg; - uint64_t superTblCount; - SSuperTable superTbls[MAX_SUPER_TABLE_COUNT]; + char dbName[TSDB_DB_NAME_LEN]; + bool drop; // 0: use exists, 1: if exists, drop then new create + SDbCfg dbCfg; + uint64_t superTblCount; + SSuperTable superTbls[MAX_SUPER_TABLE_COUNT]; } SDataBase; typedef struct SDbs_S { - char cfgDir[MAX_FILE_NAME_LEN]; - char host[MAX_HOSTNAME_SIZE]; - struct sockaddr_in serv_addr; + char cfgDir[MAX_FILE_NAME_LEN]; + char host[MAX_HOSTNAME_SIZE]; + struct sockaddr_in serv_addr; - uint16_t port; - char user[MAX_USERNAME_SIZE]; - char password[MAX_PASSWORD_SIZE]; - char resultFile[MAX_FILE_NAME_LEN]; - bool use_metric; - bool insert_only; - bool do_aggreFunc; - bool asyncMode; + uint16_t port; + char user[MAX_USERNAME_SIZE]; + char password[MAX_PASSWORD_SIZE]; + char resultFile[MAX_FILE_NAME_LEN]; + bool use_metric; + bool insert_only; + bool do_aggreFunc; + bool asyncMode; - uint32_t threadCount; - uint32_t threadCountByCreateTbl; - uint32_t dbCount; - SDataBase db[MAX_DB_COUNT]; + uint32_t threadCount; + uint32_t threadCountByCreateTbl; + uint32_t dbCount; + SDataBase db[MAX_DB_COUNT]; - // statistics - uint64_t totalInsertRows; - uint64_t totalAffectedRows; + // statistics + uint64_t totalInsertRows; + uint64_t totalAffectedRows; } SDbs; typedef struct SpecifiedQueryInfo_S { - uint64_t queryInterval; // 0: unlimit > 0 loop/s - uint32_t concurrent; - int sqlCount; - uint32_t asyncMode; // 0: sync, 1: async - uint64_t subscribeInterval; // ms - uint64_t queryTimes; - bool subscribeRestart; - int subscribeKeepProgress; - char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; - char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN]; - int resubAfterConsume[MAX_QUERY_SQL_COUNT]; - int endAfterConsume[MAX_QUERY_SQL_COUNT]; - TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; - char topic[MAX_QUERY_SQL_COUNT][32]; - int consumed[MAX_QUERY_SQL_COUNT]; - TAOS_RES* res[MAX_QUERY_SQL_COUNT]; - uint64_t totalQueried; + uint64_t queryInterval; // 0: unlimit > 0 loop/s + uint32_t concurrent; + int sqlCount; + uint32_t asyncMode; // 0: sync, 1: async + uint64_t subscribeInterval; // ms + uint64_t queryTimes; + bool subscribeRestart; + int subscribeKeepProgress; + char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; + char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN]; + int resubAfterConsume[MAX_QUERY_SQL_COUNT]; + int endAfterConsume[MAX_QUERY_SQL_COUNT]; + TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; + char topic[MAX_QUERY_SQL_COUNT][32]; + int consumed[MAX_QUERY_SQL_COUNT]; + TAOS_RES* res[MAX_QUERY_SQL_COUNT]; + uint64_t totalQueried; } SpecifiedQueryInfo; typedef struct SuperQueryInfo_S { - char sTblName[TSDB_TABLE_NAME_LEN]; - uint64_t queryInterval; // 0: unlimit > 0 loop/s - uint32_t threadCnt; - uint32_t asyncMode; // 0: sync, 1: async - uint64_t subscribeInterval; // ms - bool subscribeRestart; - int subscribeKeepProgress; - uint64_t queryTimes; - int64_t childTblCount; - char childTblPrefix[TSDB_TABLE_NAME_LEN - 20]; // 20 characters reserved for seq - int sqlCount; - char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; - char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN]; - int resubAfterConsume; - int endAfterConsume; - TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; + char sTblName[TSDB_TABLE_NAME_LEN]; + uint64_t queryInterval; // 0: unlimit > 0 loop/s + uint32_t threadCnt; + uint32_t asyncMode; // 0: sync, 1: async + uint64_t subscribeInterval; // ms + bool subscribeRestart; + int subscribeKeepProgress; + uint64_t queryTimes; + int64_t childTblCount; + char childTblPrefix[TSDB_TABLE_NAME_LEN - 20]; // 20 characters reserved for seq + int sqlCount; + char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; + char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN]; + int resubAfterConsume; + int endAfterConsume; + TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; - char* childTblName; - uint64_t totalQueried; + char* childTblName; + uint64_t totalQueried; } SuperQueryInfo; typedef struct SQueryMetaInfo_S { - char cfgDir[MAX_FILE_NAME_LEN]; - char host[MAX_HOSTNAME_SIZE]; - uint16_t port; - struct sockaddr_in serv_addr; - char user[MAX_USERNAME_SIZE]; - char password[MAX_PASSWORD_SIZE]; - char dbName[TSDB_DB_NAME_LEN]; - char queryMode[MAX_TB_NAME_SIZE]; // taosc, rest + char cfgDir[MAX_FILE_NAME_LEN]; + char host[MAX_HOSTNAME_SIZE]; + uint16_t port; + struct sockaddr_in serv_addr; + char user[MAX_USERNAME_SIZE]; + char password[MAX_PASSWORD_SIZE]; + char dbName[TSDB_DB_NAME_LEN]; + char queryMode[MAX_TB_NAME_SIZE]; // taosc, rest - SpecifiedQueryInfo specifiedQueryInfo; - SuperQueryInfo superQueryInfo; - uint64_t totalQueried; + SpecifiedQueryInfo specifiedQueryInfo; + SuperQueryInfo superQueryInfo; + uint64_t totalQueried; } SQueryMetaInfo; typedef struct SThreadInfo_S { @@ -1514,7 +1515,10 @@ static int printfInsertMeta() { } if (g_Dbs.db[i].dbCfg.precision[0] != 0) { if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) - || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { +#if NANO_SECOND_ENABLED == 1 + || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2)) +#endif + || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ns", 2))) { printf(" precision: \033[33m%s\033[0m\n", g_Dbs.db[i].dbCfg.precision); } else { @@ -1704,6 +1708,9 @@ static void printfInsertMetaToFile(FILE* fp) { } if (g_Dbs.db[i].dbCfg.precision[0] != 0) { if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) +#if NANO_SECOND_ENABLED == 1 + || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ns", 2)) +#endif || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { fprintf(fp, " precision: %s\n", g_Dbs.db[i].dbCfg.precision); @@ -1904,10 +1911,12 @@ static void printfQueryMeta() { static char* formatTimestamp(char* buf, int64_t val, int precision) { time_t tt; - if (precision == TSDB_TIME_PRECISION_NANO) { - tt = (time_t)(val / 1000000000); - } else if (precision == TSDB_TIME_PRECISION_MICRO) { + if (precision == TSDB_TIME_PRECISION_MICRO) { tt = (time_t)(val / 1000000); +#if NANO_SECOND_ENABLED == 1 + } if (precision == TSDB_TIME_PRECISION_NANO) { + tt = (time_t)(val / 1000000000); +#endif } else { tt = (time_t)(val / 1000); } @@ -1927,10 +1936,12 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { struct tm* ptm = localtime(&tt); size_t pos = strftime(buf, 32, "%Y-%m-%d %H:%M:%S", ptm); - if (precision == TSDB_TIME_PRECISION_NANO) { - sprintf(buf + pos, ".%09d", (int)(val % 1000000000)); - } else if (precision == TSDB_TIME_PRECISION_MICRO) { + if (precision == TSDB_TIME_PRECISION_MICRO) { sprintf(buf + pos, ".%06d", (int)(val % 1000000)); +#if NANO_SECOND_ENABLED == 1 + } else if (precision == TSDB_TIME_PRECISION_NANO) { + sprintf(buf + pos, ".%09d", (int)(val % 1000000000)); +#endif } else { sprintf(buf + pos, ".%03d", (int)(val % 1000)); } @@ -2876,134 +2887,138 @@ static int createSuperTable( } static int createDatabasesAndStables() { - TAOS * taos = NULL; - int ret = 0; - taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port); - if (taos == NULL) { - errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); - return -1; - } - char command[BUFFER_SIZE] = "\0"; - - for (int i = 0; i < g_Dbs.dbCount; i++) { - if (g_Dbs.db[i].drop) { - sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName); - if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) { - taos_close(taos); + TAOS * taos = NULL; + int ret = 0; + taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port); + if (taos == NULL) { + errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); return -1; - } - - int dataLen = 0; - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName); - - if (g_Dbs.db[i].dbCfg.blocks > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks); - } - if (g_Dbs.db[i].dbCfg.cache > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache); - } - if (g_Dbs.db[i].dbCfg.days > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days); - } - if (g_Dbs.db[i].dbCfg.keep > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep); - } - if (g_Dbs.db[i].dbCfg.quorum > 1) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum); - } - if (g_Dbs.db[i].dbCfg.replica > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica); - } - if (g_Dbs.db[i].dbCfg.update > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update); - } - //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) { - // dataLen += snprintf(command + dataLen, - // BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode); - //} - if (g_Dbs.db[i].dbCfg.minRows > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows); - } - if (g_Dbs.db[i].dbCfg.maxRows > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows); - } - if (g_Dbs.db[i].dbCfg.comp > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp); - } - if (g_Dbs.db[i].dbCfg.walLevel > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel); - } - if (g_Dbs.db[i].dbCfg.cacheLast > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast); - } - if (g_Dbs.db[i].dbCfg.fsync > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, - " fsync %d", g_Dbs.db[i].dbCfg.fsync); - } - if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms"))) - || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, - "us", strlen("us")))) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, - " precision \'%s\';", g_Dbs.db[i].dbCfg.precision); - } - - if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) { - taos_close(taos); - errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName); - return -1; - } - printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName); } + char command[BUFFER_SIZE] = "\0"; - debugPrint("%s() LN%d supertbl count:%"PRIu64"\n", - __func__, __LINE__, g_Dbs.db[i].superTblCount); + for (int i = 0; i < g_Dbs.dbCount; i++) { + if (g_Dbs.db[i].drop) { + sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName); + if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) { + taos_close(taos); + return -1; + } - int validStbCount = 0; + int dataLen = 0; + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName); - for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) { - sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName, - g_Dbs.db[i].superTbls[j].sTblName); - ret = queryDbExec(taos, command, NO_INSERT_TYPE, true); + if (g_Dbs.db[i].dbCfg.blocks > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks); + } + if (g_Dbs.db[i].dbCfg.cache > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache); + } + if (g_Dbs.db[i].dbCfg.days > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days); + } + if (g_Dbs.db[i].dbCfg.keep > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep); + } + if (g_Dbs.db[i].dbCfg.quorum > 1) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum); + } + if (g_Dbs.db[i].dbCfg.replica > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica); + } + if (g_Dbs.db[i].dbCfg.update > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update); + } + //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) { + // dataLen += snprintf(command + dataLen, + // BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode); + //} + if (g_Dbs.db[i].dbCfg.minRows > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows); + } + if (g_Dbs.db[i].dbCfg.maxRows > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows); + } + if (g_Dbs.db[i].dbCfg.comp > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp); + } + if (g_Dbs.db[i].dbCfg.walLevel > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel); + } + if (g_Dbs.db[i].dbCfg.cacheLast > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast); + } + if (g_Dbs.db[i].dbCfg.fsync > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, + " fsync %d", g_Dbs.db[i].dbCfg.fsync); + } + if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms"))) +#if NANO_SECOND_ENABLED == 1 + || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, + "ns", strlen("ns"))) +#endif + || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, + "us", strlen("us")))) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, + " precision \'%s\';", g_Dbs.db[i].dbCfg.precision); + } - if ((ret != 0) || (g_Dbs.db[i].drop)) { - ret = createSuperTable(taos, g_Dbs.db[i].dbName, - &g_Dbs.db[i].superTbls[j]); - - if (0 != ret) { - errorPrint("create super table %"PRIu64" failed!\n\n", j); - continue; + if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) { + taos_close(taos); + errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName); + return -1; + } + printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName); } - } - ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName, - &g_Dbs.db[i].superTbls[j]); - if (0 != ret) { - errorPrint("\nget super table %s.%s info failed!\n\n", - g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName); - continue; - } + debugPrint("%s() LN%d supertbl count:%"PRIu64"\n", + __func__, __LINE__, g_Dbs.db[i].superTblCount); - validStbCount ++; + int validStbCount = 0; + + for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) { + sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName, + g_Dbs.db[i].superTbls[j].sTblName); + ret = queryDbExec(taos, command, NO_INSERT_TYPE, true); + + if ((ret != 0) || (g_Dbs.db[i].drop)) { + ret = createSuperTable(taos, g_Dbs.db[i].dbName, + &g_Dbs.db[i].superTbls[j]); + + if (0 != ret) { + errorPrint("create super table %"PRIu64" failed!\n\n", j); + continue; + } + } + + ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName, + &g_Dbs.db[i].superTbls[j]); + if (0 != ret) { + errorPrint("\nget super table %s.%s info failed!\n\n", + g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName); + continue; + } + + validStbCount ++; + } + + g_Dbs.db[i].superTblCount = validStbCount; } - g_Dbs.db[i].superTblCount = validStbCount; - } - - taos_close(taos); - return 0; + taos_close(taos); + return 0; } static void* createTable(void *sarg) @@ -3531,733 +3546,733 @@ PARSE_OVER: } static bool getMetaFromInsertJsonFile(cJSON* root) { - bool ret = false; + bool ret = false; - cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir"); - if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) { - tstrncpy(g_Dbs.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN); - } - - cJSON* host = cJSON_GetObjectItem(root, "host"); - if (host && host->type == cJSON_String && host->valuestring != NULL) { - tstrncpy(g_Dbs.host, host->valuestring, MAX_HOSTNAME_SIZE); - } else if (!host) { - tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE); - } else { - printf("ERROR: failed to read json, host not found\n"); - goto PARSE_OVER; - } - - cJSON* port = cJSON_GetObjectItem(root, "port"); - if (port && port->type == cJSON_Number) { - g_Dbs.port = port->valueint; - } else if (!port) { - g_Dbs.port = 6030; - } - - cJSON* user = cJSON_GetObjectItem(root, "user"); - if (user && user->type == cJSON_String && user->valuestring != NULL) { - tstrncpy(g_Dbs.user, user->valuestring, MAX_USERNAME_SIZE); - } else if (!user) { - tstrncpy(g_Dbs.user, "root", MAX_USERNAME_SIZE); - } - - cJSON* password = cJSON_GetObjectItem(root, "password"); - if (password && password->type == cJSON_String && password->valuestring != NULL) { - tstrncpy(g_Dbs.password, password->valuestring, MAX_PASSWORD_SIZE); - } else if (!password) { - tstrncpy(g_Dbs.password, "taosdata", MAX_PASSWORD_SIZE); - } - - cJSON* resultfile = cJSON_GetObjectItem(root, "result_file"); - if (resultfile && resultfile->type == cJSON_String && resultfile->valuestring != NULL) { - tstrncpy(g_Dbs.resultFile, resultfile->valuestring, MAX_FILE_NAME_LEN); - } else if (!resultfile) { - tstrncpy(g_Dbs.resultFile, "./insert_res.txt", MAX_FILE_NAME_LEN); - } - - cJSON* threads = cJSON_GetObjectItem(root, "thread_count"); - if (threads && threads->type == cJSON_Number) { - g_Dbs.threadCount = threads->valueint; - } else if (!threads) { - g_Dbs.threadCount = 1; - } else { - printf("ERROR: failed to read json, threads not found\n"); - goto PARSE_OVER; - } - - cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl"); - if (threads2 && threads2->type == cJSON_Number) { - g_Dbs.threadCountByCreateTbl = threads2->valueint; - } else if (!threads2) { - g_Dbs.threadCountByCreateTbl = 1; - } else { - errorPrint("%s() LN%d, failed to read json, threads2 not found\n", - __func__, __LINE__); - goto PARSE_OVER; - } - - cJSON* gInsertInterval = cJSON_GetObjectItem(root, "insert_interval"); - if (gInsertInterval && gInsertInterval->type == cJSON_Number) { - if (gInsertInterval->valueint <0) { - errorPrint("%s() LN%d, failed to read json, insert interval input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; + cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir"); + if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) { + tstrncpy(g_Dbs.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN); } - g_args.insert_interval = gInsertInterval->valueint; - } else if (!gInsertInterval) { - g_args.insert_interval = 0; - } else { - errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - cJSON* interlaceRows = cJSON_GetObjectItem(root, "interlace_rows"); - if (interlaceRows && interlaceRows->type == cJSON_Number) { - if (interlaceRows->valueint < 0) { - errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - - } - g_args.interlace_rows = interlaceRows->valueint; - } else if (!interlaceRows) { - g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req - } else { - errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - - cJSON* maxSqlLen = cJSON_GetObjectItem(root, "max_sql_len"); - if (maxSqlLen && maxSqlLen->type == cJSON_Number) { - if (maxSqlLen->valueint < 0) { - errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - g_args.max_sql_len = maxSqlLen->valueint; - } else if (!maxSqlLen) { - g_args.max_sql_len = (1024*1024); - } else { - errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - - cJSON* numRecPerReq = cJSON_GetObjectItem(root, "num_of_records_per_req"); - if (numRecPerReq && numRecPerReq->type == cJSON_Number) { - if (numRecPerReq->valueint <= 0) { - errorPrint("%s() LN%d, failed to read json, num_of_records_per_req input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } else if (numRecPerReq->valueint > MAX_RECORDS_PER_REQ) { - printf("NOTICE: number of records per request value %"PRIu64" > %d\n\n", - numRecPerReq->valueint, MAX_RECORDS_PER_REQ); - printf(" number of records per request value will be set to %d\n\n", - MAX_RECORDS_PER_REQ); - prompt(); - numRecPerReq->valueint = MAX_RECORDS_PER_REQ; - } - g_args.num_of_RPR = numRecPerReq->valueint; - } else if (!numRecPerReq) { - g_args.num_of_RPR = MAX_RECORDS_PER_REQ; - } else { - errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n", - __func__, __LINE__); - goto PARSE_OVER; - } - - cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no, - if (answerPrompt - && answerPrompt->type == cJSON_String - && answerPrompt->valuestring != NULL) { - if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) { - g_args.answer_yes = false; - } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) { - g_args.answer_yes = true; + cJSON* host = cJSON_GetObjectItem(root, "host"); + if (host && host->type == cJSON_String && host->valuestring != NULL) { + tstrncpy(g_Dbs.host, host->valuestring, MAX_HOSTNAME_SIZE); + } else if (!host) { + tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE); } else { - g_args.answer_yes = false; - } - } else if (!answerPrompt) { - g_args.answer_yes = true; // default is no, mean answer_yes. - } else { - errorPrint("%s", "failed to read json, confirm_parameter_prompt input mistake\n"); - goto PARSE_OVER; - } - - // rows per table need be less than insert batch - if (g_args.interlace_rows > g_args.num_of_RPR) { - printf("NOTICE: interlace rows value %u > num_of_records_per_req %u\n\n", - g_args.interlace_rows, g_args.num_of_RPR); - printf(" interlace rows value will be set to num_of_records_per_req %u\n\n", - g_args.num_of_RPR); - prompt(); - g_args.interlace_rows = g_args.num_of_RPR; - } - - cJSON* dbs = cJSON_GetObjectItem(root, "databases"); - if (!dbs || dbs->type != cJSON_Array) { - printf("ERROR: failed to read json, databases not found\n"); - goto PARSE_OVER; - } - - int dbSize = cJSON_GetArraySize(dbs); - if (dbSize > MAX_DB_COUNT) { - errorPrint( - "ERROR: failed to read json, databases size overflow, max database is %d\n", - MAX_DB_COUNT); - goto PARSE_OVER; - } - - g_Dbs.dbCount = dbSize; - for (int i = 0; i < dbSize; ++i) { - cJSON* dbinfos = cJSON_GetArrayItem(dbs, i); - if (dbinfos == NULL) continue; - - // dbinfo - cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo"); - if (!dbinfo || dbinfo->type != cJSON_Object) { - printf("ERROR: failed to read json, dbinfo not found\n"); - goto PARSE_OVER; + printf("ERROR: failed to read json, host not found\n"); + goto PARSE_OVER; } - cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name"); - if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) { - printf("ERROR: failed to read json, db name not found\n"); - goto PARSE_OVER; + cJSON* port = cJSON_GetObjectItem(root, "port"); + if (port && port->type == cJSON_Number) { + g_Dbs.port = port->valueint; + } else if (!port) { + g_Dbs.port = 6030; } - tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, TSDB_DB_NAME_LEN); - cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop"); - if (drop && drop->type == cJSON_String && drop->valuestring != NULL) { - if (0 == strncasecmp(drop->valuestring, "yes", strlen("yes"))) { - g_Dbs.db[i].drop = true; - } else { - g_Dbs.db[i].drop = false; - } - } else if (!drop) { - g_Dbs.db[i].drop = g_args.drop_database; + cJSON* user = cJSON_GetObjectItem(root, "user"); + if (user && user->type == cJSON_String && user->valuestring != NULL) { + tstrncpy(g_Dbs.user, user->valuestring, MAX_USERNAME_SIZE); + } else if (!user) { + tstrncpy(g_Dbs.user, "root", MAX_USERNAME_SIZE); + } + + cJSON* password = cJSON_GetObjectItem(root, "password"); + if (password && password->type == cJSON_String && password->valuestring != NULL) { + tstrncpy(g_Dbs.password, password->valuestring, MAX_PASSWORD_SIZE); + } else if (!password) { + tstrncpy(g_Dbs.password, "taosdata", MAX_PASSWORD_SIZE); + } + + cJSON* resultfile = cJSON_GetObjectItem(root, "result_file"); + if (resultfile && resultfile->type == cJSON_String && resultfile->valuestring != NULL) { + tstrncpy(g_Dbs.resultFile, resultfile->valuestring, MAX_FILE_NAME_LEN); + } else if (!resultfile) { + tstrncpy(g_Dbs.resultFile, "./insert_res.txt", MAX_FILE_NAME_LEN); + } + + cJSON* threads = cJSON_GetObjectItem(root, "thread_count"); + if (threads && threads->type == cJSON_Number) { + g_Dbs.threadCount = threads->valueint; + } else if (!threads) { + g_Dbs.threadCount = 1; } else { - errorPrint("%s() LN%d, failed to read json, drop input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; + printf("ERROR: failed to read json, threads not found\n"); + goto PARSE_OVER; } - cJSON *precision = cJSON_GetObjectItem(dbinfo, "precision"); - if (precision && precision->type == cJSON_String - && precision->valuestring != NULL) { - tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring, - 8); - } else if (!precision) { - memset(g_Dbs.db[i].dbCfg.precision, 0, 8); + cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl"); + if (threads2 && threads2->type == cJSON_Number) { + g_Dbs.threadCountByCreateTbl = threads2->valueint; + } else if (!threads2) { + g_Dbs.threadCountByCreateTbl = 1; } else { - printf("ERROR: failed to read json, precision not found\n"); - goto PARSE_OVER; - } - - cJSON* update = cJSON_GetObjectItem(dbinfo, "update"); - if (update && update->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.update = update->valueint; - } else if (!update) { - g_Dbs.db[i].dbCfg.update = -1; - } else { - printf("ERROR: failed to read json, update not found\n"); - goto PARSE_OVER; - } - - cJSON* replica = cJSON_GetObjectItem(dbinfo, "replica"); - if (replica && replica->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.replica = replica->valueint; - } else if (!replica) { - g_Dbs.db[i].dbCfg.replica = -1; - } else { - printf("ERROR: failed to read json, replica not found\n"); - goto PARSE_OVER; - } - - cJSON* keep = cJSON_GetObjectItem(dbinfo, "keep"); - if (keep && keep->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.keep = keep->valueint; - } else if (!keep) { - g_Dbs.db[i].dbCfg.keep = -1; - } else { - printf("ERROR: failed to read json, keep not found\n"); - goto PARSE_OVER; - } - - cJSON* days = cJSON_GetObjectItem(dbinfo, "days"); - if (days && days->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.days = days->valueint; - } else if (!days) { - g_Dbs.db[i].dbCfg.days = -1; - } else { - printf("ERROR: failed to read json, days not found\n"); - goto PARSE_OVER; - } - - cJSON* cache = cJSON_GetObjectItem(dbinfo, "cache"); - if (cache && cache->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.cache = cache->valueint; - } else if (!cache) { - g_Dbs.db[i].dbCfg.cache = -1; - } else { - printf("ERROR: failed to read json, cache not found\n"); - goto PARSE_OVER; - } - - cJSON* blocks= cJSON_GetObjectItem(dbinfo, "blocks"); - if (blocks && blocks->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.blocks = blocks->valueint; - } else if (!blocks) { - g_Dbs.db[i].dbCfg.blocks = -1; - } else { - printf("ERROR: failed to read json, block not found\n"); - goto PARSE_OVER; - } - - //cJSON* maxtablesPerVnode= cJSON_GetObjectItem(dbinfo, "maxtablesPerVnode"); - //if (maxtablesPerVnode && maxtablesPerVnode->type == cJSON_Number) { - // g_Dbs.db[i].dbCfg.maxtablesPerVnode = maxtablesPerVnode->valueint; - //} else if (!maxtablesPerVnode) { - // g_Dbs.db[i].dbCfg.maxtablesPerVnode = TSDB_DEFAULT_TABLES; - //} else { - // printf("failed to read json, maxtablesPerVnode not found"); - // goto PARSE_OVER; - //} - - cJSON* minRows= cJSON_GetObjectItem(dbinfo, "minRows"); - if (minRows && minRows->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.minRows = minRows->valueint; - } else if (!minRows) { - g_Dbs.db[i].dbCfg.minRows = 0; // 0 means default - } else { - printf("ERROR: failed to read json, minRows not found\n"); - goto PARSE_OVER; - } - - cJSON* maxRows= cJSON_GetObjectItem(dbinfo, "maxRows"); - if (maxRows && maxRows->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.maxRows = maxRows->valueint; - } else if (!maxRows) { - g_Dbs.db[i].dbCfg.maxRows = 0; // 0 means default - } else { - printf("ERROR: failed to read json, maxRows not found\n"); - goto PARSE_OVER; - } - - cJSON* comp= cJSON_GetObjectItem(dbinfo, "comp"); - if (comp && comp->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.comp = comp->valueint; - } else if (!comp) { - g_Dbs.db[i].dbCfg.comp = -1; - } else { - printf("ERROR: failed to read json, comp not found\n"); - goto PARSE_OVER; - } - - cJSON* walLevel= cJSON_GetObjectItem(dbinfo, "walLevel"); - if (walLevel && walLevel->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.walLevel = walLevel->valueint; - } else if (!walLevel) { - g_Dbs.db[i].dbCfg.walLevel = -1; - } else { - printf("ERROR: failed to read json, walLevel not found\n"); - goto PARSE_OVER; - } - - cJSON* cacheLast= cJSON_GetObjectItem(dbinfo, "cachelast"); - if (cacheLast && cacheLast->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.cacheLast = cacheLast->valueint; - } else if (!cacheLast) { - g_Dbs.db[i].dbCfg.cacheLast = -1; - } else { - printf("ERROR: failed to read json, cacheLast not found\n"); - goto PARSE_OVER; - } - - cJSON* quorum= cJSON_GetObjectItem(dbinfo, "quorum"); - if (quorum && quorum->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.quorum = quorum->valueint; - } else if (!quorum) { - g_Dbs.db[i].dbCfg.quorum = 1; - } else { - printf("failed to read json, quorum input mistake"); - goto PARSE_OVER; - } - - cJSON* fsync= cJSON_GetObjectItem(dbinfo, "fsync"); - if (fsync && fsync->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.fsync = fsync->valueint; - } else if (!fsync) { - g_Dbs.db[i].dbCfg.fsync = -1; - } else { - errorPrint("%s() LN%d, failed to read json, fsync input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - - // super_talbes - cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables"); - if (!stables || stables->type != cJSON_Array) { - errorPrint("%s() LN%d, failed to read json, super_tables not found\n", - __func__, __LINE__); - goto PARSE_OVER; - } - - int stbSize = cJSON_GetArraySize(stables); - if (stbSize > MAX_SUPER_TABLE_COUNT) { - errorPrint( - "%s() LN%d, failed to read json, supertable size overflow, max supertable is %d\n", - __func__, __LINE__, MAX_SUPER_TABLE_COUNT); - goto PARSE_OVER; - } - - g_Dbs.db[i].superTblCount = stbSize; - for (int j = 0; j < stbSize; ++j) { - cJSON* stbInfo = cJSON_GetArrayItem(stables, j); - if (stbInfo == NULL) continue; - - // dbinfo - cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name"); - if (!stbName || stbName->type != cJSON_String - || stbName->valuestring == NULL) { - errorPrint("%s() LN%d, failed to read json, stb name not found\n", + errorPrint("%s() LN%d, failed to read json, threads2 not found\n", __func__, __LINE__); goto PARSE_OVER; - } - tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, - TSDB_TABLE_NAME_LEN); + } - cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix"); - if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) { - printf("ERROR: failed to read json, childtable_prefix not found\n"); - goto PARSE_OVER; - } - tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, - TSDB_TABLE_NAME_LEN - 20); - - cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); - if (autoCreateTbl - && autoCreateTbl->type == cJSON_String - && autoCreateTbl->valuestring != NULL) { - if ((0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3)) - && (TBL_ALREADY_EXISTS != g_Dbs.db[i].superTbls[j].childTblExists)) { - g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL; - } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) { - g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; - } else { - g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; - } - } else if (!autoCreateTbl) { - g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; - } else { - printf("ERROR: failed to read json, auto_create_table not found\n"); - goto PARSE_OVER; - } - - cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num"); - if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) { - g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint; - } else if (!batchCreateTbl) { - g_Dbs.db[i].superTbls[j].batchCreateTableNum = 1000; - } else { - printf("ERROR: failed to read json, batch_create_tbl_num not found\n"); - goto PARSE_OVER; - } - - cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no - if (childTblExists - && childTblExists->type == cJSON_String - && childTblExists->valuestring != NULL) { - if ((0 == strncasecmp(childTblExists->valuestring, "yes", 3)) - && (g_Dbs.db[i].drop == false)) { - g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS; - } else if ((0 == strncasecmp(childTblExists->valuestring, "no", 2) - || (g_Dbs.db[i].drop == true))) { - g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; - } else { - g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; - } - } else if (!childTblExists) { - g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; - } else { - errorPrint("%s() LN%d, failed to read json, child_table_exists not found\n", - __func__, __LINE__); - goto PARSE_OVER; - } - - if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { - g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; - } - - cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count"); - if (!count || count->type != cJSON_Number || 0 >= count->valueint) { - errorPrint("%s() LN%d, failed to read json, childtable_count input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - g_Dbs.db[i].superTbls[j].childTblCount = count->valueint; - - cJSON *dataSource = cJSON_GetObjectItem(stbInfo, "data_source"); - if (dataSource && dataSource->type == cJSON_String - && dataSource->valuestring != NULL) { - tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, - dataSource->valuestring, TSDB_DB_NAME_LEN); - } else if (!dataSource) { - tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", TSDB_DB_NAME_LEN); - } else { - errorPrint("%s() LN%d, failed to read json, data_source not found\n", - __func__, __LINE__); - goto PARSE_OVER; - } - - cJSON *stbIface = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , rest, stmt - if (stbIface && stbIface->type == cJSON_String - && stbIface->valuestring != NULL) { - if (0 == strcasecmp(stbIface->valuestring, "taosc")) { - g_Dbs.db[i].superTbls[j].iface= TAOSC_IFACE; - } else if (0 == strcasecmp(stbIface->valuestring, "rest")) { - g_Dbs.db[i].superTbls[j].iface= REST_IFACE; -#if STMT_IFACE_ENABLED == 1 - } else if (0 == strcasecmp(stbIface->valuestring, "stmt")) { - g_Dbs.db[i].superTbls[j].iface= STMT_IFACE; -#endif - } else { - errorPrint("%s() LN%d, failed to read json, insert_mode %s not recognized\n", - __func__, __LINE__, stbIface->valuestring); + cJSON* gInsertInterval = cJSON_GetObjectItem(root, "insert_interval"); + if (gInsertInterval && gInsertInterval->type == cJSON_Number) { + if (gInsertInterval->valueint <0) { + errorPrint("%s() LN%d, failed to read json, insert interval input mistake\n", + __func__, __LINE__); goto PARSE_OVER; } - } else if (!stbIface) { - g_Dbs.db[i].superTbls[j].iface = TAOSC_IFACE; - } else { - errorPrint("%s", "failed to read json, insert_mode not found\n"); - goto PARSE_OVER; - } - - cJSON* childTbl_limit = cJSON_GetObjectItem(stbInfo, "childtable_limit"); - if ((childTbl_limit) && (g_Dbs.db[i].drop != true) - && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) { - if (childTbl_limit->type != cJSON_Number) { - printf("ERROR: failed to read json, childtable_limit\n"); - goto PARSE_OVER; - } - g_Dbs.db[i].superTbls[j].childTblLimit = childTbl_limit->valueint; - } else { - g_Dbs.db[i].superTbls[j].childTblLimit = -1; // select ... limit -1 means all query result, drop = yes mean all table need recreate, limit value is invalid. - } - - cJSON* childTbl_offset = cJSON_GetObjectItem(stbInfo, "childtable_offset"); - if ((childTbl_offset) && (g_Dbs.db[i].drop != true) - && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) { - if ((childTbl_offset->type != cJSON_Number) - || (0 > childTbl_offset->valueint)) { - printf("ERROR: failed to read json, childtable_offset\n"); - goto PARSE_OVER; - } - g_Dbs.db[i].superTbls[j].childTblOffset = childTbl_offset->valueint; - } else { - g_Dbs.db[i].superTbls[j].childTblOffset = 0; - } - - cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp"); - if (ts && ts->type == cJSON_String && ts->valuestring != NULL) { - tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp, - ts->valuestring, TSDB_DB_NAME_LEN); - } else if (!ts) { - tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp, - "now", TSDB_DB_NAME_LEN); - } else { - printf("ERROR: failed to read json, start_timestamp not found\n"); - goto PARSE_OVER; - } - - cJSON* timestampStep = cJSON_GetObjectItem(stbInfo, "timestamp_step"); - if (timestampStep && timestampStep->type == cJSON_Number) { - g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint; - } else if (!timestampStep) { - g_Dbs.db[i].superTbls[j].timeStampStep = DEFAULT_TIMESTAMP_STEP; - } else { - printf("ERROR: failed to read json, timestamp_step not found\n"); - goto PARSE_OVER; - } - - cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format"); - if (sampleFormat && sampleFormat->type - == cJSON_String && sampleFormat->valuestring != NULL) { - tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, - sampleFormat->valuestring, TSDB_DB_NAME_LEN); - } else if (!sampleFormat) { - tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", TSDB_DB_NAME_LEN); - } else { - printf("ERROR: failed to read json, sample_format not found\n"); - goto PARSE_OVER; - } - - cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file"); - if (sampleFile && sampleFile->type == cJSON_String - && sampleFile->valuestring != NULL) { - tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile, - sampleFile->valuestring, MAX_FILE_NAME_LEN); - } else if (!sampleFile) { - memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN); - } else { - printf("ERROR: failed to read json, sample_file not found\n"); - goto PARSE_OVER; - } - - cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file"); - if ((tagsFile && tagsFile->type == cJSON_String) - && (tagsFile->valuestring != NULL)) { - tstrncpy(g_Dbs.db[i].superTbls[j].tagsFile, - tagsFile->valuestring, MAX_FILE_NAME_LEN); - if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) { - g_Dbs.db[i].superTbls[j].tagSource = 0; - } else { - g_Dbs.db[i].superTbls[j].tagSource = 1; - } - } else if (!tagsFile) { - memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN); - g_Dbs.db[i].superTbls[j].tagSource = 0; - } else { - printf("ERROR: failed to read json, tags_file not found\n"); - goto PARSE_OVER; - } - - cJSON* stbMaxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len"); - if (stbMaxSqlLen && stbMaxSqlLen->type == cJSON_Number) { - int32_t len = stbMaxSqlLen->valueint; - if (len > TSDB_MAX_ALLOWED_SQL_LEN) { - len = TSDB_MAX_ALLOWED_SQL_LEN; - } else if (len < 5) { - len = 5; - } - g_Dbs.db[i].superTbls[j].maxSqlLen = len; - } else if (!maxSqlLen) { - g_Dbs.db[i].superTbls[j].maxSqlLen = g_args.max_sql_len; - } else { - errorPrint("%s() LN%d, failed to read json, stbMaxSqlLen input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } -/* - cJSON *multiThreadWriteOneTbl = - cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes - if (multiThreadWriteOneTbl - && multiThreadWriteOneTbl->type == cJSON_String - && multiThreadWriteOneTbl->valuestring != NULL) { - if (0 == strncasecmp(multiThreadWriteOneTbl->valuestring, "yes", 3)) { - g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1; - } else { - g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0; - } - } else if (!multiThreadWriteOneTbl) { - g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0; - } else { - printf("ERROR: failed to read json, multiThreadWriteOneTbl not found\n"); - goto PARSE_OVER; - } -*/ - cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows"); - if (insertRows && insertRows->type == cJSON_Number) { - if (insertRows->valueint < 0) { - errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint; - } else if (!insertRows) { - g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF; - } else { - errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - - cJSON* stbInterlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows"); - if (stbInterlaceRows && stbInterlaceRows->type == cJSON_Number) { - if (stbInterlaceRows->valueint < 0) { - errorPrint("%s() LN%d, failed to read json, interlace rows input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - g_Dbs.db[i].superTbls[j].interlaceRows = stbInterlaceRows->valueint; - - if (g_Dbs.db[i].superTbls[j].interlaceRows > g_Dbs.db[i].superTbls[j].insertRows) { - printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %u > insert_rows %"PRId64"\n\n", - i, j, g_Dbs.db[i].superTbls[j].interlaceRows, - g_Dbs.db[i].superTbls[j].insertRows); - printf(" interlace rows value will be set to insert_rows %"PRId64"\n\n", - g_Dbs.db[i].superTbls[j].insertRows); - prompt(); - g_Dbs.db[i].superTbls[j].interlaceRows = g_Dbs.db[i].superTbls[j].insertRows; - } - } else if (!stbInterlaceRows) { - g_Dbs.db[i].superTbls[j].interlaceRows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req - } else { - errorPrint( - "%s() LN%d, failed to read json, interlace rows input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - - cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio"); - if (disorderRatio && disorderRatio->type == cJSON_Number) { - if (disorderRatio->valueint > 50) - disorderRatio->valueint = 50; - - if (disorderRatio->valueint < 0) - disorderRatio->valueint = 0; - - g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint; - } else if (!disorderRatio) { - g_Dbs.db[i].superTbls[j].disorderRatio = 0; - } else { - printf("ERROR: failed to read json, disorderRatio not found\n"); - goto PARSE_OVER; - } - - cJSON* disorderRange = cJSON_GetObjectItem(stbInfo, "disorder_range"); - if (disorderRange && disorderRange->type == cJSON_Number) { - g_Dbs.db[i].superTbls[j].disorderRange = disorderRange->valueint; - } else if (!disorderRange) { - g_Dbs.db[i].superTbls[j].disorderRange = 1000; - } else { - printf("ERROR: failed to read json, disorderRange not found\n"); - goto PARSE_OVER; - } - - cJSON* insertInterval = cJSON_GetObjectItem(stbInfo, "insert_interval"); - if (insertInterval && insertInterval->type == cJSON_Number) { - g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint; - if (insertInterval->valueint < 0) { - errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - } else if (!insertInterval) { - verbosePrint("%s() LN%d: stable insert interval be overrided by global %"PRIu64".\n", - __func__, __LINE__, g_args.insert_interval); - g_Dbs.db[i].superTbls[j].insertInterval = g_args.insert_interval; - } else { + g_args.insert_interval = gInsertInterval->valueint; + } else if (!gInsertInterval) { + g_args.insert_interval = 0; + } else { errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n", __func__, __LINE__); goto PARSE_OVER; - } - - int retVal = getColumnAndTagTypeFromInsertJsonFile( - stbInfo, &g_Dbs.db[i].superTbls[j]); - if (false == retVal) { - goto PARSE_OVER; - } } - } - ret = true; + cJSON* interlaceRows = cJSON_GetObjectItem(root, "interlace_rows"); + if (interlaceRows && interlaceRows->type == cJSON_Number) { + if (interlaceRows->valueint < 0) { + errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + + } + g_args.interlace_rows = interlaceRows->valueint; + } else if (!interlaceRows) { + g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req + } else { + errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON* maxSqlLen = cJSON_GetObjectItem(root, "max_sql_len"); + if (maxSqlLen && maxSqlLen->type == cJSON_Number) { + if (maxSqlLen->valueint < 0) { + errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + g_args.max_sql_len = maxSqlLen->valueint; + } else if (!maxSqlLen) { + g_args.max_sql_len = (1024*1024); + } else { + errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON* numRecPerReq = cJSON_GetObjectItem(root, "num_of_records_per_req"); + if (numRecPerReq && numRecPerReq->type == cJSON_Number) { + if (numRecPerReq->valueint <= 0) { + errorPrint("%s() LN%d, failed to read json, num_of_records_per_req input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } else if (numRecPerReq->valueint > MAX_RECORDS_PER_REQ) { + printf("NOTICE: number of records per request value %"PRIu64" > %d\n\n", + numRecPerReq->valueint, MAX_RECORDS_PER_REQ); + printf(" number of records per request value will be set to %d\n\n", + MAX_RECORDS_PER_REQ); + prompt(); + numRecPerReq->valueint = MAX_RECORDS_PER_REQ; + } + g_args.num_of_RPR = numRecPerReq->valueint; + } else if (!numRecPerReq) { + g_args.num_of_RPR = MAX_RECORDS_PER_REQ; + } else { + errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no, + if (answerPrompt + && answerPrompt->type == cJSON_String + && answerPrompt->valuestring != NULL) { + if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) { + g_args.answer_yes = false; + } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) { + g_args.answer_yes = true; + } else { + g_args.answer_yes = false; + } + } else if (!answerPrompt) { + g_args.answer_yes = true; // default is no, mean answer_yes. + } else { + errorPrint("%s", "failed to read json, confirm_parameter_prompt input mistake\n"); + goto PARSE_OVER; + } + + // rows per table need be less than insert batch + if (g_args.interlace_rows > g_args.num_of_RPR) { + printf("NOTICE: interlace rows value %u > num_of_records_per_req %u\n\n", + g_args.interlace_rows, g_args.num_of_RPR); + printf(" interlace rows value will be set to num_of_records_per_req %u\n\n", + g_args.num_of_RPR); + prompt(); + g_args.interlace_rows = g_args.num_of_RPR; + } + + cJSON* dbs = cJSON_GetObjectItem(root, "databases"); + if (!dbs || dbs->type != cJSON_Array) { + printf("ERROR: failed to read json, databases not found\n"); + goto PARSE_OVER; + } + + int dbSize = cJSON_GetArraySize(dbs); + if (dbSize > MAX_DB_COUNT) { + errorPrint( + "ERROR: failed to read json, databases size overflow, max database is %d\n", + MAX_DB_COUNT); + goto PARSE_OVER; + } + + g_Dbs.dbCount = dbSize; + for (int i = 0; i < dbSize; ++i) { + cJSON* dbinfos = cJSON_GetArrayItem(dbs, i); + if (dbinfos == NULL) continue; + + // dbinfo + cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo"); + if (!dbinfo || dbinfo->type != cJSON_Object) { + printf("ERROR: failed to read json, dbinfo not found\n"); + goto PARSE_OVER; + } + + cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name"); + if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) { + printf("ERROR: failed to read json, db name not found\n"); + goto PARSE_OVER; + } + tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, TSDB_DB_NAME_LEN); + + cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop"); + if (drop && drop->type == cJSON_String && drop->valuestring != NULL) { + if (0 == strncasecmp(drop->valuestring, "yes", strlen("yes"))) { + g_Dbs.db[i].drop = true; + } else { + g_Dbs.db[i].drop = false; + } + } else if (!drop) { + g_Dbs.db[i].drop = g_args.drop_database; + } else { + errorPrint("%s() LN%d, failed to read json, drop input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON *precision = cJSON_GetObjectItem(dbinfo, "precision"); + if (precision && precision->type == cJSON_String + && precision->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring, + 8); + } else if (!precision) { + memset(g_Dbs.db[i].dbCfg.precision, 0, 8); + } else { + printf("ERROR: failed to read json, precision not found\n"); + goto PARSE_OVER; + } + + cJSON* update = cJSON_GetObjectItem(dbinfo, "update"); + if (update && update->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.update = update->valueint; + } else if (!update) { + g_Dbs.db[i].dbCfg.update = -1; + } else { + printf("ERROR: failed to read json, update not found\n"); + goto PARSE_OVER; + } + + cJSON* replica = cJSON_GetObjectItem(dbinfo, "replica"); + if (replica && replica->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.replica = replica->valueint; + } else if (!replica) { + g_Dbs.db[i].dbCfg.replica = -1; + } else { + printf("ERROR: failed to read json, replica not found\n"); + goto PARSE_OVER; + } + + cJSON* keep = cJSON_GetObjectItem(dbinfo, "keep"); + if (keep && keep->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.keep = keep->valueint; + } else if (!keep) { + g_Dbs.db[i].dbCfg.keep = -1; + } else { + printf("ERROR: failed to read json, keep not found\n"); + goto PARSE_OVER; + } + + cJSON* days = cJSON_GetObjectItem(dbinfo, "days"); + if (days && days->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.days = days->valueint; + } else if (!days) { + g_Dbs.db[i].dbCfg.days = -1; + } else { + printf("ERROR: failed to read json, days not found\n"); + goto PARSE_OVER; + } + + cJSON* cache = cJSON_GetObjectItem(dbinfo, "cache"); + if (cache && cache->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.cache = cache->valueint; + } else if (!cache) { + g_Dbs.db[i].dbCfg.cache = -1; + } else { + printf("ERROR: failed to read json, cache not found\n"); + goto PARSE_OVER; + } + + cJSON* blocks= cJSON_GetObjectItem(dbinfo, "blocks"); + if (blocks && blocks->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.blocks = blocks->valueint; + } else if (!blocks) { + g_Dbs.db[i].dbCfg.blocks = -1; + } else { + printf("ERROR: failed to read json, block not found\n"); + goto PARSE_OVER; + } + + //cJSON* maxtablesPerVnode= cJSON_GetObjectItem(dbinfo, "maxtablesPerVnode"); + //if (maxtablesPerVnode && maxtablesPerVnode->type == cJSON_Number) { + // g_Dbs.db[i].dbCfg.maxtablesPerVnode = maxtablesPerVnode->valueint; + //} else if (!maxtablesPerVnode) { + // g_Dbs.db[i].dbCfg.maxtablesPerVnode = TSDB_DEFAULT_TABLES; + //} else { + // printf("failed to read json, maxtablesPerVnode not found"); + // goto PARSE_OVER; + //} + + cJSON* minRows= cJSON_GetObjectItem(dbinfo, "minRows"); + if (minRows && minRows->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.minRows = minRows->valueint; + } else if (!minRows) { + g_Dbs.db[i].dbCfg.minRows = 0; // 0 means default + } else { + printf("ERROR: failed to read json, minRows not found\n"); + goto PARSE_OVER; + } + + cJSON* maxRows= cJSON_GetObjectItem(dbinfo, "maxRows"); + if (maxRows && maxRows->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.maxRows = maxRows->valueint; + } else if (!maxRows) { + g_Dbs.db[i].dbCfg.maxRows = 0; // 0 means default + } else { + printf("ERROR: failed to read json, maxRows not found\n"); + goto PARSE_OVER; + } + + cJSON* comp= cJSON_GetObjectItem(dbinfo, "comp"); + if (comp && comp->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.comp = comp->valueint; + } else if (!comp) { + g_Dbs.db[i].dbCfg.comp = -1; + } else { + printf("ERROR: failed to read json, comp not found\n"); + goto PARSE_OVER; + } + + cJSON* walLevel= cJSON_GetObjectItem(dbinfo, "walLevel"); + if (walLevel && walLevel->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.walLevel = walLevel->valueint; + } else if (!walLevel) { + g_Dbs.db[i].dbCfg.walLevel = -1; + } else { + printf("ERROR: failed to read json, walLevel not found\n"); + goto PARSE_OVER; + } + + cJSON* cacheLast= cJSON_GetObjectItem(dbinfo, "cachelast"); + if (cacheLast && cacheLast->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.cacheLast = cacheLast->valueint; + } else if (!cacheLast) { + g_Dbs.db[i].dbCfg.cacheLast = -1; + } else { + printf("ERROR: failed to read json, cacheLast not found\n"); + goto PARSE_OVER; + } + + cJSON* quorum= cJSON_GetObjectItem(dbinfo, "quorum"); + if (quorum && quorum->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.quorum = quorum->valueint; + } else if (!quorum) { + g_Dbs.db[i].dbCfg.quorum = 1; + } else { + printf("failed to read json, quorum input mistake"); + goto PARSE_OVER; + } + + cJSON* fsync= cJSON_GetObjectItem(dbinfo, "fsync"); + if (fsync && fsync->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.fsync = fsync->valueint; + } else if (!fsync) { + g_Dbs.db[i].dbCfg.fsync = -1; + } else { + errorPrint("%s() LN%d, failed to read json, fsync input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + // super_talbes + cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables"); + if (!stables || stables->type != cJSON_Array) { + errorPrint("%s() LN%d, failed to read json, super_tables not found\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + int stbSize = cJSON_GetArraySize(stables); + if (stbSize > MAX_SUPER_TABLE_COUNT) { + errorPrint( + "%s() LN%d, failed to read json, supertable size overflow, max supertable is %d\n", + __func__, __LINE__, MAX_SUPER_TABLE_COUNT); + goto PARSE_OVER; + } + + g_Dbs.db[i].superTblCount = stbSize; + for (int j = 0; j < stbSize; ++j) { + cJSON* stbInfo = cJSON_GetArrayItem(stables, j); + if (stbInfo == NULL) continue; + + // dbinfo + cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name"); + if (!stbName || stbName->type != cJSON_String + || stbName->valuestring == NULL) { + errorPrint("%s() LN%d, failed to read json, stb name not found\n", + __func__, __LINE__); + goto PARSE_OVER; + } + tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, + TSDB_TABLE_NAME_LEN); + + cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix"); + if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) { + printf("ERROR: failed to read json, childtable_prefix not found\n"); + goto PARSE_OVER; + } + tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, + TSDB_TABLE_NAME_LEN - 20); + + cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); + if (autoCreateTbl + && autoCreateTbl->type == cJSON_String + && autoCreateTbl->valuestring != NULL) { + if ((0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3)) + && (TBL_ALREADY_EXISTS != g_Dbs.db[i].superTbls[j].childTblExists)) { + g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL; + } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) { + g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; + } else { + g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; + } + } else if (!autoCreateTbl) { + g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; + } else { + printf("ERROR: failed to read json, auto_create_table not found\n"); + goto PARSE_OVER; + } + + cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num"); + if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint; + } else if (!batchCreateTbl) { + g_Dbs.db[i].superTbls[j].batchCreateTableNum = 1000; + } else { + printf("ERROR: failed to read json, batch_create_tbl_num not found\n"); + goto PARSE_OVER; + } + + cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no + if (childTblExists + && childTblExists->type == cJSON_String + && childTblExists->valuestring != NULL) { + if ((0 == strncasecmp(childTblExists->valuestring, "yes", 3)) + && (g_Dbs.db[i].drop == false)) { + g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS; + } else if ((0 == strncasecmp(childTblExists->valuestring, "no", 2) + || (g_Dbs.db[i].drop == true))) { + g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; + } else { + g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; + } + } else if (!childTblExists) { + g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; + } else { + errorPrint("%s() LN%d, failed to read json, child_table_exists not found\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; + } + + cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count"); + if (!count || count->type != cJSON_Number || 0 >= count->valueint) { + errorPrint("%s() LN%d, failed to read json, childtable_count input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + g_Dbs.db[i].superTbls[j].childTblCount = count->valueint; + + cJSON *dataSource = cJSON_GetObjectItem(stbInfo, "data_source"); + if (dataSource && dataSource->type == cJSON_String + && dataSource->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, + dataSource->valuestring, TSDB_DB_NAME_LEN); + } else if (!dataSource) { + tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", TSDB_DB_NAME_LEN); + } else { + errorPrint("%s() LN%d, failed to read json, data_source not found\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON *stbIface = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , rest, stmt + if (stbIface && stbIface->type == cJSON_String + && stbIface->valuestring != NULL) { + if (0 == strcasecmp(stbIface->valuestring, "taosc")) { + g_Dbs.db[i].superTbls[j].iface= TAOSC_IFACE; + } else if (0 == strcasecmp(stbIface->valuestring, "rest")) { + g_Dbs.db[i].superTbls[j].iface= REST_IFACE; +#if STMT_IFACE_ENABLED == 1 + } else if (0 == strcasecmp(stbIface->valuestring, "stmt")) { + g_Dbs.db[i].superTbls[j].iface= STMT_IFACE; +#endif + } else { + errorPrint("%s() LN%d, failed to read json, insert_mode %s not recognized\n", + __func__, __LINE__, stbIface->valuestring); + goto PARSE_OVER; + } + } else if (!stbIface) { + g_Dbs.db[i].superTbls[j].iface = TAOSC_IFACE; + } else { + errorPrint("%s", "failed to read json, insert_mode not found\n"); + goto PARSE_OVER; + } + + cJSON* childTbl_limit = cJSON_GetObjectItem(stbInfo, "childtable_limit"); + if ((childTbl_limit) && (g_Dbs.db[i].drop != true) + && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) { + if (childTbl_limit->type != cJSON_Number) { + printf("ERROR: failed to read json, childtable_limit\n"); + goto PARSE_OVER; + } + g_Dbs.db[i].superTbls[j].childTblLimit = childTbl_limit->valueint; + } else { + g_Dbs.db[i].superTbls[j].childTblLimit = -1; // select ... limit -1 means all query result, drop = yes mean all table need recreate, limit value is invalid. + } + + cJSON* childTbl_offset = cJSON_GetObjectItem(stbInfo, "childtable_offset"); + if ((childTbl_offset) && (g_Dbs.db[i].drop != true) + && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) { + if ((childTbl_offset->type != cJSON_Number) + || (0 > childTbl_offset->valueint)) { + printf("ERROR: failed to read json, childtable_offset\n"); + goto PARSE_OVER; + } + g_Dbs.db[i].superTbls[j].childTblOffset = childTbl_offset->valueint; + } else { + g_Dbs.db[i].superTbls[j].childTblOffset = 0; + } + + cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp"); + if (ts && ts->type == cJSON_String && ts->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp, + ts->valuestring, TSDB_DB_NAME_LEN); + } else if (!ts) { + tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp, + "now", TSDB_DB_NAME_LEN); + } else { + printf("ERROR: failed to read json, start_timestamp not found\n"); + goto PARSE_OVER; + } + + cJSON* timestampStep = cJSON_GetObjectItem(stbInfo, "timestamp_step"); + if (timestampStep && timestampStep->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint; + } else if (!timestampStep) { + g_Dbs.db[i].superTbls[j].timeStampStep = DEFAULT_TIMESTAMP_STEP; + } else { + printf("ERROR: failed to read json, timestamp_step not found\n"); + goto PARSE_OVER; + } + + cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format"); + if (sampleFormat && sampleFormat->type + == cJSON_String && sampleFormat->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, + sampleFormat->valuestring, TSDB_DB_NAME_LEN); + } else if (!sampleFormat) { + tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", TSDB_DB_NAME_LEN); + } else { + printf("ERROR: failed to read json, sample_format not found\n"); + goto PARSE_OVER; + } + + cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file"); + if (sampleFile && sampleFile->type == cJSON_String + && sampleFile->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile, + sampleFile->valuestring, MAX_FILE_NAME_LEN); + } else if (!sampleFile) { + memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN); + } else { + printf("ERROR: failed to read json, sample_file not found\n"); + goto PARSE_OVER; + } + + cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file"); + if ((tagsFile && tagsFile->type == cJSON_String) + && (tagsFile->valuestring != NULL)) { + tstrncpy(g_Dbs.db[i].superTbls[j].tagsFile, + tagsFile->valuestring, MAX_FILE_NAME_LEN); + if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) { + g_Dbs.db[i].superTbls[j].tagSource = 0; + } else { + g_Dbs.db[i].superTbls[j].tagSource = 1; + } + } else if (!tagsFile) { + memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN); + g_Dbs.db[i].superTbls[j].tagSource = 0; + } else { + printf("ERROR: failed to read json, tags_file not found\n"); + goto PARSE_OVER; + } + + cJSON* stbMaxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len"); + if (stbMaxSqlLen && stbMaxSqlLen->type == cJSON_Number) { + int32_t len = stbMaxSqlLen->valueint; + if (len > TSDB_MAX_ALLOWED_SQL_LEN) { + len = TSDB_MAX_ALLOWED_SQL_LEN; + } else if (len < 5) { + len = 5; + } + g_Dbs.db[i].superTbls[j].maxSqlLen = len; + } else if (!maxSqlLen) { + g_Dbs.db[i].superTbls[j].maxSqlLen = g_args.max_sql_len; + } else { + errorPrint("%s() LN%d, failed to read json, stbMaxSqlLen input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + /* + cJSON *multiThreadWriteOneTbl = + cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes + if (multiThreadWriteOneTbl + && multiThreadWriteOneTbl->type == cJSON_String + && multiThreadWriteOneTbl->valuestring != NULL) { + if (0 == strncasecmp(multiThreadWriteOneTbl->valuestring, "yes", 3)) { + g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1; + } else { + g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0; + } + } else if (!multiThreadWriteOneTbl) { + g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0; + } else { + printf("ERROR: failed to read json, multiThreadWriteOneTbl not found\n"); + goto PARSE_OVER; + } + */ + cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows"); + if (insertRows && insertRows->type == cJSON_Number) { + if (insertRows->valueint < 0) { + errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint; + } else if (!insertRows) { + g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF; + } else { + errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON* stbInterlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows"); + if (stbInterlaceRows && stbInterlaceRows->type == cJSON_Number) { + if (stbInterlaceRows->valueint < 0) { + errorPrint("%s() LN%d, failed to read json, interlace rows input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + g_Dbs.db[i].superTbls[j].interlaceRows = stbInterlaceRows->valueint; + + if (g_Dbs.db[i].superTbls[j].interlaceRows > g_Dbs.db[i].superTbls[j].insertRows) { + printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %u > insert_rows %"PRId64"\n\n", + i, j, g_Dbs.db[i].superTbls[j].interlaceRows, + g_Dbs.db[i].superTbls[j].insertRows); + printf(" interlace rows value will be set to insert_rows %"PRId64"\n\n", + g_Dbs.db[i].superTbls[j].insertRows); + prompt(); + g_Dbs.db[i].superTbls[j].interlaceRows = g_Dbs.db[i].superTbls[j].insertRows; + } + } else if (!stbInterlaceRows) { + g_Dbs.db[i].superTbls[j].interlaceRows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req + } else { + errorPrint( + "%s() LN%d, failed to read json, interlace rows input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio"); + if (disorderRatio && disorderRatio->type == cJSON_Number) { + if (disorderRatio->valueint > 50) + disorderRatio->valueint = 50; + + if (disorderRatio->valueint < 0) + disorderRatio->valueint = 0; + + g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint; + } else if (!disorderRatio) { + g_Dbs.db[i].superTbls[j].disorderRatio = 0; + } else { + printf("ERROR: failed to read json, disorderRatio not found\n"); + goto PARSE_OVER; + } + + cJSON* disorderRange = cJSON_GetObjectItem(stbInfo, "disorder_range"); + if (disorderRange && disorderRange->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].disorderRange = disorderRange->valueint; + } else if (!disorderRange) { + g_Dbs.db[i].superTbls[j].disorderRange = 1000; + } else { + printf("ERROR: failed to read json, disorderRange not found\n"); + goto PARSE_OVER; + } + + cJSON* insertInterval = cJSON_GetObjectItem(stbInfo, "insert_interval"); + if (insertInterval && insertInterval->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint; + if (insertInterval->valueint < 0) { + errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + } else if (!insertInterval) { + verbosePrint("%s() LN%d: stable insert interval be overrided by global %"PRIu64".\n", + __func__, __LINE__, g_args.insert_interval); + g_Dbs.db[i].superTbls[j].insertInterval = g_args.insert_interval; + } else { + errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + int retVal = getColumnAndTagTypeFromInsertJsonFile( + stbInfo, &g_Dbs.db[i].superTbls[j]); + if (false == retVal) { + goto PARSE_OVER; + } + } + } + + ret = true; PARSE_OVER: - return ret; + return ret; } static bool getMetaFromQueryJsonFile(cJSON* root) { @@ -6536,347 +6551,351 @@ static int convertHostToServAddr(char *host, uint16_t port, struct sockaddr_in * static void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSuperTable* superTblInfo) { - int32_t timePrec = TSDB_TIME_PRECISION_MILLI; - if (0 != precision[0]) { - if (0 == strncasecmp(precision, "ms", 2)) { - timePrec = TSDB_TIME_PRECISION_MILLI; - } else if (0 == strncasecmp(precision, "us", 2)) { - timePrec = TSDB_TIME_PRECISION_MICRO; - } else { - errorPrint("Not support precision: %s\n", precision); - exit(-1); - } - } - - int64_t start_time; - if (superTblInfo) { - if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { - start_time = taosGetTimestamp(timePrec); - } else { - if (TSDB_CODE_SUCCESS != taosParseTime( - superTblInfo->startTimestamp, - &start_time, - strlen(superTblInfo->startTimestamp), - timePrec, 0)) { - ERROR_EXIT("failed to parse time!\n"); - } - } - } else { - start_time = 1500000000000; - } - - int64_t start = taosGetTimestampMs(); - - // read sample data from file first - if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource, - "sample", strlen("sample")))) { - if (0 != prepareSampleDataForSTable(superTblInfo)) { - errorPrint("%s() LN%d, prepare sample data for stable failed!\n", - __func__, __LINE__); - exit(-1); - } - } - - TAOS* taos0 = taos_connect( - g_Dbs.host, g_Dbs.user, - g_Dbs.password, db_name, g_Dbs.port); - if (NULL == taos0) { - errorPrint("%s() LN%d, connect to server fail , reason: %s\n", - __func__, __LINE__, taos_errstr(NULL)); - exit(-1); - } - - int64_t ntables = 0; - uint64_t tableFrom; - - if (superTblInfo) { - int64_t limit; - uint64_t offset; - - if ((NULL != g_args.sqlFile) && (superTblInfo->childTblExists == TBL_NO_EXISTS) && - ((superTblInfo->childTblOffset != 0) || (superTblInfo->childTblLimit >= 0))) { - printf("WARNING: offset and limit will not be used since the child tables not exists!\n"); - } - - if (superTblInfo->childTblExists == TBL_ALREADY_EXISTS) { - if ((superTblInfo->childTblLimit < 0) - || ((superTblInfo->childTblOffset + superTblInfo->childTblLimit) - > (superTblInfo->childTblCount))) { - superTblInfo->childTblLimit = - superTblInfo->childTblCount - superTblInfo->childTblOffset; - } - - offset = superTblInfo->childTblOffset; - limit = superTblInfo->childTblLimit; - } else { - limit = superTblInfo->childTblCount; - offset = 0; - } - - ntables = limit; - tableFrom = offset; - - if ((superTblInfo->childTblExists != TBL_NO_EXISTS) - && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit ) - > superTblInfo->childTblCount)) { - printf("WARNING: specified offset + limit > child table count!\n"); - prompt(); - } - - if ((superTblInfo->childTblExists != TBL_NO_EXISTS) - && (0 == superTblInfo->childTblLimit)) { - printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n"); - prompt(); - } - - superTblInfo->childTblName = (char*)calloc(1, - limit * TSDB_TABLE_NAME_LEN); - if (superTblInfo->childTblName == NULL) { - errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__); - taos_close(taos0); - exit(-1); - } - - int64_t childTblCount; - getChildNameOfSuperTableWithLimitAndOffset( - taos0, - db_name, superTblInfo->sTblName, - &superTblInfo->childTblName, &childTblCount, - limit, - offset); - } else { - ntables = g_args.num_of_tables; - tableFrom = 0; - } - - taos_close(taos0); - - int64_t a = ntables / threads; - if (a < 1) { - threads = ntables; - a = 1; - } - - int64_t b = 0; - if (threads != 0) { - b = ntables % threads; - } - - if ((superTblInfo) - && (superTblInfo->iface == REST_IFACE)) { - if (convertHostToServAddr( - g_Dbs.host, g_Dbs.port, &(g_Dbs.serv_addr)) != 0) { - exit(-1); - } - } - - pthread_t *pids = calloc(1, threads * sizeof(pthread_t)); - assert(pids != NULL); - - threadInfo *infos = calloc(1, threads * sizeof(threadInfo)); - assert(infos != NULL); - - memset(pids, 0, threads * sizeof(pthread_t)); - memset(infos, 0, threads * sizeof(threadInfo)); - - for (int i = 0; i < threads; i++) { - threadInfo *pThreadInfo = infos + i; - pThreadInfo->threadID = i; - tstrncpy(pThreadInfo->db_name, db_name, TSDB_DB_NAME_LEN); - pThreadInfo->time_precision = timePrec; - pThreadInfo->superTblInfo = superTblInfo; - - pThreadInfo->start_time = start_time; - pThreadInfo->minDelay = UINT64_MAX; - - if ((NULL == superTblInfo) || - (superTblInfo->iface != REST_IFACE)) { - //t_info->taos = taos; - pThreadInfo->taos = taos_connect( - g_Dbs.host, g_Dbs.user, - g_Dbs.password, db_name, g_Dbs.port); - if (NULL == pThreadInfo->taos) { - errorPrint( - "%s() LN%d, connect to server fail from insert sub thread, reason: %s\n", - __func__, __LINE__, - taos_errstr(NULL)); - free(infos); - exit(-1); - } - -#if STMT_IFACE_ENABLED == 1 - if ((g_args.iface == STMT_IFACE) - || ((superTblInfo) && (superTblInfo->iface == STMT_IFACE))) { - - int columnCount; - if (superTblInfo) { - columnCount = superTblInfo->columnCount; - } else { - columnCount = g_args.num_of_CPR; - } - - pThreadInfo->stmt = taos_stmt_init(pThreadInfo->taos); - if (NULL == pThreadInfo->stmt) { - errorPrint( - "%s() LN%d, failed init stmt, reason: %s\n", - __func__, __LINE__, - taos_errstr(NULL)); - free(pids); - free(infos); - exit(-1); - } - - char buffer[3000]; - char *pstr = buffer; - - if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { - pstr += sprintf(pstr, "INSERT INTO ? USING %s TAGS(?", - superTblInfo->sTblName); - for (int tag = 0; tag < (superTblInfo->tagCount - 1); tag ++ ) { - pstr += sprintf(pstr, ",?"); - } - pstr += sprintf(pstr, ") VALUES(?"); + int32_t timePrec = TSDB_TIME_PRECISION_MILLI; + if (0 != precision[0]) { + if (0 == strncasecmp(precision, "ms", 2)) { + timePrec = TSDB_TIME_PRECISION_MILLI; + } else if (0 == strncasecmp(precision, "us", 2)) { + timePrec = TSDB_TIME_PRECISION_MICRO; +#if NANO_SECOND_ENABLED == 1 + } else if (0 == strncasecmp(precision, "ns", 2)) { + timePrec = TSDB_TIME_PRECISION_NANO; +#endif } else { - pstr += sprintf(pstr, "INSERT INTO ? VALUES(?"); - } - - for (int col = 0; col < columnCount; col ++) { - pstr += sprintf(pstr, ",?"); - } - pstr += sprintf(pstr, ")"); - - debugPrint("%s() LN%d, buffer: %s", __func__, __LINE__, buffer); - int ret = taos_stmt_prepare(pThreadInfo->stmt, buffer, 0); - if (ret != 0){ - errorPrint("failed to execute taos_stmt_prepare. return 0x%x. reason: %s\n", - ret, taos_errstr(NULL)); - free(pids); - free(infos); + errorPrint("Not support precision: %s\n", precision); exit(-1); } - } -#endif + } + + int64_t start_time; + if (superTblInfo) { + if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { + start_time = taosGetTimestamp(timePrec); + } else { + if (TSDB_CODE_SUCCESS != taosParseTime( + superTblInfo->startTimestamp, + &start_time, + strlen(superTblInfo->startTimestamp), + timePrec, 0)) { + ERROR_EXIT("failed to parse time!\n"); + } + } } else { - pThreadInfo->taos = NULL; + start_time = 1500000000000; } -/* if ((NULL == superTblInfo) - || (0 == superTblInfo->multiThreadWriteOneTbl)) { - */ - pThreadInfo->start_table_from = tableFrom; - pThreadInfo->ntables = iend_table_to = i < b ? tableFrom + a : tableFrom + a - 1; - tableFrom = pThreadInfo->end_table_to + 1; -/* } else { - pThreadInfo->start_table_from = 0; - pThreadInfo->ntables = superTblInfo->childTblCount; - pThreadInfo->start_time = pThreadInfo->start_time + rand_int() % 10000 - rand_tinyint(); + int64_t start = taosGetTimestampMs(); + + // read sample data from file first + if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource, + "sample", strlen("sample")))) { + if (0 != prepareSampleDataForSTable(superTblInfo)) { + errorPrint("%s() LN%d, prepare sample data for stable failed!\n", + __func__, __LINE__); + exit(-1); + } } -*/ - tsem_init(&(pThreadInfo->lock_sem), 0, 0); - if (ASYNC_MODE == g_Dbs.asyncMode) { - pthread_create(pids + i, NULL, asyncWrite, pThreadInfo); + + TAOS* taos0 = taos_connect( + g_Dbs.host, g_Dbs.user, + g_Dbs.password, db_name, g_Dbs.port); + if (NULL == taos0) { + errorPrint("%s() LN%d, connect to server fail , reason: %s\n", + __func__, __LINE__, taos_errstr(NULL)); + exit(-1); + } + + int64_t ntables = 0; + uint64_t tableFrom; + + if (superTblInfo) { + int64_t limit; + uint64_t offset; + + if ((NULL != g_args.sqlFile) && (superTblInfo->childTblExists == TBL_NO_EXISTS) && + ((superTblInfo->childTblOffset != 0) || (superTblInfo->childTblLimit >= 0))) { + printf("WARNING: offset and limit will not be used since the child tables not exists!\n"); + } + + if (superTblInfo->childTblExists == TBL_ALREADY_EXISTS) { + if ((superTblInfo->childTblLimit < 0) + || ((superTblInfo->childTblOffset + superTblInfo->childTblLimit) + > (superTblInfo->childTblCount))) { + superTblInfo->childTblLimit = + superTblInfo->childTblCount - superTblInfo->childTblOffset; + } + + offset = superTblInfo->childTblOffset; + limit = superTblInfo->childTblLimit; + } else { + limit = superTblInfo->childTblCount; + offset = 0; + } + + ntables = limit; + tableFrom = offset; + + if ((superTblInfo->childTblExists != TBL_NO_EXISTS) + && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit ) + > superTblInfo->childTblCount)) { + printf("WARNING: specified offset + limit > child table count!\n"); + prompt(); + } + + if ((superTblInfo->childTblExists != TBL_NO_EXISTS) + && (0 == superTblInfo->childTblLimit)) { + printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n"); + prompt(); + } + + superTblInfo->childTblName = (char*)calloc(1, + limit * TSDB_TABLE_NAME_LEN); + if (superTblInfo->childTblName == NULL) { + errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__); + taos_close(taos0); + exit(-1); + } + + int64_t childTblCount; + getChildNameOfSuperTableWithLimitAndOffset( + taos0, + db_name, superTblInfo->sTblName, + &superTblInfo->childTblName, &childTblCount, + limit, + offset); } else { - pthread_create(pids + i, NULL, syncWrite, pThreadInfo); + ntables = g_args.num_of_tables; + tableFrom = 0; } - } - for (int i = 0; i < threads; i++) { - pthread_join(pids[i], NULL); - } + taos_close(taos0); - uint64_t totalDelay = 0; - uint64_t maxDelay = 0; - uint64_t minDelay = UINT64_MAX; - uint64_t cntDelay = 1; - double avgDelay = 0; + int64_t a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } - for (int i = 0; i < threads; i++) { - threadInfo *pThreadInfo = infos + i; + int64_t b = 0; + if (threads != 0) { + b = ntables % threads; + } - tsem_destroy(&(pThreadInfo->lock_sem)); + if ((superTblInfo) + && (superTblInfo->iface == REST_IFACE)) { + if (convertHostToServAddr( + g_Dbs.host, g_Dbs.port, &(g_Dbs.serv_addr)) != 0) { + exit(-1); + } + } + + pthread_t *pids = calloc(1, threads * sizeof(pthread_t)); + assert(pids != NULL); + + threadInfo *infos = calloc(1, threads * sizeof(threadInfo)); + assert(infos != NULL); + + memset(pids, 0, threads * sizeof(pthread_t)); + memset(infos, 0, threads * sizeof(threadInfo)); + + for (int i = 0; i < threads; i++) { + threadInfo *pThreadInfo = infos + i; + pThreadInfo->threadID = i; + tstrncpy(pThreadInfo->db_name, db_name, TSDB_DB_NAME_LEN); + pThreadInfo->time_precision = timePrec; + pThreadInfo->superTblInfo = superTblInfo; + + pThreadInfo->start_time = start_time; + pThreadInfo->minDelay = UINT64_MAX; + + if ((NULL == superTblInfo) || + (superTblInfo->iface != REST_IFACE)) { + //t_info->taos = taos; + pThreadInfo->taos = taos_connect( + g_Dbs.host, g_Dbs.user, + g_Dbs.password, db_name, g_Dbs.port); + if (NULL == pThreadInfo->taos) { + errorPrint( + "%s() LN%d, connect to server fail from insert sub thread, reason: %s\n", + __func__, __LINE__, + taos_errstr(NULL)); + free(infos); + exit(-1); + } #if STMT_IFACE_ENABLED == 1 - if (pThreadInfo->stmt) { - taos_stmt_close(pThreadInfo->stmt); - } + if ((g_args.iface == STMT_IFACE) + || ((superTblInfo) && (superTblInfo->iface == STMT_IFACE))) { + + int columnCount; + if (superTblInfo) { + columnCount = superTblInfo->columnCount; + } else { + columnCount = g_args.num_of_CPR; + } + + pThreadInfo->stmt = taos_stmt_init(pThreadInfo->taos); + if (NULL == pThreadInfo->stmt) { + errorPrint( + "%s() LN%d, failed init stmt, reason: %s\n", + __func__, __LINE__, + taos_errstr(NULL)); + free(pids); + free(infos); + exit(-1); + } + + char buffer[3000]; + char *pstr = buffer; + + if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { + pstr += sprintf(pstr, "INSERT INTO ? USING %s TAGS(?", + superTblInfo->sTblName); + for (int tag = 0; tag < (superTblInfo->tagCount - 1); tag ++ ) { + pstr += sprintf(pstr, ",?"); + } + pstr += sprintf(pstr, ") VALUES(?"); + } else { + pstr += sprintf(pstr, "INSERT INTO ? VALUES(?"); + } + + for (int col = 0; col < columnCount; col ++) { + pstr += sprintf(pstr, ",?"); + } + pstr += sprintf(pstr, ")"); + + debugPrint("%s() LN%d, buffer: %s", __func__, __LINE__, buffer); + int ret = taos_stmt_prepare(pThreadInfo->stmt, buffer, 0); + if (ret != 0){ + errorPrint("failed to execute taos_stmt_prepare. return 0x%x. reason: %s\n", + ret, taos_errstr(NULL)); + free(pids); + free(infos); + exit(-1); + } + } #endif - tsem_destroy(&(pThreadInfo->lock_sem)); - taos_close(pThreadInfo->taos); + } else { + pThreadInfo->taos = NULL; + } + + /* if ((NULL == superTblInfo) + || (0 == superTblInfo->multiThreadWriteOneTbl)) { + */ + pThreadInfo->start_table_from = tableFrom; + pThreadInfo->ntables = iend_table_to = i < b ? tableFrom + a : tableFrom + a - 1; + tableFrom = pThreadInfo->end_table_to + 1; + /* } else { + pThreadInfo->start_table_from = 0; + pThreadInfo->ntables = superTblInfo->childTblCount; + pThreadInfo->start_time = pThreadInfo->start_time + rand_int() % 10000 - rand_tinyint(); + } + */ + tsem_init(&(pThreadInfo->lock_sem), 0, 0); + if (ASYNC_MODE == g_Dbs.asyncMode) { + pthread_create(pids + i, NULL, asyncWrite, pThreadInfo); + } else { + pthread_create(pids + i, NULL, syncWrite, pThreadInfo); + } + } + + for (int i = 0; i < threads; i++) { + pthread_join(pids[i], NULL); + } + + uint64_t totalDelay = 0; + uint64_t maxDelay = 0; + uint64_t minDelay = UINT64_MAX; + uint64_t cntDelay = 1; + double avgDelay = 0; + + for (int i = 0; i < threads; i++) { + threadInfo *pThreadInfo = infos + i; + + tsem_destroy(&(pThreadInfo->lock_sem)); + +#if STMT_IFACE_ENABLED == 1 + if (pThreadInfo->stmt) { + taos_stmt_close(pThreadInfo->stmt); + } +#endif + tsem_destroy(&(pThreadInfo->lock_sem)); + taos_close(pThreadInfo->taos); + + debugPrint("%s() LN%d, [%d] totalInsert=%"PRIu64" totalAffected=%"PRIu64"\n", + __func__, __LINE__, + pThreadInfo->threadID, pThreadInfo->totalInsertRows, + pThreadInfo->totalAffectedRows); + if (superTblInfo) { + superTblInfo->totalAffectedRows += pThreadInfo->totalAffectedRows; + superTblInfo->totalInsertRows += pThreadInfo->totalInsertRows; + } else { + g_args.totalAffectedRows += pThreadInfo->totalAffectedRows; + g_args.totalInsertRows += pThreadInfo->totalInsertRows; + } + + totalDelay += pThreadInfo->totalDelay; + cntDelay += pThreadInfo->cntDelay; + if (pThreadInfo->maxDelay > maxDelay) maxDelay = pThreadInfo->maxDelay; + if (pThreadInfo->minDelay < minDelay) minDelay = pThreadInfo->minDelay; + } + cntDelay -= 1; + + if (cntDelay == 0) cntDelay = 1; + avgDelay = (double)totalDelay / cntDelay; + + int64_t end = taosGetTimestampMs(); + int64_t t = end - start; + + double tInMs = t/1000.0; - debugPrint("%s() LN%d, [%d] totalInsert=%"PRIu64" totalAffected=%"PRIu64"\n", - __func__, __LINE__, - pThreadInfo->threadID, pThreadInfo->totalInsertRows, - pThreadInfo->totalAffectedRows); if (superTblInfo) { - superTblInfo->totalAffectedRows += pThreadInfo->totalAffectedRows; - superTblInfo->totalInsertRows += pThreadInfo->totalInsertRows; + fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n", + tInMs, superTblInfo->totalInsertRows, + superTblInfo->totalAffectedRows, + threads, db_name, superTblInfo->sTblName, + (tInMs)? + (double)(superTblInfo->totalInsertRows/tInMs):FLT_MAX); + + if (g_fpOfInsertResult) { + fprintf(g_fpOfInsertResult, + "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n", + tInMs, superTblInfo->totalInsertRows, + superTblInfo->totalAffectedRows, + threads, db_name, superTblInfo->sTblName, + (tInMs)? + (double)(superTblInfo->totalInsertRows/tInMs):FLT_MAX); + } } else { - g_args.totalAffectedRows += pThreadInfo->totalAffectedRows; - g_args.totalInsertRows += pThreadInfo->totalInsertRows; + fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n", + tInMs, g_args.totalInsertRows, + g_args.totalAffectedRows, + threads, db_name, + (tInMs)? + (double)(g_args.totalInsertRows/tInMs):FLT_MAX); + if (g_fpOfInsertResult) { + fprintf(g_fpOfInsertResult, + "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n", + tInMs, g_args.totalInsertRows, + g_args.totalAffectedRows, + threads, db_name, + (tInMs)? + (double)(g_args.totalInsertRows/tInMs):FLT_MAX); + } } - totalDelay += pThreadInfo->totalDelay; - cntDelay += pThreadInfo->cntDelay; - if (pThreadInfo->maxDelay > maxDelay) maxDelay = pThreadInfo->maxDelay; - if (pThreadInfo->minDelay < minDelay) minDelay = pThreadInfo->minDelay; - } - cntDelay -= 1; - - if (cntDelay == 0) cntDelay = 1; - avgDelay = (double)totalDelay / cntDelay; - - int64_t end = taosGetTimestampMs(); - int64_t t = end - start; - - double tInMs = t/1000.0; - - if (superTblInfo) { - fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n", - tInMs, superTblInfo->totalInsertRows, - superTblInfo->totalAffectedRows, - threads, db_name, superTblInfo->sTblName, - (tInMs)? - (double)(superTblInfo->totalInsertRows/tInMs):FLT_MAX); - + fprintf(stderr, "insert delay, avg: %10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n", + avgDelay, maxDelay, minDelay); if (g_fpOfInsertResult) { - fprintf(g_fpOfInsertResult, - "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n", - tInMs, superTblInfo->totalInsertRows, - superTblInfo->totalAffectedRows, - threads, db_name, superTblInfo->sTblName, - (tInMs)? - (double)(superTblInfo->totalInsertRows/tInMs):FLT_MAX); + fprintf(g_fpOfInsertResult, "insert delay, avg:%10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n", + avgDelay, maxDelay, minDelay); } - } else { - fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n", - tInMs, g_args.totalInsertRows, - g_args.totalAffectedRows, - threads, db_name, - (tInMs)? - (double)(g_args.totalInsertRows/tInMs):FLT_MAX); - if (g_fpOfInsertResult) { - fprintf(g_fpOfInsertResult, - "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n", - tInMs, g_args.totalInsertRows, - g_args.totalAffectedRows, - threads, db_name, - (tInMs)? - (double)(g_args.totalInsertRows/tInMs):FLT_MAX); - } - } - fprintf(stderr, "insert delay, avg: %10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n", - avgDelay, maxDelay, minDelay); - if (g_fpOfInsertResult) { - fprintf(g_fpOfInsertResult, "insert delay, avg:%10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n", - avgDelay, maxDelay, minDelay); - } + //taos_close(taos); - //taos_close(taos); - - free(pids); - free(infos); + free(pids); + free(infos); } static void *readTable(void *sarg) { @@ -7034,98 +7053,98 @@ static void prompt() static int insertTestProcess() { - setupForAnsiEscape(); - int ret = printfInsertMeta(); - resetAfterAnsiEscape(); + setupForAnsiEscape(); + int ret = printfInsertMeta(); + resetAfterAnsiEscape(); - if (ret == -1) - exit(EXIT_FAILURE); + if (ret == -1) + exit(EXIT_FAILURE); - debugPrint("%d result file: %s\n", __LINE__, g_Dbs.resultFile); - g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a"); - if (NULL == g_fpOfInsertResult) { - errorPrint( "Failed to open %s for save result\n", g_Dbs.resultFile); - return -1; - } - - if (g_fpOfInsertResult) - printfInsertMetaToFile(g_fpOfInsertResult); - - prompt(); - - init_rand_data(); - - // create database and super tables - if(createDatabasesAndStables() != 0) { - if (g_fpOfInsertResult) - fclose(g_fpOfInsertResult); - return -1; - } - - // pretreatement - if (prepareSampleData() != 0) { - if (g_fpOfInsertResult) - fclose(g_fpOfInsertResult); - return -1; - } - - double start; - double end; - - // create child tables - start = taosGetTimestampMs(); - createChildTables(); - end = taosGetTimestampMs(); - - if (g_totalChildTables > 0) { - fprintf(stderr, "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n", - (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl); - if (g_fpOfInsertResult) { - fprintf(g_fpOfInsertResult, - "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n", - (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl); + debugPrint("%d result file: %s\n", __LINE__, g_Dbs.resultFile); + g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a"); + if (NULL == g_fpOfInsertResult) { + errorPrint( "Failed to open %s for save result\n", g_Dbs.resultFile); + return -1; } - } - // create sub threads for inserting data - //start = taosGetTimestampMs(); - for (int i = 0; i < g_Dbs.dbCount; i++) { - if (g_Dbs.use_metric) { - if (g_Dbs.db[i].superTblCount > 0) { - for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) { + if (g_fpOfInsertResult) + printfInsertMetaToFile(g_fpOfInsertResult); - SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j]; + prompt(); - if (superTblInfo && (superTblInfo->insertRows > 0)) { - startMultiThreadInsertData( - g_Dbs.threadCount, - g_Dbs.db[i].dbName, - g_Dbs.db[i].dbCfg.precision, - superTblInfo); - } + init_rand_data(); + + // create database and super tables + if(createDatabasesAndStables() != 0) { + if (g_fpOfInsertResult) + fclose(g_fpOfInsertResult); + return -1; + } + + // pretreatement + if (prepareSampleData() != 0) { + if (g_fpOfInsertResult) + fclose(g_fpOfInsertResult); + return -1; + } + + double start; + double end; + + // create child tables + start = taosGetTimestampMs(); + createChildTables(); + end = taosGetTimestampMs(); + + if (g_totalChildTables > 0) { + fprintf(stderr, "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n", + (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl); + if (g_fpOfInsertResult) { + fprintf(g_fpOfInsertResult, + "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n", + (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl); } - } - } else { - startMultiThreadInsertData( - g_Dbs.threadCount, - g_Dbs.db[i].dbName, - g_Dbs.db[i].dbCfg.precision, - NULL); } - } - //end = taosGetTimestampMs(); - //int64_t totalInsertRows = 0; - //int64_t totalAffectedRows = 0; - //for (int i = 0; i < g_Dbs.dbCount; i++) { - // for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { - // totalInsertRows+= g_Dbs.db[i].superTbls[j].totalInsertRows; - // totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows; - //} - //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalInsertRows, totalAffectedRows, g_Dbs.threadCount); - postFreeResource(); + // create sub threads for inserting data + //start = taosGetTimestampMs(); + for (int i = 0; i < g_Dbs.dbCount; i++) { + if (g_Dbs.use_metric) { + if (g_Dbs.db[i].superTblCount > 0) { + for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) { - return 0; + SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j]; + + if (superTblInfo && (superTblInfo->insertRows > 0)) { + startMultiThreadInsertData( + g_Dbs.threadCount, + g_Dbs.db[i].dbName, + g_Dbs.db[i].dbCfg.precision, + superTblInfo); + } + } + } + } else { + startMultiThreadInsertData( + g_Dbs.threadCount, + g_Dbs.db[i].dbName, + g_Dbs.db[i].dbCfg.precision, + NULL); + } + } + //end = taosGetTimestampMs(); + + //int64_t totalInsertRows = 0; + //int64_t totalAffectedRows = 0; + //for (int i = 0; i < g_Dbs.dbCount; i++) { + // for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + // totalInsertRows+= g_Dbs.db[i].superTbls[j].totalInsertRows; + // totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows; + //} + //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalInsertRows, totalAffectedRows, g_Dbs.threadCount); + postFreeResource(); + + return 0; } static void *specifiedTableQuery(void *sarg) { @@ -7961,116 +7980,116 @@ static void initOfQueryMeta() { } static void setParaFromArg(){ - if (g_args.host) { - tstrncpy(g_Dbs.host, g_args.host, MAX_HOSTNAME_SIZE); - } else { - tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE); - } + if (g_args.host) { + tstrncpy(g_Dbs.host, g_args.host, MAX_HOSTNAME_SIZE); + } else { + tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE); + } - if (g_args.user) { - tstrncpy(g_Dbs.user, g_args.user, MAX_USERNAME_SIZE); - } + if (g_args.user) { + tstrncpy(g_Dbs.user, g_args.user, MAX_USERNAME_SIZE); + } - if (g_args.password) { - tstrncpy(g_Dbs.password, g_args.password, MAX_PASSWORD_SIZE); - } + if (g_args.password) { + tstrncpy(g_Dbs.password, g_args.password, MAX_PASSWORD_SIZE); + } - if (g_args.port) { - g_Dbs.port = g_args.port; - } + if (g_args.port) { + g_Dbs.port = g_args.port; + } - g_Dbs.threadCount = g_args.num_of_threads; - g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; - - g_Dbs.dbCount = 1; - g_Dbs.db[0].drop = true; - - tstrncpy(g_Dbs.db[0].dbName, g_args.database, TSDB_DB_NAME_LEN); - g_Dbs.db[0].dbCfg.replica = g_args.replica; - tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", 8); - - tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN); - - g_Dbs.use_metric = g_args.use_metric; - g_Dbs.insert_only = g_args.insert_only; - - g_Dbs.do_aggreFunc = true; - - char dataString[STRING_LEN]; - char **data_type = g_args.datatype; - - memset(dataString, 0, STRING_LEN); - - if (strcasecmp(data_type[0], "BINARY") == 0 - || strcasecmp(data_type[0], "BOOL") == 0 - || strcasecmp(data_type[0], "NCHAR") == 0 ) { - g_Dbs.do_aggreFunc = false; - } - - if (g_args.use_metric) { - g_Dbs.db[0].superTblCount = 1; - tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", TSDB_TABLE_NAME_LEN); - g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables; g_Dbs.threadCount = g_args.num_of_threads; g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; - g_Dbs.asyncMode = g_args.async_mode; - g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL; - g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS; - g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange; - g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio; - tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, - g_args.tb_prefix, TSDB_TABLE_NAME_LEN - 20); - tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE); + g_Dbs.dbCount = 1; + g_Dbs.db[0].drop = true; - if (g_args.iface == INTERFACE_BUT) { - g_Dbs.db[0].superTbls[0].iface = TAOSC_IFACE; - } else { - g_Dbs.db[0].superTbls[0].iface = g_args.iface; - } - tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp, - "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE); - g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP; + tstrncpy(g_Dbs.db[0].dbName, g_args.database, TSDB_DB_NAME_LEN); + g_Dbs.db[0].dbCfg.replica = g_args.replica; + tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", 8); - g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT; - g_Dbs.db[0].superTbls[0].maxSqlLen = g_args.max_sql_len; + tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN); - g_Dbs.db[0].superTbls[0].columnCount = 0; - for (int i = 0; i < MAX_NUM_COLUMNS; i++) { - if (data_type[i] == NULL) { - break; - } + g_Dbs.use_metric = g_args.use_metric; + g_Dbs.insert_only = g_args.insert_only; - tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, - data_type[i], strlen(data_type[i]) + 1); - g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary; - g_Dbs.db[0].superTbls[0].columnCount++; + g_Dbs.do_aggreFunc = true; + + char dataString[STRING_LEN]; + char **data_type = g_args.datatype; + + memset(dataString, 0, STRING_LEN); + + if (strcasecmp(data_type[0], "BINARY") == 0 + || strcasecmp(data_type[0], "BOOL") == 0 + || strcasecmp(data_type[0], "NCHAR") == 0 ) { + g_Dbs.do_aggreFunc = false; } - if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) { - g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR; - } else { - for (int i = g_Dbs.db[0].superTbls[0].columnCount; - i < g_args.num_of_CPR; i++) { - tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, + if (g_args.use_metric) { + g_Dbs.db[0].superTblCount = 1; + tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", TSDB_TABLE_NAME_LEN); + g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables; + g_Dbs.threadCount = g_args.num_of_threads; + g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; + g_Dbs.asyncMode = g_args.async_mode; + + g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL; + g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS; + g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange; + g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio; + tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, + g_args.tb_prefix, TSDB_TABLE_NAME_LEN - 20); + tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE); + + if (g_args.iface == INTERFACE_BUT) { + g_Dbs.db[0].superTbls[0].iface = TAOSC_IFACE; + } else { + g_Dbs.db[0].superTbls[0].iface = g_args.iface; + } + tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp, + "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP; + + g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT; + g_Dbs.db[0].superTbls[0].maxSqlLen = g_args.max_sql_len; + + g_Dbs.db[0].superTbls[0].columnCount = 0; + for (int i = 0; i < MAX_NUM_COLUMNS; i++) { + if (data_type[i] == NULL) { + break; + } + + tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, + data_type[i], strlen(data_type[i]) + 1); + g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary; + g_Dbs.db[0].superTbls[0].columnCount++; + } + + if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) { + g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR; + } else { + for (int i = g_Dbs.db[0].superTbls[0].columnCount; + i < g_args.num_of_CPR; i++) { + tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, + "INT", strlen("INT") + 1); + g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0; + g_Dbs.db[0].superTbls[0].columnCount++; + } + } + + tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, "INT", strlen("INT") + 1); - g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0; - g_Dbs.db[0].superTbls[0].columnCount++; - } + g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0; + + tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, + "BINARY", strlen("BINARY") + 1); + g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary; + g_Dbs.db[0].superTbls[0].tagCount = 2; + } else { + g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; + g_Dbs.db[0].superTbls[0].tagCount = 0; } - - tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, - "INT", strlen("INT") + 1); - g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0; - - tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, - "BINARY", strlen("BINARY") + 1); - g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary; - g_Dbs.db[0].superTbls[0].tagCount = 2; - } else { - g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; - g_Dbs.db[0].superTbls[0].tagCount = 0; - } } /* Function to do regular expression check */ From 278fbeaa4a2491460419afca2965174545223f17 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 18 Jul 2021 09:26:13 +0800 Subject: [PATCH 007/106] Hotfix/sangshuduo/td 3197 taosdemo coverity scan for master (#6906) * [TD-3197]: taosdemo and taosdump coverity scan issues. * exit if read sample file failed. * fix converity scan issue. * fix coverity scan issue. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 106 +++++++++++++++++++----------------- 1 file changed, 55 insertions(+), 51 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 2e0d8538b3..ccb889ee49 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -4973,62 +4973,64 @@ static int64_t generateStbRowData( static int64_t generateData(char *recBuf, char **data_type, int64_t timestamp, int lenOfBinary) { - memset(recBuf, 0, MAX_DATA_SIZE); - char *pstr = recBuf; - pstr += sprintf(pstr, "(%" PRId64, timestamp); + memset(recBuf, 0, MAX_DATA_SIZE); + char *pstr = recBuf; + pstr += sprintf(pstr, "(%" PRId64, timestamp); - int columnCount = g_args.num_of_CPR; + int columnCount = g_args.num_of_CPR; - for (int i = 0; i < columnCount; i++) { - if (strcasecmp(data_type[i % columnCount], "TINYINT") == 0) { - pstr += sprintf(pstr, ",%d", rand_tinyint() ); - } else if (strcasecmp(data_type[i % columnCount], "SMALLINT") == 0) { - pstr += sprintf(pstr, ",%d", rand_smallint()); - } else if (strcasecmp(data_type[i % columnCount], "INT") == 0) { - pstr += sprintf(pstr, ",%d", rand_int()); - } else if (strcasecmp(data_type[i % columnCount], "BIGINT") == 0) { - pstr += sprintf(pstr, ",%" PRId64, rand_bigint()); - } else if (strcasecmp(data_type[i % columnCount], "TIMESTAMP") == 0) { - pstr += sprintf(pstr, ",%" PRId64, rand_bigint()); - } else if (strcasecmp(data_type[i % columnCount], "FLOAT") == 0) { - pstr += sprintf(pstr, ",%10.4f", rand_float()); - } else if (strcasecmp(data_type[i % columnCount], "DOUBLE") == 0) { - double t = rand_double(); - pstr += sprintf(pstr, ",%20.8f", t); - } else if (strcasecmp(data_type[i % columnCount], "BOOL") == 0) { - bool b = rand_bool() & 1; - pstr += sprintf(pstr, ",%s", b ? "true" : "false"); - } else if (strcasecmp(data_type[i % columnCount], "BINARY") == 0) { - char *s = malloc(lenOfBinary + 1); - if (s == NULL) { - errorPrint("%s() LN%d, memory allocation %d bytes failed\n", - __func__, __LINE__, lenOfBinary + 1); - } - rand_string(s, lenOfBinary); - pstr += sprintf(pstr, ",\"%s\"", s); - free(s); - } else if (strcasecmp(data_type[i % columnCount], "NCHAR") == 0) { - char *s = malloc(lenOfBinary + 1); - if (s == NULL) { - errorPrint("%s() LN%d, memory allocation %d bytes failed\n", - __func__, __LINE__, lenOfBinary + 1); - } - rand_string(s, lenOfBinary); - pstr += sprintf(pstr, ",\"%s\"", s); - free(s); + for (int i = 0; i < columnCount; i++) { + if (strcasecmp(data_type[i % columnCount], "TINYINT") == 0) { + pstr += sprintf(pstr, ",%d", rand_tinyint() ); + } else if (strcasecmp(data_type[i % columnCount], "SMALLINT") == 0) { + pstr += sprintf(pstr, ",%d", rand_smallint()); + } else if (strcasecmp(data_type[i % columnCount], "INT") == 0) { + pstr += sprintf(pstr, ",%d", rand_int()); + } else if (strcasecmp(data_type[i % columnCount], "BIGINT") == 0) { + pstr += sprintf(pstr, ",%" PRId64, rand_bigint()); + } else if (strcasecmp(data_type[i % columnCount], "TIMESTAMP") == 0) { + pstr += sprintf(pstr, ",%" PRId64, rand_bigint()); + } else if (strcasecmp(data_type[i % columnCount], "FLOAT") == 0) { + pstr += sprintf(pstr, ",%10.4f", rand_float()); + } else if (strcasecmp(data_type[i % columnCount], "DOUBLE") == 0) { + double t = rand_double(); + pstr += sprintf(pstr, ",%20.8f", t); + } else if (strcasecmp(data_type[i % columnCount], "BOOL") == 0) { + bool b = rand_bool() & 1; + pstr += sprintf(pstr, ",%s", b ? "true" : "false"); + } else if (strcasecmp(data_type[i % columnCount], "BINARY") == 0) { + char *s = malloc(lenOfBinary + 1); + if (s == NULL) { + errorPrint("%s() LN%d, memory allocation %d bytes failed\n", + __func__, __LINE__, lenOfBinary + 1); + exit(-1); + } + rand_string(s, lenOfBinary); + pstr += sprintf(pstr, ",\"%s\"", s); + free(s); + } else if (strcasecmp(data_type[i % columnCount], "NCHAR") == 0) { + char *s = malloc(lenOfBinary + 1); + if (s == NULL) { + errorPrint("%s() LN%d, memory allocation %d bytes failed\n", + __func__, __LINE__, lenOfBinary + 1); + exit(-1); + } + rand_string(s, lenOfBinary); + pstr += sprintf(pstr, ",\"%s\"", s); + free(s); + } + + if (strlen(recBuf) > MAX_DATA_SIZE) { + perror("column length too long, abort"); + exit(-1); + } } - if (strlen(recBuf) > MAX_DATA_SIZE) { - perror("column length too long, abort"); - exit(-1); - } - } + pstr += sprintf(pstr, ")"); - pstr += sprintf(pstr, ")"); + verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf); - verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf); - - return (int32_t)strlen(recBuf); + return (int32_t)strlen(recBuf); } static int prepareSampleDataForSTable(SSuperTable *superTblInfo) { @@ -6748,7 +6750,9 @@ static void startMultiThreadInsertData(int threads, char* db_name, char buffer[3000]; char *pstr = buffer; - if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { + if ((superTblInfo) + && (AUTO_CREATE_SUBTBL + == superTblInfo->autoCreateTable)) { pstr += sprintf(pstr, "INSERT INTO ? USING %s TAGS(?", superTblInfo->sTblName); for (int tag = 0; tag < (superTblInfo->tagCount - 1); tag ++ ) { From 47a7136dbfe3842f783f0f52d6050461ca85bbb6 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 19 Jul 2021 10:05:28 +0800 Subject: [PATCH 008/106] Hotfix/sangshuduo/td 5136 taosdemo rework for master (#6913) * cherry pick from develop branch. * [TD-5136]: taosdemo simulate real senario. * update test case according to taosdemo change * adjust range of semi-random data. * make demo mode use different tag name and value. * change malloc to calloc for pid allocation. * fix typo. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index ccb889ee49..4a739d5337 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -2818,7 +2818,7 @@ static int createSuperTable( if (strcasecmp(dataType, "BINARY") == 0) { if ((g_args.demo_mode) && (tagIndex == 1)) { len += snprintf(tags + len, STRING_LEN - len, - "loction BINARY(%d), ", + "location BINARY(%d), ", superTbl->tags[tagIndex].dataLen); } else { len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", From a8cc70f245753cf2388f8a93c0c13d57ba75ea92 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Mon, 19 Jul 2021 10:41:14 +0800 Subject: [PATCH 009/106] [TD-5206]:test query stable join function --- tests/pytest/fulltest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index e1880a3750..c2221f521e 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -226,6 +226,7 @@ python3 ./test.py -f query/queryStddevWithGroupby.py python3 ./test.py -f query/querySecondtscolumnTowherenow.py python3 ./test.py -f query/queryFilterTswithDateUnit.py python3 ./test.py -f query/queryTscomputWithNow.py +python3 ./test.py -f query/queryStableJoin.py python3 ./test.py -f query/computeErrorinWhere.py python3 ./test.py -f query/queryTsisNull.py From b5724bb8e9e538d9419a10932e2ffb764a155965 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Mon, 19 Jul 2021 10:41:40 +0800 Subject: [PATCH 010/106] [TD-5206]:test query stable join function --- tests/pytest/query/queryStableJoin.py | 300 ++++++++++++++++++++++++++ 1 file changed, 300 insertions(+) create mode 100644 tests/pytest/query/queryStableJoin.py diff --git a/tests/pytest/query/queryStableJoin.py b/tests/pytest/query/queryStableJoin.py new file mode 100644 index 0000000000..825942bad8 --- /dev/null +++ b/tests/pytest/query/queryStableJoin.py @@ -0,0 +1,300 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +import random + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1600000000000 + self.num = 10 + + def run(self): + tdSql.prepare() + # test case for https://jira.taosdata.com:18080/browse/TD-5206 + + tdSql.execute('''create stable stable_1 + (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint, + q_bool bool , q_binary binary(20) , q_nchar nchar(20) ,q_float float , q_double double , q_ts timestamp) + tags(loc nchar(20) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, + t_bool bool , t_binary binary(20) , t_nchar nchar(20) ,t_float float , t_double double );''') + tdSql.execute('''create stable stable_2 + (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint, + q_bool bool , q_binary binary(20) , q_nchar nchar(20) ,q_float float , q_double double , q_ts timestamp) + tags(loc nchar(20) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, + t_bool bool , t_binary binary(20) , t_nchar nchar(20) ,t_float float , t_double double );''') + tdSql.execute('''create table table_0 using stable_1 + tags('table_0' , '0' , '0' , '0' , '0' , 0 , '0' , '0' , '0' , '0' )''') + tdSql.execute('''create table table_1 using stable_1 + tags('table_1' , '2147483647' , '9223372036854775807' , '32767' , '127' , 1 , + 'binary1' , 'nchar1' , '1' , '11' )''') + tdSql.execute('''create table table_2 using stable_1 + tags('table_2' , '-2147483647' , '-9223372036854775807' , '-32767' , '-127' , false , + 'binary2' , 'nchar2nchar2' , '-2.2' , '-22.22')''') + tdSql.execute('''create table table_3 using stable_1 + tags('table_3' , '3' , '3' , '3' , '3' , true , 'binary3' , 'nchar3' , '33.33' , '3333.3333' )''') + tdSql.execute('''create table table_4 using stable_1 + tags('table_4' , '4' , '4' , '4' , '4' , false , 'binary4' , 'nchar4' , '-444.444' , '-444444.444444' )''') + tdSql.execute('''create table table_5 using stable_1 + tags('table_5' , '5' , '5' , '5' , '5' , true , 'binary5' , 'nchar5' , '5555.5555' , '55555555.55555555' )''') + tdSql.execute('''create table table_21 using stable_2 + tags('table_5' , '5' , '5' , '5' , '5' , true , 'binary5' , 'nchar5' , '5555.5555' , '55555555.55555555' )''') + + for i in range(self.num): + tdSql.execute('''insert into table_0 values(%d, %d, %d, %d, %d, 0, 'binary.%s', 'nchar.%s', %f, %f, %d)''' + % (self.ts + i, i, i, i, i, i, i, i, i, self.ts + i)) + tdSql.execute('''insert into table_1 values(%d, %d, %d, %d, %d, 1, 'binary1.%s', 'nchar1.%s', %f, %f, %d)''' + % (self.ts + i, 2147483647-i, 9223372036854775807-i, 32767-i, 127-i, + i, i, random.random(), random.random(), 1262304000001 + i)) + tdSql.execute('''insert into table_2 values(%d, %d, %d, %d, %d, true, 'binary2.%s', 'nchar2nchar2.%s', %f, %f, %d)''' + % (self.ts + i, -2147483647+i, -9223372036854775807+i, -32767+i, -127+i, + i, i, random.uniform(-1,0), random.uniform(-1,0), 1577836800001 + i)) + tdSql.execute('''insert into table_3 values(%d, %d, %d, %d, %d, false, 'binary3.%s', 'nchar3.%s', %f, %f, %d)''' + % (self.ts + i, random.randint(-2147483647, 2147483647), + random.randint(-9223372036854775807, 9223372036854775807), random.randint(-32767, 32767), + random.randint(-127, 127), random.randint(-100, 100), random.randint(-10000, 10000), + random.uniform(-100000,100000), random.uniform(-1000000000,1000000000), self.ts + i)) + tdSql.execute('''insert into table_4 values(%d, %d, %d, %d, %d, true, 'binary4.%s', 'nchar4.%s', %f, %f, %d)''' + % (self.ts + i, i, i, i, i, i, i, i, i, self.ts + i)) + tdSql.execute('''insert into table_5 values(%d, %d, %d, %d, %d, false, 'binary5.%s', 'nchar5.%s', %f, %f, %d)''' + % (self.ts + i, i, i, i, i, i, i, i, i, self.ts + i)) + tdSql.execute('''insert into table_21 values(%d, %d, %d, %d, %d, false, 'binary5.%s', 'nchar5.%s', %f, %f, %d)''' + % (self.ts + i, i, i, i, i, i, i, i, i, self.ts + i)) + + + tdLog.info("==========TEST1:test all table data==========") + sql = '''select * from stable_1,stable_2 where stable_1.t_nchar = stable_2.t_nchar and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_tinyint = stable_2.t_tinyint and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_binary = stable_2.t_binary and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_double = stable_2.t_double and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_smallint = stable_2.t_smallint and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_bigint = stable_2.t_bigint and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_int = stable_2.t_int and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_float = stable_2.t_float and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_bool = stable_2.t_bool and stable_1.ts = stable_2.ts;''' + tdSql.error(sql) + + tdLog.info("==========TEST1:test drop table_0 data==========") + sql = '''drop table table_0;''' + tdSql.execute(sql) + sql = '''select * from stable_1,stable_2 where stable_1.t_nchar = stable_2.t_nchar and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_tinyint = stable_2.t_tinyint and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_binary = stable_2.t_binary and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_double = stable_2.t_double and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_smallint = stable_2.t_smallint and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_bigint = stable_2.t_bigint and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_int = stable_2.t_int and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_float = stable_2.t_float and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_bool = stable_2.t_bool and stable_1.ts = stable_2.ts;''' + tdSql.error(sql) + + tdLog.info("==========TEST1:test drop table_1 data==========") + sql = '''drop table table_1;''' + tdSql.execute(sql) + sql = '''select * from stable_1,stable_2 where stable_1.t_nchar = stable_2.t_nchar and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_tinyint = stable_2.t_tinyint and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_binary = stable_2.t_binary and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_double = stable_2.t_double and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_smallint = stable_2.t_smallint and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_bigint = stable_2.t_bigint and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_int = stable_2.t_int and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_float = stable_2.t_float and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_bool = stable_2.t_bool and stable_1.ts = stable_2.ts;''' + tdSql.error(sql) + + tdLog.info("==========TEST1:test drop table_2 data==========") + sql = '''drop table table_2;''' + tdSql.execute(sql) + sql = '''select * from stable_1,stable_2 where stable_1.t_nchar = stable_2.t_nchar and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_tinyint = stable_2.t_tinyint and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_binary = stable_2.t_binary and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_double = stable_2.t_double and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_smallint = stable_2.t_smallint and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_bigint = stable_2.t_bigint and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_int = stable_2.t_int and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_float = stable_2.t_float and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_bool = stable_2.t_bool and stable_1.ts = stable_2.ts;''' + tdSql.error(sql) + + tdLog.info("==========TEST1:test drop table_3 data==========") + sql = '''drop table table_3;''' + tdSql.execute(sql) + sql = '''select * from stable_1,stable_2 where stable_1.t_nchar = stable_2.t_nchar and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_tinyint = stable_2.t_tinyint and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_binary = stable_2.t_binary and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_double = stable_2.t_double and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_smallint = stable_2.t_smallint and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_bigint = stable_2.t_bigint and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_int = stable_2.t_int and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_float = stable_2.t_float and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_bool = stable_2.t_bool and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + + tdLog.info("==========TEST1:test drop table_4 data==========") + sql = '''drop table table_4;''' + tdSql.execute(sql) + sql = '''select * from stable_1,stable_2 where stable_1.t_nchar = stable_2.t_nchar and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_tinyint = stable_2.t_tinyint and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_binary = stable_2.t_binary and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_double = stable_2.t_double and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_smallint = stable_2.t_smallint and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_bigint = stable_2.t_bigint and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_int = stable_2.t_int and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_float = stable_2.t_float and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select * from stable_1,stable_2 where stable_1.t_bool = stable_2.t_bool and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + + tdLog.info("==========TEST1:test drop table_5 data==========") + sql = '''drop table table_5;''' + tdSql.execute(sql) + sql = '''select * from stable_1,stable_2 where stable_1.t_nchar = stable_2.t_nchar and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(0) + sql = '''select * from stable_1,stable_2 where stable_1.t_tinyint = stable_2.t_tinyint and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(0) + sql = '''select * from stable_1,stable_2 where stable_1.t_binary = stable_2.t_binary and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(0) + sql = '''select * from stable_1,stable_2 where stable_1.t_double = stable_2.t_double and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(0) + sql = '''select * from stable_1,stable_2 where stable_1.t_smallint = stable_2.t_smallint and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(0) + sql = '''select * from stable_1,stable_2 where stable_1.t_bigint = stable_2.t_bigint and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(0) + sql = '''select * from stable_1,stable_2 where stable_1.t_int = stable_2.t_int and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(0) + sql = '''select * from stable_1,stable_2 where stable_1.t_float = stable_2.t_float and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(0) + sql = '''select * from stable_1,stable_2 where stable_1.t_bool = stable_2.t_bool and stable_1.ts = stable_2.ts;''' + tdSql.query(sql) + tdSql.checkRows(0) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From f2e8ec524b6799ddf6c93bf7c8d8768a3fdb0c72 Mon Sep 17 00:00:00 2001 From: tomchon Date: Mon, 19 Jul 2021 14:01:39 +0800 Subject: [PATCH 011/106] modify all release scripts to adding "beta" in package name --- alert/release.sh | 2 +- packaging/deb/makedeb.sh | 4 ++- packaging/docker/dockerManifest.sh | 4 ++- packaging/docker/dockerbuild.sh | 2 +- packaging/rpm/makerpm.sh | 2 +- packaging/tools/makearbi.sh | 26 +++++++++++--------- packaging/tools/makeclient.sh | 39 ++++++++++++++++-------------- packaging/tools/makepkg.sh | 27 ++++++++++++--------- 8 files changed, 60 insertions(+), 46 deletions(-) diff --git a/alert/release.sh b/alert/release.sh index 77f6798480..20317e4166 100755 --- a/alert/release.sh +++ b/alert/release.sh @@ -69,7 +69,7 @@ rm -rf ./TDengine-alert pkg_name=TDengine-alert-${version}-${osType^}-${archMap[${cpuType}]} if [ "$verType" == "beta" ]; then - pkg_name=${pkg_name}-${verType} + pkg_name=TDengine-alert-${version}-${verType}-${osType^}-${archMap[${cpuType}]} elif [ "$verType" == "stable" ]; then pkg_name=${pkg_name} else diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index e6ddb6d742..a169bf2ba0 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -123,7 +123,7 @@ else fi if [ "$verType" == "beta" ]; then - debname=${debname}-${verType}".deb" + debname="TDengine-server-"${tdengine_ver}-${verType}-${osType}-${cpuType}".deb" elif [ "$verType" == "stable" ]; then debname=${debname}".deb" else @@ -131,6 +131,8 @@ else exit 1 fi + + # make deb package dpkg -b ${pkg_dir} $debname echo "make deb package success!" diff --git a/packaging/docker/dockerManifest.sh b/packaging/docker/dockerManifest.sh index 8bf449c8f2..0b30dd702c 100755 --- a/packaging/docker/dockerManifest.sh +++ b/packaging/docker/dockerManifest.sh @@ -49,12 +49,14 @@ if [ "$verType" == "beta" ]; then docker login -u tdengine -p ${passWord} #replace the docker registry username and password docker manifest push tdengine/tdengine-beta:latest docker manifest push tdengine/tdengine-beta:${version} + elif [ "$verType" == "stable" ]; then docker manifest create -a tdengine/tdengine:${version} tdengine/tdengine-amd64:${version} tdengine/tdengine-aarch64:${version} tdengine/tdengine-aarch32:${version} docker manifest create -a tdengine/tdengine:latest tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest docker login -u tdengine -p ${passWord} #replace the docker registry username and password docker manifest push tdengine/tdengine:latest - docker manifest push tdengine/tdengine:${version} + docker manifest push tdengine/tdengine:${version} + else echo "unknow verType, nor stabel or beta" exit 1 diff --git a/packaging/docker/dockerbuild.sh b/packaging/docker/dockerbuild.sh index 76717d5892..861f00f854 100755 --- a/packaging/docker/dockerbuild.sh +++ b/packaging/docker/dockerbuild.sh @@ -16,7 +16,7 @@ cpuType="" version="" passWord="" pkgFile="" -verType="" +verType="stable" while getopts "hc:n:p:f:V:" arg do diff --git a/packaging/rpm/makerpm.sh b/packaging/rpm/makerpm.sh index 7c3272f8d0..4cc7daf1a4 100755 --- a/packaging/rpm/makerpm.sh +++ b/packaging/rpm/makerpm.sh @@ -73,7 +73,7 @@ else fi if [ "$verType" == "beta" ]; then - rpmname=${rpmname}-${verType}".rpm" + rpmname="TDengine-server-"${tdengine_ver}-${verType}-${osType}-${cpuType}".rpm" elif [ "$verType" == "stable" ]; then rpmname=${rpmname}".rpm" else diff --git a/packaging/tools/makearbi.sh b/packaging/tools/makearbi.sh index 82808bcae9..5346a79c8f 100755 --- a/packaging/tools/makearbi.sh +++ b/packaging/tools/makearbi.sh @@ -47,24 +47,28 @@ mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir} cd ${release_dir} -if [ "$verMode" == "cluster" ]; then - pkg_name=${install_dir}-${osType}-${cpuType} -elif [ "$verMode" == "edge" ]; then - pkg_name=${install_dir}-${osType}-${cpuType} -else - echo "unknow verMode, nor cluster or edge" - exit 1 -fi +# install_dir has been distinguishes cluster from edege, so comments this code +pkg_name=${install_dir}-${osType}-${cpuType} + +# if [ "$verMode" == "cluster" ]; then +# pkg_name=${install_dir}-${osType}-${cpuType} +# elif [ "$verMode" == "edge" ]; then +# pkg_name=${install_dir}-${osType}-${cpuType} +# else +# echo "unknow verMode, nor cluster or edge" +# exit 1 +# fi if [ "$verType" == "beta" ]; then - pkg_name=${pkg_name}-${verType} -elif [ "$verType" == "stable" ]; then - pkg_name=${pkg_name} + pkg_name=${install_dir}-${verType}-${osType}-${cpuType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} else echo "unknow verType, nor stabel or beta" exit 1 fi + tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : exitcode=$? if [ "$exitcode" != "0" ]; then diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 2e0376f80e..2c36e85970 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -41,10 +41,10 @@ fi if [ "$osType" != "Darwin" ]; then if [ "$pagMode" == "lite" ]; then - #strip ${build_dir}/bin/taosd + #strip ${build_dir}/bin/taosd strip ${build_dir}/bin/taos bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh" - else + else bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo \ ${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh ${script_dir}/taosd-dump-cfg.gdb" fi @@ -139,7 +139,7 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then cp -r ${examples_dir}/C# ${install_dir}/examples fi # Copy driver -mkdir -p ${install_dir}/driver +mkdir -p ${install_dir}/driver cp ${lib_files} ${install_dir}/driver # Copy connector @@ -168,14 +168,26 @@ fi # exit 1 -cd ${release_dir} +cd ${release_dir} -if [ "$verMode" == "cluster" ]; then - pkg_name=${install_dir}-${osType}-${cpuType} -elif [ "$verMode" == "edge" ]; then - pkg_name=${install_dir}-${osType}-${cpuType} +# install_dir has been distinguishes cluster from edege, so comments this code +pkg_name=${install_dir}-${osType}-${cpuType} + +# if [ "$verMode" == "cluster" ]; then +# pkg_name=${install_dir}-${osType}-${cpuType} +# elif [ "$verMode" == "edge" ]; then +# pkg_name=${install_dir}-${osType}-${cpuType} +# else +# echo "unknow verMode, nor cluster or edge" +# exit 1 +# fi + +if [ "$verType" == "beta" ]; then + pkg_name=${install_dir}-${verType}-${osType}-${cpuType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} else - echo "unknow verMode, nor cluster or edge" + echo "unknow verType, nor stabel or beta" exit 1 fi @@ -183,15 +195,6 @@ if [ "$pagMode" == "lite" ]; then pkg_name=${pkg_name}-Lite fi -if [ "$verType" == "beta" ]; then - pkg_name=${pkg_name}-${verType} -elif [ "$verType" == "stable" ]; then - pkg_name=${pkg_name} -else - echo "unknow verType, nor stable or beta" - exit 1 -fi - if [ "$osType" != "Darwin" ]; then tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : else diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 81061416a2..56ab24426f 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -203,21 +203,20 @@ fi cd ${release_dir} -if [ "$verMode" == "cluster" ]; then - pkg_name=${install_dir}-${osType}-${cpuType} -elif [ "$verMode" == "edge" ]; then - pkg_name=${install_dir}-${osType}-${cpuType} -else - echo "unknow verMode, nor cluster or edge" - exit 1 -fi +# install_dir has been distinguishes cluster from edege, so comments this code +pkg_name=${install_dir}-${osType}-${cpuType} -if [ "$pagMode" == "lite" ]; then - pkg_name=${pkg_name}-Lite -fi +# if [ "$verMode" == "cluster" ]; then +# pkg_name=${install_dir}-${osType}-${cpuType} +# elif [ "$verMode" == "edge" ]; then +# pkg_name=${install_dir}-${osType}-${cpuType} +# else +# echo "unknow verMode, nor cluster or edge" +# exit 1 +# fi if [ "$verType" == "beta" ]; then - pkg_name=${pkg_name}-${verType} + pkg_name=${install_dir}-${verType}-${osType}-${cpuType} elif [ "$verType" == "stable" ]; then pkg_name=${pkg_name} else @@ -225,6 +224,10 @@ else exit 1 fi +if [ "$pagMode" == "lite" ]; then + pkg_name=${pkg_name}-Lite +fi + tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : exitcode=$? if [ "$exitcode" != "0" ]; then From b74bfad64f99b61f2fd912f1bab99030a181a005 Mon Sep 17 00:00:00 2001 From: zyyang Date: Mon, 19 Jul 2021 15:36:58 +0800 Subject: [PATCH 012/106] [TD-5388]: fix isUpdateQuery for 'reset query cache' statement --- src/client/src/tscUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 720327e327..749658eca3 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2518,7 +2518,7 @@ bool tscIsUpdateQuery(SSqlObj* pSql) { } SSqlCmd* pCmd = &pSql->cmd; - return ((pCmd->command >= TSDB_SQL_INSERT && pCmd->command <= TSDB_SQL_DROP_DNODE) || TSDB_SQL_USE_DB == pCmd->command); + return ((pCmd->command >= TSDB_SQL_INSERT && pCmd->command <= TSDB_SQL_DROP_DNODE) || TSDB_SQL_RESET_CACHE == pCmd->command || TSDB_SQL_USE_DB == pCmd->command); } char* tscGetSqlStr(SSqlObj* pSql) { From 72512255bb5bae0db622c0f35ad42401a493bb5e Mon Sep 17 00:00:00 2001 From: zhaoyanggh Date: Mon, 19 Jul 2021 15:40:29 +0800 Subject: [PATCH 013/106] add test case for reset query cache using jdbc --- .../jdbc/cases/ResetQueryCacheTest.java | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResetQueryCacheTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResetQueryCacheTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResetQueryCacheTest.java new file mode 100644 index 0000000000..52db8052e3 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResetQueryCacheTest.java @@ -0,0 +1,53 @@ +package com.taosdata.jdbc.cases; + +import com.taosdata.jdbc.TSDBDriver; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.sql.*; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; + +public class ResetQueryCacheTest { + + static Connection connection; + static Statement statement; + static String host = "127.0.0.1"; + + @Before + public void init() { + try { + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); + statement = connection.createStatement(); + } catch (SQLException e) { + return; + } + } + + @Test + public void testResetQueryCache() throws SQLException { + String resetSql = "reset query cache"; + statement.execute(resetSql); + // ResultSet rs = statement.executeQuery(resetSql); + // rs.close(); + } + + @After + public void close() { + try { + if (statement != null) + statement.close(); + if (connection != null) + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file From 2ae80b66a3007adec0fb1a536cb41d6305ae174e Mon Sep 17 00:00:00 2001 From: wpan Date: Mon, 19 Jul 2021 16:26:27 +0800 Subject: [PATCH 014/106] support dummy tag in sub-table interval query --- src/query/src/qExecutor.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 21aab9fdbc..9b5b23d9aa 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4885,6 +4885,8 @@ static SSDataBlock* doIntervalAgg(void* param) { break; } + setTagValue(pOperator, pRuntimeEnv->pQuery->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); + // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQuery->order.order); hashIntervalAgg(pOperator, &pIntervalInfo->resultRowInfo, pBlock, 0); From 43a3ee069000606bbc570b014aa3d5aa9e366472 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 19 Jul 2021 16:48:40 +0800 Subject: [PATCH 015/106] Hotfix/sangshuduo/td 5300 taosdemo stmt print for master (#6919) * [TD-5300]: taosdemo stmt debug print. * fix default iface is unknown. * merge with TD-5300 fix against develop --- src/kit/taosdemo/taosdemo.c | 766 ++++++++++++++++++------------------ 1 file changed, 386 insertions(+), 380 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 4a739d5337..59c49a0c93 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -5228,63 +5228,69 @@ static int32_t generateStbDataTail( int64_t remainderBufLen, int64_t insertRows, uint64_t recordFrom, int64_t startTime, int64_t *pSamplePos, int64_t *dataLen) { - uint64_t len = 0; + uint64_t len = 0; - char *pstr = buffer; + char *pstr = buffer; - bool tsRand; - if (0 == strncasecmp(superTblInfo->dataSource, "rand", strlen("rand"))) { - tsRand = true; - } else { - tsRand = false; - } - verbosePrint("%s() LN%d batch=%u buflen=%"PRId64"\n", - __func__, __LINE__, batch, remainderBufLen); - - int32_t k; - for (k = 0; k < batch;) { - char data[MAX_DATA_SIZE]; - memset(data, 0, MAX_DATA_SIZE); - - int64_t lenOfRow = 0; - - if (tsRand) { - lenOfRow = generateStbRowData(superTblInfo, data, - startTime + getTSRandTail( - superTblInfo->timeStampStep, k, - superTblInfo->disorderRatio, - superTblInfo->disorderRange) - ); + bool tsRand; + if (0 == strncasecmp(superTblInfo->dataSource, "rand", strlen("rand"))) { + tsRand = true; } else { - lenOfRow = getRowDataFromSample( - data, - (remainderBufLen < MAX_DATA_SIZE)?remainderBufLen:MAX_DATA_SIZE, - startTime + superTblInfo->timeStampStep * k, - superTblInfo, - pSamplePos); + tsRand = false; + } + verbosePrint("%s() LN%d batch=%u buflen=%"PRId64"\n", + __func__, __LINE__, batch, remainderBufLen); + + int32_t k; + for (k = 0; k < batch;) { + char data[MAX_DATA_SIZE]; + memset(data, 0, MAX_DATA_SIZE); + + int64_t lenOfRow = 0; + + if (tsRand) { + if (superTblInfo->disorderRatio > 0) { + lenOfRow = generateStbRowData(superTblInfo, data, + startTime + getTSRandTail( + superTblInfo->timeStampStep, k, + superTblInfo->disorderRatio, + superTblInfo->disorderRange) + ); + } else { + lenOfRow = generateStbRowData(superTblInfo, data, + startTime + superTblInfo->timeStampStep * k + ); + } + } else { + lenOfRow = getRowDataFromSample( + data, + (remainderBufLen < MAX_DATA_SIZE)?remainderBufLen:MAX_DATA_SIZE, + startTime + superTblInfo->timeStampStep * k, + superTblInfo, + pSamplePos); + } + + if ((lenOfRow + 1) > remainderBufLen) { + break; + } + + pstr += snprintf(pstr , lenOfRow + 1, "%s", data); + k++; + len += lenOfRow; + remainderBufLen -= lenOfRow; + + verbosePrint("%s() LN%d len=%"PRIu64" k=%u \nbuffer=%s\n", + __func__, __LINE__, len, k, buffer); + + recordFrom ++; + + if (recordFrom >= insertRows) { + break; + } } - if ((lenOfRow + 1) > remainderBufLen) { - break; - } - - pstr += snprintf(pstr , lenOfRow + 1, "%s", data); - k++; - len += lenOfRow; - remainderBufLen -= lenOfRow; - - verbosePrint("%s() LN%d len=%"PRIu64" k=%u \nbuffer=%s\n", - __func__, __LINE__, len, k, buffer); - - recordFrom ++; - - if (recordFrom >= insertRows) { - break; - } - } - - *dataLen = len; - return k; + *dataLen = len; + return k; } @@ -5381,52 +5387,52 @@ static int32_t generateStbInterlaceData( int64_t startTime, uint64_t *pRemainderBufLen) { - assert(buffer); - char *pstr = buffer; + assert(buffer); + char *pstr = buffer; - int headLen = generateStbSQLHead( - superTblInfo, - tableName, tableSeq, pThreadInfo->db_name, - pstr, *pRemainderBufLen); + int headLen = generateStbSQLHead( + superTblInfo, + tableName, tableSeq, pThreadInfo->db_name, + pstr, *pRemainderBufLen); - if (headLen <= 0) { - return 0; - } - // generate data buffer - verbosePrint("[%d] %s() LN%d i=%"PRIu64" buffer:\n%s\n", + if (headLen <= 0) { + return 0; + } + // generate data buffer + verbosePrint("[%d] %s() LN%d i=%"PRIu64" buffer:\n%s\n", pThreadInfo->threadID, __func__, __LINE__, i, buffer); - pstr += headLen; - *pRemainderBufLen -= headLen; + pstr += headLen; + *pRemainderBufLen -= headLen; - int64_t dataLen = 0; + int64_t dataLen = 0; - verbosePrint("[%d] %s() LN%d i=%"PRIu64" batchPerTblTimes=%u batchPerTbl = %u\n", + verbosePrint("[%d] %s() LN%d i=%"PRIu64" batchPerTblTimes=%u batchPerTbl = %u\n", pThreadInfo->threadID, __func__, __LINE__, i, batchPerTblTimes, batchPerTbl); - if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { - startTime = taosGetTimestamp(pThreadInfo->time_precision); - } + if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { + startTime = taosGetTimestamp(pThreadInfo->time_precision); + } - int32_t k = generateStbDataTail( + int32_t k = generateStbDataTail( superTblInfo, batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0, startTime, &(pThreadInfo->samplePos), &dataLen); - if (k == batchPerTbl) { - pstr += dataLen; - *pRemainderBufLen -= dataLen; - } else { - debugPrint("%s() LN%d, generated data tail: %u, not equal batch per table: %u\n", - __func__, __LINE__, k, batchPerTbl); - pstr -= headLen; - pstr[0] = '\0'; - k = 0; - } + if (k == batchPerTbl) { + pstr += dataLen; + *pRemainderBufLen -= dataLen; + } else { + debugPrint("%s() LN%d, generated data tail: %u, not equal batch per table: %u\n", + __func__, __LINE__, k, batchPerTbl); + pstr -= headLen; + pstr[0] = '\0'; + k = 0; + } - return k; + return k; } static int64_t generateInterlaceDataWithoutStb( @@ -5874,7 +5880,7 @@ static int32_t prepareStbStmtInterlace( stbInfo, stmt, tableName, - g_args.num_of_RPR, + batch, insertRows, 0, startTime, pSamplePos); } @@ -5892,7 +5898,7 @@ static int32_t prepareStbStmtProgressive( stbInfo, stmt, tableName, - g_args.num_of_RPR, + batch, insertRows, recordFrom, startTime, pSamplePos); } @@ -5908,29 +5914,29 @@ static int32_t generateStbProgressiveData( uint64_t recordFrom, int64_t startTime, int64_t *pSamplePos, int64_t *pRemainderBufLen) { - assert(buffer != NULL); - char *pstr = buffer; + assert(buffer != NULL); + char *pstr = buffer; - memset(buffer, 0, *pRemainderBufLen); + memset(buffer, 0, *pRemainderBufLen); - int64_t headLen = generateStbSQLHead( - superTblInfo, - tableName, tableSeq, dbName, - buffer, *pRemainderBufLen); + int64_t headLen = generateStbSQLHead( + superTblInfo, + tableName, tableSeq, dbName, + buffer, *pRemainderBufLen); - if (headLen <= 0) { - return 0; - } - pstr += headLen; - *pRemainderBufLen -= headLen; + if (headLen <= 0) { + return 0; + } + pstr += headLen; + *pRemainderBufLen -= headLen; - int64_t dataLen; + int64_t dataLen; - return generateStbDataTail(superTblInfo, - g_args.num_of_RPR, pstr, *pRemainderBufLen, - insertRows, recordFrom, - startTime, - pSamplePos, &dataLen); + return generateStbDataTail(superTblInfo, + g_args.num_of_RPR, pstr, *pRemainderBufLen, + insertRows, recordFrom, + startTime, + pSamplePos, &dataLen); } static int32_t generateProgressiveDataWithoutStb( @@ -5977,283 +5983,283 @@ static void printStatPerThread(threadInfo *pThreadInfo) // sync write interlace data static void* syncWriteInterlace(threadInfo *pThreadInfo) { - debugPrint("[%d] %s() LN%d: ### interlace write\n", - pThreadInfo->threadID, __func__, __LINE__); - - int64_t insertRows; - uint32_t interlaceRows; - uint64_t maxSqlLen; - int64_t nTimeStampStep; - uint64_t insert_interval; - - SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - - if (superTblInfo) { - insertRows = superTblInfo->insertRows; - - if ((superTblInfo->interlaceRows == 0) - && (g_args.interlace_rows > 0)) { - interlaceRows = g_args.interlace_rows; - } else { - interlaceRows = superTblInfo->interlaceRows; - } - maxSqlLen = superTblInfo->maxSqlLen; - nTimeStampStep = superTblInfo->timeStampStep; - insert_interval = superTblInfo->insertInterval; - } else { - insertRows = g_args.num_of_DPT; - interlaceRows = g_args.interlace_rows; - maxSqlLen = g_args.max_sql_len; - nTimeStampStep = DEFAULT_TIMESTAMP_STEP; - insert_interval = g_args.insert_interval; - } - - debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n", - pThreadInfo->threadID, __func__, __LINE__, - pThreadInfo->start_table_from, - pThreadInfo->ntables, insertRows); - - if (interlaceRows > insertRows) - interlaceRows = insertRows; - - if (interlaceRows > g_args.num_of_RPR) - interlaceRows = g_args.num_of_RPR; - - uint32_t batchPerTbl = interlaceRows; - uint32_t batchPerTblTimes; - - if ((interlaceRows > 0) && (pThreadInfo->ntables > 1)) { - batchPerTblTimes = - g_args.num_of_RPR / interlaceRows; - } else { - batchPerTblTimes = 1; - } - - pThreadInfo->buffer = calloc(maxSqlLen, 1); - if (NULL == pThreadInfo->buffer) { - errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n", - __func__, __LINE__, maxSqlLen, strerror(errno)); - return NULL; - } - - pThreadInfo->totalInsertRows = 0; - pThreadInfo->totalAffectedRows = 0; - - uint64_t st = 0; - uint64_t et = UINT64_MAX; - - uint64_t lastPrintTime = taosGetTimestampMs(); - uint64_t startTs = taosGetTimestampMs(); - uint64_t endTs; - - uint64_t tableSeq = pThreadInfo->start_table_from; - int64_t startTime = pThreadInfo->start_time; - - uint64_t generatedRecPerTbl = 0; - bool flagSleep = true; - uint64_t sleepTimeTotal = 0; - - while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) { - if ((flagSleep) && (insert_interval)) { - st = taosGetTimestampMs(); - flagSleep = false; - } - // generate data - memset(pThreadInfo->buffer, 0, maxSqlLen); - uint64_t remainderBufLen = maxSqlLen; - - char *pstr = pThreadInfo->buffer; - - int len = snprintf(pstr, - strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO); - pstr += len; - remainderBufLen -= len; - - uint32_t recOfBatch = 0; - - for (uint64_t i = 0; i < batchPerTblTimes; i ++) { - char tableName[TSDB_TABLE_NAME_LEN]; - - getTableName(tableName, pThreadInfo, tableSeq); - if (0 == strlen(tableName)) { - errorPrint("[%d] %s() LN%d, getTableName return null\n", + debugPrint("[%d] %s() LN%d: ### interlace write\n", pThreadInfo->threadID, __func__, __LINE__); - free(pThreadInfo->buffer); - return NULL; - } - uint64_t oldRemainderLen = remainderBufLen; + int64_t insertRows; + uint32_t interlaceRows; + uint64_t maxSqlLen; + int64_t nTimeStampStep; + uint64_t insert_interval; - int32_t generated; - if (superTblInfo) { - if (superTblInfo->iface == STMT_IFACE) { -#if STMT_IFACE_ENABLED == 1 - generated = prepareStbStmtInterlace( - superTblInfo, - pThreadInfo->stmt, - tableName, - batchPerTbl, - insertRows, i, - startTime, - &(pThreadInfo->samplePos)); -#else - generated = -1; -#endif - } else { - generated = generateStbInterlaceData( - superTblInfo, - tableName, batchPerTbl, i, - batchPerTblTimes, - tableSeq, - pThreadInfo, pstr, - insertRows, - startTime, - &remainderBufLen); - } - } else { - if (g_args.iface == STMT_IFACE) { - debugPrint("[%d] %s() LN%d, tableName:%s, batch:%d startTime:%"PRId64"\n", - pThreadInfo->threadID, - __func__, __LINE__, - tableName, batchPerTbl, startTime); -#if STMT_IFACE_ENABLED == 1 - generated = prepareStmtWithoutStb( - pThreadInfo->stmt, tableName, - batchPerTbl, - insertRows, i, - startTime); -#else - generated = -1; -#endif - } else { - generated = generateInterlaceDataWithoutStb( - tableName, batchPerTbl, - tableSeq, - pThreadInfo->db_name, pstr, - insertRows, - startTime, - &remainderBufLen); - } - } + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - debugPrint("[%d] %s() LN%d, generated records is %d\n", - pThreadInfo->threadID, __func__, __LINE__, generated); - if (generated < 0) { - errorPrint("[%d] %s() LN%d, generated records is %d\n", - pThreadInfo->threadID, __func__, __LINE__, generated); - goto free_of_interlace; - } else if (generated == 0) { - break; - } + if (superTblInfo) { + insertRows = superTblInfo->insertRows; - tableSeq ++; - recOfBatch += batchPerTbl; - - pstr += (oldRemainderLen - remainderBufLen); - pThreadInfo->totalInsertRows += batchPerTbl; - - verbosePrint("[%d] %s() LN%d batchPerTbl=%d recOfBatch=%d\n", - pThreadInfo->threadID, __func__, __LINE__, - batchPerTbl, recOfBatch); - - if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) { - // turn to first table - tableSeq = pThreadInfo->start_table_from; - generatedRecPerTbl += batchPerTbl; - - startTime = pThreadInfo->start_time - + generatedRecPerTbl * nTimeStampStep; - - flagSleep = true; - if (generatedRecPerTbl >= insertRows) - break; - - int64_t remainRows = insertRows - generatedRecPerTbl; - if ((remainRows > 0) && (batchPerTbl > remainRows)) - batchPerTbl = remainRows; - - if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR) - break; - } - - verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%"PRId64" insertRows=%"PRId64"\n", - pThreadInfo->threadID, __func__, __LINE__, - generatedRecPerTbl, insertRows); - - if ((g_args.num_of_RPR - recOfBatch) < batchPerTbl) - break; - } - - verbosePrint("[%d] %s() LN%d recOfBatch=%d totalInsertRows=%"PRIu64"\n", - pThreadInfo->threadID, __func__, __LINE__, recOfBatch, - pThreadInfo->totalInsertRows); - verbosePrint("[%d] %s() LN%d, buffer=%s\n", - pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->buffer); - - startTs = taosGetTimestampMs(); - - if (recOfBatch == 0) { - errorPrint("[%d] %s() LN%d Failed to insert records of batch %d\n", - pThreadInfo->threadID, __func__, __LINE__, - batchPerTbl); - if (batchPerTbl > 0) { - errorPrint("\tIf the batch is %d, the length of the SQL to insert a row must be less then %"PRId64"\n", - batchPerTbl, maxSqlLen / batchPerTbl); + if ((superTblInfo->interlaceRows == 0) + && (g_args.interlace_rows > 0)) { + interlaceRows = g_args.interlace_rows; + } else { + interlaceRows = superTblInfo->interlaceRows; } - errorPrint("\tPlease check if the buffer length(%"PRId64") or batch(%d) is set with proper value!\n", - maxSqlLen, batchPerTbl); - goto free_of_interlace; - } - int64_t affectedRows = execInsert(pThreadInfo, recOfBatch); - - endTs = taosGetTimestampMs(); - uint64_t delay = endTs - startTs; - performancePrint("%s() LN%d, insert execution time is %"PRIu64"ms\n", - __func__, __LINE__, delay); - verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n", - pThreadInfo->threadID, - __func__, __LINE__, affectedRows); - - if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay; - if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay; - pThreadInfo->cntDelay++; - pThreadInfo->totalDelay += delay; - - if (recOfBatch != affectedRows) { - errorPrint("[%d] %s() LN%d execInsert insert %d, affected rows: %"PRId64"\n%s\n", - pThreadInfo->threadID, __func__, __LINE__, - recOfBatch, affectedRows, pThreadInfo->buffer); - goto free_of_interlace; + maxSqlLen = superTblInfo->maxSqlLen; + nTimeStampStep = superTblInfo->timeStampStep; + insert_interval = superTblInfo->insertInterval; + } else { + insertRows = g_args.num_of_DPT; + interlaceRows = g_args.interlace_rows; + maxSqlLen = g_args.max_sql_len; + nTimeStampStep = DEFAULT_TIMESTAMP_STEP; + insert_interval = g_args.insert_interval; } - pThreadInfo->totalAffectedRows += affectedRows; + debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n", + pThreadInfo->threadID, __func__, __LINE__, + pThreadInfo->start_table_from, + pThreadInfo->ntables, insertRows); - int64_t currentPrintTime = taosGetTimestampMs(); - if (currentPrintTime - lastPrintTime > 30*1000) { - printf("thread[%d] has currently inserted rows: %"PRIu64 ", affected rows: %"PRIu64 "\n", + if (interlaceRows > insertRows) + interlaceRows = insertRows; + + if (interlaceRows > g_args.num_of_RPR) + interlaceRows = g_args.num_of_RPR; + + uint32_t batchPerTbl = interlaceRows; + uint32_t batchPerTblTimes; + + if ((interlaceRows > 0) && (pThreadInfo->ntables > 1)) { + batchPerTblTimes = + g_args.num_of_RPR / interlaceRows; + } else { + batchPerTblTimes = 1; + } + + pThreadInfo->buffer = calloc(maxSqlLen, 1); + if (NULL == pThreadInfo->buffer) { + errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n", + __func__, __LINE__, maxSqlLen, strerror(errno)); + return NULL; + } + + pThreadInfo->totalInsertRows = 0; + pThreadInfo->totalAffectedRows = 0; + + uint64_t st = 0; + uint64_t et = UINT64_MAX; + + uint64_t lastPrintTime = taosGetTimestampMs(); + uint64_t startTs = taosGetTimestampMs(); + uint64_t endTs; + + uint64_t tableSeq = pThreadInfo->start_table_from; + int64_t startTime = pThreadInfo->start_time; + + uint64_t generatedRecPerTbl = 0; + bool flagSleep = true; + uint64_t sleepTimeTotal = 0; + + while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) { + if ((flagSleep) && (insert_interval)) { + st = taosGetTimestampMs(); + flagSleep = false; + } + // generate data + memset(pThreadInfo->buffer, 0, maxSqlLen); + uint64_t remainderBufLen = maxSqlLen; + + char *pstr = pThreadInfo->buffer; + + int len = snprintf(pstr, + strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO); + pstr += len; + remainderBufLen -= len; + + uint32_t recOfBatch = 0; + + for (uint64_t i = 0; i < batchPerTblTimes; i ++) { + char tableName[TSDB_TABLE_NAME_LEN]; + + getTableName(tableName, pThreadInfo, tableSeq); + if (0 == strlen(tableName)) { + errorPrint("[%d] %s() LN%d, getTableName return null\n", + pThreadInfo->threadID, __func__, __LINE__); + free(pThreadInfo->buffer); + return NULL; + } + + uint64_t oldRemainderLen = remainderBufLen; + + int32_t generated; + if (superTblInfo) { + if (superTblInfo->iface == STMT_IFACE) { +#if STMT_IFACE_ENABLED == 1 + generated = prepareStbStmtInterlace( + superTblInfo, + pThreadInfo->stmt, + tableName, + batchPerTbl, + insertRows, i, + startTime, + &(pThreadInfo->samplePos)); +#else + generated = -1; +#endif + } else { + generated = generateStbInterlaceData( + superTblInfo, + tableName, batchPerTbl, i, + batchPerTblTimes, + tableSeq, + pThreadInfo, pstr, + insertRows, + startTime, + &remainderBufLen); + } + } else { + if (g_args.iface == STMT_IFACE) { + debugPrint("[%d] %s() LN%d, tableName:%s, batch:%d startTime:%"PRId64"\n", + pThreadInfo->threadID, + __func__, __LINE__, + tableName, batchPerTbl, startTime); +#if STMT_IFACE_ENABLED == 1 + generated = prepareStmtWithoutStb( + pThreadInfo->stmt, tableName, + batchPerTbl, + insertRows, i, + startTime); +#else + generated = -1; +#endif + } else { + generated = generateInterlaceDataWithoutStb( + tableName, batchPerTbl, + tableSeq, + pThreadInfo->db_name, pstr, + insertRows, + startTime, + &remainderBufLen); + } + } + + debugPrint("[%d] %s() LN%d, generated records is %d\n", + pThreadInfo->threadID, __func__, __LINE__, generated); + if (generated < 0) { + errorPrint("[%d] %s() LN%d, generated records is %d\n", + pThreadInfo->threadID, __func__, __LINE__, generated); + goto free_of_interlace; + } else if (generated == 0) { + break; + } + + tableSeq ++; + recOfBatch += batchPerTbl; + + pstr += (oldRemainderLen - remainderBufLen); + pThreadInfo->totalInsertRows += batchPerTbl; + + verbosePrint("[%d] %s() LN%d batchPerTbl=%d recOfBatch=%d\n", + pThreadInfo->threadID, __func__, __LINE__, + batchPerTbl, recOfBatch); + + if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) { + // turn to first table + tableSeq = pThreadInfo->start_table_from; + generatedRecPerTbl += batchPerTbl; + + startTime = pThreadInfo->start_time + + generatedRecPerTbl * nTimeStampStep; + + flagSleep = true; + if (generatedRecPerTbl >= insertRows) + break; + + int64_t remainRows = insertRows - generatedRecPerTbl; + if ((remainRows > 0) && (batchPerTbl > remainRows)) + batchPerTbl = remainRows; + + if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR) + break; + } + + verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%"PRId64" insertRows=%"PRId64"\n", + pThreadInfo->threadID, __func__, __LINE__, + generatedRecPerTbl, insertRows); + + if ((g_args.num_of_RPR - recOfBatch) < batchPerTbl) + break; + } + + verbosePrint("[%d] %s() LN%d recOfBatch=%d totalInsertRows=%"PRIu64"\n", + pThreadInfo->threadID, __func__, __LINE__, recOfBatch, + pThreadInfo->totalInsertRows); + verbosePrint("[%d] %s() LN%d, buffer=%s\n", + pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->buffer); + + startTs = taosGetTimestampMs(); + + if (recOfBatch == 0) { + errorPrint("[%d] %s() LN%d Failed to insert records of batch %d\n", + pThreadInfo->threadID, __func__, __LINE__, + batchPerTbl); + if (batchPerTbl > 0) { + errorPrint("\tIf the batch is %d, the length of the SQL to insert a row must be less then %"PRId64"\n", + batchPerTbl, maxSqlLen / batchPerTbl); + } + errorPrint("\tPlease check if the buffer length(%"PRId64") or batch(%d) is set with proper value!\n", + maxSqlLen, batchPerTbl); + goto free_of_interlace; + } + int64_t affectedRows = execInsert(pThreadInfo, recOfBatch); + + endTs = taosGetTimestampMs(); + uint64_t delay = endTs - startTs; + performancePrint("%s() LN%d, insert execution time is %"PRIu64"ms\n", + __func__, __LINE__, delay); + verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n", + pThreadInfo->threadID, + __func__, __LINE__, affectedRows); + + if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay; + if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay; + pThreadInfo->cntDelay++; + pThreadInfo->totalDelay += delay; + + if (recOfBatch != affectedRows) { + errorPrint("[%d] %s() LN%d execInsert insert %d, affected rows: %"PRId64"\n%s\n", + pThreadInfo->threadID, __func__, __LINE__, + recOfBatch, affectedRows, pThreadInfo->buffer); + goto free_of_interlace; + } + + pThreadInfo->totalAffectedRows += affectedRows; + + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30*1000) { + printf("thread[%d] has currently inserted rows: %"PRIu64 ", affected rows: %"PRIu64 "\n", pThreadInfo->threadID, pThreadInfo->totalInsertRows, pThreadInfo->totalAffectedRows); - lastPrintTime = currentPrintTime; - } + lastPrintTime = currentPrintTime; + } - if ((insert_interval) && flagSleep) { - et = taosGetTimestampMs(); + if ((insert_interval) && flagSleep) { + et = taosGetTimestampMs(); - if (insert_interval > (et - st) ) { - uint64_t sleepTime = insert_interval - (et -st); - performancePrint("%s() LN%d sleep: %"PRId64" ms for insert interval\n", - __func__, __LINE__, sleepTime); - taosMsleep(sleepTime); // ms - sleepTimeTotal += insert_interval; - } + if (insert_interval > (et - st) ) { + uint64_t sleepTime = insert_interval - (et -st); + performancePrint("%s() LN%d sleep: %"PRId64" ms for insert interval\n", + __func__, __LINE__, sleepTime); + taosMsleep(sleepTime); // ms + sleepTimeTotal += insert_interval; + } + } } - } free_of_interlace: - tmfree(pThreadInfo->buffer); - printStatPerThread(pThreadInfo); - return NULL; + tmfree(pThreadInfo->buffer); + printStatPerThread(pThreadInfo); + return NULL; } // sync insertion progressive data @@ -6420,29 +6426,29 @@ free_of_progressive: static void* syncWrite(void *sarg) { - threadInfo *pThreadInfo = (threadInfo *)sarg; - SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + threadInfo *pThreadInfo = (threadInfo *)sarg; + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - uint32_t interlaceRows; + uint32_t interlaceRows; - if (superTblInfo) { - if ((superTblInfo->interlaceRows == 0) - && (g_args.interlace_rows > 0)) { - interlaceRows = g_args.interlace_rows; + if (superTblInfo) { + if ((superTblInfo->interlaceRows == 0) + && (g_args.interlace_rows > 0)) { + interlaceRows = g_args.interlace_rows; + } else { + interlaceRows = superTblInfo->interlaceRows; + } } else { - interlaceRows = superTblInfo->interlaceRows; + interlaceRows = g_args.interlace_rows; } - } else { - interlaceRows = g_args.interlace_rows; - } - if (interlaceRows > 0) { - // interlace mode - return syncWriteInterlace(pThreadInfo); - } else { - // progressive mode - return syncWriteProgressive(pThreadInfo); - } + if (interlaceRows > 0) { + // interlace mode + return syncWriteInterlace(pThreadInfo); + } else { + // progressive mode + return syncWriteProgressive(pThreadInfo); + } } static void callBack(void *param, TAOS_RES *res, int code) { From 211c7f38ce80029658ffb04427a96167d636fef7 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 20 Jul 2021 11:49:58 +0800 Subject: [PATCH 016/106] [TD-5398]: taosdemo 4096 columns table creation failed. (#6931) * [TD-5398]: taosdemo 4096 columns table creation failed. * change column name prefix from col to c for maxium column number to fix buffer length. * use accurate buffer length. * fix test case for column name be shorter. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 392 +++++++++--------- tests/pytest/query/query1970YearsAf.py | 8 +- .../TD-4985/query-limit-offset.py | 51 ++- .../taosdemoTestInsertWithJson.py | 152 ++++--- tests/pytest/tools/taosdemoTest.py | 4 +- tests/pytest/tools/taosdemoTestTblAlt.py | 4 +- 6 files changed, 304 insertions(+), 307 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 59c49a0c93..3d953edc61 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -71,16 +71,15 @@ extern char configDir[]; #define HEAD_BUFF_LEN TSDB_MAX_COLUMNS*24 // 16*MAX_COLUMNS + (192+32)*2 + insert into .. -#define MAX_SQL_SIZE 65536 -#define BUFFER_SIZE (65536*2) -#define COND_BUF_LEN (BUFFER_SIZE - 30) +#define COL_BUFFER_LEN (TSDB_MAX_BYTES_PER_ROW - 50) +#define BUFFER_SIZE (50 + TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_MAX_BYTES_PER_ROW + TSDB_MAX_TAGS_LEN) +#define COND_BUF_LEN (BUFFER_SIZE - 30) #define MAX_USERNAME_SIZE 64 #define MAX_PASSWORD_SIZE 64 #define MAX_HOSTNAME_SIZE 64 #define MAX_TB_NAME_SIZE 64 #define MAX_DATA_SIZE (16*TSDB_MAX_COLUMNS)+20 // max record len: 16*MAX_COLUMNS, timestamp string and ,('') need extra space #define OPT_ABORT 1 /* –abort */ -#define STRING_LEN 60000 #define MAX_PREPARED_RAND 1000000 #define MAX_FILE_NAME_LEN 256 // max file name length on linux is 255. @@ -2710,7 +2709,7 @@ static int createSuperTable( char command[BUFFER_SIZE] = "\0"; - char cols[STRING_LEN] = "\0"; + char cols[COL_BUFFER_LEN] = "\0"; int colIndex; int len = 0; @@ -2726,55 +2725,55 @@ static int createSuperTable( char* dataType = superTbl->columns[colIndex].dataType; if (strcasecmp(dataType, "BINARY") == 0) { - len += snprintf(cols + len, STRING_LEN - len, - ", col%d %s(%d)", colIndex, "BINARY", + len += snprintf(cols + len, COL_BUFFER_LEN - len, + ", C%d %s(%d)", colIndex, "BINARY", superTbl->columns[colIndex].dataLen); lenOfOneRow += superTbl->columns[colIndex].dataLen + 3; } else if (strcasecmp(dataType, "NCHAR") == 0) { - len += snprintf(cols + len, STRING_LEN - len, - ", col%d %s(%d)", colIndex, "NCHAR", + len += snprintf(cols + len, COL_BUFFER_LEN - len, + ", C%d %s(%d)", colIndex, "NCHAR", superTbl->columns[colIndex].dataLen); lenOfOneRow += superTbl->columns[colIndex].dataLen + 3; } else if (strcasecmp(dataType, "INT") == 0) { if ((g_args.demo_mode) && (colIndex == 1)) { - len += snprintf(cols + len, STRING_LEN - len, + len += snprintf(cols + len, COL_BUFFER_LEN - len, ", VOLTAGE INT"); } else { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT"); + len += snprintf(cols + len, COL_BUFFER_LEN - len, ", C%d %s", colIndex, "INT"); } lenOfOneRow += 11; } else if (strcasecmp(dataType, "BIGINT") == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", + len += snprintf(cols + len, COL_BUFFER_LEN - len, ", C%d %s", colIndex, "BIGINT"); lenOfOneRow += 21; } else if (strcasecmp(dataType, "SMALLINT") == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", + len += snprintf(cols + len, COL_BUFFER_LEN - len, ", C%d %s", colIndex, "SMALLINT"); lenOfOneRow += 6; } else if (strcasecmp(dataType, "TINYINT") == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TINYINT"); + len += snprintf(cols + len, COL_BUFFER_LEN - len, ", C%d %s", colIndex, "TINYINT"); lenOfOneRow += 4; } else if (strcasecmp(dataType, "BOOL") == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BOOL"); + len += snprintf(cols + len, COL_BUFFER_LEN - len, ", C%d %s", colIndex, "BOOL"); lenOfOneRow += 6; } else if (strcasecmp(dataType, "FLOAT") == 0) { if (g_args.demo_mode) { if (colIndex == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", CURRENT FLOAT"); + len += snprintf(cols + len, COL_BUFFER_LEN - len, ", CURRENT FLOAT"); } else if (colIndex == 2) { - len += snprintf(cols + len, STRING_LEN - len, ", PHASE FLOAT"); + len += snprintf(cols + len, COL_BUFFER_LEN - len, ", PHASE FLOAT"); } } else { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "FLOAT"); + len += snprintf(cols + len, COL_BUFFER_LEN - len, ", C%d %s", colIndex, "FLOAT"); } lenOfOneRow += 22; } else if (strcasecmp(dataType, "DOUBLE") == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", + len += snprintf(cols + len, COL_BUFFER_LEN - len, ", C%d %s", colIndex, "DOUBLE"); lenOfOneRow += 42; } else if (strcasecmp(dataType, "TIMESTAMP") == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", + len += snprintf(cols + len, COL_BUFFER_LEN - len, ", C%d %s", colIndex, "TIMESTAMP"); lenOfOneRow += 21; } else { @@ -2806,60 +2805,63 @@ static int createSuperTable( return -1; } - char tags[STRING_LEN] = "\0"; + char tags[TSDB_MAX_TAGS_LEN] = "\0"; int tagIndex; len = 0; int lenOfTagOfOneRow = 0; - len += snprintf(tags + len, STRING_LEN - len, "("); + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, "("); for (tagIndex = 0; tagIndex < superTbl->tagCount; tagIndex++) { char* dataType = superTbl->tags[tagIndex].dataType; if (strcasecmp(dataType, "BINARY") == 0) { if ((g_args.demo_mode) && (tagIndex == 1)) { - len += snprintf(tags + len, STRING_LEN - len, + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, "location BINARY(%d), ", superTbl->tags[tagIndex].dataLen); } else { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", - tagIndex, "BINARY", superTbl->tags[tagIndex].dataLen); + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, + "t%d %s(%d), ", tagIndex, "BINARY", + superTbl->tags[tagIndex].dataLen); } lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3; } else if (strcasecmp(dataType, "NCHAR") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex, + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, + "t%d %s(%d), ", tagIndex, "NCHAR", superTbl->tags[tagIndex].dataLen); lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3; } else if (strcasecmp(dataType, "INT") == 0) { if ((g_args.demo_mode) && (tagIndex == 0)) { - len += snprintf(tags + len, STRING_LEN - len, "groupId INT, "); + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, + "groupId INT, "); } else { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, - "INT"); + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, + "t%d %s, ", tagIndex, "INT"); } lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 11; } else if (strcasecmp(dataType, "BIGINT") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, - "BIGINT"); + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, + "t%d %s, ", tagIndex, "BIGINT"); lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 21; } else if (strcasecmp(dataType, "SMALLINT") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, - "SMALLINT"); + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, + "t%d %s, ", tagIndex, "SMALLINT"); lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6; } else if (strcasecmp(dataType, "TINYINT") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, - "TINYINT"); + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, + "t%d %s, ", tagIndex, "TINYINT"); lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 4; } else if (strcasecmp(dataType, "BOOL") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, - "BOOL"); + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, + "t%d %s, ", tagIndex, "BOOL"); lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6; } else if (strcasecmp(dataType, "FLOAT") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, - "FLOAT"); + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, + "t%d %s, ", tagIndex, "FLOAT"); lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 22; } else if (strcasecmp(dataType, "DOUBLE") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, - "DOUBLE"); + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, + "t%d %s, ", tagIndex, "DOUBLE"); lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 42; } else { taos_close(taos); @@ -2870,7 +2872,7 @@ static int createSuperTable( } len -= 2; - len += snprintf(tags + len, STRING_LEN - len, ")"); + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, ")"); superTbl->lenOfTagOfOneRow = lenOfTagOfOneRow; @@ -3023,175 +3025,175 @@ static int createDatabasesAndStables() { static void* createTable(void *sarg) { - threadInfo *pThreadInfo = (threadInfo *)sarg; - SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + threadInfo *pThreadInfo = (threadInfo *)sarg; + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - uint64_t lastPrintTime = taosGetTimestampMs(); + uint64_t lastPrintTime = taosGetTimestampMs(); - int buff_len; - buff_len = BUFFER_SIZE / 8; + int buff_len; + buff_len = BUFFER_SIZE; - pThreadInfo->buffer = calloc(buff_len, 1); - if (pThreadInfo->buffer == NULL) { - errorPrint("%s() LN%d, Memory allocated failed!\n", __func__, __LINE__); - exit(-1); - } - - int len = 0; - int batchNum = 0; - - verbosePrint("%s() LN%d: Creating table from %"PRIu64" to %"PRIu64"\n", - __func__, __LINE__, - pThreadInfo->start_table_from, pThreadInfo->end_table_to); - - for (uint64_t i = pThreadInfo->start_table_from; - i <= pThreadInfo->end_table_to; i++) { - if (0 == g_Dbs.use_metric) { - snprintf(pThreadInfo->buffer, buff_len, - "create table if not exists %s.%s%"PRIu64" %s;", - pThreadInfo->db_name, - g_args.tb_prefix, i, - pThreadInfo->cols); - } else { - if (superTblInfo == NULL) { - errorPrint("%s() LN%d, use metric, but super table info is NULL\n", - __func__, __LINE__); - free(pThreadInfo->buffer); + pThreadInfo->buffer = calloc(buff_len, 1); + if (pThreadInfo->buffer == NULL) { + errorPrint("%s() LN%d, Memory allocated failed!\n", __func__, __LINE__); exit(-1); - } else { - if (0 == len) { - batchNum = 0; - memset(pThreadInfo->buffer, 0, buff_len); - len += snprintf(pThreadInfo->buffer + len, - buff_len - len, "create table "); - } - char* tagsValBuf = NULL; - if (0 == superTblInfo->tagSource) { - tagsValBuf = generateTagVaulesForStb(superTblInfo, i); + } + + int len = 0; + int batchNum = 0; + + verbosePrint("%s() LN%d: Creating table from %"PRIu64" to %"PRIu64"\n", + __func__, __LINE__, + pThreadInfo->start_table_from, pThreadInfo->end_table_to); + + for (uint64_t i = pThreadInfo->start_table_from; + i <= pThreadInfo->end_table_to; i++) { + if (0 == g_Dbs.use_metric) { + snprintf(pThreadInfo->buffer, buff_len, + "create table if not exists %s.%s%"PRIu64" %s;", + pThreadInfo->db_name, + g_args.tb_prefix, i, + pThreadInfo->cols); } else { - tagsValBuf = getTagValueFromTagSample( - superTblInfo, - i % superTblInfo->tagSampleCount); + if (superTblInfo == NULL) { + errorPrint("%s() LN%d, use metric, but super table info is NULL\n", + __func__, __LINE__); + free(pThreadInfo->buffer); + exit(-1); + } else { + if (0 == len) { + batchNum = 0; + memset(pThreadInfo->buffer, 0, buff_len); + len += snprintf(pThreadInfo->buffer + len, + buff_len - len, "create table "); + } + char* tagsValBuf = NULL; + if (0 == superTblInfo->tagSource) { + tagsValBuf = generateTagVaulesForStb(superTblInfo, i); + } else { + tagsValBuf = getTagValueFromTagSample( + superTblInfo, + i % superTblInfo->tagSampleCount); + } + if (NULL == tagsValBuf) { + free(pThreadInfo->buffer); + return NULL; + } + len += snprintf(pThreadInfo->buffer + len, + buff_len - len, + "if not exists %s.%s%"PRIu64" using %s.%s tags %s ", + pThreadInfo->db_name, superTblInfo->childTblPrefix, + i, pThreadInfo->db_name, + superTblInfo->sTblName, tagsValBuf); + free(tagsValBuf); + batchNum++; + if ((batchNum < superTblInfo->batchCreateTableNum) + && ((buff_len - len) + >= (superTblInfo->lenOfTagOfOneRow + 256))) { + continue; + } + } } - if (NULL == tagsValBuf) { - free(pThreadInfo->buffer); - return NULL; + + len = 0; + if (0 != queryDbExec(pThreadInfo->taos, pThreadInfo->buffer, + NO_INSERT_TYPE, false)){ + errorPrint( "queryDbExec() failed. buffer:\n%s\n", pThreadInfo->buffer); + free(pThreadInfo->buffer); + return NULL; } - len += snprintf(pThreadInfo->buffer + len, - buff_len - len, - "if not exists %s.%s%"PRIu64" using %s.%s tags %s ", - pThreadInfo->db_name, superTblInfo->childTblPrefix, - i, pThreadInfo->db_name, - superTblInfo->sTblName, tagsValBuf); - free(tagsValBuf); - batchNum++; - if ((batchNum < superTblInfo->batchCreateTableNum) - && ((buff_len - len) - >= (superTblInfo->lenOfTagOfOneRow + 256))) { - continue; + + uint64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30*1000) { + printf("thread[%d] already create %"PRIu64" - %"PRIu64" tables\n", + pThreadInfo->threadID, pThreadInfo->start_table_from, i); + lastPrintTime = currentPrintTime; } - } } - len = 0; - if (0 != queryDbExec(pThreadInfo->taos, pThreadInfo->buffer, - NO_INSERT_TYPE, false)){ - errorPrint( "queryDbExec() failed. buffer:\n%s\n", pThreadInfo->buffer); - free(pThreadInfo->buffer); - return NULL; + if (0 != len) { + if (0 != queryDbExec(pThreadInfo->taos, pThreadInfo->buffer, + NO_INSERT_TYPE, false)) { + errorPrint( "queryDbExec() failed. buffer:\n%s\n", pThreadInfo->buffer); + } } - uint64_t currentPrintTime = taosGetTimestampMs(); - if (currentPrintTime - lastPrintTime > 30*1000) { - printf("thread[%d] already create %"PRIu64" - %"PRIu64" tables\n", - pThreadInfo->threadID, pThreadInfo->start_table_from, i); - lastPrintTime = currentPrintTime; - } - } - - if (0 != len) { - if (0 != queryDbExec(pThreadInfo->taos, pThreadInfo->buffer, - NO_INSERT_TYPE, false)) { - errorPrint( "queryDbExec() failed. buffer:\n%s\n", pThreadInfo->buffer); - } - } - - free(pThreadInfo->buffer); - return NULL; + free(pThreadInfo->buffer); + return NULL; } static int startMultiThreadCreateChildTable( char* cols, int threads, uint64_t tableFrom, int64_t ntables, char* db_name, SSuperTable* superTblInfo) { - pthread_t *pids = calloc(1, threads * sizeof(pthread_t)); - threadInfo *infos = calloc(1, threads * sizeof(threadInfo)); + pthread_t *pids = calloc(1, threads * sizeof(pthread_t)); + threadInfo *infos = calloc(1, threads * sizeof(threadInfo)); - if ((NULL == pids) || (NULL == infos)) { - printf("malloc failed\n"); - exit(-1); - } - - if (threads < 1) { - threads = 1; - } - - int64_t a = ntables / threads; - if (a < 1) { - threads = ntables; - a = 1; - } - - int64_t b = 0; - b = ntables % threads; - - for (int64_t i = 0; i < threads; i++) { - threadInfo *pThreadInfo = infos + i; - pThreadInfo->threadID = i; - tstrncpy(pThreadInfo->db_name, db_name, TSDB_DB_NAME_LEN); - pThreadInfo->superTblInfo = superTblInfo; - verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name); - pThreadInfo->taos = taos_connect( - g_Dbs.host, - g_Dbs.user, - g_Dbs.password, - db_name, - g_Dbs.port); - if (pThreadInfo->taos == NULL) { - errorPrint( "%s() LN%d, Failed to connect to TDengine, reason:%s\n", - __func__, __LINE__, taos_errstr(NULL)); - free(pids); - free(infos); - return -1; + if ((NULL == pids) || (NULL == infos)) { + printf("malloc failed\n"); + exit(-1); } - pThreadInfo->start_table_from = tableFrom; - pThreadInfo->ntables = iend_table_to = i < b ? tableFrom + a : tableFrom + a - 1; - tableFrom = pThreadInfo->end_table_to + 1; - pThreadInfo->use_metric = true; - pThreadInfo->cols = cols; - pThreadInfo->minDelay = UINT64_MAX; - pthread_create(pids + i, NULL, createTable, pThreadInfo); - } + if (threads < 1) { + threads = 1; + } - for (int i = 0; i < threads; i++) { - pthread_join(pids[i], NULL); - } + int64_t a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } - for (int i = 0; i < threads; i++) { - threadInfo *pThreadInfo = infos + i; - taos_close(pThreadInfo->taos); - } + int64_t b = 0; + b = ntables % threads; - free(pids); - free(infos); + for (int64_t i = 0; i < threads; i++) { + threadInfo *pThreadInfo = infos + i; + pThreadInfo->threadID = i; + tstrncpy(pThreadInfo->db_name, db_name, TSDB_DB_NAME_LEN); + pThreadInfo->superTblInfo = superTblInfo; + verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name); + pThreadInfo->taos = taos_connect( + g_Dbs.host, + g_Dbs.user, + g_Dbs.password, + db_name, + g_Dbs.port); + if (pThreadInfo->taos == NULL) { + errorPrint( "%s() LN%d, Failed to connect to TDengine, reason:%s\n", + __func__, __LINE__, taos_errstr(NULL)); + free(pids); + free(infos); + return -1; + } - return 0; + pThreadInfo->start_table_from = tableFrom; + pThreadInfo->ntables = iend_table_to = i < b ? tableFrom + a : tableFrom + a - 1; + tableFrom = pThreadInfo->end_table_to + 1; + pThreadInfo->use_metric = true; + pThreadInfo->cols = cols; + pThreadInfo->minDelay = UINT64_MAX; + pthread_create(pids + i, NULL, createTable, pThreadInfo); + } + + for (int i = 0; i < threads; i++) { + pthread_join(pids[i], NULL); + } + + for (int i = 0; i < threads; i++) { + threadInfo *pThreadInfo = infos + i; + taos_close(pThreadInfo->taos); + } + + free(pids); + free(infos); + + return 0; } static void createChildTables() { - char tblColsBuf[MAX_SQL_SIZE]; + char tblColsBuf[TSDB_MAX_BYTES_PER_ROW]; int len; for (int i = 0; i < g_Dbs.dbCount; i++) { @@ -3223,21 +3225,21 @@ static void createChildTables() { } } else { // normal table - len = snprintf(tblColsBuf, MAX_SQL_SIZE, "(TS TIMESTAMP"); + len = snprintf(tblColsBuf, TSDB_MAX_BYTES_PER_ROW, "(TS TIMESTAMP"); for (int j = 0; j < g_args.num_of_CPR; j++) { if ((strncasecmp(g_args.datatype[j], "BINARY", strlen("BINARY")) == 0) || (strncasecmp(g_args.datatype[j], "NCHAR", strlen("NCHAR")) == 0)) { - snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, - ", COL%d %s(%d)", j, g_args.datatype[j], g_args.len_of_binary); + snprintf(tblColsBuf + len, TSDB_MAX_BYTES_PER_ROW - len, + ",C%d %s(%d)", j, g_args.datatype[j], g_args.len_of_binary); } else { - snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, - ", COL%d %s", j, g_args.datatype[j]); + snprintf(tblColsBuf + len, TSDB_MAX_BYTES_PER_ROW - len, + ",C%d %s", j, g_args.datatype[j]); } len = strlen(tblColsBuf); } - snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")"); + snprintf(tblColsBuf + len, TSDB_MAX_BYTES_PER_ROW - len, ")"); verbosePrint("%s() LN%d: dbName: %s num of tb: %"PRId64" schema: %s\n", __func__, __LINE__, @@ -7989,7 +7991,7 @@ static void initOfQueryMeta() { tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE); } -static void setParaFromArg(){ +static void setParaFromArg() { if (g_args.host) { tstrncpy(g_Dbs.host, g_args.host, MAX_HOSTNAME_SIZE); } else { @@ -8025,10 +8027,10 @@ static void setParaFromArg(){ g_Dbs.do_aggreFunc = true; - char dataString[STRING_LEN]; + char dataString[TSDB_MAX_BYTES_PER_ROW]; char **data_type = g_args.datatype; - memset(dataString, 0, STRING_LEN); + memset(dataString, 0, TSDB_MAX_BYTES_PER_ROW); if (strcasecmp(data_type[0], "BINARY") == 0 || strcasecmp(data_type[0], "BOOL") == 0 @@ -8146,7 +8148,7 @@ static void querySqlFile(TAOS* taos, char* sqlFile) } int read_len = 0; - char * cmd = calloc(1, MAX_SQL_SIZE); + char * cmd = calloc(1, TSDB_MAX_BYTES_PER_ROW); size_t cmd_len = 0; char * line = NULL; size_t line_len = 0; @@ -8154,7 +8156,7 @@ static void querySqlFile(TAOS* taos, char* sqlFile) double t = taosGetTimestampMs(); while((read_len = tgetline(&line, &line_len, fp)) != -1) { - if (read_len >= MAX_SQL_SIZE) continue; + if (read_len >= TSDB_MAX_BYTES_PER_ROW) continue; line[--read_len] = '\0'; if (read_len == 0 || isCommentLine(line)) { // line starts with # @@ -8177,7 +8179,7 @@ static void querySqlFile(TAOS* taos, char* sqlFile) tmfclose(fp); return; } - memset(cmd, 0, MAX_SQL_SIZE); + memset(cmd, 0, TSDB_MAX_BYTES_PER_ROW); cmd_len = 0; } diff --git a/tests/pytest/query/query1970YearsAf.py b/tests/pytest/query/query1970YearsAf.py index 93404afd59..a365369b21 100644 --- a/tests/pytest/query/query1970YearsAf.py +++ b/tests/pytest/query/query1970YearsAf.py @@ -187,19 +187,19 @@ class TDTestCase: "select * from t9 where t9.ts > '1969-12-31 22:00:00.000' and t9.ts <'1970-01-01 02:00:00.000' " ) tdSql.checkRows(719) - + tdSql.query( "select * from t0,t1 where t0.ts=t1.ts and t1.ts >= '1970-01-01 00:00:00.000' " ) tdSql.checkRows(680) - + tdSql.query( - "select diff(col1) from t0 where t0.ts >= '1970-01-01 00:00:00.000' " + "select diff(c1) from t0 where t0.ts >= '1970-01-01 00:00:00.000' " ) tdSql.checkRows(679) tdSql.query( - "select t0,col1 from stb2 where stb2.ts < '1970-01-01 00:00:00.000' order by ts" + "select t0,c1 from stb2 where stb2.ts < '1970-01-01 00:00:00.000' order by ts" ) tdSql.checkRows(43200) diff --git a/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.py b/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.py index 081057f180..aa2f96940c 100644 --- a/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.py +++ b/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -39,7 +39,7 @@ class TDTestCase: buildPath = root[:len(root)-len("/build/bin")] break return buildPath - + def run(self): buildPath = self.getBuildPath() if (buildPath == ""): @@ -48,7 +48,7 @@ class TDTestCase: tdLog.info("taosd found in %s" % buildPath) binPath = buildPath+ "/build/bin/" - # insert: create one or mutiple tables per sql and insert multiple rows per sql + # insert: create one or mutiple tables per sql and insert multiple rows per sql # test case for https://jira.taosdata.com:18080/browse/TD-4985 os.system("%staosdemo -f tools/taosdemoAllTest/TD-4985/query-limit-offset.json -y " % binPath) tdSql.execute("use db") @@ -56,27 +56,27 @@ class TDTestCase: tdSql.checkData(0, 0, 10000) for i in range(1000): - tdSql.execute('''insert into stb00_9999 values(%d, %d, %d,'test99.%s')''' + tdSql.execute('''insert into stb00_9999 values(%d, %d, %d,'test99.%s')''' % (1600000000000 + i, i, -10000+i, i)) - tdSql.execute('''insert into stb00_8888 values(%d, %d, %d,'test98.%s')''' + tdSql.execute('''insert into stb00_8888 values(%d, %d, %d,'test98.%s')''' % (1600000000000 + i, i, -10000+i, i)) - tdSql.execute('''insert into stb00_7777 values(%d, %d, %d,'test97.%s')''' + tdSql.execute('''insert into stb00_7777 values(%d, %d, %d,'test97.%s')''' % (1600000000000 + i, i, -10000+i, i)) - tdSql.execute('''insert into stb00_6666 values(%d, %d, %d,'test96.%s')''' + tdSql.execute('''insert into stb00_6666 values(%d, %d, %d,'test96.%s')''' % (1600000000000 + i, i, -10000+i, i)) - tdSql.execute('''insert into stb00_5555 values(%d, %d, %d,'test95.%s')''' + tdSql.execute('''insert into stb00_5555 values(%d, %d, %d,'test95.%s')''' % (1600000000000 + i, i, -10000+i, i)) - tdSql.execute('''insert into stb00_4444 values(%d, %d, %d,'test94.%s')''' + tdSql.execute('''insert into stb00_4444 values(%d, %d, %d,'test94.%s')''' % (1600000000000 + i, i, -10000+i, i)) - tdSql.execute('''insert into stb00_3333 values(%d, %d, %d,'test93.%s')''' + tdSql.execute('''insert into stb00_3333 values(%d, %d, %d,'test93.%s')''' % (1600000000000 + i, i, -10000+i, i)) - tdSql.execute('''insert into stb00_2222 values(%d, %d, %d,'test92.%s')''' + tdSql.execute('''insert into stb00_2222 values(%d, %d, %d,'test92.%s')''' % (1600000000000 + i, i, -10000+i, i)) - tdSql.execute('''insert into stb00_1111 values(%d, %d, %d,'test91.%s')''' + tdSql.execute('''insert into stb00_1111 values(%d, %d, %d,'test91.%s')''' % (1600000000000 + i, i, -10000+i, i)) - tdSql.execute('''insert into stb00_100 values(%d, %d, %d,'test90.%s')''' + tdSql.execute('''insert into stb00_100 values(%d, %d, %d,'test90.%s')''' % (1600000000000 + i, i, -10000+i, i)) - tdSql.query("select * from stb0 where col2 like 'test99%' ") + tdSql.query("select * from stb0 where c2 like 'test99%' ") tdSql.checkRows(1000) tdSql.query("select * from stb0 where tbname like 'stb00_9999' limit 10" ) tdSql.checkData(0, 1, 0) @@ -86,7 +86,7 @@ class TDTestCase: tdSql.checkData(0, 1, 5) tdSql.checkData(1, 1, 6) tdSql.checkData(2, 1, 7) - tdSql.query("select * from stb0 where col2 like 'test98%' ") + tdSql.query("select * from stb0 where c2 like 'test98%' ") tdSql.checkRows(1000) tdSql.query("select * from stb0 where tbname like 'stb00_8888' limit 10" ) tdSql.checkData(0, 1, 0) @@ -96,7 +96,7 @@ class TDTestCase: tdSql.checkData(0, 1, 5) tdSql.checkData(1, 1, 6) tdSql.checkData(2, 1, 7) - tdSql.query("select * from stb0 where col2 like 'test97%' ") + tdSql.query("select * from stb0 where c2 like 'test97%' ") tdSql.checkRows(1000) tdSql.query("select * from stb0 where tbname like 'stb00_7777' limit 10" ) tdSql.checkData(0, 1, 0) @@ -106,7 +106,7 @@ class TDTestCase: tdSql.checkData(0, 1, 5) tdSql.checkData(1, 1, 6) tdSql.checkData(2, 1, 7) - tdSql.query("select * from stb0 where col2 like 'test96%' ") + tdSql.query("select * from stb0 where c2 like 'test96%' ") tdSql.checkRows(1000) tdSql.query("select * from stb0 where tbname like 'stb00_6666' limit 10" ) tdSql.checkData(0, 1, 0) @@ -116,7 +116,7 @@ class TDTestCase: tdSql.checkData(0, 1, 5) tdSql.checkData(1, 1, 6) tdSql.checkData(2, 1, 7) - tdSql.query("select * from stb0 where col2 like 'test95%' ") + tdSql.query("select * from stb0 where c2 like 'test95%' ") tdSql.checkRows(1000) tdSql.query("select * from stb0 where tbname like 'stb00_5555' limit 10" ) tdSql.checkData(0, 1, 0) @@ -126,7 +126,7 @@ class TDTestCase: tdSql.checkData(0, 1, 5) tdSql.checkData(1, 1, 6) tdSql.checkData(2, 1, 7) - tdSql.query("select * from stb0 where col2 like 'test94%' ") + tdSql.query("select * from stb0 where c2 like 'test94%' ") tdSql.checkRows(1000) tdSql.query("select * from stb0 where tbname like 'stb00_4444' limit 10" ) tdSql.checkData(0, 1, 0) @@ -136,7 +136,7 @@ class TDTestCase: tdSql.checkData(0, 1, 5) tdSql.checkData(1, 1, 6) tdSql.checkData(2, 1, 7) - tdSql.query("select * from stb0 where col2 like 'test93%' ") + tdSql.query("select * from stb0 where c2 like 'test93%' ") tdSql.checkRows(1000) tdSql.query("select * from stb0 where tbname like 'stb00_3333' limit 100" ) tdSql.checkData(0, 1, 0) @@ -146,7 +146,7 @@ class TDTestCase: tdSql.checkData(0, 1, 5) tdSql.checkData(1, 1, 6) tdSql.checkData(2, 1, 7) - tdSql.query("select * from stb0 where col2 like 'test92%' ") + tdSql.query("select * from stb0 where c2 like 'test92%' ") tdSql.checkRows(1000) tdSql.query("select * from stb0 where tbname like 'stb00_2222' limit 100" ) tdSql.checkData(0, 1, 0) @@ -156,7 +156,7 @@ class TDTestCase: tdSql.checkData(0, 1, 5) tdSql.checkData(1, 1, 6) tdSql.checkData(2, 1, 7) - tdSql.query("select * from stb0 where col2 like 'test91%' ") + tdSql.query("select * from stb0 where c2 like 'test91%' ") tdSql.checkRows(1000) tdSql.query("select * from stb0 where tbname like 'stb00_1111' limit 100" ) tdSql.checkData(0, 1, 0) @@ -166,7 +166,7 @@ class TDTestCase: tdSql.checkData(0, 1, 5) tdSql.checkData(1, 1, 6) tdSql.checkData(2, 1, 7) - tdSql.query("select * from stb0 where col2 like 'test90%' ") + tdSql.query("select * from stb0 where c2 like 'test90%' ") tdSql.checkRows(1000) tdSql.query("select * from stb0 where tbname like 'stb00_100' limit 100" ) tdSql.checkData(0, 1, 0) @@ -177,11 +177,8 @@ class TDTestCase: tdSql.checkData(1, 1, 6) tdSql.checkData(2, 1, 7) + os.system("rm -rf tools/taosdemoAllTest/TD-4985/query-limit-offset.py.sql") - os.system("rm -rf tools/taosdemoAllTest/TD-4985/query-limit-offset.py.sql") - - - def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJson.py b/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJson.py index 01e46eaaa0..ec1df420ae 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJson.py +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJson.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -39,7 +39,7 @@ class TDTestCase: buildPath = root[:len(root)-len("/build/bin")] break return buildPath - + def run(self): buildPath = self.getBuildPath() if (buildPath == ""): @@ -48,7 +48,7 @@ class TDTestCase: tdLog.info("taosd found in %s" % buildPath) binPath = buildPath+ "/build/bin/" - # insert: create one or mutiple tables per sql and insert multiple rows per sql + # insert: create one or mutiple tables per sql and insert multiple rows per sql os.system("%staosdemo -f tools/taosdemoAllTest/insert-1s1tnt1r.json -y " % binPath) tdSql.execute("use db") tdSql.query("select count (tbname) from stb0") @@ -62,7 +62,7 @@ class TDTestCase: tdSql.query("select count(*) from stb01_1") tdSql.checkData(0, 0, 200) tdSql.query("select count(*) from stb1") - tdSql.checkData(0, 0, 200000) + tdSql.checkData(0, 0, 200000) # restful connector insert data os.system("%staosdemo -f tools/taosdemoAllTest/insertRestful.json -y " % binPath) @@ -81,7 +81,7 @@ class TDTestCase: tdSql.checkData(0, 0, 200) - # insert: create mutiple tables per sql and insert one rows per sql . + # insert: create mutiple tables per sql and insert one rows per sql . os.system("%staosdemo -f tools/taosdemoAllTest/insert-1s1tntmr.json -y " % binPath) tdSql.execute("use db") tdSql.query("select count (tbname) from stb0") @@ -89,34 +89,34 @@ class TDTestCase: tdSql.query("select count (tbname) from stb1") tdSql.checkData(0, 0, 20) tdSql.query("select count(*) from stb00_0") - tdSql.checkData(0, 0, 10000) + tdSql.checkData(0, 0, 10000) tdSql.query("select count(*) from stb0") - tdSql.checkData(0, 0, 100000) + tdSql.checkData(0, 0, 100000) tdSql.query("select count(*) from stb01_0") - tdSql.checkData(0, 0, 20000) + tdSql.checkData(0, 0, 20000) tdSql.query("select count(*) from stb1") - tdSql.checkData(0, 0, 400000) + tdSql.checkData(0, 0, 400000) - # insert: using parament "insert_interval to controls spped of insert. + # insert: using parament "insert_interval to controls spped of insert. # but We need to have accurate methods to control the speed, such as getting the speed value, checking the count and so on。 os.system("%staosdemo -f tools/taosdemoAllTest/insert-interval-speed.json -y" % binPath) tdSql.execute("use db") tdSql.query("show stables") tdSql.checkData(0, 4, 100) tdSql.query("select count(*) from stb00_0") - tdSql.checkData(0, 0, 20000) + tdSql.checkData(0, 0, 20000) tdSql.query("select count(*) from stb0") - tdSql.checkData(0, 0, 2000000) + tdSql.checkData(0, 0, 2000000) tdSql.query("show stables") tdSql.checkData(1, 4, 100) tdSql.query("select count(*) from stb01_0") - tdSql.checkData(0, 0, 20000) + tdSql.checkData(0, 0, 20000) tdSql.query("select count(*) from stb1") - tdSql.checkData(0, 0, 2000000) - + tdSql.checkData(0, 0, 2000000) + # spend 2min30s for 3 testcases. # insert: drop and child_table_exists combination test - # insert: using parament "childtable_offset and childtable_limit" to control table'offset point and offset + # insert: using parament "childtable_offset and childtable_limit" to control table'offset point and offset os.system("%staosdemo -f tools/taosdemoAllTest/insert-nodbnodrop.json -y" % binPath) tdSql.error("show dbno.stables") os.system("%staosdemo -f tools/taosdemoAllTest/insert-newdb.json -y" % binPath) @@ -128,41 +128,41 @@ class TDTestCase: tdSql.query("select count (tbname) from stb2") tdSql.checkData(0, 0, 7) tdSql.query("select count (tbname) from stb3") - tdSql.checkData(0, 0, 8) + tdSql.checkData(0, 0, 8) tdSql.query("select count (tbname) from stb4") - tdSql.checkData(0, 0, 8) + tdSql.checkData(0, 0, 8) os.system("%staosdemo -f tools/taosdemoAllTest/insert-offset.json -y" % binPath) - tdSql.execute("use db") + tdSql.execute("use db") tdSql.query("select count(*) from stb0") - tdSql.checkData(0, 0, 50) + tdSql.checkData(0, 0, 50) tdSql.query("select count(*) from stb1") - tdSql.checkData(0, 0, 240) + tdSql.checkData(0, 0, 240) tdSql.query("select count(*) from stb2") - tdSql.checkData(0, 0, 220) + tdSql.checkData(0, 0, 220) tdSql.query("select count(*) from stb3") tdSql.checkData(0, 0, 180) tdSql.query("select count(*) from stb4") tdSql.checkData(0, 0, 160) os.system("%staosdemo -f tools/taosdemoAllTest/insert-newtable.json -y" % binPath) - tdSql.execute("use db") + tdSql.execute("use db") tdSql.query("select count(*) from stb0") - tdSql.checkData(0, 0, 150) + tdSql.checkData(0, 0, 150) tdSql.query("select count(*) from stb1") - tdSql.checkData(0, 0, 360) + tdSql.checkData(0, 0, 360) tdSql.query("select count(*) from stb2") - tdSql.checkData(0, 0, 360) + tdSql.checkData(0, 0, 360) tdSql.query("select count(*) from stb3") tdSql.checkData(0, 0, 340) tdSql.query("select count(*) from stb4") tdSql.checkData(0, 0, 400) os.system("%staosdemo -f tools/taosdemoAllTest/insert-renewdb.json -y" % binPath) - tdSql.execute("use db") + tdSql.execute("use db") tdSql.query("select count(*) from stb0") - tdSql.checkData(0, 0, 50) + tdSql.checkData(0, 0, 50) tdSql.query("select count(*) from stb1") - tdSql.checkData(0, 0, 120) + tdSql.checkData(0, 0, 120) tdSql.query("select count(*) from stb2") - tdSql.checkData(0, 0, 140) + tdSql.checkData(0, 0, 140) tdSql.query("select count(*) from stb3") tdSql.checkData(0, 0, 160) tdSql.query("select count(*) from stb4") @@ -170,59 +170,59 @@ class TDTestCase: # insert: let parament in json file is illegal, it'll expect error. - tdSql.execute("drop database if exists db") + tdSql.execute("drop database if exists db") os.system("%staosdemo -f tools/taosdemoAllTest/insertColumnsAndTagNumLarge1024.json -y " % binPath) tdSql.error("use db") - tdSql.execute("drop database if exists db") + tdSql.execute("drop database if exists db") os.system("%staosdemo -f tools/taosdemoAllTest/insertSigcolumnsNum1024.json -y " % binPath) tdSql.error("select * from db.stb0") - tdSql.execute("drop database if exists db") + tdSql.execute("drop database if exists db") os.system("%staosdemo -f tools/taosdemoAllTest/insertColumnsAndTagNum1024.json -y " % binPath) tdSql.query("select count(*) from db.stb0") - tdSql.checkData(0, 0, 10000) - tdSql.execute("drop database if exists db") + tdSql.checkData(0, 0, 10000) + tdSql.execute("drop database if exists db") os.system("%staosdemo -f tools/taosdemoAllTest/insertInterlaceRowsLarge1M.json -y " % binPath) tdSql.query("select count(*) from db.stb0") tdSql.checkRows(0) - tdSql.execute("drop database if exists db") + tdSql.execute("drop database if exists db") os.system("%staosdemo -f tools/taosdemoAllTest/insertColumnsNum0.json -y " % binPath) - tdSql.execute("use db") + tdSql.execute("use db") tdSql.query("show stables like 'stb0%' ") tdSql.checkData(0, 2, 11) - tdSql.execute("drop database if exists db") - os.system("%staosdemo -f tools/taosdemoAllTest/insertTagsNumLarge128.json -y " % binPath) - tdSql.error("use db1") - tdSql.execute("drop database if exists db") - os.system("%staosdemo -f tools/taosdemoAllTest/insertBinaryLenLarge16374AllcolLar16384.json -y " % binPath) - tdSql.query("select count(*) from db.stb0") + tdSql.execute("drop database if exists db") + os.system("%staosdemo -f tools/taosdemoAllTest/insertTagsNumLarge128.json -y " % binPath) + tdSql.error("use db1") + tdSql.execute("drop database if exists db") + os.system("%staosdemo -f tools/taosdemoAllTest/insertBinaryLenLarge16374AllcolLar16384.json -y " % binPath) + tdSql.query("select count(*) from db.stb0") tdSql.checkRows(1) - tdSql.query("select count(*) from db.stb1") + tdSql.query("select count(*) from db.stb1") tdSql.checkRows(1) tdSql.error("select * from db.stb3") tdSql.error("select * from db.stb2") - tdSql.execute("drop database if exists db") - os.system("%staosdemo -f tools/taosdemoAllTest/insertNumOfrecordPerReq0.json -y " % binPath) - tdSql.error("select count(*) from db.stb0") - tdSql.execute("drop database if exists db") - os.system("%staosdemo -f tools/taosdemoAllTest/insertNumOfrecordPerReqless0.json -y " % binPath) - tdSql.error("use db") - tdSql.execute("drop database if exists db") - os.system("%staosdemo -f tools/taosdemoAllTest/insertChildTab0.json -y " % binPath) - tdSql.error("use db") - tdSql.execute("drop database if exists db") - os.system("%staosdemo -f tools/taosdemoAllTest/insertChildTabLess0.json -y " % binPath) - tdSql.error("use db") - tdSql.execute("drop database if exists blf") - os.system("%staosdemo -f tools/taosdemoAllTest/insertTimestepMulRowsLargeint16.json -y " % binPath) - tdSql.execute("use blf") - tdSql.query("select ts from blf.p_0_topics_7 limit 262800,1") + tdSql.execute("drop database if exists db") + os.system("%staosdemo -f tools/taosdemoAllTest/insertNumOfrecordPerReq0.json -y " % binPath) + tdSql.error("select count(*) from db.stb0") + tdSql.execute("drop database if exists db") + os.system("%staosdemo -f tools/taosdemoAllTest/insertNumOfrecordPerReqless0.json -y " % binPath) + tdSql.error("use db") + tdSql.execute("drop database if exists db") + os.system("%staosdemo -f tools/taosdemoAllTest/insertChildTab0.json -y " % binPath) + tdSql.error("use db") + tdSql.execute("drop database if exists db") + os.system("%staosdemo -f tools/taosdemoAllTest/insertChildTabLess0.json -y " % binPath) + tdSql.error("use db") + tdSql.execute("drop database if exists blf") + os.system("%staosdemo -f tools/taosdemoAllTest/insertTimestepMulRowsLargeint16.json -y " % binPath) + tdSql.execute("use blf") + tdSql.query("select ts from blf.p_0_topics_7 limit 262800,1") tdSql.checkData(0, 0, "2020-03-31 12:00:00.000") tdSql.query("select first(ts) from blf.p_0_topics_2") tdSql.checkData(0, 0, "2019-10-01 00:00:00") - tdSql.query("select last(ts) from blf.p_0_topics_6 ") + tdSql.query("select last(ts) from blf.p_0_topics_6 ") tdSql.checkData(0, 0, "2020-09-29 23:59:00") - os.system("%staosdemo -f tools/taosdemoAllTest/insertMaxNumPerReq.json -y " % binPath) - tdSql.execute("use db") + os.system("%staosdemo -f tools/taosdemoAllTest/insertMaxNumPerReq.json -y " % binPath) + tdSql.execute("use db") tdSql.query("select count(*) from stb0") tdSql.checkData(0, 0, 5000000) tdSql.query("select count(*) from stb1") @@ -230,7 +230,7 @@ class TDTestCase: - # insert: timestamp and step + # insert: timestamp and step os.system("%staosdemo -f tools/taosdemoAllTest/insert-timestep.json -y " % binPath) tdSql.execute("use db") tdSql.query("show stables") @@ -239,13 +239,13 @@ class TDTestCase: tdSql.query("select count (tbname) from stb1") tdSql.checkData(0, 0, 20) tdSql.query("select last(ts) from db.stb00_0") - tdSql.checkData(0, 0, "2020-10-01 00:00:00.019000") + tdSql.checkData(0, 0, "2020-10-01 00:00:00.019000") tdSql.query("select count(*) from stb0") - tdSql.checkData(0, 0, 200) + tdSql.checkData(0, 0, 200) tdSql.query("select last(ts) from db.stb01_0") - tdSql.checkData(0, 0, "2020-11-01 00:00:00.190000") + tdSql.checkData(0, 0, "2020-11-01 00:00:00.190000") tdSql.query("select count(*) from stb1") - tdSql.checkData(0, 0, 400) + tdSql.checkData(0, 0, 400) # # insert: disorder_ratio os.system("%staosdemo -f tools/taosdemoAllTest/insert-disorder.json -g 2>&1 -y " % binPath) @@ -255,14 +255,14 @@ class TDTestCase: tdSql.query("select count (tbname) from stb1") tdSql.checkData(0, 0, 1) tdSql.query("select count(*) from stb0") - tdSql.checkData(0, 0, 10) + tdSql.checkData(0, 0, 10) tdSql.query("select count(*) from stb1") - tdSql.checkData(0, 0, 10) + tdSql.checkData(0, 0, 10) # insert: sample json os.system("%staosdemo -f tools/taosdemoAllTest/insert-sample.json -y " % binPath) tdSql.execute("use dbtest123") - tdSql.query("select col2 from stb0") + tdSql.query("select c2 from stb0") tdSql.checkData(0, 0, 2147483647) tdSql.query("select * from stb1 where t1=-127") tdSql.checkRows(20) @@ -271,13 +271,13 @@ class TDTestCase: tdSql.query("select * from stb1 where t2=126") tdSql.checkRows(10) - # insert: test interlace parament + # insert: test interlace parament os.system("%staosdemo -f tools/taosdemoAllTest/insert-interlace-row.json -y " % binPath) tdSql.execute("use db") tdSql.query("select count (tbname) from stb0") tdSql.checkData(0, 0, 100) tdSql.query("select count (*) from stb0") - tdSql.checkData(0, 0, 15000) + tdSql.checkData(0, 0, 15000) # # insert: auto_create @@ -317,10 +317,8 @@ class TDTestCase: tdSql.checkRows(20) os.system("rm -rf ./insert_res.txt") - os.system("rm -rf tools/taosdemoAllTest/taosdemoTestInsertWithJson.py.sql") - - - + os.system("rm -rf tools/taosdemoAllTest/taosdemoTestInsertWithJson.py.sql") + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/tools/taosdemoTest.py b/tests/pytest/tools/taosdemoTest.py index 4cae8dfd3c..5662881031 100644 --- a/tests/pytest/tools/taosdemoTest.py +++ b/tests/pytest/tools/taosdemoTest.py @@ -59,11 +59,11 @@ class TDTestCase: tdSql.checkData(0, 0, self.numberOfTables * self.numberOfRecords) tdSql.query( - "select sum(col1) from test.meters interval(1h) sliding(30m)") + "select sum(c1) from test.meters interval(1h) sliding(30m)") tdSql.checkRows(2) tdSql.query( - "select apercentile(col1, 1) from test.meters interval(100s)") + "select apercentile(c1, 1) from test.meters interval(100s)") tdSql.checkRows(1) tdSql.error("select loc, count(loc) from test.meters") diff --git a/tests/pytest/tools/taosdemoTestTblAlt.py b/tests/pytest/tools/taosdemoTestTblAlt.py index 56c535916a..b70525ae4d 100644 --- a/tests/pytest/tools/taosdemoTestTblAlt.py +++ b/tests/pytest/tools/taosdemoTestTblAlt.py @@ -98,8 +98,8 @@ class TDTestCase: break time.sleep(1) - print("alter table test.meters add column col10 int") - tdSql.execute("alter table test.meters add column col10 int") + print("alter table test.meters add column c10 int") + tdSql.execute("alter table test.meters add column c10 int") print("insert into test.t9 values (now, 1, 2, 3, 4, 0)") tdSql.execute("insert into test.t9 values (now, 1, 2, 3, 4, 0)") From d8cbde67c49983e282fd5beaa79bd31e7f5da9fc Mon Sep 17 00:00:00 2001 From: Linhe Huo Date: Tue, 20 Jul 2021 15:43:17 +0800 Subject: [PATCH 017/106] [TD-5406]: dll export stmt_* api for windows (#6937) --- src/inc/taos.h | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/inc/taos.h b/src/inc/taos.h index cd8e116053..3d3797b50b 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -99,16 +99,16 @@ typedef struct TAOS_BIND { unsigned int allocated; } TAOS_BIND; -TAOS_STMT *taos_stmt_init(TAOS *taos); -int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); -int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); -int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); +DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos); +DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); +DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); +DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); -int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind); -int taos_stmt_add_batch(TAOS_STMT *stmt); -int taos_stmt_execute(TAOS_STMT *stmt); -TAOS_RES * taos_stmt_use_result(TAOS_STMT *stmt); -int taos_stmt_close(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind); +DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt); +DLL_EXPORT TAOS_RES * taos_stmt_use_result(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt); DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql); DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res); @@ -123,7 +123,7 @@ DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int n DLL_EXPORT void taos_stop_query(TAOS_RES *res); DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col); -int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); +DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); int taos_validate_sql(TAOS *taos, const char *sql); int* taos_fetch_lengths(TAOS_RES *res); From 234aad8ee75d34862ab70bfd652f8079a2fc1456 Mon Sep 17 00:00:00 2001 From: tickduan <417921451@qq.com> Date: Tue, 20 Jul 2021 21:07:13 +0800 Subject: [PATCH 018/106] db name len and metrics_del.sim two problem fixed --- src/inc/tcq.h | 2 +- tests/script/general/stream/metrics_del.sim | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/inc/tcq.h b/src/inc/tcq.h index 7549c3d498..7338cccfee 100644 --- a/src/inc/tcq.h +++ b/src/inc/tcq.h @@ -27,7 +27,7 @@ typedef struct { int32_t vgId; char user[TSDB_USER_LEN]; char pass[TSDB_KEY_LEN]; - char db[TSDB_DB_NAME_LEN]; + char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; // size must same with SVnodeObj.db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN] FCqWrite cqWrite; } SCqCfg; diff --git a/tests/script/general/stream/metrics_del.sim b/tests/script/general/stream/metrics_del.sim index 321658cd8d..18e777cd2c 100644 --- a/tests/script/general/stream/metrics_del.sim +++ b/tests/script/general/stream/metrics_del.sim @@ -34,11 +34,11 @@ while $i < $tbNum $tb = $tbPrefix . $i sql create table $tb using $mt tags( $i ) - $x = -1440 + $x = 1 $y = 0 while $y < $rowNum - $ms = $x . m - sql insert into $tb values (now $ms , $y , $y ) + $ts = 1626710400000 + $x + sql insert into $tb values ($ts , $y , $y ) $x = $x + 1 $y = $y + 1 endw From d39ff9193bd7a1057581a8394cb17b080484ce20 Mon Sep 17 00:00:00 2001 From: tomchon Date: Tue, 20 Jul 2021 23:21:01 +0800 Subject: [PATCH 019/106] @tomchon modify release scripts and change version number --- cmake/version.inc | 2 +- packaging/docker/dockerbuild.sh | 6 ++++-- snap/snapcraft.yaml | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cmake/version.inc b/cmake/version.inc index 6e71ac55ef..3bb06bfcf2 100755 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.0.20.9") + SET(TD_VER_NUMBER "2.0.20.10") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/packaging/docker/dockerbuild.sh b/packaging/docker/dockerbuild.sh index 861f00f854..1c440f8315 100755 --- a/packaging/docker/dockerbuild.sh +++ b/packaging/docker/dockerbuild.sh @@ -67,8 +67,11 @@ done if [ "$verType" == "beta" ]; then dockername=${cpuType}-${verType} + dirName=${pkgFile%-beta*} elif [ "$verType" == "stable" ]; then dockername=${cpuType} + dirName=${pkgFile%-Linux*} + else echo "unknow verType, nor stabel or beta" exit 1 @@ -84,11 +87,10 @@ comunityArchiveDir=/nas/TDengine/v$version/community # community version’pac cd ${scriptDir} cp -f ${comunityArchiveDir}/${pkgFile} . -dirName=${pkgFile%-Linux*} echo "dirName=${dirName}" -docker build --rm -f "Dockerfile" -t tdengine/tdengine-${dockername}:${version} "." --build-arg pkgFile=${pkgFile} --build-arg dirName=${dirName} +docker build --rm -f "Dockerfile" --network=host -t tdengine/tdengine-${dockername}:${version} "." --build-arg pkgFile=${pkgFile} --build-arg dirName=${dirName} docker login -u tdengine -p ${passWord} #replace the docker registry username and password docker push tdengine/tdengine-${dockername}:${version} diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 16fd9145fa..b06864fe5d 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: tdengine base: core18 -version: '2.0.20.9' +version: '2.0.20.10' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | @@ -72,7 +72,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.0.20.9 + - usr/lib/libtaos.so.2.0.20.10 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so From 151f3ac16116c8683b29d8d6c72253c64aa7f873 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 21 Jul 2021 05:51:43 +0800 Subject: [PATCH 020/106] Hotfix/sangshuduo/td 5398 taosdemo 4096col for master (#6940) * [TD-5398]: taosdemo 4096 columns table creation failed. * change column name prefix from col to c for maxium column number to fix buffer length. * use accurate buffer length. * fix test case for column name be shorter. * remove space between columns and tags. * change sql buffer to 1M align with taosdef.h. * column buffer length be calculated. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 89 ++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 45 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 3d953edc61..9b1c4ba444 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -71,9 +71,9 @@ extern char configDir[]; #define HEAD_BUFF_LEN TSDB_MAX_COLUMNS*24 // 16*MAX_COLUMNS + (192+32)*2 + insert into .. -#define COL_BUFFER_LEN (TSDB_MAX_BYTES_PER_ROW - 50) -#define BUFFER_SIZE (50 + TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_MAX_BYTES_PER_ROW + TSDB_MAX_TAGS_LEN) +#define BUFFER_SIZE TSDB_MAX_ALLOWED_SQL_LEN #define COND_BUF_LEN (BUFFER_SIZE - 30) +#define COL_BUFFER_LEN ((TSDB_COL_NAME_LEN + 15) * TSDB_MAX_COLUMNS) #define MAX_USERNAME_SIZE 64 #define MAX_PASSWORD_SIZE 64 #define MAX_HOSTNAME_SIZE 64 @@ -946,7 +946,6 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { for (int col = arguments->num_of_CPR; col < MAX_NUM_COLUMNS; col++) { arguments->datatype[col] = NULL; } - } else if (strcmp(argv[i], "-b") == 0) { arguments->demo_mode = false; if (argc == i+1) { @@ -1141,7 +1140,6 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { if (arguments->disorderRatio) { printf("# Data order: %d\n", arguments->disorderRatio); printf("# Data out of order rate: %d\n", arguments->disorderRange); - } printf("# Delete method: %d\n", arguments->method_of_delete); printf("# Answer yes when prompt: %d\n", arguments->answer_yes); @@ -2196,7 +2194,6 @@ static void printfQuerySystemInfo(TAOS * taos) { snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.stables;", dbInfos[i]->name); res = taos_query(taos, buffer); xDumpResultToFile(filename, res); - free(dbInfos[i]); } @@ -2370,7 +2367,7 @@ static char* getTagValueFromTagSample(SSuperTable* stbInfo, int tagUsePos) { return dataBuf; } -static char* generateTagVaulesForStb(SSuperTable* stbInfo, int32_t tableSeq) { +static char* generateTagValuesForStb(SSuperTable* stbInfo, int32_t tableSeq) { char* dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1); if (NULL == dataBuf) { printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1); @@ -2406,47 +2403,47 @@ static char* generateTagVaulesForStb(SSuperTable* stbInfo, int32_t tableSeq) { } //rand_string(buf, stbInfo->tags[i].dataLen); dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, - "\'%s\', ", buf); + "\'%s\',", buf); tmfree(buf); } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "int", strlen("int"))) { if ((g_args.demo_mode) && (i == 0)) { dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, - "%d, ", tableSeq % 10); + "%d,", tableSeq % 10); } else { dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, - "%d, ", tableSeq); + "%d,", tableSeq); } } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "bigint", strlen("bigint"))) { dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, - "%"PRId64", ", rand_bigint()); + "%"PRId64",", rand_bigint()); } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "float", strlen("float"))) { dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, - "%f, ", rand_float()); + "%f,", rand_float()); } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "double", strlen("double"))) { dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, - "%f, ", rand_double()); + "%f,", rand_double()); } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "smallint", strlen("smallint"))) { dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, - "%d, ", rand_smallint()); + "%d,", rand_smallint()); } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "tinyint", strlen("tinyint"))) { dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, - "%d, ", rand_tinyint()); + "%d,", rand_tinyint()); } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "bool", strlen("bool"))) { dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, - "%d, ", rand_bool()); + "%d,", rand_bool()); } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "timestamp", strlen("timestamp"))) { dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, - "%"PRId64", ", rand_bigint()); + "%"PRId64",", rand_bigint()); } else { printf("No support data type: %s\n", stbInfo->tags[i].dataType); tmfree(dataBuf); @@ -2454,7 +2451,7 @@ static char* generateTagVaulesForStb(SSuperTable* stbInfo, int32_t tableSeq) { } } - dataLen -= 2; + dataLen -= 1; dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")"); return dataBuf; } @@ -2726,54 +2723,54 @@ static int createSuperTable( if (strcasecmp(dataType, "BINARY") == 0) { len += snprintf(cols + len, COL_BUFFER_LEN - len, - ", C%d %s(%d)", colIndex, "BINARY", + ",C%d %s(%d)", colIndex, "BINARY", superTbl->columns[colIndex].dataLen); lenOfOneRow += superTbl->columns[colIndex].dataLen + 3; } else if (strcasecmp(dataType, "NCHAR") == 0) { len += snprintf(cols + len, COL_BUFFER_LEN - len, - ", C%d %s(%d)", colIndex, "NCHAR", + ",C%d %s(%d)", colIndex, "NCHAR", superTbl->columns[colIndex].dataLen); lenOfOneRow += superTbl->columns[colIndex].dataLen + 3; } else if (strcasecmp(dataType, "INT") == 0) { if ((g_args.demo_mode) && (colIndex == 1)) { len += snprintf(cols + len, COL_BUFFER_LEN - len, - ", VOLTAGE INT"); + ",VOLTAGE INT"); } else { - len += snprintf(cols + len, COL_BUFFER_LEN - len, ", C%d %s", colIndex, "INT"); + len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "INT"); } lenOfOneRow += 11; } else if (strcasecmp(dataType, "BIGINT") == 0) { - len += snprintf(cols + len, COL_BUFFER_LEN - len, ", C%d %s", + len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "BIGINT"); lenOfOneRow += 21; } else if (strcasecmp(dataType, "SMALLINT") == 0) { - len += snprintf(cols + len, COL_BUFFER_LEN - len, ", C%d %s", + len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "SMALLINT"); lenOfOneRow += 6; } else if (strcasecmp(dataType, "TINYINT") == 0) { - len += snprintf(cols + len, COL_BUFFER_LEN - len, ", C%d %s", colIndex, "TINYINT"); + len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "TINYINT"); lenOfOneRow += 4; } else if (strcasecmp(dataType, "BOOL") == 0) { - len += snprintf(cols + len, COL_BUFFER_LEN - len, ", C%d %s", colIndex, "BOOL"); + len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "BOOL"); lenOfOneRow += 6; } else if (strcasecmp(dataType, "FLOAT") == 0) { if (g_args.demo_mode) { if (colIndex == 0) { - len += snprintf(cols + len, COL_BUFFER_LEN - len, ", CURRENT FLOAT"); + len += snprintf(cols + len, COL_BUFFER_LEN - len, ",CURRENT FLOAT"); } else if (colIndex == 2) { - len += snprintf(cols + len, COL_BUFFER_LEN - len, ", PHASE FLOAT"); + len += snprintf(cols + len, COL_BUFFER_LEN - len, ",PHASE FLOAT"); } } else { - len += snprintf(cols + len, COL_BUFFER_LEN - len, ", C%d %s", colIndex, "FLOAT"); + len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "FLOAT"); } lenOfOneRow += 22; } else if (strcasecmp(dataType, "DOUBLE") == 0) { - len += snprintf(cols + len, COL_BUFFER_LEN - len, ", C%d %s", + len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "DOUBLE"); lenOfOneRow += 42; } else if (strcasecmp(dataType, "TIMESTAMP") == 0) { - len += snprintf(cols + len, COL_BUFFER_LEN - len, ", C%d %s", + len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "TIMESTAMP"); lenOfOneRow += 21; } else { @@ -2817,17 +2814,17 @@ static int createSuperTable( if (strcasecmp(dataType, "BINARY") == 0) { if ((g_args.demo_mode) && (tagIndex == 1)) { len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, - "location BINARY(%d), ", + "location BINARY(%d),", superTbl->tags[tagIndex].dataLen); } else { len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, - "t%d %s(%d), ", tagIndex, "BINARY", + "T%d %s(%d),", tagIndex, "BINARY", superTbl->tags[tagIndex].dataLen); } lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3; } else if (strcasecmp(dataType, "NCHAR") == 0) { len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, - "t%d %s(%d), ", tagIndex, + "T%d %s(%d),", tagIndex, "NCHAR", superTbl->tags[tagIndex].dataLen); lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3; } else if (strcasecmp(dataType, "INT") == 0) { @@ -2836,32 +2833,32 @@ static int createSuperTable( "groupId INT, "); } else { len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, - "t%d %s, ", tagIndex, "INT"); + "T%d %s,", tagIndex, "INT"); } lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 11; } else if (strcasecmp(dataType, "BIGINT") == 0) { len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, - "t%d %s, ", tagIndex, "BIGINT"); + "T%d %s,", tagIndex, "BIGINT"); lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 21; } else if (strcasecmp(dataType, "SMALLINT") == 0) { len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, - "t%d %s, ", tagIndex, "SMALLINT"); + "T%d %s,", tagIndex, "SMALLINT"); lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6; } else if (strcasecmp(dataType, "TINYINT") == 0) { len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, - "t%d %s, ", tagIndex, "TINYINT"); + "T%d %s,", tagIndex, "TINYINT"); lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 4; } else if (strcasecmp(dataType, "BOOL") == 0) { len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, - "t%d %s, ", tagIndex, "BOOL"); + "T%d %s,", tagIndex, "BOOL"); lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6; } else if (strcasecmp(dataType, "FLOAT") == 0) { len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, - "t%d %s, ", tagIndex, "FLOAT"); + "T%d %s,", tagIndex, "FLOAT"); lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 22; } else if (strcasecmp(dataType, "DOUBLE") == 0) { len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, - "t%d %s, ", tagIndex, "DOUBLE"); + "T%d %s,", tagIndex, "DOUBLE"); lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 42; } else { taos_close(taos); @@ -2871,7 +2868,7 @@ static int createSuperTable( } } - len -= 2; + len -= 1; len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, ")"); superTbl->lenOfTagOfOneRow = lenOfTagOfOneRow; @@ -3069,7 +3066,7 @@ static void* createTable(void *sarg) } char* tagsValBuf = NULL; if (0 == superTblInfo->tagSource) { - tagsValBuf = generateTagVaulesForStb(superTblInfo, i); + tagsValBuf = generateTagValuesForStb(superTblInfo, i); } else { tagsValBuf = getTagValueFromTagSample( superTblInfo, @@ -5332,7 +5329,7 @@ static int generateStbSQLHead( if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { char* tagsValBuf = NULL; if (0 == superTblInfo->tagSource) { - tagsValBuf = generateTagVaulesForStb(superTblInfo, tableSeq); + tagsValBuf = generateTagValuesForStb(superTblInfo, tableSeq); } else { tagsValBuf = getTagValueFromTagSample( superTblInfo, @@ -5347,7 +5344,7 @@ static int generateStbSQLHead( len = snprintf( headBuf, HEAD_BUFF_LEN, - "%s.%s using %s.%s tags %s values", + "%s.%s using %s.%s TAGS%s values", dbName, tableName, dbName, @@ -5900,7 +5897,7 @@ static int32_t prepareStbStmtProgressive( stbInfo, stmt, tableName, - batch, + g_args.num_of_RPR, insertRows, recordFrom, startTime, pSamplePos); } @@ -6593,6 +6590,8 @@ static void startMultiThreadInsertData(int threads, char* db_name, } else { start_time = 1500000000000; } + debugPrint("%s() LN%d, start_time= %"PRId64"\n", + __func__, __LINE__, start_time); int64_t start = taosGetTimestampMs(); From c71d9050e92518566b445cb5653fb930245a5b6b Mon Sep 17 00:00:00 2001 From: tickduan <417921451@qq.com> Date: Wed, 21 Jul 2021 09:33:21 +0800 Subject: [PATCH 021/106] trigger PR build --- tests/script/general/stream/metrics_del.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/general/stream/metrics_del.sim b/tests/script/general/stream/metrics_del.sim index 18e777cd2c..6cc3da71e9 100644 --- a/tests/script/general/stream/metrics_del.sim +++ b/tests/script/general/stream/metrics_del.sim @@ -34,7 +34,7 @@ while $i < $tbNum $tb = $tbPrefix . $i sql create table $tb using $mt tags( $i ) - $x = 1 + $x = 0 $y = 0 while $y < $rowNum $ts = 1626710400000 + $x From b6edb74b3bdc350b151b375a418d3b22ddd0773a Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 21 Jul 2021 14:49:36 +0800 Subject: [PATCH 022/106] [TD-5431]:taken into accout data length when compare nchar bytes --- src/query/src/qExtbuffer.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/query/src/qExtbuffer.c b/src/query/src/qExtbuffer.c index 90734e2e0e..e14ffa3a28 100644 --- a/src/query/src/qExtbuffer.c +++ b/src/query/src/qExtbuffer.c @@ -378,11 +378,18 @@ static FORCE_INLINE int32_t columnValueAscendingComparator(char *f1, char *f2, i }; case TSDB_DATA_TYPE_NCHAR: { // todo handle the var string compare - int32_t ret = tasoUcs4Compare(f1, f2, bytes); - if (ret == 0) { - return 0; + int32_t len1 = varDataLen(f1); + int32_t len2 = varDataLen(f2); + + if (len1 != len2) { + return len1 > len2 ? 1 : -1; + } else { + int32_t ret = tasoUcs4Compare(varDataVal(f1), varDataVal(f2), len1); + if (ret == 0) { + return 0; + } + return (ret < 0) ? -1 : 1; } - return (ret < 0) ? -1 : 1; }; case TSDB_DATA_TYPE_UTINYINT: DEFAULT_COMP(GET_UINT8_VAL(f1), GET_UINT8_VAL(f2)); case TSDB_DATA_TYPE_USMALLINT: DEFAULT_COMP(GET_UINT16_VAL(f1), GET_UINT16_VAL(f2)); From f24be8751f1b632c43299b455ad6faf834c004bd Mon Sep 17 00:00:00 2001 From: tomchon Date: Wed, 21 Jul 2021 15:19:14 +0800 Subject: [PATCH 023/106] modify dockerManifest.sh --- packaging/docker/dockerManifest.sh | 4 +++- packaging/docker/dockerbuild.sh | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packaging/docker/dockerManifest.sh b/packaging/docker/dockerManifest.sh index 0b30dd702c..9c5a618f34 100755 --- a/packaging/docker/dockerManifest.sh +++ b/packaging/docker/dockerManifest.sh @@ -44,6 +44,7 @@ echo "version=${version}" #docker manifest rm tdengine/tdengine #docker manifest rm tdengine/tdengine:${version} if [ "$verType" == "beta" ]; then + docker manifest rm tdengine/tdengine:latest docker manifest create -a tdengine/tdengine-beta:${version} tdengine/tdengine-amd64-beta:${version} tdengine/tdengine-aarch64-beta:${version} tdengine/tdengine-aarch32-beta:${version} docker manifest create -a tdengine/tdengine-beta:latest tdengine/tdengine-amd64-beta:latest tdengine/tdengine-aarch64-beta:latest tdengine/tdengine-aarch32-beta:latest docker login -u tdengine -p ${passWord} #replace the docker registry username and password @@ -51,11 +52,12 @@ if [ "$verType" == "beta" ]; then docker manifest push tdengine/tdengine-beta:${version} elif [ "$verType" == "stable" ]; then + docker manifest rm tdengine/tdengine:latest docker manifest create -a tdengine/tdengine:${version} tdengine/tdengine-amd64:${version} tdengine/tdengine-aarch64:${version} tdengine/tdengine-aarch32:${version} docker manifest create -a tdengine/tdengine:latest tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest docker login -u tdengine -p ${passWord} #replace the docker registry username and password docker manifest push tdengine/tdengine:latest - docker manifest push tdengine/tdengine:${version} + docker manifest push tdengine/tdengine:${version} else echo "unknow verType, nor stabel or beta" diff --git a/packaging/docker/dockerbuild.sh b/packaging/docker/dockerbuild.sh index 1c440f8315..c0e52f6dce 100755 --- a/packaging/docker/dockerbuild.sh +++ b/packaging/docker/dockerbuild.sh @@ -71,7 +71,6 @@ if [ "$verType" == "beta" ]; then elif [ "$verType" == "stable" ]; then dockername=${cpuType} dirName=${pkgFile%-Linux*} - else echo "unknow verType, nor stabel or beta" exit 1 @@ -90,7 +89,7 @@ cp -f ${comunityArchiveDir}/${pkgFile} . echo "dirName=${dirName}" -docker build --rm -f "Dockerfile" --network=host -t tdengine/tdengine-${dockername}:${version} "." --build-arg pkgFile=${pkgFile} --build-arg dirName=${dirName} +docker build --rm -f "Dockerfile" --network=host -t tdengine/tdengine-${dockername}:${version} "." --build-arg pkgFile=${pkgFile} --build-arg dirName=${dirName} docker login -u tdengine -p ${passWord} #replace the docker registry username and password docker push tdengine/tdengine-${dockername}:${version} From c41d23daaa68aed0a00a2eca6d6a6814eb4b49f7 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 21 Jul 2021 18:55:10 +0800 Subject: [PATCH 024/106] copy from develop branch. stmt: 0, nanosec: 0 (#6955) * copy from develop branch. stmt: 0, nanosec: 0 * set thread name disabled on master branch. --- src/kit/taosdemo/taosdemo.c | 1969 ++++++++++++++++++----------------- 1 file changed, 1041 insertions(+), 928 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 9b1c4ba444..de894434ad 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -55,6 +55,11 @@ #define STMT_IFACE_ENABLED 0 #define NANO_SECOND_ENABLED 0 +#define SET_THREADNAME_ENABLED 0 + +#if SET_THREADNAME_ENABLED == 0 +#define setThreadName(name) +#endif #define REQ_EXTRA_BUF_LEN 1024 #define RESP_BUF_LEN 4096 @@ -90,7 +95,7 @@ extern char configDir[]; #define MAX_SUPER_TABLE_COUNT 200 #define MAX_QUERY_SQL_COUNT 100 -#define MAX_QUERY_SQL_LENGTH 1024 +#define MAX_QUERY_SQL_LENGTH BUFFER_SIZE #define MAX_DATABASE_COUNT 256 #define INPUT_BUF_LEN 256 @@ -2367,7 +2372,25 @@ static char* getTagValueFromTagSample(SSuperTable* stbInfo, int tagUsePos) { return dataBuf; } -static char* generateTagValuesForStb(SSuperTable* stbInfo, int32_t tableSeq) { +static char *generateBinaryNCharTagValues(int64_t tableSeq, uint32_t len) +{ + char* buf = (char*)calloc(len, 1); + if (NULL == buf) { + printf("calloc failed! size:%d\n", len); + return NULL; + } + + if (tableSeq % 2) { + tstrncpy(buf, "beijing", len); + } else { + tstrncpy(buf, "shanghai", len); + } + //rand_string(buf, stbInfo->tags[i].dataLen); + + return buf; +} + +static char* generateTagValuesForStb(SSuperTable* stbInfo, int64_t tableSeq) { char* dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1); if (NULL == dataBuf) { printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1); @@ -2388,20 +2411,12 @@ static char* generateTagValuesForStb(SSuperTable* stbInfo, int32_t tableSeq) { return NULL; } - int tagBufLen = stbInfo->tags[i].dataLen + 1; - char* buf = (char*)calloc(tagBufLen, 1); + int32_t tagBufLen = stbInfo->tags[i].dataLen + 1; + char *buf = generateBinaryNCharTagValues(tableSeq, tagBufLen); if (NULL == buf) { - printf("calloc failed! size:%d\n", stbInfo->tags[i].dataLen); tmfree(dataBuf); return NULL; } - - if (tableSeq % 2) { - tstrncpy(buf, "beijing", tagBufLen); - } else { - tstrncpy(buf, "shanghai", tagBufLen); - } - //rand_string(buf, stbInfo->tags[i].dataLen); dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "\'%s\',", buf); tmfree(buf); @@ -2410,11 +2425,11 @@ static char* generateTagValuesForStb(SSuperTable* stbInfo, int32_t tableSeq) { if ((g_args.demo_mode) && (i == 0)) { dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, - "%d,", tableSeq % 10); + "%"PRId64",", tableSeq % 10); } else { dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, - "%d,", tableSeq); + "%"PRId64",", tableSeq); } } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "bigint", strlen("bigint"))) { @@ -2445,7 +2460,7 @@ static char* generateTagValuesForStb(SSuperTable* stbInfo, int32_t tableSeq) { dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%"PRId64",", rand_bigint()); } else { - printf("No support data type: %s\n", stbInfo->tags[i].dataType); + errorPrint("No support data type: %s\n", stbInfo->tags[i].dataType); tmfree(dataBuf); return NULL; } @@ -2734,7 +2749,7 @@ static int createSuperTable( } else if (strcasecmp(dataType, "INT") == 0) { if ((g_args.demo_mode) && (colIndex == 1)) { len += snprintf(cols + len, COL_BUFFER_LEN - len, - ",VOLTAGE INT"); + ", VOLTAGE INT"); } else { len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "INT"); } @@ -2756,9 +2771,9 @@ static int createSuperTable( } else if (strcasecmp(dataType, "FLOAT") == 0) { if (g_args.demo_mode) { if (colIndex == 0) { - len += snprintf(cols + len, COL_BUFFER_LEN - len, ",CURRENT FLOAT"); + len += snprintf(cols + len, COL_BUFFER_LEN - len, ", CURRENT FLOAT"); } else if (colIndex == 2) { - len += snprintf(cols + len, COL_BUFFER_LEN - len, ",PHASE FLOAT"); + len += snprintf(cols + len, COL_BUFFER_LEN - len, ", PHASE FLOAT"); } } else { len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "FLOAT"); @@ -3025,10 +3040,11 @@ static void* createTable(void *sarg) threadInfo *pThreadInfo = (threadInfo *)sarg; SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + setThreadName("createTable"); + uint64_t lastPrintTime = taosGetTimestampMs(); - int buff_len; - buff_len = BUFFER_SIZE; + int buff_len = BUFFER_SIZE; pThreadInfo->buffer = calloc(buff_len, 1); if (pThreadInfo->buffer == NULL) { @@ -4275,577 +4291,577 @@ PARSE_OVER: } static bool getMetaFromQueryJsonFile(cJSON* root) { - bool ret = false; + bool ret = false; - cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir"); - if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) { - tstrncpy(g_queryInfo.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN); - } + cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir"); + if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) { + tstrncpy(g_queryInfo.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN); + } - cJSON* host = cJSON_GetObjectItem(root, "host"); - if (host && host->type == cJSON_String && host->valuestring != NULL) { - tstrncpy(g_queryInfo.host, host->valuestring, MAX_HOSTNAME_SIZE); - } else if (!host) { - tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE); - } else { - printf("ERROR: failed to read json, host not found\n"); - goto PARSE_OVER; - } - - cJSON* port = cJSON_GetObjectItem(root, "port"); - if (port && port->type == cJSON_Number) { - g_queryInfo.port = port->valueint; - } else if (!port) { - g_queryInfo.port = 6030; - } - - cJSON* user = cJSON_GetObjectItem(root, "user"); - if (user && user->type == cJSON_String && user->valuestring != NULL) { - tstrncpy(g_queryInfo.user, user->valuestring, MAX_USERNAME_SIZE); - } else if (!user) { - tstrncpy(g_queryInfo.user, "root", MAX_USERNAME_SIZE); ; - } - - cJSON* password = cJSON_GetObjectItem(root, "password"); - if (password && password->type == cJSON_String && password->valuestring != NULL) { - tstrncpy(g_queryInfo.password, password->valuestring, MAX_PASSWORD_SIZE); - } else if (!password) { - tstrncpy(g_queryInfo.password, "taosdata", MAX_PASSWORD_SIZE);; - } - - cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no, - if (answerPrompt && answerPrompt->type == cJSON_String - && answerPrompt->valuestring != NULL) { - if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) { - g_args.answer_yes = false; - } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) { - g_args.answer_yes = true; + cJSON* host = cJSON_GetObjectItem(root, "host"); + if (host && host->type == cJSON_String && host->valuestring != NULL) { + tstrncpy(g_queryInfo.host, host->valuestring, MAX_HOSTNAME_SIZE); + } else if (!host) { + tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE); } else { - g_args.answer_yes = false; - } - } else if (!answerPrompt) { - g_args.answer_yes = false; - } else { - printf("ERROR: failed to read json, confirm_parameter_prompt not found\n"); - goto PARSE_OVER; - } - - cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times"); - if (gQueryTimes && gQueryTimes->type == cJSON_Number) { - if (gQueryTimes->valueint <= 0) { - errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - g_args.query_times = gQueryTimes->valueint; - } else if (!gQueryTimes) { - g_args.query_times = 1; - } else { - errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - - cJSON* dbs = cJSON_GetObjectItem(root, "databases"); - if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) { - tstrncpy(g_queryInfo.dbName, dbs->valuestring, TSDB_DB_NAME_LEN); - } else if (!dbs) { - printf("ERROR: failed to read json, databases not found\n"); - goto PARSE_OVER; - } - - cJSON* queryMode = cJSON_GetObjectItem(root, "query_mode"); - if (queryMode && queryMode->type == cJSON_String && queryMode->valuestring != NULL) { - tstrncpy(g_queryInfo.queryMode, queryMode->valuestring, MAX_TB_NAME_SIZE); - } else if (!queryMode) { - tstrncpy(g_queryInfo.queryMode, "taosc", MAX_TB_NAME_SIZE); - } else { - printf("ERROR: failed to read json, query_mode not found\n"); - goto PARSE_OVER; - } - - // specified_table_query - cJSON *specifiedQuery = cJSON_GetObjectItem(root, "specified_table_query"); - if (!specifiedQuery) { - g_queryInfo.specifiedQueryInfo.concurrent = 1; - g_queryInfo.specifiedQueryInfo.sqlCount = 0; - } else if (specifiedQuery->type != cJSON_Object) { - printf("ERROR: failed to read json, super_table_query not found\n"); - goto PARSE_OVER; - } else { - cJSON* queryInterval = cJSON_GetObjectItem(specifiedQuery, "query_interval"); - if (queryInterval && queryInterval->type == cJSON_Number) { - g_queryInfo.specifiedQueryInfo.queryInterval = queryInterval->valueint; - } else if (!queryInterval) { - g_queryInfo.specifiedQueryInfo.queryInterval = 0; - } - - cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery, - "query_times"); - if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) { - if (specifiedQueryTimes->valueint <= 0) { - errorPrint( - "%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", - __func__, __LINE__, specifiedQueryTimes->valueint); + printf("ERROR: failed to read json, host not found\n"); goto PARSE_OVER; - - } - g_queryInfo.specifiedQueryInfo.queryTimes = specifiedQueryTimes->valueint; - } else if (!specifiedQueryTimes) { - g_queryInfo.specifiedQueryInfo.queryTimes = g_args.query_times; - } else { - errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; } - cJSON* concurrent = cJSON_GetObjectItem(specifiedQuery, "concurrent"); - if (concurrent && concurrent->type == cJSON_Number) { - if (concurrent->valueint <= 0) { - errorPrint( - "%s() LN%d, query sqlCount %d or concurrent %d is not correct.\n", - __func__, __LINE__, - g_queryInfo.specifiedQueryInfo.sqlCount, - g_queryInfo.specifiedQueryInfo.concurrent); - goto PARSE_OVER; - } - g_queryInfo.specifiedQueryInfo.concurrent = concurrent->valueint; - } else if (!concurrent) { - g_queryInfo.specifiedQueryInfo.concurrent = 1; + cJSON* port = cJSON_GetObjectItem(root, "port"); + if (port && port->type == cJSON_Number) { + g_queryInfo.port = port->valueint; + } else if (!port) { + g_queryInfo.port = 6030; } - cJSON* specifiedAsyncMode = cJSON_GetObjectItem(specifiedQuery, "mode"); - if (specifiedAsyncMode && specifiedAsyncMode->type == cJSON_String - && specifiedAsyncMode->valuestring != NULL) { - if (0 == strcmp("sync", specifiedAsyncMode->valuestring)) { - g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE; - } else if (0 == strcmp("async", specifiedAsyncMode->valuestring)) { - g_queryInfo.specifiedQueryInfo.asyncMode = ASYNC_MODE; - } else { - errorPrint("%s() LN%d, failed to read json, async mode input error\n", - __func__, __LINE__); - goto PARSE_OVER; - } - } else { - g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE; + cJSON* user = cJSON_GetObjectItem(root, "user"); + if (user && user->type == cJSON_String && user->valuestring != NULL) { + tstrncpy(g_queryInfo.user, user->valuestring, MAX_USERNAME_SIZE); + } else if (!user) { + tstrncpy(g_queryInfo.user, "root", MAX_USERNAME_SIZE); ; } - cJSON* interval = cJSON_GetObjectItem(specifiedQuery, "interval"); - if (interval && interval->type == cJSON_Number) { - g_queryInfo.specifiedQueryInfo.subscribeInterval = interval->valueint; - } else if (!interval) { - //printf("failed to read json, subscribe interval no found\n"); - //goto PARSE_OVER; - g_queryInfo.specifiedQueryInfo.subscribeInterval = 10000; + cJSON* password = cJSON_GetObjectItem(root, "password"); + if (password && password->type == cJSON_String && password->valuestring != NULL) { + tstrncpy(g_queryInfo.password, password->valuestring, MAX_PASSWORD_SIZE); + } else if (!password) { + tstrncpy(g_queryInfo.password, "taosdata", MAX_PASSWORD_SIZE);; } - cJSON* restart = cJSON_GetObjectItem(specifiedQuery, "restart"); - if (restart && restart->type == cJSON_String && restart->valuestring != NULL) { - if (0 == strcmp("yes", restart->valuestring)) { - g_queryInfo.specifiedQueryInfo.subscribeRestart = true; - } else if (0 == strcmp("no", restart->valuestring)) { - g_queryInfo.specifiedQueryInfo.subscribeRestart = false; - } else { - printf("ERROR: failed to read json, subscribe restart error\n"); - goto PARSE_OVER; - } - } else { - g_queryInfo.specifiedQueryInfo.subscribeRestart = true; - } - - cJSON* keepProgress = cJSON_GetObjectItem(specifiedQuery, "keepProgress"); - if (keepProgress - && keepProgress->type == cJSON_String - && keepProgress->valuestring != NULL) { - if (0 == strcmp("yes", keepProgress->valuestring)) { - g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 1; - } else if (0 == strcmp("no", keepProgress->valuestring)) { - g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0; - } else { - printf("ERROR: failed to read json, subscribe keepProgress error\n"); - goto PARSE_OVER; - } - } else { - g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0; - } - - // sqls - cJSON* specifiedSqls = cJSON_GetObjectItem(specifiedQuery, "sqls"); - if (!specifiedSqls) { - g_queryInfo.specifiedQueryInfo.sqlCount = 0; - } else if (specifiedSqls->type != cJSON_Array) { - errorPrint("%s() LN%d, failed to read json, super sqls not found\n", - __func__, __LINE__); - goto PARSE_OVER; - } else { - int superSqlSize = cJSON_GetArraySize(specifiedSqls); - if (superSqlSize * g_queryInfo.specifiedQueryInfo.concurrent - > MAX_QUERY_SQL_COUNT) { - errorPrint("%s() LN%d, failed to read json, query sql(%d) * concurrent(%d) overflow, max is %d\n", - __func__, __LINE__, - superSqlSize, - g_queryInfo.specifiedQueryInfo.concurrent, - MAX_QUERY_SQL_COUNT); - goto PARSE_OVER; - } - - g_queryInfo.specifiedQueryInfo.sqlCount = superSqlSize; - for (int j = 0; j < superSqlSize; ++j) { - cJSON* sql = cJSON_GetArrayItem(specifiedSqls, j); - if (sql == NULL) continue; - - cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); - if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) { - printf("ERROR: failed to read json, sql not found\n"); - goto PARSE_OVER; - } - tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j], - sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); - - // default value is -1, which mean infinite loop - g_queryInfo.specifiedQueryInfo.endAfterConsume[j] = -1; - cJSON* endAfterConsume = - cJSON_GetObjectItem(specifiedQuery, "endAfterConsume"); - if (endAfterConsume - && endAfterConsume->type == cJSON_Number) { - g_queryInfo.specifiedQueryInfo.endAfterConsume[j] - = endAfterConsume->valueint; - } - if (g_queryInfo.specifiedQueryInfo.endAfterConsume[j] < -1) - g_queryInfo.specifiedQueryInfo.endAfterConsume[j] = -1; - - g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] = -1; - cJSON* resubAfterConsume = - cJSON_GetObjectItem(specifiedQuery, "resubAfterConsume"); - if ((resubAfterConsume) - && (resubAfterConsume->type == cJSON_Number) - && (resubAfterConsume->valueint >= 0)) { - g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] - = resubAfterConsume->valueint; - } - - if (g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] < -1) - g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] = -1; - - cJSON *result = cJSON_GetObjectItem(sql, "result"); - if ((NULL != result) && (result->type == cJSON_String) - && (result->valuestring != NULL)) { - tstrncpy(g_queryInfo.specifiedQueryInfo.result[j], - result->valuestring, MAX_FILE_NAME_LEN); - } else if (NULL == result) { - memset(g_queryInfo.specifiedQueryInfo.result[j], - 0, MAX_FILE_NAME_LEN); + cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no, + if (answerPrompt && answerPrompt->type == cJSON_String + && answerPrompt->valuestring != NULL) { + if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) { + g_args.answer_yes = false; + } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) { + g_args.answer_yes = true; } else { - printf("ERROR: failed to read json, super query result file not found\n"); - goto PARSE_OVER; + g_args.answer_yes = false; } - } - } - } - - // super_table_query - cJSON *superQuery = cJSON_GetObjectItem(root, "super_table_query"); - if (!superQuery) { - g_queryInfo.superQueryInfo.threadCnt = 1; - g_queryInfo.superQueryInfo.sqlCount = 0; - } else if (superQuery->type != cJSON_Object) { - printf("ERROR: failed to read json, sub_table_query not found\n"); - ret = true; - goto PARSE_OVER; - } else { - cJSON* subrate = cJSON_GetObjectItem(superQuery, "query_interval"); - if (subrate && subrate->type == cJSON_Number) { - g_queryInfo.superQueryInfo.queryInterval = subrate->valueint; - } else if (!subrate) { - g_queryInfo.superQueryInfo.queryInterval = 0; - } - - cJSON* superQueryTimes = cJSON_GetObjectItem(superQuery, "query_times"); - if (superQueryTimes && superQueryTimes->type == cJSON_Number) { - if (superQueryTimes->valueint <= 0) { - errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", - __func__, __LINE__, superQueryTimes->valueint); - goto PARSE_OVER; - } - g_queryInfo.superQueryInfo.queryTimes = superQueryTimes->valueint; - } else if (!superQueryTimes) { - g_queryInfo.superQueryInfo.queryTimes = g_args.query_times; + } else if (!answerPrompt) { + g_args.answer_yes = false; } else { - errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - - cJSON* threads = cJSON_GetObjectItem(superQuery, "threads"); - if (threads && threads->type == cJSON_Number) { - if (threads->valueint <= 0) { - errorPrint("%s() LN%d, failed to read json, threads input mistake\n", - __func__, __LINE__); + printf("ERROR: failed to read json, confirm_parameter_prompt not found\n"); goto PARSE_OVER; - - } - g_queryInfo.superQueryInfo.threadCnt = threads->valueint; - } else if (!threads) { - g_queryInfo.superQueryInfo.threadCnt = 1; } - //cJSON* subTblCnt = cJSON_GetObjectItem(superQuery, "childtable_count"); - //if (subTblCnt && subTblCnt->type == cJSON_Number) { - // g_queryInfo.superQueryInfo.childTblCount = subTblCnt->valueint; - //} else if (!subTblCnt) { - // g_queryInfo.superQueryInfo.childTblCount = 0; - //} - - cJSON* stblname = cJSON_GetObjectItem(superQuery, "stblname"); - if (stblname && stblname->type == cJSON_String - && stblname->valuestring != NULL) { - tstrncpy(g_queryInfo.superQueryInfo.sTblName, stblname->valuestring, - TSDB_TABLE_NAME_LEN); + cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times"); + if (gQueryTimes && gQueryTimes->type == cJSON_Number) { + if (gQueryTimes->valueint <= 0) { + errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + g_args.query_times = gQueryTimes->valueint; + } else if (!gQueryTimes) { + g_args.query_times = 1; } else { - errorPrint("%s() LN%d, failed to read json, super table name input error\n", - __func__, __LINE__); - goto PARSE_OVER; + errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; } - cJSON* superAsyncMode = cJSON_GetObjectItem(superQuery, "mode"); - if (superAsyncMode && superAsyncMode->type == cJSON_String - && superAsyncMode->valuestring != NULL) { - if (0 == strcmp("sync", superAsyncMode->valuestring)) { - g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE; - } else if (0 == strcmp("async", superAsyncMode->valuestring)) { - g_queryInfo.superQueryInfo.asyncMode = ASYNC_MODE; - } else { - errorPrint("%s() LN%d, failed to read json, async mode input error\n", - __func__, __LINE__); + cJSON* dbs = cJSON_GetObjectItem(root, "databases"); + if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) { + tstrncpy(g_queryInfo.dbName, dbs->valuestring, TSDB_DB_NAME_LEN); + } else if (!dbs) { + printf("ERROR: failed to read json, databases not found\n"); goto PARSE_OVER; - } + } + + cJSON* queryMode = cJSON_GetObjectItem(root, "query_mode"); + if (queryMode && queryMode->type == cJSON_String && queryMode->valuestring != NULL) { + tstrncpy(g_queryInfo.queryMode, queryMode->valuestring, MAX_TB_NAME_SIZE); + } else if (!queryMode) { + tstrncpy(g_queryInfo.queryMode, "taosc", MAX_TB_NAME_SIZE); } else { - g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE; + printf("ERROR: failed to read json, query_mode not found\n"); + goto PARSE_OVER; } - cJSON* superInterval = cJSON_GetObjectItem(superQuery, "interval"); - if (superInterval && superInterval->type == cJSON_Number) { - if (superInterval->valueint < 0) { - errorPrint("%s() LN%d, failed to read json, interval input mistake\n", - __func__, __LINE__); + // specified_table_query + cJSON *specifiedQuery = cJSON_GetObjectItem(root, "specified_table_query"); + if (!specifiedQuery) { + g_queryInfo.specifiedQueryInfo.concurrent = 1; + g_queryInfo.specifiedQueryInfo.sqlCount = 0; + } else if (specifiedQuery->type != cJSON_Object) { + printf("ERROR: failed to read json, super_table_query not found\n"); goto PARSE_OVER; - } - g_queryInfo.superQueryInfo.subscribeInterval = superInterval->valueint; - } else if (!superInterval) { - //printf("failed to read json, subscribe interval no found\n"); - //goto PARSE_OVER; - g_queryInfo.superQueryInfo.subscribeInterval = 10000; - } - - cJSON* subrestart = cJSON_GetObjectItem(superQuery, "restart"); - if (subrestart && subrestart->type == cJSON_String - && subrestart->valuestring != NULL) { - if (0 == strcmp("yes", subrestart->valuestring)) { - g_queryInfo.superQueryInfo.subscribeRestart = true; - } else if (0 == strcmp("no", subrestart->valuestring)) { - g_queryInfo.superQueryInfo.subscribeRestart = false; - } else { - printf("ERROR: failed to read json, subscribe restart error\n"); - goto PARSE_OVER; - } } else { - g_queryInfo.superQueryInfo.subscribeRestart = true; + cJSON* queryInterval = cJSON_GetObjectItem(specifiedQuery, "query_interval"); + if (queryInterval && queryInterval->type == cJSON_Number) { + g_queryInfo.specifiedQueryInfo.queryInterval = queryInterval->valueint; + } else if (!queryInterval) { + g_queryInfo.specifiedQueryInfo.queryInterval = 0; + } + + cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery, + "query_times"); + if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) { + if (specifiedQueryTimes->valueint <= 0) { + errorPrint( + "%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", + __func__, __LINE__, specifiedQueryTimes->valueint); + goto PARSE_OVER; + + } + g_queryInfo.specifiedQueryInfo.queryTimes = specifiedQueryTimes->valueint; + } else if (!specifiedQueryTimes) { + g_queryInfo.specifiedQueryInfo.queryTimes = g_args.query_times; + } else { + errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON* concurrent = cJSON_GetObjectItem(specifiedQuery, "concurrent"); + if (concurrent && concurrent->type == cJSON_Number) { + if (concurrent->valueint <= 0) { + errorPrint( + "%s() LN%d, query sqlCount %d or concurrent %d is not correct.\n", + __func__, __LINE__, + g_queryInfo.specifiedQueryInfo.sqlCount, + g_queryInfo.specifiedQueryInfo.concurrent); + goto PARSE_OVER; + } + g_queryInfo.specifiedQueryInfo.concurrent = concurrent->valueint; + } else if (!concurrent) { + g_queryInfo.specifiedQueryInfo.concurrent = 1; + } + + cJSON* specifiedAsyncMode = cJSON_GetObjectItem(specifiedQuery, "mode"); + if (specifiedAsyncMode && specifiedAsyncMode->type == cJSON_String + && specifiedAsyncMode->valuestring != NULL) { + if (0 == strcmp("sync", specifiedAsyncMode->valuestring)) { + g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE; + } else if (0 == strcmp("async", specifiedAsyncMode->valuestring)) { + g_queryInfo.specifiedQueryInfo.asyncMode = ASYNC_MODE; + } else { + errorPrint("%s() LN%d, failed to read json, async mode input error\n", + __func__, __LINE__); + goto PARSE_OVER; + } + } else { + g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE; + } + + cJSON* interval = cJSON_GetObjectItem(specifiedQuery, "interval"); + if (interval && interval->type == cJSON_Number) { + g_queryInfo.specifiedQueryInfo.subscribeInterval = interval->valueint; + } else if (!interval) { + //printf("failed to read json, subscribe interval no found\n"); + //goto PARSE_OVER; + g_queryInfo.specifiedQueryInfo.subscribeInterval = 10000; + } + + cJSON* restart = cJSON_GetObjectItem(specifiedQuery, "restart"); + if (restart && restart->type == cJSON_String && restart->valuestring != NULL) { + if (0 == strcmp("yes", restart->valuestring)) { + g_queryInfo.specifiedQueryInfo.subscribeRestart = true; + } else if (0 == strcmp("no", restart->valuestring)) { + g_queryInfo.specifiedQueryInfo.subscribeRestart = false; + } else { + printf("ERROR: failed to read json, subscribe restart error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.specifiedQueryInfo.subscribeRestart = true; + } + + cJSON* keepProgress = cJSON_GetObjectItem(specifiedQuery, "keepProgress"); + if (keepProgress + && keepProgress->type == cJSON_String + && keepProgress->valuestring != NULL) { + if (0 == strcmp("yes", keepProgress->valuestring)) { + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 1; + } else if (0 == strcmp("no", keepProgress->valuestring)) { + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0; + } else { + printf("ERROR: failed to read json, subscribe keepProgress error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0; + } + + // sqls + cJSON* specifiedSqls = cJSON_GetObjectItem(specifiedQuery, "sqls"); + if (!specifiedSqls) { + g_queryInfo.specifiedQueryInfo.sqlCount = 0; + } else if (specifiedSqls->type != cJSON_Array) { + errorPrint("%s() LN%d, failed to read json, super sqls not found\n", + __func__, __LINE__); + goto PARSE_OVER; + } else { + int superSqlSize = cJSON_GetArraySize(specifiedSqls); + if (superSqlSize * g_queryInfo.specifiedQueryInfo.concurrent + > MAX_QUERY_SQL_COUNT) { + errorPrint("%s() LN%d, failed to read json, query sql(%d) * concurrent(%d) overflow, max is %d\n", + __func__, __LINE__, + superSqlSize, + g_queryInfo.specifiedQueryInfo.concurrent, + MAX_QUERY_SQL_COUNT); + goto PARSE_OVER; + } + + g_queryInfo.specifiedQueryInfo.sqlCount = superSqlSize; + for (int j = 0; j < superSqlSize; ++j) { + cJSON* sql = cJSON_GetArrayItem(specifiedSqls, j); + if (sql == NULL) continue; + + cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); + if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) { + printf("ERROR: failed to read json, sql not found\n"); + goto PARSE_OVER; + } + tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j], + sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); + + // default value is -1, which mean infinite loop + g_queryInfo.specifiedQueryInfo.endAfterConsume[j] = -1; + cJSON* endAfterConsume = + cJSON_GetObjectItem(specifiedQuery, "endAfterConsume"); + if (endAfterConsume + && endAfterConsume->type == cJSON_Number) { + g_queryInfo.specifiedQueryInfo.endAfterConsume[j] + = endAfterConsume->valueint; + } + if (g_queryInfo.specifiedQueryInfo.endAfterConsume[j] < -1) + g_queryInfo.specifiedQueryInfo.endAfterConsume[j] = -1; + + g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] = -1; + cJSON* resubAfterConsume = + cJSON_GetObjectItem(specifiedQuery, "resubAfterConsume"); + if ((resubAfterConsume) + && (resubAfterConsume->type == cJSON_Number) + && (resubAfterConsume->valueint >= 0)) { + g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] + = resubAfterConsume->valueint; + } + + if (g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] < -1) + g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] = -1; + + cJSON *result = cJSON_GetObjectItem(sql, "result"); + if ((NULL != result) && (result->type == cJSON_String) + && (result->valuestring != NULL)) { + tstrncpy(g_queryInfo.specifiedQueryInfo.result[j], + result->valuestring, MAX_FILE_NAME_LEN); + } else if (NULL == result) { + memset(g_queryInfo.specifiedQueryInfo.result[j], + 0, MAX_FILE_NAME_LEN); + } else { + printf("ERROR: failed to read json, super query result file not found\n"); + goto PARSE_OVER; + } + } + } } - cJSON* superkeepProgress = cJSON_GetObjectItem(superQuery, "keepProgress"); - if (superkeepProgress && - superkeepProgress->type == cJSON_String - && superkeepProgress->valuestring != NULL) { - if (0 == strcmp("yes", superkeepProgress->valuestring)) { - g_queryInfo.superQueryInfo.subscribeKeepProgress = 1; - } else if (0 == strcmp("no", superkeepProgress->valuestring)) { - g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; - } else { - printf("ERROR: failed to read json, subscribe super table keepProgress error\n"); + // super_table_query + cJSON *superQuery = cJSON_GetObjectItem(root, "super_table_query"); + if (!superQuery) { + g_queryInfo.superQueryInfo.threadCnt = 1; + g_queryInfo.superQueryInfo.sqlCount = 0; + } else if (superQuery->type != cJSON_Object) { + printf("ERROR: failed to read json, sub_table_query not found\n"); + ret = true; goto PARSE_OVER; - } } else { - g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; - } + cJSON* subrate = cJSON_GetObjectItem(superQuery, "query_interval"); + if (subrate && subrate->type == cJSON_Number) { + g_queryInfo.superQueryInfo.queryInterval = subrate->valueint; + } else if (!subrate) { + g_queryInfo.superQueryInfo.queryInterval = 0; + } - // default value is -1, which mean do not resub - g_queryInfo.superQueryInfo.endAfterConsume = -1; - cJSON* superEndAfterConsume = - cJSON_GetObjectItem(superQuery, "endAfterConsume"); - if (superEndAfterConsume - && superEndAfterConsume->type == cJSON_Number) { - g_queryInfo.superQueryInfo.endAfterConsume = - superEndAfterConsume->valueint; - } - if (g_queryInfo.superQueryInfo.endAfterConsume < -1) + cJSON* superQueryTimes = cJSON_GetObjectItem(superQuery, "query_times"); + if (superQueryTimes && superQueryTimes->type == cJSON_Number) { + if (superQueryTimes->valueint <= 0) { + errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", + __func__, __LINE__, superQueryTimes->valueint); + goto PARSE_OVER; + } + g_queryInfo.superQueryInfo.queryTimes = superQueryTimes->valueint; + } else if (!superQueryTimes) { + g_queryInfo.superQueryInfo.queryTimes = g_args.query_times; + } else { + errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON* threads = cJSON_GetObjectItem(superQuery, "threads"); + if (threads && threads->type == cJSON_Number) { + if (threads->valueint <= 0) { + errorPrint("%s() LN%d, failed to read json, threads input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + + } + g_queryInfo.superQueryInfo.threadCnt = threads->valueint; + } else if (!threads) { + g_queryInfo.superQueryInfo.threadCnt = 1; + } + + //cJSON* subTblCnt = cJSON_GetObjectItem(superQuery, "childtable_count"); + //if (subTblCnt && subTblCnt->type == cJSON_Number) { + // g_queryInfo.superQueryInfo.childTblCount = subTblCnt->valueint; + //} else if (!subTblCnt) { + // g_queryInfo.superQueryInfo.childTblCount = 0; + //} + + cJSON* stblname = cJSON_GetObjectItem(superQuery, "stblname"); + if (stblname && stblname->type == cJSON_String + && stblname->valuestring != NULL) { + tstrncpy(g_queryInfo.superQueryInfo.sTblName, stblname->valuestring, + TSDB_TABLE_NAME_LEN); + } else { + errorPrint("%s() LN%d, failed to read json, super table name input error\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON* superAsyncMode = cJSON_GetObjectItem(superQuery, "mode"); + if (superAsyncMode && superAsyncMode->type == cJSON_String + && superAsyncMode->valuestring != NULL) { + if (0 == strcmp("sync", superAsyncMode->valuestring)) { + g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE; + } else if (0 == strcmp("async", superAsyncMode->valuestring)) { + g_queryInfo.superQueryInfo.asyncMode = ASYNC_MODE; + } else { + errorPrint("%s() LN%d, failed to read json, async mode input error\n", + __func__, __LINE__); + goto PARSE_OVER; + } + } else { + g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE; + } + + cJSON* superInterval = cJSON_GetObjectItem(superQuery, "interval"); + if (superInterval && superInterval->type == cJSON_Number) { + if (superInterval->valueint < 0) { + errorPrint("%s() LN%d, failed to read json, interval input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + g_queryInfo.superQueryInfo.subscribeInterval = superInterval->valueint; + } else if (!superInterval) { + //printf("failed to read json, subscribe interval no found\n"); + //goto PARSE_OVER; + g_queryInfo.superQueryInfo.subscribeInterval = 10000; + } + + cJSON* subrestart = cJSON_GetObjectItem(superQuery, "restart"); + if (subrestart && subrestart->type == cJSON_String + && subrestart->valuestring != NULL) { + if (0 == strcmp("yes", subrestart->valuestring)) { + g_queryInfo.superQueryInfo.subscribeRestart = true; + } else if (0 == strcmp("no", subrestart->valuestring)) { + g_queryInfo.superQueryInfo.subscribeRestart = false; + } else { + printf("ERROR: failed to read json, subscribe restart error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.superQueryInfo.subscribeRestart = true; + } + + cJSON* superkeepProgress = cJSON_GetObjectItem(superQuery, "keepProgress"); + if (superkeepProgress && + superkeepProgress->type == cJSON_String + && superkeepProgress->valuestring != NULL) { + if (0 == strcmp("yes", superkeepProgress->valuestring)) { + g_queryInfo.superQueryInfo.subscribeKeepProgress = 1; + } else if (0 == strcmp("no", superkeepProgress->valuestring)) { + g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; + } else { + printf("ERROR: failed to read json, subscribe super table keepProgress error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; + } + + // default value is -1, which mean do not resub g_queryInfo.superQueryInfo.endAfterConsume = -1; + cJSON* superEndAfterConsume = + cJSON_GetObjectItem(superQuery, "endAfterConsume"); + if (superEndAfterConsume + && superEndAfterConsume->type == cJSON_Number) { + g_queryInfo.superQueryInfo.endAfterConsume = + superEndAfterConsume->valueint; + } + if (g_queryInfo.superQueryInfo.endAfterConsume < -1) + g_queryInfo.superQueryInfo.endAfterConsume = -1; - // default value is -1, which mean do not resub - g_queryInfo.superQueryInfo.resubAfterConsume = -1; - cJSON* superResubAfterConsume = - cJSON_GetObjectItem(superQuery, "resubAfterConsume"); - if ((superResubAfterConsume) - && (superResubAfterConsume->type == cJSON_Number) - && (superResubAfterConsume->valueint >= 0)) { - g_queryInfo.superQueryInfo.resubAfterConsume = - superResubAfterConsume->valueint; - } - if (g_queryInfo.superQueryInfo.resubAfterConsume < -1) + // default value is -1, which mean do not resub g_queryInfo.superQueryInfo.resubAfterConsume = -1; - - // supert table sqls - cJSON* superSqls = cJSON_GetObjectItem(superQuery, "sqls"); - if (!superSqls) { - g_queryInfo.superQueryInfo.sqlCount = 0; - } else if (superSqls->type != cJSON_Array) { - errorPrint("%s() LN%d: failed to read json, super sqls not found\n", - __func__, __LINE__); - goto PARSE_OVER; - } else { - int superSqlSize = cJSON_GetArraySize(superSqls); - if (superSqlSize > MAX_QUERY_SQL_COUNT) { - errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n", - __func__, __LINE__, MAX_QUERY_SQL_COUNT); - goto PARSE_OVER; - } - - g_queryInfo.superQueryInfo.sqlCount = superSqlSize; - for (int j = 0; j < superSqlSize; ++j) { - cJSON* sql = cJSON_GetArrayItem(superSqls, j); - if (sql == NULL) continue; - - cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); - if (!sqlStr || sqlStr->type != cJSON_String - || sqlStr->valuestring == NULL) { - errorPrint("%s() LN%d, failed to read json, sql not found\n", - __func__, __LINE__); - goto PARSE_OVER; + cJSON* superResubAfterConsume = + cJSON_GetObjectItem(superQuery, "resubAfterConsume"); + if ((superResubAfterConsume) + && (superResubAfterConsume->type == cJSON_Number) + && (superResubAfterConsume->valueint >= 0)) { + g_queryInfo.superQueryInfo.resubAfterConsume = + superResubAfterConsume->valueint; } - tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring, - MAX_QUERY_SQL_LENGTH); + if (g_queryInfo.superQueryInfo.resubAfterConsume < -1) + g_queryInfo.superQueryInfo.resubAfterConsume = -1; - cJSON *result = cJSON_GetObjectItem(sql, "result"); - if (result != NULL && result->type == cJSON_String - && result->valuestring != NULL){ - tstrncpy(g_queryInfo.superQueryInfo.result[j], - result->valuestring, MAX_FILE_NAME_LEN); - } else if (NULL == result) { - memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN); - } else { - errorPrint("%s() LN%d, failed to read json, sub query result file not found\n", - __func__, __LINE__); - goto PARSE_OVER; + // supert table sqls + cJSON* superSqls = cJSON_GetObjectItem(superQuery, "sqls"); + if (!superSqls) { + g_queryInfo.superQueryInfo.sqlCount = 0; + } else if (superSqls->type != cJSON_Array) { + errorPrint("%s() LN%d: failed to read json, super sqls not found\n", + __func__, __LINE__); + goto PARSE_OVER; + } else { + int superSqlSize = cJSON_GetArraySize(superSqls); + if (superSqlSize > MAX_QUERY_SQL_COUNT) { + errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n", + __func__, __LINE__, MAX_QUERY_SQL_COUNT); + goto PARSE_OVER; + } + + g_queryInfo.superQueryInfo.sqlCount = superSqlSize; + for (int j = 0; j < superSqlSize; ++j) { + cJSON* sql = cJSON_GetArrayItem(superSqls, j); + if (sql == NULL) continue; + + cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); + if (!sqlStr || sqlStr->type != cJSON_String + || sqlStr->valuestring == NULL) { + errorPrint("%s() LN%d, failed to read json, sql not found\n", + __func__, __LINE__); + goto PARSE_OVER; + } + tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring, + MAX_QUERY_SQL_LENGTH); + + cJSON *result = cJSON_GetObjectItem(sql, "result"); + if (result != NULL && result->type == cJSON_String + && result->valuestring != NULL){ + tstrncpy(g_queryInfo.superQueryInfo.result[j], + result->valuestring, MAX_FILE_NAME_LEN); + } else if (NULL == result) { + memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN); + } else { + errorPrint("%s() LN%d, failed to read json, sub query result file not found\n", + __func__, __LINE__); + goto PARSE_OVER; + } + } } - } } - } - ret = true; + ret = true; PARSE_OVER: - return ret; + return ret; } static bool getInfoFromJsonFile(char* file) { debugPrint("%s %d %s\n", __func__, __LINE__, file); - FILE *fp = fopen(file, "r"); - if (!fp) { - printf("failed to read %s, reason:%s\n", file, strerror(errno)); - return false; - } - - bool ret = false; - int maxLen = 6400000; - char *content = calloc(1, maxLen + 1); - int len = fread(content, 1, maxLen, fp); - if (len <= 0) { - free(content); - fclose(fp); - printf("failed to read %s, content is null", file); - return false; - } - - content[len] = 0; - cJSON* root = cJSON_Parse(content); - if (root == NULL) { - printf("ERROR: failed to cjson parse %s, invalid json format\n", file); - goto PARSE_OVER; - } - - cJSON* filetype = cJSON_GetObjectItem(root, "filetype"); - if (filetype && filetype->type == cJSON_String && filetype->valuestring != NULL) { - if (0 == strcasecmp("insert", filetype->valuestring)) { - g_args.test_mode = INSERT_TEST; - } else if (0 == strcasecmp("query", filetype->valuestring)) { - g_args.test_mode = QUERY_TEST; - } else if (0 == strcasecmp("subscribe", filetype->valuestring)) { - g_args.test_mode = SUBSCRIBE_TEST; - } else { - printf("ERROR: failed to read json, filetype not support\n"); - goto PARSE_OVER; + FILE *fp = fopen(file, "r"); + if (!fp) { + printf("failed to read %s, reason:%s\n", file, strerror(errno)); + return false; } - } else if (!filetype) { - g_args.test_mode = INSERT_TEST; - } else { - printf("ERROR: failed to read json, filetype not found\n"); - goto PARSE_OVER; - } - if (INSERT_TEST == g_args.test_mode) { - ret = getMetaFromInsertJsonFile(root); - } else if ((QUERY_TEST == g_args.test_mode) - || (SUBSCRIBE_TEST == g_args.test_mode)) { - ret = getMetaFromQueryJsonFile(root); - } else { - errorPrint("%s() LN%d, input json file type error! please input correct file type: insert or query or subscribe\n", - __func__, __LINE__); - goto PARSE_OVER; - } + bool ret = false; + int maxLen = 6400000; + char *content = calloc(1, maxLen + 1); + int len = fread(content, 1, maxLen, fp); + if (len <= 0) { + free(content); + fclose(fp); + printf("failed to read %s, content is null", file); + return false; + } + + content[len] = 0; + cJSON* root = cJSON_Parse(content); + if (root == NULL) { + printf("ERROR: failed to cjson parse %s, invalid json format\n", file); + goto PARSE_OVER; + } + + cJSON* filetype = cJSON_GetObjectItem(root, "filetype"); + if (filetype && filetype->type == cJSON_String && filetype->valuestring != NULL) { + if (0 == strcasecmp("insert", filetype->valuestring)) { + g_args.test_mode = INSERT_TEST; + } else if (0 == strcasecmp("query", filetype->valuestring)) { + g_args.test_mode = QUERY_TEST; + } else if (0 == strcasecmp("subscribe", filetype->valuestring)) { + g_args.test_mode = SUBSCRIBE_TEST; + } else { + printf("ERROR: failed to read json, filetype not support\n"); + goto PARSE_OVER; + } + } else if (!filetype) { + g_args.test_mode = INSERT_TEST; + } else { + printf("ERROR: failed to read json, filetype not found\n"); + goto PARSE_OVER; + } + + if (INSERT_TEST == g_args.test_mode) { + ret = getMetaFromInsertJsonFile(root); + } else if ((QUERY_TEST == g_args.test_mode) + || (SUBSCRIBE_TEST == g_args.test_mode)) { + ret = getMetaFromQueryJsonFile(root); + } else { + errorPrint("%s() LN%d, input json file type error! please input correct file type: insert or query or subscribe\n", + __func__, __LINE__); + goto PARSE_OVER; + } PARSE_OVER: - free(content); - cJSON_Delete(root); - fclose(fp); - return ret; + free(content); + cJSON_Delete(root); + fclose(fp); + return ret; } static int prepareSampleData() { - for (int i = 0; i < g_Dbs.dbCount; i++) { - for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { - if (g_Dbs.db[i].superTbls[j].tagsFile[0] != 0) { - if (readTagFromCsvFileToMem(&g_Dbs.db[i].superTbls[j]) != 0) { - return -1; + for (int i = 0; i < g_Dbs.dbCount; i++) { + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + if (g_Dbs.db[i].superTbls[j].tagsFile[0] != 0) { + if (readTagFromCsvFileToMem(&g_Dbs.db[i].superTbls[j]) != 0) { + return -1; + } + } } - } } - } - return 0; + return 0; } static void postFreeResource() { - tmfclose(g_fpOfInsertResult); - for (int i = 0; i < g_Dbs.dbCount; i++) { - for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) { - if (0 != g_Dbs.db[i].superTbls[j].colsOfCreateChildTable) { - free(g_Dbs.db[i].superTbls[j].colsOfCreateChildTable); - g_Dbs.db[i].superTbls[j].colsOfCreateChildTable = NULL; - } - if (0 != g_Dbs.db[i].superTbls[j].sampleDataBuf) { - free(g_Dbs.db[i].superTbls[j].sampleDataBuf); - g_Dbs.db[i].superTbls[j].sampleDataBuf = NULL; - } - if (0 != g_Dbs.db[i].superTbls[j].tagDataBuf) { - free(g_Dbs.db[i].superTbls[j].tagDataBuf); - g_Dbs.db[i].superTbls[j].tagDataBuf = NULL; - } - if (0 != g_Dbs.db[i].superTbls[j].childTblName) { - free(g_Dbs.db[i].superTbls[j].childTblName); - g_Dbs.db[i].superTbls[j].childTblName = NULL; - } + tmfclose(g_fpOfInsertResult); + for (int i = 0; i < g_Dbs.dbCount; i++) { + for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) { + if (0 != g_Dbs.db[i].superTbls[j].colsOfCreateChildTable) { + free(g_Dbs.db[i].superTbls[j].colsOfCreateChildTable); + g_Dbs.db[i].superTbls[j].colsOfCreateChildTable = NULL; + } + if (0 != g_Dbs.db[i].superTbls[j].sampleDataBuf) { + free(g_Dbs.db[i].superTbls[j].sampleDataBuf); + g_Dbs.db[i].superTbls[j].sampleDataBuf = NULL; + } + if (0 != g_Dbs.db[i].superTbls[j].tagDataBuf) { + free(g_Dbs.db[i].superTbls[j].tagDataBuf); + g_Dbs.db[i].superTbls[j].tagDataBuf = NULL; + } + if (0 != g_Dbs.db[i].superTbls[j].childTblName) { + free(g_Dbs.db[i].superTbls[j].childTblName); + g_Dbs.db[i].superTbls[j].childTblName = NULL; + } + } } - } } static int getRowDataFromSample( @@ -5033,30 +5049,30 @@ static int64_t generateData(char *recBuf, char **data_type, } static int prepareSampleDataForSTable(SSuperTable *superTblInfo) { - char* sampleDataBuf = NULL; + char* sampleDataBuf = NULL; - sampleDataBuf = calloc( + sampleDataBuf = calloc( superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1); - if (sampleDataBuf == NULL) { - errorPrint("%s() LN%d, Failed to calloc %"PRIu64" Bytes, reason:%s\n", - __func__, __LINE__, - superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, - strerror(errno)); - return -1; - } + if (sampleDataBuf == NULL) { + errorPrint("%s() LN%d, Failed to calloc %"PRIu64" Bytes, reason:%s\n", + __func__, __LINE__, + superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, + strerror(errno)); + return -1; + } - superTblInfo->sampleDataBuf = sampleDataBuf; - int ret = readSampleFromCsvFileToMem(superTblInfo); + superTblInfo->sampleDataBuf = sampleDataBuf; + int ret = readSampleFromCsvFileToMem(superTblInfo); - if (0 != ret) { - errorPrint("%s() LN%d, read sample from csv file failed.\n", - __func__, __LINE__); - tmfree(sampleDataBuf); - superTblInfo->sampleDataBuf = NULL; - return -1; - } + if (0 != ret) { + errorPrint("%s() LN%d, read sample from csv file failed.\n", + __func__, __LINE__); + tmfree(sampleDataBuf); + superTblInfo->sampleDataBuf = NULL; + return -1; + } - return 0; + return 0; } static int32_t execInsert(threadInfo *pThreadInfo, uint32_t k) @@ -5155,54 +5171,54 @@ static int32_t generateDataTailWithoutStb( uint64_t recordFrom, int64_t startTime, /* int64_t *pSamplePos, */int64_t *dataLen) { - uint64_t len = 0; - char *pstr = buffer; + uint64_t len = 0; + char *pstr = buffer; - verbosePrint("%s() LN%d batch=%d\n", __func__, __LINE__, batch); + verbosePrint("%s() LN%d batch=%d\n", __func__, __LINE__, batch); - int32_t k = 0; - for (k = 0; k < batch;) { - char data[MAX_DATA_SIZE]; - memset(data, 0, MAX_DATA_SIZE); + int32_t k = 0; + for (k = 0; k < batch;) { + char data[MAX_DATA_SIZE]; + memset(data, 0, MAX_DATA_SIZE); - int64_t retLen = 0; + int64_t retLen = 0; - char **data_type = g_args.datatype; - int lenOfBinary = g_args.len_of_binary; + char **data_type = g_args.datatype; + int lenOfBinary = g_args.len_of_binary; - if (g_args.disorderRatio) { - retLen = generateData(data, data_type, - startTime + getTSRandTail( - (int64_t) DEFAULT_TIMESTAMP_STEP, k, - g_args.disorderRatio, - g_args.disorderRange), - lenOfBinary); - } else { - retLen = generateData(data, data_type, - startTime + (int64_t) (DEFAULT_TIMESTAMP_STEP* k), - lenOfBinary); + if (g_args.disorderRatio) { + retLen = generateData(data, data_type, + startTime + getTSRandTail( + (int64_t) DEFAULT_TIMESTAMP_STEP, k, + g_args.disorderRatio, + g_args.disorderRange), + lenOfBinary); + } else { + retLen = generateData(data, data_type, + startTime + (int64_t) (DEFAULT_TIMESTAMP_STEP* k), + lenOfBinary); + } + + if (len > remainderBufLen) + break; + + pstr += sprintf(pstr, "%s", data); + k++; + len += retLen; + remainderBufLen -= retLen; + + verbosePrint("%s() LN%d len=%"PRIu64" k=%d \nbuffer=%s\n", + __func__, __LINE__, len, k, buffer); + + recordFrom ++; + + if (recordFrom >= insertRows) { + break; + } } - if (len > remainderBufLen) - break; - - pstr += sprintf(pstr, "%s", data); - k++; - len += retLen; - remainderBufLen -= retLen; - - verbosePrint("%s() LN%d len=%"PRIu64" k=%d \nbuffer=%s\n", - __func__, __LINE__, len, k, buffer); - - recordFrom ++; - - if (recordFrom >= insertRows) { - break; - } - } - - *dataLen = len; - return k; + *dataLen = len; + return k; } static int64_t getTSRandTail(int64_t timeStampStep, int32_t seq, @@ -5297,82 +5313,82 @@ static int generateSQLHeadWithoutStb(char *tableName, char *dbName, char *buffer, int remainderBufLen) { - int len; + int len; - char headBuf[HEAD_BUFF_LEN]; + char headBuf[HEAD_BUFF_LEN]; - len = snprintf( - headBuf, - HEAD_BUFF_LEN, - "%s.%s values", - dbName, - tableName); + len = snprintf( + headBuf, + HEAD_BUFF_LEN, + "%s.%s values", + dbName, + tableName); - if (len > remainderBufLen) - return -1; + if (len > remainderBufLen) + return -1; - tstrncpy(buffer, headBuf, len + 1); + tstrncpy(buffer, headBuf, len + 1); - return len; + return len; } static int generateStbSQLHead( SSuperTable* superTblInfo, - char *tableName, int32_t tableSeq, + char *tableName, int64_t tableSeq, char *dbName, char *buffer, int remainderBufLen) { - int len; + int len; - char headBuf[HEAD_BUFF_LEN]; + char headBuf[HEAD_BUFF_LEN]; - if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { - char* tagsValBuf = NULL; - if (0 == superTblInfo->tagSource) { + if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { + char* tagsValBuf = NULL; + if (0 == superTblInfo->tagSource) { tagsValBuf = generateTagValuesForStb(superTblInfo, tableSeq); - } else { + } else { tagsValBuf = getTagValueFromTagSample( superTblInfo, tableSeq % superTblInfo->tagSampleCount); - } - if (NULL == tagsValBuf) { - errorPrint("%s() LN%d, tag buf failed to allocate memory\n", - __func__, __LINE__); - return -1; - } + } + if (NULL == tagsValBuf) { + errorPrint("%s() LN%d, tag buf failed to allocate memory\n", + __func__, __LINE__); + return -1; + } - len = snprintf( - headBuf, - HEAD_BUFF_LEN, - "%s.%s using %s.%s TAGS%s values", - dbName, - tableName, - dbName, - superTblInfo->sTblName, - tagsValBuf); - tmfree(tagsValBuf); + len = snprintf( + headBuf, + HEAD_BUFF_LEN, + "%s.%s using %s.%s TAGS%s values", + dbName, + tableName, + dbName, + superTblInfo->sTblName, + tagsValBuf); + tmfree(tagsValBuf); } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) { - len = snprintf( - headBuf, - HEAD_BUFF_LEN, - "%s.%s values", - dbName, - tableName); + len = snprintf( + headBuf, + HEAD_BUFF_LEN, + "%s.%s values", + dbName, + tableName); } else { - len = snprintf( - headBuf, - HEAD_BUFF_LEN, - "%s.%s values", - dbName, - tableName); - } + len = snprintf( + headBuf, + HEAD_BUFF_LEN, + "%s.%s values", + dbName, + tableName); + } - if (len > remainderBufLen) - return -1; + if (len > remainderBufLen) + return -1; - tstrncpy(buffer, headBuf, len + 1); + tstrncpy(buffer, headBuf, len + 1); - return len; + return len; } static int32_t generateStbInterlaceData( @@ -5650,8 +5666,7 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, *ptr += bind->buffer_length; } else { - errorPrint( "No support data type: %s\n", - dataType); + errorPrint( "No support data type: %s\n", dataType); return -1; } @@ -5737,28 +5752,120 @@ static int32_t prepareStmtWithoutStb( return k; } +static int32_t prepareStbStmtBind( + char *bindArray, SSuperTable *stbInfo, bool sourceRand, + int64_t startTime, int32_t recSeq, + bool isColumn) +{ + char *bindBuffer = calloc(1, g_args.len_of_binary); + if (bindBuffer == NULL) { + errorPrint("%s() LN%d, Failed to allocate %d bind buffer\n", + __func__, __LINE__, g_args.len_of_binary); + return -1; + } + + char data[MAX_DATA_SIZE]; + memset(data, 0, MAX_DATA_SIZE); + char *ptr = data; + + TAOS_BIND *bind; + + if (isColumn) { + for (int i = 0; i < stbInfo->columnCount + 1; i ++) { + bind = (TAOS_BIND *)((char *)bindArray + (sizeof(TAOS_BIND) * i)); + + if (i == 0) { + int64_t *bind_ts; + + bind_ts = (int64_t *)ptr; + bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + if (stbInfo->disorderRatio) { + *bind_ts = startTime + getTSRandTail( + stbInfo->timeStampStep, recSeq, + stbInfo->disorderRatio, + stbInfo->disorderRange); + } else { + *bind_ts = startTime + stbInfo->timeStampStep * recSeq; + } + bind->buffer_length = sizeof(int64_t); + bind->buffer = bind_ts; + bind->length = &bind->buffer_length; + bind->is_null = NULL; + + ptr += bind->buffer_length; + } else { + int cursor = 0; + + if (sourceRand) { + if ( -1 == prepareStmtBindArrayByType( + bind, + stbInfo->columns[i-1].dataType, + stbInfo->columns[i-1].dataLen, + &ptr, + NULL)) { + free(bindBuffer); + return -1; + } + } else { + char *restStr = stbInfo->sampleDataBuf + cursor; + int lengthOfRest = strlen(restStr); + + int index = 0; + for (index = 0; index < lengthOfRest; index ++) { + if (restStr[index] == ',') { + break; + } + } + + memset(bindBuffer, 0, g_args.len_of_binary); + strncpy(bindBuffer, restStr, index); + cursor += index + 1; // skip ',' too + + if ( -1 == prepareStmtBindArrayByType( + bind, + stbInfo->columns[i-1].dataType, + stbInfo->columns[i-1].dataLen, + &ptr, + bindBuffer)) { + free(bindBuffer); + return -1; + } + } + } + } + } else { + TAOS_BIND *tag; + + for (int t = 0; t < stbInfo->tagCount; t ++) { + tag = (TAOS_BIND *)((char *)bindArray + (sizeof(TAOS_BIND) * t)); + if ( -1 == prepareStmtBindArrayByType( + tag, + stbInfo->tags[t].dataType, + stbInfo->tags[t].dataLen, + &ptr, + NULL)) { + free(bindBuffer); + return -1; + } + } + + } + + return 0; +} + static int32_t prepareStbStmt( SSuperTable *stbInfo, TAOS_STMT *stmt, - char *tableName, uint32_t batch, + char *tableName, + int64_t tableSeq, + uint32_t batch, uint64_t insertRows, uint64_t recordFrom, int64_t startTime, int64_t *pSamplePos) { - int ret = taos_stmt_set_tbname(stmt, tableName); - if (ret != 0) { - errorPrint("failed to execute taos_stmt_set_tbname(%s). return 0x%x. reason: %s\n", - tableName, ret, taos_errstr(NULL)); - return ret; - } - - char *bindArray = malloc(sizeof(TAOS_BIND) * (stbInfo->columnCount + 1)); - if (bindArray == NULL) { - errorPrint("%s() LN%d, Failed to allocate %d bind params\n", - __func__, __LINE__, (stbInfo->columnCount + 1)); - return -1; - } + int ret; bool sourceRand; if (0 == strncasecmp(stbInfo->dataSource, "rand", strlen("rand"))) { @@ -5767,83 +5874,68 @@ static int32_t prepareStbStmt( sourceRand = false; // from sample data file } - char *bindBuffer = malloc(g_args.len_of_binary); - if (bindBuffer == NULL) { - errorPrint("%s() LN%d, Failed to allocate %d bind buffer\n", - __func__, __LINE__, g_args.len_of_binary); - free(bindArray); + if (AUTO_CREATE_SUBTBL == stbInfo->autoCreateTable) { + char* tagsValBuf = NULL; + + bool tagRand; + if (0 == stbInfo->tagSource) { + tagRand = true; + tagsValBuf = generateTagValuesForStb(stbInfo, tableSeq); + } else { + tagRand = false; + tagsValBuf = getTagValueFromTagSample( + stbInfo, + tableSeq % stbInfo->tagSampleCount); + } + + if (NULL == tagsValBuf) { + errorPrint("%s() LN%d, tag buf failed to allocate memory\n", + __func__, __LINE__); + return -1; + } + + char *tagsArray = calloc(1, sizeof(TAOS_BIND) * stbInfo->tagCount); + if (NULL == tagsArray) { + tmfree(tagsValBuf); + errorPrint("%s() LN%d, tag buf failed to allocate memory\n", + __func__, __LINE__); + return -1; + } + + if (-1 == prepareStbStmtBind( + tagsArray, stbInfo, tagRand, -1, -1, false /* is tag */)) { + free(tagsArray); + return -1; + } + + ret = taos_stmt_set_tbname_tags(stmt, tableName, (TAOS_BIND *)tagsArray); + + tmfree(tagsValBuf); + tmfree((char *)tagsArray); + } else { + ret = taos_stmt_set_tbname(stmt, tableName); + } + + if (ret != 0) { + errorPrint("failed to execute taos_stmt_set_tbname(%s). return 0x%x. reason: %s\n", + tableName, ret, taos_errstr(NULL)); + return ret; + } + + char *bindArray = calloc(1, sizeof(TAOS_BIND) * (stbInfo->columnCount + 1)); + if (bindArray == NULL) { + errorPrint("%s() LN%d, Failed to allocate %d bind params\n", + __func__, __LINE__, (stbInfo->columnCount + 1)); return -1; } uint32_t k; for (k = 0; k < batch;) { /* columnCount + 1 (ts) */ - char data[MAX_DATA_SIZE]; - memset(data, 0, MAX_DATA_SIZE); - - char *ptr = data; - TAOS_BIND *bind = (TAOS_BIND *)(bindArray + 0); - - int64_t *bind_ts; - - bind_ts = (int64_t *)ptr; - bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - if (stbInfo->disorderRatio) { - *bind_ts = startTime + getTSRandTail( - stbInfo->timeStampStep, k, - stbInfo->disorderRatio, - stbInfo->disorderRange); - } else { - *bind_ts = startTime + stbInfo->timeStampStep * k; - } - bind->buffer_length = sizeof(int64_t); - bind->buffer = bind_ts; - bind->length = &bind->buffer_length; - bind->is_null = NULL; - - ptr += bind->buffer_length; - - int cursor = 0; - for (int i = 0; i < stbInfo->columnCount; i ++) { - bind = (TAOS_BIND *)((char *)bindArray + (sizeof(TAOS_BIND) * (i + 1))); - - if (sourceRand) { - if ( -1 == prepareStmtBindArrayByType( - bind, - stbInfo->columns[i].dataType, - stbInfo->columns[i].dataLen, - &ptr, - NULL)) { - free(bindArray); - free(bindBuffer); - return -1; - } - } else { - char *restStr = stbInfo->sampleDataBuf + cursor; - int lengthOfRest = strlen(restStr); - - int index = 0; - for (index = 0; index < lengthOfRest; index ++) { - if (restStr[index] == ',') { - break; - } - } - - memset(bindBuffer, 0, g_args.len_of_binary); - strncpy(bindBuffer, restStr, index); - cursor += index + 1; // skip ',' too - - if ( -1 == prepareStmtBindArrayByType( - bind, - stbInfo->columns[i].dataType, - stbInfo->columns[i].dataLen, - &ptr, - bindBuffer)) { - free(bindArray); - free(bindBuffer); - return -1; - } - } + if (-1 == prepareStbStmtBind(bindArray, stbInfo, sourceRand, + startTime, k, true /* is column */)) { + free(bindArray); + return -1; } taos_stmt_bind_param(stmt, (TAOS_BIND *)bindArray); // if msg > 3MB, break @@ -5861,7 +5953,6 @@ static int32_t prepareStbStmt( } } - free(bindBuffer); free(bindArray); return k; } @@ -5869,7 +5960,9 @@ static int32_t prepareStbStmt( static int32_t prepareStbStmtInterlace( SSuperTable *stbInfo, TAOS_STMT *stmt, - char *tableName, uint32_t batch, + char *tableName, + int64_t tableSeq, + uint32_t batch, uint64_t insertRows, uint64_t recordFrom, int64_t startTime, @@ -5879,6 +5972,7 @@ static int32_t prepareStbStmtInterlace( stbInfo, stmt, tableName, + tableSeq, batch, insertRows, 0, startTime, pSamplePos); @@ -5887,7 +5981,9 @@ static int32_t prepareStbStmtInterlace( static int32_t prepareStbStmtProgressive( SSuperTable *stbInfo, TAOS_STMT *stmt, - char *tableName, uint32_t batch, + char *tableName, + int64_t tableSeq, + uint32_t batch, uint64_t insertRows, uint64_t recordFrom, int64_t startTime, @@ -5897,6 +5993,7 @@ static int32_t prepareStbStmtProgressive( stbInfo, stmt, tableName, + tableSeq, g_args.num_of_RPR, insertRows, recordFrom, startTime, pSamplePos); @@ -6097,6 +6194,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { superTblInfo, pThreadInfo->stmt, tableName, + tableSeq, batchPerTbl, insertRows, i, startTime, @@ -6263,164 +6361,165 @@ free_of_interlace: // sync insertion progressive data static void* syncWriteProgressive(threadInfo *pThreadInfo) { - debugPrint("%s() LN%d: ### progressive write\n", __func__, __LINE__); + debugPrint("%s() LN%d: ### progressive write\n", __func__, __LINE__); - SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len; - int64_t timeStampStep = - superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP; - int64_t insertRows = + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len; + int64_t timeStampStep = + superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP; + int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; - verbosePrint("%s() LN%d insertRows=%"PRId64"\n", + verbosePrint("%s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, insertRows); - pThreadInfo->buffer = calloc(maxSqlLen, 1); - if (NULL == pThreadInfo->buffer) { - errorPrint( "Failed to alloc %"PRIu64" Bytes, reason:%s\n", - maxSqlLen, - strerror(errno)); - return NULL; - } - - uint64_t lastPrintTime = taosGetTimestampMs(); - uint64_t startTs = taosGetTimestampMs(); - uint64_t endTs; - - pThreadInfo->totalInsertRows = 0; - pThreadInfo->totalAffectedRows = 0; - - pThreadInfo->samplePos = 0; - - for (uint64_t tableSeq = pThreadInfo->start_table_from; - tableSeq <= pThreadInfo->end_table_to; - tableSeq ++) { - int64_t start_time = pThreadInfo->start_time; - - for (uint64_t i = 0; i < insertRows;) { - char tableName[TSDB_TABLE_NAME_LEN]; - getTableName(tableName, pThreadInfo, tableSeq); - verbosePrint("%s() LN%d: tid=%d seq=%"PRId64" tableName=%s\n", - __func__, __LINE__, - pThreadInfo->threadID, tableSeq, tableName); - if (0 == strlen(tableName)) { - errorPrint("[%d] %s() LN%d, getTableName return null\n", - pThreadInfo->threadID, __func__, __LINE__); - free(pThreadInfo->buffer); + pThreadInfo->buffer = calloc(maxSqlLen, 1); + if (NULL == pThreadInfo->buffer) { + errorPrint( "Failed to alloc %"PRIu64" Bytes, reason:%s\n", + maxSqlLen, + strerror(errno)); return NULL; - } - - int64_t remainderBufLen = maxSqlLen; - char *pstr = pThreadInfo->buffer; - - int len = snprintf(pstr, - strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO); - - pstr += len; - remainderBufLen -= len; - - int32_t generated; - if (superTblInfo) { - if (superTblInfo->iface == STMT_IFACE) { -#if STMT_IFACE_ENABLED == 1 - generated = prepareStbStmtProgressive( - superTblInfo, - pThreadInfo->stmt, - tableName, - g_args.num_of_RPR, - insertRows, i, start_time, - &(pThreadInfo->samplePos)); -#else - generated = -1; -#endif - } else { - generated = generateStbProgressiveData( - superTblInfo, - tableName, tableSeq, pThreadInfo->db_name, pstr, - insertRows, i, start_time, - &(pThreadInfo->samplePos), - &remainderBufLen); - } - } else { - if (g_args.iface == STMT_IFACE) { -#if STMT_IFACE_ENABLED == 1 - generated = prepareStmtWithoutStb( - pThreadInfo->stmt, - tableName, - g_args.num_of_RPR, - insertRows, i, - start_time); -#else - generated = -1; -#endif - } else { - generated = generateProgressiveDataWithoutStb( - tableName, - /* tableSeq, */ - pThreadInfo, pstr, insertRows, - i, start_time, - /* &(pThreadInfo->samplePos), */ - &remainderBufLen); - } - } - if (generated > 0) - i += generated; - else - goto free_of_progressive; - - start_time += generated * timeStampStep; - pThreadInfo->totalInsertRows += generated; - - startTs = taosGetTimestampMs(); - - int32_t affectedRows = execInsert(pThreadInfo, generated); - - endTs = taosGetTimestampMs(); - uint64_t delay = endTs - startTs; - performancePrint("%s() LN%d, insert execution time is %"PRId64"ms\n", - __func__, __LINE__, delay); - verbosePrint("[%d] %s() LN%d affectedRows=%d\n", - pThreadInfo->threadID, - __func__, __LINE__, affectedRows); - - if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay; - if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay; - pThreadInfo->cntDelay++; - pThreadInfo->totalDelay += delay; - - if (affectedRows < 0) { - errorPrint("%s() LN%d, affected rows: %d\n", - __func__, __LINE__, affectedRows); - goto free_of_progressive; - } - - pThreadInfo->totalAffectedRows += affectedRows; - - int64_t currentPrintTime = taosGetTimestampMs(); - if (currentPrintTime - lastPrintTime > 30*1000) { - printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n", - pThreadInfo->threadID, - pThreadInfo->totalInsertRows, - pThreadInfo->totalAffectedRows); - lastPrintTime = currentPrintTime; - } - - if (i >= insertRows) - break; - } // num_of_DPT - - if ((g_args.verbose_print) && - (tableSeq == pThreadInfo->ntables - 1) && (superTblInfo) && - (0 == strncasecmp( - superTblInfo->dataSource, "sample", strlen("sample")))) { - verbosePrint("%s() LN%d samplePos=%"PRId64"\n", - __func__, __LINE__, pThreadInfo->samplePos); } - } // tableSeq + + uint64_t lastPrintTime = taosGetTimestampMs(); + uint64_t startTs = taosGetTimestampMs(); + uint64_t endTs; + + pThreadInfo->totalInsertRows = 0; + pThreadInfo->totalAffectedRows = 0; + + pThreadInfo->samplePos = 0; + + for (uint64_t tableSeq = pThreadInfo->start_table_from; + tableSeq <= pThreadInfo->end_table_to; + tableSeq ++) { + int64_t start_time = pThreadInfo->start_time; + + for (uint64_t i = 0; i < insertRows;) { + char tableName[TSDB_TABLE_NAME_LEN]; + getTableName(tableName, pThreadInfo, tableSeq); + verbosePrint("%s() LN%d: tid=%d seq=%"PRId64" tableName=%s\n", + __func__, __LINE__, + pThreadInfo->threadID, tableSeq, tableName); + if (0 == strlen(tableName)) { + errorPrint("[%d] %s() LN%d, getTableName return null\n", + pThreadInfo->threadID, __func__, __LINE__); + free(pThreadInfo->buffer); + return NULL; + } + + int64_t remainderBufLen = maxSqlLen; + char *pstr = pThreadInfo->buffer; + + int len = snprintf(pstr, + strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO); + + pstr += len; + remainderBufLen -= len; + + int32_t generated; + if (superTblInfo) { + if (superTblInfo->iface == STMT_IFACE) { +#if STMT_IFACE_ENABLED == 1 + generated = prepareStbStmtProgressive( + superTblInfo, + pThreadInfo->stmt, + tableName, + tableSeq, + g_args.num_of_RPR, + insertRows, i, start_time, + &(pThreadInfo->samplePos)); +#else + generated = -1; +#endif + } else { + generated = generateStbProgressiveData( + superTblInfo, + tableName, tableSeq, pThreadInfo->db_name, pstr, + insertRows, i, start_time, + &(pThreadInfo->samplePos), + &remainderBufLen); + } + } else { + if (g_args.iface == STMT_IFACE) { +#if STMT_IFACE_ENABLED == 1 + generated = prepareStmtWithoutStb( + pThreadInfo->stmt, + tableName, + g_args.num_of_RPR, + insertRows, i, + start_time); +#else + generated = -1; +#endif + } else { + generated = generateProgressiveDataWithoutStb( + tableName, + /* tableSeq, */ + pThreadInfo, pstr, insertRows, + i, start_time, + /* &(pThreadInfo->samplePos), */ + &remainderBufLen); + } + } + if (generated > 0) + i += generated; + else + goto free_of_progressive; + + start_time += generated * timeStampStep; + pThreadInfo->totalInsertRows += generated; + + startTs = taosGetTimestampMs(); + + int32_t affectedRows = execInsert(pThreadInfo, generated); + + endTs = taosGetTimestampMs(); + uint64_t delay = endTs - startTs; + performancePrint("%s() LN%d, insert execution time is %"PRId64"ms\n", + __func__, __LINE__, delay); + verbosePrint("[%d] %s() LN%d affectedRows=%d\n", + pThreadInfo->threadID, + __func__, __LINE__, affectedRows); + + if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay; + if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay; + pThreadInfo->cntDelay++; + pThreadInfo->totalDelay += delay; + + if (affectedRows < 0) { + errorPrint("%s() LN%d, affected rows: %d\n", + __func__, __LINE__, affectedRows); + goto free_of_progressive; + } + + pThreadInfo->totalAffectedRows += affectedRows; + + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30*1000) { + printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n", + pThreadInfo->threadID, + pThreadInfo->totalInsertRows, + pThreadInfo->totalAffectedRows); + lastPrintTime = currentPrintTime; + } + + if (i >= insertRows) + break; + } // num_of_DPT + + if ((g_args.verbose_print) && + (tableSeq == pThreadInfo->ntables - 1) && (superTblInfo) && + (0 == strncasecmp( + superTblInfo->dataSource, "sample", strlen("sample")))) { + verbosePrint("%s() LN%d samplePos=%"PRId64"\n", + __func__, __LINE__, pThreadInfo->samplePos); + } + } // tableSeq free_of_progressive: - tmfree(pThreadInfo->buffer); - printStatPerThread(pThreadInfo); - return NULL; + tmfree(pThreadInfo->buffer); + printStatPerThread(pThreadInfo); + return NULL; } static void* syncWrite(void *sarg) { @@ -6428,6 +6527,8 @@ static void* syncWrite(void *sarg) { threadInfo *pThreadInfo = (threadInfo *)sarg; SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + setThreadName("syncWrite"); + uint32_t interlaceRows; if (superTblInfo) { @@ -6513,6 +6614,8 @@ static void *asyncWrite(void *sarg) { threadInfo *pThreadInfo = (threadInfo *)sarg; SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + setThreadName("asyncWrite"); + pThreadInfo->st = 0; pThreadInfo->et = 0; pThreadInfo->lastTs = pThreadInfo->start_time; @@ -6754,7 +6857,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, exit(-1); } - char buffer[3000]; + char buffer[BUFFER_SIZE]; char *pstr = buffer; if ((superTblInfo) @@ -6913,6 +7016,7 @@ static void *readTable(void *sarg) { #if 1 threadInfo *pThreadInfo = (threadInfo *)sarg; TAOS *taos = pThreadInfo->taos; + setThreadName("readTable"); char command[BUFFER_SIZE] = "\0"; uint64_t sTime = pThreadInfo->start_time; char *tb_prefix = pThreadInfo->tb_prefix; @@ -6985,6 +7089,7 @@ static void *readMetric(void *sarg) { #if 1 threadInfo *pThreadInfo = (threadInfo *)sarg; TAOS *taos = pThreadInfo->taos; + setThreadName("readMetric"); char command[BUFFER_SIZE] = "\0"; FILE *fp = fopen(pThreadInfo->filePath, "a"); if (NULL == fp) { @@ -7161,6 +7266,8 @@ static int insertTestProcess() { static void *specifiedTableQuery(void *sarg) { threadInfo *pThreadInfo = (threadInfo *)sarg; + setThreadName("specTableQuery"); + if (pThreadInfo->taos == NULL) { TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, @@ -7260,6 +7367,8 @@ static void *superTableQuery(void *sarg) { char sqlstr[MAX_QUERY_SQL_LENGTH]; threadInfo *pThreadInfo = (threadInfo *)sarg; + setThreadName("superTableQuery"); + if (pThreadInfo->taos == NULL) { TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, @@ -7562,6 +7671,8 @@ static void *superSubscribe(void *sarg) { TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT] = {0}; uint64_t tsubSeq; + setThreadName("superSub"); + if (pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) { errorPrint("The table number(%"PRId64") of the thread is more than max query sql count: %d\n", pThreadInfo->ntables, MAX_QUERY_SQL_COUNT); @@ -7708,6 +7819,8 @@ static void *specifiedSubscribe(void *sarg) { threadInfo *pThreadInfo = (threadInfo *)sarg; // TAOS_SUB* tsub = NULL; + setThreadName("specSub"); + if (pThreadInfo->taos == NULL) { pThreadInfo->taos = taos_connect(g_queryInfo.host, g_queryInfo.user, @@ -8140,55 +8253,55 @@ static int isCommentLine(char *line) { static void querySqlFile(TAOS* taos, char* sqlFile) { - FILE *fp = fopen(sqlFile, "r"); - if (fp == NULL) { - printf("failed to open file %s, reason:%s\n", sqlFile, strerror(errno)); - return; - } - - int read_len = 0; - char * cmd = calloc(1, TSDB_MAX_BYTES_PER_ROW); - size_t cmd_len = 0; - char * line = NULL; - size_t line_len = 0; - - double t = taosGetTimestampMs(); - - while((read_len = tgetline(&line, &line_len, fp)) != -1) { - if (read_len >= TSDB_MAX_BYTES_PER_ROW) continue; - line[--read_len] = '\0'; - - if (read_len == 0 || isCommentLine(line)) { // line starts with # - continue; - } - - if (line[read_len - 1] == '\\') { - line[read_len - 1] = ' '; - memcpy(cmd + cmd_len, line, read_len); - cmd_len += read_len; - continue; - } - - memcpy(cmd + cmd_len, line, read_len); - if (0 != queryDbExec(taos, cmd, NO_INSERT_TYPE, false)) { - errorPrint("%s() LN%d, queryDbExec %s failed!\n", - __func__, __LINE__, cmd); - tmfree(cmd); - tmfree(line); - tmfclose(fp); + FILE *fp = fopen(sqlFile, "r"); + if (fp == NULL) { + printf("failed to open file %s, reason:%s\n", sqlFile, strerror(errno)); return; } - memset(cmd, 0, TSDB_MAX_BYTES_PER_ROW); - cmd_len = 0; - } - t = taosGetTimestampMs() - t; - printf("run %s took %.6f second(s)\n\n", sqlFile, t); + int read_len = 0; + char * cmd = calloc(1, TSDB_MAX_BYTES_PER_ROW); + size_t cmd_len = 0; + char * line = NULL; + size_t line_len = 0; - tmfree(cmd); - tmfree(line); - tmfclose(fp); - return; + double t = taosGetTimestampMs(); + + while((read_len = tgetline(&line, &line_len, fp)) != -1) { + if (read_len >= TSDB_MAX_BYTES_PER_ROW) continue; + line[--read_len] = '\0'; + + if (read_len == 0 || isCommentLine(line)) { // line starts with # + continue; + } + + if (line[read_len - 1] == '\\') { + line[read_len - 1] = ' '; + memcpy(cmd + cmd_len, line, read_len); + cmd_len += read_len; + continue; + } + + memcpy(cmd + cmd_len, line, read_len); + if (0 != queryDbExec(taos, cmd, NO_INSERT_TYPE, false)) { + errorPrint("%s() LN%d, queryDbExec %s failed!\n", + __func__, __LINE__, cmd); + tmfree(cmd); + tmfree(line); + tmfclose(fp); + return; + } + memset(cmd, 0, TSDB_MAX_BYTES_PER_ROW); + cmd_len = 0; + } + + t = taosGetTimestampMs() - t; + printf("run %s took %.6f second(s)\n\n", sqlFile, t); + + tmfree(cmd); + tmfree(line); + tmfclose(fp); + return; } static void testMetaFile() { From 199bf6e0d5258160e6d42f5187d83b676b9d18f6 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 22 Jul 2021 16:52:00 +0800 Subject: [PATCH 025/106] [TD-5445]: taosdemo bug stmt interface with sample data. (#6971) sync from develop branch. --- src/kit/taosdemo/taosdemo.c | 2914 ++++++++++++++++++----------------- 1 file changed, 1461 insertions(+), 1453 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index de894434ad..444a59f6d7 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -16,7 +16,7 @@ /* when in some thread query return error, thread don't exit, but return, otherwise coredump in other thread. -*/ + */ #include #include @@ -24,24 +24,24 @@ #define CURL_STATICLIB #ifdef LINUX - #include - #include - #ifndef _ALPINE - #include - #endif - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include +#include +#include +#ifndef _ALPINE +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #else - #include - #include +#include +#include #endif #include @@ -485,42 +485,42 @@ typedef unsigned __int32 uint32_t; #pragma comment ( lib, "ws2_32.lib" ) // Some old MinGW/CYGWIN distributions don't define this: #ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING - #define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004 +#define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004 #endif // ENABLE_VIRTUAL_TERMINAL_PROCESSING static HANDLE g_stdoutHandle; static DWORD g_consoleMode; static void setupForAnsiEscape(void) { - DWORD mode = 0; - g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE); + DWORD mode = 0; + g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE); - if(g_stdoutHandle == INVALID_HANDLE_VALUE) { - exit(GetLastError()); - } + if(g_stdoutHandle == INVALID_HANDLE_VALUE) { + exit(GetLastError()); + } - if(!GetConsoleMode(g_stdoutHandle, &mode)) { - exit(GetLastError()); - } + if(!GetConsoleMode(g_stdoutHandle, &mode)) { + exit(GetLastError()); + } - g_consoleMode = mode; + g_consoleMode = mode; - // Enable ANSI escape codes - mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING; + // Enable ANSI escape codes + mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING; - if(!SetConsoleMode(g_stdoutHandle, mode)) { - exit(GetLastError()); - } + if(!SetConsoleMode(g_stdoutHandle, mode)) { + exit(GetLastError()); + } } static void resetAfterAnsiEscape(void) { - // Reset colors - printf("\x1b[0m"); + // Reset colors + printf("\x1b[0m"); - // Reset console mode - if(!SetConsoleMode(g_stdoutHandle, g_consoleMode)) { - exit(GetLastError()); - } + // Reset console mode + if(!SetConsoleMode(g_stdoutHandle, g_consoleMode)) { + exit(GetLastError()); + } } static int taosRandom() @@ -534,15 +534,15 @@ static int taosRandom() static void setupForAnsiEscape(void) {} static void resetAfterAnsiEscape(void) { - // Reset colors - printf("\x1b[0m"); + // Reset colors + printf("\x1b[0m"); } #include static int taosRandom() { - return rand(); + return rand(); } #endif // ifdef Windows @@ -634,7 +634,7 @@ static FILE * g_fpOfInsertResult = NULL; #define debugPrint(fmt, ...) \ do { if (g_args.debug_print || g_args.verbose_print) \ - fprintf(stderr, "DEBG: "fmt, __VA_ARGS__); } while(0) + fprintf(stderr, "DEBG: "fmt, __VA_ARGS__); } while(0) #define verbosePrint(fmt, ...) \ do { if (g_args.verbose_print) \ @@ -1363,14 +1363,14 @@ static const char charNum[] = "0123456789"; static void nonrand_string(char *, int) __attribute__ ((unused)); // reserve for debugging purpose static void nonrand_string(char *str, int size) { - str[0] = 0; - if (size > 0) { - int n; - for (n = 0; n < size; n++) { - str[n] = charNum[n % 10]; + str[0] = 0; + if (size > 0) { + int n; + for (n = 0; n < size; n++) { + str[n] = charNum[n % 10]; + } + str[n] = 0; } - str[n] = 0; - } } #endif @@ -1436,8 +1436,8 @@ static int printfInsertMeta() { if (g_args.iface != INTERFACE_BUT) { // first time if no iface specified printf("interface: \033[33m%s\033[0m\n", - (g_args.iface==TAOSC_IFACE)?"taosc": - (g_args.iface==REST_IFACE)?"rest":"stmt"); + (g_args.iface==TAOSC_IFACE)?"taosc": + (g_args.iface==REST_IFACE)?"rest":"stmt"); } printf("host: \033[33m%s:%u\033[0m\n", @@ -2221,24 +2221,24 @@ static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port request_buf = malloc(req_buf_len); if (NULL == request_buf) { - errorPrint("%s", "ERROR, cannot allocate memory.\n"); - exit(EXIT_FAILURE); + errorPrint("%s", "ERROR, cannot allocate memory.\n"); + exit(EXIT_FAILURE); } char userpass_buf[INPUT_BUF_LEN]; int mod_table[] = {0, 2, 1}; static char base64[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', - 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', - 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', - 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', - 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', - 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', - 'w', 'x', 'y', 'z', '0', '1', '2', '3', - '4', '5', '6', '7', '8', '9', '+', '/'}; + 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', + 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', + 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', + 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', + 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '+', '/'}; snprintf(userpass_buf, INPUT_BUF_LEN, "%s:%s", - g_Dbs.user, g_Dbs.password); + g_Dbs.user, g_Dbs.password); size_t userpass_buf_len = strlen(userpass_buf); size_t encoded_len = 4 * ((userpass_buf_len +2) / 3); @@ -2270,22 +2270,22 @@ static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port memset(base64_buf, 0, INPUT_BUF_LEN); for (int n = 0, m = 0; n < userpass_buf_len;) { - uint32_t oct_a = n < userpass_buf_len ? - (unsigned char) userpass_buf[n++]:0; - uint32_t oct_b = n < userpass_buf_len ? - (unsigned char) userpass_buf[n++]:0; - uint32_t oct_c = n < userpass_buf_len ? - (unsigned char) userpass_buf[n++]:0; - uint32_t triple = (oct_a << 0x10) + (oct_b << 0x08) + oct_c; + uint32_t oct_a = n < userpass_buf_len ? + (unsigned char) userpass_buf[n++]:0; + uint32_t oct_b = n < userpass_buf_len ? + (unsigned char) userpass_buf[n++]:0; + uint32_t oct_c = n < userpass_buf_len ? + (unsigned char) userpass_buf[n++]:0; + uint32_t triple = (oct_a << 0x10) + (oct_b << 0x08) + oct_c; - base64_buf[m++] = base64[(triple >> 3* 6) & 0x3f]; - base64_buf[m++] = base64[(triple >> 2* 6) & 0x3f]; - base64_buf[m++] = base64[(triple >> 1* 6) & 0x3f]; - base64_buf[m++] = base64[(triple >> 0* 6) & 0x3f]; + base64_buf[m++] = base64[(triple >> 3* 6) & 0x3f]; + base64_buf[m++] = base64[(triple >> 2* 6) & 0x3f]; + base64_buf[m++] = base64[(triple >> 1* 6) & 0x3f]; + base64_buf[m++] = base64[(triple >> 0* 6) & 0x3f]; } for (int l = 0; l < mod_table[userpass_buf_len % 3]; l++) - base64_buf[encoded_len - 1 - l] = '='; + base64_buf[encoded_len - 1 - l] = '='; debugPrint("%s() LN%d: auth string base64 encoded: %s\n", __func__, __LINE__, base64_buf); @@ -2343,7 +2343,7 @@ static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port printf("Response:\n%s\n", response_buf); if (strlen(pThreadInfo->filePath) > 0) { - appendResultBufToFile(response_buf, pThreadInfo); + appendResultBufToFile(response_buf, pThreadInfo); } free(request_buf); @@ -2425,11 +2425,11 @@ static char* generateTagValuesForStb(SSuperTable* stbInfo, int64_t tableSeq) { if ((g_args.demo_mode) && (i == 0)) { dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, - "%"PRId64",", tableSeq % 10); + "%"PRId64",", tableSeq % 10); } else { dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, - "%"PRId64",", tableSeq); + "%"PRId64",", tableSeq); } } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "bigint", strlen("bigint"))) { @@ -2472,72 +2472,72 @@ static char* generateTagValuesForStb(SSuperTable* stbInfo, int64_t tableSeq) { } static int calcRowLen(SSuperTable* superTbls) { - int colIndex; - int lenOfOneRow = 0; + int colIndex; + int lenOfOneRow = 0; - for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) { - char* dataType = superTbls->columns[colIndex].dataType; + for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) { + char* dataType = superTbls->columns[colIndex].dataType; - if (strcasecmp(dataType, "BINARY") == 0) { - lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; - } else if (strcasecmp(dataType, "NCHAR") == 0) { - lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; - } else if (strcasecmp(dataType, "INT") == 0) { - lenOfOneRow += 11; - } else if (strcasecmp(dataType, "BIGINT") == 0) { - lenOfOneRow += 21; - } else if (strcasecmp(dataType, "SMALLINT") == 0) { - lenOfOneRow += 6; - } else if (strcasecmp(dataType, "TINYINT") == 0) { - lenOfOneRow += 4; - } else if (strcasecmp(dataType, "BOOL") == 0) { - lenOfOneRow += 6; - } else if (strcasecmp(dataType, "FLOAT") == 0) { - lenOfOneRow += 22; - } else if (strcasecmp(dataType, "DOUBLE") == 0) { - lenOfOneRow += 42; - } else if (strcasecmp(dataType, "TIMESTAMP") == 0) { - lenOfOneRow += 21; - } else { - printf("get error data type : %s\n", dataType); - exit(-1); + if (strcasecmp(dataType, "BINARY") == 0) { + lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; + } else if (strcasecmp(dataType, "NCHAR") == 0) { + lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; + } else if (strcasecmp(dataType, "INT") == 0) { + lenOfOneRow += 11; + } else if (strcasecmp(dataType, "BIGINT") == 0) { + lenOfOneRow += 21; + } else if (strcasecmp(dataType, "SMALLINT") == 0) { + lenOfOneRow += 6; + } else if (strcasecmp(dataType, "TINYINT") == 0) { + lenOfOneRow += 4; + } else if (strcasecmp(dataType, "BOOL") == 0) { + lenOfOneRow += 6; + } else if (strcasecmp(dataType, "FLOAT") == 0) { + lenOfOneRow += 22; + } else if (strcasecmp(dataType, "DOUBLE") == 0) { + lenOfOneRow += 42; + } else if (strcasecmp(dataType, "TIMESTAMP") == 0) { + lenOfOneRow += 21; + } else { + printf("get error data type : %s\n", dataType); + exit(-1); + } } - } - superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp + superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp - int tagIndex; - int lenOfTagOfOneRow = 0; - for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) { - char* dataType = superTbls->tags[tagIndex].dataType; + int tagIndex; + int lenOfTagOfOneRow = 0; + for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) { + char* dataType = superTbls->tags[tagIndex].dataType; - if (strcasecmp(dataType, "BINARY") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; - } else if (strcasecmp(dataType, "NCHAR") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; - } else if (strcasecmp(dataType, "INT") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11; - } else if (strcasecmp(dataType, "BIGINT") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21; - } else if (strcasecmp(dataType, "SMALLINT") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; - } else if (strcasecmp(dataType, "TINYINT") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4; - } else if (strcasecmp(dataType, "BOOL") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; - } else if (strcasecmp(dataType, "FLOAT") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22; - } else if (strcasecmp(dataType, "DOUBLE") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42; - } else { - printf("get error tag type : %s\n", dataType); - exit(-1); + if (strcasecmp(dataType, "BINARY") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; + } else if (strcasecmp(dataType, "NCHAR") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; + } else if (strcasecmp(dataType, "INT") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11; + } else if (strcasecmp(dataType, "BIGINT") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21; + } else if (strcasecmp(dataType, "SMALLINT") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; + } else if (strcasecmp(dataType, "TINYINT") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4; + } else if (strcasecmp(dataType, "BOOL") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; + } else if (strcasecmp(dataType, "FLOAT") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22; + } else if (strcasecmp(dataType, "DOUBLE") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42; + } else { + printf("get error tag type : %s\n", dataType); + exit(-1); + } } - } - superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow; + superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow; - return 0; + return 0; } @@ -2545,84 +2545,84 @@ static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, char* dbName, char* sTblName, char** childTblNameOfSuperTbl, int64_t* childTblCountOfSuperTbl, int64_t limit, uint64_t offset) { - char command[BUFFER_SIZE] = "\0"; - char limitBuf[100] = "\0"; + char command[BUFFER_SIZE] = "\0"; + char limitBuf[100] = "\0"; - TAOS_RES * res; - TAOS_ROW row = NULL; + TAOS_RES * res; + TAOS_ROW row = NULL; - char* childTblName = *childTblNameOfSuperTbl; + char* childTblName = *childTblNameOfSuperTbl; - if (offset >= 0) { - snprintf(limitBuf, 100, " limit %"PRId64" offset %"PRIu64"", - limit, offset); - } - - //get all child table name use cmd: select tbname from superTblName; - snprintf(command, BUFFER_SIZE, "select tbname from %s.%s %s", - dbName, sTblName, limitBuf); - - res = taos_query(taos, command); - int32_t code = taos_errno(res); - if (code != 0) { - taos_free_result(res); - taos_close(taos); - errorPrint("%s() LN%d, failed to run command %s\n", - __func__, __LINE__, command); - exit(-1); - } - - int64_t childTblCount = (limit < 0)?10000:limit; - int64_t count = 0; - if (childTblName == NULL) { - childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN); - if (NULL == childTblName) { - taos_free_result(res); - taos_close(taos); - errorPrint("%s() LN%d, failed to allocate memory!\n", __func__, __LINE__); - exit(-1); - } - } - - char* pTblName = childTblName; - while((row = taos_fetch_row(res)) != NULL) { - int32_t* len = taos_fetch_lengths(res); - - if (0 == strlen((char *)row[0])) { - errorPrint("%s() LN%d, No.%"PRId64" table return empty name\n", - __func__, __LINE__, count); - exit(-1); + if (offset >= 0) { + snprintf(limitBuf, 100, " limit %"PRId64" offset %"PRIu64"", + limit, offset); } - tstrncpy(pTblName, (char *)row[0], len[0]+1); - //printf("==== sub table name: %s\n", pTblName); - count++; - if (count >= childTblCount - 1) { - char *tmp = realloc(childTblName, - (size_t)childTblCount*1.5*TSDB_TABLE_NAME_LEN+1); - if (tmp != NULL) { - childTblName = tmp; - childTblCount = (int)(childTblCount*1.5); - memset(childTblName + count*TSDB_TABLE_NAME_LEN, 0, - (size_t)((childTblCount-count)*TSDB_TABLE_NAME_LEN)); - } else { - // exit, if allocate more memory failed - errorPrint("%s() LN%d, realloc fail for save child table name of %s.%s\n", - __func__, __LINE__, dbName, sTblName); - tmfree(childTblName); + //get all child table name use cmd: select tbname from superTblName; + snprintf(command, BUFFER_SIZE, "select tbname from %s.%s %s", + dbName, sTblName, limitBuf); + + res = taos_query(taos, command); + int32_t code = taos_errno(res); + if (code != 0) { taos_free_result(res); taos_close(taos); + errorPrint("%s() LN%d, failed to run command %s\n", + __func__, __LINE__, command); exit(-1); - } } - pTblName = childTblName + count * TSDB_TABLE_NAME_LEN; - } - *childTblCountOfSuperTbl = count; - *childTblNameOfSuperTbl = childTblName; + int64_t childTblCount = (limit < 0)?10000:limit; + int64_t count = 0; + if (childTblName == NULL) { + childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN); + if (NULL == childTblName) { + taos_free_result(res); + taos_close(taos); + errorPrint("%s() LN%d, failed to allocate memory!\n", __func__, __LINE__); + exit(-1); + } + } - taos_free_result(res); - return 0; + char* pTblName = childTblName; + while((row = taos_fetch_row(res)) != NULL) { + int32_t* len = taos_fetch_lengths(res); + + if (0 == strlen((char *)row[0])) { + errorPrint("%s() LN%d, No.%"PRId64" table return empty name\n", + __func__, __LINE__, count); + exit(-1); + } + + tstrncpy(pTblName, (char *)row[0], len[0]+1); + //printf("==== sub table name: %s\n", pTblName); + count++; + if (count >= childTblCount - 1) { + char *tmp = realloc(childTblName, + (size_t)childTblCount*1.5*TSDB_TABLE_NAME_LEN+1); + if (tmp != NULL) { + childTblName = tmp; + childTblCount = (int)(childTblCount*1.5); + memset(childTblName + count*TSDB_TABLE_NAME_LEN, 0, + (size_t)((childTblCount-count)*TSDB_TABLE_NAME_LEN)); + } else { + // exit, if allocate more memory failed + errorPrint("%s() LN%d, realloc fail for save child table name of %s.%s\n", + __func__, __LINE__, dbName, sTblName); + tmfree(childTblName); + taos_free_result(res); + taos_close(taos); + exit(-1); + } + } + pTblName = childTblName + count * TSDB_TABLE_NAME_LEN; + } + + *childTblCountOfSuperTbl = count; + *childTblNameOfSuperTbl = childTblName; + + taos_free_result(res); + return 0; } static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName, @@ -2637,82 +2637,82 @@ static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName, static int getSuperTableFromServer(TAOS * taos, char* dbName, SSuperTable* superTbls) { - char command[BUFFER_SIZE] = "\0"; - TAOS_RES * res; - TAOS_ROW row = NULL; - int count = 0; + char command[BUFFER_SIZE] = "\0"; + TAOS_RES * res; + TAOS_ROW row = NULL; + int count = 0; - //get schema use cmd: describe superTblName; - snprintf(command, BUFFER_SIZE, "describe %s.%s", dbName, superTbls->sTblName); - res = taos_query(taos, command); - int32_t code = taos_errno(res); - if (code != 0) { - printf("failed to run command %s\n", command); + //get schema use cmd: describe superTblName; + snprintf(command, BUFFER_SIZE, "describe %s.%s", dbName, superTbls->sTblName); + res = taos_query(taos, command); + int32_t code = taos_errno(res); + if (code != 0) { + printf("failed to run command %s\n", command); + taos_free_result(res); + return -1; + } + + int tagIndex = 0; + int columnIndex = 0; + TAOS_FIELD *fields = taos_fetch_fields(res); + while((row = taos_fetch_row(res)) != NULL) { + if (0 == count) { + count++; + continue; + } + + if (strcmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "TAG") == 0) { + tstrncpy(superTbls->tags[tagIndex].field, + (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], + fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); + tstrncpy(superTbls->tags[tagIndex].dataType, + (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + min(15, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes)); + superTbls->tags[tagIndex].dataLen = + *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); + tstrncpy(superTbls->tags[tagIndex].note, + (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], + fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); + tagIndex++; + } else { + tstrncpy(superTbls->columns[columnIndex].field, + (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], + fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); + tstrncpy(superTbls->columns[columnIndex].dataType, + (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + min(15, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes)); + superTbls->columns[columnIndex].dataLen = + *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); + tstrncpy(superTbls->columns[columnIndex].note, + (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], + fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); + columnIndex++; + } + count++; + } + + superTbls->columnCount = columnIndex; + superTbls->tagCount = tagIndex; taos_free_result(res); - return -1; - } - int tagIndex = 0; - int columnIndex = 0; - TAOS_FIELD *fields = taos_fetch_fields(res); - while((row = taos_fetch_row(res)) != NULL) { - if (0 == count) { - count++; - continue; - } + calcRowLen(superTbls); - if (strcmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "TAG") == 0) { - tstrncpy(superTbls->tags[tagIndex].field, - (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], - fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); - tstrncpy(superTbls->tags[tagIndex].dataType, - (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - min(15, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes)); - superTbls->tags[tagIndex].dataLen = - *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); - tstrncpy(superTbls->tags[tagIndex].note, - (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], - fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); - tagIndex++; - } else { - tstrncpy(superTbls->columns[columnIndex].field, - (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], - fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); - tstrncpy(superTbls->columns[columnIndex].dataType, - (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - min(15, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes)); - superTbls->columns[columnIndex].dataLen = - *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); - tstrncpy(superTbls->columns[columnIndex].note, - (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], - fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); - columnIndex++; - } - count++; - } - - superTbls->columnCount = columnIndex; - superTbls->tagCount = tagIndex; - taos_free_result(res); - - calcRowLen(superTbls); - -/* - if (TBL_ALREADY_EXISTS == superTbls->childTblExists) { + /* + if (TBL_ALREADY_EXISTS == superTbls->childTblExists) { //get all child table name use cmd: select tbname from superTblName; int childTblCount = 10000; superTbls->childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN); if (superTbls->childTblName == NULL) { - errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__); - return -1; + errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__); + return -1; } getAllChildNameOfSuperTable(taos, dbName, - superTbls->sTblName, - &superTbls->childTblName, - &superTbls->childTblCount); - } - */ - return 0; + superTbls->sTblName, + &superTbls->childTblName, + &superTbls->childTblCount); + } + */ + return 0; } static int createSuperTable( @@ -2748,8 +2748,8 @@ static int createSuperTable( lenOfOneRow += superTbl->columns[colIndex].dataLen + 3; } else if (strcasecmp(dataType, "INT") == 0) { if ((g_args.demo_mode) && (colIndex == 1)) { - len += snprintf(cols + len, COL_BUFFER_LEN - len, - ", VOLTAGE INT"); + len += snprintf(cols + len, COL_BUFFER_LEN - len, + ", VOLTAGE INT"); } else { len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "INT"); } @@ -3221,7 +3221,7 @@ static void createChildTables() { continue; } verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, - g_Dbs.db[i].superTbls[j].colsOfCreateChildTable); + g_Dbs.db[i].superTbls[j].colsOfCreateChildTable); uint64_t startFrom = 0; g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount; @@ -3269,295 +3269,295 @@ static void createChildTables() { } /* - Read 10000 lines at most. If more than 10000 lines, continue to read after using -*/ + Read 10000 lines at most. If more than 10000 lines, continue to read after using + */ static int readTagFromCsvFileToMem(SSuperTable * superTblInfo) { - size_t n = 0; - ssize_t readLen = 0; - char * line = NULL; + size_t n = 0; + ssize_t readLen = 0; + char * line = NULL; - FILE *fp = fopen(superTblInfo->tagsFile, "r"); - if (fp == NULL) { - printf("Failed to open tags file: %s, reason:%s\n", - superTblInfo->tagsFile, strerror(errno)); - return -1; - } - - if (superTblInfo->tagDataBuf) { - free(superTblInfo->tagDataBuf); - superTblInfo->tagDataBuf = NULL; - } - - int tagCount = 10000; - int count = 0; - char* tagDataBuf = calloc(1, superTblInfo->lenOfTagOfOneRow * tagCount); - if (tagDataBuf == NULL) { - printf("Failed to calloc, reason:%s\n", strerror(errno)); - fclose(fp); - return -1; - } - - while((readLen = tgetline(&line, &n, fp)) != -1) { - if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { - line[--readLen] = 0; + FILE *fp = fopen(superTblInfo->tagsFile, "r"); + if (fp == NULL) { + printf("Failed to open tags file: %s, reason:%s\n", + superTblInfo->tagsFile, strerror(errno)); + return -1; } - if (readLen == 0) { - continue; + if (superTblInfo->tagDataBuf) { + free(superTblInfo->tagDataBuf); + superTblInfo->tagDataBuf = NULL; } - memcpy(tagDataBuf + count * superTblInfo->lenOfTagOfOneRow, line, readLen); - count++; - - if (count >= tagCount - 1) { - char *tmp = realloc(tagDataBuf, - (size_t)tagCount*1.5*superTblInfo->lenOfTagOfOneRow); - if (tmp != NULL) { - tagDataBuf = tmp; - tagCount = (int)(tagCount*1.5); - memset(tagDataBuf + count*superTblInfo->lenOfTagOfOneRow, - 0, (size_t)((tagCount-count)*superTblInfo->lenOfTagOfOneRow)); - } else { - // exit, if allocate more memory failed - printf("realloc fail for save tag val from %s\n", superTblInfo->tagsFile); - tmfree(tagDataBuf); - free(line); + int tagCount = 10000; + int count = 0; + char* tagDataBuf = calloc(1, superTblInfo->lenOfTagOfOneRow * tagCount); + if (tagDataBuf == NULL) { + printf("Failed to calloc, reason:%s\n", strerror(errno)); fclose(fp); return -1; - } } - } - superTblInfo->tagDataBuf = tagDataBuf; - superTblInfo->tagSampleCount = count; + while((readLen = tgetline(&line, &n, fp)) != -1) { + if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { + line[--readLen] = 0; + } - free(line); - fclose(fp); - return 0; + if (readLen == 0) { + continue; + } + + memcpy(tagDataBuf + count * superTblInfo->lenOfTagOfOneRow, line, readLen); + count++; + + if (count >= tagCount - 1) { + char *tmp = realloc(tagDataBuf, + (size_t)tagCount*1.5*superTblInfo->lenOfTagOfOneRow); + if (tmp != NULL) { + tagDataBuf = tmp; + tagCount = (int)(tagCount*1.5); + memset(tagDataBuf + count*superTblInfo->lenOfTagOfOneRow, + 0, (size_t)((tagCount-count)*superTblInfo->lenOfTagOfOneRow)); + } else { + // exit, if allocate more memory failed + printf("realloc fail for save tag val from %s\n", superTblInfo->tagsFile); + tmfree(tagDataBuf); + free(line); + fclose(fp); + return -1; + } + } + } + + superTblInfo->tagDataBuf = tagDataBuf; + superTblInfo->tagSampleCount = count; + + free(line); + fclose(fp); + return 0; } /* - Read 10000 lines at most. If more than 10000 lines, continue to read after using -*/ + Read 10000 lines at most. If more than 10000 lines, continue to read after using + */ static int readSampleFromCsvFileToMem( SSuperTable* superTblInfo) { - size_t n = 0; - ssize_t readLen = 0; - char * line = NULL; - int getRows = 0; + size_t n = 0; + ssize_t readLen = 0; + char * line = NULL; + int getRows = 0; - FILE* fp = fopen(superTblInfo->sampleFile, "r"); - if (fp == NULL) { - errorPrint( "Failed to open sample file: %s, reason:%s\n", - superTblInfo->sampleFile, strerror(errno)); - return -1; - } - - assert(superTblInfo->sampleDataBuf); - memset(superTblInfo->sampleDataBuf, 0, - MAX_SAMPLES_ONCE_FROM_FILE * superTblInfo->lenOfOneRow); - while(1) { - readLen = tgetline(&line, &n, fp); - if (-1 == readLen) { - if(0 != fseek(fp, 0, SEEK_SET)) { - errorPrint( "Failed to fseek file: %s, reason:%s\n", + FILE* fp = fopen(superTblInfo->sampleFile, "r"); + if (fp == NULL) { + errorPrint( "Failed to open sample file: %s, reason:%s\n", superTblInfo->sampleFile, strerror(errno)); - fclose(fp); return -1; - } - continue; } - if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { - line[--readLen] = 0; + assert(superTblInfo->sampleDataBuf); + memset(superTblInfo->sampleDataBuf, 0, + MAX_SAMPLES_ONCE_FROM_FILE * superTblInfo->lenOfOneRow); + while(1) { + readLen = tgetline(&line, &n, fp); + if (-1 == readLen) { + if(0 != fseek(fp, 0, SEEK_SET)) { + errorPrint( "Failed to fseek file: %s, reason:%s\n", + superTblInfo->sampleFile, strerror(errno)); + fclose(fp); + return -1; + } + continue; + } + + if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { + line[--readLen] = 0; + } + + if (readLen == 0) { + continue; + } + + if (readLen > superTblInfo->lenOfOneRow) { + printf("sample row len[%d] overflow define schema len[%"PRIu64"], so discard this row\n", + (int32_t)readLen, superTblInfo->lenOfOneRow); + continue; + } + + memcpy(superTblInfo->sampleDataBuf + getRows * superTblInfo->lenOfOneRow, + line, readLen); + getRows++; + + if (getRows == MAX_SAMPLES_ONCE_FROM_FILE) { + break; + } } - if (readLen == 0) { - continue; - } - - if (readLen > superTblInfo->lenOfOneRow) { - printf("sample row len[%d] overflow define schema len[%"PRIu64"], so discard this row\n", - (int32_t)readLen, superTblInfo->lenOfOneRow); - continue; - } - - memcpy(superTblInfo->sampleDataBuf + getRows * superTblInfo->lenOfOneRow, - line, readLen); - getRows++; - - if (getRows == MAX_SAMPLES_ONCE_FROM_FILE) { - break; - } - } - - fclose(fp); - tmfree(line); - return 0; + fclose(fp); + tmfree(line); + return 0; } static bool getColumnAndTagTypeFromInsertJsonFile( cJSON* stbInfo, SSuperTable* superTbls) { - bool ret = false; + bool ret = false; - // columns - cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns"); - if (columns && columns->type != cJSON_Array) { - printf("ERROR: failed to read json, columns not found\n"); - goto PARSE_OVER; - } else if (NULL == columns) { - superTbls->columnCount = 0; - superTbls->tagCount = 0; - return true; - } + // columns + cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns"); + if (columns && columns->type != cJSON_Array) { + printf("ERROR: failed to read json, columns not found\n"); + goto PARSE_OVER; + } else if (NULL == columns) { + superTbls->columnCount = 0; + superTbls->tagCount = 0; + return true; + } - int columnSize = cJSON_GetArraySize(columns); - if ((columnSize + 1/* ts */) > TSDB_MAX_COLUMNS) { - errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n", - __func__, __LINE__, TSDB_MAX_COLUMNS); - goto PARSE_OVER; - } + int columnSize = cJSON_GetArraySize(columns); + if ((columnSize + 1/* ts */) > TSDB_MAX_COLUMNS) { + errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n", + __func__, __LINE__, TSDB_MAX_COLUMNS); + goto PARSE_OVER; + } - int count = 1; - int index = 0; - StrColumn columnCase; + int count = 1; + int index = 0; + StrColumn columnCase; - //superTbls->columnCount = columnSize; - for (int k = 0; k < columnSize; ++k) { - cJSON* column = cJSON_GetArrayItem(columns, k); - if (column == NULL) continue; + //superTbls->columnCount = columnSize; + for (int k = 0; k < columnSize; ++k) { + cJSON* column = cJSON_GetArrayItem(columns, k); + if (column == NULL) continue; + + count = 1; + cJSON* countObj = cJSON_GetObjectItem(column, "count"); + if (countObj && countObj->type == cJSON_Number) { + count = countObj->valueint; + } else if (countObj && countObj->type != cJSON_Number) { + errorPrint("%s() LN%d, failed to read json, column count not found\n", + __func__, __LINE__); + goto PARSE_OVER; + } else { + count = 1; + } + + // column info + memset(&columnCase, 0, sizeof(StrColumn)); + cJSON *dataType = cJSON_GetObjectItem(column, "type"); + if (!dataType || dataType->type != cJSON_String + || dataType->valuestring == NULL) { + errorPrint("%s() LN%d: failed to read json, column type not found\n", + __func__, __LINE__); + goto PARSE_OVER; + } + //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE); + tstrncpy(columnCase.dataType, dataType->valuestring, strlen(dataType->valuestring) + 1); + + cJSON* dataLen = cJSON_GetObjectItem(column, "len"); + if (dataLen && dataLen->type == cJSON_Number) { + columnCase.dataLen = dataLen->valueint; + } else if (dataLen && dataLen->type != cJSON_Number) { + debugPrint("%s() LN%d: failed to read json, column len not found\n", + __func__, __LINE__); + goto PARSE_OVER; + } else { + columnCase.dataLen = 8; + } + + for (int n = 0; n < count; ++n) { + tstrncpy(superTbls->columns[index].dataType, + columnCase.dataType, strlen(columnCase.dataType) + 1); + superTbls->columns[index].dataLen = columnCase.dataLen; + index++; + } + } + + if ((index + 1 /* ts */) > MAX_NUM_COLUMNS) { + errorPrint("%s() LN%d, failed to read json, column size overflow, allowed max column size is %d\n", + __func__, __LINE__, MAX_NUM_COLUMNS); + goto PARSE_OVER; + } + + superTbls->columnCount = index; count = 1; - cJSON* countObj = cJSON_GetObjectItem(column, "count"); - if (countObj && countObj->type == cJSON_Number) { - count = countObj->valueint; - } else if (countObj && countObj->type != cJSON_Number) { - errorPrint("%s() LN%d, failed to read json, column count not found\n", - __func__, __LINE__); - goto PARSE_OVER; - } else { - count = 1; + index = 0; + // tags + cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags"); + if (!tags || tags->type != cJSON_Array) { + errorPrint("%s() LN%d, failed to read json, tags not found\n", + __func__, __LINE__); + goto PARSE_OVER; } - // column info - memset(&columnCase, 0, sizeof(StrColumn)); - cJSON *dataType = cJSON_GetObjectItem(column, "type"); - if (!dataType || dataType->type != cJSON_String - || dataType->valuestring == NULL) { - errorPrint("%s() LN%d: failed to read json, column type not found\n", - __func__, __LINE__); - goto PARSE_OVER; - } - //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE); - tstrncpy(columnCase.dataType, dataType->valuestring, strlen(dataType->valuestring) + 1); - - cJSON* dataLen = cJSON_GetObjectItem(column, "len"); - if (dataLen && dataLen->type == cJSON_Number) { - columnCase.dataLen = dataLen->valueint; - } else if (dataLen && dataLen->type != cJSON_Number) { - debugPrint("%s() LN%d: failed to read json, column len not found\n", - __func__, __LINE__); - goto PARSE_OVER; - } else { - columnCase.dataLen = 8; + int tagSize = cJSON_GetArraySize(tags); + if (tagSize > TSDB_MAX_TAGS) { + errorPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n", + __func__, __LINE__, TSDB_MAX_TAGS); + goto PARSE_OVER; } - for (int n = 0; n < count; ++n) { - tstrncpy(superTbls->columns[index].dataType, - columnCase.dataType, strlen(columnCase.dataType) + 1); - superTbls->columns[index].dataLen = columnCase.dataLen; - index++; - } - } + //superTbls->tagCount = tagSize; + for (int k = 0; k < tagSize; ++k) { + cJSON* tag = cJSON_GetArrayItem(tags, k); + if (tag == NULL) continue; - if ((index + 1 /* ts */) > MAX_NUM_COLUMNS) { - errorPrint("%s() LN%d, failed to read json, column size overflow, allowed max column size is %d\n", - __func__, __LINE__, MAX_NUM_COLUMNS); - goto PARSE_OVER; - } + count = 1; + cJSON* countObj = cJSON_GetObjectItem(tag, "count"); + if (countObj && countObj->type == cJSON_Number) { + count = countObj->valueint; + } else if (countObj && countObj->type != cJSON_Number) { + printf("ERROR: failed to read json, column count not found\n"); + goto PARSE_OVER; + } else { + count = 1; + } - superTbls->columnCount = index; + // column info + memset(&columnCase, 0, sizeof(StrColumn)); + cJSON *dataType = cJSON_GetObjectItem(tag, "type"); + if (!dataType || dataType->type != cJSON_String + || dataType->valuestring == NULL) { + errorPrint("%s() LN%d, failed to read json, tag type not found\n", + __func__, __LINE__); + goto PARSE_OVER; + } + tstrncpy(columnCase.dataType, dataType->valuestring, strlen(dataType->valuestring) + 1); - count = 1; - index = 0; - // tags - cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags"); - if (!tags || tags->type != cJSON_Array) { - errorPrint("%s() LN%d, failed to read json, tags not found\n", - __func__, __LINE__); - goto PARSE_OVER; - } + cJSON* dataLen = cJSON_GetObjectItem(tag, "len"); + if (dataLen && dataLen->type == cJSON_Number) { + columnCase.dataLen = dataLen->valueint; + } else if (dataLen && dataLen->type != cJSON_Number) { + errorPrint("%s() LN%d, failed to read json, column len not found\n", + __func__, __LINE__); + goto PARSE_OVER; + } else { + columnCase.dataLen = 0; + } - int tagSize = cJSON_GetArraySize(tags); - if (tagSize > TSDB_MAX_TAGS) { - errorPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n", - __func__, __LINE__, TSDB_MAX_TAGS); - goto PARSE_OVER; - } - - //superTbls->tagCount = tagSize; - for (int k = 0; k < tagSize; ++k) { - cJSON* tag = cJSON_GetArrayItem(tags, k); - if (tag == NULL) continue; - - count = 1; - cJSON* countObj = cJSON_GetObjectItem(tag, "count"); - if (countObj && countObj->type == cJSON_Number) { - count = countObj->valueint; - } else if (countObj && countObj->type != cJSON_Number) { - printf("ERROR: failed to read json, column count not found\n"); - goto PARSE_OVER; - } else { - count = 1; + for (int n = 0; n < count; ++n) { + tstrncpy(superTbls->tags[index].dataType, columnCase.dataType, + strlen(columnCase.dataType) + 1); + superTbls->tags[index].dataLen = columnCase.dataLen; + index++; + } } - // column info - memset(&columnCase, 0, sizeof(StrColumn)); - cJSON *dataType = cJSON_GetObjectItem(tag, "type"); - if (!dataType || dataType->type != cJSON_String - || dataType->valuestring == NULL) { - errorPrint("%s() LN%d, failed to read json, tag type not found\n", - __func__, __LINE__); - goto PARSE_OVER; - } - tstrncpy(columnCase.dataType, dataType->valuestring, strlen(dataType->valuestring) + 1); - - cJSON* dataLen = cJSON_GetObjectItem(tag, "len"); - if (dataLen && dataLen->type == cJSON_Number) { - columnCase.dataLen = dataLen->valueint; - } else if (dataLen && dataLen->type != cJSON_Number) { - errorPrint("%s() LN%d, failed to read json, column len not found\n", - __func__, __LINE__); - goto PARSE_OVER; - } else { - columnCase.dataLen = 0; + if (index > TSDB_MAX_TAGS) { + errorPrint("%s() LN%d, failed to read json, tags size overflow, allowed max tag count is %d\n", + __func__, __LINE__, TSDB_MAX_TAGS); + goto PARSE_OVER; } - for (int n = 0; n < count; ++n) { - tstrncpy(superTbls->tags[index].dataType, columnCase.dataType, - strlen(columnCase.dataType) + 1); - superTbls->tags[index].dataLen = columnCase.dataLen; - index++; + superTbls->tagCount = index; + + if ((superTbls->columnCount + superTbls->tagCount + 1 /* ts */) > TSDB_MAX_COLUMNS) { + errorPrint("%s() LN%d, columns + tags is more than allowed max columns count: %d\n", + __func__, __LINE__, TSDB_MAX_COLUMNS); + goto PARSE_OVER; } - } - - if (index > TSDB_MAX_TAGS) { - errorPrint("%s() LN%d, failed to read json, tags size overflow, allowed max tag count is %d\n", - __func__, __LINE__, TSDB_MAX_TAGS); - goto PARSE_OVER; - } - - superTbls->tagCount = index; - - if ((superTbls->columnCount + superTbls->tagCount + 1 /* ts */) > TSDB_MAX_COLUMNS) { - errorPrint("%s() LN%d, columns + tags is more than allowed max columns count: %d\n", - __func__, __LINE__, TSDB_MAX_COLUMNS); - goto PARSE_OVER; - } - ret = true; + ret = true; PARSE_OVER: - return ret; + return ret; } static bool getMetaFromInsertJsonFile(cJSON* root) { @@ -4866,7 +4866,7 @@ static void postFreeResource() { static int getRowDataFromSample( char* dataBuf, int64_t maxLen, int64_t timestamp, - SSuperTable* superTblInfo, int64_t* sampleUsePos) + SSuperTable* superTblInfo, int64_t* sampleUsePos) { if ((*sampleUsePos) == MAX_SAMPLES_ONCE_FROM_FILE) { /* int ret = readSampleFromCsvFileToMem(superTblInfo); @@ -5118,7 +5118,8 @@ static int32_t execInsert(threadInfo *pThreadInfo, uint32_t k) #if STMT_IFACE_ENABLED == 1 case STMT_IFACE: - debugPrint("%s() LN%d, stmt=%p", __func__, __LINE__, pThreadInfo->stmt); + debugPrint("%s() LN%d, stmt=%p", + __func__, __LINE__, pThreadInfo->stmt); if (0 != taos_stmt_execute(pThreadInfo->stmt)) { errorPrint("%s() LN%d, failied to execute insert statement\n", __func__, __LINE__); @@ -5458,39 +5459,39 @@ static int64_t generateInterlaceDataWithoutStb( int64_t startTime, uint64_t *pRemainderBufLen) { - assert(buffer); - char *pstr = buffer; + assert(buffer); + char *pstr = buffer; - int headLen = generateSQLHeadWithoutStb( - tableName, dbName, + int headLen = generateSQLHeadWithoutStb( + tableName, dbName, pstr, *pRemainderBufLen); - if (headLen <= 0) { - return 0; - } + if (headLen <= 0) { + return 0; + } - pstr += headLen; - *pRemainderBufLen -= headLen; + pstr += headLen; + *pRemainderBufLen -= headLen; - int64_t dataLen = 0; + int64_t dataLen = 0; - int32_t k = generateDataTailWithoutStb( + int32_t k = generateDataTailWithoutStb( batch, pstr, *pRemainderBufLen, insertRows, 0, startTime, &dataLen); - if (k == batch) { - pstr += dataLen; - *pRemainderBufLen -= dataLen; - } else { - debugPrint("%s() LN%d, generated data tail: %d, not equal batch per table: %u\n", - __func__, __LINE__, k, batch); - pstr -= headLen; - pstr[0] = '\0'; - k = 0; - } + if (k == batch) { + pstr += dataLen; + *pRemainderBufLen -= dataLen; + } else { + debugPrint("%s() LN%d, generated data tail: %d, not equal batch per table: %u\n", + __func__, __LINE__, k, batch); + pstr -= headLen; + pstr[0] = '\0'; + k = 0; + } - return k; + return k; } #if STMT_IFACE_ENABLED == 1 @@ -5771,6 +5772,8 @@ static int32_t prepareStbStmtBind( TAOS_BIND *bind; if (isColumn) { + int cursor = 0; + for (int i = 0; i < stbInfo->columnCount + 1; i ++) { bind = (TAOS_BIND *)((char *)bindArray + (sizeof(TAOS_BIND) * i)); @@ -5794,7 +5797,6 @@ static int32_t prepareStbStmtBind( ptr += bind->buffer_length; } else { - int cursor = 0; if (sourceRand) { if ( -1 == prepareStmtBindArrayByType( @@ -5851,6 +5853,7 @@ static int32_t prepareStbStmtBind( } + free(bindBuffer); return 0; } @@ -6043,27 +6046,27 @@ static int32_t generateProgressiveDataWithoutStb( uint64_t recordFrom, int64_t startTime, /*int64_t *pSamplePos, */ int64_t *pRemainderBufLen) { - assert(buffer != NULL); - char *pstr = buffer; + assert(buffer != NULL); + char *pstr = buffer; - memset(buffer, 0, *pRemainderBufLen); + memset(buffer, 0, *pRemainderBufLen); - int64_t headLen = generateSQLHeadWithoutStb( - tableName, pThreadInfo->db_name, - buffer, *pRemainderBufLen); + int64_t headLen = generateSQLHeadWithoutStb( + tableName, pThreadInfo->db_name, + buffer, *pRemainderBufLen); - if (headLen <= 0) { - return 0; - } - pstr += headLen; - *pRemainderBufLen -= headLen; + if (headLen <= 0) { + return 0; + } + pstr += headLen; + *pRemainderBufLen -= headLen; - int64_t dataLen; + int64_t dataLen; - return generateDataTailWithoutStb( - g_args.num_of_RPR, pstr, *pRemainderBufLen, insertRows, recordFrom, - startTime, - /*pSamplePos, */&dataLen); + return generateDataTailWithoutStb( + g_args.num_of_RPR, pstr, *pRemainderBufLen, insertRows, recordFrom, + startTime, + /*pSamplePos, */&dataLen); } static void printStatPerThread(threadInfo *pThreadInfo) @@ -6552,110 +6555,110 @@ static void* syncWrite(void *sarg) { } static void callBack(void *param, TAOS_RES *res, int code) { - threadInfo* pThreadInfo = (threadInfo*)param; - SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + threadInfo* pThreadInfo = (threadInfo*)param; + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - int insert_interval = - superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; - if (insert_interval) { - pThreadInfo->et = taosGetTimestampMs(); - if ((pThreadInfo->et - pThreadInfo->st) < insert_interval) { - taosMsleep(insert_interval - (pThreadInfo->et - pThreadInfo->st)); // ms + int insert_interval = + superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; + if (insert_interval) { + pThreadInfo->et = taosGetTimestampMs(); + if ((pThreadInfo->et - pThreadInfo->st) < insert_interval) { + taosMsleep(insert_interval - (pThreadInfo->et - pThreadInfo->st)); // ms + } } - } - char *buffer = calloc(1, pThreadInfo->superTblInfo->maxSqlLen); - char data[MAX_DATA_SIZE]; - char *pstr = buffer; - pstr += sprintf(pstr, "insert into %s.%s%"PRId64" values", - pThreadInfo->db_name, pThreadInfo->tb_prefix, - pThreadInfo->start_table_from); -// if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) { - if (pThreadInfo->counter >= g_args.num_of_RPR) { - pThreadInfo->start_table_from++; - pThreadInfo->counter = 0; - } - if (pThreadInfo->start_table_from > pThreadInfo->end_table_to) { - tsem_post(&pThreadInfo->lock_sem); + char *buffer = calloc(1, pThreadInfo->superTblInfo->maxSqlLen); + char data[MAX_DATA_SIZE]; + char *pstr = buffer; + pstr += sprintf(pstr, "insert into %s.%s%"PRId64" values", + pThreadInfo->db_name, pThreadInfo->tb_prefix, + pThreadInfo->start_table_from); + // if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) { + if (pThreadInfo->counter >= g_args.num_of_RPR) { + pThreadInfo->start_table_from++; + pThreadInfo->counter = 0; + } + if (pThreadInfo->start_table_from > pThreadInfo->end_table_to) { + tsem_post(&pThreadInfo->lock_sem); + free(buffer); + taos_free_result(res); + return; + } + + for (int i = 0; i < g_args.num_of_RPR; i++) { + int rand_num = taosRandom() % 100; + if (0 != pThreadInfo->superTblInfo->disorderRatio + && rand_num < pThreadInfo->superTblInfo->disorderRatio) { + int64_t d = pThreadInfo->lastTs + - (taosRandom() % pThreadInfo->superTblInfo->disorderRange + 1); + generateStbRowData(pThreadInfo->superTblInfo, data, d); + } else { + generateStbRowData(pThreadInfo->superTblInfo, + data, pThreadInfo->lastTs += 1000); + } + pstr += sprintf(pstr, "%s", data); + pThreadInfo->counter++; + + if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) { + break; + } + } + + if (insert_interval) { + pThreadInfo->st = taosGetTimestampMs(); + } + taos_query_a(pThreadInfo->taos, buffer, callBack, pThreadInfo); free(buffer); + taos_free_result(res); - return; - } - - for (int i = 0; i < g_args.num_of_RPR; i++) { - int rand_num = taosRandom() % 100; - if (0 != pThreadInfo->superTblInfo->disorderRatio - && rand_num < pThreadInfo->superTblInfo->disorderRatio) { - int64_t d = pThreadInfo->lastTs - - (taosRandom() % pThreadInfo->superTblInfo->disorderRange + 1); - generateStbRowData(pThreadInfo->superTblInfo, data, d); - } else { - generateStbRowData(pThreadInfo->superTblInfo, - data, pThreadInfo->lastTs += 1000); - } - pstr += sprintf(pstr, "%s", data); - pThreadInfo->counter++; - - if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) { - break; - } - } - - if (insert_interval) { - pThreadInfo->st = taosGetTimestampMs(); - } - taos_query_a(pThreadInfo->taos, buffer, callBack, pThreadInfo); - free(buffer); - - taos_free_result(res); } static void *asyncWrite(void *sarg) { - threadInfo *pThreadInfo = (threadInfo *)sarg; - SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + threadInfo *pThreadInfo = (threadInfo *)sarg; + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - setThreadName("asyncWrite"); + setThreadName("asyncWrite"); - pThreadInfo->st = 0; - pThreadInfo->et = 0; - pThreadInfo->lastTs = pThreadInfo->start_time; + pThreadInfo->st = 0; + pThreadInfo->et = 0; + pThreadInfo->lastTs = pThreadInfo->start_time; - int insert_interval = - superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; - if (insert_interval) { - pThreadInfo->st = taosGetTimestampMs(); - } - taos_query_a(pThreadInfo->taos, "show databases", callBack, pThreadInfo); + int insert_interval = + superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; + if (insert_interval) { + pThreadInfo->st = taosGetTimestampMs(); + } + taos_query_a(pThreadInfo->taos, "show databases", callBack, pThreadInfo); - tsem_wait(&(pThreadInfo->lock_sem)); + tsem_wait(&(pThreadInfo->lock_sem)); - return NULL; + return NULL; } static int convertHostToServAddr(char *host, uint16_t port, struct sockaddr_in *serv_addr) { - uint16_t rest_port = port + TSDB_PORT_HTTP; - struct hostent *server = gethostbyname(host); - if ((server == NULL) || (server->h_addr == NULL)) { - errorPrint("%s", "ERROR, no such host"); - return -1; - } + uint16_t rest_port = port + TSDB_PORT_HTTP; + struct hostent *server = gethostbyname(host); + if ((server == NULL) || (server->h_addr == NULL)) { + errorPrint("%s", "ERROR, no such host"); + return -1; + } - debugPrint("h_name: %s\nh_addr=%p\nh_addretype: %s\nh_length: %d\n", + debugPrint("h_name: %s\nh_addr=%p\nh_addretype: %s\nh_length: %d\n", server->h_name, server->h_addr, (server->h_addrtype == AF_INET)?"ipv4":"ipv6", server->h_length); - memset(serv_addr, 0, sizeof(struct sockaddr_in)); - serv_addr->sin_family = AF_INET; - serv_addr->sin_port = htons(rest_port); + memset(serv_addr, 0, sizeof(struct sockaddr_in)); + serv_addr->sin_family = AF_INET; + serv_addr->sin_port = htons(rest_port); #ifdef WINDOWS - serv_addr->sin_addr.s_addr = inet_addr(host); + serv_addr->sin_addr.s_addr = inet_addr(host); #else - memcpy(&(serv_addr->sin_addr.s_addr), server->h_addr, server->h_length); + memcpy(&(serv_addr->sin_addr.s_addr), server->h_addr, server->h_length); #endif - return 0; + return 0; } static void startMultiThreadInsertData(int threads, char* db_name, @@ -6724,14 +6727,17 @@ static void startMultiThreadInsertData(int threads, char* db_name, int64_t limit; uint64_t offset; - if ((NULL != g_args.sqlFile) && (superTblInfo->childTblExists == TBL_NO_EXISTS) && - ((superTblInfo->childTblOffset != 0) || (superTblInfo->childTblLimit >= 0))) { + if ((NULL != g_args.sqlFile) + && (superTblInfo->childTblExists == TBL_NO_EXISTS) + && ((superTblInfo->childTblOffset != 0) + || (superTblInfo->childTblLimit >= 0))) { printf("WARNING: offset and limit will not be used since the child tables not exists!\n"); } if (superTblInfo->childTblExists == TBL_ALREADY_EXISTS) { if ((superTblInfo->childTblLimit < 0) - || ((superTblInfo->childTblOffset + superTblInfo->childTblLimit) + || ((superTblInfo->childTblOffset + + superTblInfo->childTblLimit) > (superTblInfo->childTblCount))) { superTblInfo->childTblLimit = superTblInfo->childTblCount - superTblInfo->childTblOffset; @@ -6837,7 +6843,8 @@ static void startMultiThreadInsertData(int threads, char* db_name, #if STMT_IFACE_ENABLED == 1 if ((g_args.iface == STMT_IFACE) - || ((superTblInfo) && (superTblInfo->iface == STMT_IFACE))) { + || ((superTblInfo) + && (superTblInfo->iface == STMT_IFACE))) { int columnCount; if (superTblInfo) { @@ -6865,7 +6872,8 @@ static void startMultiThreadInsertData(int threads, char* db_name, == superTblInfo->autoCreateTable)) { pstr += sprintf(pstr, "INSERT INTO ? USING %s TAGS(?", superTblInfo->sTblName); - for (int tag = 0; tag < (superTblInfo->tagCount - 1); tag ++ ) { + for (int tag = 0; tag < (superTblInfo->tagCount - 1); + tag ++ ) { pstr += sprintf(pstr, ",?"); } pstr += sprintf(pstr, ") VALUES(?"); @@ -7014,157 +7022,157 @@ static void startMultiThreadInsertData(int threads, char* db_name, static void *readTable(void *sarg) { #if 1 - threadInfo *pThreadInfo = (threadInfo *)sarg; - TAOS *taos = pThreadInfo->taos; - setThreadName("readTable"); - char command[BUFFER_SIZE] = "\0"; - uint64_t sTime = pThreadInfo->start_time; - char *tb_prefix = pThreadInfo->tb_prefix; - FILE *fp = fopen(pThreadInfo->filePath, "a"); - if (NULL == fp) { - errorPrint( "fopen %s fail, reason:%s.\n", pThreadInfo->filePath, strerror(errno)); - return NULL; - } - - int64_t num_of_DPT; -/* if (pThreadInfo->superTblInfo) { - num_of_DPT = pThreadInfo->superTblInfo->insertRows; // nrecords_per_table; - } else { - */ - num_of_DPT = g_args.num_of_DPT; -// } - - int64_t num_of_tables = pThreadInfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; - int64_t totalData = num_of_DPT * num_of_tables; - bool do_aggreFunc = g_Dbs.do_aggreFunc; - - int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; - if (!do_aggreFunc) { - printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n"); - } - printf("%"PRId64" records:\n", totalData); - fprintf(fp, "| QFunctions | QRecords | QSpeed(R/s) | QLatency(ms) |\n"); - - for (int j = 0; j < n; j++) { - double totalT = 0; - uint64_t count = 0; - for (int64_t i = 0; i < num_of_tables; i++) { - sprintf(command, "select %s from %s%"PRId64" where ts>= %" PRIu64, - aggreFunc[j], tb_prefix, i, sTime); - - double t = taosGetTimestampMs(); - TAOS_RES *pSql = taos_query(taos, command); - int32_t code = taos_errno(pSql); - - if (code != 0) { - errorPrint( "Failed to query:%s\n", taos_errstr(pSql)); - taos_free_result(pSql); - taos_close(taos); - fclose(fp); + threadInfo *pThreadInfo = (threadInfo *)sarg; + TAOS *taos = pThreadInfo->taos; + setThreadName("readTable"); + char command[BUFFER_SIZE] = "\0"; + uint64_t sTime = pThreadInfo->start_time; + char *tb_prefix = pThreadInfo->tb_prefix; + FILE *fp = fopen(pThreadInfo->filePath, "a"); + if (NULL == fp) { + errorPrint( "fopen %s fail, reason:%s.\n", pThreadInfo->filePath, strerror(errno)); return NULL; - } - - while(taos_fetch_row(pSql) != NULL) { - count++; - } - - t = taosGetTimestampMs() - t; - totalT += t; - - taos_free_result(pSql); } - fprintf(fp, "|%10s | %"PRId64" | %12.2f | %10.2f |\n", - aggreFunc[j][0] == '*' ? " * " : aggreFunc[j], totalData, - (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000); - printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT * 1000); - } - fprintf(fp, "\n"); - fclose(fp); + int64_t num_of_DPT; + /* if (pThreadInfo->superTblInfo) { + num_of_DPT = pThreadInfo->superTblInfo->insertRows; // nrecords_per_table; + } else { + */ + num_of_DPT = g_args.num_of_DPT; + // } + + int64_t num_of_tables = pThreadInfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; + int64_t totalData = num_of_DPT * num_of_tables; + bool do_aggreFunc = g_Dbs.do_aggreFunc; + + int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; + if (!do_aggreFunc) { + printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n"); + } + printf("%"PRId64" records:\n", totalData); + fprintf(fp, "| QFunctions | QRecords | QSpeed(R/s) | QLatency(ms) |\n"); + + for (int j = 0; j < n; j++) { + double totalT = 0; + uint64_t count = 0; + for (int64_t i = 0; i < num_of_tables; i++) { + sprintf(command, "select %s from %s%"PRId64" where ts>= %" PRIu64, + aggreFunc[j], tb_prefix, i, sTime); + + double t = taosGetTimestampMs(); + TAOS_RES *pSql = taos_query(taos, command); + int32_t code = taos_errno(pSql); + + if (code != 0) { + errorPrint( "Failed to query:%s\n", taos_errstr(pSql)); + taos_free_result(pSql); + taos_close(taos); + fclose(fp); + return NULL; + } + + while(taos_fetch_row(pSql) != NULL) { + count++; + } + + t = taosGetTimestampMs() - t; + totalT += t; + + taos_free_result(pSql); + } + + fprintf(fp, "|%10s | %"PRId64" | %12.2f | %10.2f |\n", + aggreFunc[j][0] == '*' ? " * " : aggreFunc[j], totalData, + (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000); + printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT * 1000); + } + fprintf(fp, "\n"); + fclose(fp); #endif - return NULL; + return NULL; } static void *readMetric(void *sarg) { #if 1 - threadInfo *pThreadInfo = (threadInfo *)sarg; - TAOS *taos = pThreadInfo->taos; - setThreadName("readMetric"); - char command[BUFFER_SIZE] = "\0"; - FILE *fp = fopen(pThreadInfo->filePath, "a"); - if (NULL == fp) { - printf("fopen %s fail, reason:%s.\n", pThreadInfo->filePath, strerror(errno)); - return NULL; - } - - int64_t num_of_DPT = pThreadInfo->superTblInfo->insertRows; - int64_t num_of_tables = pThreadInfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; - int64_t totalData = num_of_DPT * num_of_tables; - bool do_aggreFunc = g_Dbs.do_aggreFunc; - - int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; - if (!do_aggreFunc) { - printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n"); - } - printf("%"PRId64" records:\n", totalData); - fprintf(fp, "Querying On %"PRId64" records:\n", totalData); - - for (int j = 0; j < n; j++) { - char condition[COND_BUF_LEN] = "\0"; - char tempS[64] = "\0"; - - int64_t m = 10 < num_of_tables ? 10 : num_of_tables; - - for (int64_t i = 1; i <= m; i++) { - if (i == 1) { - sprintf(tempS, "t1 = %"PRId64"", i); - } else { - sprintf(tempS, " or t1 = %"PRId64" ", i); - } - strncat(condition, tempS, COND_BUF_LEN - 1); - - sprintf(command, "select %s from meters where %s", aggreFunc[j], condition); - - printf("Where condition: %s\n", condition); - fprintf(fp, "%s\n", command); - - double t = taosGetTimestampMs(); - - TAOS_RES *pSql = taos_query(taos, command); - int32_t code = taos_errno(pSql); - - if (code != 0) { - errorPrint( "Failed to query:%s\n", taos_errstr(pSql)); - taos_free_result(pSql); - taos_close(taos); - fclose(fp); + threadInfo *pThreadInfo = (threadInfo *)sarg; + TAOS *taos = pThreadInfo->taos; + setThreadName("readMetric"); + char command[BUFFER_SIZE] = "\0"; + FILE *fp = fopen(pThreadInfo->filePath, "a"); + if (NULL == fp) { + printf("fopen %s fail, reason:%s.\n", pThreadInfo->filePath, strerror(errno)); return NULL; - } - int count = 0; - while(taos_fetch_row(pSql) != NULL) { - count++; - } - t = taosGetTimestampMs() - t; - - fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n", - num_of_tables * num_of_DPT / (t * 1000.0), t); - printf("select %10s took %.6f second(s)\n\n", aggreFunc[j], t * 1000.0); - - taos_free_result(pSql); } - fprintf(fp, "\n"); - } - fclose(fp); + + int64_t num_of_DPT = pThreadInfo->superTblInfo->insertRows; + int64_t num_of_tables = pThreadInfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; + int64_t totalData = num_of_DPT * num_of_tables; + bool do_aggreFunc = g_Dbs.do_aggreFunc; + + int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; + if (!do_aggreFunc) { + printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n"); + } + printf("%"PRId64" records:\n", totalData); + fprintf(fp, "Querying On %"PRId64" records:\n", totalData); + + for (int j = 0; j < n; j++) { + char condition[COND_BUF_LEN] = "\0"; + char tempS[64] = "\0"; + + int64_t m = 10 < num_of_tables ? 10 : num_of_tables; + + for (int64_t i = 1; i <= m; i++) { + if (i == 1) { + sprintf(tempS, "t1 = %"PRId64"", i); + } else { + sprintf(tempS, " or t1 = %"PRId64" ", i); + } + strncat(condition, tempS, COND_BUF_LEN - 1); + + sprintf(command, "select %s from meters where %s", aggreFunc[j], condition); + + printf("Where condition: %s\n", condition); + fprintf(fp, "%s\n", command); + + double t = taosGetTimestampMs(); + + TAOS_RES *pSql = taos_query(taos, command); + int32_t code = taos_errno(pSql); + + if (code != 0) { + errorPrint( "Failed to query:%s\n", taos_errstr(pSql)); + taos_free_result(pSql); + taos_close(taos); + fclose(fp); + return NULL; + } + int count = 0; + while(taos_fetch_row(pSql) != NULL) { + count++; + } + t = taosGetTimestampMs() - t; + + fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n", + num_of_tables * num_of_DPT / (t * 1000.0), t); + printf("select %10s took %.6f second(s)\n\n", aggreFunc[j], t * 1000.0); + + taos_free_result(pSql); + } + fprintf(fp, "\n"); + } + fclose(fp); #endif - return NULL; + return NULL; } static void prompt() { - if (!g_args.answer_yes) { - printf(" Press enter key to continue or Ctrl-C to stop\n\n"); - (void)getchar(); - } + if (!g_args.answer_yes) { + printf(" Press enter key to continue or Ctrl-C to stop\n\n"); + (void)getchar(); + } } static int insertTestProcess() { @@ -7264,369 +7272,369 @@ static int insertTestProcess() { } static void *specifiedTableQuery(void *sarg) { - threadInfo *pThreadInfo = (threadInfo *)sarg; + threadInfo *pThreadInfo = (threadInfo *)sarg; - setThreadName("specTableQuery"); + setThreadName("specTableQuery"); - if (pThreadInfo->taos == NULL) { - TAOS * taos = NULL; - taos = taos_connect(g_queryInfo.host, - g_queryInfo.user, - g_queryInfo.password, - NULL, - g_queryInfo.port); - if (taos == NULL) { - errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", - pThreadInfo->threadID, taos_errstr(NULL)); - return NULL; - } else { - pThreadInfo->taos = taos; + if (pThreadInfo->taos == NULL) { + TAOS * taos = NULL; + taos = taos_connect(g_queryInfo.host, + g_queryInfo.user, + g_queryInfo.password, + NULL, + g_queryInfo.port); + if (taos == NULL) { + errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", + pThreadInfo->threadID, taos_errstr(NULL)); + return NULL; + } else { + pThreadInfo->taos = taos; + } } - } - char sqlStr[TSDB_DB_NAME_LEN + 5]; - sprintf(sqlStr, "use %s", g_queryInfo.dbName); - if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) { - taos_close(pThreadInfo->taos); - errorPrint( "use database %s failed!\n\n", + char sqlStr[TSDB_DB_NAME_LEN + 5]; + sprintf(sqlStr, "use %s", g_queryInfo.dbName); + if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) { + taos_close(pThreadInfo->taos); + errorPrint( "use database %s failed!\n\n", g_queryInfo.dbName); - return NULL; - } + return NULL; + } - uint64_t st = 0; - uint64_t et = 0; + uint64_t st = 0; + uint64_t et = 0; - uint64_t queryTimes = g_queryInfo.specifiedQueryInfo.queryTimes; + uint64_t queryTimes = g_queryInfo.specifiedQueryInfo.queryTimes; - uint64_t totalQueried = 0; - uint64_t lastPrintTime = taosGetTimestampMs(); - uint64_t startTs = taosGetTimestampMs(); + uint64_t totalQueried = 0; + uint64_t lastPrintTime = taosGetTimestampMs(); + uint64_t startTs = taosGetTimestampMs(); - if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != '\0') { - sprintf(pThreadInfo->filePath, "%s-%d", + if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != '\0') { + sprintf(pThreadInfo->filePath, "%s-%d", g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); - } - - while(queryTimes --) { - if (g_queryInfo.specifiedQueryInfo.queryInterval && (et - st) < - (int64_t)g_queryInfo.specifiedQueryInfo.queryInterval) { - taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval - (et - st)); // ms } - st = taosGetTimestampMs(); + while(queryTimes --) { + if (g_queryInfo.specifiedQueryInfo.queryInterval && (et - st) < + (int64_t)g_queryInfo.specifiedQueryInfo.queryInterval) { + taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval - (et - st)); // ms + } - selectAndGetResult(pThreadInfo, - g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq]); + st = taosGetTimestampMs(); - et = taosGetTimestampMs(); - printf("=thread[%"PRId64"] use %s complete one sql, Spent %10.3f s\n", - taosGetSelfPthreadId(), g_queryInfo.queryMode, (et - st)/1000.0); + selectAndGetResult(pThreadInfo, + g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq]); - totalQueried ++; - g_queryInfo.specifiedQueryInfo.totalQueried ++; + et = taosGetTimestampMs(); + printf("=thread[%"PRId64"] use %s complete one sql, Spent %10.3f s\n", + taosGetSelfPthreadId(), g_queryInfo.queryMode, (et - st)/1000.0); - uint64_t currentPrintTime = taosGetTimestampMs(); - uint64_t endTs = taosGetTimestampMs(); - if (currentPrintTime - lastPrintTime > 30*1000) { - debugPrint("%s() LN%d, endTs=%"PRIu64"ms, startTs=%"PRIu64"ms\n", - __func__, __LINE__, endTs, startTs); - printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.6f\n", + totalQueried ++; + g_queryInfo.specifiedQueryInfo.totalQueried ++; + + uint64_t currentPrintTime = taosGetTimestampMs(); + uint64_t endTs = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30*1000) { + debugPrint("%s() LN%d, endTs=%"PRIu64"ms, startTs=%"PRIu64"ms\n", + __func__, __LINE__, endTs, startTs); + printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.6f\n", pThreadInfo->threadID, totalQueried, (double)(totalQueried/((endTs-startTs)/1000.0))); - lastPrintTime = currentPrintTime; + lastPrintTime = currentPrintTime; + } } - } - return NULL; + return NULL; } static void replaceChildTblName(char* inSql, char* outSql, int tblIndex) { - char sourceString[32] = "xxxx"; - char subTblName[MAX_TB_NAME_SIZE*3]; - sprintf(subTblName, "%s.%s", - g_queryInfo.dbName, - g_queryInfo.superQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN); + char sourceString[32] = "xxxx"; + char subTblName[MAX_TB_NAME_SIZE*3]; + sprintf(subTblName, "%s.%s", + g_queryInfo.dbName, + g_queryInfo.superQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN); - //printf("inSql: %s\n", inSql); + //printf("inSql: %s\n", inSql); - char* pos = strstr(inSql, sourceString); - if (0 == pos) { - return; - } + char* pos = strstr(inSql, sourceString); + if (0 == pos) { + return; + } - tstrncpy(outSql, inSql, pos - inSql + 1); - //printf("1: %s\n", outSql); - strncat(outSql, subTblName, MAX_QUERY_SQL_LENGTH - 1); - //printf("2: %s\n", outSql); - strncat(outSql, pos+strlen(sourceString), MAX_QUERY_SQL_LENGTH - 1); - //printf("3: %s\n", outSql); + tstrncpy(outSql, inSql, pos - inSql + 1); + //printf("1: %s\n", outSql); + strncat(outSql, subTblName, MAX_QUERY_SQL_LENGTH - 1); + //printf("2: %s\n", outSql); + strncat(outSql, pos+strlen(sourceString), MAX_QUERY_SQL_LENGTH - 1); + //printf("3: %s\n", outSql); } static void *superTableQuery(void *sarg) { - char sqlstr[MAX_QUERY_SQL_LENGTH]; - threadInfo *pThreadInfo = (threadInfo *)sarg; + char sqlstr[MAX_QUERY_SQL_LENGTH]; + threadInfo *pThreadInfo = (threadInfo *)sarg; - setThreadName("superTableQuery"); + setThreadName("superTableQuery"); - if (pThreadInfo->taos == NULL) { - TAOS * taos = NULL; - taos = taos_connect(g_queryInfo.host, - g_queryInfo.user, - g_queryInfo.password, - NULL, - g_queryInfo.port); - if (taos == NULL) { - errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", - pThreadInfo->threadID, taos_errstr(NULL)); - return NULL; - } else { - pThreadInfo->taos = taos; - } - } - - uint64_t st = 0; - uint64_t et = (int64_t)g_queryInfo.superQueryInfo.queryInterval; - - uint64_t queryTimes = g_queryInfo.superQueryInfo.queryTimes; - uint64_t totalQueried = 0; - uint64_t startTs = taosGetTimestampMs(); - - uint64_t lastPrintTime = taosGetTimestampMs(); - while(queryTimes --) { - if (g_queryInfo.superQueryInfo.queryInterval - && (et - st) < (int64_t)g_queryInfo.superQueryInfo.queryInterval) { - taosMsleep(g_queryInfo.superQueryInfo.queryInterval - (et - st)); // ms - //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, pThreadInfo->start_table_from, pThreadInfo->end_table_to); - } - - st = taosGetTimestampMs(); - for (int i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { - for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { - memset(sqlstr,0,sizeof(sqlstr)); - replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i); - if (g_queryInfo.superQueryInfo.result[j][0] != '\0') { - sprintf(pThreadInfo->filePath, "%s-%d", - g_queryInfo.superQueryInfo.result[j], - pThreadInfo->threadID); + if (pThreadInfo->taos == NULL) { + TAOS * taos = NULL; + taos = taos_connect(g_queryInfo.host, + g_queryInfo.user, + g_queryInfo.password, + NULL, + g_queryInfo.port); + if (taos == NULL) { + errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", + pThreadInfo->threadID, taos_errstr(NULL)); + return NULL; + } else { + pThreadInfo->taos = taos; } - selectAndGetResult(pThreadInfo, sqlstr); - - totalQueried++; - g_queryInfo.superQueryInfo.totalQueried ++; - - int64_t currentPrintTime = taosGetTimestampMs(); - int64_t endTs = taosGetTimestampMs(); - if (currentPrintTime - lastPrintTime > 30*1000) { - printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.3f\n", - pThreadInfo->threadID, - totalQueried, - (double)(totalQueried/((endTs-startTs)/1000.0))); - lastPrintTime = currentPrintTime; - } - } } - et = taosGetTimestampMs(); - printf("####thread[%"PRId64"] complete all sqls to allocate all sub-tables[%"PRIu64" - %"PRIu64"] once queries duration:%.4fs\n\n", - taosGetSelfPthreadId(), - pThreadInfo->start_table_from, - pThreadInfo->end_table_to, - (double)(et - st)/1000.0); - } - return NULL; + uint64_t st = 0; + uint64_t et = (int64_t)g_queryInfo.superQueryInfo.queryInterval; + + uint64_t queryTimes = g_queryInfo.superQueryInfo.queryTimes; + uint64_t totalQueried = 0; + uint64_t startTs = taosGetTimestampMs(); + + uint64_t lastPrintTime = taosGetTimestampMs(); + while(queryTimes --) { + if (g_queryInfo.superQueryInfo.queryInterval + && (et - st) < (int64_t)g_queryInfo.superQueryInfo.queryInterval) { + taosMsleep(g_queryInfo.superQueryInfo.queryInterval - (et - st)); // ms + //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, pThreadInfo->start_table_from, pThreadInfo->end_table_to); + } + + st = taosGetTimestampMs(); + for (int i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { + for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { + memset(sqlstr,0,sizeof(sqlstr)); + replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i); + if (g_queryInfo.superQueryInfo.result[j][0] != '\0') { + sprintf(pThreadInfo->filePath, "%s-%d", + g_queryInfo.superQueryInfo.result[j], + pThreadInfo->threadID); + } + selectAndGetResult(pThreadInfo, sqlstr); + + totalQueried++; + g_queryInfo.superQueryInfo.totalQueried ++; + + int64_t currentPrintTime = taosGetTimestampMs(); + int64_t endTs = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30*1000) { + printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.3f\n", + pThreadInfo->threadID, + totalQueried, + (double)(totalQueried/((endTs-startTs)/1000.0))); + lastPrintTime = currentPrintTime; + } + } + } + et = taosGetTimestampMs(); + printf("####thread[%"PRId64"] complete all sqls to allocate all sub-tables[%"PRIu64" - %"PRIu64"] once queries duration:%.4fs\n\n", + taosGetSelfPthreadId(), + pThreadInfo->start_table_from, + pThreadInfo->end_table_to, + (double)(et - st)/1000.0); + } + + return NULL; } static int queryTestProcess() { - setupForAnsiEscape(); - printfQueryMeta(); - resetAfterAnsiEscape(); + setupForAnsiEscape(); + printfQueryMeta(); + resetAfterAnsiEscape(); - TAOS * taos = NULL; - taos = taos_connect(g_queryInfo.host, - g_queryInfo.user, - g_queryInfo.password, - NULL, - g_queryInfo.port); - if (taos == NULL) { - errorPrint( "Failed to connect to TDengine, reason:%s\n", - taos_errstr(NULL)); - exit(-1); - } - - if (0 != g_queryInfo.superQueryInfo.sqlCount) { - getAllChildNameOfSuperTable(taos, - g_queryInfo.dbName, - g_queryInfo.superQueryInfo.sTblName, - &g_queryInfo.superQueryInfo.childTblName, - &g_queryInfo.superQueryInfo.childTblCount); - } - - prompt(); - - if (g_args.debug_print || g_args.verbose_print) { - printfQuerySystemInfo(taos); - } - - if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) { - if (convertHostToServAddr( - g_queryInfo.host, g_queryInfo.port, &g_queryInfo.serv_addr) != 0) - exit(-1); - } - - pthread_t *pids = NULL; - threadInfo *infos = NULL; - //==== create sub threads for query from specify table - int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent; - uint64_t nSqlCount = g_queryInfo.specifiedQueryInfo.sqlCount; - - uint64_t startTs = taosGetTimestampMs(); - - if ((nSqlCount > 0) && (nConcurrent > 0)) { - - pids = calloc(1, nConcurrent * nSqlCount * sizeof(pthread_t)); - infos = calloc(1, nConcurrent * nSqlCount * sizeof(threadInfo)); - - if ((NULL == pids) || (NULL == infos)) { - taos_close(taos); - ERROR_EXIT("memory allocation failed for create threads\n"); + TAOS * taos = NULL; + taos = taos_connect(g_queryInfo.host, + g_queryInfo.user, + g_queryInfo.password, + NULL, + g_queryInfo.port); + if (taos == NULL) { + errorPrint( "Failed to connect to TDengine, reason:%s\n", + taos_errstr(NULL)); + exit(-1); } - for (uint64_t i = 0; i < nSqlCount; i++) { - for (int j = 0; j < nConcurrent; j++) { - uint64_t seq = i * nConcurrent + j; - threadInfo *pThreadInfo = infos + seq; - pThreadInfo->threadID = seq; - pThreadInfo->querySeq = i; + if (0 != g_queryInfo.superQueryInfo.sqlCount) { + getAllChildNameOfSuperTable(taos, + g_queryInfo.dbName, + g_queryInfo.superQueryInfo.sTblName, + &g_queryInfo.superQueryInfo.childTblName, + &g_queryInfo.superQueryInfo.childTblCount); + } - if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { + prompt(); - char sqlStr[MAX_TB_NAME_SIZE*2]; - sprintf(sqlStr, "use %s", g_queryInfo.dbName); - if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) { - taos_close(taos); - free(infos); - free(pids); - errorPrint( "use database %s failed!\n\n", - g_queryInfo.dbName); - return -1; + if (g_args.debug_print || g_args.verbose_print) { + printfQuerySystemInfo(taos); + } + + if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) { + if (convertHostToServAddr( + g_queryInfo.host, g_queryInfo.port, &g_queryInfo.serv_addr) != 0) + exit(-1); + } + + pthread_t *pids = NULL; + threadInfo *infos = NULL; + //==== create sub threads for query from specify table + int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent; + uint64_t nSqlCount = g_queryInfo.specifiedQueryInfo.sqlCount; + + uint64_t startTs = taosGetTimestampMs(); + + if ((nSqlCount > 0) && (nConcurrent > 0)) { + + pids = calloc(1, nConcurrent * nSqlCount * sizeof(pthread_t)); + infos = calloc(1, nConcurrent * nSqlCount * sizeof(threadInfo)); + + if ((NULL == pids) || (NULL == infos)) { + taos_close(taos); + ERROR_EXIT("memory allocation failed for create threads\n"); + } + + for (uint64_t i = 0; i < nSqlCount; i++) { + for (int j = 0; j < nConcurrent; j++) { + uint64_t seq = i * nConcurrent + j; + threadInfo *pThreadInfo = infos + seq; + pThreadInfo->threadID = seq; + pThreadInfo->querySeq = i; + + if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { + + char sqlStr[MAX_TB_NAME_SIZE*2]; + sprintf(sqlStr, "use %s", g_queryInfo.dbName); + if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) { + taos_close(taos); + free(infos); + free(pids); + errorPrint( "use database %s failed!\n\n", + g_queryInfo.dbName); + return -1; + } } + + pThreadInfo->taos = NULL;// TODO: workaround to use separate taos connection; + + pthread_create(pids + seq, NULL, specifiedTableQuery, + pThreadInfo); } + } + } else { + g_queryInfo.specifiedQueryInfo.concurrent = 0; + } - pThreadInfo->taos = NULL;// TODO: workaround to use separate taos connection; + taos_close(taos); - pthread_create(pids + seq, NULL, specifiedTableQuery, - pThreadInfo); + pthread_t *pidsOfSub = NULL; + threadInfo *infosOfSub = NULL; + //==== create sub threads for query from all sub table of the super table + if ((g_queryInfo.superQueryInfo.sqlCount > 0) + && (g_queryInfo.superQueryInfo.threadCnt > 0)) { + pidsOfSub = calloc(1, g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t)); + infosOfSub = calloc(1, g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo)); + + if ((NULL == pidsOfSub) || (NULL == infosOfSub)) { + free(infos); + free(pids); + + ERROR_EXIT("memory allocation failed for create threads\n"); + } + + int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; + int threads = g_queryInfo.superQueryInfo.threadCnt; + + int64_t a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } + + int64_t b = 0; + if (threads != 0) { + b = ntables % threads; + } + + uint64_t tableFrom = 0; + for (int i = 0; i < threads; i++) { + threadInfo *pThreadInfo = infosOfSub + i; + pThreadInfo->threadID = i; + + pThreadInfo->start_table_from = tableFrom; + pThreadInfo->ntables = iend_table_to = i < b ? tableFrom + a : tableFrom + a - 1; + tableFrom = pThreadInfo->end_table_to + 1; + pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection; + pthread_create(pidsOfSub + i, NULL, superTableQuery, pThreadInfo); + } + + g_queryInfo.superQueryInfo.threadCnt = threads; + } else { + g_queryInfo.superQueryInfo.threadCnt = 0; + } + + if ((nSqlCount > 0) && (nConcurrent > 0)) { + for (int i = 0; i < nConcurrent; i++) { + for (int j = 0; j < nSqlCount; j++) { + pthread_join(pids[i * nSqlCount + j], NULL); + } } } - } else { - g_queryInfo.specifiedQueryInfo.concurrent = 0; - } - taos_close(taos); + tmfree((char*)pids); + tmfree((char*)infos); - pthread_t *pidsOfSub = NULL; - threadInfo *infosOfSub = NULL; - //==== create sub threads for query from all sub table of the super table - if ((g_queryInfo.superQueryInfo.sqlCount > 0) - && (g_queryInfo.superQueryInfo.threadCnt > 0)) { - pidsOfSub = calloc(1, g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t)); - infosOfSub = calloc(1, g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo)); - - if ((NULL == pidsOfSub) || (NULL == infosOfSub)) { - free(infos); - free(pids); - - ERROR_EXIT("memory allocation failed for create threads\n"); + for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) { + pthread_join(pidsOfSub[i], NULL); } - int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; - int threads = g_queryInfo.superQueryInfo.threadCnt; + tmfree((char*)pidsOfSub); + tmfree((char*)infosOfSub); - int64_t a = ntables / threads; - if (a < 1) { - threads = ntables; - a = 1; - } + // taos_close(taos);// TODO: workaround to use separate taos connection; + uint64_t endTs = taosGetTimestampMs(); - int64_t b = 0; - if (threads != 0) { - b = ntables % threads; - } + uint64_t totalQueried = g_queryInfo.specifiedQueryInfo.totalQueried + + g_queryInfo.superQueryInfo.totalQueried; - uint64_t tableFrom = 0; - for (int i = 0; i < threads; i++) { - threadInfo *pThreadInfo = infosOfSub + i; - pThreadInfo->threadID = i; - - pThreadInfo->start_table_from = tableFrom; - pThreadInfo->ntables = iend_table_to = i < b ? tableFrom + a : tableFrom + a - 1; - tableFrom = pThreadInfo->end_table_to + 1; - pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection; - pthread_create(pidsOfSub + i, NULL, superTableQuery, pThreadInfo); - } - - g_queryInfo.superQueryInfo.threadCnt = threads; - } else { - g_queryInfo.superQueryInfo.threadCnt = 0; - } - - if ((nSqlCount > 0) && (nConcurrent > 0)) { - for (int i = 0; i < nConcurrent; i++) { - for (int j = 0; j < nSqlCount; j++) { - pthread_join(pids[i * nSqlCount + j], NULL); - } - } - } - - tmfree((char*)pids); - tmfree((char*)infos); - - for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) { - pthread_join(pidsOfSub[i], NULL); - } - - tmfree((char*)pidsOfSub); - tmfree((char*)infosOfSub); - -// taos_close(taos);// TODO: workaround to use separate taos connection; - uint64_t endTs = taosGetTimestampMs(); - - uint64_t totalQueried = g_queryInfo.specifiedQueryInfo.totalQueried + - g_queryInfo.superQueryInfo.totalQueried; - - fprintf(stderr, "==== completed total queries: %"PRIu64", the QPS of all threads: %10.3f====\n", - totalQueried, - (double)(totalQueried/((endTs-startTs)/1000.0))); - return 0; + fprintf(stderr, "==== completed total queries: %"PRIu64", the QPS of all threads: %10.3f====\n", + totalQueried, + (double)(totalQueried/((endTs-startTs)/1000.0))); + return 0; } static void stable_sub_callback( TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { - if (res == NULL || taos_errno(res) != 0) { - errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n", - __func__, __LINE__, code, taos_errstr(res)); - return; - } + if (res == NULL || taos_errno(res) != 0) { + errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n", + __func__, __LINE__, code, taos_errstr(res)); + return; + } - if (param) - fetchResult(res, (threadInfo *)param); - // tao_unscribe() will free result. + if (param) + fetchResult(res, (threadInfo *)param); + // tao_unscribe() will free result. } static void specified_sub_callback( TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { - if (res == NULL || taos_errno(res) != 0) { - errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n", - __func__, __LINE__, code, taos_errstr(res)); - return; - } + if (res == NULL || taos_errno(res) != 0) { + errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n", + __func__, __LINE__, code, taos_errstr(res)); + return; + } - if (param) - fetchResult(res, (threadInfo *)param); - // tao_unscribe() will free result. + if (param) + fetchResult(res, (threadInfo *)param); + // tao_unscribe() will free result. } static TAOS_SUB* subscribeImpl( @@ -7634,35 +7642,35 @@ static TAOS_SUB* subscribeImpl( threadInfo *pThreadInfo, char *sql, char* topic, bool restart, uint64_t interval) { - TAOS_SUB* tsub = NULL; + TAOS_SUB* tsub = NULL; - if ((SPECIFIED_CLASS == class) - && (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode)) { - tsub = taos_subscribe( - pThreadInfo->taos, - restart, - topic, sql, specified_sub_callback, (void*)pThreadInfo, - g_queryInfo.specifiedQueryInfo.subscribeInterval); - } else if ((STABLE_CLASS == class) - && (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode)) { - tsub = taos_subscribe( - pThreadInfo->taos, - restart, - topic, sql, stable_sub_callback, (void*)pThreadInfo, - g_queryInfo.superQueryInfo.subscribeInterval); - } else { - tsub = taos_subscribe( - pThreadInfo->taos, - restart, - topic, sql, NULL, NULL, interval); - } + if ((SPECIFIED_CLASS == class) + && (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode)) { + tsub = taos_subscribe( + pThreadInfo->taos, + restart, + topic, sql, specified_sub_callback, (void*)pThreadInfo, + g_queryInfo.specifiedQueryInfo.subscribeInterval); + } else if ((STABLE_CLASS == class) + && (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode)) { + tsub = taos_subscribe( + pThreadInfo->taos, + restart, + topic, sql, stable_sub_callback, (void*)pThreadInfo, + g_queryInfo.superQueryInfo.subscribeInterval); + } else { + tsub = taos_subscribe( + pThreadInfo->taos, + restart, + topic, sql, NULL, NULL, interval); + } - if (tsub == NULL) { - errorPrint("failed to create subscription. topic:%s, sql:%s\n", topic, sql); - return NULL; - } + if (tsub == NULL) { + errorPrint("failed to create subscription. topic:%s, sql:%s\n", topic, sql); + return NULL; + } - return tsub; + return tsub; } static void *superSubscribe(void *sarg) { @@ -7816,291 +7824,291 @@ static void *superSubscribe(void *sarg) { } static void *specifiedSubscribe(void *sarg) { - threadInfo *pThreadInfo = (threadInfo *)sarg; -// TAOS_SUB* tsub = NULL; + threadInfo *pThreadInfo = (threadInfo *)sarg; + // TAOS_SUB* tsub = NULL; - setThreadName("specSub"); + setThreadName("specSub"); - if (pThreadInfo->taos == NULL) { - pThreadInfo->taos = taos_connect(g_queryInfo.host, - g_queryInfo.user, - g_queryInfo.password, - g_queryInfo.dbName, - g_queryInfo.port); if (pThreadInfo->taos == NULL) { - errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", - pThreadInfo->threadID, taos_errstr(NULL)); - return NULL; + pThreadInfo->taos = taos_connect(g_queryInfo.host, + g_queryInfo.user, + g_queryInfo.password, + g_queryInfo.dbName, + g_queryInfo.port); + if (pThreadInfo->taos == NULL) { + errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", + pThreadInfo->threadID, taos_errstr(NULL)); + return NULL; + } } - } - char sqlStr[MAX_TB_NAME_SIZE*2]; - sprintf(sqlStr, "use %s", g_queryInfo.dbName); - if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) { - taos_close(pThreadInfo->taos); - return NULL; - } + char sqlStr[MAX_TB_NAME_SIZE*2]; + sprintf(sqlStr, "use %s", g_queryInfo.dbName); + if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) { + taos_close(pThreadInfo->taos); + return NULL; + } - sprintf(g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID], - "taosdemo-subscribe-%"PRIu64"-%d", - pThreadInfo->querySeq, - pThreadInfo->threadID); - if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != '\0') { - sprintf(pThreadInfo->filePath, "%s-%d", + sprintf(g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID], + "taosdemo-subscribe-%"PRIu64"-%d", + pThreadInfo->querySeq, + pThreadInfo->threadID); + if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != '\0') { + sprintf(pThreadInfo->filePath, "%s-%d", g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); - } - g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] = subscribeImpl( - SPECIFIED_CLASS, pThreadInfo, - g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], - g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID], - g_queryInfo.specifiedQueryInfo.subscribeRestart, - g_queryInfo.specifiedQueryInfo.subscribeInterval); - if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) { - taos_close(pThreadInfo->taos); - return NULL; - } + } + g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] = subscribeImpl( + SPECIFIED_CLASS, pThreadInfo, + g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], + g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID], + g_queryInfo.specifiedQueryInfo.subscribeRestart, + g_queryInfo.specifiedQueryInfo.subscribeInterval); + if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) { + taos_close(pThreadInfo->taos); + return NULL; + } - // start loop to consume result + // start loop to consume result - g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0; - while((g_queryInfo.specifiedQueryInfo.endAfterConsume[pThreadInfo->querySeq] == -1) - || (g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] < - g_queryInfo.specifiedQueryInfo.endAfterConsume[pThreadInfo->querySeq])) { + g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0; + while((g_queryInfo.specifiedQueryInfo.endAfterConsume[pThreadInfo->querySeq] == -1) + || (g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] < + g_queryInfo.specifiedQueryInfo.endAfterConsume[pThreadInfo->querySeq])) { - printf("consumed[%d]: %d, endAfterConsum[%"PRId64"]: %d\n", - pThreadInfo->threadID, - g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID], - pThreadInfo->querySeq, - g_queryInfo.specifiedQueryInfo.endAfterConsume[pThreadInfo->querySeq]); - if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) { - continue; - } + printf("consumed[%d]: %d, endAfterConsum[%"PRId64"]: %d\n", + pThreadInfo->threadID, + g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID], + pThreadInfo->querySeq, + g_queryInfo.specifiedQueryInfo.endAfterConsume[pThreadInfo->querySeq]); + if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) { + continue; + } - g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID] = taos_consume( - g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]); - if (g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]) { - if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] - != 0) { - sprintf(pThreadInfo->filePath, "%s-%d", - g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], - pThreadInfo->threadID); - } - fetchResult( - g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID], - pThreadInfo); + g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID] = taos_consume( + g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]); + if (g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]) { + if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] + != 0) { + sprintf(pThreadInfo->filePath, "%s-%d", + g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); + } + fetchResult( + g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID], + pThreadInfo); - g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] ++; - if ((g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq] != -1) - && (g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] >= - g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) { - printf("keepProgress:%d, resub specified query: %"PRIu64"\n", - g_queryInfo.specifiedQueryInfo.subscribeKeepProgress, - pThreadInfo->querySeq); - g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0; - taos_unsubscribe(g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID], - g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); - g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] = - subscribeImpl( - SPECIFIED_CLASS, - pThreadInfo, - g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], - g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID], - g_queryInfo.specifiedQueryInfo.subscribeRestart, - g_queryInfo.specifiedQueryInfo.subscribeInterval); - if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) { - taos_close(pThreadInfo->taos); - return NULL; - } - } - } - } - taos_free_result(g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]); - taos_close(pThreadInfo->taos); + g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] ++; + if ((g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq] != -1) + && (g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] >= + g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) { + printf("keepProgress:%d, resub specified query: %"PRIu64"\n", + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress, + pThreadInfo->querySeq); + g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0; + taos_unsubscribe(g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID], + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); + g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] = + subscribeImpl( + SPECIFIED_CLASS, + pThreadInfo, + g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], + g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID], + g_queryInfo.specifiedQueryInfo.subscribeRestart, + g_queryInfo.specifiedQueryInfo.subscribeInterval); + if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) { + taos_close(pThreadInfo->taos); + return NULL; + } + } + } + } + taos_free_result(g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]); + taos_close(pThreadInfo->taos); - return NULL; + return NULL; } static int subscribeTestProcess() { - setupForAnsiEscape(); - printfQueryMeta(); - resetAfterAnsiEscape(); + setupForAnsiEscape(); + printfQueryMeta(); + resetAfterAnsiEscape(); - prompt(); + prompt(); - TAOS * taos = NULL; - taos = taos_connect(g_queryInfo.host, - g_queryInfo.user, - g_queryInfo.password, - g_queryInfo.dbName, - g_queryInfo.port); - if (taos == NULL) { - errorPrint( "Failed to connect to TDengine, reason:%s\n", - taos_errstr(NULL)); - exit(-1); - } - - if (0 != g_queryInfo.superQueryInfo.sqlCount) { - getAllChildNameOfSuperTable(taos, + TAOS * taos = NULL; + taos = taos_connect(g_queryInfo.host, + g_queryInfo.user, + g_queryInfo.password, g_queryInfo.dbName, - g_queryInfo.superQueryInfo.sTblName, - &g_queryInfo.superQueryInfo.childTblName, - &g_queryInfo.superQueryInfo.childTblCount); - } - - taos_close(taos); // TODO: workaround to use separate taos connection; - - pthread_t *pids = NULL; - threadInfo *infos = NULL; - - pthread_t *pidsOfStable = NULL; - threadInfo *infosOfStable = NULL; - - //==== create threads for query for specified table - if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) { - debugPrint("%s() LN%d, sepcified query sqlCount %d.\n", - __func__, __LINE__, - g_queryInfo.specifiedQueryInfo.sqlCount); - } else { - if (g_queryInfo.specifiedQueryInfo.concurrent <= 0) { - errorPrint("%s() LN%d, sepcified query sqlCount %d.\n", - __func__, __LINE__, - g_queryInfo.specifiedQueryInfo.sqlCount); + g_queryInfo.port); + if (taos == NULL) { + errorPrint( "Failed to connect to TDengine, reason:%s\n", + taos_errstr(NULL)); exit(-1); } - pids = calloc( - 1, - g_queryInfo.specifiedQueryInfo.sqlCount * - g_queryInfo.specifiedQueryInfo.concurrent * - sizeof(pthread_t)); - infos = calloc( - 1, - g_queryInfo.specifiedQueryInfo.sqlCount * - g_queryInfo.specifiedQueryInfo.concurrent * - sizeof(threadInfo)); - if ((NULL == pids) || (NULL == infos)) { - errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__); - exit(-1); + if (0 != g_queryInfo.superQueryInfo.sqlCount) { + getAllChildNameOfSuperTable(taos, + g_queryInfo.dbName, + g_queryInfo.superQueryInfo.sTblName, + &g_queryInfo.superQueryInfo.childTblName, + &g_queryInfo.superQueryInfo.childTblCount); + } + + taos_close(taos); // TODO: workaround to use separate taos connection; + + pthread_t *pids = NULL; + threadInfo *infos = NULL; + + pthread_t *pidsOfStable = NULL; + threadInfo *infosOfStable = NULL; + + //==== create threads for query for specified table + if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) { + debugPrint("%s() LN%d, sepcified query sqlCount %d.\n", + __func__, __LINE__, + g_queryInfo.specifiedQueryInfo.sqlCount); + } else { + if (g_queryInfo.specifiedQueryInfo.concurrent <= 0) { + errorPrint("%s() LN%d, sepcified query sqlCount %d.\n", + __func__, __LINE__, + g_queryInfo.specifiedQueryInfo.sqlCount); + exit(-1); + } + + pids = calloc( + 1, + g_queryInfo.specifiedQueryInfo.sqlCount * + g_queryInfo.specifiedQueryInfo.concurrent * + sizeof(pthread_t)); + infos = calloc( + 1, + g_queryInfo.specifiedQueryInfo.sqlCount * + g_queryInfo.specifiedQueryInfo.concurrent * + sizeof(threadInfo)); + if ((NULL == pids) || (NULL == infos)) { + errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__); + exit(-1); + } + + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { + for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) { + uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j; + threadInfo *pThreadInfo = infos + seq; + pThreadInfo->threadID = seq; + pThreadInfo->querySeq = i; + pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection; + pthread_create(pids + seq, NULL, specifiedSubscribe, pThreadInfo); + } + } + } + + //==== create threads for super table query + if (g_queryInfo.superQueryInfo.sqlCount <= 0) { + debugPrint("%s() LN%d, super table query sqlCount %d.\n", + __func__, __LINE__, + g_queryInfo.superQueryInfo.sqlCount); + } else { + if ((g_queryInfo.superQueryInfo.sqlCount > 0) + && (g_queryInfo.superQueryInfo.threadCnt > 0)) { + pidsOfStable = calloc( + 1, + g_queryInfo.superQueryInfo.sqlCount * + g_queryInfo.superQueryInfo.threadCnt * + sizeof(pthread_t)); + infosOfStable = calloc( + 1, + g_queryInfo.superQueryInfo.sqlCount * + g_queryInfo.superQueryInfo.threadCnt * + sizeof(threadInfo)); + if ((NULL == pidsOfStable) || (NULL == infosOfStable)) { + errorPrint("%s() LN%d, malloc failed for create threads\n", + __func__, __LINE__); + // taos_close(taos); + exit(-1); + } + + int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; + int threads = g_queryInfo.superQueryInfo.threadCnt; + + int64_t a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } + + int64_t b = 0; + if (threads != 0) { + b = ntables % threads; + } + + for (uint64_t i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + uint64_t tableFrom = 0; + for (int j = 0; j < threads; j++) { + uint64_t seq = i * threads + j; + threadInfo *pThreadInfo = infosOfStable + seq; + pThreadInfo->threadID = seq; + pThreadInfo->querySeq = i; + + pThreadInfo->start_table_from = tableFrom; + pThreadInfo->ntables = jend_table_to = jend_table_to + 1; + pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection; + pthread_create(pidsOfStable + seq, + NULL, superSubscribe, pThreadInfo); + } + } + + g_queryInfo.superQueryInfo.threadCnt = threads; + + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + for (int j = 0; j < threads; j++) { + uint64_t seq = i * threads + j; + pthread_join(pidsOfStable[seq], NULL); + } + } + } } for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) { uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j; - threadInfo *pThreadInfo = infos + seq; - pThreadInfo->threadID = seq; - pThreadInfo->querySeq = i; - pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection; - pthread_create(pids + seq, NULL, specifiedSubscribe, pThreadInfo); + pthread_join(pids[seq], NULL); } } - } - //==== create threads for super table query - if (g_queryInfo.superQueryInfo.sqlCount <= 0) { - debugPrint("%s() LN%d, super table query sqlCount %d.\n", - __func__, __LINE__, - g_queryInfo.superQueryInfo.sqlCount); - } else { - if ((g_queryInfo.superQueryInfo.sqlCount > 0) - && (g_queryInfo.superQueryInfo.threadCnt > 0)) { - pidsOfStable = calloc( - 1, - g_queryInfo.superQueryInfo.sqlCount * - g_queryInfo.superQueryInfo.threadCnt * - sizeof(pthread_t)); - infosOfStable = calloc( - 1, - g_queryInfo.superQueryInfo.sqlCount * - g_queryInfo.superQueryInfo.threadCnt * - sizeof(threadInfo)); - if ((NULL == pidsOfStable) || (NULL == infosOfStable)) { - errorPrint("%s() LN%d, malloc failed for create threads\n", - __func__, __LINE__); - // taos_close(taos); - exit(-1); - } + tmfree((char*)pids); + tmfree((char*)infos); - int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; - int threads = g_queryInfo.superQueryInfo.threadCnt; - - int64_t a = ntables / threads; - if (a < 1) { - threads = ntables; - a = 1; - } - - int64_t b = 0; - if (threads != 0) { - b = ntables % threads; - } - - for (uint64_t i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - uint64_t tableFrom = 0; - for (int j = 0; j < threads; j++) { - uint64_t seq = i * threads + j; - threadInfo *pThreadInfo = infosOfStable + seq; - pThreadInfo->threadID = seq; - pThreadInfo->querySeq = i; - - pThreadInfo->start_table_from = tableFrom; - pThreadInfo->ntables = jend_table_to = jend_table_to + 1; - pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection; - pthread_create(pidsOfStable + seq, - NULL, superSubscribe, pThreadInfo); - } - } - - g_queryInfo.superQueryInfo.threadCnt = threads; - - for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - for (int j = 0; j < threads; j++) { - uint64_t seq = i * threads + j; - pthread_join(pidsOfStable[seq], NULL); - } - } - } - } - - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { - for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) { - uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j; - pthread_join(pids[seq], NULL); - } - } - - tmfree((char*)pids); - tmfree((char*)infos); - - tmfree((char*)pidsOfStable); - tmfree((char*)infosOfStable); -// taos_close(taos); - return 0; + tmfree((char*)pidsOfStable); + tmfree((char*)infosOfStable); + // taos_close(taos); + return 0; } static void initOfInsertMeta() { - memset(&g_Dbs, 0, sizeof(SDbs)); + memset(&g_Dbs, 0, sizeof(SDbs)); - // set default values - tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE); - g_Dbs.port = 6030; - tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE); - tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE); - g_Dbs.threadCount = 2; + // set default values + tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE); + g_Dbs.port = 6030; + tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE); + tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE); + g_Dbs.threadCount = 2; - g_Dbs.use_metric = g_args.use_metric; + g_Dbs.use_metric = g_args.use_metric; } static void initOfQueryMeta() { - memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo)); + memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo)); - // set default values - tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE); - g_queryInfo.port = 6030; - tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE); - tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE); + // set default values + tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE); + g_queryInfo.port = 6030; + tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE); + tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE); } static void setParaFromArg() { @@ -8218,37 +8226,37 @@ static void setParaFromArg() { /* Function to do regular expression check */ static int regexMatch(const char *s, const char *reg, int cflags) { - regex_t regex; - char msgbuf[100] = {0}; + regex_t regex; + char msgbuf[100] = {0}; - /* Compile regular expression */ - if (regcomp(®ex, reg, cflags) != 0) { - printf("Fail to compile regex\n"); - exit(-1); - } + /* Compile regular expression */ + if (regcomp(®ex, reg, cflags) != 0) { + printf("Fail to compile regex\n"); + exit(-1); + } + + /* Execute regular expression */ + int reti = regexec(®ex, s, 0, NULL, 0); + if (!reti) { + regfree(®ex); + return 1; + } else if (reti == REG_NOMATCH) { + regfree(®ex); + return 0; + } else { + regerror(reti, ®ex, msgbuf, sizeof(msgbuf)); + printf("Regex match failed: %s\n", msgbuf); + regfree(®ex); + exit(-1); + } - /* Execute regular expression */ - int reti = regexec(®ex, s, 0, NULL, 0); - if (!reti) { - regfree(®ex); - return 1; - } else if (reti == REG_NOMATCH) { - regfree(®ex); return 0; - } else { - regerror(reti, ®ex, msgbuf, sizeof(msgbuf)); - printf("Regex match failed: %s\n", msgbuf); - regfree(®ex); - exit(-1); - } - - return 0; } static int isCommentLine(char *line) { - if (line == NULL) return 1; + if (line == NULL) return 1; - return regexMatch(line, "^\\s*#.*", REG_EXTENDED); + return regexMatch(line, "^\\s*#.*", REG_EXTENDED); } static void querySqlFile(TAOS* taos, char* sqlFile) @@ -8306,131 +8314,131 @@ static void querySqlFile(TAOS* taos, char* sqlFile) static void testMetaFile() { if (INSERT_TEST == g_args.test_mode) { - if (g_Dbs.cfgDir[0]) - taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir); + if (g_Dbs.cfgDir[0]) + taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir); - insertTestProcess(); + insertTestProcess(); } else if (QUERY_TEST == g_args.test_mode) { - if (g_queryInfo.cfgDir[0]) - taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); + if (g_queryInfo.cfgDir[0]) + taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); - queryTestProcess(); + queryTestProcess(); } else if (SUBSCRIBE_TEST == g_args.test_mode) { - if (g_queryInfo.cfgDir[0]) - taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); + if (g_queryInfo.cfgDir[0]) + taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); - subscribeTestProcess(); + subscribeTestProcess(); } else { - ; + ; } } static void queryResult() { - // query data + // query data - pthread_t read_id; - threadInfo *pThreadInfo = calloc(1, sizeof(threadInfo)); - assert(pThreadInfo); - pThreadInfo->start_time = 1500000000000; // 2017-07-14 10:40:00.000 - pThreadInfo->start_table_from = 0; + pthread_t read_id; + threadInfo *pThreadInfo = calloc(1, sizeof(threadInfo)); + assert(pThreadInfo); + pThreadInfo->start_time = 1500000000000; // 2017-07-14 10:40:00.000 + pThreadInfo->start_table_from = 0; - //pThreadInfo->do_aggreFunc = g_Dbs.do_aggreFunc; - if (g_args.use_metric) { - pThreadInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount; - pThreadInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1; - pThreadInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; - tstrncpy(pThreadInfo->tb_prefix, - g_Dbs.db[0].superTbls[0].childTblPrefix, TSDB_TABLE_NAME_LEN - 20); - } else { - pThreadInfo->ntables = g_args.num_of_tables; - pThreadInfo->end_table_to = g_args.num_of_tables -1; - tstrncpy(pThreadInfo->tb_prefix, g_args.tb_prefix, TSDB_TABLE_NAME_LEN); - } + //pThreadInfo->do_aggreFunc = g_Dbs.do_aggreFunc; + if (g_args.use_metric) { + pThreadInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount; + pThreadInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1; + pThreadInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; + tstrncpy(pThreadInfo->tb_prefix, + g_Dbs.db[0].superTbls[0].childTblPrefix, TSDB_TABLE_NAME_LEN - 20); + } else { + pThreadInfo->ntables = g_args.num_of_tables; + pThreadInfo->end_table_to = g_args.num_of_tables -1; + tstrncpy(pThreadInfo->tb_prefix, g_args.tb_prefix, TSDB_TABLE_NAME_LEN); + } - pThreadInfo->taos = taos_connect( - g_Dbs.host, - g_Dbs.user, - g_Dbs.password, - g_Dbs.db[0].dbName, - g_Dbs.port); - if (pThreadInfo->taos == NULL) { - errorPrint( "Failed to connect to TDengine, reason:%s\n", - taos_errstr(NULL)); + pThreadInfo->taos = taos_connect( + g_Dbs.host, + g_Dbs.user, + g_Dbs.password, + g_Dbs.db[0].dbName, + g_Dbs.port); + if (pThreadInfo->taos == NULL) { + errorPrint( "Failed to connect to TDengine, reason:%s\n", + taos_errstr(NULL)); + free(pThreadInfo); + exit(-1); + } + + tstrncpy(pThreadInfo->filePath, g_Dbs.resultFile, MAX_FILE_NAME_LEN); + + if (!g_Dbs.use_metric) { + pthread_create(&read_id, NULL, readTable, pThreadInfo); + } else { + pthread_create(&read_id, NULL, readMetric, pThreadInfo); + } + pthread_join(read_id, NULL); + taos_close(pThreadInfo->taos); free(pThreadInfo); - exit(-1); - } - - tstrncpy(pThreadInfo->filePath, g_Dbs.resultFile, MAX_FILE_NAME_LEN); - - if (!g_Dbs.use_metric) { - pthread_create(&read_id, NULL, readTable, pThreadInfo); - } else { - pthread_create(&read_id, NULL, readMetric, pThreadInfo); - } - pthread_join(read_id, NULL); - taos_close(pThreadInfo->taos); - free(pThreadInfo); } static void testCmdLine() { - if (strlen(configDir)) { - wordexp_t full_path; - if (wordexp(configDir, &full_path, 0) != 0) { - errorPrint( "Invalid path %s\n", configDir); - return; + if (strlen(configDir)) { + wordexp_t full_path; + if (wordexp(configDir, &full_path, 0) != 0) { + errorPrint( "Invalid path %s\n", configDir); + return; + } + taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]); + wordfree(&full_path); } - taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]); - wordfree(&full_path); - } - g_args.test_mode = INSERT_TEST; - insertTestProcess(); + g_args.test_mode = INSERT_TEST; + insertTestProcess(); - if (false == g_Dbs.insert_only) - queryResult(); + if (false == g_Dbs.insert_only) + queryResult(); } int main(int argc, char *argv[]) { - parse_args(argc, argv, &g_args); + parse_args(argc, argv, &g_args); - debugPrint("meta file: %s\n", g_args.metaFile); + debugPrint("meta file: %s\n", g_args.metaFile); - if (g_args.metaFile) { - initOfInsertMeta(); - initOfQueryMeta(); + if (g_args.metaFile) { + initOfInsertMeta(); + initOfQueryMeta(); - if (false == getInfoFromJsonFile(g_args.metaFile)) { - printf("Failed to read %s\n", g_args.metaFile); - return 1; - } - - testMetaFile(); - } else { - memset(&g_Dbs, 0, sizeof(SDbs)); - setParaFromArg(); - - if (NULL != g_args.sqlFile) { - TAOS* qtaos = taos_connect( - g_Dbs.host, - g_Dbs.user, - g_Dbs.password, - g_Dbs.db[0].dbName, - g_Dbs.port); - querySqlFile(qtaos, g_args.sqlFile); - taos_close(qtaos); + if (false == getInfoFromJsonFile(g_args.metaFile)) { + printf("Failed to read %s\n", g_args.metaFile); + return 1; + } + testMetaFile(); } else { - testCmdLine(); + memset(&g_Dbs, 0, sizeof(SDbs)); + setParaFromArg(); + + if (NULL != g_args.sqlFile) { + TAOS* qtaos = taos_connect( + g_Dbs.host, + g_Dbs.user, + g_Dbs.password, + g_Dbs.db[0].dbName, + g_Dbs.port); + querySqlFile(qtaos, g_args.sqlFile); + taos_close(qtaos); + + } else { + testCmdLine(); + } + + if (g_dupstr) + free(g_dupstr); } - if (g_dupstr) - free(g_dupstr); - } - - return 0; + return 0; } From 9df622ef80a3fbc516bf2d5dc1dafd2baedd0ae7 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Thu, 22 Jul 2021 18:03:23 +0800 Subject: [PATCH 026/106] [TD-5338]:test stable interval function --- tests/pytest/query/queryInterval.py | 46 +++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/pytest/query/queryInterval.py b/tests/pytest/query/queryInterval.py index ce8d05ae50..be79050d71 100644 --- a/tests/pytest/query/queryInterval.py +++ b/tests/pytest/query/queryInterval.py @@ -73,6 +73,52 @@ class TDTestCase: tdSql.checkData(6, 0, "2020-09-16 00:00:00") tdSql.checkData(6, 1, 222.0) + # test case for https://jira.taosdata.com:18080/browse/TD-5338 + tdSql.query("select loc,max(voltage) from st interval(1m);") + tdSql.checkRows(8) + tdSql.checkData(0, 0, "2020-07-01 04:24:00.000") + tdSql.checkData(0, 1, "beijing") + tdSql.checkData(0, 2, 220) + tdSql.checkData(1, 0, "2020-07-12 18:11:00.000") + tdSql.checkData(1, 1, "beijing") + tdSql.checkData(1, 2, 221) + tdSql.checkData(2, 0, "2020-07-24 07:58:00.000") + tdSql.checkData(2, 1, "beijing") + tdSql.checkData(2, 2, 225) + tdSql.checkData(3, 0, "2020-08-04 21:44:00.000") + tdSql.checkData(2, 1, "beijing") + tdSql.checkData(3, 2, 228) + tdSql.checkData(4, 0, "2020-08-16 11:31:00.000") + tdSql.checkData(4, 1, "shanghai") + tdSql.checkData(4, 2, 225) + tdSql.checkData(5, 0, "2020-08-28 01:18:00.000") + tdSql.checkData(5, 1, "shanghai") + tdSql.checkData(5, 2, 228) + tdSql.checkData(6, 0, "2020-09-08 15:04:00.000") + tdSql.checkData(6, 1, "beijing") + tdSql.checkData(6, 2, 222) + tdSql.checkData(7, 0, "2020-09-20 04:51:00.000") + tdSql.checkData(7, 1, "shanghai") + tdSql.checkData(7, 2, 222) + tdSql.query("select loc,max(voltage) from t0 interval(1m);") + tdSql.checkRows(5) + tdSql.checkData(0, 0, "2020-07-01 04:24:00.000") + tdSql.checkData(0, 1, "beijing") + tdSql.checkData(0, 2, 220) + tdSql.checkData(1, 0, "2020-07-12 18:11:00.000") + tdSql.checkData(1, 1, "beijing") + tdSql.checkData(1, 2, 221) + tdSql.checkData(2, 0, "2020-07-24 07:58:00.000") + tdSql.checkData(2, 1, "beijing") + tdSql.checkData(2, 2, 225) + tdSql.checkData(3, 0, "2020-08-04 21:44:00.000") + tdSql.checkData(2, 1, "beijing") + tdSql.checkData(3, 2, 228) + tdSql.checkData(4, 0, "2020-09-08 15:04:00.000") + tdSql.checkData(4, 1, "beijing") + tdSql.checkData(4, 2, 222) + + # test case for https://jira.taosdata.com:18080/browse/TD-2298 tdSql.execute("create database test keep 36500") tdSql.execute("use test") From 1bdd6838724d0fa00f4948bd3c702c59b97fe44a Mon Sep 17 00:00:00 2001 From: wenzhouwww Date: Thu, 22 Jul 2021 19:59:36 +0800 Subject: [PATCH 027/106] [TD-5369] taosdemo test case for nano support , the sql files is must be here! --- tests/pytest/fulltest.sh | 8 + .../tools/taosdemoAllTest/nano_samples.csv | 100 +++++++++++ .../tools/taosdemoAllTest/nano_sampletags.csv | 100 +++++++++++ .../taosdemoTestNanoCreateDB.sql | 7 + .../taosdemoTestNanoDatabase.json | 88 ++++++++++ .../taosdemoTestNanoDatabaseInsertForSub.json | 84 +++++++++ .../taosdemoTestNanoDatabaseNow.json | 62 +++++++ .../taosdemoTestNanoDatabasecsv.json | 84 +++++++++ .../taosdemoTestSupportNanoInsert.py | 162 ++++++++++++++++++ .../taosdemoTestSupportNanoQuery.json | 92 ++++++++++ .../taosdemoTestSupportNanoQuery.py | 157 +++++++++++++++++ .../taosdemoTestSupportNanoQuerycsv.json | 110 ++++++++++++ .../taosdemoTestSupportNanoSubscribe.json | 32 ++++ .../taosdemoTestSupportNanosubscribe.py | 123 +++++++++++++ 14 files changed, 1209 insertions(+) create mode 100644 tests/pytest/tools/taosdemoAllTest/nano_samples.csv create mode 100644 tests/pytest/tools/taosdemoAllTest/nano_sampletags.csv create mode 100644 tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoCreateDB.sql create mode 100644 tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabase.json create mode 100644 tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseInsertForSub.json create mode 100644 tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseNow.json create mode 100644 tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabasecsv.json create mode 100644 tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py create mode 100644 tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoQuery.json create mode 100644 tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoQuery.py create mode 100644 tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoQuerycsv.json create mode 100644 tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoSubscribe.json create mode 100644 tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanosubscribe.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 80d45fbc31..99a15e2e71 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -361,4 +361,12 @@ python3 test.py -f alter/alter_keep.py python3 test.py -f alter/alter_cacheLastRow.py python3 ./test.py -f query/querySession.py python3 test.py -f alter/alter_create_exception.py + +# nano support +python3 test.py -f tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py +python3 test.py -f tools/taosdemoAllTest/taosdemoTestSupportNanoQuery.py +python3 test.py -f tools/taosdemoAllTest/taosdemoTestSupportNanosubscribe.py + + #======================p4-end=============== +python3 test.py -f tools/taosdemoAllTest/pytest.py \ No newline at end of file diff --git a/tests/pytest/tools/taosdemoAllTest/nano_samples.csv b/tests/pytest/tools/taosdemoAllTest/nano_samples.csv new file mode 100644 index 0000000000..5fc779b41b --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/nano_samples.csv @@ -0,0 +1,100 @@ +8.855,"binary_str0" ,1626870128248246976 +8.75,"binary_str1" ,1626870128249060032 +5.44,"binary_str2" ,1626870128249067968 +8.45,"binary_str3" ,1626870128249072064 +4.07,"binary_str4" ,1626870128249075904 +6.97,"binary_str5" ,1626870128249078976 +6.86,"binary_str6" ,1626870128249082048 +1.585,"binary_str7" ,1626870128249085120 +1.4,"binary_str8" ,1626870128249087936 +5.135,"binary_str9" ,1626870128249092032 +3.15,"binary_str10" ,1626870128249095104 +1.765,"binary_str11" ,1626870128249097920 +7.71,"binary_str12" ,1626870128249100992 +3.91,"binary_str13" ,1626870128249104064 +5.615,"binary_str14" ,1626870128249106880 +9.495,"binary_str15" ,1626870128249109952 +3.825,"binary_str16" ,1626870128249113024 +1.94,"binary_str17" ,1626870128249117120 +5.385,"binary_str18" ,1626870128249119936 +7.075,"binary_str19" ,1626870128249123008 +5.715,"binary_str20" ,1626870128249126080 +1.83,"binary_str21" ,1626870128249128896 +6.365,"binary_str22" ,1626870128249131968 +6.55,"binary_str23" ,1626870128249135040 +6.315,"binary_str24" ,1626870128249138112 +3.82,"binary_str25" ,1626870128249140928 +2.455,"binary_str26" ,1626870128249145024 +7.795,"binary_str27" ,1626870128249148096 +2.47,"binary_str28" ,1626870128249150912 +1.37,"binary_str29" ,1626870128249155008 +5.39,"binary_str30" ,1626870128249158080 +5.13,"binary_str31" ,1626870128249160896 +4.09,"binary_str32" ,1626870128249163968 +5.855,"binary_str33" ,1626870128249167040 +0.17,"binary_str34" ,1626870128249170112 +1.955,"binary_str35" ,1626870128249173952 +0.585,"binary_str36" ,1626870128249178048 +0.33,"binary_str37" ,1626870128249181120 +7.925,"binary_str38" ,1626870128249183936 +9.685,"binary_str39" ,1626870128249187008 +2.6,"binary_str40" ,1626870128249191104 +5.705,"binary_str41" ,1626870128249193920 +3.965,"binary_str42" ,1626870128249196992 +4.43,"binary_str43" ,1626870128249200064 +8.73,"binary_str44" ,1626870128249202880 +3.105,"binary_str45" ,1626870128249205952 +9.39,"binary_str46" ,1626870128249209024 +2.825,"binary_str47" ,1626870128249212096 +9.675,"binary_str48" ,1626870128249214912 +9.99,"binary_str49" ,1626870128249217984 +4.51,"binary_str50" ,1626870128249221056 +4.94,"binary_str51" ,1626870128249223872 +7.72,"binary_str52" ,1626870128249226944 +4.135,"binary_str53" ,1626870128249231040 +2.325,"binary_str54" ,1626870128249234112 +4.585,"binary_str55" ,1626870128249236928 +8.76,"binary_str56" ,1626870128249240000 +4.715,"binary_str57" ,1626870128249243072 +0.56,"binary_str58" ,1626870128249245888 +5.35,"binary_str59" ,1626870128249249984 +5.075,"binary_str60" ,1626870128249253056 +6.665,"binary_str61" ,1626870128249256128 +7.13,"binary_str62" ,1626870128249258944 +2.775,"binary_str63" ,1626870128249262016 +5.775,"binary_str64" ,1626870128249265088 +1.62,"binary_str65" ,1626870128249267904 +1.625,"binary_str66" ,1626870128249270976 +8.15,"binary_str67" ,1626870128249274048 +0.75,"binary_str68" ,1626870128249277120 +3.265,"binary_str69" ,1626870128249280960 +8.585,"binary_str70" ,1626870128249284032 +1.88,"binary_str71" ,1626870128249287104 +8.44,"binary_str72" ,1626870128249289920 +5.12,"binary_str73" ,1626870128249295040 +2.58,"binary_str74" ,1626870128249298112 +9.42,"binary_str75" ,1626870128249300928 +1.765,"binary_str76" ,1626870128249304000 +2.66,"binary_str77" ,1626870128249308096 +1.405,"binary_str78" ,1626870128249310912 +5.595,"binary_str79" ,1626870128249315008 +2.28,"binary_str80" ,1626870128249318080 +9.24,"binary_str81" ,1626870128249320896 +9.03,"binary_str82" ,1626870128249323968 +6.055,"binary_str83" ,1626870128249327040 +1.74,"binary_str84" ,1626870128249330112 +5.77,"binary_str85" ,1626870128249332928 +1.97,"binary_str86" ,1626870128249336000 +0.3,"binary_str87" ,1626870128249339072 +7.145,"binary_str88" ,1626870128249342912 +0.88,"binary_str89" ,1626870128249345984 +8.025,"binary_str90" ,1626870128249349056 +4.81,"binary_str91" ,1626870128249351872 +0.725,"binary_str92" ,1626870128249355968 +3.85,"binary_str93" ,1626870128249359040 +9.455,"binary_str94" ,1626870128249362112 +2.265,"binary_str95" ,1626870128249364928 +3.985,"binary_str96" ,1626870128249368000 +9.375,"binary_str97" ,1626870128249371072 +0.2,"binary_str98" ,1626870128249373888 +6.95,"binary_str99" ,1626870128249377984 diff --git a/tests/pytest/tools/taosdemoAllTest/nano_sampletags.csv b/tests/pytest/tools/taosdemoAllTest/nano_sampletags.csv new file mode 100644 index 0000000000..18fb855d6d --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/nano_sampletags.csv @@ -0,0 +1,100 @@ +"string0",7,8.615 +"string1",4,9.895 +"string2",3,2.92 +"string3",3,5.62 +"string4",7,1.615 +"string5",6,1.45 +"string6",5,7.48 +"string7",7,3.01 +"string8",5,4.76 +"string9",10,7.09 +"string10",2,8.38 +"string11",7,8.65 +"string12",5,5.025 +"string13",10,5.765 +"string14",2,4.57 +"string15",2,1.03 +"string16",7,6.98 +"string17",10,0.23 +"string18",7,5.815 +"string19",1,2.37 +"string20",10,8.865 +"string21",3,1.235 +"string22",2,8.62 +"string23",9,1.045 +"string24",8,4.34 +"string25",1,5.455 +"string26",2,4.475 +"string27",1,6.95 +"string28",2,3.39 +"string29",3,6.79 +"string30",7,9.735 +"string31",1,9.79 +"string32",10,9.955 +"string33",1,5.095 +"string34",3,3.86 +"string35",9,5.105 +"string36",10,4.22 +"string37",1,2.78 +"string38",9,6.345 +"string39",1,0.975 +"string40",5,6.16 +"string41",4,7.735 +"string42",5,6.6 +"string43",8,2.845 +"string44",1,0.655 +"string45",3,2.995 +"string46",9,3.6 +"string47",8,3.47 +"string48",3,7.98 +"string49",6,2.225 +"string50",9,5.44 +"string51",4,6.335 +"string52",3,2.955 +"string53",1,0.565 +"string54",6,5.575 +"string55",6,9.905 +"string56",9,6.025 +"string57",8,0.94 +"string58",10,0.15 +"string59",8,1.555 +"string60",4,2.28 +"string61",2,8.29 +"string62",9,6.22 +"string63",6,3.35 +"string64",10,6.7 +"string65",3,9.345 +"string66",7,9.815 +"string67",1,5.365 +"string68",10,3.81 +"string69",1,6.405 +"string70",8,2.715 +"string71",3,8.58 +"string72",8,6.34 +"string73",2,7.49 +"string74",4,8.64 +"string75",3,8.995 +"string76",7,3.465 +"string77",1,7.64 +"string78",6,3.65 +"string79",6,1.4 +"string80",6,5.875 +"string81",2,1.22 +"string82",5,7.87 +"string83",9,8.41 +"string84",9,8.9 +"string85",9,3.89 +"string86",2,5.0 +"string87",2,4.495 +"string88",4,2.835 +"string89",3,5.895 +"string90",7,8.41 +"string91",5,5.125 +"string92",7,9.165 +"string93",5,8.315 +"string94",10,7.485 +"string95",7,4.635 +"string96",2,6.015 +"string97",8,0.595 +"string98",3,8.79 +"string99",4,1.72 diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoCreateDB.sql b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoCreateDB.sql new file mode 100644 index 0000000000..e79e09592c --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoCreateDB.sql @@ -0,0 +1,7 @@ +drop database if exists nsdbsql; +create database nsdbsql precision "ns" keep 36 days 6 update 1; +use nsdbsql; +CREATE STABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupdId int); +CREATE TABLE d1001 USING meters TAGS ("Beijing.Chaoyang", 2); +INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 219, 0.32); +INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 85, 32, 0.76); diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabase.json b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabase.json new file mode 100644 index 0000000000..9010415fe6 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabase.json @@ -0,0 +1,88 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 10, + "thread_count_create_tbl": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "nsdb", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ns", + "keep": 36, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 1000, + "childtable_prefix": "tb0_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 10000, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 10000000, + "start_timestamp": "2021-07-01 00:00:00.000", + "sample_format": "", + "sample_file": "", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":3}, {"type": "BINARY", "len": 16, "count":2}, {"type": "BINARY", "len": 32, "count":2}, + {"type": "TIMESTAMP"}, {"type": "BIGINT", "count":3},{"type": "FLOAT", "count":1},{"type": "SMALLINT", "count":1},{"type": "TINYINT", "count":1}, + {"type": "BOOL"},{"type": "NCHAR","len":16}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5},{"type": "NCHAR","len":16, "count":1}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 1000, + "childtable_prefix": "tb1_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 10000, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 10, + "disorder_range": 1000, + "timestamp_step": 10000000, + "start_timestamp": "2021-07-01 00:00:00.000", + "sample_format": "", + "sample_file": "", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":3}, {"type": "BINARY", "len": 16, "count":2}, {"type": "BINARY", "len": 32, "count":2}, + {"type": "TIMESTAMP"}, {"type": "BIGINT", "count":3},{"type": "FLOAT", "count":1},{"type": "SMALLINT", "count":1},{"type": "TINYINT", "count":1}, + {"type": "BOOL"},{"type": "NCHAR","len":16}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5},{"type": "NCHAR","len":16, "count":1}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseInsertForSub.json b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseInsertForSub.json new file mode 100644 index 0000000000..0726f3905d --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseInsertForSub.json @@ -0,0 +1,84 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 10, + "thread_count_create_tbl": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "subnsdb", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ns", + "keep": 36, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 10, + "childtable_prefix": "tb0_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "samples", + "insert_mode": "taosc", + "insert_rows": 10, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 10000000, + "start_timestamp": "2021-07-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./tools/taosdemoAllTest/nano_samples.csv", + "tags_file": "./tools/taosdemoAllTest/nano_sampletags.csv", + "columns": [{"type": "DOUBLE"}, {"type": "BINARY", "len": 64, "count":1}, {"type": "TIMESTAMP", "count":1}], + "tags": [{"type": "BINARY", "len": 16, "count":1},{"type": "INT"},{"type": "DOUBLE"}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 10, + "childtable_prefix": "tb1_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "samples", + "insert_mode": "taosc", + "insert_rows": 10, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 10, + "disorder_range": 1000, + "timestamp_step": 10000000, + "start_timestamp": "2021-07-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./tools/taosdemoAllTest/nano_samples.csv", + "tags_file": "./tools/taosdemoAllTest/nano_sampletags.csv", + "columns": [{"type": "DOUBLE"}, {"type": "BINARY", "len": 64, "count":1}, {"type": "TIMESTAMP", "count":1}], + "tags": [{"type": "BINARY", "len": 16, "count":1},{"type": "INT"},{"type": "DOUBLE"}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseNow.json b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseNow.json new file mode 100644 index 0000000000..d2542a0eba --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseNow.json @@ -0,0 +1,62 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 10, + "thread_count_create_tbl": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "nsdb2", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ns", + "keep": 36, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 1000, + "childtable_prefix": "tb0_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 10, + "start_timestamp": "now", + "sample_format": "", + "sample_file": "", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":3}, {"type": "BINARY", "len": 16, "count":2}, {"type": "BINARY", "len": 32, "count":2}, + {"type": "TIMESTAMP"}, {"type": "BIGINT", "count":3},{"type": "FLOAT", "count":1},{"type": "SMALLINT", "count":1},{"type": "TINYINT", "count":1}, + {"type": "BOOL"},{"type": "NCHAR","len":16}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5},{"type": "NCHAR","len":16, "count":1}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabasecsv.json b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabasecsv.json new file mode 100644 index 0000000000..867619ed8c --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabasecsv.json @@ -0,0 +1,84 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 10, + "thread_count_create_tbl": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "nsdbcsv", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ns", + "keep": 36, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "tb0_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "samples", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 10000000, + "start_timestamp": "2021-07-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./tools/taosdemoAllTest/nano_samples.csv", + "tags_file": "./tools/taosdemoAllTest/nano_sampletags.csv", + "columns": [{"type": "DOUBLE"}, {"type": "BINARY", "len": 64, "count":1}, {"type": "TIMESTAMP", "count":1}], + "tags": [{"type": "BINARY", "len": 16, "count":1},{"type": "INT"},{"type": "DOUBLE"}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "tb1_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "samples", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 10, + "disorder_range": 1000, + "timestamp_step": 10000000, + "start_timestamp": "2021-07-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./tools/taosdemoAllTest/nano_samples.csv", + "tags_file": "./tools/taosdemoAllTest/nano_sampletags.csv", + "columns": [{"type": "DOUBLE"}, {"type": "BINARY", "len": 64, "count":1}, {"type": "TIMESTAMP", "count":1}], + "tags": [{"type": "BINARY", "len": 16, "count":1},{"type": "INT"},{"type": "DOUBLE"}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py new file mode 100644 index 0000000000..010308d037 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py @@ -0,0 +1,162 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + + # insert: create one or mutiple tables per sql and insert multiple rows per sql + # insert data from a special timestamp + # check stable stb0 + + os.system("%staosdemo -f tools/taosdemoAllTest/taosdemoTestNanoDatabase.json -y " % binPath) + tdSql.execute("use nsdb") + tdSql.query("show stables") + tdSql.checkData(0, 4, 1000) + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 1000) + tdSql.query("select count(*) from tb0_0") + tdSql.checkData(0, 0, 10000) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 10000000) + tdSql.query("describe stb0") + tdSql.getData(8, 1) + tdSql.checkDataType(8, 1,"TIMESTAMP") + tdSql.query("select last(ts) from stb0") + tdSql.getData(0, 0) + + # check stable stb1 which is insert with disord + + tdSql.query("select count (tbname) from stb1") + tdSql.checkData(0, 0, 1000) + tdSql.query("select count(*) from tb1_0") + tdSql.checkData(0, 0, 10000) + tdSql.query("select count(*) from stb1") + tdSql.checkData(0, 0, 10000000) + # check c8 is an nano timestamp + tdSql.query("describe stb1") + tdSql.checkDataType(8, 1,"TIMESTAMP") + # check insert timestamp_step is nano_second + tdSql.query("select last(ts) from stb1") + tdSql.checkData(0, 0,"2021-07-01 00:01:39.990000000") + + # insert data from now time + + # check stable stb0 + os.system("%staosdemo -f tools/taosdemoAllTest/taosdemoTestNanoDatabaseNow.json -y " % binPath) + tdSql.execute("use nsdb2") + tdSql.query("show stables") + tdSql.checkData(0, 4, 1000) + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 1000) + tdSql.query("select count(*) from tb0_0") + tdSql.checkData(0, 0, 100) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 100000) + # check c8 is an nano timestamp + tdSql.query("describe stb0") + tdSql.checkDataType(8,1,"TIMESTAMP") + + # insert by csv files and timetamp is long int , strings in ts and cols + + os.system("%staosdemo -f tools/taosdemoAllTest/taosdemoTestNanoDatabasecsv.json -y " % binPath) + tdSql.execute("use nsdbcsv") + tdSql.query("show stables") + tdSql.checkData(0, 4, 100) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 10000) + tdSql.query("describe stb0") + tdSql.checkDataType(3, 1, "TIMESTAMP") + tdSql.query("select count(*) from stb0 where ts > \"2021-07-01 00:00:00.490000000\"") + tdSql.checkData(0, 0, 5000) + tdSql.query("select count(*) from stb0 where ts >now -22d-1h-3s") + tdSql.checkData(0, 0, 10000) + tdSql.query("select count(*) from stb0 where ts >now -22d-1h-3s") + tdSql.checkData(0, 0, 10000) + tdSql.query("select count(*) from stb0 where ts < 1626918583000000000") + tdSql.checkData(0, 0, 10000) + + os.system("rm -rf ./insert_res.txt") + os.system("rm -rf tools/taosdemoAllTest/taosdemoTestSupportNano*.py.sql") + + + + + + + + + + + # taosdemo test insert with command and parameter , detals show taosdemo --help + + os.system("%staosdemo -u root -P taosdata -p 6030 -h vm84 -a 1 -m pre -n 10 -T 20 -t 60 -o res.txt -y " % binPath) + tdSql.query("select count(*) from test.meters") + tdSql.checkData(0, 0, 600) + + os.system("%staosdemo -u root -P taosdata -p 6030 -h vm84 -a 1 -m pre -n 10 -T 20 -t 60 -o res.txt -y " % binPath) + tdSql.query("select count(*) from test.meters") + tdSql.checkData(0, 0, 600) + + # check taosdemo -s + + os.system("%staosdemo -s tools/taosdemoAllTest/taosdemoTestNanoCreateDB.sql -y " % binPath) + tdSql.query("select count(*) from nsdbsql.meters") + tdSql.checkData(0, 0, 2) + + + + os.system("rm -rf ./res.txt") + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoQuery.json b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoQuery.json new file mode 100644 index 0000000000..fff1017588 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoQuery.json @@ -0,0 +1,92 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "nsdb", + "query_times": 10, + "query_mode": "taosc", + "specified_table_query": { + "query_interval": 1, + "concurrent": 2, + "sqls": [ + { + "sql": "select count(*) from stb0 where ts>\"2021-07-01 00:01:00.000000000 \" ;", + "result": "./query_res0.txt" + }, + { + "sql": "select count(*) from stb0 where ts>\"2021-07-01 00:01:00.000000000\" and ts <=\"2021-07-01 00:01:10.000000000\" ;", + "result": "./query_res1.txt" + }, + { + "sql": "select count(*) from stb0 where ts>now-20d ;", + "result": "./query_res2.txt" + }, + { + "sql": "select max(c10) from stb0;", + "result": "./query_res3.txt" + }, + { + "sql": "select min(c1) from stb0;", + "result": "./query_res4.txt" + }, + { + "sql": "select avg(c1) from stb0;", + "result": "./query_res5.txt" + }, + { + "sql":"select count(*) from stb0 group by tbname;", + "result":"./query_res6.txt" + } + + ] + }, + "super_table_query": { + "stblname": "stb0", + "query_interval": 0, + "threads": 4, + "sqls": [ + { + "sql": "select count(*) from xxxx where ts>\"2021-07-01 00:01:00.000000000 \" ;", + "result": "./query_res_tb0.txt" + }, + { + "sql":"select count(*) from xxxx where ts>\"2021-07-01 00:01:00.000000000\" and ts <=\"2021-07-01 00:01:10.000000000\" ;", + "result": "./query_res_tb1.txt" + }, + { + "sql":"select first(*) from xxxx ;", + "result": "./query_res_tb2.txt" + }, + { + "sql":"select last(*) from xxxx;", + "result": "./query_res_tb3.txt" + + }, + { + "sql":"select last_row(*) from xxxx ;", + "result": "./query_res_tb4.txt" + + }, + { + "sql":"select max(c10) from xxxx ;", + "result": "./query_res_tb5.txt" + + }, + { + "sql":"select min(c1) from xxxx ;", + "result": "./query_res_tb6.txt" + + }, + { + "sql":"select avg(c10) from xxxx ;", + "result": "./query_res_tb7.txt" + + } + + ] + } + } \ No newline at end of file diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoQuery.py b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoQuery.py new file mode 100644 index 0000000000..21a7037ce6 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoQuery.py @@ -0,0 +1,157 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + # query: query test for nanoSecond with where and max min groupby order + os.system("%staosdemo -f tools/taosdemoAllTest/taosdemoTestNanoDatabase.json -y " % binPath) + + tdSql.execute("use nsdb") + + # use where to filter + + tdSql.query("select count(*) from stb0 where ts>\"2021-07-01 00:01:00.000000000 \" ;") + tdSql.checkData(0, 0, 3999000) + tdSql.query("select count(*) from stb0 where ts>\"2021-07-01 00:01:00.000000000\" and ts <=\"2021-07-01 00:01:10.000000000\" ;") + tdSql.checkData(0, 0, 1000000) + + tdSql.query("select count(*) from tb0_0 where ts>\"2021-07-01 00:01:00.000000000 \" ;") + tdSql.checkData(0, 0, 3999) + tdSql.query("select count(*) from tb0_0 where ts>\"2021-07-01 00:01:00.000000000\" and ts <=\"2021-07-01 00:01:10.000000000\" ;") + tdSql.checkData(0, 0, 1000) + + + # select max min avg from special col + tdSql.query("select max(c10) from stb0;") + print("select max(c10) from stb0 : " , tdSql.getData(0, 0)) + + tdSql.query("select max(c10) from tb0_0;") + print("select max(c10) from tb0_0 : " , tdSql.getData(0, 0)) + + + tdSql.query("select min(c1) from stb0;") + print( "select min(c1) from stb0 : " , tdSql.getData(0, 0)) + + tdSql.query("select min(c1) from tb0_0;") + print( "select min(c1) from tb0_0 : " , tdSql.getData(0, 0)) + + tdSql.query("select avg(c1) from stb0;") + print( "select avg(c1) from stb0 : " , tdSql.getData(0, 0)) + + tdSql.query("select avg(c1) from tb0_0;") + print( "select avg(c1) from tb0_0 : " , tdSql.getData(0, 0)) + + tdSql.query("select count(*) from stb0 group by tbname;") + tdSql.checkData(0, 0, 10000) + tdSql.checkData(100, 0, 10000) + + # query : query above sqls by taosdemo and continuously + + os.system("%staosdemo -f tools/taosdemoAllTest/taosdemoTestSupportNanoQuery.json -y " % binPath) + + + os.system("%staosdemo -f tools/taosdemoAllTest/taosdemoTestNanoDatabasecsv.json -y " % binPath) + tdSql.execute("use nsdbcsv") + tdSql.query("show stables") + tdSql.checkData(0, 4, 100) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 10000) + tdSql.query("describe stb0") + tdSql.checkDataType(3, 1, "TIMESTAMP") + tdSql.query("select count(*) from stb0 where ts >\"2021-07-01 00:00:00.490000000\"") + tdSql.checkData(0, 0, 5000) + tdSql.query("select count(*) from stb0 where ts >now -22d-1h-3s") + tdSql.checkData(0, 0, 10000) + tdSql.query("select count(*) from stb0 where ts < 1626918583000000000") + tdSql.checkData(0, 0, 10000) + tdSql.execute('select count(*) from stb0 where c2 > 162687012800000000') + tdSql.execute('select count(*) from stb0 where c2 < 162687012800000000') + tdSql.execute('select count(*) from stb0 where c2 = 162687012800000000') + tdSql.execute('select count(*) from stb0 where c2 != 162687012800000000') + tdSql.execute('select count(*) from stb0 where c2 <> 162687012800000000') + tdSql.execute('select count(*) from stb0 where c2 > "2021-07-21 20:22:08.248246976"') + tdSql.execute('select count(*) from stb0 where c2 < "2021-07-21 20:22:08.248246976"') + tdSql.execute('select count(*) from stb0 where c2 = "2021-07-21 20:22:08.248246976"') + tdSql.execute('select count(*) from stb0 where c2 != "2021-07-21 20:22:08.248246976"') + tdSql.execute('select count(*) from stb0 where c2 <> "2021-07-21 20:22:08.248246976"') + tdSql.execute('select count(*) from stb0 where ts between "2021-07-01 00:00:00.000000000" and "2021-07-01 00:00:00.990000000"') + tdSql.execute('select count(*) from stb0 where ts between 1625068800000000000 and 1625068801000000000') + tdSql.query('select avg(c0) from stb0 interval(5000000000b)') + tdSql.checkRows(1) + + tdSql.query('select avg(c0) from stb0 interval(100000000b)') + tdSql.checkRows(10) + + tdSql.error('select avg(c0) from stb0 interval(1b)') + tdSql.error('select avg(c0) from stb0 interval(999b)') + + tdSql.query('select avg(c0) from stb0 interval(1000b)') + tdSql.checkRows(100) + + tdSql.query('select avg(c0) from stb0 interval(1u)') + tdSql.checkRows(100) + + tdSql.query('select avg(c0) from stb0 interval(100000000b) sliding (100000000b)') + tdSql.checkRows(10) + + # query : query above sqls by taosdemo and continuously + os.system("%staosdemo -f tools/taosdemoAllTest/taosdemoTestSupportNanoQuerycsv.json -y " % binPath) + + os.system("rm -rf ./query_res*.txt*") + os.system("rm -rf tools/taosdemoAllTest/*.py.sql") + + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoQuerycsv.json b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoQuerycsv.json new file mode 100644 index 0000000000..aa95837a33 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoQuerycsv.json @@ -0,0 +1,110 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "nsdbcsv", + "query_times": 10, + "query_mode": "taosc", + "specified_table_query": { + "query_interval": 1, + "concurrent": 2, + "sqls": [ + { + "sql": "select count(*) from stb0 where ts> \"2021-07-01 00:00:00.490000000\" ;", + "result": "./query_res0.txt" + }, + { + "sql": "select count(*) from stb0 where ts > now -22d-1h-3s ;", + "result": "./query_res1.txt" + }, + { + "sql": "select count(*) from stb0 where ts < 1626918583000000000 ;", + "result": "./query_res2.txt" + }, + { + "sql": "select count(*) from stb0 where c2 <> 162687012800000000';", + "result": "./query_res3.txt" + }, + { + "sql": "select count(*) from stb0 where c2 != \"2021-07-21 20:22:08.248246976\";", + "result": "./query_res4.txt" + }, + { + "sql": "select count(*) from stb0 where ts between \"2021-07-01 00:00:00.000000000\" and \"2021-07-01 00:00:00.990000000\";", + "result": "./query_res5.txt" + }, + { + "sql":"select count(*) from stb0 group by tbname;", + "result":"./query_res6.txt" + }, + { + "sql":"select count(*) from stb0 where ts between 1625068800000000000 and 1625068801000000000;", + "result":"./query_res7.txt" + }, + { + "sql":"select avg(c0) from stb0 interval(5000000000b);", + "result":"./query_res8.txt" + }, + { + "sql":"select avg(c0) from stb0 interval(100000000b) sliding (100000000b);", + "result":"./query_res9.txt" + } + + ] + }, + "super_table_query": { + "stblname": "stb0", + "query_interval": 0, + "threads": 4, + "sqls": [ + { + "sql": "select count(*) from xxxx where ts > \"2021-07-01 00:00:00.490000000\" ;", + "result": "./query_res_tb0.txt" + }, + { + "sql":"select count(*) from xxxx where ts between \"2021-07-01 00:00:00.000000000\" and \"2021-07-01 00:00:00.990000000\" ;", + "result": "./query_res_tb1.txt" + }, + { + "sql":"select first(*) from xxxx ;", + "result": "./query_res_tb2.txt" + }, + { + "sql":"select last(*) from xxxx;", + "result": "./query_res_tb3.txt" + + }, + { + "sql":"select last_row(*) from xxxx ;", + "result": "./query_res_tb4.txt" + + }, + { + "sql":"select max(c0) from xxxx ;", + "result": "./query_res_tb5.txt" + + }, + { + "sql":"select min(c0) from xxxx ;", + "result": "./query_res_tb6.txt" + + }, + { + "sql":"select avg(c0) from xxxx ;", + "result": "./query_res_tb7.txt" + + }, + { + "sql":"select avg(c0) from xxxx interval(100000000b) sliding (100000000b) ;", + "result": "./query_res_tb8.txt" + + } + + + ] + } + } \ No newline at end of file diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoSubscribe.json b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoSubscribe.json new file mode 100644 index 0000000000..26d405b65b --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoSubscribe.json @@ -0,0 +1,32 @@ +{ + "filetype":"subscribe", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "databases": "subnsdb", + "confirm_parameter_prompt": "no", + "specified_table_query": + { + "concurrent":2, + "mode":"sync", + "interval":10000, + "restart":"yes", + "keepProgress":"yes", + "sqls": [ + { + "sql": "select * from stb0 where ts>= \"2021-07-01 00:00:00.000000000\" ;", + "result": "./subscribe_res0.txt" + }, + { + "sql": "select * from stb0 where ts > now -20d-1h-3s ;", + "result": "./subscribe_res1.txt" + }, + { + "sql": "select * from stb0 where ts < 1626918583000000000 ;", + "result": "./subscribe_res2.txt" + }] + + } +} diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanosubscribe.py b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanosubscribe.py new file mode 100644 index 0000000000..f6324577c1 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanosubscribe.py @@ -0,0 +1,123 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import time +from datetime import datetime +import subprocess + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + # get the number of subscriptions + def subTimes(self,filename): + self.filename = filename + command = 'cat %s |wc -l'% filename + times = int(subprocess.getstatusoutput(command)[1]) + return times + + # assert results + def assertCheck(self,filename,subResult,expectResult): + self.filename = filename + self.subResult = subResult + self.expectResult = expectResult + args0 = (filename, subResult, expectResult) + assert subResult == expectResult , "Queryfile:%s ,result is %s != expect: %s" % args0 + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + # clear env + os.system("ps -ef |grep 'taosdemoAllTest/taosdemoTestSupportNanoSubscribe.json' |grep -v 'grep' |awk '{print $2}'|xargs kill -9") + os.system("rm -rf ./subscribe_res*") + os.system("rm -rf ./all_subscribe_res*") + + + # insert data + os.system("%staosdemo -f tools/taosdemoAllTest/taosdemoTestNanoDatabaseInsertForSub.json" % binPath) + os.system("nohup %staosdemo -f tools/taosdemoAllTest/taosdemoTestSupportNanoSubscribe.json &" % binPath) + query_pid = int(subprocess.getstatusoutput('ps aux|grep "taosdemoAllTest/taosdemoTestSupportNanoSubscribe.json" |grep -v "grep"|awk \'{print $2}\'')[1]) + + + # merge result files + sleep(20) + os.system("cat subscribe_res0.txt* > all_subscribe_res0.txt") + os.system("cat subscribe_res1.txt* > all_subscribe_res1.txt") + os.system("cat subscribe_res2.txt* > all_subscribe_res2.txt") + + + # correct subscribeTimes testcase + subTimes0 = self.subTimes("all_subscribe_res0.txt") + self.assertCheck("all_subscribe_res0.txt",subTimes0 ,200) + + subTimes1 = self.subTimes("all_subscribe_res1.txt") + self.assertCheck("all_subscribe_res1.txt",subTimes1 ,0) + + subTimes2 = self.subTimes("all_subscribe_res2.txt") + self.assertCheck("all_subscribe_res2.txt",subTimes2 ,200) + + + # insert extral data + tdSql.execute("use subnsdb") + tdSql.execute("insert into tb0_0 values(now,100.1000,'subtest1',now-1s)") + sleep(1) + + os.system("cat subscribe_res0.txt* > all_subscribe_res0.txt") + subTimes0 = self.subTimes("all_subscribe_res0.txt") + self.assertCheck("all_subscribe_res0.txt",subTimes0 ,202) + + + + # correct data testcase + os.system("kill -9 %d" % query_pid) + sleep(3) + os.system("rm -rf ./subscribe_res*") + os.system("rm -rf ./all_subscribe*") + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 7c817e248651b23b254ba242aa7b44e4ab7b2b97 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 23 Jul 2021 10:04:42 +0800 Subject: [PATCH 028/106] Hotfix/sangshuduo/td 5479 taosdump s e for master (#6986) * [TD-5479]: taosdump -S -E does not work. * taosdemo support timestamp step from command line. --- src/kit/taosdemo/taosdemo.c | 29 +++++++++++++++++++++-------- src/kit/taosdump/taosdump.c | 2 +- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 444a59f6d7..400799b065 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -225,6 +225,7 @@ typedef struct SArguments_S { uint32_t num_of_CPR; uint32_t num_of_threads; uint64_t insert_interval; + uint64_t timestamp_step; int64_t query_times; uint32_t interlace_rows; uint32_t num_of_RPR; // num_of_records_per_req @@ -605,6 +606,7 @@ SArguments g_args = { 4, // num_of_CPR 10, // num_of_connections/thread 0, // insert_interval + DEFAULT_TIMESTAMP_STEP, // timestamp_step 1, // query_times 0, // interlace_rows; 30000, // num_of_RPR @@ -740,6 +742,9 @@ static void printHelp() { "The number of threads. Default is 10."); printf("%s%s%s%s\n", indent, "-i", indent, "The sleep time (ms) between insertion. Default is 0."); + printf("%s%s%s%s%d\n", indent, "-S", indent, + "The timestamp step between insertion. Default is %d.", + DEFAULT_TIMESTAMP_STEP); printf("%s%s%s%s\n", indent, "-r", indent, "The number of records per request. Default is 30000."); printf("%s%s%s%s\n", indent, "-t", indent, @@ -881,6 +886,14 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } arguments->insert_interval = atoi(argv[++i]); + } else if (strcmp(argv[i], "-S") == 0) { + if ((argc == i+1) || + (!isStringNumber(argv[i+1]))) { + printHelp(); + errorPrint("\n\t%s%s", argv[i], " need a number following!\n"); + exit(EXIT_FAILURE); + } + arguments->timestamp_step = atoi(argv[++i]); } else if (strcmp(argv[i], "-qt") == 0) { if ((argc == i+1) || (!isStringNumber(argv[i+1]))) { @@ -4107,7 +4120,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (timestampStep && timestampStep->type == cJSON_Number) { g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint; } else if (!timestampStep) { - g_Dbs.db[i].superTbls[j].timeStampStep = DEFAULT_TIMESTAMP_STEP; + g_Dbs.db[i].superTbls[j].timeStampStep = g_args.timestamp_step; } else { printf("ERROR: failed to read json, timestamp_step not found\n"); goto PARSE_OVER; @@ -5190,13 +5203,13 @@ static int32_t generateDataTailWithoutStb( if (g_args.disorderRatio) { retLen = generateData(data, data_type, startTime + getTSRandTail( - (int64_t) DEFAULT_TIMESTAMP_STEP, k, + g_args.timestamp_step, k, g_args.disorderRatio, g_args.disorderRange), lenOfBinary); } else { retLen = generateData(data, data_type, - startTime + (int64_t) (DEFAULT_TIMESTAMP_STEP* k), + startTime + g_args.timestamp_step * k, lenOfBinary); } @@ -5714,11 +5727,11 @@ static int32_t prepareStmtWithoutStb( if (g_args.disorderRatio) { *bind_ts = startTime + getTSRandTail( - (int64_t)DEFAULT_TIMESTAMP_STEP, k, + g_args.timestamp_step, k, g_args.disorderRatio, g_args.disorderRange); } else { - *bind_ts = startTime + (int64_t)(DEFAULT_TIMESTAMP_STEP * k); + *bind_ts = startTime + g_args.timestamp_step * k; } bind->buffer_length = sizeof(int64_t); bind->buffer = bind_ts; @@ -6109,7 +6122,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { insertRows = g_args.num_of_DPT; interlaceRows = g_args.interlace_rows; maxSqlLen = g_args.max_sql_len; - nTimeStampStep = DEFAULT_TIMESTAMP_STEP; + nTimeStampStep = g_args.timestamp_step; insert_interval = g_args.insert_interval; } @@ -6369,7 +6382,7 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { SSuperTable* superTblInfo = pThreadInfo->superTblInfo; uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len; int64_t timeStampStep = - superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP; + superTblInfo?superTblInfo->timeStampStep:g_args.timestamp_step; int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; verbosePrint("%s() LN%d insertRows=%"PRId64"\n", @@ -8181,7 +8194,7 @@ static void setParaFromArg() { } tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp, "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE); - g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP; + g_Dbs.db[0].superTbls[0].timeStampStep = g_args.timestamp_step; g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT; g_Dbs.db[0].superTbls[0].maxSqlLen = g_args.max_sql_len; diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index d6dec54af8..17a4d7d19f 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -600,7 +600,7 @@ static void parse_timestamp( tmpEpoch = atoll(tmp); } - sprintf(argv[i], "%"PRId64"", tmpEpoch); + sprintf(argv[i+1], "%"PRId64"", tmpEpoch); debugPrint("%s() LN%d, tmp is: %s, argv[%d]: %s\n", __func__, __LINE__, tmp, i, argv[i]); free(tmp); From c133eb1cfea4803c8b851eb4540de82f2bf5daf8 Mon Sep 17 00:00:00 2001 From: wenzhouwww Date: Fri, 23 Jul 2021 10:07:02 +0800 Subject: [PATCH 029/106] [TD-5369] modify an error about insert --- .../tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py index 010308d037..76b5871de6 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py @@ -113,10 +113,6 @@ class TDTestCase: tdSql.checkDataType(3, 1, "TIMESTAMP") tdSql.query("select count(*) from stb0 where ts > \"2021-07-01 00:00:00.490000000\"") tdSql.checkData(0, 0, 5000) - tdSql.query("select count(*) from stb0 where ts >now -22d-1h-3s") - tdSql.checkData(0, 0, 10000) - tdSql.query("select count(*) from stb0 where ts >now -22d-1h-3s") - tdSql.checkData(0, 0, 10000) tdSql.query("select count(*) from stb0 where ts < 1626918583000000000") tdSql.checkData(0, 0, 10000) From ea0cc35af4d2db6f92f0a0b74e511deb01d2633c Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Fri, 23 Jul 2021 14:49:15 +0800 Subject: [PATCH 030/106] [TD-5074]test operator --- tests/pytest/fulltest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index e452ed3de4..eb068b6585 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -241,6 +241,7 @@ python3 ./test.py -f query/queryStateWindow.py python3 ./test.py -f query/nestedQuery/queryWithOrderLimit.py python3 ./test.py -f query/nestquery_last_row.py python3 ./test.py -f query/queryCnameDisplay.py +python3 ./test.py -f query/operator.py python3 test.py -f query/nestedQuery/queryWithSpread.py #stream From 11d5b3231d91d78609a0141ef3f3200579f1813d Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Fri, 23 Jul 2021 14:49:27 +0800 Subject: [PATCH 031/106] [TD-5074]test operator --- tests/pytest/query/operator.py | 539 +++++++++++++++++++++++++++++++++ 1 file changed, 539 insertions(+) create mode 100644 tests/pytest/query/operator.py diff --git a/tests/pytest/query/operator.py b/tests/pytest/query/operator.py new file mode 100644 index 0000000000..b94d5fa3b3 --- /dev/null +++ b/tests/pytest/query/operator.py @@ -0,0 +1,539 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +import random +import time + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1600000000000 + self.num = 10 + + def run(self): + tdSql.prepare() + # test case for https://jira.taosdata.com:18080/browse/TD-5074 + + startTime = time.time() + + tdSql.execute('''create stable stable_1 + (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint, + q_bool bool , q_binary binary(20) , q_nchar nchar(20) , + q_float float , q_double double , q_ts timestamp) + tags(loc nchar(20) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, + t_bool bool , t_binary binary(20) , t_nchar nchar(20) , + t_float float , t_double double , t_ts timestamp);''') + tdSql.execute('''create stable stable_2 + (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint, + q_bool bool , q_binary binary(20) , q_nchar nchar(20) , + q_float float , q_double double , q_ts timestamp) + tags(loc nchar(20) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, + t_bool bool , t_binary binary(20) , t_nchar nchar(20) , + t_float float , t_double double , t_ts timestamp);''') + tdSql.execute('''create table table_0 using stable_1 + tags('table_0' , '0' , '0' , '0' , '0' , 0 , '0' , '0' , '0' , '0' ,'0')''') + tdSql.execute('''create table table_1 using stable_1 + tags('table_1' , '2147483647' , '9223372036854775807' , '32767' , '127' , 1 , + 'binary1' , 'nchar1' , '1' , '11' , \'1999-09-09 09:09:09.090\')''') + tdSql.execute('''create table table_2 using stable_1 + tags('table_2' , '-2147483647' , '-9223372036854775807' , '-32767' , '-127' , false , + 'binary2' , 'nchar2nchar2' , '-2.2' , '-22.22' , \'2099-09-09 09:09:09.090\')''') + tdSql.execute('''create table table_3 using stable_1 + tags('table_3' , '3' , '3' , '3' , '3' , true , 'binary3' , 'nchar3' , '33.33' , '3333.3333' , '0')''') + tdSql.execute('''create table table_4 using stable_1 + tags('table_4' , '4' , '4' , '4' , '4' , false , 'binary4' , 'nchar4' , '-444.444' , '-444444.444444' , '0')''') + tdSql.execute('''create table table_5 using stable_1 + tags('table_5' , '5' , '5' , '5' , '5' , true , 'binary5' , 'nchar5' , '5555.5555' , '55555555.55555555' , '0')''') + tdSql.execute('''create table table_21 using stable_2 + tags('table_5' , '5' , '5' , '5' , '5' , true , 'binary5' , 'nchar5' , '5555.5555' , '55555555.55555555' , '0')''') + #regular table + tdSql.execute('''create table regular_table_1 + (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint, + q_bool bool , q_binary binary(20) , q_nchar nchar(20) , + q_float float , q_double double , q_ts timestamp) ;''') + + for i in range(self.num): + tdSql.execute('''insert into table_0 values(%d, %d, %d, %d, %d, 0, 'binary.%s', 'nchar.%s', %f, %f, %d)''' + % (self.ts + i, i, i, i, i, i, i, i, i, self.ts + i)) + tdSql.execute('''insert into table_1 values(%d, %d, %d, %d, %d, 1, 'binary1.%s', 'nchar1.%s', %f, %f, %d)''' + % (self.ts + i, 2147483647-i, 9223372036854775807-i, 32767-i, 127-i, + i, i, random.random(), random.random(), 1262304000001 + i)) + tdSql.execute('''insert into table_2 values(%d, %d, %d, %d, %d, true, 'binary2.%s', 'nchar2nchar2.%s', %f, %f, %d)''' + % (self.ts + i, -2147483647+i, -9223372036854775807+i, -32767+i, -127+i, + i, i, random.uniform(-1,0), random.uniform(-1,0), 1577836800001 + i)) + tdSql.execute('''insert into table_3 values(%d, %d, %d, %d, %d, false, 'binary3.%s', 'nchar3.%s', %f, %f, %d)''' + % (self.ts + i, random.randint(-2147483647, 2147483647), + random.randint(-9223372036854775807, 9223372036854775807), random.randint(-32767, 32767), + random.randint(-127, 127), random.randint(-100, 100), random.randint(-10000, 10000), + random.uniform(-100000,100000), random.uniform(-1000000000,1000000000), self.ts + i)) + tdSql.execute('''insert into table_4 values(%d, %d, %d, %d, %d, true, 'binary4.%s', 'nchar4.%s', %f, %f, %d)''' + % (self.ts + i, i, i, i, i, i, i, i, i, self.ts + i)) + tdSql.execute('''insert into table_5 values(%d, %d, %d, %d, %d, false, 'binary5.%s', 'nchar5.%s', %f, %f, %d)''' + % (self.ts + i, i, i, i, i, i, i, i, i, self.ts + i)) + tdSql.execute('''insert into table_21 values(%d, %d, %d, %d, %d, false, 'binary5.%s', 'nchar5.%s', %f, %f, %d)''' + % (self.ts + i, i, i, i, i, i, i, i, i, self.ts + i)) + + tdSql.execute('''insert into regular_table_1 values(%d, %d, %d, %d, %d, 0, 'binary.%s', 'nchar.%s', %f, %f, %d)''' + % (self.ts + i, i, i, i, i, i, i, i, i, self.ts + i)) + tdSql.execute('''insert into regular_table_1 values(%d, %d, %d, %d, %d, 1, 'binary1.%s', 'nchar1.%s', %f, %f, %d)''' + % (self.ts + 100 + i, 2147483647-i, 9223372036854775807-i, 32767-i, 127-i, + i, i, random.random(), random.random(), 1262304000001 + i)) + tdSql.execute('''insert into regular_table_1 values(%d, %d, %d, %d, %d, true, 'binary2.%s', 'nchar2nchar2.%s', %f, %f, %d)''' + % (self.ts + 200 + i, -2147483647+i, -9223372036854775807+i, -32767+i, -127+i, + i, i, random.uniform(-1,0), random.uniform(-1,0), 1577836800001 + i)) + tdSql.execute('''insert into regular_table_1 values(%d, %d, %d, %d, %d, false, 'binary3.%s', 'nchar3.%s', %f, %f, %d)''' + % (self.ts + 300 + i, random.randint(-2147483647, 2147483647), + random.randint(-9223372036854775807, 9223372036854775807), random.randint(-32767, 32767), + random.randint(-127, 127), random.randint(-100, 100), random.randint(-10000, 10000), + random.uniform(-100000,100000), random.uniform(-1000000000,1000000000), self.ts + i)) + tdSql.execute('''insert into regular_table_1 values(%d, %d, %d, %d, %d, true, 'binary4.%s', 'nchar4.%s', %f, %f, %d)''' + % (self.ts + 400 + i, i, i, i, i, i, i, i, i, self.ts + i)) + tdSql.execute('''insert into regular_table_1 values(%d, %d, %d, %d, %d, false, 'binary5.%s', 'nchar5.%s', %f, %f, %d)''' + % (self.ts + 500 + i, i, i, i, i, i, i, i, i, self.ts + i)) + + tdLog.info("========== operator=1(OP_TableScan) ==========") + tdLog.info("========== operator=7(OP_Project) ==========") + sql = '''select * from stable_1''' + tdSql.query(sql) + tdSql.checkRows(6*self.num) + sql = '''select * from regular_table_1''' + tdSql.query(sql) + tdSql.checkRows(6*self.num) + + tdLog.info("========== operator=14(OP_MultiTableAggregate ) ==========") + sql = '''select last_row(*) from stable_1;''' + tdSql.query(sql) + tdSql.checkData(0,1,self.num-1) + + tdLog.info("========== operator=6(OP_Aggregate) ==========") + sql = '''select last_row(*) from regular_table_1;''' + tdSql.query(sql) + tdSql.checkData(0,1,self.num-1) + + tdLog.info("========== operator=9(OP_Limit) ==========") + sql = '''select * from stable_1 where loc = 'table_0' limit 5;''' + tdSql.query(sql) + tdSql.checkRows(5) + sql = '''select last_row(*) from (select * from stable_1 where loc = 'table_0');''' + tdSql.query(sql) + tdSql.checkRows(1) + + sql = '''select * from regular_table_1 ;''' + tdSql.query(sql) + tdSql.checkRows(6*self.num) + sql = '''select last_row(*) from (select * from regular_table_1);''' + tdSql.query(sql) + tdSql.checkRows(1) + tdSql.checkData(0,1,self.num-1) + + + sql = '''select last_row(*) from + ((select * from table_0) union all + (select * from table_1) union all + (select * from table_2));''' + tdSql.error(sql) + + tdLog.info("========== operator=16(OP_DummyInput) ==========") + sql = '''select last_row(*) from + ((select last_row(*) from table_0) union all + (select last_row(*) from table_1) union all + (select last_row(*) from table_2));''' + tdSql.error(sql) + + sql = '''select last_row(*) from + ((select * from table_0 limit 5 offset 5) union all + (select * from table_1 limit 5 offset 5) union all + (select * from regular_table_1 limit 5 offset 5));''' + tdSql.error(sql) + + tdLog.info("========== operator=10(OP_SLimit) ==========") + sql = '''select count(*) from stable_1 group by loc slimit 3 soffset 2 ;''' + tdSql.query(sql) + tdSql.checkRows(3) + + sql = '''select last_row(*) from + ((select * from table_0) union all + (select * from table_1) union all + (select * from table_2));''' + tdSql.error(sql) + + tdLog.info("========== operator=20(OP_Distinct) ==========") + tdLog.info("========== operator=4(OP_TagScan) ==========") + sql = '''select distinct(t_bool) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(2) + sql = '''select distinct(loc) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(6) + sql = '''select distinct(t_int) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(6) + sql = '''select distinct(t_bigint) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(6) + sql = '''select distinct(t_smallint) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(6) + sql = '''select distinct(t_tinyint) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(6) + sql = '''select distinct(t_binary) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(6) + sql = '''select distinct(t_nchar) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(6) + sql = '''select distinct(t_float) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(6) + sql = '''select distinct(t_double) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(6) + sql = '''select distinct(t_ts) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(3) + sql = '''select distinct(tbname) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(6) + + tdLog.info("========== operator=2(OP_DataBlocksOptScan) ==========") + sql = '''select last(q_int),first(q_int) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_bigint),first(q_bigint) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_smallint),first(q_smallint) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_tinyint),first(q_tinyint) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_bool),first(q_bool) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_binary),first(q_binary) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_nchar),first(q_nchar) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_float),first(q_float) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_double),first(q_double) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_ts),first(q_ts) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_int),last(q_bigint), last(q_smallint),last(q_tinyint),last(q_bool),last(q_binary),last(q_nchar), + last(q_float),last(q_double),last(q_ts),first(q_int),first(q_bigint),first(q_smallint),first(q_tinyint), + first(q_bool),first(q_binary),first(q_nchar),first(q_float),first(q_float),first(q_double),first(q_ts) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_int),last(q_bigint), last(q_smallint),last(q_tinyint),last(q_bool),last(q_binary),last(q_nchar), + last(q_float),last(q_double),last(q_ts),first(q_int),first(q_bigint),first(q_smallint),first(q_tinyint),first(q_bool), + first(q_binary),first(q_nchar),first(q_float),first(q_float),first(q_double),first(q_ts) from regular_table_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + + tdLog.info("========== operator=8(OP_Groupby) ==========") + sql = '''select stddev(q_int) from table_0 group by q_int;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select stddev(q_int),stddev(q_bigint),stddev(q_smallint),stddev(q_tinyint),stddev(q_float),stddev(q_double) from stable_1 group by q_int;''' + tdSql.query(sql) + sql = '''select stddev(q_int),stddev(q_bigint),stddev(q_smallint),stddev(q_tinyint),stddev(q_float),stddev(q_double) from table_1 group by q_bigint;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select stddev(q_int),stddev(q_bigint),stddev(q_smallint),stddev(q_tinyint),stddev(q_float),stddev(q_double) from regular_table_1 group by q_smallint;''' + tdSql.query(sql) + + tdLog.info("========== operator=11(OP_TimeWindow) ==========") + sql = '''select last(q_int) from table_0 interval(1m);''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_int),last(q_bigint), last(q_smallint),last(q_tinyint), + first(q_int),first(q_bigint),first(q_smallint),first(q_tinyint) from table_1 interval(1m);''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_int),last(q_bigint), last(q_smallint),last(q_tinyint), + first(q_int),first(q_bigint),first(q_smallint),first(q_tinyint) from stable_1 interval(1m);''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_int),last(q_bigint), last(q_smallint),last(q_tinyint), + first(q_int),first(q_bigint),first(q_smallint),first(q_tinyint) from regular_table_1 interval(1m);''' + tdSql.query(sql) + tdSql.checkRows(1) + + tdLog.info("========== operator=12(OP_SessionWindow) ==========") + sql = '''select count(*) from table_1 session(ts,1s);''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select count(*) from regular_table_1 session(ts,1s);''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select count(*),sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from table_1 session(ts,1s);''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select count(*),sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from regular_table_1 session(ts,1s);''' + tdSql.query(sql) + tdSql.checkRows(1) + + tdLog.info("========== operator=13(OP_Fill) ==========") + sql = '''select sum(q_int) from table_0 + where ts >='1970-10-01 00:00:00' and ts <=now interval(1n) fill(NULL);''' + tdSql.query(sql) + tdSql.checkData(0,1,'None') + sql = '''select sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from stable_1 where ts >='1970-10-01 00:00:00' and ts <=now interval(1n) fill(NULL);''' + tdSql.query(sql) + tdSql.checkData(0,1,'None') + sql = '''select sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from regular_table_1 where ts >='1970-10-01 00:00:00' and ts <=now interval(1n) fill(NULL);''' + tdSql.query(sql) + tdSql.checkData(0,1,'None') + sql = '''select sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from table_0 where ts >='1970-10-01 00:00:00' and ts <=now interval(1n) fill(NULL);''' + tdSql.query(sql) + tdSql.checkData(0,1,'None') + #TD-5190 + sql = '''select sum(q_tinyint),stddev(q_float) from stable_1 + where ts >='1970-10-01 00:00:00' and ts <=now interval(1n) fill(NULL);''' + tdSql.query(sql) + tdSql.checkData(0,1,'None') + + tdLog.info("========== operator=15(OP_MultiTableTimeInterval) ==========") + sql = '''select avg(q_int) from stable_1 where ts=0;''' + tdSql.query(sql) + tdSql.checkData(0,0,'table_0') + sql = '''select loc, sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from stable_1 group by loc having sum(q_int)>=0;''' + tdSql.query(sql) + tdSql.checkData(0,0,'table_0') + sql = '''select loc, sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from stable_1 group by loc having avg(q_int)>=0;''' + tdSql.query(sql) + tdSql.checkData(0,0,'table_0') + sql = '''select loc, sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from stable_1 group by loc having min(q_int)>=0;''' + tdSql.query(sql) + tdSql.checkData(0,0,'table_0') + sql = '''select loc, sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from stable_1 group by loc having max(q_int)>=0;''' + tdSql.query(sql) + tdSql.checkData(0,0,'table_0') + sql = '''select loc, sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from stable_1 group by loc having first(q_int)>=0;''' + tdSql.query(sql) + tdSql.checkData(0,0,'table_0') + sql = '''select loc, sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from stable_1 group by loc having last(q_int)>=0;''' + tdSql.query(sql) + tdSql.checkData(0,0,'table_0') + + tdLog.info("========== operator=21(OP_Join) ==========") + sql = '''select t1.q_int,t2.q_int from + (select ts,q_int from table_1) t1 , (select ts,q_int from table_2) t2 + where t2.ts = t1.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select t1.*,t2.* from + (select * from table_1) t1 , (select * from table_2) t2 + where t2.ts = t1.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select t1.*,t2.* from + (select * from regular_table_1) t1 , (select * from table_0) t2 + where t2.ts = t1.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select t1.*,t2.* from + (select * from stable_1) t1 , (select * from table_2) t2 + where t2.ts = t1.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select t1.*,t2.* from + (select * from regular_table_1) t1 , (select * from stable_1) t2 + where t2.ts = t1.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select t1.*,t2.*,t3.* from + (select * from regular_table_1) t1 , (select * from stable_1) t2, (select * from table_0) t3 + where t2.ts = t1.ts and t3.ts = t1.ts and t2.ts = t3.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + + tdLog.info("========== operator=22(OP_StateWindow) ==========") + sql = '''select avg(q_int),sum(q_smallint) from table_1 state_window(q_int);''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from table_1 state_window(q_bigint);''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from regular_table_1 state_window(q_smallint);''' + tdSql.query(sql) + tdSql.checkRows(6*self.num) + + endTime = time.time() + print("total time %ds" % (endTime - startTime)) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 8c62d84a3d5f1d52191bcf4a99a9b4d75b21eca9 Mon Sep 17 00:00:00 2001 From: wenzhouwww Date: Fri, 23 Jul 2021 20:05:08 +0800 Subject: [PATCH 032/106] [TD-5369] add test case about taosdemo params 'time_step' --- tests/pytest/fulltest.sh | 1 + .../taosdemoAllTest/taosdemoInsertMSDB.json | 63 ++++++++++ .../taosdemoAllTest/taosdemoInsertNanoDB.json | 63 ++++++++++ .../taosdemoAllTest/taosdemoInsertUSDB.json | 63 ++++++++++ .../taosdemoTestInsertTime_step.py | 115 ++++++++++++++++++ .../taosdemoTestNanoCreateDB.sql | 7 -- .../taosdemoTestSupportNanoInsert.py | 36 +++--- 7 files changed, 324 insertions(+), 24 deletions(-) create mode 100644 tests/pytest/tools/taosdemoAllTest/taosdemoInsertMSDB.json create mode 100644 tests/pytest/tools/taosdemoAllTest/taosdemoInsertNanoDB.json create mode 100644 tests/pytest/tools/taosdemoAllTest/taosdemoInsertUSDB.json create mode 100644 tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertTime_step.py delete mode 100644 tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoCreateDB.sql diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index c9dd6d8e79..ab7dedc959 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -368,6 +368,7 @@ python3 test.py -f alter/alter_create_exception.py python3 test.py -f tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py python3 test.py -f tools/taosdemoAllTest/taosdemoTestSupportNanoQuery.py python3 test.py -f tools/taosdemoAllTest/taosdemoTestSupportNanosubscribe.py +python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertTime_step.py python3 ./test.py -f insert/flushwhiledrop.py #======================p4-end=============== diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoInsertMSDB.json b/tests/pytest/tools/taosdemoAllTest/taosdemoInsertMSDB.json new file mode 100644 index 0000000000..49ab6f3a43 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoInsertMSDB.json @@ -0,0 +1,63 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 10, + "thread_count_create_tbl": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "testdb3", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 36, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "tb0_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1000, + "start_timestamp": "2021-07-01 00:00:00.000", + "sample_format": "", + "sample_file": "", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":3}, {"type": "BINARY", "len": 16, "count":2}, {"type": "BINARY", "len": 32, "count":2}, + {"type": "TIMESTAMP"}, {"type": "BIGINT", "count":3},{"type": "FLOAT", "count":1},{"type": "SMALLINT", "count":1},{"type": "TINYINT", "count":1}, + {"type": "BOOL"},{"type": "NCHAR","len":16}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5},{"type": "NCHAR","len":16, "count":1}] + }] + }] +} + diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoInsertNanoDB.json b/tests/pytest/tools/taosdemoAllTest/taosdemoInsertNanoDB.json new file mode 100644 index 0000000000..9a35df917d --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoInsertNanoDB.json @@ -0,0 +1,63 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 10, + "thread_count_create_tbl": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "testdb1", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ns", + "keep": 36, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "tb0_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1000, + "start_timestamp": "2021-07-01 00:00:00.000", + "sample_format": "", + "sample_file": "", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":3}, {"type": "BINARY", "len": 16, "count":2}, {"type": "BINARY", "len": 32, "count":2}, + {"type": "TIMESTAMP"}, {"type": "BIGINT", "count":3},{"type": "FLOAT", "count":1},{"type": "SMALLINT", "count":1},{"type": "TINYINT", "count":1}, + {"type": "BOOL"},{"type": "NCHAR","len":16}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5},{"type": "NCHAR","len":16, "count":1}] + }] + }] +} + diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoInsertUSDB.json b/tests/pytest/tools/taosdemoAllTest/taosdemoInsertUSDB.json new file mode 100644 index 0000000000..631179dbae --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoInsertUSDB.json @@ -0,0 +1,63 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 10, + "thread_count_create_tbl": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "testdb2", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "us", + "keep": 36, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "tb0_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1000, + "start_timestamp": "2021-07-01 00:00:00.000", + "sample_format": "", + "sample_file": "", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":3}, {"type": "BINARY", "len": 16, "count":2}, {"type": "BINARY", "len": 32, "count":2}, + {"type": "TIMESTAMP"}, {"type": "BIGINT", "count":3},{"type": "FLOAT", "count":1},{"type": "SMALLINT", "count":1},{"type": "TINYINT", "count":1}, + {"type": "BOOL"},{"type": "NCHAR","len":16}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5},{"type": "NCHAR","len":16, "count":1}] + }] + }] +} + diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertTime_step.py b/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertTime_step.py new file mode 100644 index 0000000000..7b3b865df9 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertTime_step.py @@ -0,0 +1,115 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + # insert: create one or mutiple tables per sql and insert multiple rows per sql + + # check the params of taosdemo about time_step is nano + os.system("%staosdemo -f tools/taosdemoAllTest/taosdemoInsertNanoDB.json -y " % binPath) + tdSql.execute("use testdb1") + tdSql.query("show stables") + tdSql.checkData(0, 4, 100) + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 100) + tdSql.query("select count(*) from tb0_0") + tdSql.checkData(0, 0, 100) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 10000) + tdSql.query("describe stb0") + tdSql.getData(9, 1) + tdSql.checkDataType(9, 1,"TIMESTAMP") + tdSql.query("select last(ts) from stb0") + tdSql.checkData(0, 0,"2021-07-01 00:00:00.000099000") + + # check the params of taosdemo about time_step is us + os.system("%staosdemo -f tools/taosdemoAllTest/taosdemoInsertUSDB.json -y " % binPath) + tdSql.execute("use testdb2") + tdSql.query("show stables") + tdSql.checkData(0, 4, 100) + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 100) + tdSql.query("select count(*) from tb0_0") + tdSql.checkData(0, 0, 100) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 10000) + tdSql.query("describe stb0") + tdSql.getData(9, 1) + tdSql.checkDataType(9, 1,"TIMESTAMP") + tdSql.query("select last(ts) from stb0") + tdSql.checkData(0, 0,"2021-07-01 00:00:00.099000") + + # check the params of taosdemo about time_step is ms + os.system("%staosdemo -f tools/taosdemoAllTest/taosdemoInsertMSDB.json -y " % binPath) + tdSql.execute("use testdb3") + tdSql.query("show stables") + tdSql.checkData(0, 4, 100) + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 100) + tdSql.query("select count(*) from tb0_0") + tdSql.checkData(0, 0, 100) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 10000) + tdSql.query("describe stb0") + tdSql.checkDataType(9, 1,"TIMESTAMP") + tdSql.query("select last(ts) from stb0") + tdSql.checkData(0, 0,"2021-07-01 00:01:39.000") + + + os.system("rm -rf ./res.txt") + os.system("rm -rf ./*.py.sql") + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoCreateDB.sql b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoCreateDB.sql deleted file mode 100644 index e79e09592c..0000000000 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoCreateDB.sql +++ /dev/null @@ -1,7 +0,0 @@ -drop database if exists nsdbsql; -create database nsdbsql precision "ns" keep 36 days 6 update 1; -use nsdbsql; -CREATE STABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupdId int); -CREATE TABLE d1001 USING meters TAGS ("Beijing.Chaoyang", 2); -INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 219, 0.32); -INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 85, 32, 0.76); diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py index 76b5871de6..88a917da85 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py @@ -64,8 +64,7 @@ class TDTestCase: tdSql.query("select count(*) from stb0") tdSql.checkData(0, 0, 10000000) tdSql.query("describe stb0") - tdSql.getData(8, 1) - tdSql.checkDataType(8, 1,"TIMESTAMP") + tdSql.checkDataType(9, 1,"TIMESTAMP") tdSql.query("select last(ts) from stb0") tdSql.getData(0, 0) @@ -79,7 +78,7 @@ class TDTestCase: tdSql.checkData(0, 0, 10000000) # check c8 is an nano timestamp tdSql.query("describe stb1") - tdSql.checkDataType(8, 1,"TIMESTAMP") + tdSql.checkDataType(9, 1,"TIMESTAMP") # check insert timestamp_step is nano_second tdSql.query("select last(ts) from stb1") tdSql.checkData(0, 0,"2021-07-01 00:01:39.990000000") @@ -99,7 +98,7 @@ class TDTestCase: tdSql.checkData(0, 0, 100000) # check c8 is an nano timestamp tdSql.query("describe stb0") - tdSql.checkDataType(8,1,"TIMESTAMP") + tdSql.checkDataType(9,1,"TIMESTAMP") # insert by csv files and timetamp is long int , strings in ts and cols @@ -118,16 +117,6 @@ class TDTestCase: os.system("rm -rf ./insert_res.txt") os.system("rm -rf tools/taosdemoAllTest/taosdemoTestSupportNano*.py.sql") - - - - - - - - - - # taosdemo test insert with command and parameter , detals show taosdemo --help os.system("%staosdemo -u root -P taosdata -p 6030 -h vm84 -a 1 -m pre -n 10 -T 20 -t 60 -o res.txt -y " % binPath) @@ -140,13 +129,26 @@ class TDTestCase: # check taosdemo -s - os.system("%staosdemo -s tools/taosdemoAllTest/taosdemoTestNanoCreateDB.sql -y " % binPath) + sqls_ls = ['drop database if exists nsdbsql;','create database nsdbsql precision "ns" keep 36 days 6 update 1;', + 'use nsdbsql;','CREATE STABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupdId int);', + 'CREATE TABLE d1001 USING meters TAGS ("Beijing.Chaoyang", 2);', + 'INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 219, 0.32);', + 'INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 85, 32, 0.76);'] + + with open("./taosdemoTestNanoCreateDB.sql",mode ="a" ) as sql_files: + for sql in sqls_ls: + sql_files.write(sql+"\n") + sql_files.close() + + sleep(10) + + os.system("%staosdemo -s taosdemoTestNanoCreateDB.sql -y " % binPath) tdSql.query("select count(*) from nsdbsql.meters") tdSql.checkData(0, 0, 2) - - os.system("rm -rf ./res.txt") + os.system("rm -rf ./*.py.sql") + os.system("rm -rf ./taosdemoTestNanoCreateDB.sql") def stop(self): From fa756766139563ea0a86dbbc417d90bf1474a460 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 23 Jul 2021 18:43:42 +0800 Subject: [PATCH 033/106] [TD-5512]: fix arbitrator offline caused by vnode destroying --- src/sync/inc/syncTcp.h | 2 +- src/sync/src/syncArbitrator.c | 6 +++--- src/sync/src/syncMain.c | 9 +++++---- src/sync/src/syncTcp.c | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/sync/inc/syncTcp.h b/src/sync/inc/syncTcp.h index b322c3440c..e2e5234d39 100644 --- a/src/sync/inc/syncTcp.h +++ b/src/sync/inc/syncTcp.h @@ -25,7 +25,7 @@ typedef struct { uint32_t serverIp; int16_t port; int32_t bufferSize; - void (*processBrokenLink)(int64_t handleId); + void (*processBrokenLink)(int64_t handleId, int32_t closedByApp); int32_t (*processIncomingMsg)(int64_t handleId, void *buffer); void (*processIncomingConn)(SOCKET fd, uint32_t ip); } SPoolInfo; diff --git a/src/sync/src/syncArbitrator.c b/src/sync/src/syncArbitrator.c index fdbef4c9f8..08ebe7ae3e 100644 --- a/src/sync/src/syncArbitrator.c +++ b/src/sync/src/syncArbitrator.c @@ -30,7 +30,7 @@ extern void syncProcessTestMsg(SSyncMsg *pMsg, SOCKET connFd); static void arbSignalHandler(int32_t signum, void *sigInfo, void *context); static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); -static void arbProcessBrokenLink(int64_t rid); +static void arbProcessBrokenLink(int64_t rid, int32_t closedByApp); static int32_t arbProcessPeerMsg(int64_t rid, void *buffer); static tsem_t tsArbSem; static void * tsArbTcpPool; @@ -147,10 +147,10 @@ static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp) { return; } -static void arbProcessBrokenLink(int64_t rid) { +static void arbProcessBrokenLink(int64_t rid, int32_t closedByApp) { SNodeConn *pNode = (SNodeConn *)rid; - sDebug("%s, TCP link is broken since %s, close connection", pNode->id, strerror(errno)); + sDebug("%s, TCP link is broken since %s, closedByApp:%d", pNode->id, strerror(errno), closedByApp); tfree(pNode); } diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 821d526d5e..d93fcdc3ca 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -43,7 +43,7 @@ static void syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer); static void syncRecoverFromMaster(SSyncPeer *pPeer); static void syncCheckPeerConnection(void *param, void *tmrId); static int32_t syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId); -static void syncProcessBrokenLink(int64_t rid); +static void syncProcessBrokenLink(int64_t rid, int32_t closedByApp); static int32_t syncProcessPeerMsg(int64_t rid, void *buffer); static void syncProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); static void syncRemovePeer(SSyncPeer *pPeer); @@ -1308,7 +1308,7 @@ static void syncProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp) { pthread_mutex_unlock(&pNode->mutex); } -static void syncProcessBrokenLink(int64_t rid) { +static void syncProcessBrokenLink(int64_t rid, int32_t closedByApp) { SSyncPeer *pPeer = syncAcquirePeer(rid); if (pPeer == NULL) return; @@ -1316,9 +1316,10 @@ static void syncProcessBrokenLink(int64_t rid) { pthread_mutex_lock(&pNode->mutex); - sDebug("%s, TCP link is broken since %s, pfd:%d sfd:%d", pPeer->id, strerror(errno), pPeer->peerFd, pPeer->syncFd); + sDebug("%s, TCP link is broken since %s, pfd:%d sfd:%d closedByApp:%d", + pPeer->id, strerror(errno), pPeer->peerFd, pPeer->syncFd, closedByApp); pPeer->peerFd = -1; - if (pPeer->isArb) { + if (!closedByApp && pPeer->isArb) { tsArbOnline = 0; } diff --git a/src/sync/src/syncTcp.c b/src/sync/src/syncTcp.c index 3ad9e9bba0..e0deb64e6c 100644 --- a/src/sync/src/syncTcp.c +++ b/src/sync/src/syncTcp.c @@ -177,7 +177,7 @@ static void taosProcessBrokenLink(SConnObj *pConn) { SPoolInfo * pInfo = &pPool->info; if (pConn->closedByApp == 0) shutdown(pConn->fd, SHUT_WR); - (*pInfo->processBrokenLink)(pConn->handleId); + (*pInfo->processBrokenLink)(pConn->handleId, pConn->closedByApp); pThread->numOfFds--; epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pConn->fd, NULL); From 763b4f3b1d91c905a1b4ebd37c66de9f9f0efe9c Mon Sep 17 00:00:00 2001 From: Zhiyu Yang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Fri, 23 Jul 2021 23:04:29 +0800 Subject: [PATCH 034/106] Release/td 5493 (#6993) * merge jdbc update on develop into master * remove useless test cases * change * change * remove uneccesary test case * change --- src/connector/jdbc/CMakeLists.txt | 2 +- src/connector/jdbc/deploy-pom.xml | 2 +- src/connector/jdbc/pom.xml | 35 +- .../com/taosdata/jdbc/AbstractConnection.java | 64 +- .../jdbc/AbstractDatabaseMetaData.java | 4 +- .../com/taosdata/jdbc/AbstractDriver.java | 4 +- .../jdbc/AbstractParameterMetaData.java | 2 - .../com/taosdata/jdbc/AbstractStatement.java | 44 +- .../jdbc/DatabaseMetaDataResultSet.java | 12 +- .../com/taosdata/jdbc/EmptyResultSet.java | 10 +- .../com/taosdata/jdbc/TSDBConnection.java | 2 +- .../java/com/taosdata/jdbc/TSDBConstants.java | 9 +- .../taosdata/jdbc/TSDBDatabaseMetaData.java | 4 +- .../java/com/taosdata/jdbc/TSDBDriver.java | 11 +- .../java/com/taosdata/jdbc/TSDBError.java | 6 +- .../com/taosdata/jdbc/TSDBErrorNumbers.java | 7 +- .../com/taosdata/jdbc/TSDBJNIConnector.java | 47 +- .../taosdata/jdbc/TSDBPreparedStatement.java | 73 +-- .../java/com/taosdata/jdbc/TSDBResultSet.java | 4 +- .../taosdata/jdbc/TSDBResultSetBlockData.java | 20 +- .../taosdata/jdbc/TSDBResultSetMetaData.java | 5 +- .../taosdata/jdbc/TSDBResultSetRowData.java | 131 ++-- .../java/com/taosdata/jdbc/TSDBSubscribe.java | 3 - .../taosdata/jdbc/enums/TimestampFormat.java | 7 + .../jdbc/enums/TimestampPrecision.java | 8 + .../taosdata/jdbc/rs/RestfulConnection.java | 2 +- .../jdbc/rs/RestfulPreparedStatement.java | 27 +- .../taosdata/jdbc/rs/RestfulResultSet.java | 199 +++--- .../jdbc/rs/RestfulResultSetMetaData.java | 17 +- .../taosdata/jdbc/rs/RestfulStatement.java | 7 - .../jdbc/utils/HttpClientPoolUtil.java | 29 +- .../java/com/taosdata/jdbc/utils/OSUtils.java | 6 +- .../jdbc/utils/SqlSyntaxValidator.java | 30 +- .../com/taosdata/jdbc/utils/TaosInfo.java | 6 +- .../java/com/taosdata/jdbc/utils/Utils.java | 133 ++-- .../java/com/taosdata/jdbc/StatementTest.java | 108 ---- .../com/taosdata/jdbc/TSDBConnectionTest.java | 8 +- .../taosdata/jdbc/TSDBJNIConnectorTest.java | 75 ++- .../jdbc/TSDBParameterMetaDataTest.java | 1 - .../jdbc/TSDBPreparedStatementTest.java | 326 +++++++++- .../com/taosdata/jdbc/TSDBResultSetTest.java | 14 +- .../com/taosdata/jdbc/TSDBStatementTest.java | 22 - .../jdbc/cases/AppMemoryLeakTest.java | 7 +- .../{rs => cases}/AuthenticationTest.java | 2 +- .../jdbc/cases/BadLocaleSettingTest.java | 59 ++ .../jdbc/cases/BatchErrorIgnoreTest.java | 144 +++++ .../taosdata/jdbc/cases/BatchInsertTest.java | 3 - ...iTaosdByRestfulWithDifferentTokenTest.java | 17 +- .../jdbc/cases/DatetimeBefore1970Test.java | 66 +- ...174Test.java => DoubleQuoteInSqlTest.java} | 62 +- .../jdbc/cases/DriverAutoloadTest.java | 2 +- .../com/taosdata/jdbc/cases/ImportTest.java | 5 - .../jdbc/cases/InsertDbwithoutUseDbTest.java | 4 +- .../cases/InsertSpecialCharacterJniTest.java | 16 +- .../InsertSpecialCharacterRestfulTest.java | 16 +- .../cases/InvalidResultSetPointerTest.java | 8 +- ....java => MicroSecondPrecisionJNITest.java} | 3 +- ...a => MicroSecondPrecisionRestfulTest.java} | 3 +- ...=> MultiThreadsWithSameStatementTest.java} | 12 +- .../cases/NanoSecondTimestampJNITest.java | 182 ++++++ .../cases/NanoSecondTimestampRestfulTest.java | 182 ++++++ ....java => NullValueInResultSetJNITest.java} | 7 +- ...a => NullValueInResultSetRestfulTest.java} | 4 +- ...est.java => NullValueInResultSetTest.java} | 2 +- .../PreparedStatementBatchInsertJNITest.java | 98 +++ ...eparedStatementBatchInsertRestfulTest.java | 98 +++ .../taosdata/jdbc/cases/QueryDataTest.java | 5 +- .../jdbc/cases/ResetQueryCacheTest.java | 2 - ...sultSetMetaShouldNotBeNullRestfulTest.java | 86 +++ .../com/taosdata/jdbc/cases/SelectTest.java | 4 - .../com/taosdata/jdbc/cases/StableTest.java | 15 +- .../com/taosdata/jdbc/cases/TD4144Test.java | 105 ---- .../jdbc/cases/TaosInfoMonitorTest.java | 4 +- .../TimestampPrecisionInNanoInJniTest.java | 570 ++++++++++++++++++ .../jdbc/cases/UnsignedNumberJniTest.java | 2 +- .../jdbc/cases/UnsignedNumberRestfulTest.java | 2 +- .../taosdata/jdbc/rs/RestfulDriverTest.java | 9 +- .../com/taosdata/jdbc/rs/RestfulJDBCTest.java | 207 ++++--- .../jdbc/rs/RestfulPreparedStatementTest.java | 34 +- .../jdbc/rs/RestfulResultSetTest.java | 14 +- .../jdbc/rs/RestfulStatementTest.java | 20 +- .../java/com/taosdata/jdbc/rs/SQLTest.java | 366 +++++++++-- .../com/taosdata/jdbc/utils/OSUtilsTest.java | 6 +- .../com/taosdata/jdbc/utils/SQLExecutor.java | 74 --- .../taosdata/jdbc/utils/TimestampUtil.java | 43 -- .../com/taosdata/jdbc/utils/UtilsTest.java | 122 +++- 86 files changed, 3006 insertions(+), 1212 deletions(-) mode change 100755 => 100644 src/connector/jdbc/pom.xml create mode 100644 src/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampFormat.java create mode 100644 src/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampPrecision.java delete mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java rename src/connector/jdbc/src/test/java/com/taosdata/jdbc/{rs => cases}/AuthenticationTest.java (98%) create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BadLocaleSettingTest.java create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchErrorIgnoreTest.java rename src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/{TD4174Test.java => DoubleQuoteInSqlTest.java} (55%) rename src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/{TwoTypeTimestampPercisionInJniTest.java => MicroSecondPrecisionJNITest.java} (97%) rename src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/{TwoTypeTimestampPercisionInRestfulTest.java => MicroSecondPrecisionRestfulTest.java} (99%) rename src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/{MultiThreadsWithSameStatmentTest.java => MultiThreadsWithSameStatementTest.java} (83%) create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampJNITest.java create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampRestfulTest.java rename src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/{NullValueInResultSetForJdbcJniTest.java => NullValueInResultSetJNITest.java} (87%) rename src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/{NullValueInResultSetForJdbcRestfulTest.java => NullValueInResultSetRestfulTest.java} (92%) rename src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/{TD3841Test.java => NullValueInResultSetTest.java} (99%) create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertJNITest.java create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertRestfulTest.java create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResultSetMetaShouldNotBeNullRestfulTest.java delete mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD4144Test.java create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java delete mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SQLExecutor.java diff --git a/src/connector/jdbc/CMakeLists.txt b/src/connector/jdbc/CMakeLists.txt index 3fabd751d0..8e30481575 100644 --- a/src/connector/jdbc/CMakeLists.txt +++ b/src/connector/jdbc/CMakeLists.txt @@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED) ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME} POST_BUILD COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.31.jar ${LIBRARY_OUTPUT_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.33.jar ${LIBRARY_OUTPUT_PATH} COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMENT "build jdbc driver") ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME}) diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index 8f77582d30..86f498b24a 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -5,7 +5,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.31 + 2.0.33 jar JDBCDriver diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml old mode 100755 new mode 100644 index d08d5502c2..ac2f99b0c5 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.taosdata.jdbc taos-jdbcdriver - 2.0.31 + 2.0.33 jar JDBCDriver https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc @@ -37,27 +37,26 @@ - - junit - junit - 4.13 - test - - org.apache.httpcomponents httpclient - 4.5.8 + 4.5.13 com.alibaba fastjson - 1.2.58 + 1.2.76 com.google.guava guava - 29.0-jre + 30.1.1-jre + + + junit + junit + 4.13.1 + test @@ -113,17 +112,19 @@ **/*Test.java - **/TSDBJNIConnectorTest.java - **/UnsignedNumberJniTest.java - **/DatetimeBefore1970Test.java + **/TimestampPrecisionInNanoInJniTest.java + **/NanoSecondTimestampJNITest.java + **/NanoSecondTimestampRestfulTest.java **/AppMemoryLeakTest.java **/AuthenticationTest.java - **/TaosInfoMonitorTest.java + **/ConnectMultiTaosdByRestfulWithDifferentTokenTest.java + **/DatetimeBefore1970Test.java **/FailOverTest.java **/InvalidResultSetPointerTest.java **/RestfulConnectionTest.java - **/TD4144Test.java - **/ConnectMultiTaosdByRestfulWithDifferentTokenTest.java + **/TSDBJNIConnectorTest.java + **/TaosInfoMonitorTest.java + **/UnsignedNumberJniTest.java true diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java index 2970f6c2d3..9950dbeb64 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java @@ -1,5 +1,7 @@ package com.taosdata.jdbc; +import com.taosdata.jdbc.enums.TimestampFormat; + import java.sql.*; import java.util.Enumeration; import java.util.Map; @@ -18,7 +20,7 @@ public abstract class AbstractConnection extends WrapperImpl implements Connecti for (String propName : propNames) { clientInfoProps.setProperty(propName, properties.getProperty(propName)); } - String timestampFormat = properties.getProperty(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT, "STRING"); + String timestampFormat = properties.getProperty(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT, String.valueOf(TimestampFormat.STRING)); clientInfoProps.setProperty(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT, timestampFormat); } @@ -169,11 +171,7 @@ public abstract class AbstractConnection extends WrapperImpl implements Connecti // do nothing } - @Override - public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); - + private void checkResultSetTypeAndResultSetConcurrency(int resultSetType, int resultSetConcurrency) throws SQLException { switch (resultSetType) { case ResultSet.TYPE_FORWARD_ONLY: break; @@ -192,7 +190,14 @@ public abstract class AbstractConnection extends WrapperImpl implements Connecti default: throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); } + } + @Override + public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + checkResultSetTypeAndResultSetConcurrency(resultSetType, resultSetConcurrency); return createStatement(); } @@ -201,24 +206,7 @@ public abstract class AbstractConnection extends WrapperImpl implements Connecti if (isClosed()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); - switch (resultSetType) { - case ResultSet.TYPE_FORWARD_ONLY: - break; - case ResultSet.TYPE_SCROLL_INSENSITIVE: - case ResultSet.TYPE_SCROLL_SENSITIVE: - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); - default: - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); - } - - switch (resultSetConcurrency) { - case ResultSet.CONCUR_READ_ONLY: - break; - case ResultSet.CONCUR_UPDATABLE: - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); - default: - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); - } + checkResultSetTypeAndResultSetConcurrency(resultSetType, resultSetConcurrency); return prepareStatement(sql); } @@ -304,9 +292,6 @@ public abstract class AbstractConnection extends WrapperImpl implements Connecti @Override public Statement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); - switch (resultSetHoldability) { case ResultSet.HOLD_CURSORS_OVER_COMMIT: break; @@ -320,11 +305,7 @@ public abstract class AbstractConnection extends WrapperImpl implements Connecti } @Override - public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) - throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); - + public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { switch (resultSetHoldability) { case ResultSet.HOLD_CURSORS_OVER_COMMIT: break; @@ -423,7 +404,7 @@ public abstract class AbstractConnection extends WrapperImpl implements Connecti status = resultSet.getInt("server_status()"); resultSet.close(); } - return status == 1 ? true : false; + return status == 1; }); boolean status = false; @@ -432,9 +413,7 @@ public abstract class AbstractConnection extends WrapperImpl implements Connecti status = future.get(); else status = future.get(timeout, TimeUnit.SECONDS); - } catch (InterruptedException e) { - e.printStackTrace(); - } catch (ExecutionException e) { + } catch (InterruptedException | ExecutionException e) { e.printStackTrace(); } catch (TimeoutException e) { future.cancel(true); @@ -450,8 +429,7 @@ public abstract class AbstractConnection extends WrapperImpl implements Connecti if (isClosed) throw (SQLClientInfoException) TSDBError.createSQLException(TSDBErrorNumbers.ERROR_SQLCLIENT_EXCEPTION_ON_CONNECTION_CLOSED); - if (clientInfoProps != null) - clientInfoProps.setProperty(name, value); + clientInfoProps.setProperty(name, value); } @Override @@ -459,8 +437,8 @@ public abstract class AbstractConnection extends WrapperImpl implements Connecti if (isClosed) throw (SQLClientInfoException) TSDBError.createSQLException(TSDBErrorNumbers.ERROR_SQLCLIENT_EXCEPTION_ON_CONNECTION_CLOSED); - for (Enumeration enumer = properties.keys(); enumer.hasMoreElements(); ) { - String name = (String) enumer.nextElement(); + for (Enumeration enumeration = properties.keys(); enumeration.hasMoreElements(); ) { + String name = (String) enumeration.nextElement(); clientInfoProps.put(name, properties.getProperty(name)); } } @@ -516,7 +494,6 @@ public abstract class AbstractConnection extends WrapperImpl implements Connecti public void abort(Executor executor) throws SQLException { if (isClosed()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); - // do nothing } @@ -527,14 +504,13 @@ public abstract class AbstractConnection extends WrapperImpl implements Connecti if (milliseconds < 0) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + // do nothing } @Override public int getNetworkTimeout() throws SQLException { if (isClosed()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); - - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + return 0; } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDatabaseMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDatabaseMetaData.java index 48811d30a6..3c9c784f59 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDatabaseMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDatabaseMetaData.java @@ -12,8 +12,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da private final static int DRIVER_MAJAR_VERSION = 2; private final static int DRIVER_MINOR_VERSION = 0; - private String precision = "ms"; - private String database; + private String precision = TSDBConstants.DEFAULT_PRECISION; public boolean allProceduresAreCallable() throws SQLException { return false; @@ -1223,7 +1222,6 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da ResultSet databases = stmt.executeQuery("show databases"); while (databases.next()) { String dbname = databases.getString("name"); - this.database = dbname; this.precision = databases.getString("precision"); if (dbname.equalsIgnoreCase(catalog)) return true; diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDriver.java index 5eaac1cd3b..7d4a268391 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDriver.java @@ -58,7 +58,7 @@ public abstract class AbstractDriver implements Driver { value = parameterValuePair.substring(indexOfEqual + 1); } } - if ((value != null && value.length() > 0) && (parameter != null && parameter.length() > 0)) { + if (value != null && value.length() > 0 && parameter.length() > 0) { urlProps.setProperty(parameter, value); } } @@ -87,7 +87,7 @@ public abstract class AbstractDriver implements Driver { url = url.substring(0, indexOfColon); } // parse host - if (url != null && url.length() > 0 && url.trim().length() > 0) { + if (url.length() > 0 && url.trim().length() > 0) { urlProps.setProperty(TSDBDriver.PROPERTY_KEY_HOST, url); } return urlProps; diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java index cb6c7d40ad..7d9d8ee515 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java @@ -1,7 +1,5 @@ package com.taosdata.jdbc; -import com.sun.org.apache.xpath.internal.operations.Bool; - import java.sql.ParameterMetaData; import java.sql.SQLException; import java.sql.Timestamp; diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java index 9dc559339a..a801f5a674 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java @@ -2,6 +2,7 @@ package com.taosdata.jdbc; import java.sql.*; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; public abstract class AbstractStatement extends WrapperImpl implements Statement { @@ -9,8 +10,6 @@ public abstract class AbstractStatement extends WrapperImpl implements Statement protected List batchedArgs; private int fetchSize; - - @Override public abstract ResultSet executeQuery(String sql) throws SQLException; @@ -198,13 +197,44 @@ public abstract class AbstractStatement extends WrapperImpl implements Statement if (batchedArgs == null || batchedArgs.isEmpty()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_BATCH_IS_EMPTY); + String clientInfo = getConnection().getClientInfo(TSDBDriver.PROPERTY_KEY_BATCH_ERROR_IGNORE); + boolean batchErrorIgnore = clientInfo == null ? TSDBConstants.DEFAULT_BATCH_ERROR_IGNORE : Boolean.parseBoolean(clientInfo); + + if (batchErrorIgnore) { + return executeBatchIgnoreException(); + } + return executeBatchThrowException(); + } + + private int[] executeBatchIgnoreException() { + return batchedArgs.stream().mapToInt(sql -> { + try { + boolean isSelect = execute(sql); + if (isSelect) { + return SUCCESS_NO_INFO; + } else { + return getUpdateCount(); + } + } catch (SQLException e) { + return EXECUTE_FAILED; + } + }).toArray(); + } + + private int[] executeBatchThrowException() throws BatchUpdateException { int[] res = new int[batchedArgs.size()]; for (int i = 0; i < batchedArgs.size(); i++) { - boolean isSelect = execute(batchedArgs.get(i)); - if (isSelect) { - res[i] = SUCCESS_NO_INFO; - } else { - res[i] = getUpdateCount(); + try { + boolean isSelect = execute(batchedArgs.get(i)); + if (isSelect) { + res[i] = SUCCESS_NO_INFO; + } else { + res[i] = getUpdateCount(); + } + } catch (SQLException e) { + String reason = e.getMessage(); + int[] updateCounts = Arrays.copyOfRange(res, 0, i); + throw new BatchUpdateException(reason, updateCounts, e); } } return res; diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java index cd266529f3..bda3d52212 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java @@ -15,10 +15,12 @@ package com.taosdata.jdbc; import java.math.BigDecimal; -import java.sql.*; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; import java.util.ArrayList; import java.util.Calendar; -import java.util.Iterator; import java.util.List; /* @@ -130,7 +132,7 @@ public class DatabaseMetaDataResultSet extends AbstractResultSet { public Timestamp getTimestamp(int columnIndex) throws SQLException { int colType = columnMetaDataList.get(columnIndex - 1).getColType(); int nativeType = TSDBConstants.jdbcType2TaosType(colType); - return rowCursor.getTimestamp(columnIndex,nativeType); + return rowCursor.getTimestamp(columnIndex, nativeType); } @Override @@ -145,9 +147,7 @@ public class DatabaseMetaDataResultSet extends AbstractResultSet { @Override public int findColumn(String columnLabel) throws SQLException { - Iterator colMetaDataIt = this.columnMetaDataList.iterator(); - while (colMetaDataIt.hasNext()) { - ColumnMetaData colMetaData = colMetaDataIt.next(); + for (ColumnMetaData colMetaData : this.columnMetaDataList) { if (colMetaData.getColName() != null && colMetaData.getColName().equals(columnLabel)) { return colMetaData.getColIndex(); } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/EmptyResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/EmptyResultSet.java index b6587b942d..fa8bf9e7e9 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/EmptyResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/EmptyResultSet.java @@ -23,7 +23,7 @@ import java.util.Calendar; import java.util.Map; /* - * TDengine only supports a subset of the standard SQL, thus this implemetation of the + * TDengine only supports a subset of the standard SQL, thus this implementation of the * standard JDBC API contains more or less some adjustments customized for certain * compatibility needs. */ @@ -326,7 +326,7 @@ public class EmptyResultSet implements ResultSet { @Override public int getFetchDirection() throws SQLException { - return 0; + return ResultSet.FETCH_FORWARD; } @Override @@ -341,12 +341,12 @@ public class EmptyResultSet implements ResultSet { @Override public int getType() throws SQLException { - return 0; + return ResultSet.TYPE_FORWARD_ONLY; } @Override public int getConcurrency() throws SQLException { - return 0; + return ResultSet.CONCUR_READ_ONLY; } @Override @@ -746,7 +746,7 @@ public class EmptyResultSet implements ResultSet { @Override public int getHoldability() throws SQLException { - return 0; + return ResultSet.CLOSE_CURSORS_AT_COMMIT; } @Override diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java index 02fee74eb5..8cd8da6de4 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java @@ -20,7 +20,7 @@ import java.util.Properties; public class TSDBConnection extends AbstractConnection { private TSDBJNIConnector connector; - private TSDBDatabaseMetaData databaseMetaData; + private final TSDBDatabaseMetaData databaseMetaData; private boolean batchFetch; public Boolean getBatchFetch() { diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java index e6406d2c6d..f3f04eff12 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java @@ -54,10 +54,11 @@ public abstract class TSDBConstants { public static final int TSDB_DATA_TYPE_USMALLINT = 12; //unsigned smallint public static final int TSDB_DATA_TYPE_UINT = 13; //unsigned int public static final int TSDB_DATA_TYPE_UBIGINT = 14; //unsigned bigint + // nchar column max length public static final int maxFieldSize = 16 * 1024; - // precision for data types + // precision for data types, this is used for metadata public static final int BOOLEAN_PRECISION = 1; public static final int TINYINT_PRECISION = 4; public static final int SMALLINT_PRECISION = 6; @@ -67,10 +68,14 @@ public abstract class TSDBConstants { public static final int DOUBLE_PRECISION = 22; public static final int TIMESTAMP_MS_PRECISION = 23; public static final int TIMESTAMP_US_PRECISION = 26; - // scale for data types + // scale for data types, this is used for metadata public static final int FLOAT_SCALE = 31; public static final int DOUBLE_SCALE = 31; + public static final String DEFAULT_PRECISION = "ms"; + + public static final boolean DEFAULT_BATCH_ERROR_IGNORE = false; + public static int typeName2JdbcType(String type) { switch (type.toUpperCase()) { case "TIMESTAMP": diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java index 8b7ede148e..9a5eda4cd8 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java @@ -20,8 +20,8 @@ import java.sql.SQLException; public class TSDBDatabaseMetaData extends AbstractDatabaseMetaData { - private String url; - private String userName; + private final String url; + private final String userName; private Connection conn; public TSDBDatabaseMetaData(String url, String userName) { diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java index 55533bd28c..f5f16758c1 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java @@ -100,6 +100,11 @@ public class TSDBDriver extends AbstractDriver { */ public static final String PROPERTY_KEY_TIMESTAMP_FORMAT = "timestampFormat"; + /** + * continue process commands in executeBatch + */ + public static final String PROPERTY_KEY_BATCH_ERROR_IGNORE = "batchErrorIgnore"; + private TSDBDatabaseMetaData dbMetaData = null; static { @@ -176,7 +181,7 @@ public class TSDBDriver extends AbstractDriver { int beginningOfSlashes = url.indexOf("//"); int index = url.indexOf("?"); if (index != -1) { - String paramString = url.substring(index + 1, url.length()); + String paramString = url.substring(index + 1); url = url.substring(0, index); StringTokenizer queryParams = new StringTokenizer(paramString, "&"); while (queryParams.hasMoreElements()) { @@ -213,7 +218,7 @@ public class TSDBDriver extends AbstractDriver { url = url.substring(0, indexOfColon); } - if (url != null && url.length() > 0 && url.trim().length() > 0) { + if (url.length() > 0 && url.trim().length() > 0) { urlProps.setProperty(TSDBDriver.PROPERTY_KEY_HOST, url); } @@ -233,7 +238,7 @@ public class TSDBDriver extends AbstractDriver { return false; } - public Logger getParentLogger() throws SQLFeatureNotSupportedException { + public Logger getParentLogger() { return null; } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java index 90967b3620..da89081428 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java @@ -8,7 +8,7 @@ import java.util.HashMap; import java.util.Map; public class TSDBError { - private static Map TSDBErrorMap = new HashMap<>(); + private static final Map TSDBErrorMap = new HashMap<>(); static { TSDBErrorMap.put(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED, "connection already closed"); @@ -32,10 +32,10 @@ public class TSDBError { TSDBErrorMap.put(TSDBErrorNumbers.ERROR_INVALID_SQL, "invalid sql"); TSDBErrorMap.put(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE, "numeric value out of range"); TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNKNOWN_TAOS_TYPE_IN_TDENGINE, "unknown taos type in tdengine"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNKNOWN_TIMESTAMP_PERCISION, "unknown timestamp precision"); - /**************************************************/ TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNKNOWN, "unknown error"); - /**************************************************/ + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_SUBSCRIBE_FAILED, "failed to create subscription"); TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNSUPPORTED_ENCODING, "Unsupported encoding"); TSDBErrorMap.put(TSDBErrorNumbers.ERROR_JNI_TDENGINE_ERROR, "internal error of database"); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java index c978bb3a2e..a796e6d86f 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java @@ -1,6 +1,7 @@ package com.taosdata.jdbc; import java.util.HashSet; +import java.util.Set; public class TSDBErrorNumbers { @@ -25,6 +26,7 @@ public class TSDBErrorNumbers { public static final int ERROR_INVALID_SQL = 0x2313; // invalid sql public static final int ERROR_NUMERIC_VALUE_OUT_OF_RANGE = 0x2314; // numeric value out of range public static final int ERROR_UNKNOWN_TAOS_TYPE_IN_TDENGINE = 0x2315; //unknown taos type in tdengine + public static final int ERROR_UNKNOWN_TIMESTAMP_PERCISION = 0x2316; // unknown timestamp precision public static final int ERROR_UNKNOWN = 0x2350; //unknown error @@ -38,10 +40,9 @@ public class TSDBErrorNumbers { public static final int ERROR_JNI_FETCH_END = 0x2358; // fetch to the end of resultSet public static final int ERROR_JNI_OUT_OF_MEMORY = 0x2359; // JNI alloc memory failed - private static final HashSet errorNumbers; + private static final Set errorNumbers = new HashSet<>(); static { - errorNumbers = new HashSet(); errorNumbers.add(ERROR_CONNECTION_CLOSED); errorNumbers.add(ERROR_UNSUPPORTED_METHOD); errorNumbers.add(ERROR_INVALID_VARIABLE); @@ -62,8 +63,8 @@ public class TSDBErrorNumbers { errorNumbers.add(ERROR_INVALID_SQL); errorNumbers.add(ERROR_NUMERIC_VALUE_OUT_OF_RANGE); errorNumbers.add(ERROR_UNKNOWN_TAOS_TYPE_IN_TDENGINE); + errorNumbers.add(ERROR_UNKNOWN_TIMESTAMP_PERCISION); - /*****************************************************/ errorNumbers.add(ERROR_SUBSCRIBE_FAILED); errorNumbers.add(ERROR_UNSUPPORTED_ENCODING); errorNumbers.add(ERROR_JNI_TDENGINE_ERROR); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java index 92792d9751..051eca7e10 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java @@ -29,14 +29,9 @@ import java.util.List; public class TSDBJNIConnector { private static volatile Boolean isInitialized = false; - private TaosInfo taosInfo = TaosInfo.getInstance(); - - // Connection pointer used in C - private long taos = TSDBConstants.JNI_NULL_POINTER; - - // result set status in current connection - private boolean isResultsetClosed; - + private final TaosInfo taosInfo = TaosInfo.getInstance(); + private long taos = TSDBConstants.JNI_NULL_POINTER; // Connection pointer used in C + private boolean isResultsetClosed; // result set status in current connection private int affectedRows = -1; static { @@ -96,11 +91,9 @@ public class TSDBJNIConnector { /** * Execute DML/DDL operation - * - * @throws SQLException */ public long executeQuery(String sql) throws SQLException { - Long pSql = 0l; + long pSql = 0L; try { pSql = this.executeQueryImp(sql.getBytes(TaosGlobalConfig.getCharset()), this.taos); taosInfo.stmt_count_increment(); @@ -161,7 +154,7 @@ public class TSDBJNIConnector { private native long getResultSetImp(long connection, long pSql); public boolean isUpdateQuery(long pSql) { - return isUpdateQueryImp(this.taos, pSql) == 1 ? true : false; + return isUpdateQueryImp(this.taos, pSql) == 1; } private native long isUpdateQueryImp(long connection, long pSql); @@ -195,7 +188,7 @@ public class TSDBJNIConnector { */ public int getSchemaMetaData(long resultSet, List columnMetaData) { int ret = this.getSchemaMetaDataImp(this.taos, resultSet, columnMetaData); - columnMetaData.stream().forEach(column -> column.setColIndex(column.getColIndex() + 1)); + columnMetaData.forEach(column -> column.setColIndex(column.getColIndex() + 1)); return ret; } @@ -217,9 +210,18 @@ public class TSDBJNIConnector { private native int fetchBlockImp(long connection, long resultSet, TSDBResultSetBlockData blockData); /** - * Execute close operation from C to release connection pointer by JNI + * Get Result Time Precision. * - * @throws SQLException + * @return 0: ms, 1: us, 2: ns + */ + public int getResultTimePrecision(long sqlObj) { + return this.getResultTimePrecisionImp(this.taos, sqlObj); + } + + private native int getResultTimePrecisionImp(long connection, long result); + + /** + * Execute close operation from C to release connection pointer by JNI */ public void closeConnection() throws SQLException { int code = this.closeConnectionImp(this.taos); @@ -258,8 +260,6 @@ public class TSDBJNIConnector { /** * Unsubscribe, close a subscription - * - * @param subscription */ void unsubscribe(long subscription, boolean isKeep) { unsubscribeImp(subscription, isKeep); @@ -272,13 +272,13 @@ public class TSDBJNIConnector { */ public boolean validateCreateTableSql(String sql) { int res = validateCreateTableSqlImp(taos, sql.getBytes()); - return res != 0 ? false : true; + return res == 0; } private native int validateCreateTableSqlImp(long connection, byte[] sqlBytes); public long prepareStmt(String sql) throws SQLException { - Long stmt; + long stmt; try { stmt = prepareStmtImp(sql.getBytes(), this.taos); } catch (Exception e) { @@ -348,4 +348,13 @@ public class TSDBJNIConnector { } private native int closeStmt(long stmt, long con); + + public void insertLines(String[] lines) throws SQLException { + int code = insertLinesImp(lines, this.taos); + if (code != TSDBConstants.JNI_SUCCESS) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "failed to insertLines"); + } + } + + private native int insertLinesImp(String[] lines, long conn); } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java index c3d5abf35c..22fb0c4ae4 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java @@ -38,7 +38,6 @@ import java.util.regex.Pattern; public class TSDBPreparedStatement extends TSDBStatement implements PreparedStatement { private String rawSql; private Object[] parameters; - private boolean isPrepared; private ArrayList colData; private ArrayList tableTags; @@ -47,8 +46,6 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat private String tableName; private long nativeStmtHandle = 0; - private volatile TSDBParameterMetaData parameterMetaData; - TSDBPreparedStatement(TSDBConnection connection, String sql) { super(connection); init(sql); @@ -60,14 +57,13 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat parameterCnt++; } } - parameters = new Object[parameterCnt]; - this.isPrepared = true; } + parameters = new Object[parameterCnt]; if (parameterCnt > 1) { // the table name is also a parameter, so ignore it. - this.colData = new ArrayList(); - this.tableTags = new ArrayList(); + this.colData = new ArrayList<>(); + this.tableTags = new ArrayList<>(); } } @@ -76,18 +72,17 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat preprocessSql(); } - @Override - public int[] executeBatch() throws SQLException { - return super.executeBatch(); - } - /* + * + */ + + /** * Some of the SQLs sent by other popular frameworks or tools like Spark, contains syntax that cannot be parsed by * the TDengine client. Thus, some simple parsers/filters are intentionally added in this JDBC implementation in * order to process those supported SQLs. */ private void preprocessSql() { - /***** For processing some of Spark SQLs*****/ + /***For processing some of Spark SQLs*/ // should replace it first this.rawSql = this.rawSql.replaceAll("or (.*) is null", ""); this.rawSql = this.rawSql.replaceAll(" where ", " WHERE "); @@ -134,32 +129,17 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat } rawSql = rawSql.replace(matcher.group(1), tableFullName); } - /***** for inner queries *****/ - } - - /** - * Populate parameters into prepared sql statements - * - * @return a string of the native sql statement for TSDB - */ - private String getNativeSql(String rawSql) throws SQLException { - return Utils.getNativeSql(rawSql, this.parameters); } @Override public ResultSet executeQuery() throws SQLException { - if (!isPrepared) - return executeQuery(this.rawSql); - - final String sql = getNativeSql(this.rawSql); + final String sql = Utils.getNativeSql(this.rawSql, this.parameters); return executeQuery(sql); } @Override public int executeUpdate() throws SQLException { - if (!isPrepared) - return executeUpdate(this.rawSql); - String sql = getNativeSql(this.rawSql); + String sql = Utils.getNativeSql(this.rawSql, this.parameters); return executeUpdate(sql); } @@ -282,25 +262,14 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat @Override public boolean execute() throws SQLException { - if (!isPrepared) - return execute(this.rawSql); - - final String sql = getNativeSql(this.rawSql); + final String sql = Utils.getNativeSql(this.rawSql, this.parameters); return execute(sql); } @Override public void addBatch() throws SQLException { - if (this.batchedArgs == null) { - batchedArgs = new ArrayList<>(); - } - - if (!isPrepared) { - addBatch(this.rawSql); - } else { - String sql = this.getConnection().nativeSQL(this.rawSql); - addBatch(sql); - } + String sql = Utils.getNativeSql(this.rawSql, this.parameters); + addBatch(sql); } @Override @@ -553,12 +522,10 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat } } - ; - private static class TableTagInfo { private boolean isNull; - private Object value; - private int type; + private final Object value; + private final int type; public TableTagInfo(Object value, int type) { this.value = value; @@ -572,8 +539,6 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat } } - ; - public void setTableName(String name) { this.tableName = name; } @@ -661,7 +626,7 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat this.colData.addAll(Collections.nCopies(this.parameters.length - 1 - this.tableTags.size(), null)); } - ColumnInfo col = (ColumnInfo) this.colData.get(columnIndex); + ColumnInfo col = this.colData.get(columnIndex); if (col == null) { ColumnInfo p = new ColumnInfo(); p.setType(type); @@ -752,8 +717,7 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat ByteBuffer isNullList = ByteBuffer.allocate(num * Integer.BYTES); isNullList.order(ByteOrder.LITTLE_ENDIAN); - for (int i = 0; i < num; ++i) { - TableTagInfo tag = this.tableTags.get(i); + for (TableTagInfo tag : this.tableTags) { if (tag.isNull) { typeList.put((byte) tag.type); isNullList.putInt(1); @@ -852,7 +816,7 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat typeList, lengthList, isNullList); } - ColumnInfo colInfo = (ColumnInfo) this.colData.get(0); + ColumnInfo colInfo = this.colData.get(0); if (colInfo == null) { throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "column data not bind"); } @@ -988,7 +952,6 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "not support data types"); } } - ; connector.bindColumnDataArray(this.nativeStmtHandle, colDataList, lengthList, isNullList, col1.type, col1.bytes, rows, i); } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java index 59a64ad520..00a62206fc 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java @@ -339,11 +339,11 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet { case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: case TSDBConstants.TSDB_DATA_TYPE_INT: case TSDBConstants.TSDB_DATA_TYPE_BIGINT: - res = new BigDecimal(Long.valueOf(this.rowData.getObject(columnIndex).toString())); + res = new BigDecimal(Long.parseLong(this.rowData.getObject(columnIndex).toString())); break; case TSDBConstants.TSDB_DATA_TYPE_FLOAT: case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: - res = new BigDecimal(Double.valueOf(this.rowData.getObject(columnIndex).toString())); + res = BigDecimal.valueOf(Double.parseDouble(this.rowData.getObject(columnIndex).toString())); break; case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: return new BigDecimal(((Timestamp) this.rowData.getObject(columnIndex)).getTime()); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java index 7b3be5d263..6211f61dc5 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java @@ -36,23 +36,20 @@ public class TSDBResultSetBlockData { private int rowIndex = 0; private List columnMetaDataList; - private ArrayList colData = null; + private ArrayList colData; public TSDBResultSetBlockData(List colMeta, int numOfCols) { this.columnMetaDataList = colMeta; - this.colData = new ArrayList(numOfCols); + this.colData = new ArrayList<>(numOfCols); } public TSDBResultSetBlockData() { - this.colData = new ArrayList(); + this.colData = new ArrayList<>(); } public void clear() { int size = this.colData.size(); - if (this.colData != null) { - this.colData.clear(); - } - + this.colData.clear(); setNumOfCols(size); } @@ -69,7 +66,7 @@ public class TSDBResultSetBlockData { } public void setNumOfCols(int numOfCols) { - this.colData = new ArrayList(numOfCols); + this.colData = new ArrayList<>(numOfCols); this.colData.addAll(Collections.nCopies(numOfCols, null)); } @@ -166,15 +163,10 @@ public class TSDBResultSetBlockData { } } - /** * The original type may not be a string type, but will be converted to by * calling this method - * - * @param col column index - * @return - * @throws SQLException */ public String getString(int col) throws SQLException { Object obj = get(col); @@ -387,7 +379,7 @@ public class TSDBResultSetBlockData { return null; } - return (long) val; + return val; } case TSDBConstants.TSDB_DATA_TYPE_FLOAT: { diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java index 48d4247392..6292673352 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java @@ -41,10 +41,7 @@ public class TSDBResultSetMetaData extends WrapperImpl implements ResultSetMetaD } public boolean isSearchable(int column) throws SQLException { - if (column == 1) { - return true; - } - return false; + return column == 1; } public boolean isCurrency(int column) throws SQLException { diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java index 01104440ab..2ff0d86c92 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java @@ -26,7 +26,7 @@ import java.util.Collections; public class TSDBResultSetRowData { private ArrayList data; - private int colSize; + private final int colSize; public TSDBResultSetRowData(int colSize) { this.colSize = colSize; @@ -147,30 +147,14 @@ public class TSDBResultSetRowData { case TSDBConstants.TSDB_DATA_TYPE_NCHAR: case TSDBConstants.TSDB_DATA_TYPE_BINARY: return Integer.parseInt((String) obj); - case TSDBConstants.TSDB_DATA_TYPE_UTINYINT: { - Byte value = (byte) obj; - if (value < 0) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE); - return value; - } - case TSDBConstants.TSDB_DATA_TYPE_USMALLINT: { - short value = (short) obj; - if (value < 0) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE); - return value; - } - case TSDBConstants.TSDB_DATA_TYPE_UINT: { - int value = (int) obj; - if (value < 0) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE); - return value; - } - case TSDBConstants.TSDB_DATA_TYPE_UBIGINT: { - long value = (long) obj; - if (value < 0) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE); - return Long.valueOf(value).intValue(); - } + case TSDBConstants.TSDB_DATA_TYPE_UTINYINT: + return parseUnsignedTinyIntToInt(obj); + case TSDBConstants.TSDB_DATA_TYPE_USMALLINT: + return parseUnsignedSmallIntToInt(obj); + case TSDBConstants.TSDB_DATA_TYPE_UINT: + return parseUnsignedIntegerToInt(obj); + case TSDBConstants.TSDB_DATA_TYPE_UBIGINT: + return parseUnsignedBigIntToInt(obj); case TSDBConstants.TSDB_DATA_TYPE_FLOAT: return ((Float) obj).intValue(); case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: @@ -180,6 +164,35 @@ public class TSDBResultSetRowData { } } + private byte parseUnsignedTinyIntToInt(Object obj) throws SQLException { + byte value = (byte) obj; + if (value < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE); + return value; + } + + private short parseUnsignedSmallIntToInt(Object obj) throws SQLException { + short value = (short) obj; + if (value < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE); + return value; + } + + private int parseUnsignedIntegerToInt(Object obj) throws SQLException { + int value = (int) obj; + if (value < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE); + return value; + } + + private int parseUnsignedBigIntToInt(Object obj) throws SQLException { + long value = (long) obj; + if (value < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE); + return Long.valueOf(value).intValue(); + } + + /** * $$$ this method is invoked by databaseMetaDataResultSet and so on which use a index start from 1 in JDBC api */ @@ -216,7 +229,7 @@ public class TSDBResultSetRowData { case TSDBConstants.TSDB_DATA_TYPE_BINARY: return Long.parseLong((String) obj); case TSDBConstants.TSDB_DATA_TYPE_UTINYINT: { - Byte value = (byte) obj; + byte value = (byte) obj; if (value < 0) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE); return value; @@ -377,27 +390,27 @@ public class TSDBResultSetRowData { switch (nativeType) { case TSDBConstants.TSDB_DATA_TYPE_UTINYINT: { - Byte value = new Byte(String.valueOf(obj)); + byte value = new Byte(String.valueOf(obj)); if (value >= 0) - return value.toString(); + return Byte.toString(value); return Integer.toString(value & 0xff); } case TSDBConstants.TSDB_DATA_TYPE_USMALLINT: { - Short value = new Short(String.valueOf(obj)); + short value = new Short(String.valueOf(obj)); if (value >= 0) - return value.toString(); + return Short.toString(value); return Integer.toString(value & 0xffff); } case TSDBConstants.TSDB_DATA_TYPE_UINT: { - Integer value = new Integer(String.valueOf(obj)); + int value = new Integer(String.valueOf(obj)); if (value >= 0) - return value.toString(); - return Long.toString(value & 0xffffffffl); + return Integer.toString(value); + return Long.toString(value & 0xffffffffL); } case TSDBConstants.TSDB_DATA_TYPE_UBIGINT: { - Long value = new Long(String.valueOf(obj)); + long value = new Long(String.valueOf(obj)); if (value >= 0) - return value.toString(); + return Long.toString(value); long lowValue = value & 0x7fffffffffffffffL; return BigDecimal.valueOf(lowValue).add(BigDecimal.valueOf(Long.MAX_VALUE)).add(BigDecimal.valueOf(1)).toString(); } @@ -414,11 +427,45 @@ public class TSDBResultSetRowData { * $$$ this method is invoked by databaseMetaDataResultSet and so on which use a index start from 1 in JDBC api */ public void setTimestampValue(int colIndex, long value) { - setTimestamp(colIndex - 1, value); + setTimestamp(colIndex - 1, value, 0); } /** * !!! this method is invoked by JNI method and the index start from 0 in C implementations + * + * @param precision 0 : ms, 1 : us, 2 : ns + */ + public void setTimestamp(int col, long ts, int precision) { + long milliseconds; + int fracNanoseconds; + switch (precision) { + case 0: { + milliseconds = ts; + fracNanoseconds = (int) (ts * 1_000_000 % 1_000_000_000); + break; + } + case 1: { + milliseconds = ts / 1_000; + fracNanoseconds = (int) (ts * 1_000 % 1_000_000_000); + break; + } + case 2: { + milliseconds = ts / 1_000_000; + fracNanoseconds = (int) (ts % 1_000_000_000); + break; + } + default: { + throw new IllegalArgumentException("precision is not valid. precision: " + precision); + } + } + + Timestamp tsObj = new Timestamp(milliseconds); + tsObj.setNanos(fracNanoseconds); + data.set(col, tsObj); + } + + /** + * this implementation is used for TDengine old version */ public void setTimestamp(int col, long ts) { //TODO: this implementation contains logical error @@ -429,8 +476,8 @@ public class TSDBResultSetRowData { if (ts < 1_0000_0000_0000_0L) { data.set(col, new Timestamp(ts)); } else { - long epochSec = ts / 1000_000l; - long nanoAdjustment = ts % 1000_000l * 1000l; + long epochSec = ts / 1000_000L; + long nanoAdjustment = ts % 1000_000L * 1000L; Timestamp timestamp = Timestamp.from(Instant.ofEpochSecond(epochSec, nanoAdjustment)); data.set(col, timestamp); } @@ -440,12 +487,10 @@ public class TSDBResultSetRowData { Object obj = data.get(col - 1); if (obj == null) return null; - switch (nativeType) { - case TSDBConstants.TSDB_DATA_TYPE_BIGINT: - return new Timestamp((Long) obj); - default: - return (Timestamp) obj; + if (nativeType == TSDBConstants.TSDB_DATA_TYPE_BIGINT) { + return new Timestamp((Long) obj); } + return (Timestamp) obj; } public Object getObject(int col) { diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java index c5fd497ca3..d74f7755b2 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java @@ -47,9 +47,6 @@ public class TSDBSubscribe { /** * close subscription - * - * @param keepProgress - * @throws SQLException */ public void close(boolean keepProgress) throws SQLException { if (this.connecter.isClosed()) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampFormat.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampFormat.java new file mode 100644 index 0000000000..5ff0774639 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampFormat.java @@ -0,0 +1,7 @@ +package com.taosdata.jdbc.enums; + +public enum TimestampFormat { + STRING, + TIMESTAMP, + UTC +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampPrecision.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampPrecision.java new file mode 100644 index 0000000000..79350076c7 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampPrecision.java @@ -0,0 +1,8 @@ +package com.taosdata.jdbc.enums; + +public enum TimestampPrecision { + MS, + US, + NS, + UNKNOWN +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java index d6a02b7e3a..12a0ab57e2 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java @@ -16,7 +16,7 @@ public class RestfulConnection extends AbstractConnection { private final String host; private final int port; private final String url; - private volatile String database; + private final String database; private final String token; /******************************************************/ private boolean isClosed; diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java index f58e3f8cd2..f2abbd2445 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java @@ -13,7 +13,7 @@ import java.util.Calendar; public class RestfulPreparedStatement extends RestfulStatement implements PreparedStatement { - private ParameterMetaData parameterMetaData; + private final ParameterMetaData parameterMetaData; private final String rawSql; private Object[] parameters; private boolean isPrepared; @@ -22,16 +22,16 @@ public class RestfulPreparedStatement extends RestfulStatement implements Prepar super(conn, database); this.rawSql = sql; + int parameterCnt = 0; if (sql.contains("?")) { - int parameterCnt = 0; for (int i = 0; i < sql.length(); i++) { if ('?' == sql.charAt(i)) { parameterCnt++; } } - parameters = new Object[parameterCnt]; this.isPrepared = true; } + parameters = new Object[parameterCnt]; // build parameterMetaData this.parameterMetaData = new RestfulParameterMetaData(parameters); @@ -44,7 +44,7 @@ public class RestfulPreparedStatement extends RestfulStatement implements Prepar if (!isPrepared) return executeQuery(this.rawSql); - final String sql = getNativeSql(this.rawSql); + final String sql = Utils.getNativeSql(this.rawSql, this.parameters); return executeQuery(sql); } @@ -55,20 +55,10 @@ public class RestfulPreparedStatement extends RestfulStatement implements Prepar if (!isPrepared) return executeUpdate(this.rawSql); - final String sql = getNativeSql(this.rawSql); + final String sql = Utils.getNativeSql(rawSql, this.parameters); return executeUpdate(sql); } - /**** - * ĺ°†rawSql转换ć一条可执行的sql语句，使用属性parameters中的ĺŹč„¸čż›čˇŚć›żćŤ˘ - * 对于insert into ?.? (?,?,?) using ?.? (?,?,?) tags(?, ?, ?) values(?, ?, ?) - * @param rawSql，可č˝ćŻinsertă€selectć–其他，使用?ĺšĺŤ ä˝Ťç¬¦ - * @return - */ - private String getNativeSql(String rawSql) { - return Utils.getNativeSql(rawSql, this.parameters); - } - @Override public void setNull(int parameterIndex, int sqlType) throws SQLException { if (isClosed()) @@ -224,16 +214,13 @@ public class RestfulPreparedStatement extends RestfulStatement implements Prepar throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); if (!isPrepared) return execute(this.rawSql); - final String sql = getNativeSql(rawSql); + final String sql = Utils.getNativeSql(rawSql, this.parameters); return execute(sql); } @Override public void addBatch() throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - - final String sql = getNativeSql(this.rawSql); + final String sql = Utils.getNativeSql(rawSql, this.parameters); addBatch(sql); } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java index 530b433d42..f0ea03638f 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java @@ -6,6 +6,8 @@ import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import com.google.common.primitives.Shorts; import com.taosdata.jdbc.*; +import com.taosdata.jdbc.enums.TimestampPrecision; +import com.taosdata.jdbc.enums.TimestampFormat; import com.taosdata.jdbc.utils.Utils; import java.math.BigDecimal; @@ -15,19 +17,20 @@ import java.time.ZoneOffset; import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.Calendar; +import java.util.List; public class RestfulResultSet extends AbstractResultSet implements ResultSet { - private volatile boolean isClosed; - private int pos = -1; - private final String database; private final Statement statement; // data - private final ArrayList> resultSet = new ArrayList<>(); + private final List> resultSet = new ArrayList<>(); // meta - private ArrayList columnNames = new ArrayList<>(); - private ArrayList columns = new ArrayList<>(); - private RestfulResultSetMetaData metaData; + private final List columnNames = new ArrayList<>(); + private final List columns = new ArrayList<>(); + private final RestfulResultSetMetaData metaData; + + private volatile boolean isClosed; + private int pos = -1; /** * 由一个resultçš„Json构造结果集，对应执行show databases, show tables等这些语句，返回结果集，但无法获取结果集对应的meta，统一当ćStringĺ¤„ç† @@ -35,34 +38,30 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { * @param resultJson: 包ĺ«data信ćŻçš„结果集，有sql返回的结果集 ***/ public RestfulResultSet(String database, Statement statement, JSONObject resultJson) throws SQLException { - this.database = database; this.statement = statement; + // get head + JSONArray head = resultJson.getJSONArray("head"); // get column metadata JSONArray columnMeta = resultJson.getJSONArray("column_meta"); // get row data JSONArray data = resultJson.getJSONArray("data"); - if (data == null || data.isEmpty()) { - columnNames.clear(); - columns.clear(); - this.resultSet.clear(); - return; - } - // get head - JSONArray head = resultJson.getJSONArray("head"); // get rows Integer rows = resultJson.getInteger("rows"); + // parse column_meta if (columnMeta != null) { parseColumnMeta_new(columnMeta); } else { parseColumnMeta_old(head, data, rows); } - this.metaData = new RestfulResultSetMetaData(this.database, columns, this); + this.metaData = new RestfulResultSetMetaData(database, columns, this); + + if (data == null || data.isEmpty()) + return; // parse row data - resultSet.clear(); for (int rowIndex = 0; rowIndex < data.size(); rowIndex++) { - ArrayList row = new ArrayList(); + List row = new ArrayList<>(); JSONArray jsonRow = data.getJSONArray(rowIndex); for (int colIndex = 0; colIndex < this.metaData.getColumnCount(); colIndex++) { row.add(parseColumnData(jsonRow, colIndex, columns.get(colIndex).taos_type)); @@ -131,7 +130,6 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { } } - private Object parseColumnData(JSONArray row, int colIndex, int taosType) throws SQLException { switch (taosType) { case TSDBConstants.TSDB_DATA_TYPE_NULL: @@ -150,44 +148,8 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { return row.getFloat(colIndex); case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: return row.getDouble(colIndex); - case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: { - if (row.get(colIndex) == null) - return null; - String timestampFormat = this.statement.getConnection().getClientInfo(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT); - if ("TIMESTAMP".equalsIgnoreCase(timestampFormat)) { - Long value = row.getLong(colIndex); - //TODO: this implementation has bug if the timestamp bigger than 9999_9999_9999_9 - if (value < 1_0000_0000_0000_0L) - return new Timestamp(value); - long epochSec = value / 1000_000l; - long nanoAdjustment = value % 1000_000l * 1000l; - return Timestamp.from(Instant.ofEpochSecond(epochSec, nanoAdjustment)); - } - if ("UTC".equalsIgnoreCase(timestampFormat)) { - String value = row.getString(colIndex); - long epochSec = Timestamp.valueOf(value.substring(0, 19).replace("T", " ")).getTime() / 1000; - int fractionalSec = Integer.parseInt(value.substring(20, value.length() - 5)); - long nanoAdjustment = 0; - if (value.length() > 28) { - // ms timestamp: yyyy-MM-ddTHH:mm:ss.SSSSSS+0x00 - nanoAdjustment = fractionalSec * 1000l; - } else { - // ms timestamp: yyyy-MM-ddTHH:mm:ss.SSS+0x00 - nanoAdjustment = fractionalSec * 1000_000l; - } - ZoneOffset zoneOffset = ZoneOffset.of(value.substring(value.length() - 5)); - Instant instant = Instant.ofEpochSecond(epochSec, nanoAdjustment).atOffset(zoneOffset).toInstant(); - return Timestamp.from(instant); - } - String value = row.getString(colIndex); - if (value.length() <= 23) // ms timestamp: yyyy-MM-dd HH:mm:ss.SSS - return row.getTimestamp(colIndex); - // us timestamp: yyyy-MM-dd HH:mm:ss.SSSSSS - long epochSec = Timestamp.valueOf(value.substring(0, 19)).getTime() / 1000; - long nanoAdjustment = Integer.parseInt(value.substring(20)) * 1000l; - Timestamp timestamp = Timestamp.from(Instant.ofEpochSecond(epochSec, nanoAdjustment)); - return timestamp; - } + case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: + return parseTimestampColumnData(row, colIndex); case TSDBConstants.TSDB_DATA_TYPE_BINARY: return row.getString(colIndex) == null ? null : row.getString(colIndex).getBytes(); case TSDBConstants.TSDB_DATA_TYPE_NCHAR: @@ -197,7 +159,66 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { } } - public class Field { + private Timestamp parseTimestampColumnData(JSONArray row, int colIndex) throws SQLException { + if (row.get(colIndex) == null) + return null; + String tsFormatUpperCase = this.statement.getConnection().getClientInfo(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT).toUpperCase(); + TimestampFormat timestampFormat = TimestampFormat.valueOf(tsFormatUpperCase); + switch (timestampFormat) { + case TIMESTAMP: { + Long value = row.getLong(colIndex); + //TODO: this implementation has bug if the timestamp bigger than 9999_9999_9999_9 + if (value < 1_0000_0000_0000_0L) + return new Timestamp(value); + long epochSec = value / 1000_000L; + long nanoAdjustment = value % 1000_000L * 1000L; + return Timestamp.from(Instant.ofEpochSecond(epochSec, nanoAdjustment)); + } + case UTC: { + String value = row.getString(colIndex); + long epochSec = Timestamp.valueOf(value.substring(0, 19).replace("T", " ")).getTime() / 1000; + int fractionalSec = Integer.parseInt(value.substring(20, value.length() - 5)); + long nanoAdjustment; + if (value.length() > 31) { + // ns timestamp: yyyy-MM-ddTHH:mm:ss.SSSSSSSSS+0x00 + nanoAdjustment = fractionalSec; + } else if (value.length() > 28) { + // ms timestamp: yyyy-MM-ddTHH:mm:ss.SSSSSS+0x00 + nanoAdjustment = fractionalSec * 1000L; + } else { + // ms timestamp: yyyy-MM-ddTHH:mm:ss.SSS+0x00 + nanoAdjustment = fractionalSec * 1000_000L; + } + ZoneOffset zoneOffset = ZoneOffset.of(value.substring(value.length() - 5)); + Instant instant = Instant.ofEpochSecond(epochSec, nanoAdjustment).atOffset(zoneOffset).toInstant(); + return Timestamp.from(instant); + } + case STRING: + default: { + String value = row.getString(colIndex); + TimestampPrecision precision = Utils.guessTimestampPrecision(value); + if (precision == TimestampPrecision.MS) { + // ms timestamp: yyyy-MM-dd HH:mm:ss.SSS + return row.getTimestamp(colIndex); + } + if (precision == TimestampPrecision.US) { + // us timestamp: yyyy-MM-dd HH:mm:ss.SSSSSS + long epochSec = Timestamp.valueOf(value.substring(0, 19)).getTime() / 1000; + long nanoAdjustment = Integer.parseInt(value.substring(20)) * 1000L; + return Timestamp.from(Instant.ofEpochSecond(epochSec, nanoAdjustment)); + } + if (precision == TimestampPrecision.NS) { + // ms timestamp: yyyy-MM-dd HH:mm:ss.SSSSSSSSS + long epochSec = Timestamp.valueOf(value.substring(0, 19)).getTime() / 1000; + long nanoAdjustment = Integer.parseInt(value.substring(20)); + return Timestamp.from(Instant.ofEpochSecond(epochSec, nanoAdjustment)); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN_TIMESTAMP_PERCISION); + } + } + } + + public static class Field { String name; int type; int length; @@ -211,6 +232,7 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { this.note = note; this.taos_type = taos_type; } + } @Override @@ -218,10 +240,7 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { if (isClosed()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); pos++; - if (pos <= resultSet.size() - 1) { - return true; - } - return false; + return pos <= resultSet.size() - 1; } @Override @@ -231,13 +250,6 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { } } -// @Override -// public boolean wasNull() throws SQLException { -// if (isClosed()) -// throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); -// return resultSet.isEmpty(); -// } - @Override public String getString(int columnIndex) throws SQLException { checkAvailability(columnIndex, resultSet.get(pos).size()); @@ -262,7 +274,7 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { wasNull = false; if (value instanceof Boolean) return (boolean) value; - return Boolean.valueOf(value.toString()); + return Boolean.parseBoolean(value.toString()); } @Override @@ -334,6 +346,7 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { wasNull = true; return 0; } + wasNull = false; if (value instanceof Timestamp) { return ((Timestamp) value).getTime(); @@ -416,9 +429,7 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { return null; if (value instanceof Timestamp) return new Date(((Timestamp) value).getTime()); - Date date = null; - date = Utils.parseDate(value.toString()); - return date; + return Utils.parseDate(value.toString()); } @Override @@ -433,8 +444,7 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { Time time = null; try { time = Utils.parseTime(value.toString()); - } catch (DateTimeParseException e) { - time = null; + } catch (DateTimeParseException ignored) { } return time; } @@ -498,9 +508,9 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { return null; if (value instanceof Long || value instanceof Integer || value instanceof Short || value instanceof Byte) - return new BigDecimal(Long.valueOf(value.toString())); + return new BigDecimal(Long.parseLong(value.toString())); if (value instanceof Double || value instanceof Float) - return new BigDecimal(Double.valueOf(value.toString())); + return BigDecimal.valueOf(Double.parseDouble(value.toString())); if (value instanceof Timestamp) return new BigDecimal(((Timestamp) value).getTime()); BigDecimal ret; @@ -610,36 +620,6 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { if (isClosed()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); -// if (this.resultSet.size() == 0) -// return false; -// -// if (row == 0) { -// beforeFirst(); -// return false; -// } else if (row == 1) { -// return first(); -// } else if (row == -1) { -// return last(); -// } else if (row > this.resultSet.size()) { -// afterLast(); -// return false; -// } else { -// if (row < 0) { -// // adjust to reflect after end of result set -// int newRowPosition = this.resultSet.size() + row + 1; -// if (newRowPosition <= 0) { -// beforeFirst(); -// return false; -// } else { -// return absolute(newRowPosition); -// } -// } else { -// row--; // adjust for index difference -// this.pos = row; -// return true; -// } -// } - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @@ -683,5 +663,4 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { return isClosed; } - -} +} \ No newline at end of file diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java index 7ead8bd1bb..148f77974f 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java @@ -7,21 +7,20 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Timestamp; import java.sql.Types; -import java.util.ArrayList; +import java.util.Collections; +import java.util.List; public class RestfulResultSetMetaData extends WrapperImpl implements ResultSetMetaData { private final String database; - private ArrayList fields; - private final RestfulResultSet resultSet; + private final List fields; - public RestfulResultSetMetaData(String database, ArrayList fields, RestfulResultSet resultSet) { + public RestfulResultSetMetaData(String database, List fields, RestfulResultSet resultSet) { this.database = database; - this.fields = fields; - this.resultSet = resultSet; + this.fields = fields == null ? Collections.emptyList() : fields; } - public ArrayList getFields() { + public List getFields() { return fields; } @@ -139,8 +138,8 @@ public class RestfulResultSetMetaData extends WrapperImpl implements ResultSetMe @Override public String getColumnTypeName(int column) throws SQLException { - int taos_type = fields.get(column - 1).taos_type; - return TSDBConstants.taosType2JdbcTypeName(taos_type); + int taosType = fields.get(column - 1).taos_type; + return TSDBConstants.taosType2JdbcTypeName(taosType); } @Override diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java index e9d193f6b4..f8acd8f061 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java @@ -35,10 +35,6 @@ public class RestfulStatement extends AbstractStatement { if (!SqlSyntaxValidator.isValidForExecuteQuery(sql)) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_QUERY, "not a valid sql for executeQuery: " + sql); - if (SqlSyntaxValidator.isDatabaseUnspecifiedQuery(sql)) { - return executeOneQuery(sql); - } - return executeOneQuery(sql); } @@ -50,9 +46,6 @@ public class RestfulStatement extends AbstractStatement { throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_UPDATE, "not a valid sql for executeUpdate: " + sql); final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; - if (SqlSyntaxValidator.isDatabaseUnspecifiedUpdate(sql)) { - return executeOneUpdate(url, sql); - } return executeOneUpdate(url, sql); } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java index b1c6dae6b3..e10bdb5aa9 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java @@ -9,6 +9,7 @@ import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.conn.ConnectionKeepAliveStrategy; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.message.BasicHeaderElementIterator; @@ -16,31 +17,33 @@ import org.apache.http.protocol.HTTP; import org.apache.http.protocol.HttpContext; import org.apache.http.util.EntityUtils; -import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; public class HttpClientPoolUtil { private static final String DEFAULT_CONTENT_TYPE = "application/json"; - private static final String DEFAULT_TOKEN = "cm9vdDp0YW9zZGF0YQ=="; private static final int DEFAULT_TIME_OUT = 15000; private static final int DEFAULT_MAX_PER_ROUTE = 32; private static final int DEFAULT_MAX_TOTAL = 1000; private static final int DEFAULT_HTTP_KEEP_TIME = 15000; - private static PoolingHttpClientConnectionManager connectionManager; private static CloseableHttpClient httpClient; private static synchronized void initPools() { if (httpClient == null) { - connectionManager = new PoolingHttpClientConnectionManager(); + PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(); connectionManager.setDefaultMaxPerRoute(DEFAULT_MAX_PER_ROUTE); connectionManager.setMaxTotal(DEFAULT_MAX_TOTAL); - httpClient = HttpClients.custom().setKeepAliveStrategy(DEFAULT_KEEP_ALIVE_STRATEGY).setConnectionManager(connectionManager).build(); + httpClient = HttpClients.custom() + .setKeepAliveStrategy(DEFAULT_KEEP_ALIVE_STRATEGY) + .setConnectionManager(connectionManager) + .setRetryHandler(new DefaultHttpRequestRetryHandler(3, true)) + .build(); } } - private static ConnectionKeepAliveStrategy DEFAULT_KEEP_ALIVE_STRATEGY = (response, context) -> { + private static final ConnectionKeepAliveStrategy DEFAULT_KEEP_ALIVE_STRATEGY = (response, context) -> { HeaderElementIterator it = new BasicHeaderElementIterator(response.headerIterator(HTTP.CONN_KEEP_ALIVE)); int keepTime = DEFAULT_HTTP_KEEP_TIME * 1000; while (it.hasNext()) { @@ -80,7 +83,7 @@ public class HttpClientPoolUtil { method.setHeader("Connection", "keep-alive"); method.setHeader("Authorization", "Taosd " + token); - method.setEntity(new StringEntity(data, Charset.forName("UTF-8"))); + method.setEntity(new StringEntity(data, StandardCharsets.UTF_8)); HttpContext context = HttpClientContext.create(); CloseableHttpResponse httpResponse = httpClient.execute(method, context); httpEntity = httpResponse.getEntity(); @@ -165,28 +168,18 @@ public class HttpClientPoolUtil { httpEntity = httpResponse.getEntity(); if (httpEntity != null) { responseBody = EntityUtils.toString(httpEntity, "UTF-8"); -// logger.info("请求URL: " + uri + "+ 返回状ć€ç ďĽš" + httpResponse.getStatusLine().getStatusCode()); } } catch (Exception e) { if (method != null) { method.abort(); } e.printStackTrace(); -// logger.error("execute get request exception, url:" + uri + ", exception:" + e.toString() + ",cost time(ms):" -// + (System.currentTimeMillis() - startTime)); - System.out.println("log:č°ç”¨ HttpClientPoolUtil execute get request exception, url:" + uri + ", exception:" + e.toString() + ",cost time(ms):" - + (System.currentTimeMillis() - startTime)); } finally { if (httpEntity != null) { try { EntityUtils.consumeQuietly(httpEntity); } catch (Exception e) { -// e.printStackTrace(); -// logger.error("close response exception, url:" + uri + ", exception:" + e.toString() -// + ",cost time(ms):" + (System.currentTimeMillis() - startTime)); - new Exception("close response exception, url:" + uri + ", exception:" + e.toString() - + ",cost time(ms):" + (System.currentTimeMillis() - startTime)) - .printStackTrace(); + new Exception("close response exception, url:" + uri + ", exception:" + e.toString() + ",cost time(ms):" + (System.currentTimeMillis() - startTime)).printStackTrace(); } } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/OSUtils.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/OSUtils.java index a67b4763f9..1af3e9666c 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/OSUtils.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/OSUtils.java @@ -4,14 +4,14 @@ public class OSUtils { private static final String OS = System.getProperty("os.name").toLowerCase(); public static boolean isWindows() { - return OS.indexOf("win") >= 0; + return OS.contains("win"); } public static boolean isMac() { - return OS.indexOf("mac") >= 0; + return OS.contains("mac"); } public static boolean isLinux() { - return OS.indexOf("nux") >= 0; + return OS.contains("nux"); } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java index 251ca2af01..0f99ff4f66 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java @@ -14,24 +14,14 @@ *****************************************************************************/ package com.taosdata.jdbc.utils; -import com.taosdata.jdbc.TSDBConnection; - -import java.sql.Connection; - public class SqlSyntaxValidator { - private static final String[] SQL = {"select", "insert", "import", "create", "use", "alter", "drop", "set", "show", "describe"}; + private static final String[] SQL = {"select", "insert", "import", "create", "use", "alter", "drop", "set", "show", "describe", "reset"}; private static final String[] updateSQL = {"insert", "import", "create", "use", "alter", "drop", "set"}; private static final String[] querySQL = {"select", "show", "describe"}; private static final String[] databaseUnspecifiedShow = {"databases", "dnodes", "mnodes", "variables"}; - private TSDBConnection tsdbConnection; - - public SqlSyntaxValidator(Connection connection) { - this.tsdbConnection = (TSDBConnection) connection; - } - public static boolean isValidForExecuteUpdate(String sql) { for (String prefix : updateSQL) { if (sql.trim().toLowerCase().startsWith(prefix)) @@ -71,29 +61,11 @@ public class SqlSyntaxValidator { public static boolean isUseSql(String sql) { return sql.trim().toLowerCase().startsWith("use"); -// || sql.trim().toLowerCase().matches("create\\s*database.*") || sql.toLowerCase().toLowerCase().matches("drop\\s*database.*"); - } - - public static boolean isShowSql(String sql) { - return sql.trim().toLowerCase().startsWith("show"); - } - - public static boolean isDescribeSql(String sql) { - return sql.trim().toLowerCase().startsWith("describe"); - } - - - public static boolean isInsertSql(String sql) { - return sql.trim().toLowerCase().startsWith("insert") || sql.trim().toLowerCase().startsWith("import"); } public static boolean isSelectSql(String sql) { return sql.trim().toLowerCase().startsWith("select"); } - public static boolean isShowDatabaseSql(String sql) { - return sql.trim().toLowerCase().matches("show\\s*databases"); - } - } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java index ee1364ce21..a427103770 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java @@ -7,9 +7,9 @@ import java.util.concurrent.atomic.AtomicLong; public class TaosInfo implements TaosInfoMBean { private static volatile TaosInfo instance; - private AtomicLong connect_open = new AtomicLong(); - private AtomicLong connect_close = new AtomicLong(); - private AtomicLong statement_count = new AtomicLong(); + private final AtomicLong connect_open = new AtomicLong(); + private final AtomicLong connect_close = new AtomicLong(); + private final AtomicLong statement_count = new AtomicLong(); static { try { diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java index 8ab610fec6..9cf61903f0 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java @@ -3,14 +3,13 @@ package com.taosdata.jdbc.utils; import com.google.common.collect.Range; import com.google.common.collect.RangeSet; import com.google.common.collect.TreeRangeSet; +import com.taosdata.jdbc.enums.TimestampPrecision; -import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.sql.Date; import java.sql.Time; import java.sql.Timestamp; -import java.time.LocalDate; import java.time.LocalDateTime; -import java.time.LocalTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; import java.time.format.DateTimeParseException; @@ -23,46 +22,58 @@ import java.util.stream.IntStream; public class Utils { - private static Pattern ptn = Pattern.compile(".*?'"); - - private static final DateTimeFormatter formatter = new DateTimeFormatterBuilder() - .appendPattern("yyyy-MM-dd HH:mm:ss.SSS").toFormatter(); - private static final DateTimeFormatter formatter2 = new DateTimeFormatterBuilder() - .appendPattern("yyyy-MM-dd HH:mm:ss.SSSSSS").toFormatter(); + private static final Pattern ptn = Pattern.compile(".*?'"); + private static final DateTimeFormatter milliSecFormatter = new DateTimeFormatterBuilder().appendPattern("yyyy-MM-dd HH:mm:ss.SSS").toFormatter(); + private static final DateTimeFormatter microSecFormatter = new DateTimeFormatterBuilder().appendPattern("yyyy-MM-dd HH:mm:ss.SSSSSS").toFormatter(); + private static final DateTimeFormatter nanoSecFormatter = new DateTimeFormatterBuilder().appendPattern("yyyy-MM-dd HH:mm:ss.SSSSSSSSS").toFormatter(); public static Time parseTime(String timestampStr) throws DateTimeParseException { - LocalTime time; - try { - time = LocalTime.parse(timestampStr, formatter); - } catch (DateTimeParseException e) { - time = LocalTime.parse(timestampStr, formatter2); - } - return Time.valueOf(time); + LocalDateTime dateTime = parseLocalDateTime(timestampStr); + return dateTime != null ? Time.valueOf(dateTime.toLocalTime()) : null; } - public static Date parseDate(String timestampStr) throws DateTimeParseException { - LocalDate date; - try { - date = LocalDate.parse(timestampStr, formatter); - } catch (DateTimeParseException e) { - date = LocalDate.parse(timestampStr, formatter2); - } - return Date.valueOf(date); + public static Date parseDate(String timestampStr) { + LocalDateTime dateTime = parseLocalDateTime(timestampStr); + return dateTime != null ? Date.valueOf(String.valueOf(dateTime)) : null; } public static Timestamp parseTimestamp(String timeStampStr) { - LocalDateTime dateTime; + LocalDateTime dateTime = parseLocalDateTime(timeStampStr); + return dateTime != null ? Timestamp.valueOf(dateTime) : null; + } + + private static LocalDateTime parseLocalDateTime(String timeStampStr) { try { - dateTime = LocalDateTime.parse(timeStampStr, formatter); + return parseMilliSecTimestamp(timeStampStr); } catch (DateTimeParseException e) { - dateTime = LocalDateTime.parse(timeStampStr, formatter2); + try { + return parseMicroSecTimestamp(timeStampStr); + } catch (DateTimeParseException ee) { + try { + return parseNanoSecTimestamp(timeStampStr); + } catch (DateTimeParseException eee) { + eee.printStackTrace(); + } + } } - return Timestamp.valueOf(dateTime); + return null; + } + + private static LocalDateTime parseMilliSecTimestamp(String timeStampStr) throws DateTimeParseException { + return LocalDateTime.parse(timeStampStr, milliSecFormatter); + } + + private static LocalDateTime parseMicroSecTimestamp(String timeStampStr) throws DateTimeParseException { + return LocalDateTime.parse(timeStampStr, microSecFormatter); + } + + private static LocalDateTime parseNanoSecTimestamp(String timeStampStr) throws DateTimeParseException { + return LocalDateTime.parse(timeStampStr, nanoSecFormatter); } public static String escapeSingleQuota(String origin) { Matcher m = ptn.matcher(origin); - StringBuffer sb = new StringBuffer(); + StringBuilder sb = new StringBuilder(); int end = 0; while (m.find()) { end = m.end(); @@ -75,7 +86,7 @@ public class Utils { sb.append(seg); } } else { // len > 1 - sb.append(seg.substring(0, seg.length() - 2)); + sb.append(seg, 0, seg.length() - 2); char lastcSec = seg.charAt(seg.length() - 2); if (lastcSec == '\\') { sb.append("\\'"); @@ -93,17 +104,31 @@ public class Utils { } public static String getNativeSql(String rawSql, Object[] parameters) { + if (parameters == null || !rawSql.contains("?")) + return rawSql; // toLowerCase String preparedSql = rawSql.trim().toLowerCase(); - String[] clause = new String[]{"values\\s*\\(.*?\\)", "tags\\s*\\(.*?\\)", "where\\s*.*"}; + String[] clause = new String[]{"tags\\s*\\([\\s\\S]*?\\)", "where[\\s\\S]*"}; Map placeholderPositions = new HashMap<>(); RangeSet clauseRangeSet = TreeRangeSet.create(); findPlaceholderPosition(preparedSql, placeholderPositions); + // find tags and where clause's position findClauseRangeSet(preparedSql, clause, clauseRangeSet); + // find values clause's position + findValuesClauseRangeSet(preparedSql, clauseRangeSet); return transformSql(rawSql, parameters, placeholderPositions, clauseRangeSet); } + private static void findValuesClauseRangeSet(String preparedSql, RangeSet clauseRangeSet) { + Matcher matcher = Pattern.compile("(values|,)\\s*(\\([^)]*\\))").matcher(preparedSql); + while (matcher.find()) { + int start = matcher.start(2); + int end = matcher.end(2); + clauseRangeSet.add(Range.closedOpen(start, end)); + } + } + private static void findClauseRangeSet(String preparedSql, String[] regexArr, RangeSet clauseRangeSet) { clauseRangeSet.clear(); for (String regex : regexArr) { @@ -111,7 +136,7 @@ public class Utils { while (matcher.find()) { int start = matcher.start(); int end = matcher.end(); - clauseRangeSet.add(Range.closed(start, end)); + clauseRangeSet.add(Range.closedOpen(start, end)); } } } @@ -146,7 +171,7 @@ public class Utils { String paraStr; if (para != null) { if (para instanceof byte[]) { - paraStr = new String((byte[]) para, Charset.forName("UTF-8")); + paraStr = new String((byte[]) para, StandardCharsets.UTF_8); } else { paraStr = para.toString(); } @@ -167,13 +192,47 @@ public class Utils { }).collect(Collectors.joining()); } - public static String formatTimestamp(Timestamp timestamp) { int nanos = timestamp.getNanos(); - if (nanos % 1000000l != 0) - return timestamp.toLocalDateTime().format(formatter2); - return timestamp.toLocalDateTime().format(formatter); + if (nanos % 1000000L != 0) + return timestamp.toLocalDateTime().format(microSecFormatter); + return timestamp.toLocalDateTime().format(milliSecFormatter); } + public static TimestampPrecision guessTimestampPrecision(String value) { + if (isMilliSecFormat(value)) + return TimestampPrecision.MS; + if (isMicroSecFormat(value)) + return TimestampPrecision.US; + if (isNanoSecFormat(value)) + return TimestampPrecision.NS; + return TimestampPrecision.UNKNOWN; + } + private static boolean isMilliSecFormat(String timestampStr) { + try { + milliSecFormatter.parse(timestampStr); + } catch (DateTimeParseException e) { + return false; + } + return true; + } + + private static boolean isMicroSecFormat(String timestampStr) { + try { + microSecFormatter.parse(timestampStr); + } catch (DateTimeParseException e) { + return false; + } + return true; + } + + private static boolean isNanoSecFormat(String timestampStr) { + try { + nanoSecFormatter.parse(timestampStr); + } catch (DateTimeParseException e) { + return false; + } + return true; + } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java deleted file mode 100644 index 73ceafa729..0000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java +++ /dev/null @@ -1,108 +0,0 @@ -package com.taosdata.jdbc; - -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.sql.*; -import java.util.Properties; - -public class StatementTest { - static Connection connection = null; - static Statement statement = null; - static String dbName = "test"; - static String tName = "t0"; - static String host = "localhost"; - - @BeforeClass - public static void createConnection() throws SQLException { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/?user=root&password=taosdata", properties); - statement = connection.createStatement(); - statement.executeUpdate("drop database if exists " + dbName); - - } catch (ClassNotFoundException e) { - return; - } - } - - @Test - public void testCase() { - try { - ResultSet rs = statement.executeQuery("show databases"); - ResultSetMetaData metaData = rs.getMetaData(); - while (rs.next()) { - for (int i = 1; i <= metaData.getColumnCount(); i++) { - System.out.print(metaData.getColumnLabel(i) + ":" + rs.getString(i) + "\t"); - } - System.out.println(); - } - } catch (SQLException e) { - e.printStackTrace(); - } - } - - @Test - public void createTableAndQuery() throws SQLException { - long ts = System.currentTimeMillis(); - - statement.executeUpdate("create database if not exists " + dbName); - statement.executeUpdate("create table if not exists " + dbName + "." + tName + "(ts timestamp, k1 int)"); - statement.executeUpdate("insert into " + dbName + "." + tName + " values (" + ts + ", 1)"); - statement.execute("select * from " + dbName + "." + tName); - ResultSet resultSet = statement.getResultSet(); - Assert.assertNotNull(resultSet); - - boolean isClosed = statement.isClosed(); - Assert.assertEquals(false, isClosed); - } - - @Test(expected = SQLException.class) - public void testUnsupport() throws SQLException { - Assert.assertNotNull(statement.unwrap(TSDBStatement.class)); - Assert.assertTrue(statement.isWrapperFor(TSDBStatement.class)); - - statement.getMaxFieldSize(); - statement.setMaxFieldSize(0); - statement.setEscapeProcessing(true); - statement.cancel(); - statement.getWarnings(); - statement.clearWarnings(); - statement.setCursorName(null); - statement.getMoreResults(); - statement.setFetchDirection(0); - statement.getFetchDirection(); - statement.getResultSetConcurrency(); - statement.getResultSetType(); - statement.getConnection(); - statement.getMoreResults(); - statement.getGeneratedKeys(); - statement.executeUpdate(null, 0); - statement.executeUpdate(null, new int[]{0}); - statement.executeUpdate(null, new String[]{"str1", "str2"}); - statement.getResultSetHoldability(); - statement.setPoolable(true); - statement.isPoolable(); - statement.closeOnCompletion(); - statement.isCloseOnCompletion(); - } - - @AfterClass - public static void close() { - try { - statement.execute("drop database if exists " + dbName); - if (statement != null) - statement.close(); - if (connection != null) - connection.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - } -} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBConnectionTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBConnectionTest.java index dc6d0d322a..7cdda572a7 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBConnectionTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBConnectionTest.java @@ -5,7 +5,6 @@ import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; -import javax.management.OperationsException; import java.sql.*; import java.util.Properties; @@ -380,14 +379,15 @@ public class TSDBConnectionTest { conn.abort(null); } - @Test(expected = SQLFeatureNotSupportedException.class) + @Test public void setNetworkTimeout() throws SQLException { conn.setNetworkTimeout(null, 1000); } - @Test(expected = SQLFeatureNotSupportedException.class) + @Test public void getNetworkTimeout() throws SQLException { - conn.getNetworkTimeout(); + int networkTimeout = conn.getNetworkTimeout(); + Assert.assertEquals(0, networkTimeout); } @Test diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBJNIConnectorTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBJNIConnectorTest.java index 66078ef503..88ff5d3a81 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBJNIConnectorTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBJNIConnectorTest.java @@ -2,11 +2,16 @@ package com.taosdata.jdbc; import org.junit.Test; +import java.lang.management.ManagementFactory; +import java.lang.management.RuntimeMXBean; import java.sql.SQLException; import java.sql.SQLWarning; import java.util.ArrayList; import java.util.List; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + public class TSDBJNIConnectorTest { private static TSDBResultSetRowData rowData; @@ -14,17 +19,68 @@ public class TSDBJNIConnectorTest { @Test public void test() { try { + + try { + //change sleepSeconds when debugging with attach to process to find PID + int sleepSeconds = -1; + if (sleepSeconds>0) { + RuntimeMXBean runtimeBean = ManagementFactory.getRuntimeMXBean(); + String jvmName = runtimeBean.getName(); + long pid = Long.valueOf(jvmName.split("@")[0]); + System.out.println("JVM PID = " + pid); + + Thread.sleep(sleepSeconds*1000); + } + } + catch (Exception e) { + e.printStackTrace(); + } + // init - TSDBJNIConnector.init("/etc/taos/taos.cfg", null, null, null); + TSDBJNIConnector.init("/etc/taos", null, null, null); + // connect TSDBJNIConnector connector = new TSDBJNIConnector(); - connector.connect("127.0.0.1", 6030, "unsign_jni", "root", "taosdata"); + connector.connect("127.0.0.1", 6030, null, "root", "taosdata"); + + // setup + String setupSqlStrs[] = {"create database if not exists d precision \"us\"", + "create table if not exists d.t(ts timestamp, f int)", + "create database if not exists d2", + "create table if not exists d2.t2(ts timestamp, f int)", + "insert into d.t values(now+100s, 100)", + "insert into d2.t2 values(now+200s, 200)" + }; + for (String setupSqlStr : setupSqlStrs) { + long setupSql = connector.executeQuery(setupSqlStr); + + assertEquals(0, connector.getResultTimePrecision(setupSql)); + if (connector.isUpdateQuery(setupSql)) { + connector.freeResultSet(setupSql); + } + } + + { + long sqlObj1 = connector.executeQuery("select * from d2.t2"); + assertEquals(0, connector.getResultTimePrecision(sqlObj1)); + List columnMetaDataList = new ArrayList<>(); + int code = connector.getSchemaMetaData(sqlObj1, columnMetaDataList); + rowData = new TSDBResultSetRowData(columnMetaDataList.size()); + assertTrue(next(connector, sqlObj1)); + assertEquals(0, connector.getResultTimePrecision(sqlObj1)); + connector.freeResultSet(sqlObj1); + } + // executeQuery - long pSql = connector.executeQuery("select * from unsign_jni.us_table"); + long pSql = connector.executeQuery("select * from d.t"); + if (connector.isUpdateQuery(pSql)) { connector.freeResultSet(pSql); throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_WITH_EXECUTEQUERY); } + + assertEquals(1, connector.getResultTimePrecision(pSql)); + // get schema List columnMetaDataList = new ArrayList<>(); int code = connector.getSchemaMetaData(pSql, columnMetaDataList); @@ -37,6 +93,8 @@ public class TSDBJNIConnectorTest { if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) { throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_NUM_OF_FIELDS_0); } + + assertEquals(1, connector.getResultTimePrecision(pSql)); int columnSize = columnMetaDataList.size(); // print metadata for (int i = 0; i < columnSize; i++) { @@ -45,9 +103,8 @@ public class TSDBJNIConnectorTest { rowData = new TSDBResultSetRowData(columnSize); // iterate resultSet for (int i = 0; next(connector, pSql); i++) { -// System.out.println("col[" + i + "] size: " + rowData.getColSize()); -// rowData.getData().stream().forEach(col -> System.out.print(col + "\t")); -// System.out.println(); + assertEquals(1, connector.getResultTimePrecision(pSql)); + System.out.println(); } // close resultSet code = connector.freeResultSet(pSql); @@ -57,6 +114,10 @@ public class TSDBJNIConnectorTest { throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_RESULT_SET_NULL); } // close statement + connector.executeQuery("use d"); + String[] lines = new String[] {"st,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000ns", + "st,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64 1626006833640000000ns"}; + connector.insertLines(lines); // close connection connector.closeConnection(); @@ -86,4 +147,4 @@ public class TSDBJNIConnectorTest { } } -} \ No newline at end of file +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBParameterMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBParameterMetaDataTest.java index 83caf1bebb..dc41d85cf3 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBParameterMetaDataTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBParameterMetaDataTest.java @@ -1,6 +1,5 @@ package com.taosdata.jdbc; -import com.taosdata.jdbc.rs.RestfulParameterMetaData; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java index 0daf6c00bc..73853cf7a1 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java @@ -2,7 +2,6 @@ package com.taosdata.jdbc; import org.junit.*; -import java.io.IOException; import java.math.BigDecimal; import java.sql.*; import java.util.ArrayList; @@ -14,6 +13,7 @@ public class TSDBPreparedStatementTest { private static Connection conn; private static final String sql_insert = "insert into t1 values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; private static final String sql_select = "select * from t1 where ts >= ? and ts < ? and f1 >= ?"; + private static final String dbname = "test_pstmt_jni"; private PreparedStatement pstmt_insert; private PreparedStatement pstmt_select; @@ -25,7 +25,7 @@ public class TSDBPreparedStatementTest { long ts = System.currentTimeMillis(); pstmt_insert.setTimestamp(1, new Timestamp(ts)); pstmt_insert.setInt(2, 2); - pstmt_insert.setLong(3, 3l); + pstmt_insert.setLong(3, 3L); pstmt_insert.setFloat(4, 3.14f); pstmt_insert.setDouble(5, 3.1415); pstmt_insert.setShort(6, (short) 6); @@ -52,8 +52,8 @@ public class TSDBPreparedStatementTest { Assert.assertEquals(ts, rs.getTimestamp(1).getTime()); Assert.assertEquals(2, rs.getInt(2)); Assert.assertEquals(2, rs.getInt("f1")); - Assert.assertEquals(3l, rs.getLong(3)); - Assert.assertEquals(3l, rs.getLong("f2")); + Assert.assertEquals(3L, rs.getLong(3)); + Assert.assertEquals(3L, rs.getLong("f2")); Assert.assertEquals(3.14f, rs.getFloat(4), 0.0); Assert.assertEquals(3.14f, rs.getFloat("f3"), 0.0); Assert.assertEquals(3.1415, rs.getDouble(5), 0.0); @@ -296,7 +296,306 @@ public class TSDBPreparedStatementTest { assertAllNullExceptTimestamp(rs, ts); } } - + + /* + @Test + public void executeTest() throws SQLException { + Statement stmt = conn.createStatement(); + + int numOfRows = 1000; + + for (int loop = 0; loop < 10; loop++) { + stmt.execute("drop table if exists weather_test"); + stmt.execute("create table weather_test(ts timestamp, f1 nchar(4), f2 float, f3 double, f4 timestamp, f5 int, f6 bool, f7 binary(10))"); + + TSDBPreparedStatement s = (TSDBPreparedStatement) conn.prepareStatement("insert into ? values(?, ?, ?, ?, ?, ?, ?, ?)"); + Random r = new Random(); + s.setTableName("weather_test"); + + ArrayList ts = new ArrayList<>(); + for (int i = 0; i < numOfRows; i++) { + ts.add(System.currentTimeMillis() + i); + } + s.setTimestamp(0, ts); + + int random = 10 + r.nextInt(5); + ArrayList s2 = new ArrayList<>(); + for (int i = 0; i < numOfRows; i++) { + if (i % random == 0) { + s2.add(null); + } else { + s2.add("ĺ†ć”Ż" + i % 4); + } + } + s.setNString(1, s2, 4); + + random = 10 + r.nextInt(5); + ArrayList s3 = new ArrayList<>(); + for (int i = 0; i < numOfRows; i++) { + if (i % random == 0) { + s3.add(null); + } else { + s3.add(r.nextFloat()); + } + } + s.setFloat(2, s3); + + random = 10 + r.nextInt(5); + ArrayList s4 = new ArrayList<>(); + for (int i = 0; i < numOfRows; i++) { + if (i % random == 0) { + s4.add(null); + } else { + s4.add(r.nextDouble()); + } + } + s.setDouble(3, s4); + + random = 10 + r.nextInt(5); + ArrayList ts2 = new ArrayList<>(); + for (int i = 0; i < numOfRows; i++) { + if (i % random == 0) { + ts2.add(null); + } else { + ts2.add(System.currentTimeMillis() + i); + } + } + s.setTimestamp(4, ts2); + + random = 10 + r.nextInt(5); + ArrayList vals = new ArrayList<>(); + for (int i = 0; i < numOfRows; i++) { + if (i % random == 0) { + vals.add(null); + } else { + vals.add(r.nextInt()); + } + } + s.setInt(5, vals); + + random = 10 + r.nextInt(5); + ArrayList sb = new ArrayList<>(); + for (int i = 0; i < numOfRows; i++) { + if (i % random == 0) { + sb.add(null); + } else { + sb.add(i % 2 == 0); + } + } + s.setBoolean(6, sb); + + random = 10 + r.nextInt(5); + ArrayList s5 = new ArrayList<>(); + for (int i = 0; i < numOfRows; i++) { + if (i % random == 0) { + s5.add(null); + } else { + s5.add("test" + i % 10); + } + } + s.setString(7, s5, 10); + + s.columnDataAddBatch(); + s.columnDataExecuteBatch(); + s.columnDataCloseBatch(); + + String sql = "select * from weather_test"; + PreparedStatement statement = conn.prepareStatement(sql); + ResultSet rs = statement.executeQuery(); + int rows = 0; + while (rs.next()) { + rows++; + } + Assert.assertEquals(numOfRows, rows); + } + } + + @Test + public void bindDataSelectColumnTest() throws SQLException { + Statement stmt = conn.createStatement(); + + int numOfRows = 1000; + + for (int loop = 0; loop < 10; loop++) { + stmt.execute("drop table if exists weather_test"); + stmt.execute("create table weather_test(ts timestamp, f1 nchar(4), f2 float, f3 double, f4 timestamp, f5 int, f6 bool, f7 binary(10))"); + + TSDBPreparedStatement s = (TSDBPreparedStatement) conn.prepareStatement("insert into ? (ts, f1, f7) values(?, ?, ?)"); + Random r = new Random(); + s.setTableName("weather_test"); + + ArrayList ts = new ArrayList<>(); + for (int i = 0; i < numOfRows; i++) { + ts.add(System.currentTimeMillis() + i); + } + s.setTimestamp(0, ts); + + int random = 10 + r.nextInt(5); + ArrayList s2 = new ArrayList<>(); + for (int i = 0; i < numOfRows; i++) { + if (i % random == 0) { + s2.add(null); + } else { + s2.add("ĺ†ć”Ż" + i % 4); + } + } + s.setNString(1, s2, 4); + + random = 10 + r.nextInt(5); + ArrayList s3 = new ArrayList<>(); + for (int i = 0; i < numOfRows; i++) { + if (i % random == 0) { + s3.add(null); + } else { + s3.add("test" + i % 10); + } + } + s.setString(2, s3, 10); + + s.columnDataAddBatch(); + s.columnDataExecuteBatch(); + s.columnDataCloseBatch(); + + String sql = "select * from weather_test"; + PreparedStatement statement = conn.prepareStatement(sql); + ResultSet rs = statement.executeQuery(); + int rows = 0; + while (rs.next()) { + rows++; + } + Assert.assertEquals(numOfRows, rows); + } + } + + @Test + public void bindDataWithSingleTagTest() throws SQLException { + Statement stmt = conn.createStatement(); + + String[] types = new String[]{"tinyint", "smallint", "int", "bigint", "bool", "float", "double", "binary(10)", "nchar(10)"}; + + for (String type : types) { + stmt.execute("drop table if exists weather_test"); + stmt.execute("create table weather_test(ts timestamp, f1 nchar(10), f2 binary(10)) tags (t " + type + ")"); + + int numOfRows = 1; + + TSDBPreparedStatement s = (TSDBPreparedStatement) conn.prepareStatement("insert into ? using weather_test tags(?) values(?, ?, ?)"); + Random r = new Random(); + s.setTableName("w1"); + + switch (type) { + case "tinyint": + case "smallint": + case "int": + case "bigint": + s.setTagInt(0, 1); + break; + case "float": + s.setTagFloat(0, 1.23f); + break; + case "double": + s.setTagDouble(0, 3.14159265); + break; + case "bool": + s.setTagBoolean(0, true); + break; + case "binary(10)": + s.setTagString(0, "test"); + break; + case "nchar(10)": + s.setTagNString(0, "test"); + break; + default: + break; + } + + + ArrayList ts = new ArrayList<>(); + for (int i = 0; i < numOfRows; i++) { + ts.add(System.currentTimeMillis() + i); + } + s.setTimestamp(0, ts); + + int random = 10 + r.nextInt(5); + ArrayList s2 = new ArrayList<>(); + for (int i = 0; i < numOfRows; i++) { + s2.add("ĺ†ć”Ż" + i % 4); + } + s.setNString(1, s2, 10); + + random = 10 + r.nextInt(5); + ArrayList s3 = new ArrayList<>(); + for (int i = 0; i < numOfRows; i++) { + s3.add("test" + i % 4); + } + s.setString(2, s3, 10); + + s.columnDataAddBatch(); + s.columnDataExecuteBatch(); + s.columnDataCloseBatch(); + + String sql = "select * from weather_test"; + PreparedStatement statement = conn.prepareStatement(sql); + ResultSet rs = statement.executeQuery(); + int rows = 0; + while (rs.next()) { + rows++; + } + Assert.assertEquals(numOfRows, rows); + } + } + + + @Test + public void bindDataWithMultipleTagsTest() throws SQLException { + Statement stmt = conn.createStatement(); + + stmt.execute("drop table if exists weather_test"); + stmt.execute("create table weather_test(ts timestamp, f1 nchar(10), f2 binary(10)) tags (t1 int, t2 binary(10))"); + + int numOfRows = 1; + + TSDBPreparedStatement s = (TSDBPreparedStatement) conn.prepareStatement("insert into ? using weather_test tags(?,?) (ts, f2) values(?, ?)"); + s.setTableName("w2"); + s.setTagInt(0, 1); + s.setTagString(1, "test"); + + + ArrayList ts = new ArrayList<>(); + for (int i = 0; i < numOfRows; i++) { + ts.add(System.currentTimeMillis() + i); + } + s.setTimestamp(0, ts); + + ArrayList s2 = new ArrayList<>(); + for (int i = 0; i < numOfRows; i++) { + s2.add("test" + i % 4); + } + s.setString(1, s2, 10); + + s.columnDataAddBatch(); + s.columnDataExecuteBatch(); + s.columnDataCloseBatch(); + + String sql = "select * from weather_test"; + PreparedStatement statement = conn.prepareStatement(sql); + ResultSet rs = statement.executeQuery(); + int rows = 0; + while (rs.next()) { + rows++; + } + Assert.assertEquals(numOfRows, rows); + } + */ + + @Test(expected = SQLException.class) + public void createTwoSameDbTest() throws SQLException { + // when + Statement stmt = conn.createStatement(); + stmt.execute("create database dbtest"); + stmt.execute("create database dbtest"); + } + @Test public void setBoolean() throws SQLException { // given @@ -490,7 +789,7 @@ public class TSDBPreparedStatementTest { public void setBigDecimal() throws SQLException { // given long ts = System.currentTimeMillis(); - BigDecimal bigDecimal = new BigDecimal(3.14444); + BigDecimal bigDecimal = new BigDecimal("3.14444"); // when pstmt_insert.setTimestamp(1, new Timestamp(ts)); @@ -543,13 +842,13 @@ public class TSDBPreparedStatementTest { } @Test - public void setBytes() throws SQLException, IOException { + public void setBytes() throws SQLException { // given long ts = System.currentTimeMillis(); byte[] f8 = "{\"name\": \"john\", \"age\": 10, \"address\": \"192.168.1.100\"}".getBytes(); // when - pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setTimestamp(1, new Timestamp(ts)); pstmt_insert.setBytes(9, f8); int result = pstmt_insert.executeUpdate(); @@ -701,7 +1000,7 @@ public class TSDBPreparedStatementTest { long ts = System.currentTimeMillis(); pstmt_insert.setTimestamp(1, new Timestamp(ts)); pstmt_insert.setInt(2, 2); - pstmt_insert.setLong(3, 3l); + pstmt_insert.setLong(3, 3L); pstmt_insert.setFloat(4, 3.14f); pstmt_insert.setDouble(5, 3.1415); pstmt_insert.setShort(6, (short) 6); @@ -798,9 +1097,9 @@ public class TSDBPreparedStatementTest { try { conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"); try (Statement stmt = conn.createStatement()) { - stmt.execute("drop database if exists test_pstmt_jni"); - stmt.execute("create database if not exists test_pstmt_jni"); - stmt.execute("use test_pstmt_jni"); + stmt.execute("drop database if exists " + dbname); + stmt.execute("create database if not exists " + dbname); + stmt.execute("use " + dbname); } } catch (SQLException e) { e.printStackTrace(); @@ -810,6 +1109,9 @@ public class TSDBPreparedStatementTest { @AfterClass public static void afterClass() { try { + Statement statement = conn.createStatement(); + statement.execute("drop database if exists " + dbname); + statement.close(); if (conn != null) conn.close(); } catch (SQLException e) { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java index ec54a44b7c..f72cbbec8c 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java @@ -95,13 +95,13 @@ public class TSDBResultSetTest { @Test public void getBigDecimal() throws SQLException { BigDecimal f1 = rs.getBigDecimal("f1"); - Assert.assertEquals(1609430400000l, f1.longValue()); + Assert.assertEquals(1609430400000L, f1.longValue()); BigDecimal f2 = rs.getBigDecimal("f2"); Assert.assertEquals(1, f2.intValue()); BigDecimal f3 = rs.getBigDecimal("f3"); - Assert.assertEquals(100l, f3.longValue()); + Assert.assertEquals(100L, f3.longValue()); BigDecimal f4 = rs.getBigDecimal("f4"); Assert.assertEquals(3.1415f, f4.floatValue(), 0.00000f); @@ -125,13 +125,13 @@ public class TSDBResultSetTest { Assert.assertEquals(1, Ints.fromByteArray(f2)); byte[] f3 = rs.getBytes("f3"); - Assert.assertEquals(100l, Longs.fromByteArray(f3)); + Assert.assertEquals(100L, Longs.fromByteArray(f3)); byte[] f4 = rs.getBytes("f4"); - Assert.assertEquals(3.1415f, Float.valueOf(new String(f4)), 0.000000f); + Assert.assertEquals(3.1415f, Float.parseFloat(new String(f4)), 0.000000f); byte[] f5 = rs.getBytes("f5"); - Assert.assertEquals(3.1415926, Double.valueOf(new String(f5)), 0.000000f); + Assert.assertEquals(3.1415926, Double.parseDouble(new String(f5)), 0.000000f); byte[] f6 = rs.getBytes("f6"); Assert.assertTrue(Arrays.equals("abc".getBytes(), f6)); @@ -223,7 +223,7 @@ public class TSDBResultSetTest { Object f3 = rs.getObject("f3"); Assert.assertEquals(Long.class, f3.getClass()); - Assert.assertEquals(100l, f3); + Assert.assertEquals(100L, f3); Object f4 = rs.getObject("f4"); Assert.assertEquals(Float.class, f4.getClass()); @@ -421,7 +421,7 @@ public class TSDBResultSetTest { @Test(expected = SQLFeatureNotSupportedException.class) public void updateLong() throws SQLException { - rs.updateLong(1, 1l); + rs.updateLong(1, 1L); } @Test(expected = SQLFeatureNotSupportedException.class) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBStatementTest.java index 51535bc886..c1dfa42511 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBStatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBStatementTest.java @@ -14,24 +14,6 @@ public class TSDBStatementTest { private static Connection conn; private static Statement stmt; - @Test - public void executeQuery() { - try { - ResultSet rs = stmt.executeQuery("show databases"); - Assert.assertNotNull(rs); - ResultSetMetaData meta = rs.getMetaData(); - while (rs.next()) { - for (int i = 1; i <= meta.getColumnCount(); i++) { - System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); - } - System.out.println(); - } - rs.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - } - @Test public void executeUpdate() { final String dbName = ("test_" + UUID.randomUUID()).replace("-", "_").substring(0, 32); @@ -173,10 +155,6 @@ public class TSDBStatementTest { Assert.assertEquals(3, meta.getColumnCount()); int count = 0; while (rs.next()) { - for (int i = 1; i <= meta.getColumnCount(); i++) { - System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); - } - System.out.println(); count++; } Assert.assertEquals(1, count); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java index 19bc5f713f..4f37183e71 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java @@ -1,5 +1,6 @@ package com.taosdata.jdbc.cases; +import org.junit.Assert; import org.junit.Test; import java.sql.Connection; @@ -12,21 +13,19 @@ public class AppMemoryLeakTest { @Test(expected = SQLException.class) public void testCreateTooManyConnection() throws ClassNotFoundException, SQLException { Class.forName("com.taosdata.jdbc.TSDBDriver"); - int conCnt = 0; while (true) { Connection conn = DriverManager.getConnection("jdbc:TAOS://localhost:6030/?user=root&password=taosdata"); - System.out.println(conCnt++ + " : " + conn); + Assert.assertNotNull(conn); } } @Test(expected = Exception.class) public void testCreateTooManyStatement() throws ClassNotFoundException, SQLException { Class.forName("com.taosdata.jdbc.TSDBDriver"); - int stmtCnt = 0; Connection conn = DriverManager.getConnection("jdbc:TAOS://localhost:6030/?user=root&password=taosdata"); while (true) { Statement stmt = conn.createStatement(); - System.out.println(++stmtCnt + " : " + stmt); + Assert.assertNotNull(stmt); } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/AuthenticationTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java similarity index 98% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/AuthenticationTest.java rename to src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java index a6fb6cfda0..6702de9bdb 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/AuthenticationTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java @@ -1,4 +1,4 @@ -package com.taosdata.jdbc.rs; +package com.taosdata.jdbc.cases; import org.junit.Before; import org.junit.Test; diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BadLocaleSettingTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BadLocaleSettingTest.java new file mode 100644 index 0000000000..2211e0fa17 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BadLocaleSettingTest.java @@ -0,0 +1,59 @@ +package com.taosdata.jdbc.cases; + + +import com.taosdata.jdbc.TSDBDriver; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; + +public class BadLocaleSettingTest { + + private static final String host = "127.0.0.1"; + private static final String dbName = "bad_locale_test"; + private static Connection conn; + + @Test + public void canSetLocale() { + try { + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + + String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; + conn = DriverManager.getConnection(url, properties); + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists " + dbName); + stmt.execute("create database if not exists " + dbName); + stmt.execute("use " + dbName); + stmt.execute("drop table if exists weather"); + stmt.execute("create table weather(ts timestamp, temperature float, humidity int)"); + stmt.executeUpdate("insert into weather values(1624071506435, 12.3, 4)"); + stmt.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @BeforeClass + public static void beforeClass() { + System.setProperty("sun.jnu.encoding", "ANSI_X3.4-1968"); + System.setProperty("file.encoding", "ANSI_X3.4-1968"); + } + + @AfterClass + public static void afterClass() { + try { + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchErrorIgnoreTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchErrorIgnoreTest.java new file mode 100644 index 0000000000..2934b54b5b --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchErrorIgnoreTest.java @@ -0,0 +1,144 @@ +package com.taosdata.jdbc.cases; + +import org.junit.*; + +import java.sql.*; +import java.util.stream.IntStream; + +public class BatchErrorIgnoreTest { + + private static final String host = "127.0.0.1"; + + @Test + public void batchErrorThrowException() throws SQLException { + // given + Connection conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"); + + // when + try (Statement stmt = conn.createStatement()) { + IntStream.range(1, 6).mapToObj(i -> "insert into test.t" + i + " values(now, " + i + ")").forEach(sql -> { + try { + stmt.addBatch(sql); + } catch (SQLException e) { + e.printStackTrace(); + } + }); + stmt.addBatch("insert into t11 values(now, 11)"); + IntStream.range(6, 11).mapToObj(i -> "insert into test.t" + i + " values(now, " + i + "),(now + 1s, " + (10 * i) + ")").forEach(sql -> { + try { + stmt.addBatch(sql); + } catch (SQLException e) { + e.printStackTrace(); + } + }); + stmt.addBatch("select count(*) from test.weather"); + + stmt.executeBatch(); + } catch (BatchUpdateException e) { + int[] updateCounts = e.getUpdateCounts(); + Assert.assertEquals(5, updateCounts.length); + Assert.assertEquals(1, updateCounts[0]); + Assert.assertEquals(1, updateCounts[1]); + Assert.assertEquals(1, updateCounts[2]); + Assert.assertEquals(1, updateCounts[3]); + Assert.assertEquals(1, updateCounts[4]); + } + + } + + @Test + public void batchErrorIgnore() throws SQLException { + // given + Connection conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata&batchErrorIgnore=true"); + + // when + int[] results = null; + try (Statement stmt = conn.createStatement()) { + IntStream.range(1, 6).mapToObj(i -> "insert into test.t" + i + " values(now, " + i + ")").forEach(sql -> { + try { + stmt.addBatch(sql); + } catch (SQLException e) { + e.printStackTrace(); + } + }); + stmt.addBatch("insert into t11 values(now, 11)"); + IntStream.range(6, 11).mapToObj(i -> "insert into test.t" + i + " values(now, " + i + "),(now + 1s, " + (10 * i) + ")").forEach(sql -> { + try { + stmt.addBatch(sql); + } catch (SQLException e) { + e.printStackTrace(); + } + }); + stmt.addBatch("select count(*) from test.weather"); + + results = stmt.executeBatch(); + } catch (SQLException e) { + e.printStackTrace(); + } + + // then + assert results != null; + Assert.assertEquals(12, results.length); + Assert.assertEquals(1, results[0]); + Assert.assertEquals(1, results[1]); + Assert.assertEquals(1, results[2]); + Assert.assertEquals(1, results[3]); + Assert.assertEquals(1, results[4]); + Assert.assertEquals(Statement.EXECUTE_FAILED, results[5]); + Assert.assertEquals(2, results[6]); + Assert.assertEquals(2, results[7]); + Assert.assertEquals(2, results[8]); + Assert.assertEquals(2, results[9]); + Assert.assertEquals(2, results[10]); + Assert.assertEquals(Statement.SUCCESS_NO_INFO, results[11]); + } + + @Before + public void before() { + try { + Connection conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"); + Statement stmt = conn.createStatement(); + stmt.execute("use test"); + stmt.execute("drop table if exists weather"); + stmt.execute("create table weather (ts timestamp, f1 float) tags(t1 int)"); + IntStream.range(1, 11).mapToObj(i -> "create table t" + i + " using weather tags(" + i + ")").forEach(sql -> { + try { + stmt.execute(sql); + } catch (SQLException e) { + e.printStackTrace(); + } + }); + stmt.close(); + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @BeforeClass + public static void beforeClass() { + try { + Connection conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"); + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists test"); + stmt.execute("create database if not exists test"); + stmt.close(); + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + Connection conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"); + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists test"); + stmt.close(); + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java index e2541e8109..e175d6d114 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java @@ -60,7 +60,6 @@ public class BatchInsertTest { final int index = i; executorService.execute(() -> { try { - long startTime = System.currentTimeMillis(); Statement statement = connection.createStatement(); // get statement StringBuilder sb = new StringBuilder(); sb.append("INSERT INTO " + tablePrefix + index + " VALUES"); @@ -73,8 +72,6 @@ public class BatchInsertTest { } statement.addBatch(sb.toString()); statement.executeBatch(); - long endTime = System.currentTimeMillis(); - System.out.println("Thread " + index + " takes " + (endTime - startTime) + " microseconds"); connection.commit(); statement.close(); } catch (Exception e) { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectMultiTaosdByRestfulWithDifferentTokenTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectMultiTaosdByRestfulWithDifferentTokenTest.java index 7f2d367dce..fcb6ab7aaf 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectMultiTaosdByRestfulWithDifferentTokenTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectMultiTaosdByRestfulWithDifferentTokenTest.java @@ -1,6 +1,7 @@ package com.taosdata.jdbc.cases; import com.taosdata.jdbc.TSDBDriver; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -9,13 +10,13 @@ import java.util.Properties; public class ConnectMultiTaosdByRestfulWithDifferentTokenTest { - private static String host1 = "192.168.17.156"; - private static String user1 = "root"; - private static String password1 = "tqueue"; + private static final String host1 = "192.168.17.156"; + private static final String user1 = "root"; + private static final String password1 = "tqueue"; private Connection conn1; - private static String host2 = "192.168.17.82"; - private static String user2 = "root"; - private static String password2 = "taosdata"; + private static final String host2 = "192.168.17.82"; + private static final String user2 = "root"; + private static final String password2 = "taosdata"; private Connection conn2; @Test @@ -30,10 +31,8 @@ public class ConnectMultiTaosdByRestfulWithDifferentTokenTest { try (Statement stmt = connection.createStatement()) { ResultSet rs = stmt.executeQuery("select server_status()"); ResultSetMetaData meta = rs.getMetaData(); + Assert.assertNotNull(meta); while (rs.next()) { - for (int i = 1; i <= meta.getColumnCount(); i++) { - System.out.println(meta.getColumnLabel(i) + ": " + rs.getString(i)); - } } } catch (SQLException e) { e.printStackTrace(); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DatetimeBefore1970Test.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DatetimeBefore1970Test.java index f97e555ad1..14c7698548 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DatetimeBefore1970Test.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DatetimeBefore1970Test.java @@ -1,64 +1,76 @@ package com.taosdata.jdbc.cases; import com.taosdata.jdbc.utils.TimestampUtil; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.*; import java.sql.*; public class DatetimeBefore1970Test { - private static Connection conn; + private static final String host = "127.0.0.1"; + private Connection conn; @Test public void test() { try (Statement stmt = conn.createStatement()) { + // given stmt.executeUpdate("insert into weather(ts) values('1969-12-31 23:59:59.999')"); stmt.executeUpdate("insert into weather(ts) values('1970-01-01 00:00:00.000')"); stmt.executeUpdate("insert into weather(ts) values('1970-01-01 08:00:00.000')"); stmt.executeUpdate("insert into weather(ts) values('1970-01-01 07:59:59.999')"); + ResultSet rs = stmt.executeQuery("select * from weather order by ts asc"); + ResultSetMetaData metaData = rs.getMetaData(); + Assert.assertEquals(2, metaData.getColumnCount()); + + // when + rs.next(); + // then + Timestamp ts = rs.getTimestamp("ts"); + Assert.assertEquals("1969-12-31 23:59:59.999", TimestampUtil.longToDatetime(ts.getTime())); + + // when + rs.next(); + // then + ts = rs.getTimestamp("ts"); + Assert.assertEquals("1970-01-01 00:00:00.000", TimestampUtil.longToDatetime(ts.getTime())); + + // when + rs.next(); + // then + ts = rs.getTimestamp("ts"); + Assert.assertEquals("1970-01-01 08:00:00.000", TimestampUtil.longToDatetime(ts.getTime())); + + // when + rs.next(); + // then + ts = rs.getTimestamp("ts"); + Assert.assertEquals("1970-01-01 07:59:59.999", TimestampUtil.longToDatetime(ts.getTime())); - ResultSet rs = stmt.executeQuery("select * from weather"); - while (rs.next()) { - Timestamp ts = rs.getTimestamp("ts"); - System.out.println("long: " + ts.getTime() + ", string: " + TimestampUtil.longToDatetime(ts.getTime())); - } } catch (SQLException e) { e.printStackTrace(); } } - public static void main(String[] args) { - System.out.println("timestamp: " + Long.MAX_VALUE + ", string: " + TimestampUtil.longToDatetime(Long.MAX_VALUE)); - System.out.println("timestamp: " + Long.MIN_VALUE + ", string: " + TimestampUtil.longToDatetime(Long.MIN_VALUE)); - System.out.println("timestamp: " + 0 + ", string: " + TimestampUtil.longToDatetime(0)); - System.out.println("timestamp: " + -1 + ", string: " + TimestampUtil.longToDatetime(-1)); - String datetime = "1970-01-01 00:00:00.000"; - System.out.println("timestamp: " + TimestampUtil.datetimeToLong(datetime) + ", string: " + datetime); - datetime = "1969-12-31 23:59:59.999"; - System.out.println("timestamp: " + TimestampUtil.datetimeToLong(datetime) + ", string: " + datetime); - } - - @BeforeClass - public static void beforeClass() { + @Before + public void before() { try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - conn = DriverManager.getConnection("jdbc:TAOS://127.0.0.1:6030/?user=root&password=taosdata"); + conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"); Statement stmt = conn.createStatement(); stmt.execute("drop database if exists test_timestamp"); stmt.execute("create database if not exists test_timestamp keep 36500"); stmt.execute("use test_timestamp"); stmt.execute("create table weather(ts timestamp,f1 float)"); stmt.close(); - } catch (ClassNotFoundException | SQLException e) { + } catch (SQLException e) { e.printStackTrace(); } } - @AfterClass - public static void afterClass() { + @After + public void after() { try { + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists test_timestamp"); if (conn != null) conn.close(); } catch (SQLException e) { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD4174Test.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DoubleQuoteInSqlTest.java similarity index 55% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD4174Test.java rename to src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DoubleQuoteInSqlTest.java index 2704d4cfa5..212a751ec3 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD4174Test.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DoubleQuoteInSqlTest.java @@ -7,56 +7,64 @@ import org.junit.*; import java.sql.*; import java.util.Properties; -public class TD4174Test { - private Connection conn; +public class DoubleQuoteInSqlTest { private static final String host = "127.0.0.1"; + private static final String dbname = "td4174"; + + private Connection conn; @Test public void test() { + // given long ts = System.currentTimeMillis(); - try (PreparedStatement pstmt = conn.prepareStatement("insert into weather values(" + ts + ", ?)")) { - JSONObject value = new JSONObject(); - value.put("name", "John Smith"); - value.put("age", 20); - Assert.assertEquals("{\"name\":\"John Smith\",\"age\":20}",value.toJSONString()); - pstmt.setString(1, value.toJSONString()); - - int ret = pstmt.executeUpdate(); - Assert.assertEquals(1, ret); - } catch (SQLException e) { - e.printStackTrace(); - } - } - - public static void main(String[] args) { JSONObject value = new JSONObject(); value.put("name", "John Smith"); value.put("age", 20); - System.out.println(value.toJSONString()); + + // when + int ret = 0; + try (PreparedStatement pstmt = conn.prepareStatement("insert into weather values(" + ts + ", ?)")) { + pstmt.setString(1, value.toJSONString()); + ret = pstmt.executeUpdate(); + } catch (SQLException e) { + e.printStackTrace(); + } + + // then + Assert.assertEquals("{\"name\":\"John Smith\",\"age\":20}", value.toJSONString()); + Assert.assertEquals(1, ret); } @Before - public void before() throws SQLException { + public void before() { String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; Properties properties = new Properties(); properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - conn = DriverManager.getConnection(url, properties); - try (Statement stmt = conn.createStatement()) { - stmt.execute("drop database if exists td4174"); - stmt.execute("create database if not exists td4174"); - stmt.execute("use td4174"); + try { + conn = DriverManager.getConnection(url, properties); + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists " + dbname); + stmt.execute("create database if not exists " + dbname); + stmt.execute("use " + dbname); stmt.execute("create table weather(ts timestamp, text binary(64))"); + } catch (SQLException e) { + e.printStackTrace(); } } @After - public void after() throws SQLException { - if (conn != null) + public void after() { + try { + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists " + dbname); + stmt.close(); conn.close(); - + } catch (SQLException e) { + e.printStackTrace(); + } } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java index 6c8aed1b06..a00c8f4f97 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java @@ -13,7 +13,7 @@ import java.util.Properties; public class DriverAutoloadTest { private Properties properties; - private String host = "127.0.0.1"; + private final String host = "127.0.0.1"; @Test public void testRestful() throws SQLException { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java index d7603312a0..bc11c7f34e 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java @@ -20,7 +20,6 @@ public class ImportTest { @BeforeClass public static void before() { try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); Properties properties = new Properties(); properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); @@ -33,8 +32,6 @@ public class ImportTest { stmt.close(); ts = System.currentTimeMillis(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); } catch (SQLException e) { e.printStackTrace(); } @@ -47,7 +44,6 @@ public class ImportTest { for (int i = 0; i < 50; i++) { ts++; int row = stmt.executeUpdate("import into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")"); - System.out.println("import into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")\t" + row); assertEquals(1, row); } } @@ -84,7 +80,6 @@ public class ImportTest { long t = (++ts) + a; sqlBuilder.append("(").append(t).append(",").append((100 + i)).append(",").append(i).append(") "); } - System.out.println(sqlBuilder.toString()); int rows = stmt.executeUpdate(sqlBuilder.toString()); assertEquals(50, rows); } catch (SQLException e) { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java index 84149775c3..beea990456 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java @@ -13,9 +13,9 @@ import java.util.Random; @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class InsertDbwithoutUseDbTest { - private static String host = "127.0.0.1"; + private static final String host = "127.0.0.1"; private static Properties properties; - private static Random random = new Random(System.currentTimeMillis()); + private static final Random random = new Random(System.currentTimeMillis()); @Test public void case001() throws ClassNotFoundException, SQLException { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java index 4b4e83719f..9f8243542f 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java @@ -8,14 +8,14 @@ public class InsertSpecialCharacterJniTest { private static final String host = "127.0.0.1"; private static Connection conn; - private static String dbName = "spec_char_test"; - private static String tbname1 = "test"; - private static String tbname2 = "weather"; - private static String special_character_str_1 = "$asd$$fsfsf$"; - private static String special_character_str_2 = "\\\\asdfsfsf\\\\"; - private static String special_character_str_3 = "\\\\asdfsfsf\\"; - private static String special_character_str_4 = "?asd??fsf?sf?"; - private static String special_character_str_5 = "?#sd@$f(('<(s[P)>\"){]}f?s[]{}%vaew|\"fsfs^a&d*jhg)(j))(f@~!?$"; + private static final String dbName = "spec_char_test"; + private static final String tbname1 = "test"; + private static final String tbname2 = "weather"; + private static final String special_character_str_1 = "$asd$$fsfsf$"; + private static final String special_character_str_2 = "\\\\asdfsfsf\\\\"; + private static final String special_character_str_3 = "\\\\asdfsfsf\\"; + private static final String special_character_str_4 = "?asd??fsf?sf?"; + private static final String special_character_str_5 = "?#sd@$f(('<(s[P)>\"){]}f?s[]{}%vaew|\"fsfs^a&d*jhg)(j))(f@~!?$"; @Test public void testCase01() throws SQLException { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java index fa6cbd22b5..2e981e7f41 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java @@ -8,14 +8,14 @@ public class InsertSpecialCharacterRestfulTest { private static final String host = "127.0.0.1"; private static Connection conn; - private static String dbName = "spec_char_test"; - private static String tbname1 = "test"; - private static String tbname2 = "weather"; - private static String special_character_str_1 = "$asd$$fsfsf$"; - private static String special_character_str_2 = "\\\\asdfsfsf\\\\"; - private static String special_character_str_3 = "\\\\asdfsfsf\\"; - private static String special_character_str_4 = "?asd??fsf?sf?"; - private static String special_character_str_5 = "?#sd@$f(('<(s[P)>\"){]}f?s[]{}%vaew|\"fsfs^a&d*jhg)(j))(f@~!?$"; + private static final String dbName = "spec_char_test"; + private static final String tbname1 = "test"; + private static final String tbname2 = "weather"; + private static final String special_character_str_1 = "$asd$$fsfsf$"; + private static final String special_character_str_2 = "\\\\asdfsfsf\\\\"; + private static final String special_character_str_3 = "\\\\asdfsfsf\\"; + private static final String special_character_str_4 = "?asd??fsf?sf?"; + private static final String special_character_str_5 = "?#sd@$f(('<(s[P)>\"){]}f?s[]{}%vaew|\"fsfs^a&d*jhg)(j))(f@~!?$"; @Test public void testCase01() throws SQLException { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InvalidResultSetPointerTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InvalidResultSetPointerTest.java index f3d79b1df1..6febdd8497 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InvalidResultSetPointerTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InvalidResultSetPointerTest.java @@ -8,13 +8,13 @@ import java.util.Properties; public class InvalidResultSetPointerTest { - private static String host = "127.0.0.1"; + private static final String host = "127.0.0.1"; private static final String dbName = "test"; private static final String stbName = "stb"; private static final String tbName = "tb"; private static Connection connection; - private static int numOfSTb = 30000; - private static int numOfTb = 3; + private static final int numOfSTb = 30000; + private static final int numOfTb = 3; private static int numOfThreads = 100; @Test @@ -74,7 +74,7 @@ public class InvalidResultSetPointerTest { b = numOfSTb % numOfThreads; } - multiThreadingClass instance[] = new multiThreadingClass[numOfThreads]; + multiThreadingClass[] instance = new multiThreadingClass[numOfThreads]; int last = 0; for (int i = 0; i < numOfThreads; i++) { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TwoTypeTimestampPercisionInJniTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MicroSecondPrecisionJNITest.java similarity index 97% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TwoTypeTimestampPercisionInJniTest.java rename to src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MicroSecondPrecisionJNITest.java index f9b111bb12..eb8f134227 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TwoTypeTimestampPercisionInJniTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MicroSecondPrecisionJNITest.java @@ -10,7 +10,7 @@ import org.junit.Test; import java.sql.*; import java.util.Properties; -public class TwoTypeTimestampPercisionInJniTest { +public class MicroSecondPrecisionJNITest { private static final String host = "127.0.0.1"; private static final String ms_timestamp_db = "ms_precision_test"; @@ -41,7 +41,6 @@ public class TwoTypeTimestampPercisionInJniTest { rs.next(); Timestamp timestamp = rs.getTimestamp(1); - System.out.println(timestamp); long ts = timestamp.getTime(); Assert.assertEquals(timestamp1, ts); int nanos = timestamp.getNanos(); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TwoTypeTimestampPercisionInRestfulTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MicroSecondPrecisionRestfulTest.java similarity index 99% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TwoTypeTimestampPercisionInRestfulTest.java rename to src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MicroSecondPrecisionRestfulTest.java index 5c83b5a9da..7e9f04cd63 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TwoTypeTimestampPercisionInRestfulTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MicroSecondPrecisionRestfulTest.java @@ -10,10 +10,9 @@ import org.junit.Test; import java.sql.*; import java.util.Properties; -public class TwoTypeTimestampPercisionInRestfulTest { +public class MicroSecondPrecisionRestfulTest { private static final String host = "127.0.0.1"; - private static final String ms_timestamp_db = "ms_precision_test"; private static final String us_timestamp_db = "us_precision_test"; private static final long timestamp1 = System.currentTimeMillis(); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiThreadsWithSameStatmentTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiThreadsWithSameStatementTest.java similarity index 83% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiThreadsWithSameStatmentTest.java rename to src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiThreadsWithSameStatementTest.java index 5cb76cc0cb..73da79f357 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiThreadsWithSameStatmentTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiThreadsWithSameStatementTest.java @@ -7,21 +7,19 @@ import org.junit.Test; import java.sql.*; import java.util.concurrent.TimeUnit; -public class MultiThreadsWithSameStatmentTest { +public class MultiThreadsWithSameStatementTest { - - private class Service { + private static class Service { public Connection conn; public Statement stmt; public Service() { try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); conn = DriverManager.getConnection("jdbc:TAOS://localhost:6030/?user=root&password=taosdata"); stmt = conn.createStatement(); stmt.execute("create database if not exists jdbctest"); stmt.executeUpdate("create table if not exists jdbctest.weather (ts timestamp, f1 int)"); - } catch (ClassNotFoundException | SQLException e) { + } catch (SQLException e) { e.printStackTrace(); } } @@ -48,10 +46,6 @@ public class MultiThreadsWithSameStatmentTest { ResultSet resultSet = service.stmt.executeQuery("select * from jdbctest.weather"); while (resultSet.next()) { ResultSetMetaData metaData = resultSet.getMetaData(); - for (int i = 1; i <= metaData.getColumnCount(); i++) { - System.out.print(metaData.getColumnLabel(i) + ": " + resultSet.getString(i)); - } - System.out.println(); } resultSet.close(); service.release(); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampJNITest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampJNITest.java new file mode 100644 index 0000000000..4f2c87966a --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampJNITest.java @@ -0,0 +1,182 @@ +package com.taosdata.jdbc.cases; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.*; +import java.time.Instant; +import java.util.Random; + +public class NanoSecondTimestampJNITest { + + private static final String host = "127.0.0.1"; + private static final String dbname = "nano_sec_test"; + private static final Random random = new Random(System.currentTimeMillis()); + private static Connection conn; + + @Test + public void insertUsingLongValue() { + // given + long ms = System.currentTimeMillis(); + long ns = ms * 1000_000 + random.nextInt(1000_000); + + // when + int ret = 0; + try (Statement stmt = conn.createStatement()) { + ret = stmt.executeUpdate("insert into weather(ts, temperature, humidity) values(" + ns + ", 12.3, 4)"); + } catch (SQLException e) { + e.printStackTrace(); + } + + // then + Assert.assertEquals(1, ret); + } + + @Test + public void insertUsingStringValue() { + // given + + // when + int ret = 0; + try (Statement stmt = conn.createStatement()) { + ret = stmt.executeUpdate("insert into weather(ts, temperature, humidity) values('2021-01-01 12:00:00.123456789', 12.3, 4)"); + } catch (SQLException e) { + e.printStackTrace(); + } + + // then + Assert.assertEquals(1, ret); + } + + @Test + public void insertUsingTimestampValue() { + // given + long epochSec = System.currentTimeMillis() / 1000; + long nanoAdjustment = random.nextInt(1000_000_000); + Timestamp ts = Timestamp.from(Instant.ofEpochSecond(epochSec, nanoAdjustment)); + + // when + int ret = 0; + String sql = "insert into weather(ts, temperature, humidity) values( ?, ?, ?)"; + try (PreparedStatement pstmt = conn.prepareStatement(sql)) { + pstmt.setTimestamp(1, ts); + pstmt.setFloat(2, 12.34f); + pstmt.setInt(3, 55); + ret = pstmt.executeUpdate(); + } catch (SQLException e) { + e.printStackTrace(); + } + + // then + Assert.assertEquals(1, ret); + } + + @Test + public void selectUsingLongValue() throws SQLException { + // given + long ms = System.currentTimeMillis(); + long ns = ms * 1000_000L + random.nextInt(1000_000); + + // when + ResultSet rs = null; + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("insert into weather(ts, temperature, humidity) values(" + ns + ", 12.3, 4)"); + rs = stmt.executeQuery("select * from weather"); + rs.next(); + } catch (SQLException e) { + e.printStackTrace(); + } + + // then + long actual = rs.getLong(1); + Assert.assertEquals(ms, actual); + actual = rs.getLong("ts"); + Assert.assertEquals(ms, actual); + } + + @Test + public void selectUsingStringValue() throws SQLException { + // given + String timestampStr = "2021-01-01 12:00:00.123456789"; + + // when + ResultSet rs = null; + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("insert into weather(ts, temperature, humidity) values('" + timestampStr + "', 12.3, 4)"); + rs = stmt.executeQuery("select * from weather"); + rs.next(); + } catch (SQLException e) { + e.printStackTrace(); + } + + // then + String actual = rs.getString(1); + Assert.assertEquals(timestampStr, actual); + actual = rs.getString("ts"); + Assert.assertEquals(timestampStr, actual); + } + + @Test + public void selectUsingTimestampValue() throws SQLException { + // given + long timeMillis = System.currentTimeMillis(); + long epochSec = timeMillis / 1000; + long nanoAdjustment = (timeMillis % 1000) * 1000_000L + random.nextInt(1000_000); + Timestamp ts = Timestamp.from(Instant.ofEpochSecond(epochSec, nanoAdjustment)); + + // insert one row + String sql = "insert into weather(ts, temperature, humidity) values( ?, ?, ?)"; + try (PreparedStatement pstmt = conn.prepareStatement(sql)) { + pstmt.setTimestamp(1, ts); + pstmt.setFloat(2, 12.34f); + pstmt.setInt(3, 55); + pstmt.executeUpdate(); + } catch (SQLException e) { + e.printStackTrace(); + } + + // when + ResultSet rs = null; + try (Statement stmt = conn.createStatement()) { + rs = stmt.executeQuery("select * from weather"); + rs.next(); + } catch (SQLException e) { + e.printStackTrace(); + } + + // then + Timestamp actual = rs.getTimestamp(1); + Assert.assertEquals(ts, actual); + actual = rs.getTimestamp("ts"); + Assert.assertEquals(ts, actual); + Assert.assertEquals(timeMillis, actual.getTime()); + Assert.assertEquals(nanoAdjustment, actual.getNanos()); + } + + @Before + public void before() { + try (Statement stmt = conn.createStatement()) { + stmt.execute("drop table if exists weather"); + stmt.execute("create table weather(ts timestamp, temperature float, humidity int)"); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @BeforeClass + public static void beforeClass() { + final String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; + try { + conn = DriverManager.getConnection(url); + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists " + dbname); + stmt.execute("create database if not exists " + dbname + " precision 'ns'"); + stmt.execute("use " + dbname); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampRestfulTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampRestfulTest.java new file mode 100644 index 0000000000..4271f918b9 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampRestfulTest.java @@ -0,0 +1,182 @@ +package com.taosdata.jdbc.cases; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.*; +import java.time.Instant; +import java.util.Random; + +public class NanoSecondTimestampRestfulTest { + + private static final String host = "127.0.0.1"; + private static final String dbname = "nano_sec_test"; + private static final Random random = new Random(System.currentTimeMillis()); + private static Connection conn; + + @Test + public void insertUsingLongValue() { + // given + long ms = System.currentTimeMillis(); + long ns = ms * 1000_000 + random.nextInt(1000_000); + + // when + int ret = 0; + try (Statement stmt = conn.createStatement()) { + ret = stmt.executeUpdate("insert into weather(ts, temperature, humidity) values(" + ns + ", 12.3, 4)"); + } catch (SQLException e) { + e.printStackTrace(); + } + + // then + Assert.assertEquals(1, ret); + } + + @Test + public void insertUsingStringValue() { + // given + + // when + int ret = 0; + try (Statement stmt = conn.createStatement()) { + ret = stmt.executeUpdate("insert into weather(ts, temperature, humidity) values('2021-01-01 12:00:00.123456789', 12.3, 4)"); + } catch (SQLException e) { + e.printStackTrace(); + } + + // then + Assert.assertEquals(1, ret); + } + + @Test + public void insertUsingTimestampValue() { + // given + long epochSec = System.currentTimeMillis() / 1000; + long nanoAdjustment = random.nextInt(1000_000_000); + Timestamp ts = Timestamp.from(Instant.ofEpochSecond(epochSec, nanoAdjustment)); + + // when + int ret = 0; + String sql = "insert into weather(ts, temperature, humidity) values( ?, ?, ?)"; + try (PreparedStatement pstmt = conn.prepareStatement(sql)) { + pstmt.setTimestamp(1, ts); + pstmt.setFloat(2, 12.34f); + pstmt.setInt(3, 55); + ret = pstmt.executeUpdate(); + } catch (SQLException e) { + e.printStackTrace(); + } + + // then + Assert.assertEquals(1, ret); + } + + @Test + public void selectUsingLongValue() throws SQLException { + // given + long ms = System.currentTimeMillis(); + long ns = ms * 1000_000L + random.nextInt(1000_000); + + // when + ResultSet rs = null; + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("insert into weather(ts, temperature, humidity) values(" + ns + ", 12.3, 4)"); + rs = stmt.executeQuery("select * from weather"); + rs.next(); + } catch (SQLException e) { + e.printStackTrace(); + } + + // then + long actual = rs.getLong(1); + Assert.assertEquals(ms, actual); + actual = rs.getLong("ts"); + Assert.assertEquals(ms, actual); + } + + @Test + public void selectUsingStringValue() throws SQLException { + // given + String timestampStr = "2021-01-01 12:00:00.123456789"; + + // when + ResultSet rs = null; + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("insert into weather(ts, temperature, humidity) values('" + timestampStr + "', 12.3, 4)"); + rs = stmt.executeQuery("select * from weather"); + rs.next(); + } catch (SQLException e) { + e.printStackTrace(); + } + + // then + String actual = rs.getString(1); + Assert.assertEquals(timestampStr, actual); + actual = rs.getString("ts"); + Assert.assertEquals(timestampStr, actual); + } + + @Test + public void selectUsingTimestampValue() throws SQLException { + // given + long timeMillis = System.currentTimeMillis(); + long epochSec = timeMillis / 1000; + long nanoAdjustment = (timeMillis % 1000) * 1000_000L + random.nextInt(1000_000); + Timestamp ts = Timestamp.from(Instant.ofEpochSecond(epochSec, nanoAdjustment)); + + // insert one row + String sql = "insert into weather(ts, temperature, humidity) values( ?, ?, ?)"; + try (PreparedStatement pstmt = conn.prepareStatement(sql)) { + pstmt.setTimestamp(1, ts); + pstmt.setFloat(2, 12.34f); + pstmt.setInt(3, 55); + pstmt.executeUpdate(); + } catch (SQLException e) { + e.printStackTrace(); + } + + // when + ResultSet rs = null; + try (Statement stmt = conn.createStatement()) { + rs = stmt.executeQuery("select * from weather"); + rs.next(); + } catch (SQLException e) { + e.printStackTrace(); + } + + // then + Timestamp actual = rs.getTimestamp(1); + Assert.assertEquals(ts, actual); + actual = rs.getTimestamp("ts"); + Assert.assertEquals(ts, actual); + Assert.assertEquals(timeMillis, actual.getTime()); + Assert.assertEquals(nanoAdjustment, actual.getNanos()); + } + + @Before + public void before() { + try (Statement stmt = conn.createStatement()) { + stmt.execute("drop table if exists weather"); + stmt.execute("create table weather(ts timestamp, temperature float, humidity int)"); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @BeforeClass + public static void beforeClass() { + final String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; + try { + conn = DriverManager.getConnection(url); + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists " + dbname); + stmt.execute("create database if not exists " + dbname + " precision 'ns'"); + stmt.execute("use " + dbname); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetForJdbcJniTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetJNITest.java similarity index 87% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetForJdbcJniTest.java rename to src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetJNITest.java index 782125144c..ae0241bf31 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetForJdbcJniTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetJNITest.java @@ -6,7 +6,7 @@ import org.junit.Test; import java.sql.*; -public class NullValueInResultSetForJdbcJniTest { +public class NullValueInResultSetJNITest { private static final String host = "127.0.0.1"; Connection conn; @@ -17,11 +17,6 @@ public class NullValueInResultSetForJdbcJniTest { ResultSet rs = stmt.executeQuery("select * from weather"); ResultSetMetaData meta = rs.getMetaData(); while (rs.next()) { - for (int i = 1; i <= meta.getColumnCount(); i++) { - Object value = rs.getObject(i); - System.out.print(meta.getColumnLabel(i) + ": " + value + "\t"); - } - System.out.println(); } } catch (SQLException e) { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetForJdbcRestfulTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetRestfulTest.java similarity index 92% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetForJdbcRestfulTest.java rename to src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetRestfulTest.java index f2ac94adc1..7fbb30a524 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetForJdbcRestfulTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetRestfulTest.java @@ -6,7 +6,7 @@ import org.junit.Test; import java.sql.*; -public class NullValueInResultSetForJdbcRestfulTest { +public class NullValueInResultSetRestfulTest { private static final String host = "127.0.0.1"; Connection conn; @@ -19,9 +19,7 @@ public class NullValueInResultSetForJdbcRestfulTest { while (rs.next()) { for (int i = 1; i <= meta.getColumnCount(); i++) { Object value = rs.getObject(i); - System.out.print(meta.getColumnLabel(i) + ": " + value + "\t"); } - System.out.println(); } } catch (SQLException e) { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD3841Test.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetTest.java similarity index 99% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD3841Test.java rename to src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetTest.java index c6fba81eb2..61d767b5cf 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD3841Test.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetTest.java @@ -7,7 +7,7 @@ import org.junit.*; import java.sql.*; import java.util.Properties; -public class TD3841Test { +public class NullValueInResultSetTest { private static final String host = "127.0.0.1"; private static Properties properties; private static Connection conn_restful; diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertJNITest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertJNITest.java new file mode 100644 index 0000000000..ed78c2561e --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertJNITest.java @@ -0,0 +1,98 @@ +package com.taosdata.jdbc.cases; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.sql.*; +import java.util.List; +import java.util.Random; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class PreparedStatementBatchInsertJNITest { + + private static final String host = "127.0.0.1"; + private static final String dbname = "td4668"; + + private final Random random = new Random(System.currentTimeMillis()); + private Connection conn; + + @Test + public void test() { + // given + long ts = System.currentTimeMillis(); + List rows = IntStream.range(0, 10).mapToObj(i -> { + Object[] row = new Object[6]; + final String groupId = String.format("%02d", random.nextInt(100)); + // table name (d + groupId)组ĺ + row[0] = "d" + groupId; + // tag + row[1] = groupId; + // ts + row[2] = ts + i; + // current ç”µćµ + row[3] = random.nextFloat(); + // voltage 电压 + row[4] = Math.random() > 0.5 ? 220 : 380; + // phase 相位 + row[5] = random.nextInt(10); + return row; + }).collect(Collectors.toList()); + final String sql = "INSERT INTO ? (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (?) VALUES (?,?,?,?)"; + + // when + try (PreparedStatement pstmt = conn.prepareStatement(sql)) { + for (Object[] row : rows) { + for (int i = 0; i < row.length; i++) { + pstmt.setObject(i + 1, row[i]); + } + pstmt.addBatch(); + } + pstmt.executeBatch(); + } catch (SQLException e) { + e.printStackTrace(); + Assert.fail(); + } + + // then + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select * from meters"); + int count = 0; + while (rs.next()) { + count++; + } + Assert.assertEquals(10, count); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Before + public void before() { + try { + conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"); + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists " + dbname); + stmt.execute("create database if not exists " + dbname); + stmt.execute("use " + dbname); + stmt.execute("create table meters(ts timestamp, current float, voltage int, phase int) tags(groupId int)"); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @After + public void after() { + try { + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists " + dbname); + stmt.close(); + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertRestfulTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertRestfulTest.java new file mode 100644 index 0000000000..90b285381a --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertRestfulTest.java @@ -0,0 +1,98 @@ +package com.taosdata.jdbc.cases; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.sql.*; +import java.util.List; +import java.util.Random; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class PreparedStatementBatchInsertRestfulTest { + + private static final String host = "127.0.0.1"; + private static final String dbname = "td4668"; + + private final Random random = new Random(System.currentTimeMillis()); + private Connection conn; + + @Test + public void test() { + // given + long ts = System.currentTimeMillis(); + List rows = IntStream.range(0, 10).mapToObj(i -> { + Object[] row = new Object[6]; + final String groupId = String.format("%02d", random.nextInt(100)); + // table name (d + groupId)组ĺ + row[0] = "d" + groupId; + // tag + row[1] = groupId; + // ts + row[2] = ts + i; + // current ç”µćµ + row[3] = random.nextFloat(); + // voltage 电压 + row[4] = Math.random() > 0.5 ? 220 : 380; + // phase 相位 + row[5] = random.nextInt(10); + return row; + }).collect(Collectors.toList()); + final String sql = "INSERT INTO ? (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (?) VALUES (?,?,?,?)"; + + // when + try (PreparedStatement pstmt = conn.prepareStatement(sql)) { + for (Object[] row : rows) { + for (int i = 0; i < row.length; i++) { + pstmt.setObject(i + 1, row[i]); + } + pstmt.addBatch(); + } + pstmt.executeBatch(); + } catch (SQLException e) { + e.printStackTrace(); + Assert.fail(); + } + + // then + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select * from meters"); + int count = 0; + while (rs.next()) { + count++; + } + Assert.assertEquals(10, count); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Before + public void before() { + try { + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"); + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists " + dbname); + stmt.execute("create database if not exists " + dbname); + stmt.execute("use " + dbname); + stmt.execute("create table meters(ts timestamp, current float, voltage int, phase int) tags(groupId int)"); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @After + public void after() { + try { + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists " + dbname); + stmt.close(); + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java index d0ba113b7a..535e56f7d7 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java @@ -21,7 +21,6 @@ public class QueryDataTest { @Before public void createDatabase() { try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); Properties properties = new Properties(); properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); @@ -36,7 +35,7 @@ public class QueryDataTest { String createTableSql = "create table " + stbName + "(ts timestamp, name binary(64))"; statement.executeUpdate(createTableSql); - } catch (ClassNotFoundException | SQLException e) { + } catch (SQLException e) { return; } } @@ -44,7 +43,6 @@ public class QueryDataTest { @Test public void testQueryBinaryData() throws SQLException { String insertSql = "insert into " + stbName + " values(now, 'taosdata')"; - System.out.println(insertSql); statement.executeUpdate(insertSql); String querySql = "select * from " + stbName; @@ -52,7 +50,6 @@ public class QueryDataTest { while (rs.next()) { String name = rs.getString(2); - System.out.println("name = " + name); assertEquals("taosdata", name); } rs.close(); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResetQueryCacheTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResetQueryCacheTest.java index 52db8052e3..4eebbd08e8 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResetQueryCacheTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResetQueryCacheTest.java @@ -34,8 +34,6 @@ public class ResetQueryCacheTest { public void testResetQueryCache() throws SQLException { String resetSql = "reset query cache"; statement.execute(resetSql); - // ResultSet rs = statement.executeQuery(resetSql); - // rs.close(); } @After diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResultSetMetaShouldNotBeNullRestfulTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResultSetMetaShouldNotBeNullRestfulTest.java new file mode 100644 index 0000000000..be27b13507 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResultSetMetaShouldNotBeNullRestfulTest.java @@ -0,0 +1,86 @@ +package com.taosdata.jdbc.cases; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.sql.*; + +public class ResultSetMetaShouldNotBeNullRestfulTest { + + private static final String host = "127.0.0.1"; + private static final String dbname = "td4745"; + + private Connection connection; + + @Test + public void testExecuteQuery() { + // given + ResultSetMetaData metaData = null; + int columnCount = -1; + + // when + try { + Statement statement = connection.createStatement(); + metaData = statement.executeQuery("select * from weather").getMetaData(); + columnCount = metaData.getColumnCount(); + } catch (SQLException e) { + e.printStackTrace(); + } + + // then + Assert.assertNotNull(metaData); + Assert.assertEquals(2, columnCount); + } + + @Test + public void testExecute() { + // given + ResultSetMetaData metaData = null; + int columnCount = -1; + boolean execute = false; + // when + try { + Statement statement = connection.createStatement(); + execute = statement.execute("select * from weather"); + metaData = statement.getResultSet().getMetaData(); + columnCount = metaData.getColumnCount(); + } catch (SQLException e) { + e.printStackTrace(); + } + + // then + Assert.assertEquals(true, execute); + Assert.assertNotNull(metaData); + Assert.assertEquals(2, columnCount); + } + + @Before + public void before() { + try { + connection = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"); + Statement stmt = connection.createStatement(); + stmt.execute("drop database if exists " + dbname); + stmt.execute("create database if not exists " + dbname); + stmt.execute("use " + dbname); + stmt.execute("create table weather (ts timestamp, temperature float)"); + stmt.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @After + public void after() { + try { + Statement stmt = connection.createStatement(); + stmt.execute("drop database if exists " + dbname); + stmt.close(); + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java index 38c8cbb98c..0022ceaf21 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java @@ -19,7 +19,6 @@ public class SelectTest { @Before public void createDatabaseAndTable() { try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); Properties properties = new Properties(); properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); @@ -31,8 +30,6 @@ public class SelectTest { stmt.execute("create database if not exists " + dbName); stmt.execute("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)"); stmt.close(); - } catch (ClassNotFoundException e) { - return; } catch (SQLException e) { e.printStackTrace(); } @@ -47,7 +44,6 @@ public class SelectTest { for (int i = 0; i < 50; i++) { ts++; int row = stmt.executeUpdate("insert into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")"); - System.out.println("insert into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")\t" + row); assertEquals(1, row); } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java index 4575cb73a0..1600fec13d 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java @@ -16,14 +16,13 @@ import static org.junit.Assert.assertEquals; public class StableTest { private static Connection connection; - private static String dbName = "test"; - private static String stbName = "st"; - private static String host = "127.0.0.1"; + private static final String dbName = "test"; + private static final String stbName = "st"; + private static final String host = "127.0.0.1"; @BeforeClass public static void createDatabase() { try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); Properties properties = new Properties(); properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); @@ -34,8 +33,6 @@ public class StableTest { statement.execute("create database if not exists " + dbName); statement.execute("use " + dbName); statement.close(); - } catch (ClassNotFoundException e) { - return; } catch (SQLException e) { e.printStackTrace(); } @@ -68,9 +65,6 @@ public class StableTest { String sql = "describe " + stbName; ResultSet rs = stmt.executeQuery(sql); while (rs.next()) { - for (int i = 1; i <= rs.getMetaData().getColumnCount(); i++) { - System.out.println(i + ":" + rs.getString(i)); - } num++; } rs.close(); @@ -86,9 +80,6 @@ public class StableTest { try (Statement stmt = connection.createStatement()) { ResultSet rs = stmt.executeQuery("describe t1"); while (rs.next()) { - for (int i = 1; i <= rs.getMetaData().getColumnCount(); i++) { - System.out.printf("%d: %s\n", i, rs.getString(i)); - } num++; } rs.close(); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD4144Test.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD4144Test.java deleted file mode 100644 index 6f29f64111..0000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD4144Test.java +++ /dev/null @@ -1,105 +0,0 @@ -package com.taosdata.jdbc.cases; - -import com.taosdata.jdbc.TSDBConnection; -import com.taosdata.jdbc.TSDBDriver; -import com.taosdata.jdbc.TSDBResultSet; -import com.taosdata.jdbc.TSDBSubscribe; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.sql.DriverManager; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Properties; -import java.util.concurrent.TimeUnit; - -public class TD4144Test { - - private static TSDBConnection connection; - private static final String host = "127.0.0.1"; - - private static final String topic = "topic-meter-current-bg-10"; - private static final String sql = "select * from meters where current > 10"; - private static final String sql2 = "select * from meters where ts >= '2020-08-15 12:20:00.000'"; - - - @Test - public void test() throws SQLException { - TSDBSubscribe subscribe = null; - TSDBResultSet res = null; - boolean hasNext = false; - - try { - subscribe = connection.subscribe(topic, sql, false); - int count = 0; - while (true) { - // 等待1秒，éżĺ…Ťé˘‘çąč°ç”¨ consume，给服务端造ć压力 - TimeUnit.SECONDS.sleep(1); - if (res == null) { - // ć¶č´ąć•°ćŤ® - res = subscribe.consume(); - hasNext = res.next(); - } - - if (res == null) { - continue; - } - ResultSetMetaData metaData = res.getMetaData(); - int number = 0; - while (hasNext) { - int columnCount = metaData.getColumnCount(); - for (int i = 1; i <= columnCount; i++) { - System.out.print(metaData.getColumnLabel(i) + ": " + res.getString(i) + "\t"); - } - System.out.println(); - count++; - number++; - hasNext = res.next(); - if (!hasNext) { - res.close(); - res = null; - System.out.println("rows： " + count); - } - if (hasNext == true && number >= 10) { - System.out.println("batch" + number); - break; - } - } - - } - - } catch (SQLException | InterruptedException throwables) { - throwables.printStackTrace(); - } finally { - if (subscribe != null) - subscribe.close(true); - } - } - - @BeforeClass - public static void beforeClass() throws SQLException { - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; - connection = (DriverManager.getConnection(url, properties)).unwrap(TSDBConnection.class); - try (Statement stmt = connection.createStatement()) { - stmt.execute("drop database if exists power"); - stmt.execute("create database if not exists power"); - stmt.execute("use power"); - stmt.execute("create table meters(ts timestamp, current float, voltage int, phase int) tags(location binary(64), groupId int)"); - stmt.execute("create table d1001 using meters tags(\"Beijing.Chaoyang\", 2)"); - stmt.execute("create table d1002 using meters tags(\"Beijing.Haidian\", 2)"); - stmt.execute("insert into d1001 values(\"2020-08-15 12:00:00.000\", 12, 220, 1),(\"2020-08-15 12:10:00.000\", 12.3, 220, 2),(\"2020-08-15 12:20:00.000\", 12.2, 220, 1)"); - stmt.execute("insert into d1002 values(\"2020-08-15 12:00:00.000\", 9.9, 220, 1),(\"2020-08-15 12:10:00.000\", 10.3, 220, 1),(\"2020-08-15 12:20:00.000\", 11.2, 220, 1)"); - } - } - - @AfterClass - public static void afterClass() throws SQLException { - if (connection != null) - connection.close(); - } -} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java index e9e36e20c4..7df9f73807 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java @@ -25,7 +25,7 @@ public class TaosInfoMonitorTest { return null; }).collect(Collectors.toList()); - connectionList.stream().forEach(conn -> { + connectionList.forEach(conn -> { try (Statement stmt = conn.createStatement()) { ResultSet rs = stmt.executeQuery("show databases"); while (rs.next()) { @@ -37,7 +37,7 @@ public class TaosInfoMonitorTest { } }); - connectionList.stream().forEach(conn -> { + connectionList.forEach(conn -> { try { conn.close(); TimeUnit.MILLISECONDS.sleep(100); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java new file mode 100644 index 0000000000..72734cb0ec --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java @@ -0,0 +1,570 @@ +package com.taosdata.jdbc.cases; + + +import com.taosdata.jdbc.TSDBDriver; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.After; +import org.junit.Test; + +import java.sql.*; +import java.util.Properties; +import java.text.Format; +import java.text.SimpleDateFormat; + +public class TimestampPrecisionInNanoInJniTest { + + private static final String host = "127.0.0.1"; + private static final String ns_timestamp_db = "ns_precision_test"; + private static final long timestamp1 = System.currentTimeMillis(); + private static final long timestamp2 = timestamp1 * 1000_000 + 123455; + private static final long timestamp3 = (timestamp1 + 10) * 1000_000 + 123456; + private static final Format format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); + private static final String date1 = format.format(new Date(timestamp1)); + private static final String date4 = format.format(new Date(timestamp1 + 10L)); + private static final String date2 = date1 + "123455"; + private static final String date3 = date4 + "123456"; + + + private static Connection conn; + + @BeforeClass + public static void beforeClass() throws SQLException { + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + + String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; + conn = DriverManager.getConnection(url, properties); + + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists " + ns_timestamp_db); + stmt.execute("create database if not exists " + ns_timestamp_db + " precision 'ns'"); + stmt.execute("create table " + ns_timestamp_db + ".weather(ts timestamp, ts2 timestamp, f1 int)"); + stmt.executeUpdate("insert into " + ns_timestamp_db + ".weather(ts, ts2, f1) values(\"" + date3 + "\", \"" + date3 + "\", 128)"); + stmt.executeUpdate("insert into " + ns_timestamp_db + ".weather(ts, ts2, f1) values(" + timestamp2 + "," + timestamp2 + ", 127)"); + stmt.close(); + } + + @After + public void afterEach() throws SQLException { + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists " + ns_timestamp_db); + stmt.execute("create database if not exists " + ns_timestamp_db + " precision 'ns'"); + stmt.execute("create table " + ns_timestamp_db + ".weather(ts timestamp, ts2 timestamp, f1 int)"); + stmt.executeUpdate("insert into " + ns_timestamp_db + ".weather(ts, ts2, f1) values(\"" + date3 + "\", \"" + date3 + "\", 128)"); + stmt.executeUpdate("insert into " + ns_timestamp_db + ".weather(ts, ts2, f1) values(" + timestamp2 + "," + timestamp2 + ", 127)"); + stmt.close(); + } + + @AfterClass + public static void afterClass() { + try { + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + private void checkCount(long count, ResultSet rs) throws SQLException { + if (count == 0) { + Assert.fail(); + } + rs.next(); + long test_count = rs.getLong(1); + Assert.assertEquals(count, test_count); + } + + private void checkTime(long ts, ResultSet rs) throws SQLException { + rs.next(); + int nanos = rs.getTimestamp(1).getNanos(); + Assert.assertEquals(ts % 1000_000_000l, nanos); + long test_ts = rs.getLong(1); + Assert.assertEquals(ts / 1000_000l, test_ts); + } + + @Test + public void canInsertTimestampAndQueryByEqualToInDateTypeInBothFirstAndSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts = '" + date3 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts = '" + date3 + "'"); + checkTime(timestamp3, rs); + rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 = '" + date3 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 = '" + date3 + "'"); + checkTime(timestamp3, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canImportTimestampAndQueryByEqualToInDateTypeInBothFirstAndSecondCol() { + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("import into " + ns_timestamp_db + ".weather(ts, ts2, f1) values(\"" + date1 + "123123\", \"" + date1 + "123123\", 127)"); + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts = '" + date1 + "123123'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts = '" + date1 + "123123'"); + checkTime(timestamp1 * 1000_000l + 123123l, rs); + rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 = '" + date1 + "123123'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 = '" + date1 + "123123'"); + checkTime(timestamp1 * 1000_000l + 123123l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canInsertTimestampAndQueryByEqualToInNumberTypeInBothFirstAndSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts = '" + timestamp2 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts = '" + timestamp2 + "'"); + checkTime(timestamp2, rs); + rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 = '" + timestamp2 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 = '" + timestamp2 + "'"); + checkTime(timestamp2, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canImportTimestampAndQueryByEqualToInNumberTypeInBothFirstAndSecondCol() { + try (Statement stmt = conn.createStatement()) { + long timestamp4 = timestamp1 * 1000_000 + 123123; + stmt.executeUpdate("import into " + ns_timestamp_db + ".weather(ts, ts2, f1) values(" + timestamp4 + ", " + timestamp4 + ", 127)"); + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts = '" + timestamp4 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts = '" + timestamp4 + "'"); + checkTime(timestamp4, rs); + rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 = '" + timestamp4 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 = '" + timestamp4 + "'"); + checkTime(timestamp4, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canSelectLastRowFromWeatherForFirstCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select last(ts) from " + ns_timestamp_db + ".weather"); + checkTime(timestamp3, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canSelectLastRowFromWeatherForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select last(ts2) from " + ns_timestamp_db + ".weather"); + checkTime(timestamp3, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canSelectFirstRowFromWeatherForFirstCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select first(ts) from " + ns_timestamp_db + ".weather"); + checkTime(timestamp2, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canSelectFirstRowFromWeatherForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select first(ts2) from " + ns_timestamp_db + ".weather"); + checkTime(timestamp2, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLargerThanInDateTypeForFirstCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts > '" + date2 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts > '" + date2 + "'"); + checkTime(timestamp3, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLargerThanInDateTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 > '" + date2 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 > '" + date2 + "'"); + checkTime(timestamp3, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLargerThanInNumberTypeForFirstCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts > '" + timestamp2 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts > '" + timestamp2 + "'"); + checkTime(timestamp3, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLargerThanInNumberTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 > '" + timestamp2 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 > '" + timestamp2 + "'"); + checkTime(timestamp3, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLargerThanOrEqualToInDateTypeForFirstCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts >= '" + date2 + "'"); + checkCount(2l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLargerThanOrEqualToInDateTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 >= '" + date2 + "'"); + checkCount(2l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLargerThanOrEqualToInNumberTypeForFirstCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts >= '" + timestamp2 + "'"); + checkCount(2l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLargerThanOrEqualToInNumberTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 >= '" + timestamp2 + "'"); + checkCount(2l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLessThanInDateTypeForFirstCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts < '" + date3 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts < '" + date3 + "'"); + checkTime(timestamp2, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLessThanInDateTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 < '" + date3 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 < '" + date3 + "'"); + checkTime(timestamp2, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLessThanInNumberTypeForFirstCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts < '" + timestamp3 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts < '" + timestamp3 + "'"); + checkTime(timestamp2, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLessThanInNumberTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 < '" + timestamp3 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 < '" + timestamp3 + "'"); + checkTime(timestamp2, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLessThanOrEqualToInDateTypeForFirstCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts <= '" + date3 + "'"); + checkCount(2l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLessThanOrEqualToInDateTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 <= '" + date3 + "'"); + checkCount(2l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLessThanOrEqualToInNumberTypeForFirstCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts <= '" + timestamp3 + "'"); + checkCount(2l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLessThanOrEqualToInNumberTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 <= '" + timestamp3 + "'"); + checkCount(2l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryBetweenAndInDateTypeForFirstCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts <= '" + date3 + "' AND ts > '" + date2 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts <= '" + date3 + "' AND ts > '" + date2 + "'"); + checkTime(timestamp3, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryBetweenAndInDateTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 <= '" + date3 + "' AND ts2 > '" + date2 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 <= '" + date3 + "' AND ts2 > '" + date2 + "'"); + checkTime(timestamp3, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryBetweenAndInNumberTypeForFirstCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts <= '" + timestamp3 + "' AND ts > '" + timestamp2 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts <= '" + timestamp3 + "' AND ts > '" + timestamp2 + "'"); + checkTime(timestamp3, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryBetweenAndInNumberTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 <= '" + timestamp3 + "' AND ts2 > '" + timestamp2 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 <= '" + timestamp3 + "' AND ts2 > '" + timestamp2 + "'"); + checkTime(timestamp3, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryNotEqualToInDateTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 <> '" + date3 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 <> '" + date3 + "'"); + checkTime(timestamp2, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryNotEqualToInNumberTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 <> '" + timestamp3 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 <> '" + timestamp3 + "'"); + checkTime(timestamp2, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryNotEqualInDateTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 != '" + date3 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 != '" + date3 + "'"); + checkTime(timestamp2, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryNotEqualInNumberTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 != '" + timestamp3 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 != '" + timestamp3 + "'"); + checkTime(timestamp2, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canInsertTimestampWithNowAndNsOffsetInBothFirstAndSecondCol(){ + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("insert into " + ns_timestamp_db + ".weather(ts, ts2, f1) values(now + 1000b, now - 1000b, 128)"); + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather"); + checkCount(3l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canIntervalAndSlidingAcceptNsUnitForFirstCol(){ + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select sum(f1) from " + ns_timestamp_db + ".weather where ts >= '" + date2 + "' and ts <= '" + date3 + "' interval(10000000b) sliding(10000000b)"); + rs.next(); + long sum = rs.getLong(2); + Assert.assertEquals(127l, sum); + rs.next(); + sum = rs.getLong(2); + Assert.assertEquals(128l, sum); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canIntervalAndSlidingAcceptNsUnitForSecondCol(){ + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select sum(f1) from " + ns_timestamp_db + ".weather where ts2 >= '" + date2 + "' and ts <= '" + date3 + "' interval(10000000b) sliding(10000000b)"); + rs.next(); + long sum = rs.getLong(2); + Assert.assertEquals(127l, sum); + rs.next(); + sum = rs.getLong(2); + Assert.assertEquals(128l, sum); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void testDataOutOfRangeExceptionForFirstCol() { + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("insert into " + ns_timestamp_db + ".weather(ts, ts2, f1) values(123456789012345678, 1234567890123456789, 127)"); + } catch (SQLException e) { + Assert.assertEquals("TDengine ERROR (8000060b): Timestamp data out of range", e.getMessage()); + } + } + + @Test + public void testDataOutOfRangeExceptionForSecondCol() { + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("insert into " + ns_timestamp_db + ".weather(ts, ts2, f1) values(1234567890123456789, 123456789012345678, 127)"); + } catch (SQLException e) { + Assert.assertEquals("TDengine ERROR (8000060b): Timestamp data out of range", e.getMessage()); + } + } + + @Test + public void willAutomaticallyFillToNsUnitWithZerosForFirstCol() { + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("insert into " + ns_timestamp_db + ".weather(ts, ts2, f1) values('" + date1 + "', '" + date1 + "', 127)"); + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts = '" + date1 + "000000'"); + checkCount(1l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void willAutomaticallyFillToNsUnitWithZerosForSecondCol() { + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("insert into " + ns_timestamp_db + ".weather(ts, ts2, f1) values('" + date1 + "', '" + date1 + "', 127)"); + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 = '" + date1 + "000000'"); + checkCount(1l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void willAutomaticallyDropDigitExceedNsDigitNumberForFirstCol() { + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("insert into " + ns_timestamp_db + ".weather(ts, ts2, f1) values('" + date1 + "999999999', '" + date1 + "999999999', 127)"); + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts = '" + date1 + "999999'"); + checkCount(1l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void willAutomaticallyDropDigitExceedNsDigitNumberForSecondCol() { + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("insert into " + ns_timestamp_db + ".weather(ts, ts2, f1) values('" + date1 + "999999999', '" + date1 + "999999999', 127)"); + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 = '" + date1 + "999999'"); + checkCount(1l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberJniTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberJniTest.java index fb23c0e64a..e4149793ac 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberJniTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberJniTest.java @@ -44,7 +44,7 @@ public class UnsignedNumberJniTest { Assert.assertEquals(127, rs.getByte(2)); Assert.assertEquals(32767, rs.getShort(3)); Assert.assertEquals(2147483647, rs.getInt(4)); - Assert.assertEquals(9223372036854775807l, rs.getLong(5)); + Assert.assertEquals(9223372036854775807L, rs.getLong(5)); } } catch (SQLException e) { e.printStackTrace(); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java index a659a490cb..3bdf5ae4f2 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java @@ -45,7 +45,7 @@ public class UnsignedNumberRestfulTest { Assert.assertEquals(127, rs.getByte(2)); Assert.assertEquals(32767, rs.getShort(3)); Assert.assertEquals(2147483647, rs.getInt(4)); - Assert.assertEquals(9223372036854775807l, rs.getLong(5)); + Assert.assertEquals(9223372036854775807L, rs.getLong(5)); } } catch (SQLException e) { e.printStackTrace(); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java index e239aa068b..c2f732c869 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java @@ -8,11 +8,6 @@ import java.sql.*; public class RestfulDriverTest { private static final String host = "127.0.0.1"; - @Test - public void connect() { - - } - @Test public void acceptsURL() throws SQLException { Driver driver = new RestfulDriver(); @@ -27,9 +22,7 @@ public class RestfulDriverTest { Driver driver = new RestfulDriver(); final String url = ""; DriverPropertyInfo[] propertyInfo = driver.getPropertyInfo(url, null); - for (DriverPropertyInfo prop : propertyInfo) { - System.out.println(prop); - } + Assert.assertNotNull(propertyInfo); } @Test diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java index c8bb69d827..b07dae8003 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java @@ -10,132 +10,151 @@ import java.util.Random; public class RestfulJDBCTest { private static final String host = "127.0.0.1"; - private static Connection connection; - private Random random = new Random(System.currentTimeMillis()); + private final Random random = new Random(System.currentTimeMillis()); + private Connection connection; - /** - * select * from log.log - **/ @Test public void testCase001() { - try { - Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery("select * from log.log"); - ResultSetMetaData metaData = resultSet.getMetaData(); - while (resultSet.next()) { - for (int i = 1; i <= metaData.getColumnCount(); i++) { - String column = metaData.getColumnLabel(i); - String value = resultSet.getString(i); - System.out.print(column + ":" + value + "\t"); - } - System.out.println(); - } - statement.close(); - } catch (SQLException e) { - e.printStackTrace(); - } + // given + String sql = "drop database if exists restful_test"; + // when + boolean execute = execute(connection, sql); + // then + Assert.assertFalse(execute); + + // given + sql = "create database if not exists restful_test"; + // when + execute = execute(connection, sql); + // then + Assert.assertFalse(execute); + + // given + sql = "use restful_test"; + // when + execute = execute(connection, sql); + // then + Assert.assertFalse(execute); } - /** - * create database - */ @Test public void testCase002() { - try (Statement stmt = connection.createStatement()) { - stmt.execute("drop database if exists restful_test"); - stmt.execute("create database if not exists restful_test"); - stmt.execute("use restful_test"); - } catch (SQLException e) { - e.printStackTrace(); - } - } - - /** - * create super table - ***/ - @Test - public void testCase003() { - try (Statement stmt = connection.createStatement()) { - stmt.execute("create table weather(ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int)"); - } catch (SQLException e) { - e.printStackTrace(); - } + // given + String sql = "create table weather(ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int)"; + // when + boolean execute = execute(connection, sql); + // then + Assert.assertFalse(execute); } @Test public void testCase004() { - try (Statement stmt = connection.createStatement()) { - for (int i = 1; i <= 100; i++) { - stmt.execute("create table t" + i + " using weather tags('beijing', '" + i + "')"); - } - } catch (SQLException e) { - e.printStackTrace(); + for (int i = 1; i <= 100; i++) { + // given + String sql = "create table t" + i + " using weather tags('beijing', '" + i + "')"; + // when + boolean execute = execute(connection, sql); + // then + Assert.assertFalse(execute); } } - @Test public void testCase005() { - try (Statement stmt = connection.createStatement()) { - int rows = 0; - for (int i = 0; i < 10; i++) { - for (int j = 1; j <= 100; j++) { - long currentTimeMillis = System.currentTimeMillis(); - int affectRows = stmt.executeUpdate("insert into t" + j + " values(" + currentTimeMillis + "," + (random.nextFloat() * 50) + "," + random.nextInt(100) + ")"); - Assert.assertEquals(1, affectRows); - rows += affectRows; - } + int rows = 0; + for (int i = 0; i < 10; i++) { + for (int j = 1; j <= 100; j++) { + + // given + long currentTimeMillis = System.currentTimeMillis(); + String sql = "insert into t" + j + " values(" + currentTimeMillis + "," + (random.nextFloat() * 50) + "," + random.nextInt(100) + ")"; + // when + int affectRows = executeUpdate(connection, sql); + // then + Assert.assertEquals(1, affectRows); + + rows += affectRows; } - Assert.assertEquals(1000, rows); - } catch (SQLException e) { - e.printStackTrace(); } + Assert.assertEquals(1000, rows); } @Test - public void testCase006() { - try (Statement stmt = connection.createStatement()) { - ResultSet rs = stmt.executeQuery("select * from weather"); - while (rs.next()) { - System.out.print("ts: " + rs.getTimestamp("ts")); - System.out.print(", temperature: " + rs.getString("temperature")); - System.out.print(", humidity: " + rs.getString("humidity")); - System.out.println(", location: " + rs.getString("location")); - } - } catch (SQLException e) { - e.printStackTrace(); + public void testCase006() throws SQLException { + // given + String sql = "select * from weather"; + // when + ResultSet rs = executeQuery(connection, sql); + ResultSetMetaData meta = rs.getMetaData(); + + // then + Assert.assertEquals(5, meta.getColumnCount()); + + while (rs.next()) { + Assert.assertNotNull(rs.getTimestamp("ts")); + Assert.assertNotNull(rs.getFloat("temperature")); + Assert.assertNotNull(rs.getInt("humidity")); + Assert.assertNotNull(rs.getString("location")); } } @Test public void testCase007() { + // given + String sql = "drop database restful_test"; + + // when + boolean execute = execute(connection, sql); + + // then + Assert.assertFalse(execute); + } + + private int executeUpdate(Connection connection, String sql) { try (Statement stmt = connection.createStatement()) { - ResultSet rs = stmt.executeQuery("select * from weather"); - ResultSetMetaData meta = rs.getMetaData(); - while (rs.next()) { - int columnCount = meta.getColumnCount(); - for (int i = 1; i <= columnCount; i++) { - String columnLabel = meta.getColumnLabel(i); - String value = rs.getString(i); - System.out.print(columnLabel + ": " + value + "\t"); - } - System.out.println(); - } + return stmt.executeUpdate(sql); + } catch (SQLException e) { + e.printStackTrace(); + } + return 0; + } + + private boolean execute(Connection connection, String sql) { + try (Statement stmt = connection.createStatement()) { + return stmt.execute(sql); + } catch (SQLException e) { + e.printStackTrace(); + } + return false; + } + + + private ResultSet executeQuery(Connection connection, String sql) { + try (Statement statement = connection.createStatement()) { + return statement.executeQuery(sql); + } catch (SQLException e) { + e.printStackTrace(); + } + return null; + } + + @Before + public void before() { + try { + connection = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=root&password=taosdata"); } catch (SQLException e) { e.printStackTrace(); } } - @BeforeClass - public static void before() throws ClassNotFoundException, SQLException { - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); - connection = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=root&password=taosdata"); - } - - @AfterClass - public static void after() throws SQLException { - if (connection != null) - connection.close(); + @After + public void after() { + try { + if (connection != null) + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); + } } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java index ee457ff412..4760a723e4 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java @@ -15,6 +15,8 @@ public class RestfulPreparedStatementTest { private static PreparedStatement pstmt_insert; private static final String sql_select = "select * from t1 where ts > ? and ts <= ? and f1 >= ?"; private static PreparedStatement pstmt_select; + private static final String sql_without_parameters = "select count(*) from t1"; + private static PreparedStatement pstmt_without_parameters; @Test public void executeQuery() throws SQLException { @@ -27,12 +29,9 @@ public class RestfulPreparedStatementTest { ResultSet rs = pstmt_select.executeQuery(); Assert.assertNotNull(rs); ResultSetMetaData meta = rs.getMetaData(); - while (rs.next()) { - for (int i = 1; i <= meta.getColumnCount(); i++) { - System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); - } - System.out.println(); - } + int columnCount = meta.getColumnCount(); + Assert.assertEquals(10, columnCount); + Assert.assertNotNull(rs); } @Test @@ -240,6 +239,7 @@ public class RestfulPreparedStatementTest { @Test public void clearParameters() throws SQLException { pstmt_insert.clearParameters(); + pstmt_without_parameters.clearParameters(); } @Test @@ -373,18 +373,20 @@ public class RestfulPreparedStatementTest { @BeforeClass public static void beforeClass() { try { - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"); - try (Statement stmt = conn.createStatement()) { - stmt.execute("drop database if exists test_pstmt"); - stmt.execute("create database if not exists test_pstmt"); - stmt.execute("use test_pstmt"); - stmt.execute("create table weather(ts timestamp, f1 int, f2 bigint, f3 float, f4 double, f5 smallint, f6 tinyint, f7 bool, f8 binary(64), f9 nchar(64)) tags(loc nchar(64))"); - stmt.execute("create table t1 using weather tags('beijing')"); - } + + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists test_pstmt"); + stmt.execute("create database if not exists test_pstmt"); + stmt.execute("use test_pstmt"); + stmt.execute("create table weather(ts timestamp, f1 int, f2 bigint, f3 float, f4 double, f5 smallint, f6 tinyint, f7 bool, f8 binary(64), f9 nchar(64)) tags(loc nchar(64))"); + stmt.execute("create table t1 using weather tags('beijing')"); + stmt.close(); + pstmt_insert = conn.prepareStatement(sql_insert); pstmt_select = conn.prepareStatement(sql_select); - } catch (ClassNotFoundException | SQLException e) { + pstmt_without_parameters = conn.prepareStatement(sql_without_parameters); + } catch (SQLException e) { e.printStackTrace(); } } @@ -396,6 +398,8 @@ public class RestfulPreparedStatementTest { pstmt_insert.close(); if (pstmt_select != null) pstmt_select.close(); + if (pstmt_without_parameters != null) + pstmt_without_parameters.close(); if (conn != null) conn.close(); } catch (SQLException e) { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java index 81e762c5ca..21a91669b2 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java @@ -95,13 +95,13 @@ public class RestfulResultSetTest { public void getBigDecimal() throws SQLException { BigDecimal f1 = rs.getBigDecimal("f1"); long actual = (f1 == null) ? 0 : f1.longValue(); - Assert.assertEquals(1609430400000l, actual); + Assert.assertEquals(1609430400000L, actual); BigDecimal f2 = rs.getBigDecimal("f2"); Assert.assertEquals(1, f2.intValue()); BigDecimal f3 = rs.getBigDecimal("f3"); - Assert.assertEquals(100l, f3.longValue()); + Assert.assertEquals(100L, f3.longValue()); BigDecimal f4 = rs.getBigDecimal("f4"); Assert.assertEquals(3.1415f, f4.floatValue(), 0.00000f); @@ -125,13 +125,13 @@ public class RestfulResultSetTest { Assert.assertEquals(1, Ints.fromByteArray(f2)); byte[] f3 = rs.getBytes("f3"); - Assert.assertEquals(100l, Longs.fromByteArray(f3)); + Assert.assertEquals(100L, Longs.fromByteArray(f3)); byte[] f4 = rs.getBytes("f4"); - Assert.assertEquals(3.1415f, Float.valueOf(new String(f4)), 0.000000f); + Assert.assertEquals(3.1415f, Float.parseFloat(new String(f4)), 0.000000f); byte[] f5 = rs.getBytes("f5"); - Assert.assertEquals(3.1415926, Double.valueOf(new String(f5)), 0.000000f); + Assert.assertEquals(3.1415926, Double.parseDouble(new String(f5)), 0.000000f); byte[] f6 = rs.getBytes("f6"); Assert.assertEquals("abc", new String(f6)); @@ -222,7 +222,7 @@ public class RestfulResultSetTest { Object f3 = rs.getObject("f3"); Assert.assertEquals(Long.class, f3.getClass()); - Assert.assertEquals(100l, f3); + Assert.assertEquals(100L, f3); Object f4 = rs.getObject("f4"); Assert.assertEquals(Float.class, f4.getClass()); @@ -434,7 +434,7 @@ public class RestfulResultSetTest { @Test(expected = SQLFeatureNotSupportedException.class) public void updateLong() throws SQLException { - rs.updateLong(1, 1l); + rs.updateLong(1, 1L); } @Test(expected = SQLFeatureNotSupportedException.class) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulStatementTest.java index 1be32b502d..a7b3ceb9d3 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulStatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulStatementTest.java @@ -12,6 +12,7 @@ import java.util.UUID; public class RestfulStatementTest { private static final String host = "127.0.0.1"; + private static Connection conn; private static Statement stmt; @@ -21,11 +22,11 @@ public class RestfulStatementTest { ResultSet rs = stmt.executeQuery("show databases"); Assert.assertNotNull(rs); ResultSetMetaData meta = rs.getMetaData(); + int columnCount = meta.getColumnCount(); + Assert.assertTrue(columnCount > 1); while (rs.next()) { - for (int i = 1; i <= meta.getColumnCount(); i++) { - System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); - } - System.out.println(); + Assert.assertEquals("name", meta.getColumnLabel(1)); + Assert.assertNotNull(rs.getString("name")); } rs.close(); } catch (SQLException e) { @@ -174,10 +175,10 @@ public class RestfulStatementTest { Assert.assertEquals(3, meta.getColumnCount()); int count = 0; while (rs.next()) { - for (int i = 1; i <= meta.getColumnCount(); i++) { - System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); - } - System.out.println(); + Assert.assertEquals("ts", meta.getColumnLabel(1)); + Assert.assertNotNull(rs.getTimestamp(1)); + Assert.assertEquals("temperature", meta.getColumnLabel(2)); + Assert.assertEquals(22.33, rs.getFloat(2), 0.001f); count++; } Assert.assertEquals(1, count); @@ -388,15 +389,12 @@ public class RestfulStatementTest { @BeforeClass public static void beforeClass() { try { - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); Properties properties = new Properties(); properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata", properties); stmt = conn.createStatement(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); } catch (SQLException e) { e.printStackTrace(); } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java index 4ad9826384..6b88de258d 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java @@ -1,342 +1,586 @@ package com.taosdata.jdbc.rs; -import com.taosdata.jdbc.utils.SQLExecutor; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.FixMethodOrder; -import org.junit.Test; +import org.junit.*; import org.junit.runners.MethodSorters; import java.sql.*; @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class SQLTest { + private static final String host = "127.0.0.1"; private static Connection connection; @Test public void testCase001() { + // given String sql = "create database if not exists restful_test"; - SQLExecutor.execute(connection, sql); + + // when + boolean execute = execute(connection, sql); + + // then + Assert.assertFalse(execute); } @Test public void testCase002() { + // given String sql = "use restful_test"; - SQLExecutor.execute(connection, sql); + + // when + boolean execute = execute(connection, sql); + + // then + Assert.assertFalse(execute); } @Test public void testCase003() { + // given String sql = "show databases"; - SQLExecutor.executeWithResult(connection, sql); + + // when + boolean execute = execute(connection, sql); + + // then + Assert.assertTrue(execute); } @Test public void testCase004() { + // given String sql = "show tables"; - SQLExecutor.executeWithResult(connection, sql); + + // when + boolean execute = execute(connection, sql); + + // then + Assert.assertTrue(execute); } @Test public void testCase005() { + // given String sql = "show stables"; - SQLExecutor.executeWithResult(connection, sql); + + // when + boolean execute = execute(connection, sql); + + // then + Assert.assertTrue(execute); } @Test public void testCase006() { + // given String sql = "show dnodes"; - SQLExecutor.executeWithResult(connection, sql); + + // when + boolean execute = execute(connection, sql); + + // then + Assert.assertTrue(execute); } @Test public void testCase007() { + // given String sql = "show vgroups"; - SQLExecutor.executeWithResult(connection, sql); + + // when + boolean execute = execute(connection, sql); + + // then + Assert.assertTrue(execute); } @Test public void testCase008() { + // given String sql = "drop table if exists restful_test.weather"; - SQLExecutor.execute(connection, sql); + + // when + boolean execute = execute(connection, sql); + + // then + Assert.assertFalse(execute); } @Test public void testCase009() { + // given String sql = "create table if not exists restful_test.weather(ts timestamp, temperature float) tags(location nchar(64))"; - SQLExecutor.execute(connection, sql); + + // when + boolean execute = execute(connection, sql); + + // then + Assert.assertFalse(execute); } @Test public void testCase010() { + // given String sql = "create table t1 using restful_test.weather tags('北京')"; - SQLExecutor.execute(connection, sql); + + // when + boolean execute = execute(connection, sql); + + // then + Assert.assertFalse(execute); } @Test public void testCase011() { + // given String sql = "insert into restful_test.t1 values(now, 22.22)"; - SQLExecutor.executeUpdate(connection, sql); + + // when + boolean execute = execute(connection, sql); + + // then + Assert.assertFalse(execute); } @Test public void testCase012() { + // given String sql = "insert into restful_test.t1 values('2020-01-01 00:00:00.000', 22.22)"; - SQLExecutor.executeUpdate(connection, sql); + + // when + boolean execute = execute(connection, sql); + + // then + Assert.assertFalse(execute); } @Test public void testCase013() { + // given String sql = "insert into restful_test.t1 values('2020-01-01 00:01:00.000', 22.22),('2020-01-01 00:02:00.000', 22.22)"; - SQLExecutor.executeUpdate(connection, sql); + + // when + boolean execute = execute(connection, sql); + + // then + Assert.assertFalse(execute); } @Test public void testCase014() { + // given String sql = "insert into restful_test.t2 using weather tags('上海') values('2020-01-01 00:03:00.000', 22.22)"; - SQLExecutor.executeUpdate(connection, sql); + + // when + boolean execute = execute(connection, sql); + + // then + Assert.assertFalse(execute); } @Test public void testCase015() { + // given String sql = "insert into restful_test.t2 using weather tags('上海') values('2020-01-01 00:01:00.000', 22.22),('2020-01-01 00:02:00.000', 22.22)"; - SQLExecutor.executeUpdate(connection, sql); + + // when + boolean execute = execute(connection, sql); + + // then + Assert.assertFalse(execute); } @Test public void testCase016() { + // given String sql = "insert into t1 values('2020-01-01 01:0:00.000', 22.22),('2020-01-01 02:00:00.000', 22.22) t2 values('2020-01-01 01:0:00.000', 33.33),('2020-01-01 02:00:00.000', 33.33)"; - SQLExecutor.executeUpdate(connection, sql); + + // when + boolean execute = execute(connection, sql); + + // then + Assert.assertFalse(execute); } @Test public void testCase017() { + // given String sql = "Insert into t3 using weather tags('广东') values('2020-01-01 01:0:00.000', 22.22),('2020-01-01 02:00:00.000', 22.22) t4 using weather tags('天津') values('2020-01-01 01:0:00.000', 33.33),('2020-01-01 02:00:00.000', 33.33)"; - SQLExecutor.executeUpdate(connection, sql); + + // when + boolean execute = execute(connection, sql); + + // then + Assert.assertFalse(execute); } @Test public void testCase018() { + // given String sql = "select * from restful_test.t1"; - SQLExecutor.executeQuery(connection, sql); + // when + boolean execute = execute(connection, sql); + // then + Assert.assertTrue(execute); } @Test public void testCase019() { + // given String sql = "select * from restful_test.weather"; - SQLExecutor.executeQuery(connection, sql); + // when + boolean execute = execute(connection, sql); + // then + Assert.assertTrue(execute); } @Test public void testCase020() { + // given String sql = "select ts, temperature from restful_test.t1"; - SQLExecutor.executeQuery(connection, sql); + // when + boolean execute = execute(connection, sql); + // then + Assert.assertTrue(execute); } @Test public void testCase021() { + // given String sql = "select ts, temperature from restful_test.weather"; - SQLExecutor.executeQuery(connection, sql); + // when + boolean execute = execute(connection, sql); + // then + Assert.assertTrue(execute); } @Test public void testCase022() { + // given String sql = "select temperature, ts from restful_test.t1"; - SQLExecutor.executeQuery(connection, sql); + // when + boolean execute = execute(connection, sql); + // then + Assert.assertTrue(execute); } @Test public void testCase023() { + // given String sql = "select temperature, ts from restful_test.weather"; - SQLExecutor.executeQuery(connection, sql); + // when + boolean execute = execute(connection, sql); + // then + Assert.assertTrue(execute); } @Test public void testCase024() { + // given String sql = "import into restful_test.t5 using weather tags('石家庄') values('2020-01-01 00:01:00.000', 22.22)"; - SQLExecutor.executeUpdate(connection, sql); + // when + int affectedRows = executeUpdate(connection, sql); + // then + Assert.assertEquals(1, affectedRows); } @Test public void testCase025() { + // given String sql = "import into restful_test.t6 using weather tags('ć˛éł') values('2020-01-01 00:01:00.000', 22.22),('2020-01-01 00:02:00.000', 22.22)"; - SQLExecutor.executeUpdate(connection, sql); + // when + int affectedRows = executeUpdate(connection, sql); + // then + Assert.assertEquals(2, affectedRows); } @Test public void testCase026() { + // given String sql = "import into restful_test.t7 using weather tags('长沙') values('2020-01-01 00:01:00.000', 22.22) restful_test.t8 using weather tags('ĺ‰ćž—') values('2020-01-01 00:01:00.000', 22.22)"; - SQLExecutor.executeUpdate(connection, sql); + + // when + int affectedRows = executeUpdate(connection, sql); + // then + Assert.assertEquals(2, affectedRows); } @Test public void testCase027() { + // given String sql = "import into restful_test.t9 using weather tags('武汉') values('2020-01-01 00:01:00.000', 22.22) ,('2020-01-02 00:01:00.000', 22.22) restful_test.t10 using weather tags('ĺ“尔滨') values('2020-01-01 00:01:00.000', 22.22),('2020-01-02 00:01:00.000', 22.22)"; - SQLExecutor.executeUpdate(connection, sql); + // when + int affectedRows = executeUpdate(connection, sql); + // then + Assert.assertEquals(4, affectedRows); } @Test public void testCase028() { + // given String sql = "select location, temperature, ts from restful_test.weather where temperature > 1"; - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); } @Test public void testCase029() { String sql = "select location, temperature, ts from restful_test.weather where temperature < 1"; - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); } @Test public void testCase030() { String sql = "select location, temperature, ts from restful_test.weather where ts > now"; - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); } @Test public void testCase031() { String sql = "select location, temperature, ts from restful_test.weather where ts < now"; - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); } @Test public void testCase032() { String sql = "select count(*) from restful_test.weather"; - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); } @Test public void testCase033() { String sql = "select first(*) from restful_test.weather"; - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); } @Test public void testCase034() { String sql = "select last(*) from restful_test.weather"; - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); } @Test public void testCase035() { String sql = "select last_row(*) from restful_test.weather"; - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); } @Test public void testCase036() { String sql = "select ts, ts as primary_key from restful_test.weather"; - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); } @Test public void testCase037() { String sql = "select database()"; - SQLExecutor.execute(connection, "use restful_test"); - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); } @Test public void testCase038() { String sql = "select client_version()"; - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); } @Test public void testCase039() { String sql = "select server_status()"; - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); } @Test public void testCase040() { String sql = "select server_status() as status"; - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); } @Test public void testCase041() { String sql = "select tbname, location from restful_test.weather"; - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); } @Test public void testCase042() { String sql = "select count(tbname) from restful_test.weather"; - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); } @Test public void testCase043() { String sql = "select * from restful_test.weather where ts < now - 1h"; - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); } @Test public void testCase044() { String sql = "select * from restful_test.weather where ts < now - 1h and location like '%'"; - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); } @Test public void testCase045() { String sql = "select * from restful_test.weather where ts < now - 1h order by ts"; - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); } @Test public void testCase046() { String sql = "select last(*) from restful_test.weather where ts < now - 1h group by tbname order by tbname"; - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); } @Test public void testCase047() { String sql = "select * from restful_test.weather limit 2"; - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); } @Test public void testCase048() { String sql = "select * from restful_test.weather limit 2 offset 5"; - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); } @Test public void testCase049() { String sql = "select * from restful_test.t1, restful_test.t3 where t1.ts = t3.ts "; - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); } @Test public void testCase050() { String sql = "select * from restful_test.t1, restful_test.t3 where t1.ts = t3.ts and t1.location = t3.location"; - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); } @Test public void testCase051() { String sql = "select * from restful_test.t1 tt, restful_test.t3 yy where tt.ts = yy.ts"; - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); } @Test public void testCase052() { String sql = "select server_status()"; - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); } @Test public void testCase053() { String sql = "select avg(cpu_taosd), avg(cpu_system), max(cpu_cores), avg(mem_taosd), avg(mem_system), max(mem_total), avg(disk_used), max(disk_total), avg(band_speed), avg(io_read), avg(io_write), sum(req_http), sum(req_select), sum(req_insert) from log.dn1 where ts> now - 60m and ts<= now interval(1m) fill(value, 0)"; - SQLExecutor.executeQuery(connection, sql); + // when + ResultSet rs = executeQuery(connection, sql); + // then + Assert.assertNotNull(rs); + } + + private boolean execute(Connection connection, String sql) { + try (Statement statement = connection.createStatement()) { + return statement.execute(sql); + } catch (SQLException e) { + e.printStackTrace(); + } + return false; + } + + private ResultSet executeQuery(Connection connection, String sql) { + try (Statement statement = connection.createStatement()) { + return statement.executeQuery(sql); + } catch (SQLException e) { + e.printStackTrace(); + } + return null; + } + + private int executeUpdate(Connection connection, String sql) { + try (Statement statement = connection.createStatement()) { + return statement.executeUpdate(sql); + } catch (SQLException e) { + e.printStackTrace(); + } + return 0; } @BeforeClass - public static void before() throws ClassNotFoundException, SQLException { - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + public static void before() throws SQLException { connection = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=root&password=taosdata"); } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/OSUtilsTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/OSUtilsTest.java index fd6c83ad1c..269a19e7a0 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/OSUtilsTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/OSUtilsTest.java @@ -10,17 +10,17 @@ public class OSUtilsTest { @Test public void inWindows() { - Assert.assertEquals(OS.indexOf("win") >= 0, OSUtils.isWindows()); + Assert.assertEquals(OS.contains("win"), OSUtils.isWindows()); } @Test public void isMac() { - Assert.assertEquals(OS.indexOf("mac") >= 0, OSUtils.isMac()); + Assert.assertEquals(OS.contains("mac"), OSUtils.isMac()); } @Test public void isLinux() { - Assert.assertEquals(OS.indexOf("nux") >= 0, OSUtils.isLinux()); + Assert.assertEquals(OS.contains("nux"), OSUtils.isLinux()); } @Before diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SQLExecutor.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SQLExecutor.java deleted file mode 100644 index bf034bf458..0000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SQLExecutor.java +++ /dev/null @@ -1,74 +0,0 @@ -package com.taosdata.jdbc.utils; - -import java.sql.*; - -public class SQLExecutor { - - // insert, import - public static void executeUpdate(Connection connection, String sql) { - try (Statement statement = connection.createStatement()) { - long start = System.currentTimeMillis(); - int affectedRows = statement.executeUpdate(sql); - long end = System.currentTimeMillis(); - System.out.println("[ affected rows : " + affectedRows + " ] time cost: " + (end - start) + " ms, execute statement ====> " + sql); - } catch (SQLException e) { - e.printStackTrace(); - } - } - - // show databases, show tables, show stables - public static void executeWithResult(Connection connection, String sql) { - try (Statement statement = connection.createStatement()) { - statement.execute(sql); - ResultSet resultSet = statement.getResultSet(); - printResult(resultSet); - } catch (SQLException e) { - e.printStackTrace(); - } - } - - // use database, create database, create table, drop table... - public static void execute(Connection connection, String sql) { - try (Statement statement = connection.createStatement()) { - long start = System.currentTimeMillis(); - boolean execute = statement.execute(sql); - long end = System.currentTimeMillis(); - printSql(sql, execute, (end - start)); - } catch (SQLException e) { - System.out.println("ERROR execute SQL ===> " + sql); - e.printStackTrace(); - } - } - - // select - public static void executeQuery(Connection connection, String sql) { - try (Statement statement = connection.createStatement()) { - long start = System.currentTimeMillis(); - ResultSet resultSet = statement.executeQuery(sql); - long end = System.currentTimeMillis(); - printSql(sql, true, (end - start)); - printResult(resultSet); - } catch (SQLException e) { - System.out.println("ERROR execute SQL ===> " + sql); - e.printStackTrace(); - } - } - - private static void printSql(String sql, boolean succeed, long cost) { - System.out.println("[ " + (succeed ? "OK" : "ERROR!") + " ] time cost: " + cost + " ms, execute statement ====> " + sql); - } - - private static void printResult(ResultSet resultSet) throws SQLException { - ResultSetMetaData metaData = resultSet.getMetaData(); - while (resultSet.next()) { - StringBuilder sb = new StringBuilder(); - for (int i = 1; i <= metaData.getColumnCount(); i++) { - String columnLabel = metaData.getColumnLabel(i); - String value = resultSet.getString(i); - sb.append(columnLabel + ": " + value + "\t"); - } - System.out.println(sb.toString()); - } - } - -} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/TimestampUtil.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/TimestampUtil.java index 16f8269d24..eaebd92d8e 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/TimestampUtil.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/TimestampUtil.java @@ -21,47 +21,4 @@ public class TimestampUtil { SimpleDateFormat sdf = new SimpleDateFormat(datetimeFormat); return sdf.format(new Date(time)); } - - public static class TimeTuple { - public Long start; - public Long end; - public Long timeGap; - - TimeTuple(long start, long end, long timeGap) { - this.start = start; - this.end = end; - this.timeGap = timeGap; - } - } - - public static TimeTuple range(long start, long timeGap, long size) { - long now = System.currentTimeMillis(); - if (timeGap < 1) - timeGap = 1; - if (start == 0) - start = now - size * timeGap; - - // 如果size小于1异常 - if (size < 1) - throw new IllegalArgumentException("size less than 1."); - // 如果timeGap为1，已经超长，需č¦ĺ‰Ťç§»start - if (start + size > now) { - start = now - size; - return new TimeTuple(start, now, 1); - } - long end = start + (long) (timeGap * size); - if (end > now) { - //压缩timeGap - end = now; - double gap = (end - start) / (size * 1.0f); - if (gap < 1.0f) { - timeGap = 1; - start = end - size; - } else { - timeGap = (long) gap; - end = start + (long) (timeGap * size); - } - } - return new TimeTuple(start, end, timeGap); - } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java index c861ef2966..1cbd95b249 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java @@ -3,22 +3,138 @@ package com.taosdata.jdbc.utils; import org.junit.Assert; import org.junit.Test; -import static org.junit.Assert.*; +import java.util.stream.Stream; public class UtilsTest { @Test public void escapeSingleQuota() { + // given String s = "'''''a\\'"; + // when String news = Utils.escapeSingleQuota(s); + // then Assert.assertEquals("\\'\\'\\'\\'\\'a\\'", news); - s = "\'''''a\\'"; + // given + s = "'''''a\\'"; + // when news = Utils.escapeSingleQuota(s); + // then Assert.assertEquals("\\'\\'\\'\\'\\'a\\'", news); - s = "\'\'\'\''a\\'"; + // given + s = "'''''a\\'"; + // when news = Utils.escapeSingleQuota(s); + // then Assert.assertEquals("\\'\\'\\'\\'\\'a\\'", news); } + + @Test + public void lowerCase() { + // given + String nativeSql = "insert into ?.? (ts, temperature, humidity) using ?.? tags(?,?) values(now, ?, ?)"; + Object[] parameters = Stream.of("test", "t1", "test", "weather", "beijing", 1, 12.2, 4).toArray(); + + // when + String actual = Utils.getNativeSql(nativeSql, parameters); + + // then + String expected = "insert into test.t1 (ts, temperature, humidity) using test.weather tags('beijing',1) values(now, 12.2, 4)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void upperCase() { + // given + String nativeSql = "INSERT INTO ? (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (?) VALUES (?,?,?,?)"; + Object[] parameters = Stream.of("d1", 1, 123, 3.14, 220, 4).toArray(); + + // when + String actual = Utils.getNativeSql(nativeSql, parameters); + + // then + String expected = "INSERT INTO d1 (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (1) VALUES (123,3.14,220,4)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void multiValues() { + // given + String nativeSql = "INSERT INTO ? (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (?) VALUES (?,?,?,?),(?,?,?,?)"; + Object[] parameters = Stream.of("d1", 1, 100, 3.14, "abc", 4, 200, 3.1415, "xyz", 5).toArray(); + + // when + String actual = Utils.getNativeSql(nativeSql, parameters); + + // then + String expected = "INSERT INTO d1 (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (1) VALUES (100,3.14,'abc',4),(200,3.1415,'xyz',5)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void lineTerminator() { + // given + String nativeSql = "INSERT INTO ? (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (?) VALUES (?,?,\r\n?,?),(?,?,?,?)"; + Object[] parameters = Stream.of("d1", 1, 100, 3.14, "abc", 4, 200, 3.1415, "xyz", 5).toArray(); + + // when + String actual = Utils.getNativeSql(nativeSql, parameters); + + // then + String expected = "INSERT INTO d1 (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (1) VALUES (100,3.14,\r\n'abc',4),(200,3.1415,'xyz',5)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void lineTerminatorAndMultiValues() { + String nativeSql = "INSERT Into ? TAGS(?) VALUES(?,?,\r\n?,?),(?,? ,\r\n?,?) t? tags (?) Values (?,?,?\r\n,?),(?,?,?,?) t? Tags(?) values (?,?,?,?) , (?,?,?,?)"; + Object[] parameters = Stream.of("t1", "abc", 100, 1.1, "xxx", "xxx", 200, 2.2, "xxx", "xxx", 2, "bcd", 300, 3.3, "xxx", "xxx", 400, 4.4, "xxx", "xxx", 3, "cde", 500, 5.5, "xxx", "xxx", 600, 6.6, "xxx", "xxx").toArray(); + + // when + String actual = Utils.getNativeSql(nativeSql, parameters); + + // then + String expected = "INSERT Into t1 TAGS('abc') VALUES(100,1.1,\r\n'xxx','xxx'),(200,2.2 ,\r\n'xxx','xxx') t2 tags ('bcd') Values (300,3.3,'xxx'\r\n,'xxx'),(400,4.4,'xxx','xxx') t3 Tags('cde') values (500,5.5,'xxx','xxx') , (600,6.6,'xxx','xxx')"; + Assert.assertEquals(expected, actual); + } + + @Test + public void replaceNothing() { + // given + String nativeSql = "insert into test.t1 (ts, temperature, humidity) using test.weather tags('beijing',1) values(now, 12.2, 4)"; + + // when + String actual = Utils.getNativeSql(nativeSql, null); + + // then + Assert.assertEquals(nativeSql, actual); + } + + @Test + public void replaceNothing2() { + // given + String nativeSql = "insert into test.t1 (ts, temperature, humidity) using test.weather tags('beijing',1) values(now, 12.2, 4)"; + Object[] parameters = Stream.of("test", "t1", "test", "weather", "beijing", 1, 12.2, 4).toArray(); + + // when + String actual = Utils.getNativeSql(nativeSql, parameters); + + // then + Assert.assertEquals(nativeSql, actual); + } + + @Test + public void replaceNothing3() { + // given + String nativeSql = "insert into ?.? (ts, temperature, humidity) using ?.? tags(?,?) values(now, ?, ?)"; + + // when + String actual = Utils.getNativeSql(nativeSql, null); + + // then + Assert.assertEquals(nativeSql, actual); + + } } \ No newline at end of file From f026716c16d005277076ceb154c33961790ae393 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 24 Jul 2021 15:08:33 +0800 Subject: [PATCH 035/106] Hotfix/sangshuduo/td 5479 taosdump s e for master (#6995) * [TD-5479]: taosdump -S -E does not work. * taosdemo support timestamp step from command line. * fix help msg. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 4 ++-- src/kit/taosdump/taosdump.c | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 400799b065..9055f77c8a 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -742,8 +742,8 @@ static void printHelp() { "The number of threads. Default is 10."); printf("%s%s%s%s\n", indent, "-i", indent, "The sleep time (ms) between insertion. Default is 0."); - printf("%s%s%s%s%d\n", indent, "-S", indent, - "The timestamp step between insertion. Default is %d.", + printf("%s%s%s%s%d.\n", indent, "-S", indent, + "The timestamp step between insertion. Default is ", DEFAULT_TIMESTAMP_STEP); printf("%s%s%s%s\n", indent, "-r", indent, "The number of records per request. Default is 30000."); diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 17a4d7d19f..858f3c06aa 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -453,6 +453,8 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 'E': g_args.end_time = atol(arg); break; + case 'C': + break; case 'B': g_args.data_batch = atoi(arg); if (g_args.data_batch > MAX_RECORDS_PER_REQ) { From ea33ca91d31a33861278395ee89b1c0f994c5622 Mon Sep 17 00:00:00 2001 From: Linhe Huo Date: Sun, 25 Jul 2021 22:58:46 +0800 Subject: [PATCH 036/106] [TD-5476]: update to latest grafanaplugin (#6991) --- src/connector/grafanaplugin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin index 3530c6df09..4a4d79099b 160000 --- a/src/connector/grafanaplugin +++ b/src/connector/grafanaplugin @@ -1 +1 @@ -Subproject commit 3530c6df097134a410bacec6b3cd013ef38a61aa +Subproject commit 4a4d79099b076b8ff12d5b4fdbcba54049a6866d From 2d20b42588720d6c7ec9f03ea22d19a6991969ee Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 26 Jul 2021 09:51:49 +0800 Subject: [PATCH 037/106] Hotfix/sangshuduo/td 3197 taosdemo coverity scan for master (#7013) * [TD-3197]: taosdemo and taosdump coverity scan issues. * exit if read sample file failed. * fix converity scan issue. * fix coverity scan issue. * fix coverity scan memory leak. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 9055f77c8a..ed3ebed7ae 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -5920,14 +5920,15 @@ static int32_t prepareStbStmt( if (-1 == prepareStbStmtBind( tagsArray, stbInfo, tagRand, -1, -1, false /* is tag */)) { - free(tagsArray); + tmfree(tagsValBuf); + tmfree(tagsArray); return -1; } ret = taos_stmt_set_tbname_tags(stmt, tableName, (TAOS_BIND *)tagsArray); tmfree(tagsValBuf); - tmfree((char *)tagsArray); + tmfree(tagsArray); } else { ret = taos_stmt_set_tbname(stmt, tableName); } From 4eeea0d8d6971556b81c3a8cb0526ed27ff1b27c Mon Sep 17 00:00:00 2001 From: tomchon Date: Mon, 26 Jul 2021 10:14:47 +0800 Subject: [PATCH 038/106] [TD-5523]: arm32 4byte long error. --- src/query/src/qExecutor.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 982c45c441..d479e90415 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -149,12 +149,12 @@ static void getNextTimeWindow(SQueryAttr* pQueryAttr, STimeWindow* tw) { int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); tm.tm_year = mon / 12; tm.tm_mon = mon % 12; - tw->skey = convertTimePrecision(mktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, pQueryAttr->precision); + tw->skey = convertTimePrecision((int64_t)mktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, pQueryAttr->precision); mon = (int)(mon + interval); tm.tm_year = mon / 12; tm.tm_mon = mon % 12; - tw->ekey = convertTimePrecision(mktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, pQueryAttr->precision); + tw->ekey = convertTimePrecision((int64_t)mktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, pQueryAttr->precision); tw->ekey -= 1; } From c63d6f4c94344f7ec4564b137791edd1829ac0fb Mon Sep 17 00:00:00 2001 From: wenzhouwww Date: Mon, 26 Jul 2021 18:00:02 +0800 Subject: [PATCH 039/106] [TD-5369] fulltest.sh --- tests/pytest/fulltest.sh | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index ab7dedc959..923934ac69 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -161,7 +161,11 @@ python3 test.py -f tools/taosdemoTestTblAlt.py python3 test.py -f tools/taosdemoTestSampleData.py python3 test.py -f tools/taosdemoTestInterlace.py python3 test.py -f tools/taosdemoTestQuery.py - +# nano support +python3 test.py -f tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py +python3 test.py -f tools/taosdemoAllTest/taosdemoTestSupportNanoQuery.py +python3 test.py -f tools/taosdemoAllTest/taosdemoTestSupportNanosubscribe.py +python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertTime_step.py # update @@ -363,15 +367,15 @@ python3 test.py -f alter/alter_keep.py python3 test.py -f alter/alter_cacheLastRow.py python3 ./test.py -f query/querySession.py python3 test.py -f alter/alter_create_exception.py - -# nano support -python3 test.py -f tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py -python3 test.py -f tools/taosdemoAllTest/taosdemoTestSupportNanoQuery.py -python3 test.py -f tools/taosdemoAllTest/taosdemoTestSupportNanosubscribe.py -python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertTime_step.py python3 ./test.py -f insert/flushwhiledrop.py #======================p4-end=============== python3 test.py -f tools/taosdemoAllTest/pytest.py + + + + + + From a960f0ff790a3fc868171cd927753c34e8625d1c Mon Sep 17 00:00:00 2001 From: markswang <792637585@qq.com> Date: Mon, 26 Jul 2021 18:38:37 +0800 Subject: [PATCH 040/106] [TD-5534]:fix the coverity high risk of client --- src/client/src/TSDBJNIConnector.c | 15 +++++++++-- src/client/src/tscParseLineProtocol.c | 39 ++++++++++++++++++++++++--- src/client/src/tscPrepare.c | 4 +-- src/client/src/tscSQLParser.c | 13 ++++++--- src/client/src/tscServer.c | 6 ++++- src/common/src/tname.c | 2 +- 6 files changed, 67 insertions(+), 12 deletions(-) diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index c9b00800e6..667a689979 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -728,6 +728,7 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_prepareStmtImp(J int32_t code = taos_stmt_prepare(pStmt, str, len); if (code != TSDB_CODE_SUCCESS) { jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); + free(str); return JNI_TDENGINE_ERROR; } @@ -919,6 +920,10 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setTableNameTagsI char* curTags = tagsData; TAOS_BIND *tagsBind = calloc(numOfTags, sizeof(TAOS_BIND)); + if (tagsBind == NULL) { + jniError("numOfTags:%d, alloc memory failed", numOfTags); + return JNI_OUT_OF_MEMORY; + } for(int32_t i = 0; i < numOfTags; ++i) { tagsBind[i].buffer_type = typeArray[i]; tagsBind[i].buffer = curTags; @@ -941,9 +946,10 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setTableNameTagsI if (code != TSDB_CODE_SUCCESS) { jniError("jobj:%p, conn:%p, code:%s", jobj, tsconn, tstrerror(code)); + free(tagsBind); return JNI_TDENGINE_ERROR; } - + free(tagsBind); return JNI_SUCCESS; } @@ -957,7 +963,10 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_insertLinesImp(J int numLines = (*env)->GetArrayLength(env, lines); char** c_lines = calloc(numLines, sizeof(char*)); - + if (c_lines == NULL) { + jniError("c_lines:%d, alloc memory failed", c_lines); + return JNI_OUT_OF_MEMORY; + } for (int i = 0; i < numLines; ++i) { jstring line = (jstring) ((*env)->GetObjectArrayElement(env, lines, i)); c_lines[i] = (char*)(*env)->GetStringUTFChars(env, line, 0); @@ -972,8 +981,10 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_insertLinesImp(J if (code != TSDB_CODE_SUCCESS) { jniError("jobj:%p, conn:%p, code:%s", jobj, taos, tstrerror(code)); + free(c_lines); return JNI_TDENGINE_ERROR; } + free(c_lines); return code; } \ No newline at end of file diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index d5883af7f6..f0e5017355 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -411,6 +411,11 @@ int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema) { taos_free_result(res); SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); + if (pSql == NULL){ + tscError("failed to allocate memory, reason:%s", strerror(errno)); + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return code; + } pSql->pTscObj = taos; pSql->signature = pSql; pSql->fp = NULL; @@ -421,11 +426,13 @@ int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema) { if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; sprintf(pSql->cmd.payload, "table name is invalid"); + tscFreeSqlObj(pSql); return code; } SName sname = {0}; if ((code = tscSetTableFullName(&sname, &tableToken, pSql)) != TSDB_CODE_SUCCESS) { + tscFreeSqlObj(pSql); return code; } char fullTableName[TSDB_TABLE_FNAME_LEN] = {0}; @@ -607,6 +614,10 @@ static int32_t changeChildTableTagValue(TAOS* taos, const char* cTableName, cons static int32_t creatChildTableIfNotExists(TAOS* taos, const char* cTableName, const char* sTableName, SArray* tagsSchema, SArray* tagsBind) { size_t numTags = taosArrayGetSize(tagsSchema); char* sql = malloc(tsMaxSQLStringLen+1); + if (sql == NULL) { + tscError("malloc sql memory error"); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } int freeBytes = tsMaxSQLStringLen + 1; sprintf(sql, "create table if not exists %s using %s", cTableName, sTableName); @@ -628,24 +639,31 @@ static int32_t creatChildTableIfNotExists(TAOS* taos, const char* cTableName, co tscDebug("create table : %s", sql); TAOS_STMT* stmt = taos_stmt_init(taos); + if (stmt == NULL) { + free(sql); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } int32_t code; code = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql)); free(sql); if (code != 0) { tscError("%s", taos_stmt_errstr(stmt)); + free(stmt); return code; } code = taos_stmt_bind_param(stmt, TARRAY_GET_START(tagsBind)); if (code != 0) { tscError("%s", taos_stmt_errstr(stmt)); + free(stmt); return code; } code = taos_stmt_execute(stmt); if (code != 0) { tscError("%s", taos_stmt_errstr(stmt)); + free(stmt); return code; } @@ -660,6 +678,11 @@ static int32_t creatChildTableIfNotExists(TAOS* taos, const char* cTableName, co static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* colsSchema, SArray* rowsBind) { size_t numCols = taosArrayGetSize(colsSchema); char* sql = malloc(tsMaxSQLStringLen+1); + if (sql == NULL) { + tscError("malloc sql memory error"); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + int32_t freeBytes = tsMaxSQLStringLen + 1 ; sprintf(sql, "insert into ? ("); @@ -681,11 +704,15 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols int32_t try = 0; TAOS_STMT* stmt = taos_stmt_init(taos); - + if (stmt == NULL) { + free(sql); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } code = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql)); free(sql); if (code != 0) { + free(stmt); tscError("%s", taos_stmt_errstr(stmt)); return code; } @@ -694,6 +721,7 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols code = taos_stmt_set_tbname(stmt, cTableName); if (code != 0) { tscError("%s", taos_stmt_errstr(stmt)); + free(stmt); return code; } @@ -703,11 +731,13 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols code = taos_stmt_bind_param(stmt, colsBinds); if (code != 0) { tscError("%s", taos_stmt_errstr(stmt)); + free(stmt); return code; } code = taos_stmt_add_batch(stmt); if (code != 0) { tscError("%s", taos_stmt_errstr(stmt)); + free(stmt); return code; } } @@ -1627,7 +1657,7 @@ static int32_t parseSmlTimeStamp(TAOS_SML_KV **pTS, const char **index) { static int32_t parseSmlKey(TAOS_SML_KV *pKV, const char **index) { const char *cur = *index; - char key[TSDB_COL_NAME_LEN]; + char key[TSDB_COL_NAME_LEN + 1]; // +1 to avoid 1685 line over write uint16_t len = 0; //key field cannot start with digit @@ -1704,7 +1734,10 @@ static int32_t parseSmlMeasurement(TAOS_SML_DATA_POINT *pSml, const char **index const char *cur = *index; uint16_t len = 0; - pSml->stableName = calloc(TSDB_TABLE_NAME_LEN, 1); + pSml->stableName = calloc(TSDB_TABLE_NAME_LEN + 1, 1); // +1 to avoid 1772 line over write + if (pSml->stableName == NULL){ + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } if (isdigit(*cur)) { tscError("Measurement field cannnot start with digit"); free(pSml->stableName); diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 7306523660..efdc24c899 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -1628,8 +1628,8 @@ int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags if (pStmt->mtb.subSet && taosHashGetSize(pStmt->mtb.pTableHash) > 0) { STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - char sTableName[TSDB_TABLE_FNAME_LEN]; - strncpy(sTableName, pTableMeta->sTableName, sizeof(sTableName)); + char sTableName[TSDB_TABLE_FNAME_LEN + 1] = {0}; + strncpy(sTableName, pTableMeta->sTableName, sizeof(sTableName) - 1); SStrToken tname = {0}; tname.type = TK_STRING; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 9a3b36895d..2d5a3d524e 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -421,7 +421,8 @@ int32_t readFromFile(char *name, uint32_t *len, void **buf) { tfree(*buf); return TSDB_CODE_TSC_APP_ERROR; } - + close(fd); + tfree(*buf); return TSDB_CODE_SUCCESS; } @@ -8110,7 +8111,8 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { assert(maxSize < 80 * TSDB_MAX_COLUMNS); if (!pSql->pBuf) { if (NULL == (pSql->pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _end; } } pTableMeta = calloc(1, maxSize); @@ -8351,14 +8353,18 @@ static int32_t doValidateSubquery(SSqlNode* pSqlNode, int32_t index, SSqlObj* pS // create dummy table meta info STableMetaInfo* pTableMetaInfo1 = calloc(1, sizeof(STableMetaInfo)); + if (pTableMetaInfo1 == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } pTableMetaInfo1->pTableMeta = extractTempTableMetaFromSubquery(pSub); if (subInfo->aliasName.n > 0) { if (subInfo->aliasName.n >= TSDB_TABLE_FNAME_LEN) { + free(pTableMetaInfo1); return invalidOperationMsg(msgBuf, "subquery alias name too long"); } - strncpy(pTableMetaInfo1->aliasName, subInfo->aliasName.z, subInfo->aliasName.n); + strncpy(pTableMetaInfo1->aliasName, subInfo->aliasName.z, MIN(subInfo->aliasName.n, sizeof(pTableMetaInfo1->aliasName) - 1)); } taosArrayPush(pQueryInfo->pUpstream, &pSub); @@ -8368,6 +8374,7 @@ static int32_t doValidateSubquery(SSqlNode* pSqlNode, int32_t index, SSqlObj* pS STableMetaInfo** tmp = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES); if (tmp == NULL) { + free(pTableMetaInfo1); return TSDB_CODE_TSC_OUT_OF_MEMORY; } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index f5d6765a5d..401e65efd4 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -164,7 +164,7 @@ static void tscUpdateVgroupInfo(SSqlObj *pSql, SRpcEpSet *pEpSet) { vgroupInfo.inUse = pEpSet->inUse; vgroupInfo.numOfEps = pEpSet->numOfEps; for (int32_t i = 0; i < vgroupInfo.numOfEps; i++) { - strncpy(vgroupInfo.ep[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN); + strncpy(vgroupInfo.ep[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN); // buffer not null terminated risk vgroupInfo.ep[i].port = pEpSet->port[i]; } @@ -2048,8 +2048,12 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { assert(pTableMetaInfo->pTableMeta == NULL); STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); + if (pTableMeta == NULL){ + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) { tscError("0x%"PRIx64" invalid table meta from mnode, name:%s", pSql->self, tNameGetTableName(&pTableMetaInfo->name)); + free(pTableMeta); return TSDB_CODE_TSC_INVALID_VALUE; } diff --git a/src/common/src/tname.c b/src/common/src/tname.c index 26502c5d9c..5da48b2e9a 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -319,7 +319,7 @@ int32_t tNameGetDbName(const SName* name, char* dst) { int32_t tNameGetFullDbName(const SName* name, char* dst) { assert(name != NULL && dst != NULL); - snprintf(dst, TSDB_ACCT_ID_LEN + TS_PATH_DELIMITER_LEN + TSDB_DB_NAME_LEN, + snprintf(dst, TSDB_ACCT_ID_LEN + TS_PATH_DELIMITER_LEN + TSDB_DB_NAME_LEN, // there is a over write risk "%s.%s", name->acctId, name->dbname); return 0; } From 10956b4a5e53512c546b429b82b6f71b200b069d Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 27 Jul 2021 07:59:28 +0800 Subject: [PATCH 041/106] Feature/sangshuduo/td 4068 taosdemo stmt for master (#7020) * [TD-4068]: taosdemo support stmt. for easy merge purpose. disabled in master. * fix clang compile error. * fix memory leak, add more macros. change sqlcount to int * fix rest segfault. * add stmt_errstr() for reason print. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 51 +++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index ed3ebed7ae..64adc814e2 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -5134,8 +5134,8 @@ static int32_t execInsert(threadInfo *pThreadInfo, uint32_t k) debugPrint("%s() LN%d, stmt=%p", __func__, __LINE__, pThreadInfo->stmt); if (0 != taos_stmt_execute(pThreadInfo->stmt)) { - errorPrint("%s() LN%d, failied to execute insert statement\n", - __func__, __LINE__); + errorPrint("%s() LN%d, failied to execute insert statement. reason: %s\n", + __func__, __LINE__, taos_stmt_errstr(pThreadInfo->stmt)); exit(-1); } affectedRows = k; @@ -5698,7 +5698,7 @@ static int32_t prepareStmtWithoutStb( int ret = taos_stmt_set_tbname(stmt, tableName); if (ret != 0) { errorPrint("failed to execute taos_stmt_set_tbname(%s). return 0x%x. reason: %s\n", - tableName, ret, taos_errstr(NULL)); + tableName, ret, taos_stmt_errstr(stmt)); return ret; } @@ -5751,9 +5751,17 @@ static int32_t prepareStmtWithoutStb( return -1; } } - taos_stmt_bind_param(stmt, (TAOS_BIND *)bindArray); + if (0 != taos_stmt_bind_param(stmt, (TAOS_BIND *)bindArray)) { + errorPrint("%s() LN%d, stmt_bind_param() failed! reason: %s\n", + __func__, __LINE__, taos_stmt_errstr(stmt)); + break; + } // if msg > 3MB, break - taos_stmt_add_batch(stmt); + if (0 != taos_stmt_add_batch(stmt)) { + errorPrint("%s() LN%d, stmt_add_batch() failed! reason: %s\n", + __func__, __LINE__, taos_stmt_errstr(stmt)); + break; + } k++; recordFrom ++; @@ -5929,14 +5937,19 @@ static int32_t prepareStbStmt( tmfree(tagsValBuf); tmfree(tagsArray); + + if (0 != ret) { + errorPrint("%s() LN%d, stmt_set_tbname_tags() failed! reason: %s\n", + __func__, __LINE__, taos_stmt_errstr(stmt)); + return -1; + } } else { ret = taos_stmt_set_tbname(stmt, tableName); - } - - if (ret != 0) { - errorPrint("failed to execute taos_stmt_set_tbname(%s). return 0x%x. reason: %s\n", - tableName, ret, taos_errstr(NULL)); - return ret; + if (0 != ret) { + errorPrint("%s() LN%d, stmt_set_tbname() failed! reason: %s\n", + __func__, __LINE__, taos_stmt_errstr(stmt)); + return -1; + } } char *bindArray = calloc(1, sizeof(TAOS_BIND) * (stbInfo->columnCount + 1)); @@ -5954,9 +5967,19 @@ static int32_t prepareStbStmt( free(bindArray); return -1; } - taos_stmt_bind_param(stmt, (TAOS_BIND *)bindArray); + ret = taos_stmt_bind_param(stmt, (TAOS_BIND *)bindArray); + if (0 != ret) { + errorPrint("%s() LN%d, stmt_bind_param() failed! reason: %s\n", + __func__, __LINE__, taos_stmt_errstr(stmt)); + return -1; + } // if msg > 3MB, break - taos_stmt_add_batch(stmt); + ret = taos_stmt_add_batch(stmt); + if (0 != ret) { + errorPrint("%s() LN%d, stmt_add_batch() failed! reason: %s\n", + __func__, __LINE__, taos_stmt_errstr(stmt)); + return -1; + } k++; recordFrom ++; @@ -6904,7 +6927,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, int ret = taos_stmt_prepare(pThreadInfo->stmt, buffer, 0); if (ret != 0){ errorPrint("failed to execute taos_stmt_prepare. return 0x%x. reason: %s\n", - ret, taos_errstr(NULL)); + ret, taos_stmt_errstr(pThreadInfo->stmt)); free(pids); free(infos); exit(-1); From 4833f5eaa3a5d5368dc86ef86be0667be7dc7ffe Mon Sep 17 00:00:00 2001 From: tickduan <417921451@qq.com> Date: Tue, 27 Jul 2021 10:22:29 +0800 Subject: [PATCH 042/106] metrics_del.sim 23:50 insert two days bug fixed --- tests/script/general/stream/metrics_del.sim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/script/general/stream/metrics_del.sim b/tests/script/general/stream/metrics_del.sim index 321658cd8d..6cc3da71e9 100644 --- a/tests/script/general/stream/metrics_del.sim +++ b/tests/script/general/stream/metrics_del.sim @@ -34,11 +34,11 @@ while $i < $tbNum $tb = $tbPrefix . $i sql create table $tb using $mt tags( $i ) - $x = -1440 + $x = 0 $y = 0 while $y < $rowNum - $ms = $x . m - sql insert into $tb values (now $ms , $y , $y ) + $ts = 1626710400000 + $x + sql insert into $tb values ($ts , $y , $y ) $x = $x + 1 $y = $y + 1 endw From 74fa1114f314738539819e519d5e56daab909bf4 Mon Sep 17 00:00:00 2001 From: wenzhouwww Date: Tue, 27 Jul 2021 11:06:04 +0800 Subject: [PATCH 043/106] [TD-5369] change the nums of tables and insert rows! --- .../taosdemoTestNanoDatabase.json | 8 ++--- .../taosdemoTestNanoDatabaseNow.json | 2 +- .../taosdemoTestSupportNanoInsert.py | 36 +++++++++---------- .../taosdemoTestSupportNanoQuery.py | 22 ++++++------ .../taosdemoTestSupportNanoQuerycsv.json | 2 +- .../taosdemoTestSupportNanoSubscribe.json | 2 +- .../taosdemoTestSupportNanosubscribe.py | 6 ++-- 7 files changed, 38 insertions(+), 40 deletions(-) diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabase.json b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabase.json index 9010415fe6..246f1c35f2 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabase.json +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabase.json @@ -35,13 +35,13 @@ "super_tables": [{ "name": "stb0", "child_table_exists":"no", - "childtable_count": 1000, + "childtable_count": 100, "childtable_prefix": "tb0_", "auto_create_table": "no", "batch_create_tbl_num": 20, "data_source": "rand", "insert_mode": "taosc", - "insert_rows": 10000, + "insert_rows": 100, "childtable_offset":0, "multi_thread_write_one_tbl": "no", "insert_interval":0, @@ -61,13 +61,13 @@ { "name": "stb1", "child_table_exists":"no", - "childtable_count": 1000, + "childtable_count": 100, "childtable_prefix": "tb1_", "auto_create_table": "no", "batch_create_tbl_num": 20, "data_source": "rand", "insert_mode": "taosc", - "insert_rows": 10000, + "insert_rows": 100, "childtable_offset":0, "multi_thread_write_one_tbl": "no", "insert_interval":0, diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseNow.json b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseNow.json index d2542a0eba..f36b1f9b4c 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseNow.json +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseNow.json @@ -35,7 +35,7 @@ "super_tables": [{ "name": "stb0", "child_table_exists":"no", - "childtable_count": 1000, + "childtable_count": 100, "childtable_prefix": "tb0_", "auto_create_table": "no", "batch_create_tbl_num": 20, diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py index 88a917da85..266a8fa712 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py @@ -56,46 +56,47 @@ class TDTestCase: os.system("%staosdemo -f tools/taosdemoAllTest/taosdemoTestNanoDatabase.json -y " % binPath) tdSql.execute("use nsdb") tdSql.query("show stables") - tdSql.checkData(0, 4, 1000) + tdSql.checkData(0, 4, 100) tdSql.query("select count (tbname) from stb0") - tdSql.checkData(0, 0, 1000) + tdSql.checkData(0, 0, 100) tdSql.query("select count(*) from tb0_0") - tdSql.checkData(0, 0, 10000) + tdSql.checkData(0, 0, 100) tdSql.query("select count(*) from stb0") - tdSql.checkData(0, 0, 10000000) + tdSql.checkData(0, 0, 10000) tdSql.query("describe stb0") tdSql.checkDataType(9, 1,"TIMESTAMP") tdSql.query("select last(ts) from stb0") - tdSql.getData(0, 0) + tdSql.checkData(0, 0,"2021-07-01 00:00:00.990000000") # check stable stb1 which is insert with disord tdSql.query("select count (tbname) from stb1") - tdSql.checkData(0, 0, 1000) + tdSql.checkData(0, 0, 100) tdSql.query("select count(*) from tb1_0") - tdSql.checkData(0, 0, 10000) + tdSql.checkData(0, 0, 100) tdSql.query("select count(*) from stb1") - tdSql.checkData(0, 0, 10000000) + tdSql.checkData(0, 0, 10000) # check c8 is an nano timestamp tdSql.query("describe stb1") tdSql.checkDataType(9, 1,"TIMESTAMP") # check insert timestamp_step is nano_second tdSql.query("select last(ts) from stb1") - tdSql.checkData(0, 0,"2021-07-01 00:01:39.990000000") - + tdSql.checkData(0, 0,"2021-07-01 00:00:00.990000000") + # insert data from now time # check stable stb0 os.system("%staosdemo -f tools/taosdemoAllTest/taosdemoTestNanoDatabaseNow.json -y " % binPath) + tdSql.execute("use nsdb2") tdSql.query("show stables") - tdSql.checkData(0, 4, 1000) + tdSql.checkData(0, 4, 100) tdSql.query("select count (tbname) from stb0") - tdSql.checkData(0, 0, 1000) + tdSql.checkData(0, 0, 100) tdSql.query("select count(*) from tb0_0") tdSql.checkData(0, 0, 100) tdSql.query("select count(*) from stb0") - tdSql.checkData(0, 0, 100000) + tdSql.checkData(0, 0, 10000) # check c8 is an nano timestamp tdSql.query("describe stb0") tdSql.checkDataType(9,1,"TIMESTAMP") @@ -117,16 +118,11 @@ class TDTestCase: os.system("rm -rf ./insert_res.txt") os.system("rm -rf tools/taosdemoAllTest/taosdemoTestSupportNano*.py.sql") + # taosdemo test insert with command and parameter , detals show taosdemo --help - - os.system("%staosdemo -u root -P taosdata -p 6030 -h vm84 -a 1 -m pre -n 10 -T 20 -t 60 -o res.txt -y " % binPath) + os.system("%staosdemo -u root -P taosdata -p 6030 -a 1 -m pre -n 10 -T 20 -t 60 -o res.txt -y " % binPath) tdSql.query("select count(*) from test.meters") tdSql.checkData(0, 0, 600) - - os.system("%staosdemo -u root -P taosdata -p 6030 -h vm84 -a 1 -m pre -n 10 -T 20 -t 60 -o res.txt -y " % binPath) - tdSql.query("select count(*) from test.meters") - tdSql.checkData(0, 0, 600) - # check taosdemo -s sqls_ls = ['drop database if exists nsdbsql;','create database nsdbsql precision "ns" keep 36 days 6 update 1;', diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoQuery.py b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoQuery.py index 21a7037ce6..5a37cf9c7c 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoQuery.py +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoQuery.py @@ -55,15 +55,15 @@ class TDTestCase: # use where to filter - tdSql.query("select count(*) from stb0 where ts>\"2021-07-01 00:01:00.000000000 \" ;") - tdSql.checkData(0, 0, 3999000) - tdSql.query("select count(*) from stb0 where ts>\"2021-07-01 00:01:00.000000000\" and ts <=\"2021-07-01 00:01:10.000000000\" ;") - tdSql.checkData(0, 0, 1000000) + tdSql.query("select count(*) from stb0 where ts>\"2021-07-01 00:00:00.590000000 \" ") + tdSql.checkData(0, 0, 4000) + tdSql.query("select count(*) from stb0 where ts>\"2021-07-01 00:00:00.000000000\" and ts <=\"2021-07-01 00:00:00.590000000\" ") + tdSql.checkData(0, 0, 5900) - tdSql.query("select count(*) from tb0_0 where ts>\"2021-07-01 00:01:00.000000000 \" ;") - tdSql.checkData(0, 0, 3999) - tdSql.query("select count(*) from tb0_0 where ts>\"2021-07-01 00:01:00.000000000\" and ts <=\"2021-07-01 00:01:10.000000000\" ;") - tdSql.checkData(0, 0, 1000) + tdSql.query("select count(*) from tb0_0 where ts>\"2021-07-01 00:00:00.590000000 \" ;") + tdSql.checkData(0, 0, 40) + tdSql.query("select count(*) from tb0_0 where ts>\"2021-07-01 00:00:00.000000000\" and ts <=\"2021-07-01 00:00:00.590000000\" ") + tdSql.checkData(0, 0, 59) # select max min avg from special col @@ -87,8 +87,8 @@ class TDTestCase: print( "select avg(c1) from tb0_0 : " , tdSql.getData(0, 0)) tdSql.query("select count(*) from stb0 group by tbname;") - tdSql.checkData(0, 0, 10000) - tdSql.checkData(100, 0, 10000) + tdSql.checkData(0, 0, 100) + tdSql.checkData(10, 0, 100) # query : query above sqls by taosdemo and continuously @@ -105,7 +105,7 @@ class TDTestCase: tdSql.checkDataType(3, 1, "TIMESTAMP") tdSql.query("select count(*) from stb0 where ts >\"2021-07-01 00:00:00.490000000\"") tdSql.checkData(0, 0, 5000) - tdSql.query("select count(*) from stb0 where ts >now -22d-1h-3s") + tdSql.query("select count(*) from stb0 where ts now -22d-1h-3s ;", + "sql": "select count(*) from stb0 where ts < now -22d-1h-3s ;", "result": "./query_res1.txt" }, { diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoSubscribe.json b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoSubscribe.json index 26d405b65b..1cc834164e 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoSubscribe.json +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoSubscribe.json @@ -20,7 +20,7 @@ "result": "./subscribe_res0.txt" }, { - "sql": "select * from stb0 where ts > now -20d-1h-3s ;", + "sql": "select * from stb0 where ts < now -2d-1h-3s ;", "result": "./subscribe_res1.txt" }, { diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanosubscribe.py b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanosubscribe.py index f6324577c1..6dcea6e7e0 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanosubscribe.py +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanosubscribe.py @@ -79,7 +79,7 @@ class TDTestCase: # merge result files - sleep(20) + sleep(10) os.system("cat subscribe_res0.txt* > all_subscribe_res0.txt") os.system("cat subscribe_res1.txt* > all_subscribe_res1.txt") os.system("cat subscribe_res2.txt* > all_subscribe_res2.txt") @@ -90,7 +90,7 @@ class TDTestCase: self.assertCheck("all_subscribe_res0.txt",subTimes0 ,200) subTimes1 = self.subTimes("all_subscribe_res1.txt") - self.assertCheck("all_subscribe_res1.txt",subTimes1 ,0) + self.assertCheck("all_subscribe_res1.txt",subTimes1 ,200) subTimes2 = self.subTimes("all_subscribe_res2.txt") self.assertCheck("all_subscribe_res2.txt",subTimes2 ,200) @@ -103,6 +103,7 @@ class TDTestCase: os.system("cat subscribe_res0.txt* > all_subscribe_res0.txt") subTimes0 = self.subTimes("all_subscribe_res0.txt") + print("pass") self.assertCheck("all_subscribe_res0.txt",subTimes0 ,202) @@ -112,6 +113,7 @@ class TDTestCase: sleep(3) os.system("rm -rf ./subscribe_res*") os.system("rm -rf ./all_subscribe*") + os.system("rm -rf ./*.py.sql") From 8aa0a37392c978a1f93448d839bc732e93c3d5c1 Mon Sep 17 00:00:00 2001 From: tomchon Date: Tue, 27 Jul 2021 11:40:47 +0800 Subject: [PATCH 044/106] [TD-4432]: add taodemo-testcase that using stmt interface --- .../taosdemoTestInsertWithJsonStmt.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJsonStmt.py b/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJsonStmt.py index cce6c83a07..0aade43183 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJsonStmt.py +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJsonStmt.py @@ -244,17 +244,17 @@ class TDTestCase: tdSql.query("select count(*) from stb1") tdSql.checkData(0, 0, 10) - # # insert: sample json - # os.system("%staosdemo -f tools/taosdemoAllTest/stmt/insert-sample-stmt.json -y " % binPath) - # tdSql.execute("use dbtest123") - # tdSql.query("select c2 from stb0") - # tdSql.checkData(0, 0, 2147483647) - # tdSql.query("select * from stb1 where t1=-127") - # tdSql.checkRows(20) - # tdSql.query("select * from stb1 where t2=127") - # tdSql.checkRows(10) - # tdSql.query("select * from stb1 where t2=126") - # tdSql.checkRows(10) + # insert: sample json + os.system("%staosdemo -f tools/taosdemoAllTest/stmt/insert-sample-stmt.json -y " % binPath) + tdSql.execute("use dbtest123") + tdSql.query("select c2 from stb0") + tdSql.checkData(0, 0, 2147483647) + tdSql.query("select * from stb1 where t1=-127") + tdSql.checkRows(20) + tdSql.query("select * from stb1 where t2=127") + tdSql.checkRows(10) + tdSql.query("select * from stb1 where t2=126") + tdSql.checkRows(10) # insert: test interlace parament os.system("%staosdemo -f tools/taosdemoAllTest/stmt/insert-interlace-row-stmt.json -y " % binPath) From 15b616c32e141b54fece444698358934f0af0b11 Mon Sep 17 00:00:00 2001 From: Zhiyu Yang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Tue, 27 Jul 2021 13:19:47 +0800 Subject: [PATCH 045/106] Fix/td 5506 (#7022) * slove conflict * [TD-5506]: throw SQLException in HttpClientPoolUtil --- cmake/install.inc | 2 +- src/connector/jdbc/CMakeLists.txt | 2 +- src/connector/jdbc/deploy-pom.xml | 2 +- src/connector/jdbc/pom.xml | 2 +- .../java/com/taosdata/jdbc/TSDBConstants.java | 4 +- .../java/com/taosdata/jdbc/TSDBError.java | 4 +- .../com/taosdata/jdbc/TSDBErrorNumbers.java | 13 +- .../taosdata/jdbc/rs/RestfulResultSet.java | 2 +- .../jdbc/utils/HttpClientPoolUtil.java | 202 ++++++++---------- 9 files changed, 106 insertions(+), 127 deletions(-) diff --git a/cmake/install.inc b/cmake/install.inc index b37cf751fb..e9ad240a79 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS) #INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS shell RUNTIME DESTINATION .) IF (TD_MVN_INSTALLED) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.31.jar DESTINATION connector/jdbc) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.34-dist.jar DESTINATION connector/jdbc) ENDIF () ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") diff --git a/src/connector/jdbc/CMakeLists.txt b/src/connector/jdbc/CMakeLists.txt index 8e30481575..2f211e9fba 100644 --- a/src/connector/jdbc/CMakeLists.txt +++ b/src/connector/jdbc/CMakeLists.txt @@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED) ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME} POST_BUILD COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.33.jar ${LIBRARY_OUTPUT_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.34-dist.jar ${LIBRARY_OUTPUT_PATH} COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMENT "build jdbc driver") ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME}) diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index 86f498b24a..ef57198e78 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -5,7 +5,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.33 + 2.0.34 jar JDBCDriver diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index ac2f99b0c5..5a6d007f9a 100644 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.taosdata.jdbc taos-jdbcdriver - 2.0.33 + 2.0.34 jar JDBCDriver https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java index f3f04eff12..740e3c6c21 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java @@ -130,7 +130,7 @@ public abstract class TSDBConstants { case TSDBConstants.TSDB_DATA_TYPE_NCHAR: return Types.NCHAR; } - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN_TAOS_TYPE_IN_TDENGINE); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN_TAOS_TYPE); } public static String taosType2JdbcTypeName(int taosType) throws SQLException { @@ -160,7 +160,7 @@ public abstract class TSDBConstants { case TSDBConstants.TSDB_DATA_TYPE_NCHAR: return "NCHAR"; default: - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN_TAOS_TYPE_IN_TDENGINE); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN_TAOS_TYPE); } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java index da89081428..d626698663 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java @@ -31,8 +31,8 @@ public class TSDBError { TSDBErrorMap.put(TSDBErrorNumbers.ERROR_URL_NOT_SET, "url is not set"); TSDBErrorMap.put(TSDBErrorNumbers.ERROR_INVALID_SQL, "invalid sql"); TSDBErrorMap.put(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE, "numeric value out of range"); - TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNKNOWN_TAOS_TYPE_IN_TDENGINE, "unknown taos type in tdengine"); - TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNKNOWN_TIMESTAMP_PERCISION, "unknown timestamp precision"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNKNOWN_TAOS_TYPE, "unknown taos type in tdengine"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNKNOWN_TIMESTAMP_PRECISION, "unknown timestamp precision"); TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNKNOWN, "unknown error"); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java index a796e6d86f..3c44d69be5 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java @@ -25,8 +25,10 @@ public class TSDBErrorNumbers { public static final int ERROR_URL_NOT_SET = 0x2312; // url is not set public static final int ERROR_INVALID_SQL = 0x2313; // invalid sql public static final int ERROR_NUMERIC_VALUE_OUT_OF_RANGE = 0x2314; // numeric value out of range - public static final int ERROR_UNKNOWN_TAOS_TYPE_IN_TDENGINE = 0x2315; //unknown taos type in tdengine - public static final int ERROR_UNKNOWN_TIMESTAMP_PERCISION = 0x2316; // unknown timestamp precision + public static final int ERROR_UNKNOWN_TAOS_TYPE = 0x2315; //unknown taos type in tdengine + public static final int ERROR_UNKNOWN_TIMESTAMP_PRECISION = 0x2316; // unknown timestamp precision + public static final int ERROR_RESTFul_Client_Protocol_Exception = 0x2317; + public static final int ERROR_RESTFul_Client_IOException = 0x2318; public static final int ERROR_UNKNOWN = 0x2350; //unknown error @@ -62,8 +64,11 @@ public class TSDBErrorNumbers { errorNumbers.add(ERROR_URL_NOT_SET); errorNumbers.add(ERROR_INVALID_SQL); errorNumbers.add(ERROR_NUMERIC_VALUE_OUT_OF_RANGE); - errorNumbers.add(ERROR_UNKNOWN_TAOS_TYPE_IN_TDENGINE); - errorNumbers.add(ERROR_UNKNOWN_TIMESTAMP_PERCISION); + errorNumbers.add(ERROR_UNKNOWN_TAOS_TYPE); + errorNumbers.add(ERROR_UNKNOWN_TIMESTAMP_PRECISION); + errorNumbers.add(ERROR_RESTFul_Client_IOException); + + errorNumbers.add(ERROR_RESTFul_Client_Protocol_Exception); errorNumbers.add(ERROR_SUBSCRIBE_FAILED); errorNumbers.add(ERROR_UNSUPPORTED_ENCODING); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java index f0ea03638f..1ea39236b6 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java @@ -213,7 +213,7 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { long nanoAdjustment = Integer.parseInt(value.substring(20)); return Timestamp.from(Instant.ofEpochSecond(epochSec, nanoAdjustment)); } - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN_TIMESTAMP_PERCISION); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN_TIMESTAMP_PRECISION); } } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java index e10bdb5aa9..de26ab7f1f 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java @@ -1,15 +1,18 @@ package com.taosdata.jdbc.utils; +import com.taosdata.jdbc.TSDBError; +import com.taosdata.jdbc.TSDBErrorNumbers; import org.apache.http.HeaderElement; import org.apache.http.HeaderElementIterator; import org.apache.http.HttpEntity; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.HttpRequestRetryHandler; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.*; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.conn.ConnectionKeepAliveStrategy; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.message.BasicHeaderElementIterator; @@ -17,35 +20,24 @@ import org.apache.http.protocol.HTTP; import org.apache.http.protocol.HttpContext; import org.apache.http.util.EntityUtils; +import javax.net.ssl.SSLException; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; - +import java.sql.SQLException; public class HttpClientPoolUtil { private static final String DEFAULT_CONTENT_TYPE = "application/json"; + private static final int DEFAULT_MAX_TOTAL = 200; + private static final int DEFAULT_MAX_PER_ROUTE = 20; private static final int DEFAULT_TIME_OUT = 15000; - private static final int DEFAULT_MAX_PER_ROUTE = 32; - private static final int DEFAULT_MAX_TOTAL = 1000; private static final int DEFAULT_HTTP_KEEP_TIME = 15000; - - private static CloseableHttpClient httpClient; - - private static synchronized void initPools() { - if (httpClient == null) { - PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(); - connectionManager.setDefaultMaxPerRoute(DEFAULT_MAX_PER_ROUTE); - connectionManager.setMaxTotal(DEFAULT_MAX_TOTAL); - httpClient = HttpClients.custom() - .setKeepAliveStrategy(DEFAULT_KEEP_ALIVE_STRATEGY) - .setConnectionManager(connectionManager) - .setRetryHandler(new DefaultHttpRequestRetryHandler(3, true)) - .build(); - } - } + private static final int DEFAULT_MAX_RETRY_COUNT = 5; private static final ConnectionKeepAliveStrategy DEFAULT_KEEP_ALIVE_STRATEGY = (response, context) -> { HeaderElementIterator it = new BasicHeaderElementIterator(response.headerIterator(HTTP.CONN_KEEP_ALIVE)); - int keepTime = DEFAULT_HTTP_KEEP_TIME * 1000; while (it.hasNext()) { HeaderElement headerElement = it.nextElement(); String param = headerElement.getName(); @@ -53,34 +45,73 @@ public class HttpClientPoolUtil { if (value != null && param.equalsIgnoreCase("timeout")) { try { return Long.parseLong(value) * 1000; - } catch (Exception e) { - new Exception("format KeepAlive timeout exception, exception:" + e.toString()).printStackTrace(); + } catch (NumberFormatException ignore) { } } } - return keepTime; + return DEFAULT_HTTP_KEEP_TIME * 1000; }; - /** - * 执行http post请求 - * é»č®¤é‡‡ç”¨Content-Type：application/json，Accept：application/json - * - * @param uri 请求地址 - * @param data 请求数据 - * @return responseBody - */ - public static String execute(String uri, String data, String token) { - long startTime = System.currentTimeMillis(); + private static final HttpRequestRetryHandler retryHandler = (exception, executionCount, httpContext) -> { + if (executionCount >= DEFAULT_MAX_RETRY_COUNT) + // do not retry if over max retry count + return false; + if (exception instanceof InterruptedIOException) + // timeout + return false; + if (exception instanceof UnknownHostException) + // unknown host + return false; + if (exception instanceof SSLException) + // SSL handshake exception + return false; + return true; + }; + + private static CloseableHttpClient httpClient; + + static { + PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(); + connectionManager.setMaxTotal(DEFAULT_MAX_TOTAL); + connectionManager.setDefaultMaxPerRoute(DEFAULT_MAX_PER_ROUTE); + httpClient = HttpClients.custom().setKeepAliveStrategy(DEFAULT_KEEP_ALIVE_STRATEGY).setConnectionManager(connectionManager).setRetryHandler(retryHandler).build(); + } + + /*** execute GET request ***/ + public static String execute(String uri) throws SQLException { HttpEntity httpEntity = null; - HttpEntityEnclosingRequestBase method = null; String responseBody = ""; try { - if (httpClient == null) { - initPools(); + HttpRequestBase method = getRequest(uri, HttpGet.METHOD_NAME); + HttpContext context = HttpClientContext.create(); + CloseableHttpResponse httpResponse = httpClient.execute(method, context); + httpEntity = httpResponse.getEntity(); + if (httpEntity != null) { + responseBody = EntityUtils.toString(httpEntity, StandardCharsets.UTF_8); } - method = (HttpEntityEnclosingRequestBase) getRequest(uri, HttpPost.METHOD_NAME, DEFAULT_CONTENT_TYPE, 0); - method.setHeader("Content-Type", "text/plain"); - method.setHeader("Connection", "keep-alive"); + } catch (ClientProtocolException e) { + e.printStackTrace(); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESTFul_Client_Protocol_Exception, e.getMessage()); + } catch (IOException exception) { + exception.printStackTrace(); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESTFul_Client_IOException, exception.getMessage()); + } finally { + if (httpEntity != null) { + EntityUtils.consumeQuietly(httpEntity); + } + } + return responseBody; + } + + + /*** execute POST request ***/ + public static String execute(String uri, String data, String token) throws SQLException { + HttpEntity httpEntity = null; + String responseBody = ""; + try { + HttpEntityEnclosingRequestBase method = (HttpEntityEnclosingRequestBase) getRequest(uri, HttpPost.METHOD_NAME); + method.setHeader(HTTP.CONTENT_TYPE, "text/plain"); + method.setHeader(HTTP.CONN_DIRECTIVE, HTTP.CONN_KEEP_ALIVE); method.setHeader("Authorization", "Taosd " + token); method.setEntity(new StringEntity(data, StandardCharsets.UTF_8)); @@ -88,46 +119,31 @@ public class HttpClientPoolUtil { CloseableHttpResponse httpResponse = httpClient.execute(method, context); httpEntity = httpResponse.getEntity(); if (httpEntity != null) { - responseBody = EntityUtils.toString(httpEntity, "UTF-8"); + responseBody = EntityUtils.toString(httpEntity, StandardCharsets.UTF_8); } - } catch (Exception e) { - if (method != null) { - method.abort(); - } - new Exception("execute post request exception, url:" + uri + ", exception:" + e.toString() + ", cost time(ms):" + (System.currentTimeMillis() - startTime)).printStackTrace(); + } catch (ClientProtocolException e) { + e.printStackTrace(); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESTFul_Client_Protocol_Exception, e.getMessage()); + } catch (IOException exception) { + exception.printStackTrace(); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESTFul_Client_IOException, exception.getMessage()); } finally { if (httpEntity != null) { - try { - EntityUtils.consumeQuietly(httpEntity); - } catch (Exception e) { - new Exception("close response exception, url:" + uri + ", exception:" + e.toString() + ", cost time(ms):" + (System.currentTimeMillis() - startTime)).printStackTrace(); - } + EntityUtils.consumeQuietly(httpEntity); } } return responseBody; } - /** - * * ĺ›ĺ»şčŻ·ć±‚ - * - * @param uri 请求url - * @param methodName 请求的方法类型 - * @param contentType contentType类型 - * @param timeout 超时时间 - * @return HttpRequestBase 返回类型 - * @author lisc - */ - private static HttpRequestBase getRequest(String uri, String methodName, String contentType, int timeout) { - if (httpClient == null) { - initPools(); - } + /*** create http request ***/ + private static HttpRequestBase getRequest(String uri, String methodName) { HttpRequestBase method; - if (timeout <= 0) { - timeout = DEFAULT_TIME_OUT; - } - RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(timeout * 1000) - .setConnectTimeout(timeout * 1000).setConnectionRequestTimeout(timeout * 1000) - .setExpectContinueEnabled(false).build(); + RequestConfig requestConfig = RequestConfig.custom() + .setSocketTimeout(DEFAULT_TIME_OUT * 1000) + .setConnectTimeout(DEFAULT_TIME_OUT * 1000) + .setConnectionRequestTimeout(DEFAULT_TIME_OUT * 1000) + .setExpectContinueEnabled(false) + .build(); if (HttpPut.METHOD_NAME.equalsIgnoreCase(methodName)) { method = new HttpPut(uri); } else if (HttpPost.METHOD_NAME.equalsIgnoreCase(methodName)) { @@ -137,52 +153,10 @@ public class HttpClientPoolUtil { } else { method = new HttpPost(uri); } - - if (contentType == null || contentType.isEmpty() || contentType.replaceAll("\\s", "").isEmpty()) { - contentType = DEFAULT_CONTENT_TYPE; - } - method.addHeader("Content-Type", contentType); - method.addHeader("Accept", contentType); + method.addHeader(HTTP.CONTENT_TYPE, DEFAULT_CONTENT_TYPE); + method.addHeader("Accept", DEFAULT_CONTENT_TYPE); method.setConfig(requestConfig); return method; } - /** - * 执行GET 请求 - * - * @param uri 网址 - * @return responseBody - */ - public static String execute(String uri) { - long startTime = System.currentTimeMillis(); - HttpEntity httpEntity = null; - HttpRequestBase method = null; - String responseBody = ""; - try { - if (httpClient == null) { - initPools(); - } - method = getRequest(uri, HttpGet.METHOD_NAME, DEFAULT_CONTENT_TYPE, 0); - HttpContext context = HttpClientContext.create(); - CloseableHttpResponse httpResponse = httpClient.execute(method, context); - httpEntity = httpResponse.getEntity(); - if (httpEntity != null) { - responseBody = EntityUtils.toString(httpEntity, "UTF-8"); - } - } catch (Exception e) { - if (method != null) { - method.abort(); - } - e.printStackTrace(); - } finally { - if (httpEntity != null) { - try { - EntityUtils.consumeQuietly(httpEntity); - } catch (Exception e) { - new Exception("close response exception, url:" + uri + ", exception:" + e.toString() + ",cost time(ms):" + (System.currentTimeMillis() - startTime)).printStackTrace(); - } - } - } - return responseBody; - } } \ No newline at end of file From eaed6da8cede10b8960d69791950a9731735ab78 Mon Sep 17 00:00:00 2001 From: markswang <792637585@qq.com> Date: Tue, 27 Jul 2021 13:57:22 +0800 Subject: [PATCH 046/106] [TD-5534]:fix the coverity high risk of client --- src/client/src/tscGlobalmerge.c | 7 ++++++- src/client/src/tscSql.c | 10 +++++++++- src/client/src/tscSubquery.c | 3 ++- src/client/src/tscUtil.c | 8 ++++++-- src/query/inc/qExecutor.h | 1 + src/query/src/qExecutor.c | 23 ++++++++++++----------- src/query/src/qPlan.c | 4 ++++ 7 files changed, 40 insertions(+), 16 deletions(-) diff --git a/src/client/src/tscGlobalmerge.c b/src/client/src/tscGlobalmerge.c index 8ca8c688f0..9c9bac5951 100644 --- a/src/client/src/tscGlobalmerge.c +++ b/src/client/src/tscGlobalmerge.c @@ -135,7 +135,7 @@ int32_t tscCreateGlobalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, t SLocalDataSource *ds = (SLocalDataSource *)malloc(sizeof(SLocalDataSource) + pMemBuffer[0]->pageSize); if (ds == NULL) { tscError("0x%"PRIx64" failed to create merge structure", id); - tfree(pMerger); + tfree(*pMerger); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -443,6 +443,10 @@ int32_t tscCreateGlobalMergerEnv(SQueryInfo *pQueryInfo, tExtMemBuffer ***pMemBu } pModel = createColumnModel(pSchema, (int32_t)size, capacity); + if (pModel == NULL){ + tfree(pSchema); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } tfree(pSchema); int32_t pg = DEFAULT_PAGE_SIZE; @@ -458,6 +462,7 @@ int32_t tscCreateGlobalMergerEnv(SQueryInfo *pQueryInfo, tExtMemBuffer ***pMemBu } if (createOrderDescriptor(pOrderDesc, pQueryInfo, pModel) != TSDB_CODE_SUCCESS) { + tfree(pModel); return TSDB_CODE_TSC_OUT_OF_MEMORY; } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 7675bded65..b6e2604bd2 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -963,8 +963,14 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { strtolower(str, tableNameList); SArray* plist = taosArrayInit(4, POINTER_BYTES); + if (plist == NULL) { + tfree(str); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + SArray* vgroupList = taosArrayInit(4, POINTER_BYTES); - if (plist == NULL || vgroupList == NULL) { + if (vgroupList == NULL) { + taosArrayDestroy(plist); tfree(str); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -980,6 +986,8 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { if (code != TSDB_CODE_SUCCESS) { tscFreeSqlObj(pSql); + taosArrayDestroyEx(plist); + taosArrayDestroyEx(vgroupList); return code; } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index b72bd78b1b..73794a3aa4 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2476,8 +2476,9 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { pState->states = calloc(pState->numOfSub, sizeof(*pState->states)); if (pState->states == NULL) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + tscDestroyGlobalMergerEnv(pMemoryBuf, pDesc,pState->numOfSub); + tscAsyncResultOnError(pSql); - tfree(pMemoryBuf); return ret; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 0d69fe173f..08950b93e3 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1246,6 +1246,10 @@ void handleDownstreamOperator(SSqlObj** pSqlObjList, int32_t numOfUpstream, SQue } pSourceOperator = createJoinOperatorInfo(p, px->numOfTables, schema, num); + + for(int32_t i = 0; i < px->numOfTables; ++i) { + destroyOperatorInfo(p[i]); + } tfree(p); } else { size_t num = taosArrayGetSize(px->colList); @@ -4368,7 +4372,7 @@ int32_t tscCreateTableMetaFromSTableMeta(STableMeta* pChild, const char* name, v pChild->sversion = p->sversion; pChild->tversion = p->tversion; - memcpy(&pChild->tableInfo, &p->tableInfo, sizeof(STableInfo)); + memcpy(&pChild->tableInfo, &p->tableInfo, sizeof(STableComInfo)); int32_t total = pChild->tableInfo.numOfColumns + pChild->tableInfo.numOfTags; memcpy(pChild->schema, p->schema, sizeof(SSchema) *total); @@ -4719,7 +4723,7 @@ static int32_t doAddTableName(char* nextStr, char** str, SArray* pNameArray, SSq int32_t len = 0; if (nextStr == NULL) { - strncpy(tablename, *str, TSDB_TABLE_FNAME_LEN); + strncpy(tablename, *str, TSDB_TABLE_FNAME_LEN - 1); len = (int32_t) strlen(tablename); } else { len = (int32_t)(nextStr - (*str)); diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 4581ba258d..71e9792bd8 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -578,6 +578,7 @@ void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p); SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows); void* destroyOutputBuf(SSDataBlock* pBlock); void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols); +void destroyOperatorInfo(SOperatorInfo* pOperator); void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); int32_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 982c45c441..7e3e542f5e 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -191,8 +191,6 @@ static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput); static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyAggOperatorInfo(void* param, int32_t numOfOutput); -static void destroyOperatorInfo(SOperatorInfo* pOperator); - static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock); @@ -3563,6 +3561,7 @@ STableQueryInfo* createTmpTableQueryInfo(STimeWindow win) { int32_t initialSize = 16; int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize, TSDB_DATA_TYPE_INT); if (code != TSDB_CODE_SUCCESS) { + tfree(pTableQueryInfo); return NULL; } @@ -5945,7 +5944,7 @@ static int32_t getNumOfScanTimes(SQueryAttr* pQueryAttr) { return 1; } -static void destroyOperatorInfo(SOperatorInfo* pOperator) { +void destroyOperatorInfo(SOperatorInfo* pOperator) { if (pOperator == NULL) { return; } @@ -7241,9 +7240,7 @@ void destroyUdfInfo(SUdfInfo* pUdfInfo) { tfree(pUdfInfo); } -static char* getUdfFuncName(char* name, int type) { - char* funcname = calloc(1, TSDB_FUNCTIONS_NAME_MAX_LENGTH + 10); - +static char* getUdfFuncName(char* funcname, char* name, int type) { switch (type) { case TSDB_UDF_FUNC_NORMAL: strcpy(funcname, name); @@ -7314,19 +7311,20 @@ int32_t initUdfInfo(SUdfInfo* pUdfInfo) { return TSDB_CODE_QRY_SYS_ERROR; } - pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(pUdfInfo->name, TSDB_UDF_FUNC_NORMAL)); + char funcname[TSDB_FUNCTIONS_NAME_MAX_LENGTH + 10] = {0}; + pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_NORMAL)); if (NULL == pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL]) { return TSDB_CODE_QRY_SYS_ERROR; } - pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(pUdfInfo->name, TSDB_UDF_FUNC_INIT)); + pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_INIT)); if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { - pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(pUdfInfo->name, TSDB_UDF_FUNC_FINALIZE)); - pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(pUdfInfo->name, TSDB_UDF_FUNC_MERGE)); + pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_FINALIZE)); + pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_MERGE)); } - pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(pUdfInfo->name, TSDB_UDF_FUNC_DESTROY)); + pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_DESTROY)); if (pUdfInfo->funcs[TSDB_UDF_FUNC_INIT]) { return (*(udfInitFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_INIT])(&pUdfInfo->init); @@ -7398,10 +7396,12 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp int32_t j = getColumnIndexInSource(pTableInfo, &pExprs[i].base, pTagCols); if (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag)) { if (j < TSDB_TBNAME_COLUMN_INDEX || j >= pTableInfo->numOfTags) { + tfree(pExprs); return TSDB_CODE_QRY_INVALID_MSG; } } else { if (j < PRIMARYKEY_TIMESTAMP_COL_INDEX || j >= pTableInfo->numOfCols) { + tfree(pExprs); return TSDB_CODE_QRY_INVALID_MSG; } } @@ -7421,6 +7421,7 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp int32_t ret = cloneExprFilterInfo(&pExprs[i].base.flist.filterInfo, pExprMsg[i]->flist.filterInfo, pExprMsg[i]->flist.numOfFilters); if (ret) { + tfree(pExprs); return ret; } } diff --git a/src/query/src/qPlan.c b/src/query/src/qPlan.c index e724b0418c..2772e76abe 100644 --- a/src/query/src/qPlan.c +++ b/src/query/src/qPlan.c @@ -222,6 +222,7 @@ SArray* createQueryPlanImpl(SQueryInfo* pQueryInfo) { if (pQueryInfo->numOfTables > 1) { // it is a join query // 1. separate the select clause according to table + taosArrayDestroy(upstream); upstream = taosArrayInit(5, POINTER_BYTES); for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { @@ -231,6 +232,7 @@ SArray* createQueryPlanImpl(SQueryInfo* pQueryInfo) { SArray* exprList = taosArrayInit(4, POINTER_BYTES); if (tscExprCopy(exprList, pQueryInfo->exprList, uid, true) != 0) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + tscExprDestroy(exprList); exit(-1); } @@ -245,6 +247,8 @@ SArray* createQueryPlanImpl(SQueryInfo* pQueryInfo) { // 4. add the projection query node SQueryNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, exprList, tableColumnList); + tscColumnListDestroy(tableColumnList); + tscExprDestroy(exprList); taosArrayPush(upstream, &pNode); } From 2d6e305118383ecbd8f99cb1ca3dbfe8e3f8ffff Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 27 Jul 2021 14:03:50 +0800 Subject: [PATCH 047/106] add prompt msg if WAL size exceeds limit. (#7033) --- src/kit/taosdemo/taosdemo.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 64adc814e2..5164d1bcc3 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -647,7 +647,7 @@ static FILE * g_fpOfInsertResult = NULL; fprintf(stderr, "PERF: "fmt, __VA_ARGS__); } while(0) #define errorPrint(fmt, ...) \ - do { fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); } while(0) + do { fprintf(stderr, " \033[31m"); fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); fprintf(stderr, " \033[0m"); } while(0) // for strncpy buffer overflow #define min(a, b) (((a) < (b)) ? (a) : (b)) @@ -5136,6 +5136,8 @@ static int32_t execInsert(threadInfo *pThreadInfo, uint32_t k) if (0 != taos_stmt_execute(pThreadInfo->stmt)) { errorPrint("%s() LN%d, failied to execute insert statement. reason: %s\n", __func__, __LINE__, taos_stmt_errstr(pThreadInfo->stmt)); + + fprintf(stderr, "\n\033[31m === Please reduce batch number if WAL size exceeds limit. ===\033[0m\n\n"); exit(-1); } affectedRows = k; From b993a059601cbfde5962d1790e4482cd104422e3 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 27 Jul 2021 14:49:38 +0800 Subject: [PATCH 048/106] [TD-5543]: taosdump precision description and display. (#7031) Co-authored-by: Shuduo Sang --- src/kit/taosdump/taosdump.c | 60 +++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 858f3c06aa..165268e231 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -60,7 +60,7 @@ typedef struct { fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0) #define errorPrint(fmt, ...) \ - do { fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); } while(0) + do { fprintf(stderr, "\033[31m"); fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); fprintf(stderr, "\033[0m"); } while(0) // -------------------------- SHOW DATABASE INTERFACE----------------------- @@ -234,9 +234,9 @@ static struct argp_option options[] = { {"start-time", 'S', "START_TIME", 0, "Start time to dump. Either epoch or ISO8601/RFC3339 format is acceptable. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 4}, {"end-time", 'E', "END_TIME", 0, "End time to dump. Either epoch or ISO8601/RFC3339 format is acceptable. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 5}, #if TSDB_SUPPORT_NANOSECOND == 1 - {"precision", 'C', "PRECISION", 0, "Epoch precision. Valid value is one of ms, us, and ns. Default is ms.", 6}, + {"precision", 'C', "PRECISION", 0, "Specify precision for converting human-readable time to epoch. Valid value is one of ms, us, and ns. Default is ms.", 6}, #else - {"precision", 'C', "PRECISION", 0, "Epoch precision. Valid value is one of ms and us. Default is ms.", 6}, + {"precision", 'C', "PRECISION", 0, "Use specified precision to convert human-readable time. Valid value is one of ms and us. Default is ms.", 6}, #endif {"data-batch", 'B', "DATA_BATCH", 0, "Number of data point per insert statement. Max value is 32766. Default is 1.", 3}, {"max-sql-len", 'L', "SQL_LEN", 0, "Max length of one sql. Default is 65480.", 3}, @@ -547,8 +547,8 @@ static void parse_precision_first( free(tmp); exit(-1); } - strncpy(g_args.precision, tmp, - min(DB_PRECISION_LEN - 1, strlen(tmp))); + tstrncpy(g_args.precision, tmp, + min(DB_PRECISION_LEN, strlen(tmp) + 1)); free(tmp); } } @@ -599,6 +599,7 @@ static void parse_timestamp( return; } } else { + tstrncpy(arguments->precision, "n/a", strlen("n/a") + 1); tmpEpoch = atoll(tmp); } @@ -794,12 +795,14 @@ static int taosGetTableRecordInfo( while ((row = taos_fetch_row(result)) != NULL) { isSet = true; pTableRecordInfo->isMetric = false; - strncpy(pTableRecordInfo->tableRecord.name, + tstrncpy(pTableRecordInfo->tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], - fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); - strncpy(pTableRecordInfo->tableRecord.metric, + min(TSDB_TABLE_NAME_LEN, + fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes) + 1); + tstrncpy(pTableRecordInfo->tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], - fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes); + min(TSDB_TABLE_NAME_LEN, + fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes) + 1); break; } @@ -1080,8 +1083,8 @@ _dump_db_point: goto _exit_failure; } - strncpy(g_dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], - fields[TSDB_SHOW_DB_NAME_INDEX].bytes); + tstrncpy(g_dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], + min(TSDB_DB_NAME_LEN, fields[TSDB_SHOW_DB_NAME_INDEX].bytes) + 1); if (g_args.with_property) { g_dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]); g_dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]); @@ -1089,8 +1092,8 @@ _dump_db_point: g_dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]); g_dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]); - strncpy(g_dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX], - fields[TSDB_SHOW_DB_KEEP_INDEX].bytes); + tstrncpy(g_dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX], + min(32, fields[TSDB_SHOW_DB_KEEP_INDEX].bytes) + 1); //g_dbInfos[count]->daysToKeep = *((int16_t *)row[TSDB_SHOW_DB_KEEP_INDEX]); //g_dbInfos[count]->daysToKeep1; //g_dbInfos[count]->daysToKeep2; @@ -1103,8 +1106,8 @@ _dump_db_point: g_dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX])); g_dbInfos[count]->cachelast = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX])); - strncpy(g_dbInfos[count]->precision, (char *)row[TSDB_SHOW_DB_PRECISION_INDEX], - fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes); + tstrncpy(g_dbInfos[count]->precision, (char *)row[TSDB_SHOW_DB_PRECISION_INDEX], + min(8, fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes) + 1); //g_dbInfos[count]->precision = *((int8_t *)row[TSDB_SHOW_DB_PRECISION_INDEX]); g_dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]); } @@ -1255,17 +1258,19 @@ static int taosGetTableDes( tstrncpy(tableDes->name, table, TSDB_TABLE_NAME_LEN); while ((row = taos_fetch_row(res)) != NULL) { - strncpy(tableDes->cols[count].field, + tstrncpy(tableDes->cols[count].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], - fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); - strncpy(tableDes->cols[count].type, + min(TSDB_COL_NAME_LEN + 1, + fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes + 1)); + tstrncpy(tableDes->cols[count].type, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - min(15, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes)); + min(16, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes + 1)); tableDes->cols[count].length = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); - strncpy(tableDes->cols[count].note, + tstrncpy(tableDes->cols[count].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], - fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); + min(COL_NOTE_LEN, + fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes + 1)); count++; } @@ -1698,8 +1703,9 @@ static int32_t taosDumpCreateSuperTableClause(TAOS* taosCon, char* dbName, FILE while ((row = taos_fetch_row(res)) != NULL) { memset(&tableRecord, 0, sizeof(STableRecord)); - strncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], - fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); + tstrncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], + min(TSDB_TABLE_NAME_LEN, + fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes) + 1); taosWrite(fd, &tableRecord, sizeof(STableRecord)); } @@ -1773,9 +1779,11 @@ static int taosDumpDb(SDbInfo *dbInfo, FILE *fp, TAOS *taosCon) { while ((row = taos_fetch_row(res)) != NULL) { memset(&tableRecord, 0, sizeof(STableRecord)); tstrncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], - fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); + min(TSDB_TABLE_NAME_LEN, + fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes) + 1); tstrncpy(tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], - min(TSDB_TABLE_NAME_LEN, fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes)); + min(TSDB_TABLE_NAME_LEN, + fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes) + 1); taosWrite(fd, &tableRecord, sizeof(STableRecord)); @@ -2163,7 +2171,7 @@ static int taosCheckParam(struct arguments *arguments) { if (g_args.arg_list_len == 0) { if ((!g_args.all_databases) && (!g_args.isDumpIn)) { - fprintf(stderr, "taosdump requires parameters\n"); + errorPrint("%s", "taosdump requires parameters for database and operation\n"); return -1; } } From 0d467bb0cd595569ceb8f19ba267460388893f84 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 27 Jul 2021 16:26:23 +0800 Subject: [PATCH 049/106] [TD-5558]: fix invalid read memory issue --- src/vnode/src/vnodeMgmt.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/vnode/src/vnodeMgmt.c b/src/vnode/src/vnodeMgmt.c index a30628680f..6c686d929e 100644 --- a/src/vnode/src/vnodeMgmt.c +++ b/src/vnode/src/vnodeMgmt.c @@ -110,7 +110,9 @@ void vnodeRelease(void *vparam) { if (vparam == NULL) return; int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1); - vTrace("vgId:%d, release vnode, refCount:%d pVnode:%p", pVnode->vgId, refCount, pVnode); + int32_t vgId = pVnode->vgId; + + vTrace("vgId:%d, release vnode, refCount:%d pVnode:%p", vgId, refCount, pVnode); assert(refCount >= 0); if (refCount > 0) { @@ -118,10 +120,10 @@ void vnodeRelease(void *vparam) { tsem_post(&pVnode->sem); } } else { - vDebug("vgId:%d, vnode will be destroyed, refCount:%d pVnode:%p", pVnode->vgId, refCount, pVnode); + vDebug("vgId:%d, vnode will be destroyed, refCount:%d pVnode:%p", vgId, refCount, pVnode); vnodeDestroyInMWorker(pVnode); int32_t count = taosHashGetSize(tsVnodesHash); - vDebug("vgId:%d, vnode is destroyed, vnodes:%d", pVnode->vgId, count); + vDebug("vgId:%d, vnode is destroyed, vnodes:%d", vgId, count); } } From 3e96e6452179255d88443f205a2c6e1dd352bea3 Mon Sep 17 00:00:00 2001 From: markswang <792637585@qq.com> Date: Tue, 27 Jul 2021 17:05:37 +0800 Subject: [PATCH 050/106] [TD-5534]:fix the coverity medium risk of client --- src/client/src/tscAsync.c | 3 --- src/client/src/tscLocal.c | 8 ++++++-- src/client/src/tscParseInsert.c | 2 +- src/client/src/tscParseLineProtocol.c | 4 ---- src/client/src/tscPrepare.c | 4 +++- src/client/src/tscSQLParser.c | 16 +++++++++------- src/client/src/tscServer.c | 9 +++++++-- src/client/src/tscStream.c | 2 +- src/client/src/tscSubquery.c | 3 +++ src/client/src/tscUtil.c | 17 ++++------------- src/query/inc/qTableMeta.h | 2 +- src/query/src/qExecutor.c | 11 ++++++++--- src/query/src/qPlan.c | 2 +- src/query/src/qResultbuf.c | 5 +++-- src/query/src/qScript.c | 8 +++++--- src/query/src/qSqlParser.c | 24 +++++++++++++++--------- src/query/src/qTableMeta.c | 2 +- src/query/src/queryMain.c | 2 +- src/util/inc/tlist.h | 2 +- 19 files changed, 70 insertions(+), 56 deletions(-) diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index d857d00e15..3c5036cc1a 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -493,9 +493,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { return; } - taosReleaseRef(tscObjRef, pSql->self); - return; - _error: pRes->code = code; tscAsyncResultOnError(pSql); diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index d1a325be35..e49f4e33ae 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -851,15 +851,19 @@ static int32_t tscProcessServStatus(SSqlObj *pSql) { SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid); if (pHb != NULL) { pSql->res.code = pHb->res.code; - taosReleaseRef(tscObjRef, pObj->hbrid); } if (pSql->res.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + taosReleaseRef(tscObjRef, pObj->hbrid); return pSql->res.code; } - pSql->res.code = checkForOnlineNode(pHb); + if (pHb != NULL) { + pSql->res.code = checkForOnlineNode(pHb); + } + if (pSql->res.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + taosReleaseRef(tscObjRef, pObj->hbrid); return pSql->res.code; } diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index f24f7a7ecb..8ecc58eb55 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -2105,7 +2105,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRow pParentSql->fp = pParentSql->fetchFp; // all data has been sent to vnode, call user function - int32_t v = (code != TSDB_CODE_SUCCESS) ? code : (int32_t)pParentSql->res.numOfRows; + int32_t v = (int32_t)pParentSql->res.numOfRows; (*pParentSql->fp)(pParentSql->param, pParentSql, v); return; } diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index f0e5017355..4b5416c6b9 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -438,10 +438,6 @@ int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema) { char fullTableName[TSDB_TABLE_FNAME_LEN] = {0}; memset(fullTableName, 0, tListLen(fullTableName)); tNameExtractFullName(&sname, fullTableName); - if (code != TSDB_CODE_SUCCESS) { - tscFreeSqlObj(pSql); - return code; - } tscFreeSqlObj(pSql); schema->tags = taosArrayInit(8, sizeof(SSchema)); diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index efdc24c899..9ff3a0e2dd 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -1773,7 +1773,9 @@ int taos_stmt_close(TAOS_STMT* stmt) { } tscDestroyDataBlock(pStmt->mtb.lastBlock, rmMeta); pStmt->mtb.pTableBlockHashList = tscDestroyBlockHashTable(pStmt->mtb.pTableBlockHashList, rmMeta); - taosHashCleanup(pStmt->pSql->cmd.insertParam.pTableBlockHashList); + if (pStmt->pSql){ + taosHashCleanup(pStmt->pSql->cmd.insertParam.pTableBlockHashList); + } pStmt->pSql->cmd.insertParam.pTableBlockHashList = NULL; taosArrayDestroy(pStmt->mtb.tags); tfree(pStmt->mtb.sqlstr); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 2d5a3d524e..1fac9c9a40 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -6860,7 +6860,8 @@ static int32_t doAddGroupbyColumnsOnDemand(SSqlCmd* pCmd, SQueryInfo* pQueryInfo tagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); } - SSchema* s = NULL; + SSchema tmp = {.type = 0, .name = "", .colId = 0, .bytes = 0}; + SSchema* s = &tmp; for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) { SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, i); @@ -6870,7 +6871,9 @@ static int32_t doAddGroupbyColumnsOnDemand(SSqlCmd* pCmd, SQueryInfo* pQueryInfo s = tGetTbnameColumnSchema(); } else { if (TSDB_COL_IS_TAG(pColIndex->flag)) { - s = &tagSchema[colIndex]; + if(tagSchema){ + s = &tagSchema[colIndex]; + } } else { s = &pSchema[colIndex]; } @@ -8276,17 +8279,16 @@ static int32_t doLoadAllTableMeta(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNod const char* name = tNameGetTableName(&pTableMetaInfo->name); STableMetaVgroupInfo* p = taosHashGet(pCmd->pTableMetaMap, name, strlen(name)); - + if(!p){ + tscError("taosHashGet meta null. name:%s", name); + return TSDB_CODE_TSC_APP_ERROR; + } pTableMetaInfo->pTableMeta = tscTableMetaDup(p->pTableMeta); assert(pTableMetaInfo->pTableMeta != NULL); if (p->pVgroupInfo != NULL) { pTableMetaInfo->vgroupList = tscVgroupsInfoDup(p->pVgroupInfo); } - - if (code != TSDB_CODE_SUCCESS) { - return code; - } } return TSDB_CODE_SUCCESS; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 401e65efd4..0748780e1a 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -691,7 +691,9 @@ static char *doSerializeTableInfo(SQueryTableMsg *pQueryMsg, SSqlObj *pSql, STab tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); } - pSql->epSet.inUse = rand()%pSql->epSet.numOfEps; + if (pSql->epSet.numOfEps > 0){ + pSql->epSet.inUse = rand()%pSql->epSet.numOfEps; + } pQueryMsg->head.vgId = htonl(vgId); STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; @@ -968,7 +970,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } } - if (query.numOfTags > 0) { + if (query.numOfTags > 0 && query.tagColList != NULL) { for (int32_t i = 0; i < query.numOfTags; ++i) { SColumnInfo* pTag = &query.tagColList[i]; @@ -2351,6 +2353,9 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { break; } + if (!pInfo){ + continue; + } int32_t size = 0; pInfo->vgroupList = createVgroupInfoFromMsg(pMsg, &size, pSql->self); pMsg += size; diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index da5bdf669f..a4d314cb42 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -271,7 +271,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf if (pSql == NULL || numOfRows < 0) { int64_t retryDelayTime = tscGetRetryDelayTime(pStream, pStream->interval.sliding, pStream->precision); - tscError("0x%"PRIx64" stream:%p, retrieve data failed, code:0x%08x, retry in %" PRId64 " ms", pSql->self, pStream, numOfRows, retryDelayTime); + tscError("stream:%p, retrieve data failed, code:0x%08x, retry in %" PRId64 " ms", pStream, numOfRows, retryDelayTime); tscSetRetryTimer(pStream, pStream->pSql, retryDelayTime); return; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 73794a3aa4..bef5992d53 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2723,6 +2723,9 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO } static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* pSql) { + if (trsupport->pExtMemBuffer == NULL){ + return; + } int32_t idx = trsupport->subqueryIndex; SSqlObj * pParentSql = trsupport->pParentSql; tOrderDescriptor *pDesc = trsupport->pOrderDescriptor; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 08950b93e3..7e36cce50e 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -275,16 +275,6 @@ bool tscIsProjectionQuery(SQueryInfo* pQueryInfo) { f != TSDB_FUNC_DERIVATIVE) { return false; } - - if (f < 0) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * f - 1); - if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { - return false; - } - - continue; - } - } return true; @@ -3420,7 +3410,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, SName* name, STableM return NULL; } - if (pTagCols != NULL) { + if (pTagCols != NULL && pTableMetaInfo->pTableMeta != NULL) { tscColumnListCopy(pTableMetaInfo->tagColList, pTagCols, pTableMetaInfo->pTableMeta->id.uid); } @@ -3851,7 +3841,8 @@ void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo) { SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); if (pNew == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - // return NULL; + tscError("pNew == NULL, out of memory"); + return; } pNew->pTscObj = pSql->pTscObj; @@ -4339,7 +4330,7 @@ uint32_t tscGetTableMetaSize(STableMeta* pTableMeta) { assert(pTableMeta != NULL); int32_t totalCols = 0; - if (pTableMeta->tableInfo.numOfColumns >= 0 && pTableMeta->tableInfo.numOfTags >= 0) { + if (pTableMeta->tableInfo.numOfColumns >= 0) { totalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; } diff --git a/src/query/inc/qTableMeta.h b/src/query/inc/qTableMeta.h index 56eea6429f..daef726ca6 100644 --- a/src/query/inc/qTableMeta.h +++ b/src/query/inc/qTableMeta.h @@ -60,7 +60,7 @@ typedef struct STableComInfo { typedef struct STableMeta { int32_t vgId; STableId id; - uint8_t tableType; + int8_t tableType; char sTableName[TSDB_TABLE_FNAME_LEN]; // super table name uint64_t suid; // super table id int16_t sversion; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 7e3e542f5e..11624ddffc 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -30,6 +30,7 @@ #include "tcompare.h" #include "tscompression.h" #include "qScript.h" +#include "tscLog.h" #define IS_MASTER_SCAN(runtime) ((runtime)->scanFlag == MASTER_SCAN) #define IS_REVERSE_SCAN(runtime) ((runtime)->scanFlag == REVERSE_SCAN) @@ -750,7 +751,7 @@ static int32_t getNumOfRowsInTimeWindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBloc int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - if (ekey < pDataBlockInfo->window.ekey) { + if (ekey < pDataBlockInfo->window.ekey && pPrimaryColumn) { num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); if (updateLastKey) { // update the last key item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step; @@ -762,7 +763,7 @@ static int32_t getNumOfRowsInTimeWindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBloc } } } else { // desc - if (ekey > pDataBlockInfo->window.skey) { + if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) { num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); if (updateLastKey) { // update the last key item->lastKey = pPrimaryColumn[startPos - (num - 1)] + step; @@ -1299,6 +1300,10 @@ static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBloc assert(pBlock != NULL); int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + if (pBlock->pDataBlock == NULL){ + tscError("pBlock->pDataBlock == NULL"); + return; + } SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, 0); TSKEY *tsCols = (TSKEY *)(pColInfo->pData); @@ -7540,7 +7545,7 @@ SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pCo int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId) { *pFilterInfo = calloc(1, sizeof(SSingleColumnFilterInfo) * numOfFilterCols); - if (pFilterInfo == NULL) { + if (*pFilterInfo == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } diff --git a/src/query/src/qPlan.c b/src/query/src/qPlan.c index 2772e76abe..f7fddecaf8 100644 --- a/src/query/src/qPlan.c +++ b/src/query/src/qPlan.c @@ -40,7 +40,7 @@ static SQueryNode* createQueryNode(int32_t type, const char* name, SQueryNode** pNode->info.type = type; pNode->info.name = strdup(name); - if (pTableInfo->id.uid != 0) { // it is a true table + if (pTableInfo->id.uid != 0 && pTableInfo->tableName) { // it is a true table pNode->tableInfo.id = pTableInfo->id; pNode->tableInfo.tableName = strdup(pTableInfo->tableName); } diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index 05ecf2e9b1..63eba51d6b 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -78,8 +78,9 @@ static char* doDecompressData(void* data, int32_t srcSize, int32_t *dst, SDiskba } *dst = tsDecompressString(data, srcSize, 1, pResultBuf->assistBuf, pResultBuf->pageSize, ONE_STAGE_COMP, NULL, 0); - - memcpy(data, pResultBuf->assistBuf, *dst); + if (*dst > 0) { + memcpy(data, pResultBuf->assistBuf, *dst); + } return data; } diff --git a/src/query/src/qScript.c b/src/query/src/qScript.c index 261164a84c..ed56526782 100644 --- a/src/query/src/qScript.c +++ b/src/query/src/qScript.c @@ -377,9 +377,11 @@ ScriptEnv* getScriptEnvFromPool() { return NULL; } SListNode *pNode = tdListPopHead(pool->scriptEnvs); - tdListNodeGetData(pool->scriptEnvs, pNode, (void *)(&pEnv)); - listNodeFree(pNode); - + if (pNode){ + tdListNodeGetData(pool->scriptEnvs, pNode, (void *)(&pEnv)); + listNodeFree(pNode); + } + pool->cSize--; pthread_mutex_unlock(&pool->mutex); return pEnv; diff --git a/src/query/src/qSqlParser.c b/src/query/src/qSqlParser.c index eb920b3e17..08c55d5d7e 100644 --- a/src/query/src/qSqlParser.c +++ b/src/query/src/qSqlParser.c @@ -142,14 +142,17 @@ tSqlExpr *tSqlExprCreateIdValue(SStrToken *pToken, int32_t optrType) { } if (optrType == TK_NULL) { - pToken->type = TSDB_DATA_TYPE_NULL; - tVariantCreate(&pSqlExpr->value, pToken); + if (pToken){ + pToken->type = TSDB_DATA_TYPE_NULL; + tVariantCreate(&pSqlExpr->value, pToken); + } pSqlExpr->tokenId = optrType; pSqlExpr->type = SQL_NODE_VALUE; } else if (optrType == TK_INTEGER || optrType == TK_STRING || optrType == TK_FLOAT || optrType == TK_BOOL) { - toTSDBType(pToken->type); - - tVariantCreate(&pSqlExpr->value, pToken); + if (pToken) { + toTSDBType(pToken->type); + tVariantCreate(&pSqlExpr->value, pToken); + } pSqlExpr->tokenId = optrType; pSqlExpr->type = SQL_NODE_VALUE; } else if (optrType == TK_NOW) { @@ -162,9 +165,11 @@ tSqlExpr *tSqlExprCreateIdValue(SStrToken *pToken, int32_t optrType) { } else if (optrType == TK_VARIABLE) { // use nanosecond by default // TODO set value after getting database precision - int32_t ret = parseAbsoluteDuration(pToken->z, pToken->n, &pSqlExpr->value.i64, TSDB_TIME_PRECISION_NANO); - if (ret != TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; + if (pToken) { + int32_t ret = parseAbsoluteDuration(pToken->z, pToken->n, &pSqlExpr->value.i64, TSDB_TIME_PRECISION_NANO); + if (ret != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; + } } pSqlExpr->flags |= 1 << EXPR_FLAG_NS_TIMESTAMP; @@ -340,8 +345,9 @@ static FORCE_INLINE int32_t tStrTokenCompare(SStrToken* left, SStrToken* right) return (left->type == right->type && left->n == right->n && strncasecmp(left->z, right->z, left->n) == 0) ? 0 : 1; } +// this function is not used for temporary int32_t tSqlExprCompare(tSqlExpr *left, tSqlExpr *right) { - if ((left == NULL && right) || (left && right == NULL)) { + if ((left == NULL && right) || (left && right == NULL) || (left == NULL && right == NULL)) { return 1; } diff --git a/src/query/src/qTableMeta.c b/src/query/src/qTableMeta.c index d25d6b7004..f687b8aa1f 100644 --- a/src/query/src/qTableMeta.c +++ b/src/query/src/qTableMeta.c @@ -72,7 +72,7 @@ SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId) { } STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) { - assert(pTableMetaMsg != NULL && pTableMetaMsg->numOfColumns >= 2 && pTableMetaMsg->numOfTags >= 0); + assert(pTableMetaMsg != NULL && pTableMetaMsg->numOfColumns >= 2); int32_t schemaSize = (pTableMetaMsg->numOfColumns + pTableMetaMsg->numOfTags) * sizeof(SSchema); STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + schemaSize); diff --git a/src/query/src/queryMain.c b/src/query/src/queryMain.c index 0d140d5ffb..7d30f7c668 100644 --- a/src/query/src/queryMain.c +++ b/src/query/src/queryMain.c @@ -261,7 +261,7 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex SQInfo *pQInfo = (SQInfo *)qinfo; if (pQInfo == NULL || !isValidQInfo(pQInfo)) { - qError("QInfo:0x%"PRIx64" invalid qhandle", pQInfo->qId); + qError("QInfo invalid qhandle"); return TSDB_CODE_QRY_INVALID_QHANDLE; } diff --git a/src/util/inc/tlist.h b/src/util/inc/tlist.h index 6c96ec0b13..7581904540 100644 --- a/src/util/inc/tlist.h +++ b/src/util/inc/tlist.h @@ -44,7 +44,7 @@ typedef struct { #define listNEles(l) (l)->numOfEles #define listEleSize(l) (l)->eleSize #define isListEmpty(l) ((l)->numOfEles == 0) -#define listNodeFree(n) free(n); +#define listNodeFree(n) free(n) SList * tdListNew(int eleSize); void * tdListFree(SList *list); From b2c0fbe0ce15ba13d79a4e262dee1a1d106bdb02 Mon Sep 17 00:00:00 2001 From: markswang <792637585@qq.com> Date: Tue, 27 Jul 2021 17:08:32 +0800 Subject: [PATCH 051/106] [TD-5534]:fix the coverity high risk of client --- src/client/src/TSDBJNIConnector.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index 667a689979..b373e360a1 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -964,7 +964,7 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_insertLinesImp(J int numLines = (*env)->GetArrayLength(env, lines); char** c_lines = calloc(numLines, sizeof(char*)); if (c_lines == NULL) { - jniError("c_lines:%d, alloc memory failed", c_lines); + jniError("c_lines:%p, alloc memory failed", c_lines); return JNI_OUT_OF_MEMORY; } for (int i = 0; i < numLines; ++i) { From f4efe18aa35404c2353031394177a1db79ee1ebe Mon Sep 17 00:00:00 2001 From: markswang <792637585@qq.com> Date: Tue, 27 Jul 2021 17:10:51 +0800 Subject: [PATCH 052/106] [TD-5534]:fix the coverity high risk of client --- src/client/src/tscSql.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index b6e2604bd2..6f3d5c3a63 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -986,8 +986,8 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { if (code != TSDB_CODE_SUCCESS) { tscFreeSqlObj(pSql); - taosArrayDestroyEx(plist); - taosArrayDestroyEx(vgroupList); + taosArrayDestroyEx(plist, freeElem); + taosArrayDestroyEx(vgroupList, freeElem); return code; } From 140f705c5271a601c58e176eecc69c94dcb3e9f1 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 27 Jul 2021 17:22:18 +0800 Subject: [PATCH 053/106] [TD-5356]: default numOfMnodes to 1 instead of 3 --- src/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 381d2804c1..ec7249cef5 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -38,7 +38,7 @@ uint16_t tsDnodeDnodePort = 6035; // udp/tcp uint16_t tsSyncPort = 6040; uint16_t tsArbitratorPort = 6042; int32_t tsStatusInterval = 1; // second -int32_t tsNumOfMnodes = 3; +int32_t tsNumOfMnodes = 1; int8_t tsEnableVnodeBak = 1; int8_t tsEnableTelemetryReporting = 1; int8_t tsArbOnline = 0; From ff9971ffa2838f08dbeaf761213d563e6ec15f22 Mon Sep 17 00:00:00 2001 From: markswang <792637585@qq.com> Date: Tue, 27 Jul 2021 17:43:03 +0800 Subject: [PATCH 054/106] [TD-5534]:fix the coverity high risk of client --- src/client/src/tscLocal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index e49f4e33ae..9e65094e16 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -860,10 +860,10 @@ static int32_t tscProcessServStatus(SSqlObj *pSql) { if (pHb != NULL) { pSql->res.code = checkForOnlineNode(pHb); + taosReleaseRef(tscObjRef, pObj->hbrid); } if (pSql->res.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - taosReleaseRef(tscObjRef, pObj->hbrid); return pSql->res.code; } From 6faf1d5bb3e98af1b587299b9a14bd95ec30ebca Mon Sep 17 00:00:00 2001 From: haoranchen Date: Tue, 27 Jul 2021 18:26:10 +0800 Subject: [PATCH 055/106] Update sdbCompClusterReplica2.py --- tests/pytest/wal/sdbCompClusterReplica2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytest/wal/sdbCompClusterReplica2.py b/tests/pytest/wal/sdbCompClusterReplica2.py index dd5a375151..ba80e3864a 100644 --- a/tests/pytest/wal/sdbCompClusterReplica2.py +++ b/tests/pytest/wal/sdbCompClusterReplica2.py @@ -86,7 +86,7 @@ class TwoClients: tdSql.execute("alter table stb2_0 add column col2 binary(4)") tdSql.execute("alter table stb2_0 drop column col1") tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')") - tdSql.execute("drop dnode 2") + tdSql.execute("drop dnode chenhaoran02") sleep(10) os.system("rm -rf /var/lib/taos/*") print("clear dnode chenhaoran02'data files") @@ -142,4 +142,4 @@ class TwoClients: clients = TwoClients() clients.initConnection() # clients.getBuildPath() -clients.run() \ No newline at end of file +clients.run() From 928473f0cbf77e3910539d6b8357c43c9caa9006 Mon Sep 17 00:00:00 2001 From: markswang <792637585@qq.com> Date: Tue, 27 Jul 2021 18:27:35 +0800 Subject: [PATCH 056/106] [TD-5534]:fix the coverity high risk of client --- src/client/src/tscParseLineProtocol.c | 2 +- src/client/src/tscPrepare.c | 4 ++-- src/client/src/tscSQLParser.c | 2 +- src/client/src/tscServer.c | 2 +- src/client/src/tscUtil.c | 6 +----- src/query/inc/qExecutor.h | 1 - src/query/src/qExecutor.c | 2 +- 7 files changed, 7 insertions(+), 12 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 4b5416c6b9..1790403a5e 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -1653,7 +1653,7 @@ static int32_t parseSmlTimeStamp(TAOS_SML_KV **pTS, const char **index) { static int32_t parseSmlKey(TAOS_SML_KV *pKV, const char **index) { const char *cur = *index; - char key[TSDB_COL_NAME_LEN + 1]; // +1 to avoid 1685 line over write + char key[TSDB_COL_NAME_LEN + 1]; // +1 to avoid key[len] over write uint16_t len = 0; //key field cannot start with digit diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 9ff3a0e2dd..2c2a299549 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -1628,8 +1628,8 @@ int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags if (pStmt->mtb.subSet && taosHashGetSize(pStmt->mtb.pTableHash) > 0) { STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - char sTableName[TSDB_TABLE_FNAME_LEN + 1] = {0}; - strncpy(sTableName, pTableMeta->sTableName, sizeof(sTableName) - 1); + char sTableName[TSDB_TABLE_FNAME_LEN] = {0}; + tstrncpy(sTableName, pTableMeta->sTableName, sizeof(sTableName)); SStrToken tname = {0}; tname.type = TK_STRING; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 1fac9c9a40..e899cc3a07 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -8366,7 +8366,7 @@ static int32_t doValidateSubquery(SSqlNode* pSqlNode, int32_t index, SSqlObj* pS return invalidOperationMsg(msgBuf, "subquery alias name too long"); } - strncpy(pTableMetaInfo1->aliasName, subInfo->aliasName.z, MIN(subInfo->aliasName.n, sizeof(pTableMetaInfo1->aliasName) - 1)); + tstrncpy(pTableMetaInfo1->aliasName, subInfo->aliasName.z, MIN(subInfo->aliasName.n, sizeof(pTableMetaInfo1->aliasName))); } taosArrayPush(pQueryInfo->pUpstream, &pSub); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 0748780e1a..f3c24eb407 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -164,7 +164,7 @@ static void tscUpdateVgroupInfo(SSqlObj *pSql, SRpcEpSet *pEpSet) { vgroupInfo.inUse = pEpSet->inUse; vgroupInfo.numOfEps = pEpSet->numOfEps; for (int32_t i = 0; i < vgroupInfo.numOfEps; i++) { - strncpy(vgroupInfo.ep[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN); // buffer not null terminated risk + tstrncpy(vgroupInfo.ep[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN); // buffer not null terminated risk vgroupInfo.ep[i].port = pEpSet->port[i]; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 7e36cce50e..27fc19b205 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1236,10 +1236,6 @@ void handleDownstreamOperator(SSqlObj** pSqlObjList, int32_t numOfUpstream, SQue } pSourceOperator = createJoinOperatorInfo(p, px->numOfTables, schema, num); - - for(int32_t i = 0; i < px->numOfTables; ++i) { - destroyOperatorInfo(p[i]); - } tfree(p); } else { size_t num = taosArrayGetSize(px->colList); @@ -4714,7 +4710,7 @@ static int32_t doAddTableName(char* nextStr, char** str, SArray* pNameArray, SSq int32_t len = 0; if (nextStr == NULL) { - strncpy(tablename, *str, TSDB_TABLE_FNAME_LEN - 1); + tstrncpy(tablename, *str, TSDB_TABLE_FNAME_LEN); len = (int32_t) strlen(tablename); } else { len = (int32_t)(nextStr - (*str)); diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 71e9792bd8..4581ba258d 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -578,7 +578,6 @@ void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p); SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows); void* destroyOutputBuf(SSDataBlock* pBlock); void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols); -void destroyOperatorInfo(SOperatorInfo* pOperator); void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); int32_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 11624ddffc..6c18419619 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5949,7 +5949,7 @@ static int32_t getNumOfScanTimes(SQueryAttr* pQueryAttr) { return 1; } -void destroyOperatorInfo(SOperatorInfo* pOperator) { +static void destroyOperatorInfo(SOperatorInfo* pOperator) { if (pOperator == NULL) { return; } From 4216a400a3e22d00cd308a80fc0f1f382eb9dd40 Mon Sep 17 00:00:00 2001 From: markswang <792637585@qq.com> Date: Tue, 27 Jul 2021 18:29:49 +0800 Subject: [PATCH 057/106] [TD-5534]:fix the coverity medium risk of client --- src/query/src/qExecutor.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 6c18419619..cc51cafe38 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -192,6 +192,8 @@ static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput); static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyAggOperatorInfo(void* param, int32_t numOfOutput); +static void destroyOperatorInfo(SOperatorInfo* pOperator); + static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock); From 51de627d2b036be72b40b5eb3b3b3e5fdd3ed5a1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 27 Jul 2021 18:56:02 +0800 Subject: [PATCH 058/106] [TD-5574]: fix change KV row value coredump --- src/common/inc/tdataformat.h | 1 + src/common/src/tdataformat.c | 83 +++++++++++++----------------------- 2 files changed, 31 insertions(+), 53 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index e8c0760997..2268f133d6 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -351,6 +351,7 @@ typedef struct { #define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i)) #define kvRowFree(r) tfree(r) #define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r)) +#define kvRowValLen(r) (kvRowLen(r) - TD_KV_ROW_HEAD_SIZE - sizeof(SColIdx) * kvRowNCols(r)) SKVRow tdKVRowDup(SKVRow row); int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value); diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index f5b84e4c9a..20c87d9d22 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -553,42 +553,28 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { SKVRow nrow = NULL; void * ptr = taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_GE); - if (ptr == NULL || ((SColIdx *)ptr)->colId > colId) { // need to add a column value to the row + if (ptr == NULL || ((SColIdx *)ptr)->colId > colId) { // need to add a column value to the row int diff = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type]; - nrow = malloc(kvRowLen(row) + sizeof(SColIdx) + diff); + int nRowLen = kvRowLen(row) + sizeof(SColIdx) + diff; + int oRowCols = kvRowNCols(row); + + ASSERT(diff > 0); + nrow = malloc(nRowLen); if (nrow == NULL) return -1; - kvRowSetLen(nrow, kvRowLen(row) + (int16_t)sizeof(SColIdx) + diff); - kvRowSetNCols(nrow, kvRowNCols(row) + 1); + kvRowSetLen(nrow, nRowLen); + kvRowSetNCols(nrow, oRowCols + 1); - if (ptr == NULL) { - memcpy(kvRowColIdx(nrow), kvRowColIdx(row), sizeof(SColIdx) * kvRowNCols(row)); - memcpy(kvRowValues(nrow), kvRowValues(row), POINTER_DISTANCE(kvRowEnd(row), kvRowValues(row))); - int colIdx = kvRowNCols(nrow) - 1; - kvRowColIdxAt(nrow, colIdx)->colId = colId; - kvRowColIdxAt(nrow, colIdx)->offset = (int16_t)(POINTER_DISTANCE(kvRowEnd(row), kvRowValues(row))); - memcpy(kvRowColVal(nrow, kvRowColIdxAt(nrow, colIdx)), value, diff); - } else { - int16_t tlen = (int16_t)(POINTER_DISTANCE(ptr, kvRowColIdx(row))); - if (tlen > 0) { - memcpy(kvRowColIdx(nrow), kvRowColIdx(row), tlen); - memcpy(kvRowValues(nrow), kvRowValues(row), ((SColIdx *)ptr)->offset); - } + memcpy(kvRowColIdx(nrow), kvRowColIdx(row), sizeof(SColIdx) * oRowCols); + memcpy(kvRowValues(nrow), kvRowValues(row), kvRowValLen(row)); - int colIdx = tlen / sizeof(SColIdx); - kvRowColIdxAt(nrow, colIdx)->colId = colId; - kvRowColIdxAt(nrow, colIdx)->offset = ((SColIdx *)ptr)->offset; - memcpy(kvRowColVal(nrow, kvRowColIdxAt(nrow, colIdx)), value, diff); + pColIdx = kvRowColIdxAt(nrow, oRowCols); + pColIdx->colId = colId; + pColIdx->offset = kvRowValLen(row); - for (int i = colIdx; i < kvRowNCols(row); i++) { - kvRowColIdxAt(nrow, i + 1)->colId = kvRowColIdxAt(row, i)->colId; - kvRowColIdxAt(nrow, i + 1)->offset = kvRowColIdxAt(row, i)->offset + diff; - } - memcpy(kvRowColVal(nrow, kvRowColIdxAt(nrow, colIdx + 1)), kvRowColVal(row, kvRowColIdxAt(row, colIdx)), - POINTER_DISTANCE(kvRowEnd(row), kvRowColVal(row, kvRowColIdxAt(row, colIdx))) + memcpy(kvRowColVal(nrow, pColIdx), value, diff); // copy new value - ); - } + tdSortKVRowByColIdx(nrow); *orow = nrow; free(row); @@ -599,9 +585,8 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { if (varDataTLen(value) == varDataTLen(pOldVal)) { // just update the column value in place memcpy(pOldVal, value, varDataTLen(value)); - } else { // need to reallocate the memory - int16_t diff = varDataTLen(value) - varDataTLen(pOldVal); - int16_t nlen = kvRowLen(row) + diff; + } else { // need to reallocate the memory + int16_t nlen = kvRowLen(row) + (varDataTLen(value) - varDataTLen(pOldVal)); ASSERT(nlen > 0); nrow = malloc(nlen); if (nrow == NULL) return -1; @@ -609,30 +594,22 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { kvRowSetLen(nrow, nlen); kvRowSetNCols(nrow, kvRowNCols(row)); - // Copy part ahead - nlen = (int16_t)(POINTER_DISTANCE(ptr, kvRowColIdx(row))); - ASSERT(nlen % sizeof(SColIdx) == 0); - if (nlen > 0) { - ASSERT(((SColIdx *)ptr)->offset > 0); - memcpy(kvRowColIdx(nrow), kvRowColIdx(row), nlen); - memcpy(kvRowValues(nrow), kvRowValues(row), ((SColIdx *)ptr)->offset); + int zsize = sizeof(SColIdx) * kvRowNCols(row) + ((SColIdx *)ptr)->offset; + memcpy(kvRowColIdx(nrow), kvRowColIdx(row), zsize); + memcpy(kvRowColVal(nrow, ((SColIdx *)ptr)), value, varDataTLen(value)); + // Copy left value part + int lsize = kvRowLen(row) - TD_KV_ROW_HEAD_SIZE - zsize - varDataTLen(pOldVal); + if (lsize > 0) { + memcpy(POINTER_SHIFT(nrow, TD_KV_ROW_HEAD_SIZE + zsize + varDataTLen(value)), + POINTER_SHIFT(nrow, TD_KV_ROW_HEAD_SIZE + zsize + varDataTLen(pOldVal)), lsize); } - // Construct current column value - int colIdx = nlen / sizeof(SColIdx); - pColIdx = kvRowColIdxAt(nrow, colIdx); - pColIdx->colId = ((SColIdx *)ptr)->colId; - pColIdx->offset = ((SColIdx *)ptr)->offset; - memcpy(kvRowColVal(nrow, pColIdx), value, varDataTLen(value)); - - // Construct columns after - if (kvRowNCols(nrow) - colIdx - 1 > 0) { - for (int i = colIdx + 1; i < kvRowNCols(nrow); i++) { - kvRowColIdxAt(nrow, i)->colId = kvRowColIdxAt(row, i)->colId; - kvRowColIdxAt(nrow, i)->offset = kvRowColIdxAt(row, i)->offset + diff; + for (int i = 0; i < kvRowNCols(nrow); i++) { + pColIdx = kvRowColIdxAt(nrow, i); + + if (pColIdx->offset > ((SColIdx *)ptr)->offset) { + pColIdx->offset = pColIdx->offset - varDataTLen(pOldVal) + varDataTLen(value); } - memcpy(kvRowColVal(nrow, kvRowColIdxAt(nrow, colIdx + 1)), kvRowColVal(row, kvRowColIdxAt(row, colIdx + 1)), - POINTER_DISTANCE(kvRowEnd(row), kvRowColVal(row, kvRowColIdxAt(row, colIdx + 1)))); } *orow = nrow; From 97850874a7d71ad7f4af3cf49942c14c92fb032d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 28 Jul 2021 10:55:21 +0800 Subject: [PATCH 059/106] fix typo causing coredump --- src/common/src/tdataformat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 20c87d9d22..335c2b0a92 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -601,7 +601,7 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { int lsize = kvRowLen(row) - TD_KV_ROW_HEAD_SIZE - zsize - varDataTLen(pOldVal); if (lsize > 0) { memcpy(POINTER_SHIFT(nrow, TD_KV_ROW_HEAD_SIZE + zsize + varDataTLen(value)), - POINTER_SHIFT(nrow, TD_KV_ROW_HEAD_SIZE + zsize + varDataTLen(pOldVal)), lsize); + POINTER_SHIFT(row, TD_KV_ROW_HEAD_SIZE + zsize + varDataTLen(pOldVal)), lsize); } for (int i = 0; i < kvRowNCols(nrow); i++) { From ed423748931068d3cb7b8980f19fa0cbfc689053 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 28 Jul 2021 11:14:20 +0800 Subject: [PATCH 060/106] tsim/nchar-binary: fix negative length crash --- tests/tsim/src/simExe.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index c5b8d5c5eb..c29593b6ef 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -808,6 +808,11 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: + if (length[i] < 0 || length[i] > 1 << 20) { + fprintf(stderr, "Invalid length(%d) of BINARY or NCHAR\n", length[i]); + exit(-1); + } + memset(value, 0, MAX_QUERY_VALUE_LEN); memcpy(value, row[i], length[i]); value[length[i]] = 0; From 211c079b074696a645bca76b2bbc22f28dacadb5 Mon Sep 17 00:00:00 2001 From: markswang <792637585@qq.com> Date: Wed, 28 Jul 2021 12:11:17 +0800 Subject: [PATCH 061/106] [TD-5534]:fix the coverity high risk of client --- src/client/src/TSDBJNIConnector.c | 15 ++++----------- src/client/src/tscGlobalmerge.c | 3 +-- src/client/src/tscParseLineProtocol.c | 12 ++++++------ src/client/src/tscSQLParser.c | 4 ++-- src/client/src/tscServer.c | 2 +- src/client/src/tscSubquery.c | 2 +- 6 files changed, 15 insertions(+), 23 deletions(-) diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index b373e360a1..7ba613de88 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -726,13 +726,12 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_prepareStmtImp(J TAOS_STMT* pStmt = taos_stmt_init(tscon); int32_t code = taos_stmt_prepare(pStmt, str, len); + tfree(str); if (code != TSDB_CODE_SUCCESS) { jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); - free(str); return JNI_TDENGINE_ERROR; } - free(str); return (jlong) pStmt; } @@ -920,10 +919,6 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setTableNameTagsI char* curTags = tagsData; TAOS_BIND *tagsBind = calloc(numOfTags, sizeof(TAOS_BIND)); - if (tagsBind == NULL) { - jniError("numOfTags:%d, alloc memory failed", numOfTags); - return JNI_OUT_OF_MEMORY; - } for(int32_t i = 0; i < numOfTags; ++i) { tagsBind[i].buffer_type = typeArray[i]; tagsBind[i].buffer = curTags; @@ -942,14 +937,13 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setTableNameTagsI tfree(lengthArray); tfree(typeArray); tfree(nullArray); + tfree(tagsBind); (*env)->ReleaseStringUTFChars(env, tableName, name); if (code != TSDB_CODE_SUCCESS) { jniError("jobj:%p, conn:%p, code:%s", jobj, tsconn, tstrerror(code)); - free(tagsBind); return JNI_TDENGINE_ERROR; } - free(tagsBind); return JNI_SUCCESS; } @@ -979,12 +973,11 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_insertLinesImp(J (*env)->ReleaseStringUTFChars(env, line, c_lines[i]); } + tfree(c_lines); if (code != TSDB_CODE_SUCCESS) { jniError("jobj:%p, conn:%p, code:%s", jobj, taos, tstrerror(code)); - free(c_lines); + return JNI_TDENGINE_ERROR; } - - free(c_lines); return code; } \ No newline at end of file diff --git a/src/client/src/tscGlobalmerge.c b/src/client/src/tscGlobalmerge.c index 9c9bac5951..e696d54abd 100644 --- a/src/client/src/tscGlobalmerge.c +++ b/src/client/src/tscGlobalmerge.c @@ -443,11 +443,10 @@ int32_t tscCreateGlobalMergerEnv(SQueryInfo *pQueryInfo, tExtMemBuffer ***pMemBu } pModel = createColumnModel(pSchema, (int32_t)size, capacity); + tfree(pSchema); if (pModel == NULL){ - tfree(pSchema); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - tfree(pSchema); int32_t pg = DEFAULT_PAGE_SIZE; int32_t overhead = sizeof(tFilePage); diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 1790403a5e..d12ad04aa8 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -701,14 +701,14 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols TAOS_STMT* stmt = taos_stmt_init(taos); if (stmt == NULL) { - free(sql); + tfree(sql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } code = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql)); - free(sql); + tfree(sql); if (code != 0) { - free(stmt); + tfree(stmt); tscError("%s", taos_stmt_errstr(stmt)); return code; } @@ -717,7 +717,7 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols code = taos_stmt_set_tbname(stmt, cTableName); if (code != 0) { tscError("%s", taos_stmt_errstr(stmt)); - free(stmt); + tfree(stmt); return code; } @@ -727,13 +727,13 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols code = taos_stmt_bind_param(stmt, colsBinds); if (code != 0) { tscError("%s", taos_stmt_errstr(stmt)); - free(stmt); + tfree(stmt); return code; } code = taos_stmt_add_batch(stmt); if (code != 0) { tscError("%s", taos_stmt_errstr(stmt)); - free(stmt); + tfree(stmt); return code; } } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index e899cc3a07..fb1eb56422 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -8362,7 +8362,7 @@ static int32_t doValidateSubquery(SSqlNode* pSqlNode, int32_t index, SSqlObj* pS if (subInfo->aliasName.n > 0) { if (subInfo->aliasName.n >= TSDB_TABLE_FNAME_LEN) { - free(pTableMetaInfo1); + tfree(pTableMetaInfo1); return invalidOperationMsg(msgBuf, "subquery alias name too long"); } @@ -8376,7 +8376,7 @@ static int32_t doValidateSubquery(SSqlNode* pSqlNode, int32_t index, SSqlObj* pS STableMetaInfo** tmp = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES); if (tmp == NULL) { - free(pTableMetaInfo1); + tfree(pTableMetaInfo1); return TSDB_CODE_TSC_OUT_OF_MEMORY; } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index f3c24eb407..76382d9afd 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2055,7 +2055,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { } if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) { tscError("0x%"PRIx64" invalid table meta from mnode, name:%s", pSql->self, tNameGetTableName(&pTableMetaInfo->name)); - free(pTableMeta); + tfree(pTableMeta); return TSDB_CODE_TSC_INVALID_VALUE; } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index bef5992d53..2d00253562 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2476,7 +2476,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { pState->states = calloc(pState->numOfSub, sizeof(*pState->states)); if (pState->states == NULL) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscDestroyGlobalMergerEnv(pMemoryBuf, pDesc,pState->numOfSub); + tscDestroyGlob alMergerEnv(pMemoryBuf, pDesc,pState->numOfSub); tscAsyncResultOnError(pSql); return ret; From a7dc581f23db01655ff247762dc9767bea90cab7 Mon Sep 17 00:00:00 2001 From: markswang <792637585@qq.com> Date: Wed, 28 Jul 2021 12:12:33 +0800 Subject: [PATCH 062/106] [TD-5534]:fix the coverity high risk of client --- src/client/src/tscSubquery.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 2d00253562..bef5992d53 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2476,7 +2476,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { pState->states = calloc(pState->numOfSub, sizeof(*pState->states)); if (pState->states == NULL) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscDestroyGlob alMergerEnv(pMemoryBuf, pDesc,pState->numOfSub); + tscDestroyGlobalMergerEnv(pMemoryBuf, pDesc,pState->numOfSub); tscAsyncResultOnError(pSql); return ret; From 7845fc96821ec8f29fd5ca813e67cbaa38e17f9c Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Wed, 28 Jul 2021 14:04:59 +0800 Subject: [PATCH 063/106] [TD-5074]:test operator cost --- tests/pytest/query/operator_cost.py | 539 ++++++++++++++++++++++++++++ 1 file changed, 539 insertions(+) create mode 100644 tests/pytest/query/operator_cost.py diff --git a/tests/pytest/query/operator_cost.py b/tests/pytest/query/operator_cost.py new file mode 100644 index 0000000000..b94d5fa3b3 --- /dev/null +++ b/tests/pytest/query/operator_cost.py @@ -0,0 +1,539 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +import random +import time + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1600000000000 + self.num = 10 + + def run(self): + tdSql.prepare() + # test case for https://jira.taosdata.com:18080/browse/TD-5074 + + startTime = time.time() + + tdSql.execute('''create stable stable_1 + (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint, + q_bool bool , q_binary binary(20) , q_nchar nchar(20) , + q_float float , q_double double , q_ts timestamp) + tags(loc nchar(20) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, + t_bool bool , t_binary binary(20) , t_nchar nchar(20) , + t_float float , t_double double , t_ts timestamp);''') + tdSql.execute('''create stable stable_2 + (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint, + q_bool bool , q_binary binary(20) , q_nchar nchar(20) , + q_float float , q_double double , q_ts timestamp) + tags(loc nchar(20) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, + t_bool bool , t_binary binary(20) , t_nchar nchar(20) , + t_float float , t_double double , t_ts timestamp);''') + tdSql.execute('''create table table_0 using stable_1 + tags('table_0' , '0' , '0' , '0' , '0' , 0 , '0' , '0' , '0' , '0' ,'0')''') + tdSql.execute('''create table table_1 using stable_1 + tags('table_1' , '2147483647' , '9223372036854775807' , '32767' , '127' , 1 , + 'binary1' , 'nchar1' , '1' , '11' , \'1999-09-09 09:09:09.090\')''') + tdSql.execute('''create table table_2 using stable_1 + tags('table_2' , '-2147483647' , '-9223372036854775807' , '-32767' , '-127' , false , + 'binary2' , 'nchar2nchar2' , '-2.2' , '-22.22' , \'2099-09-09 09:09:09.090\')''') + tdSql.execute('''create table table_3 using stable_1 + tags('table_3' , '3' , '3' , '3' , '3' , true , 'binary3' , 'nchar3' , '33.33' , '3333.3333' , '0')''') + tdSql.execute('''create table table_4 using stable_1 + tags('table_4' , '4' , '4' , '4' , '4' , false , 'binary4' , 'nchar4' , '-444.444' , '-444444.444444' , '0')''') + tdSql.execute('''create table table_5 using stable_1 + tags('table_5' , '5' , '5' , '5' , '5' , true , 'binary5' , 'nchar5' , '5555.5555' , '55555555.55555555' , '0')''') + tdSql.execute('''create table table_21 using stable_2 + tags('table_5' , '5' , '5' , '5' , '5' , true , 'binary5' , 'nchar5' , '5555.5555' , '55555555.55555555' , '0')''') + #regular table + tdSql.execute('''create table regular_table_1 + (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint, + q_bool bool , q_binary binary(20) , q_nchar nchar(20) , + q_float float , q_double double , q_ts timestamp) ;''') + + for i in range(self.num): + tdSql.execute('''insert into table_0 values(%d, %d, %d, %d, %d, 0, 'binary.%s', 'nchar.%s', %f, %f, %d)''' + % (self.ts + i, i, i, i, i, i, i, i, i, self.ts + i)) + tdSql.execute('''insert into table_1 values(%d, %d, %d, %d, %d, 1, 'binary1.%s', 'nchar1.%s', %f, %f, %d)''' + % (self.ts + i, 2147483647-i, 9223372036854775807-i, 32767-i, 127-i, + i, i, random.random(), random.random(), 1262304000001 + i)) + tdSql.execute('''insert into table_2 values(%d, %d, %d, %d, %d, true, 'binary2.%s', 'nchar2nchar2.%s', %f, %f, %d)''' + % (self.ts + i, -2147483647+i, -9223372036854775807+i, -32767+i, -127+i, + i, i, random.uniform(-1,0), random.uniform(-1,0), 1577836800001 + i)) + tdSql.execute('''insert into table_3 values(%d, %d, %d, %d, %d, false, 'binary3.%s', 'nchar3.%s', %f, %f, %d)''' + % (self.ts + i, random.randint(-2147483647, 2147483647), + random.randint(-9223372036854775807, 9223372036854775807), random.randint(-32767, 32767), + random.randint(-127, 127), random.randint(-100, 100), random.randint(-10000, 10000), + random.uniform(-100000,100000), random.uniform(-1000000000,1000000000), self.ts + i)) + tdSql.execute('''insert into table_4 values(%d, %d, %d, %d, %d, true, 'binary4.%s', 'nchar4.%s', %f, %f, %d)''' + % (self.ts + i, i, i, i, i, i, i, i, i, self.ts + i)) + tdSql.execute('''insert into table_5 values(%d, %d, %d, %d, %d, false, 'binary5.%s', 'nchar5.%s', %f, %f, %d)''' + % (self.ts + i, i, i, i, i, i, i, i, i, self.ts + i)) + tdSql.execute('''insert into table_21 values(%d, %d, %d, %d, %d, false, 'binary5.%s', 'nchar5.%s', %f, %f, %d)''' + % (self.ts + i, i, i, i, i, i, i, i, i, self.ts + i)) + + tdSql.execute('''insert into regular_table_1 values(%d, %d, %d, %d, %d, 0, 'binary.%s', 'nchar.%s', %f, %f, %d)''' + % (self.ts + i, i, i, i, i, i, i, i, i, self.ts + i)) + tdSql.execute('''insert into regular_table_1 values(%d, %d, %d, %d, %d, 1, 'binary1.%s', 'nchar1.%s', %f, %f, %d)''' + % (self.ts + 100 + i, 2147483647-i, 9223372036854775807-i, 32767-i, 127-i, + i, i, random.random(), random.random(), 1262304000001 + i)) + tdSql.execute('''insert into regular_table_1 values(%d, %d, %d, %d, %d, true, 'binary2.%s', 'nchar2nchar2.%s', %f, %f, %d)''' + % (self.ts + 200 + i, -2147483647+i, -9223372036854775807+i, -32767+i, -127+i, + i, i, random.uniform(-1,0), random.uniform(-1,0), 1577836800001 + i)) + tdSql.execute('''insert into regular_table_1 values(%d, %d, %d, %d, %d, false, 'binary3.%s', 'nchar3.%s', %f, %f, %d)''' + % (self.ts + 300 + i, random.randint(-2147483647, 2147483647), + random.randint(-9223372036854775807, 9223372036854775807), random.randint(-32767, 32767), + random.randint(-127, 127), random.randint(-100, 100), random.randint(-10000, 10000), + random.uniform(-100000,100000), random.uniform(-1000000000,1000000000), self.ts + i)) + tdSql.execute('''insert into regular_table_1 values(%d, %d, %d, %d, %d, true, 'binary4.%s', 'nchar4.%s', %f, %f, %d)''' + % (self.ts + 400 + i, i, i, i, i, i, i, i, i, self.ts + i)) + tdSql.execute('''insert into regular_table_1 values(%d, %d, %d, %d, %d, false, 'binary5.%s', 'nchar5.%s', %f, %f, %d)''' + % (self.ts + 500 + i, i, i, i, i, i, i, i, i, self.ts + i)) + + tdLog.info("========== operator=1(OP_TableScan) ==========") + tdLog.info("========== operator=7(OP_Project) ==========") + sql = '''select * from stable_1''' + tdSql.query(sql) + tdSql.checkRows(6*self.num) + sql = '''select * from regular_table_1''' + tdSql.query(sql) + tdSql.checkRows(6*self.num) + + tdLog.info("========== operator=14(OP_MultiTableAggregate ) ==========") + sql = '''select last_row(*) from stable_1;''' + tdSql.query(sql) + tdSql.checkData(0,1,self.num-1) + + tdLog.info("========== operator=6(OP_Aggregate) ==========") + sql = '''select last_row(*) from regular_table_1;''' + tdSql.query(sql) + tdSql.checkData(0,1,self.num-1) + + tdLog.info("========== operator=9(OP_Limit) ==========") + sql = '''select * from stable_1 where loc = 'table_0' limit 5;''' + tdSql.query(sql) + tdSql.checkRows(5) + sql = '''select last_row(*) from (select * from stable_1 where loc = 'table_0');''' + tdSql.query(sql) + tdSql.checkRows(1) + + sql = '''select * from regular_table_1 ;''' + tdSql.query(sql) + tdSql.checkRows(6*self.num) + sql = '''select last_row(*) from (select * from regular_table_1);''' + tdSql.query(sql) + tdSql.checkRows(1) + tdSql.checkData(0,1,self.num-1) + + + sql = '''select last_row(*) from + ((select * from table_0) union all + (select * from table_1) union all + (select * from table_2));''' + tdSql.error(sql) + + tdLog.info("========== operator=16(OP_DummyInput) ==========") + sql = '''select last_row(*) from + ((select last_row(*) from table_0) union all + (select last_row(*) from table_1) union all + (select last_row(*) from table_2));''' + tdSql.error(sql) + + sql = '''select last_row(*) from + ((select * from table_0 limit 5 offset 5) union all + (select * from table_1 limit 5 offset 5) union all + (select * from regular_table_1 limit 5 offset 5));''' + tdSql.error(sql) + + tdLog.info("========== operator=10(OP_SLimit) ==========") + sql = '''select count(*) from stable_1 group by loc slimit 3 soffset 2 ;''' + tdSql.query(sql) + tdSql.checkRows(3) + + sql = '''select last_row(*) from + ((select * from table_0) union all + (select * from table_1) union all + (select * from table_2));''' + tdSql.error(sql) + + tdLog.info("========== operator=20(OP_Distinct) ==========") + tdLog.info("========== operator=4(OP_TagScan) ==========") + sql = '''select distinct(t_bool) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(2) + sql = '''select distinct(loc) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(6) + sql = '''select distinct(t_int) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(6) + sql = '''select distinct(t_bigint) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(6) + sql = '''select distinct(t_smallint) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(6) + sql = '''select distinct(t_tinyint) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(6) + sql = '''select distinct(t_binary) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(6) + sql = '''select distinct(t_nchar) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(6) + sql = '''select distinct(t_float) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(6) + sql = '''select distinct(t_double) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(6) + sql = '''select distinct(t_ts) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(3) + sql = '''select distinct(tbname) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(6) + + tdLog.info("========== operator=2(OP_DataBlocksOptScan) ==========") + sql = '''select last(q_int),first(q_int) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_bigint),first(q_bigint) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_smallint),first(q_smallint) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_tinyint),first(q_tinyint) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_bool),first(q_bool) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_binary),first(q_binary) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_nchar),first(q_nchar) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_float),first(q_float) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_double),first(q_double) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_ts),first(q_ts) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_int),last(q_bigint), last(q_smallint),last(q_tinyint),last(q_bool),last(q_binary),last(q_nchar), + last(q_float),last(q_double),last(q_ts),first(q_int),first(q_bigint),first(q_smallint),first(q_tinyint), + first(q_bool),first(q_binary),first(q_nchar),first(q_float),first(q_float),first(q_double),first(q_ts) from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_int),last(q_bigint), last(q_smallint),last(q_tinyint),last(q_bool),last(q_binary),last(q_nchar), + last(q_float),last(q_double),last(q_ts),first(q_int),first(q_bigint),first(q_smallint),first(q_tinyint),first(q_bool), + first(q_binary),first(q_nchar),first(q_float),first(q_float),first(q_double),first(q_ts) from regular_table_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + + tdLog.info("========== operator=8(OP_Groupby) ==========") + sql = '''select stddev(q_int) from table_0 group by q_int;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select stddev(q_int),stddev(q_bigint),stddev(q_smallint),stddev(q_tinyint),stddev(q_float),stddev(q_double) from stable_1 group by q_int;''' + tdSql.query(sql) + sql = '''select stddev(q_int),stddev(q_bigint),stddev(q_smallint),stddev(q_tinyint),stddev(q_float),stddev(q_double) from table_1 group by q_bigint;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select stddev(q_int),stddev(q_bigint),stddev(q_smallint),stddev(q_tinyint),stddev(q_float),stddev(q_double) from regular_table_1 group by q_smallint;''' + tdSql.query(sql) + + tdLog.info("========== operator=11(OP_TimeWindow) ==========") + sql = '''select last(q_int) from table_0 interval(1m);''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_int),last(q_bigint), last(q_smallint),last(q_tinyint), + first(q_int),first(q_bigint),first(q_smallint),first(q_tinyint) from table_1 interval(1m);''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_int),last(q_bigint), last(q_smallint),last(q_tinyint), + first(q_int),first(q_bigint),first(q_smallint),first(q_tinyint) from stable_1 interval(1m);''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select last(q_int),last(q_bigint), last(q_smallint),last(q_tinyint), + first(q_int),first(q_bigint),first(q_smallint),first(q_tinyint) from regular_table_1 interval(1m);''' + tdSql.query(sql) + tdSql.checkRows(1) + + tdLog.info("========== operator=12(OP_SessionWindow) ==========") + sql = '''select count(*) from table_1 session(ts,1s);''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select count(*) from regular_table_1 session(ts,1s);''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select count(*),sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from table_1 session(ts,1s);''' + tdSql.query(sql) + tdSql.checkRows(1) + sql = '''select count(*),sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from regular_table_1 session(ts,1s);''' + tdSql.query(sql) + tdSql.checkRows(1) + + tdLog.info("========== operator=13(OP_Fill) ==========") + sql = '''select sum(q_int) from table_0 + where ts >='1970-10-01 00:00:00' and ts <=now interval(1n) fill(NULL);''' + tdSql.query(sql) + tdSql.checkData(0,1,'None') + sql = '''select sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from stable_1 where ts >='1970-10-01 00:00:00' and ts <=now interval(1n) fill(NULL);''' + tdSql.query(sql) + tdSql.checkData(0,1,'None') + sql = '''select sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from regular_table_1 where ts >='1970-10-01 00:00:00' and ts <=now interval(1n) fill(NULL);''' + tdSql.query(sql) + tdSql.checkData(0,1,'None') + sql = '''select sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from table_0 where ts >='1970-10-01 00:00:00' and ts <=now interval(1n) fill(NULL);''' + tdSql.query(sql) + tdSql.checkData(0,1,'None') + #TD-5190 + sql = '''select sum(q_tinyint),stddev(q_float) from stable_1 + where ts >='1970-10-01 00:00:00' and ts <=now interval(1n) fill(NULL);''' + tdSql.query(sql) + tdSql.checkData(0,1,'None') + + tdLog.info("========== operator=15(OP_MultiTableTimeInterval) ==========") + sql = '''select avg(q_int) from stable_1 where ts=0;''' + tdSql.query(sql) + tdSql.checkData(0,0,'table_0') + sql = '''select loc, sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from stable_1 group by loc having sum(q_int)>=0;''' + tdSql.query(sql) + tdSql.checkData(0,0,'table_0') + sql = '''select loc, sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from stable_1 group by loc having avg(q_int)>=0;''' + tdSql.query(sql) + tdSql.checkData(0,0,'table_0') + sql = '''select loc, sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from stable_1 group by loc having min(q_int)>=0;''' + tdSql.query(sql) + tdSql.checkData(0,0,'table_0') + sql = '''select loc, sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from stable_1 group by loc having max(q_int)>=0;''' + tdSql.query(sql) + tdSql.checkData(0,0,'table_0') + sql = '''select loc, sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from stable_1 group by loc having first(q_int)>=0;''' + tdSql.query(sql) + tdSql.checkData(0,0,'table_0') + sql = '''select loc, sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from stable_1 group by loc having last(q_int)>=0;''' + tdSql.query(sql) + tdSql.checkData(0,0,'table_0') + + tdLog.info("========== operator=21(OP_Join) ==========") + sql = '''select t1.q_int,t2.q_int from + (select ts,q_int from table_1) t1 , (select ts,q_int from table_2) t2 + where t2.ts = t1.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select t1.*,t2.* from + (select * from table_1) t1 , (select * from table_2) t2 + where t2.ts = t1.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select t1.*,t2.* from + (select * from regular_table_1) t1 , (select * from table_0) t2 + where t2.ts = t1.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select t1.*,t2.* from + (select * from stable_1) t1 , (select * from table_2) t2 + where t2.ts = t1.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select t1.*,t2.* from + (select * from regular_table_1) t1 , (select * from stable_1) t2 + where t2.ts = t1.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select t1.*,t2.*,t3.* from + (select * from regular_table_1) t1 , (select * from stable_1) t2, (select * from table_0) t3 + where t2.ts = t1.ts and t3.ts = t1.ts and t2.ts = t3.ts;''' + tdSql.query(sql) + tdSql.checkRows(self.num) + + tdLog.info("========== operator=22(OP_StateWindow) ==========") + sql = '''select avg(q_int),sum(q_smallint) from table_1 state_window(q_int);''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from table_1 state_window(q_bigint);''' + tdSql.query(sql) + tdSql.checkRows(self.num) + sql = '''select sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), + sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), + sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), + sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), + sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), + sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) + from regular_table_1 state_window(q_smallint);''' + tdSql.query(sql) + tdSql.checkRows(6*self.num) + + endTime = time.time() + print("total time %ds" % (endTime - startTime)) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 8b6d976e86db51c9106c054747bc48aacc243b05 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Wed, 28 Jul 2021 14:05:16 +0800 Subject: [PATCH 064/106] [TD-5074]:test operator cost --- tests/pytest/query/operator.py | 539 --------------------------------- 1 file changed, 539 deletions(-) delete mode 100644 tests/pytest/query/operator.py diff --git a/tests/pytest/query/operator.py b/tests/pytest/query/operator.py deleted file mode 100644 index b94d5fa3b3..0000000000 --- a/tests/pytest/query/operator.py +++ /dev/null @@ -1,539 +0,0 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- - -import sys -import taos -from util.log import tdLog -from util.cases import tdCases -from util.sql import tdSql -import random -import time - - -class TDTestCase: - def init(self, conn, logSql): - tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), logSql) - - self.ts = 1600000000000 - self.num = 10 - - def run(self): - tdSql.prepare() - # test case for https://jira.taosdata.com:18080/browse/TD-5074 - - startTime = time.time() - - tdSql.execute('''create stable stable_1 - (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint, - q_bool bool , q_binary binary(20) , q_nchar nchar(20) , - q_float float , q_double double , q_ts timestamp) - tags(loc nchar(20) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, - t_bool bool , t_binary binary(20) , t_nchar nchar(20) , - t_float float , t_double double , t_ts timestamp);''') - tdSql.execute('''create stable stable_2 - (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint, - q_bool bool , q_binary binary(20) , q_nchar nchar(20) , - q_float float , q_double double , q_ts timestamp) - tags(loc nchar(20) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, - t_bool bool , t_binary binary(20) , t_nchar nchar(20) , - t_float float , t_double double , t_ts timestamp);''') - tdSql.execute('''create table table_0 using stable_1 - tags('table_0' , '0' , '0' , '0' , '0' , 0 , '0' , '0' , '0' , '0' ,'0')''') - tdSql.execute('''create table table_1 using stable_1 - tags('table_1' , '2147483647' , '9223372036854775807' , '32767' , '127' , 1 , - 'binary1' , 'nchar1' , '1' , '11' , \'1999-09-09 09:09:09.090\')''') - tdSql.execute('''create table table_2 using stable_1 - tags('table_2' , '-2147483647' , '-9223372036854775807' , '-32767' , '-127' , false , - 'binary2' , 'nchar2nchar2' , '-2.2' , '-22.22' , \'2099-09-09 09:09:09.090\')''') - tdSql.execute('''create table table_3 using stable_1 - tags('table_3' , '3' , '3' , '3' , '3' , true , 'binary3' , 'nchar3' , '33.33' , '3333.3333' , '0')''') - tdSql.execute('''create table table_4 using stable_1 - tags('table_4' , '4' , '4' , '4' , '4' , false , 'binary4' , 'nchar4' , '-444.444' , '-444444.444444' , '0')''') - tdSql.execute('''create table table_5 using stable_1 - tags('table_5' , '5' , '5' , '5' , '5' , true , 'binary5' , 'nchar5' , '5555.5555' , '55555555.55555555' , '0')''') - tdSql.execute('''create table table_21 using stable_2 - tags('table_5' , '5' , '5' , '5' , '5' , true , 'binary5' , 'nchar5' , '5555.5555' , '55555555.55555555' , '0')''') - #regular table - tdSql.execute('''create table regular_table_1 - (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint, - q_bool bool , q_binary binary(20) , q_nchar nchar(20) , - q_float float , q_double double , q_ts timestamp) ;''') - - for i in range(self.num): - tdSql.execute('''insert into table_0 values(%d, %d, %d, %d, %d, 0, 'binary.%s', 'nchar.%s', %f, %f, %d)''' - % (self.ts + i, i, i, i, i, i, i, i, i, self.ts + i)) - tdSql.execute('''insert into table_1 values(%d, %d, %d, %d, %d, 1, 'binary1.%s', 'nchar1.%s', %f, %f, %d)''' - % (self.ts + i, 2147483647-i, 9223372036854775807-i, 32767-i, 127-i, - i, i, random.random(), random.random(), 1262304000001 + i)) - tdSql.execute('''insert into table_2 values(%d, %d, %d, %d, %d, true, 'binary2.%s', 'nchar2nchar2.%s', %f, %f, %d)''' - % (self.ts + i, -2147483647+i, -9223372036854775807+i, -32767+i, -127+i, - i, i, random.uniform(-1,0), random.uniform(-1,0), 1577836800001 + i)) - tdSql.execute('''insert into table_3 values(%d, %d, %d, %d, %d, false, 'binary3.%s', 'nchar3.%s', %f, %f, %d)''' - % (self.ts + i, random.randint(-2147483647, 2147483647), - random.randint(-9223372036854775807, 9223372036854775807), random.randint(-32767, 32767), - random.randint(-127, 127), random.randint(-100, 100), random.randint(-10000, 10000), - random.uniform(-100000,100000), random.uniform(-1000000000,1000000000), self.ts + i)) - tdSql.execute('''insert into table_4 values(%d, %d, %d, %d, %d, true, 'binary4.%s', 'nchar4.%s', %f, %f, %d)''' - % (self.ts + i, i, i, i, i, i, i, i, i, self.ts + i)) - tdSql.execute('''insert into table_5 values(%d, %d, %d, %d, %d, false, 'binary5.%s', 'nchar5.%s', %f, %f, %d)''' - % (self.ts + i, i, i, i, i, i, i, i, i, self.ts + i)) - tdSql.execute('''insert into table_21 values(%d, %d, %d, %d, %d, false, 'binary5.%s', 'nchar5.%s', %f, %f, %d)''' - % (self.ts + i, i, i, i, i, i, i, i, i, self.ts + i)) - - tdSql.execute('''insert into regular_table_1 values(%d, %d, %d, %d, %d, 0, 'binary.%s', 'nchar.%s', %f, %f, %d)''' - % (self.ts + i, i, i, i, i, i, i, i, i, self.ts + i)) - tdSql.execute('''insert into regular_table_1 values(%d, %d, %d, %d, %d, 1, 'binary1.%s', 'nchar1.%s', %f, %f, %d)''' - % (self.ts + 100 + i, 2147483647-i, 9223372036854775807-i, 32767-i, 127-i, - i, i, random.random(), random.random(), 1262304000001 + i)) - tdSql.execute('''insert into regular_table_1 values(%d, %d, %d, %d, %d, true, 'binary2.%s', 'nchar2nchar2.%s', %f, %f, %d)''' - % (self.ts + 200 + i, -2147483647+i, -9223372036854775807+i, -32767+i, -127+i, - i, i, random.uniform(-1,0), random.uniform(-1,0), 1577836800001 + i)) - tdSql.execute('''insert into regular_table_1 values(%d, %d, %d, %d, %d, false, 'binary3.%s', 'nchar3.%s', %f, %f, %d)''' - % (self.ts + 300 + i, random.randint(-2147483647, 2147483647), - random.randint(-9223372036854775807, 9223372036854775807), random.randint(-32767, 32767), - random.randint(-127, 127), random.randint(-100, 100), random.randint(-10000, 10000), - random.uniform(-100000,100000), random.uniform(-1000000000,1000000000), self.ts + i)) - tdSql.execute('''insert into regular_table_1 values(%d, %d, %d, %d, %d, true, 'binary4.%s', 'nchar4.%s', %f, %f, %d)''' - % (self.ts + 400 + i, i, i, i, i, i, i, i, i, self.ts + i)) - tdSql.execute('''insert into regular_table_1 values(%d, %d, %d, %d, %d, false, 'binary5.%s', 'nchar5.%s', %f, %f, %d)''' - % (self.ts + 500 + i, i, i, i, i, i, i, i, i, self.ts + i)) - - tdLog.info("========== operator=1(OP_TableScan) ==========") - tdLog.info("========== operator=7(OP_Project) ==========") - sql = '''select * from stable_1''' - tdSql.query(sql) - tdSql.checkRows(6*self.num) - sql = '''select * from regular_table_1''' - tdSql.query(sql) - tdSql.checkRows(6*self.num) - - tdLog.info("========== operator=14(OP_MultiTableAggregate ) ==========") - sql = '''select last_row(*) from stable_1;''' - tdSql.query(sql) - tdSql.checkData(0,1,self.num-1) - - tdLog.info("========== operator=6(OP_Aggregate) ==========") - sql = '''select last_row(*) from regular_table_1;''' - tdSql.query(sql) - tdSql.checkData(0,1,self.num-1) - - tdLog.info("========== operator=9(OP_Limit) ==========") - sql = '''select * from stable_1 where loc = 'table_0' limit 5;''' - tdSql.query(sql) - tdSql.checkRows(5) - sql = '''select last_row(*) from (select * from stable_1 where loc = 'table_0');''' - tdSql.query(sql) - tdSql.checkRows(1) - - sql = '''select * from regular_table_1 ;''' - tdSql.query(sql) - tdSql.checkRows(6*self.num) - sql = '''select last_row(*) from (select * from regular_table_1);''' - tdSql.query(sql) - tdSql.checkRows(1) - tdSql.checkData(0,1,self.num-1) - - - sql = '''select last_row(*) from - ((select * from table_0) union all - (select * from table_1) union all - (select * from table_2));''' - tdSql.error(sql) - - tdLog.info("========== operator=16(OP_DummyInput) ==========") - sql = '''select last_row(*) from - ((select last_row(*) from table_0) union all - (select last_row(*) from table_1) union all - (select last_row(*) from table_2));''' - tdSql.error(sql) - - sql = '''select last_row(*) from - ((select * from table_0 limit 5 offset 5) union all - (select * from table_1 limit 5 offset 5) union all - (select * from regular_table_1 limit 5 offset 5));''' - tdSql.error(sql) - - tdLog.info("========== operator=10(OP_SLimit) ==========") - sql = '''select count(*) from stable_1 group by loc slimit 3 soffset 2 ;''' - tdSql.query(sql) - tdSql.checkRows(3) - - sql = '''select last_row(*) from - ((select * from table_0) union all - (select * from table_1) union all - (select * from table_2));''' - tdSql.error(sql) - - tdLog.info("========== operator=20(OP_Distinct) ==========") - tdLog.info("========== operator=4(OP_TagScan) ==========") - sql = '''select distinct(t_bool) from stable_1;''' - tdSql.query(sql) - tdSql.checkRows(2) - sql = '''select distinct(loc) from stable_1;''' - tdSql.query(sql) - tdSql.checkRows(6) - sql = '''select distinct(t_int) from stable_1;''' - tdSql.query(sql) - tdSql.checkRows(6) - sql = '''select distinct(t_bigint) from stable_1;''' - tdSql.query(sql) - tdSql.checkRows(6) - sql = '''select distinct(t_smallint) from stable_1;''' - tdSql.query(sql) - tdSql.checkRows(6) - sql = '''select distinct(t_tinyint) from stable_1;''' - tdSql.query(sql) - tdSql.checkRows(6) - sql = '''select distinct(t_binary) from stable_1;''' - tdSql.query(sql) - tdSql.checkRows(6) - sql = '''select distinct(t_nchar) from stable_1;''' - tdSql.query(sql) - tdSql.checkRows(6) - sql = '''select distinct(t_float) from stable_1;''' - tdSql.query(sql) - tdSql.checkRows(6) - sql = '''select distinct(t_double) from stable_1;''' - tdSql.query(sql) - tdSql.checkRows(6) - sql = '''select distinct(t_ts) from stable_1;''' - tdSql.query(sql) - tdSql.checkRows(3) - sql = '''select distinct(tbname) from stable_1;''' - tdSql.query(sql) - tdSql.checkRows(6) - - tdLog.info("========== operator=2(OP_DataBlocksOptScan) ==========") - sql = '''select last(q_int),first(q_int) from stable_1;''' - tdSql.query(sql) - tdSql.checkRows(1) - sql = '''select last(q_bigint),first(q_bigint) from stable_1;''' - tdSql.query(sql) - tdSql.checkRows(1) - sql = '''select last(q_smallint),first(q_smallint) from stable_1;''' - tdSql.query(sql) - tdSql.checkRows(1) - sql = '''select last(q_tinyint),first(q_tinyint) from stable_1;''' - tdSql.query(sql) - tdSql.checkRows(1) - sql = '''select last(q_bool),first(q_bool) from stable_1;''' - tdSql.query(sql) - tdSql.checkRows(1) - sql = '''select last(q_binary),first(q_binary) from stable_1;''' - tdSql.query(sql) - tdSql.checkRows(1) - sql = '''select last(q_nchar),first(q_nchar) from stable_1;''' - tdSql.query(sql) - tdSql.checkRows(1) - sql = '''select last(q_float),first(q_float) from stable_1;''' - tdSql.query(sql) - tdSql.checkRows(1) - sql = '''select last(q_double),first(q_double) from stable_1;''' - tdSql.query(sql) - tdSql.checkRows(1) - sql = '''select last(q_ts),first(q_ts) from stable_1;''' - tdSql.query(sql) - tdSql.checkRows(1) - sql = '''select last(q_int),last(q_bigint), last(q_smallint),last(q_tinyint),last(q_bool),last(q_binary),last(q_nchar), - last(q_float),last(q_double),last(q_ts),first(q_int),first(q_bigint),first(q_smallint),first(q_tinyint), - first(q_bool),first(q_binary),first(q_nchar),first(q_float),first(q_float),first(q_double),first(q_ts) from stable_1;''' - tdSql.query(sql) - tdSql.checkRows(1) - sql = '''select last(q_int),last(q_bigint), last(q_smallint),last(q_tinyint),last(q_bool),last(q_binary),last(q_nchar), - last(q_float),last(q_double),last(q_ts),first(q_int),first(q_bigint),first(q_smallint),first(q_tinyint),first(q_bool), - first(q_binary),first(q_nchar),first(q_float),first(q_float),first(q_double),first(q_ts) from regular_table_1;''' - tdSql.query(sql) - tdSql.checkRows(1) - - tdLog.info("========== operator=8(OP_Groupby) ==========") - sql = '''select stddev(q_int) from table_0 group by q_int;''' - tdSql.query(sql) - tdSql.checkRows(self.num) - sql = '''select stddev(q_int),stddev(q_bigint),stddev(q_smallint),stddev(q_tinyint),stddev(q_float),stddev(q_double) from stable_1 group by q_int;''' - tdSql.query(sql) - sql = '''select stddev(q_int),stddev(q_bigint),stddev(q_smallint),stddev(q_tinyint),stddev(q_float),stddev(q_double) from table_1 group by q_bigint;''' - tdSql.query(sql) - tdSql.checkRows(self.num) - sql = '''select stddev(q_int),stddev(q_bigint),stddev(q_smallint),stddev(q_tinyint),stddev(q_float),stddev(q_double) from regular_table_1 group by q_smallint;''' - tdSql.query(sql) - - tdLog.info("========== operator=11(OP_TimeWindow) ==========") - sql = '''select last(q_int) from table_0 interval(1m);''' - tdSql.query(sql) - tdSql.checkRows(1) - sql = '''select last(q_int),last(q_bigint), last(q_smallint),last(q_tinyint), - first(q_int),first(q_bigint),first(q_smallint),first(q_tinyint) from table_1 interval(1m);''' - tdSql.query(sql) - tdSql.checkRows(1) - sql = '''select last(q_int),last(q_bigint), last(q_smallint),last(q_tinyint), - first(q_int),first(q_bigint),first(q_smallint),first(q_tinyint) from stable_1 interval(1m);''' - tdSql.query(sql) - tdSql.checkRows(1) - sql = '''select last(q_int),last(q_bigint), last(q_smallint),last(q_tinyint), - first(q_int),first(q_bigint),first(q_smallint),first(q_tinyint) from regular_table_1 interval(1m);''' - tdSql.query(sql) - tdSql.checkRows(1) - - tdLog.info("========== operator=12(OP_SessionWindow) ==========") - sql = '''select count(*) from table_1 session(ts,1s);''' - tdSql.query(sql) - tdSql.checkRows(1) - sql = '''select count(*) from regular_table_1 session(ts,1s);''' - tdSql.query(sql) - tdSql.checkRows(1) - sql = '''select count(*),sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), - sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), - sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), - sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), - sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), - sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) - from table_1 session(ts,1s);''' - tdSql.query(sql) - tdSql.checkRows(1) - sql = '''select count(*),sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), - sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), - sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), - sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), - sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), - sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) - from regular_table_1 session(ts,1s);''' - tdSql.query(sql) - tdSql.checkRows(1) - - tdLog.info("========== operator=13(OP_Fill) ==========") - sql = '''select sum(q_int) from table_0 - where ts >='1970-10-01 00:00:00' and ts <=now interval(1n) fill(NULL);''' - tdSql.query(sql) - tdSql.checkData(0,1,'None') - sql = '''select sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), - sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), - sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), - sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), - sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), - sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) - from stable_1 where ts >='1970-10-01 00:00:00' and ts <=now interval(1n) fill(NULL);''' - tdSql.query(sql) - tdSql.checkData(0,1,'None') - sql = '''select sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), - sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), - sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), - sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), - sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), - sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) - from regular_table_1 where ts >='1970-10-01 00:00:00' and ts <=now interval(1n) fill(NULL);''' - tdSql.query(sql) - tdSql.checkData(0,1,'None') - sql = '''select sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), - sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), - sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), - sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), - sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), - sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) - from table_0 where ts >='1970-10-01 00:00:00' and ts <=now interval(1n) fill(NULL);''' - tdSql.query(sql) - tdSql.checkData(0,1,'None') - #TD-5190 - sql = '''select sum(q_tinyint),stddev(q_float) from stable_1 - where ts >='1970-10-01 00:00:00' and ts <=now interval(1n) fill(NULL);''' - tdSql.query(sql) - tdSql.checkData(0,1,'None') - - tdLog.info("========== operator=15(OP_MultiTableTimeInterval) ==========") - sql = '''select avg(q_int) from stable_1 where ts=0;''' - tdSql.query(sql) - tdSql.checkData(0,0,'table_0') - sql = '''select loc, sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), - sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), - sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), - sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), - sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), - sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) - from stable_1 group by loc having sum(q_int)>=0;''' - tdSql.query(sql) - tdSql.checkData(0,0,'table_0') - sql = '''select loc, sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), - sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), - sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), - sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), - sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), - sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) - from stable_1 group by loc having avg(q_int)>=0;''' - tdSql.query(sql) - tdSql.checkData(0,0,'table_0') - sql = '''select loc, sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), - sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), - sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), - sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), - sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), - sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) - from stable_1 group by loc having min(q_int)>=0;''' - tdSql.query(sql) - tdSql.checkData(0,0,'table_0') - sql = '''select loc, sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), - sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), - sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), - sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), - sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), - sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) - from stable_1 group by loc having max(q_int)>=0;''' - tdSql.query(sql) - tdSql.checkData(0,0,'table_0') - sql = '''select loc, sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), - sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), - sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), - sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), - sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), - sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) - from stable_1 group by loc having first(q_int)>=0;''' - tdSql.query(sql) - tdSql.checkData(0,0,'table_0') - sql = '''select loc, sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), - sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), - sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), - sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), - sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), - sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) - from stable_1 group by loc having last(q_int)>=0;''' - tdSql.query(sql) - tdSql.checkData(0,0,'table_0') - - tdLog.info("========== operator=21(OP_Join) ==========") - sql = '''select t1.q_int,t2.q_int from - (select ts,q_int from table_1) t1 , (select ts,q_int from table_2) t2 - where t2.ts = t1.ts;''' - tdSql.query(sql) - tdSql.checkRows(self.num) - sql = '''select t1.*,t2.* from - (select * from table_1) t1 , (select * from table_2) t2 - where t2.ts = t1.ts;''' - tdSql.query(sql) - tdSql.checkRows(self.num) - sql = '''select t1.*,t2.* from - (select * from regular_table_1) t1 , (select * from table_0) t2 - where t2.ts = t1.ts;''' - tdSql.query(sql) - tdSql.checkRows(self.num) - sql = '''select t1.*,t2.* from - (select * from stable_1) t1 , (select * from table_2) t2 - where t2.ts = t1.ts;''' - tdSql.query(sql) - tdSql.checkRows(self.num) - sql = '''select t1.*,t2.* from - (select * from regular_table_1) t1 , (select * from stable_1) t2 - where t2.ts = t1.ts;''' - tdSql.query(sql) - tdSql.checkRows(self.num) - sql = '''select t1.*,t2.*,t3.* from - (select * from regular_table_1) t1 , (select * from stable_1) t2, (select * from table_0) t3 - where t2.ts = t1.ts and t3.ts = t1.ts and t2.ts = t3.ts;''' - tdSql.query(sql) - tdSql.checkRows(self.num) - - tdLog.info("========== operator=22(OP_StateWindow) ==========") - sql = '''select avg(q_int),sum(q_smallint) from table_1 state_window(q_int);''' - tdSql.query(sql) - tdSql.checkRows(self.num) - sql = '''select sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), - sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), - sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), - sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), - sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), - sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) - from table_1 state_window(q_bigint);''' - tdSql.query(sql) - tdSql.checkRows(self.num) - sql = '''select sum(q_int), avg(q_int), min(q_int), max(q_int), first(q_int), last(q_int), - sum(q_bigint), avg(q_bigint), min(q_bigint), max(q_bigint), first(q_bigint), last(q_bigint), - sum(q_smallint), avg(q_smallint), min(q_smallint), max(q_smallint), first(q_smallint), last(q_smallint), - sum(q_tinyint), avg(q_tinyint), min(q_tinyint), max(q_tinyint), first(q_tinyint), last(q_tinyint), - sum(q_float), avg(q_float), min(q_float), max(q_float), first(q_float), last(q_float), - sum(q_double), avg(q_double), min(q_double), max(q_double), first(q_double), last(q_double) - from regular_table_1 state_window(q_smallint);''' - tdSql.query(sql) - tdSql.checkRows(6*self.num) - - endTime = time.time() - print("total time %ds" % (endTime - startTime)) - - - def stop(self): - tdSql.close() - tdLog.success("%s successfully executed" % __file__) - - -tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 3647f7defcd52e1ce374e6af3c504e50383cc441 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Wed, 28 Jul 2021 14:05:42 +0800 Subject: [PATCH 065/106] [TD-5074]:test operator cost --- tests/pytest/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index eb068b6585..d54c413fba 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -241,7 +241,7 @@ python3 ./test.py -f query/queryStateWindow.py python3 ./test.py -f query/nestedQuery/queryWithOrderLimit.py python3 ./test.py -f query/nestquery_last_row.py python3 ./test.py -f query/queryCnameDisplay.py -python3 ./test.py -f query/operator.py +python3 ./test.py -f query/operator_cost.py python3 test.py -f query/nestedQuery/queryWithSpread.py #stream From e4a67f8046445013895402e47352f55a308a3044 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 28 Jul 2021 14:22:18 +0800 Subject: [PATCH 066/106] update local CI --- tests/Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile index c75427b5f4..fb96b669ff 100644 --- a/tests/Jenkinsfile +++ b/tests/Jenkinsfile @@ -48,7 +48,7 @@ pipeline { } } stage('test_b1') { - agent{label 'master'} + agent{label 'slad2'} steps { pre_test() @@ -62,7 +62,7 @@ pipeline { } stage('test_crash_gen') { - agent{label "slad2"} + agent{label "slad3"} steps { pre_test() sh ''' @@ -141,7 +141,7 @@ pipeline { } stage('test_valgrind') { - agent{label "slad3"} + agent{label "slad4"} steps { pre_test() From 8f0be342a8599ac2eb0fdc349d6f0d9d1078f142 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Wed, 28 Jul 2021 15:06:30 +0800 Subject: [PATCH 067/106] [TD-5074]:test operator cost --- tests/pytest/query/operator_cost.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/pytest/query/operator_cost.py b/tests/pytest/query/operator_cost.py index b94d5fa3b3..774a1e5f42 100644 --- a/tests/pytest/query/operator_cost.py +++ b/tests/pytest/query/operator_cost.py @@ -195,9 +195,6 @@ class TDTestCase: sql = '''select distinct(t_tinyint) from stable_1;''' tdSql.query(sql) tdSql.checkRows(6) - sql = '''select distinct(t_binary) from stable_1;''' - tdSql.query(sql) - tdSql.checkRows(6) sql = '''select distinct(t_nchar) from stable_1;''' tdSql.query(sql) tdSql.checkRows(6) From 0b1c661fff52c9ba301edbab66009b50c226edca Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 28 Jul 2021 16:03:18 +0800 Subject: [PATCH 068/106] [TD-5592]: fix change KV row value coredump --- src/common/inc/tdataformat.h | 3 +- src/common/src/tdataformat.c | 83 +++++++++++++----------------------- 2 files changed, 32 insertions(+), 54 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 829e771c70..f8394dc271 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -55,7 +55,7 @@ extern "C" { typedef struct { int8_t type; // Column type int16_t colId; // column ID - uint16_t bytes; // column bytes + int16_t bytes; // column bytes (restore to int16_t in case of misuse) uint16_t offset; // point offset in SDataRow after the header part. } STColumn; @@ -366,6 +366,7 @@ typedef struct { #define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i)) #define kvRowFree(r) tfree(r) #define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r)) +#define kvRowValLen(r) (kvRowLen(r) - TD_KV_ROW_HEAD_SIZE - sizeof(SColIdx) * kvRowNCols(r)) #define kvRowTKey(r) (*(TKEY *)(kvRowValues(r))) #define kvRowKey(r) tdGetKey(kvRowTKey(r)) #define kvRowDeleted(r) TKEY_IS_DELETED(kvRowTKey(r)) diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 9f7432c90d..5a63588611 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -634,42 +634,28 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { SKVRow nrow = NULL; void * ptr = taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_GE); - if (ptr == NULL || ((SColIdx *)ptr)->colId > colId) { // need to add a column value to the row + if (ptr == NULL || ((SColIdx *)ptr)->colId > colId) { // need to add a column value to the row int diff = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type]; - nrow = malloc(kvRowLen(row) + sizeof(SColIdx) + diff); + int nRowLen = kvRowLen(row) + sizeof(SColIdx) + diff; + int oRowCols = kvRowNCols(row); + + ASSERT(diff > 0); + nrow = malloc(nRowLen); if (nrow == NULL) return -1; - kvRowSetLen(nrow, kvRowLen(row) + (uint16_t)sizeof(SColIdx) + diff); - kvRowSetNCols(nrow, kvRowNCols(row) + 1); + kvRowSetLen(nrow, nRowLen); + kvRowSetNCols(nrow, oRowCols + 1); - if (ptr == NULL) { - memcpy(kvRowColIdx(nrow), kvRowColIdx(row), sizeof(SColIdx) * kvRowNCols(row)); - memcpy(kvRowValues(nrow), kvRowValues(row), POINTER_DISTANCE(kvRowEnd(row), kvRowValues(row))); - int colIdx = kvRowNCols(nrow) - 1; - kvRowColIdxAt(nrow, colIdx)->colId = colId; - kvRowColIdxAt(nrow, colIdx)->offset = (int16_t)(POINTER_DISTANCE(kvRowEnd(row), kvRowValues(row))); - memcpy(kvRowColVal(nrow, kvRowColIdxAt(nrow, colIdx)), value, diff); - } else { - int16_t tlen = (int16_t)(POINTER_DISTANCE(ptr, kvRowColIdx(row))); - if (tlen > 0) { - memcpy(kvRowColIdx(nrow), kvRowColIdx(row), tlen); - memcpy(kvRowValues(nrow), kvRowValues(row), ((SColIdx *)ptr)->offset); - } + memcpy(kvRowColIdx(nrow), kvRowColIdx(row), sizeof(SColIdx) * oRowCols); + memcpy(kvRowValues(nrow), kvRowValues(row), kvRowValLen(row)); - int colIdx = tlen / sizeof(SColIdx); - kvRowColIdxAt(nrow, colIdx)->colId = colId; - kvRowColIdxAt(nrow, colIdx)->offset = ((SColIdx *)ptr)->offset; - memcpy(kvRowColVal(nrow, kvRowColIdxAt(nrow, colIdx)), value, diff); + pColIdx = kvRowColIdxAt(nrow, oRowCols); + pColIdx->colId = colId; + pColIdx->offset = kvRowValLen(row); - for (int i = colIdx; i < kvRowNCols(row); i++) { - kvRowColIdxAt(nrow, i + 1)->colId = kvRowColIdxAt(row, i)->colId; - kvRowColIdxAt(nrow, i + 1)->offset = kvRowColIdxAt(row, i)->offset + diff; - } - memcpy(kvRowColVal(nrow, kvRowColIdxAt(nrow, colIdx + 1)), kvRowColVal(row, kvRowColIdxAt(row, colIdx)), - POINTER_DISTANCE(kvRowEnd(row), kvRowColVal(row, kvRowColIdxAt(row, colIdx))) + memcpy(kvRowColVal(nrow, pColIdx), value, diff); // copy new value - ); - } + tdSortKVRowByColIdx(nrow); *orow = nrow; free(row); @@ -680,9 +666,8 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { if (varDataTLen(value) == varDataTLen(pOldVal)) { // just update the column value in place memcpy(pOldVal, value, varDataTLen(value)); - } else { // need to reallocate the memory - uint16_t diff = varDataTLen(value) - varDataTLen(pOldVal); - uint16_t nlen = kvRowLen(row) + diff; + } else { // need to reallocate the memory + int16_t nlen = kvRowLen(row) + (varDataTLen(value) - varDataTLen(pOldVal)); ASSERT(nlen > 0); nrow = malloc(nlen); if (nrow == NULL) return -1; @@ -690,30 +675,22 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { kvRowSetLen(nrow, nlen); kvRowSetNCols(nrow, kvRowNCols(row)); - // Copy part ahead - nlen = (int16_t)(POINTER_DISTANCE(ptr, kvRowColIdx(row))); - ASSERT(nlen % sizeof(SColIdx) == 0); - if (nlen > 0) { - ASSERT(((SColIdx *)ptr)->offset > 0); - memcpy(kvRowColIdx(nrow), kvRowColIdx(row), nlen); - memcpy(kvRowValues(nrow), kvRowValues(row), ((SColIdx *)ptr)->offset); + int zsize = sizeof(SColIdx) * kvRowNCols(row) + ((SColIdx *)ptr)->offset; + memcpy(kvRowColIdx(nrow), kvRowColIdx(row), zsize); + memcpy(kvRowColVal(nrow, ((SColIdx *)ptr)), value, varDataTLen(value)); + // Copy left value part + int lsize = kvRowLen(row) - TD_KV_ROW_HEAD_SIZE - zsize - varDataTLen(pOldVal); + if (lsize > 0) { + memcpy(POINTER_SHIFT(nrow, TD_KV_ROW_HEAD_SIZE + zsize + varDataTLen(value)), + POINTER_SHIFT(row, TD_KV_ROW_HEAD_SIZE + zsize + varDataTLen(pOldVal)), lsize); } - // Construct current column value - int colIdx = nlen / sizeof(SColIdx); - pColIdx = kvRowColIdxAt(nrow, colIdx); - pColIdx->colId = ((SColIdx *)ptr)->colId; - pColIdx->offset = ((SColIdx *)ptr)->offset; - memcpy(kvRowColVal(nrow, pColIdx), value, varDataTLen(value)); - - // Construct columns after - if (kvRowNCols(nrow) - colIdx - 1 > 0) { - for (int i = colIdx + 1; i < kvRowNCols(nrow); i++) { - kvRowColIdxAt(nrow, i)->colId = kvRowColIdxAt(row, i)->colId; - kvRowColIdxAt(nrow, i)->offset = kvRowColIdxAt(row, i)->offset + diff; + for (int i = 0; i < kvRowNCols(nrow); i++) { + pColIdx = kvRowColIdxAt(nrow, i); + + if (pColIdx->offset > ((SColIdx *)ptr)->offset) { + pColIdx->offset = pColIdx->offset - varDataTLen(pOldVal) + varDataTLen(value); } - memcpy(kvRowColVal(nrow, kvRowColIdxAt(nrow, colIdx + 1)), kvRowColVal(row, kvRowColIdxAt(row, colIdx + 1)), - POINTER_DISTANCE(kvRowEnd(row), kvRowColVal(row, kvRowColIdxAt(row, colIdx + 1)))); } *orow = nrow; From aac0aeddfdf81725e9deb21b79301c3816cbf4bb Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 28 Jul 2021 16:10:06 +0800 Subject: [PATCH 069/106] add SANITIZER parameter --- tests/Jenkinsfile | 29 ++++++++++++++++++++++++++++- tests/mas/Jenkinsfile | 29 ++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile index fb96b669ff..ed54515ba8 100644 --- a/tests/Jenkinsfile +++ b/tests/Jenkinsfile @@ -1,5 +1,32 @@ def pre_test(){ + sh ''' + sudo rmtaos||echo 'no taosd installed' + ''' + sh ''' + cd ${WKC} + git reset --hard + git checkout $BRANCH_NAME + git pull + git submodule update + cd ${WK} + git reset --hard + git checkout $BRANCH_NAME + git pull + export TZ=Asia/Harbin + date + rm -rf ${WK}/debug + mkdir debug + cd debug + cmake -DMEMORY_SANITIZER=true .. > /dev/null + make > /dev/null + make install > /dev/null + pip3 install ${WKC}/src/connector/python + ''' + return 1 +} +def pre_test_p(){ + sh ''' sudo rmtaos||echo 'no taosd installed' ''' @@ -39,7 +66,7 @@ pipeline { stage('pytest') { agent{label 'slad1'} steps { - pre_test() + pre_test_p() sh ''' cd ${WKC}/tests find pytest -name '*'sql|xargs rm -rf diff --git a/tests/mas/Jenkinsfile b/tests/mas/Jenkinsfile index ae2286298f..4dcc1d9c18 100644 --- a/tests/mas/Jenkinsfile +++ b/tests/mas/Jenkinsfile @@ -1,5 +1,32 @@ def pre_test(){ + sh ''' + sudo rmtaos||echo 'no taosd installed' + ''' + sh ''' + cd ${WKC} + git reset --hard + git checkout $BRANCH_NAME + git pull + git submodule update + cd ${WK} + git reset --hard + git checkout $BRANCH_NAME + git pull + export TZ=Asia/Harbin + date + rm -rf ${WK}/debug + mkdir debug + cd debug + cmake -DMEMORY_SANITIZER=true .. > /dev/null + make > /dev/null + make install > /dev/null + pip3 install ${WKC}/src/connector/python/ || echo 0 + ''' + return 1 +} +def pre_test_p(){ + sh ''' sudo rmtaos||echo 'no taosd installed' ''' @@ -39,7 +66,7 @@ pipeline { stage('pytest') { agent{label 'slam1'} steps { - pre_test() + pre_test_p() sh ''' cd ${WKC}/tests find pytest -name '*'sql|xargs rm -rf From 953f72ec1d567628698aaf7449b6c34de1b3bac0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 28 Jul 2021 17:43:50 +0800 Subject: [PATCH 070/106] [td-5600]: fix fill caused client crashed in join query. --- src/client/src/tscUtil.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 749658eca3..773243891f 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1964,6 +1964,7 @@ static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) { pQueryInfo->tsBuf = tsBufDestroy(pQueryInfo->tsBuf); tfree(pQueryInfo->fillVal); + pQueryInfo->fillType = 0; tfree(pQueryInfo->buf); } From fb62d4bb8ece2cf606c86513dc05a437a1c115e9 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 28 Jul 2021 17:53:39 +0800 Subject: [PATCH 071/106] [TD-5591]: taosdemo coredump when query 4096 columns. (#7055) --- src/kit/taosdemo/taosdemo.c | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 5164d1bcc3..8a4b781d3f 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -95,7 +95,6 @@ extern char configDir[]; #define MAX_SUPER_TABLE_COUNT 200 #define MAX_QUERY_SQL_COUNT 100 -#define MAX_QUERY_SQL_LENGTH BUFFER_SIZE #define MAX_DATABASE_COUNT 256 #define INPUT_BUF_LEN 256 @@ -382,7 +381,7 @@ typedef struct SpecifiedQueryInfo_S { uint64_t queryTimes; bool subscribeRestart; int subscribeKeepProgress; - char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; + char sql[MAX_QUERY_SQL_COUNT][BUFFER_SIZE+1]; char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN]; int resubAfterConsume[MAX_QUERY_SQL_COUNT]; int endAfterConsume[MAX_QUERY_SQL_COUNT]; @@ -405,7 +404,7 @@ typedef struct SuperQueryInfo_S { int64_t childTblCount; char childTblPrefix[TSDB_TABLE_NAME_LEN - 20]; // 20 characters reserved for seq int sqlCount; - char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; + char sql[MAX_QUERY_SQL_COUNT][BUFFER_SIZE+1]; char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN]; int resubAfterConsume; int endAfterConsume; @@ -1252,14 +1251,14 @@ static void fetchResult(TAOS_RES *res, threadInfo* pThreadInfo) { // fetch the records row by row while((row = taos_fetch_row(res))) { - if (totalLen >= 100*1024*1024 - 32000) { + if (totalLen >= (100*1024*1024 - HEAD_BUFF_LEN*2)) { if (strlen(pThreadInfo->filePath) > 0) appendResultBufToFile(databuf, pThreadInfo); totalLen = 0; memset(databuf, 0, 100*1024*1024); } num_rows++; - char temp[16000] = {0}; + char temp[HEAD_BUFF_LEN] = {0}; int len = taos_print_row(temp, row, fields, num_fields); len += sprintf(temp + len, "\n"); //printf("query result:%s\n", temp); @@ -2164,15 +2163,15 @@ static void printfDbInfoForQueryToFile( } static void printfQuerySystemInfo(TAOS * taos) { - char filename[MAX_QUERY_SQL_LENGTH+1] = {0}; - char buffer[MAX_QUERY_SQL_LENGTH+1] = {0}; + char filename[BUFFER_SIZE+1] = {0}; + char buffer[BUFFER_SIZE+1] = {0}; TAOS_RES* res; time_t t; struct tm* lt; time(&t); lt = localtime(&t); - snprintf(filename, MAX_QUERY_SQL_LENGTH, "querySystemInfo-%d-%d-%d %d:%d:%d", + snprintf(filename, BUFFER_SIZE, "querySystemInfo-%d-%d-%d %d:%d:%d", lt->tm_year+1900, lt->tm_mon, lt->tm_mday, lt->tm_hour, lt->tm_min, lt->tm_sec); @@ -2204,12 +2203,12 @@ static void printfQuerySystemInfo(TAOS * taos) { printfDbInfoForQueryToFile(filename, dbInfos[i], i); // show db.vgroups - snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.vgroups;", dbInfos[i]->name); + snprintf(buffer, BUFFER_SIZE, "show %s.vgroups;", dbInfos[i]->name); res = taos_query(taos, buffer); xDumpResultToFile(filename, res); // show db.stables - snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.stables;", dbInfos[i]->name); + snprintf(buffer, BUFFER_SIZE, "show %s.stables;", dbInfos[i]->name); res = taos_query(taos, buffer); xDumpResultToFile(filename, res); free(dbInfos[i]); @@ -4529,7 +4528,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { goto PARSE_OVER; } tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j], - sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); + sqlStr->valuestring, BUFFER_SIZE); // default value is -1, which mean infinite loop g_queryInfo.specifiedQueryInfo.endAfterConsume[j] = -1; @@ -4751,7 +4750,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { goto PARSE_OVER; } tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring, - MAX_QUERY_SQL_LENGTH); + BUFFER_SIZE); cJSON *result = cJSON_GetObjectItem(sql, "result"); if (result != NULL && result->type == cJSON_String @@ -7404,14 +7403,14 @@ static void replaceChildTblName(char* inSql, char* outSql, int tblIndex) { tstrncpy(outSql, inSql, pos - inSql + 1); //printf("1: %s\n", outSql); - strncat(outSql, subTblName, MAX_QUERY_SQL_LENGTH - 1); + strncat(outSql, subTblName, BUFFER_SIZE - 1); //printf("2: %s\n", outSql); - strncat(outSql, pos+strlen(sourceString), MAX_QUERY_SQL_LENGTH - 1); + strncat(outSql, pos+strlen(sourceString), BUFFER_SIZE - 1); //printf("3: %s\n", outSql); } static void *superTableQuery(void *sarg) { - char sqlstr[MAX_QUERY_SQL_LENGTH]; + char sqlstr[BUFFER_SIZE]; threadInfo *pThreadInfo = (threadInfo *)sarg; setThreadName("superTableQuery"); @@ -7714,7 +7713,7 @@ static TAOS_SUB* subscribeImpl( static void *superSubscribe(void *sarg) { threadInfo *pThreadInfo = (threadInfo *)sarg; - char subSqlstr[MAX_QUERY_SQL_LENGTH]; + char subSqlstr[BUFFER_SIZE]; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT] = {0}; uint64_t tsubSeq; From a9c51e251a3f3c78a67168e63065a6e12283cb7b Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 28 Jul 2021 18:28:35 +0800 Subject: [PATCH 072/106] remove systemctl --- tests/Jenkinsfile | 4 ++-- tests/mas/Jenkinsfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile index ed54515ba8..b5b068381d 100644 --- a/tests/Jenkinsfile +++ b/tests/Jenkinsfile @@ -119,7 +119,7 @@ pipeline { } sh''' - systemctl start taosd + nohup taosd >/dev/null & sleep 10 ''' catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { @@ -155,7 +155,7 @@ pipeline { ''' } sh ''' - systemctl stop taosd + pkill -9 taosd cd ${WKC}/tests ./test-all.sh b2 date diff --git a/tests/mas/Jenkinsfile b/tests/mas/Jenkinsfile index 4dcc1d9c18..52b29da92e 100644 --- a/tests/mas/Jenkinsfile +++ b/tests/mas/Jenkinsfile @@ -119,7 +119,7 @@ pipeline { } sh''' - systemctl start taosd + nohup taosd >/dev/null & sleep 10 ''' catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { @@ -163,7 +163,7 @@ pipeline { ''' } sh ''' - systemctl stop taosd + pkill -9 taosd cd ${WKC}/tests ./test-all.sh b2 date From 175fd1c93d7b4073a0ffe93e4ea236f1cdb32235 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Wed, 28 Jul 2021 18:32:34 +0800 Subject: [PATCH 073/106] [TD-5074]:test operator cost --- tests/pytest/query/operator_cost.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/pytest/query/operator_cost.py b/tests/pytest/query/operator_cost.py index 774a1e5f42..27de3531eb 100644 --- a/tests/pytest/query/operator_cost.py +++ b/tests/pytest/query/operator_cost.py @@ -207,9 +207,9 @@ class TDTestCase: sql = '''select distinct(t_ts) from stable_1;''' tdSql.query(sql) tdSql.checkRows(3) - sql = '''select distinct(tbname) from stable_1;''' - tdSql.query(sql) - tdSql.checkRows(6) + # sql = '''select distinct(tbname) from stable_1;''' + # tdSql.query(sql) + # tdSql.checkRows(6) tdLog.info("========== operator=2(OP_DataBlocksOptScan) ==========") sql = '''select last(q_int),first(q_int) from stable_1;''' From f97784ad7a2c5f5b6d48a60c33ff6c12ff39e728 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 29 Jul 2021 09:36:57 +0800 Subject: [PATCH 074/106] [TD-5536]update local jenkins --- tests/Jenkinsfile | 83 ++++++++++++++++++++++++++----------------- tests/mas/Jenkinsfile | 33 +++++++++++++++-- 2 files changed, 81 insertions(+), 35 deletions(-) diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile index a16b8fc1b6..b5b068381d 100644 --- a/tests/Jenkinsfile +++ b/tests/Jenkinsfile @@ -1,5 +1,32 @@ def pre_test(){ + sh ''' + sudo rmtaos||echo 'no taosd installed' + ''' + sh ''' + cd ${WKC} + git reset --hard + git checkout $BRANCH_NAME + git pull + git submodule update + cd ${WK} + git reset --hard + git checkout $BRANCH_NAME + git pull + export TZ=Asia/Harbin + date + rm -rf ${WK}/debug + mkdir debug + cd debug + cmake -DMEMORY_SANITIZER=true .. > /dev/null + make > /dev/null + make install > /dev/null + pip3 install ${WKC}/src/connector/python + ''' + return 1 +} +def pre_test_p(){ + sh ''' sudo rmtaos||echo 'no taosd installed' ''' @@ -39,7 +66,7 @@ pipeline { stage('pytest') { agent{label 'slad1'} steps { - pre_test() + pre_test_p() sh ''' cd ${WKC}/tests find pytest -name '*'sql|xargs rm -rf @@ -48,7 +75,7 @@ pipeline { } } stage('test_b1') { - agent{label 'master'} + agent{label 'slad2'} steps { pre_test() @@ -62,7 +89,7 @@ pipeline { } stage('test_crash_gen') { - agent{label "slad2"} + agent{label "slad3"} steps { pre_test() sh ''' @@ -92,7 +119,7 @@ pipeline { } sh''' - systemctl start taosd + nohup taosd >/dev/null & sleep 10 ''' catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { @@ -110,16 +137,8 @@ pipeline { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh ''' cd ${WKC}/tests/examples/JDBC/JDBCDemo/ - mvn clean package assembly:single -DskipTests >/dev/null - java -jar target/JDBCDemo-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 - ''' - } - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - cd ${WKC}/src/connector/jdbc - mvn clean package -Dmaven.test.skip=true >/dev/null - cd ${WKC}/tests/examples/JDBC/JDBCDemo/ - java --class-path=../../../../src/connector/jdbc/target:$JAVA_HOME/jre/lib/ext -jar target/JDBCDemo-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 + mvn clean package >/dev/null + java -jar target/JdbcRestfulDemo-jar-with-dependencies.jar ''' } catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { @@ -136,7 +155,7 @@ pipeline { ''' } sh ''' - systemctl stop taosd + pkill -9 taosd cd ${WKC}/tests ./test-all.sh b2 date @@ -149,7 +168,7 @@ pipeline { } stage('test_valgrind') { - agent{label "slad3"} + agent{label "slad4"} steps { pre_test() @@ -228,11 +247,11 @@ pipeline { } } - post { + post { success { emailext ( - subject: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", - body: ''' + subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' SUCCESS", + body: """ @@ -248,29 +267,29 @@ pipeline {
    -
  • 构建ĺŤç§°>>ĺ†ć”ŻďĽš${PROJECT_NAME}
    • +
  • 构建ĺŤç§°>>ĺ†ć”ŻďĽš${env.BRANCH_NAME}
  • 构建结果： Successful
  • 构建编号：${BUILD_NUMBER}
    • -
  • 触发用ć·ďĽš${CAUSE}
    • -
  • ĺŹć›´ć¦‚č¦ďĽš${CHANGES}
    • +
  • 触发用ć·ďĽš${env.CHANGE_AUTHOR}
    • +
  • ćŹäş¤äżˇćŻďĽš${env.CHANGE_TITLE}
  • 构建地址：${BUILD_URL}
  • 构建日志：${BUILD_URL}console
    • -
  • ĺŹć›´é›†ďĽš${JELLY_SCRIPT}
    • +
- ''', + """, to: "yqliu@taosdata.com,pxiao@taosdata.com", from: "support@taosdata.com" ) } failure { emailext ( - subject: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", - body: ''' + subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' FAIL", + body: """ @@ -286,21 +305,21 @@ pipeline {
    -
  • 构建ĺŤç§°>>ĺ†ć”ŻďĽš${PROJECT_NAME}
    • -
  • 构建结果： Successful
    • +
  • 构建ĺŤç§°>>ĺ†ć”ŻďĽš${env.BRANCH_NAME}
    • +
  • 构建结果： Failure
  • 构建编号：${BUILD_NUMBER}
    • -
  • 触发用ć·ďĽš${CAUSE}
    • -
  • ĺŹć›´ć¦‚č¦ďĽš${CHANGES}
    • +
  • 触发用ć·ďĽš${env.CHANGE_AUTHOR}
    • +
  • ćŹäş¤äżˇćŻďĽš${env.CHANGE_TITLE}
  • 构建地址：${BUILD_URL}
  • 构建日志：${BUILD_URL}console
    • -
  • ĺŹć›´é›†ďĽš${JELLY_SCRIPT}
    • +
- ''', + """, to: "yqliu@taosdata.com,pxiao@taosdata.com", from: "support@taosdata.com" ) diff --git a/tests/mas/Jenkinsfile b/tests/mas/Jenkinsfile index ae2286298f..52b29da92e 100644 --- a/tests/mas/Jenkinsfile +++ b/tests/mas/Jenkinsfile @@ -1,5 +1,32 @@ def pre_test(){ + sh ''' + sudo rmtaos||echo 'no taosd installed' + ''' + sh ''' + cd ${WKC} + git reset --hard + git checkout $BRANCH_NAME + git pull + git submodule update + cd ${WK} + git reset --hard + git checkout $BRANCH_NAME + git pull + export TZ=Asia/Harbin + date + rm -rf ${WK}/debug + mkdir debug + cd debug + cmake -DMEMORY_SANITIZER=true .. > /dev/null + make > /dev/null + make install > /dev/null + pip3 install ${WKC}/src/connector/python/ || echo 0 + ''' + return 1 +} +def pre_test_p(){ + sh ''' sudo rmtaos||echo 'no taosd installed' ''' @@ -39,7 +66,7 @@ pipeline { stage('pytest') { agent{label 'slam1'} steps { - pre_test() + pre_test_p() sh ''' cd ${WKC}/tests find pytest -name '*'sql|xargs rm -rf @@ -92,7 +119,7 @@ pipeline { } sh''' - systemctl start taosd + nohup taosd >/dev/null & sleep 10 ''' catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { @@ -136,7 +163,7 @@ pipeline { ''' } sh ''' - systemctl stop taosd + pkill -9 taosd cd ${WKC}/tests ./test-all.sh b2 date From 21bdbb9e507a496cd8e7b68b8f2a67ed35fd6be6 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Thu, 29 Jul 2021 10:32:20 +0800 Subject: [PATCH 075/106] [TD-3666] : fix description about binary storage length. --- documentation20/cn/12.taos-sql/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index 72f4876dcf..6d39c25565 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -48,7 +48,7 @@ TDengine 缺çśçš„时间ćłćŻćŻ«ç§’精度，但通过在 CREATE DATABASE 时传 | 3 | BIGINT | 8 | 长整型，čŚĺ›´ [-2^63+1, 2^63-1], -2^63 用于 NULL | | 4 | FLOAT | 4 | 浮点型，有ć•ä˝Ťć•° 6-7，čŚĺ›´ [-3.4E38, 3.4E38] | | 5 | DOUBLE | 8 | 双精度浮点型，有ć•ä˝Ťć•° 15-16，čŚĺ›´ [-1.7E308, 1.7E308] | -| 6 | BINARY | 自定义 | č®°ĺ˝•ĺŤ•ĺ­—čŠ‚ĺ­—ç¬¦ä¸˛ďĽŚĺ»şč®®ĺŹŞç”¨äşŽĺ¤„ç† ASCII 可č§ĺ­—符，中文等多字节字符需使用 nchar。ç†č®şä¸ŠďĽŚćś€é•żĺŹŻä»Ąćś‰ 16374 字节，但由于每行数据最多 16K 字节，实际上é™ä¸€č¬ĺ°ŹäşŽç†č®şĺ€Ľă€‚binary 仅支ćŚĺ­—符串输入，字符串两端需使用单引号引用。使用时须指定大小，如 binary(20) 定义了最长为 20 个单字节字符的字符串，每个字符占 1 byte çš„ĺ­ĺ‚¨ç©şé—´ďĽŚć­¤ć—¶ĺ¦‚果用ć·ĺ­—符串超出 20 字节将会报错。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示，即 `\’`。 | +| 6 | BINARY | 自定义 | č®°ĺ˝•ĺŤ•ĺ­—čŠ‚ĺ­—ç¬¦ä¸˛ďĽŚĺ»şč®®ĺŹŞç”¨äşŽĺ¤„ç† ASCII 可č§ĺ­—符，中文等多字节字符需使用 nchar。ç†č®şä¸ŠďĽŚćś€é•żĺŹŻä»Ąćś‰ 16374 字节，但由于每行数据最多 16K 字节，实际上é™ä¸€č¬ĺ°ŹäşŽç†č®şĺ€Ľă€‚binary 仅支ćŚĺ­—符串输入，字符串两端需使用单引号引用。使用时须指定大小，如 binary(20) 定义了最长为 20 个单字节字符的字符串，每个字符占 1 byte çš„ĺ­ĺ‚¨ç©şé—´ďĽŚć€»ĺ…±ĺ›şĺ®šĺŤ ç”¨ 20 bytes 的空间，此时如果用ć·ĺ­—符串超出 20 字节将会报错。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示，即 `\’`。 | | 7 | SMALLINT | 2 | 短整型， čŚĺ›´ [-32767, 32767], -32768 用于 NULL | | 8 | TINYINT | 1 | 单字节整型，čŚĺ›´ [-127, 127], -128 用于 NULL | | 9 | BOOL | 1 | ĺ¸ĺ°”型，{true, false} | From c7ae4f57893a4e3af2b511b9ad0cac9d9b937ee2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 29 Jul 2021 10:47:09 +0800 Subject: [PATCH 076/106] [td-5600]: fix fill caused crashed in join query. --- src/client/src/tscUtil.c | 1 + src/os/src/detail/osTimer.c | 2 +- tests/script/general/parser/join.sim | 4 ++++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 9dc541b1a6..b9a9333fd7 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -3190,6 +3190,7 @@ static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) { } pQueryInfo->tsBuf = tsBufDestroy(pQueryInfo->tsBuf); + pQueryInfo->fillType = 0; tfree(pQueryInfo->fillVal); tfree(pQueryInfo->buf); diff --git a/src/os/src/detail/osTimer.c b/src/os/src/detail/osTimer.c index c381b3e825..618df8a8ba 100644 --- a/src/os/src/detail/osTimer.c +++ b/src/os/src/detail/osTimer.c @@ -38,7 +38,7 @@ static void *taosProcessAlarmSignal(void *tharg) { struct sigevent sevent = {{0}}; - setThreadName("alarmSignal"); + setThreadName("tmr"); #ifdef _ALPINE sevent.sigev_notify = SIGEV_THREAD; diff --git a/tests/script/general/parser/join.sim b/tests/script/general/parser/join.sim index 2c14a86c3a..e2132589bd 100644 --- a/tests/script/general/parser/join.sim +++ b/tests/script/general/parser/join.sim @@ -444,6 +444,10 @@ if $rows != $val then return -1 endi +print ================>TD-5600 +sql select first(join_tb0.c8),first(join_tb0.c9) from join_tb1 , join_tb0 where join_tb1.ts = join_tb0.ts and join_tb1.ts <= 100002 and join_tb1.ts>=100000 interval(1s) fill(linear); + + #=============================================================== sql select first(join_tb0.c8),first(join_tb0.c9) from join_tb1 , join_tb0 where join_tb1.ts = join_tb0.ts and join_tb1.ts <= 100002 and join_tb0.c7 = true From f4b07ce5d97e38fb6aada62e24786fb969f19965 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 29 Jul 2021 11:05:59 +0800 Subject: [PATCH 077/106] revert sani --- tests/Jenkinsfile | 2 +- tests/mas/Jenkinsfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile index b5b068381d..3185d9ddba 100644 --- a/tests/Jenkinsfile +++ b/tests/Jenkinsfile @@ -18,7 +18,7 @@ def pre_test(){ rm -rf ${WK}/debug mkdir debug cd debug - cmake -DMEMORY_SANITIZER=true .. > /dev/null + cmake .. > /dev/null make > /dev/null make install > /dev/null pip3 install ${WKC}/src/connector/python diff --git a/tests/mas/Jenkinsfile b/tests/mas/Jenkinsfile index 52b29da92e..e44df002ef 100644 --- a/tests/mas/Jenkinsfile +++ b/tests/mas/Jenkinsfile @@ -18,7 +18,7 @@ def pre_test(){ rm -rf ${WK}/debug mkdir debug cd debug - cmake -DMEMORY_SANITIZER=true .. > /dev/null + cmake .. > /dev/null make > /dev/null make install > /dev/null pip3 install ${WKC}/src/connector/python/ || echo 0 From c95ad354387d1a5039919a387620bbb06e837399 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 29 Jul 2021 11:07:06 +0800 Subject: [PATCH 078/106] revert sani --- tests/Jenkinsfile | 2 +- tests/mas/Jenkinsfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile index b5b068381d..3185d9ddba 100644 --- a/tests/Jenkinsfile +++ b/tests/Jenkinsfile @@ -18,7 +18,7 @@ def pre_test(){ rm -rf ${WK}/debug mkdir debug cd debug - cmake -DMEMORY_SANITIZER=true .. > /dev/null + cmake .. > /dev/null make > /dev/null make install > /dev/null pip3 install ${WKC}/src/connector/python diff --git a/tests/mas/Jenkinsfile b/tests/mas/Jenkinsfile index 52b29da92e..e44df002ef 100644 --- a/tests/mas/Jenkinsfile +++ b/tests/mas/Jenkinsfile @@ -18,7 +18,7 @@ def pre_test(){ rm -rf ${WK}/debug mkdir debug cd debug - cmake -DMEMORY_SANITIZER=true .. > /dev/null + cmake .. > /dev/null make > /dev/null make install > /dev/null pip3 install ${WKC}/src/connector/python/ || echo 0 From f77874a5f4f9d4822f6a84d1502468dd77db3648 Mon Sep 17 00:00:00 2001 From: markswang <792637585@qq.com> Date: Thu, 29 Jul 2021 13:14:52 +0800 Subject: [PATCH 079/106] [TD-5534]:fix the coverity high risk of client --- src/client/src/tscSQLParser.c | 2 +- src/client/src/tscServer.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index b7d513cfbe..7d4434a922 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -8376,7 +8376,7 @@ static int32_t doValidateSubquery(SSqlNode* pSqlNode, int32_t index, SSqlObj* pS return invalidOperationMsg(msgBuf, "subquery alias name too long"); } - tstrncpy(pTableMetaInfo1->aliasName, subInfo->aliasName.z, MIN(subInfo->aliasName.n, sizeof(pTableMetaInfo1->aliasName))); + tstrncpy(pTableMetaInfo1->aliasName, subInfo->aliasName.z, sizeof(pTableMetaInfo1->aliasName)); } taosArrayPush(pQueryInfo->pUpstream, &pSub); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 36d3d156db..01c1e4a4a4 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -164,7 +164,7 @@ static void tscUpdateVgroupInfo(SSqlObj *pSql, SRpcEpSet *pEpSet) { vgroupInfo.inUse = pEpSet->inUse; vgroupInfo.numOfEps = pEpSet->numOfEps; for (int32_t i = 0; i < vgroupInfo.numOfEps; i++) { - tstrncpy(vgroupInfo.ep[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN); // buffer not null terminated risk + tstrncpy(vgroupInfo.ep[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN); vgroupInfo.ep[i].port = pEpSet->port[i]; } From ba8d429f2ce7239cf17e99dbcc3a7db310bc337a Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 29 Jul 2021 13:48:37 +0800 Subject: [PATCH 080/106] fix error --- tests/Jenkinsfile | 2 +- tests/mas/Jenkinsfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile index 3185d9ddba..eeb56bd3b9 100644 --- a/tests/Jenkinsfile +++ b/tests/Jenkinsfile @@ -155,7 +155,7 @@ pipeline { ''' } sh ''' - pkill -9 taosd + pkill -9 taosd || echo 1 cd ${WKC}/tests ./test-all.sh b2 date diff --git a/tests/mas/Jenkinsfile b/tests/mas/Jenkinsfile index e44df002ef..84c2d8e4c2 100644 --- a/tests/mas/Jenkinsfile +++ b/tests/mas/Jenkinsfile @@ -163,7 +163,7 @@ pipeline { ''' } sh ''' - pkill -9 taosd + pkill -9 taosd || echo 1 cd ${WKC}/tests ./test-all.sh b2 date From 0c08c44f7a370fe8536a8b4b4f4df7356b9a46e1 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 29 Jul 2021 13:48:37 +0800 Subject: [PATCH 081/106] fix error --- tests/Jenkinsfile | 2 +- tests/mas/Jenkinsfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile index 3185d9ddba..eeb56bd3b9 100644 --- a/tests/Jenkinsfile +++ b/tests/Jenkinsfile @@ -155,7 +155,7 @@ pipeline { ''' } sh ''' - pkill -9 taosd + pkill -9 taosd || echo 1 cd ${WKC}/tests ./test-all.sh b2 date diff --git a/tests/mas/Jenkinsfile b/tests/mas/Jenkinsfile index e44df002ef..84c2d8e4c2 100644 --- a/tests/mas/Jenkinsfile +++ b/tests/mas/Jenkinsfile @@ -163,7 +163,7 @@ pipeline { ''' } sh ''' - pkill -9 taosd + pkill -9 taosd || echo 1 cd ${WKC}/tests ./test-all.sh b2 date From 61fb774237bb4d6d9695d4f2f0c5cc1c542a52d1 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 29 Jul 2021 14:03:59 +0800 Subject: [PATCH 082/106] [TD-5062] Feature/row partial update (#6970) * [TD-5062]: support partial update --- src/client/src/tscUtil.c | 4 +- src/common/inc/tdataformat.h | 180 ++++++++++-- src/common/src/tdataformat.c | 129 ++++++++- src/common/src/ttypes.c | 32 +-- src/inc/taosdef.h | 8 +- src/inc/tsdb.h | 2 +- src/inc/ttype.h | 8 +- src/tsdb/inc/tsdbBuffer.h | 4 +- src/tsdb/inc/tsdbReadImpl.h | 9 +- src/tsdb/inc/tsdbRowMergeBuf.h | 43 +++ src/tsdb/inc/tsdbint.h | 5 +- src/tsdb/src/tsdbBuffer.c | 4 +- src/tsdb/src/tsdbCommit.c | 4 +- src/tsdb/src/tsdbCommitQueue.c | 2 +- src/tsdb/src/tsdbCompact.c | 2 +- src/tsdb/src/tsdbMain.c | 7 +- src/tsdb/src/tsdbMemTable.c | 366 +++++++++++------------- src/tsdb/src/tsdbRead.c | 507 ++++++++++++++++++++------------- src/tsdb/src/tsdbReadImpl.c | 8 +- src/tsdb/src/tsdbRowMergeBuf.c | 30 ++ src/util/inc/tfunctional.h | 56 ++++ src/util/inc/tskiplist.h | 18 +- src/util/src/tfunctional.c | 48 ++++ src/util/src/tskiplist.c | 49 +++- 24 files changed, 1045 insertions(+), 480 deletions(-) create mode 100644 src/tsdb/inc/tsdbRowMergeBuf.h create mode 100644 src/tsdb/src/tsdbRowMergeBuf.c create mode 100644 src/util/inc/tfunctional.h create mode 100644 src/util/src/tfunctional.c diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 9dc541b1a6..1c610b67a5 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1864,8 +1864,8 @@ static SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { while (i < nColsBound) { int16_t colId = payloadColId(p); uint8_t colType = payloadColType(p); - tdAppendKvColVal(kvRow, POINTER_SHIFT(pVals,payloadColOffset(p)), colId, colType, toffset); - toffset += sizeof(SColIdx); + tdAppendKvColVal(kvRow, POINTER_SHIFT(pVals,payloadColOffset(p)), colId, colType, &toffset); + //toffset += sizeof(SColIdx); p = payloadNextCol(p); ++i; } diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index f8394dc271..47bd8a72b2 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -232,6 +232,83 @@ static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t o } } +static FORCE_INLINE void *tdGetPtrToCol(SDataRow row, STSchema *pSchema, int idx) { + return POINTER_SHIFT(row, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset); +} + +static FORCE_INLINE void *tdGetColOfRowBySchema(SDataRow row, STSchema *pSchema, int idx) { + int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset; + int8_t type = pSchema->columns[idx].type; + + return tdGetRowDataOfCol(row, type, offset); +} + +static FORCE_INLINE bool tdIsColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int idx) { + int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset; + int8_t type = pSchema->columns[idx].type; + + return isNull(tdGetRowDataOfCol(row, type, offset), type); +} + +static FORCE_INLINE void tdSetColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int idx) { + int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset; + int8_t type = pSchema->columns[idx].type; + int16_t bytes = pSchema->columns[idx].bytes; + + setNull(tdGetRowDataOfCol(row, type, offset), type, bytes); +} + +static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSchema, int dstIdx, SDataRow src, STSchema *pSrcSchema, int srcIdx) { + int8_t type = pDstSchema->columns[dstIdx].type; + ASSERT(type == pSrcSchema->columns[srcIdx].type); + void *pData = tdGetPtrToCol(dst, pDstSchema, dstIdx); + void *value = tdGetPtrToCol(src, pSrcSchema, srcIdx); + + switch (type) { + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + *(VarDataOffsetT *)pData = *(VarDataOffsetT *)value; + pData = POINTER_SHIFT(dst, *(VarDataOffsetT *)pData); + value = POINTER_SHIFT(src, *(VarDataOffsetT *)value); + memcpy(pData, value, varDataTLen(value)); + break; + case TSDB_DATA_TYPE_NULL: + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + *(uint8_t *)pData = *(uint8_t *)value; + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + *(uint16_t *)pData = *(uint16_t *)value; + break; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + *(uint32_t *)pData = *(uint32_t *)value; + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + *(uint64_t *)pData = *(uint64_t *)value; + break; + case TSDB_DATA_TYPE_FLOAT: + SET_FLOAT_PTR(pData, value); + break; + case TSDB_DATA_TYPE_DOUBLE: + SET_DOUBLE_PTR(pData, value); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + if (pSrcSchema->columns[srcIdx].colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + *(TSKEY *)pData = tdGetKey(*(TKEY *)value); + } else { + *(TSKEY *)pData = *(TSKEY *)value; + } + break; + default: + memcpy(pData, value, pSrcSchema->columns[srcIdx].bytes); + } +} + + // ----------------- Data column structure typedef struct SDataCol { int8_t type; // column type @@ -335,7 +412,7 @@ void tdResetDataCols(SDataCols *pCols); int tdInitDataCols(SDataCols *pCols, STSchema *pSchema); SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData); SDataCols *tdFreeDataCols(SDataCols *pCols); -int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset); +int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool forceSetNull); // ----------------- K-V data row structure /* |<-------------------------------------- len -------------------------------------------->| @@ -398,9 +475,9 @@ static FORCE_INLINE void *tdGetKVRowIdxOfCol(SKVRow row, int16_t colId) { } // offset here not include kvRow header length -static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, int16_t colId, int8_t type, int32_t offset) { +static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, int16_t colId, int8_t type, int32_t *offset) { ASSERT(value != NULL); - int32_t toffset = offset + TD_KV_ROW_HEAD_SIZE; + int32_t toffset = *offset + TD_KV_ROW_HEAD_SIZE; SColIdx *pColIdx = (SColIdx *)POINTER_SHIFT(row, toffset); char * ptr = (char *)POINTER_SHIFT(row, kvRowLen(row)); @@ -411,7 +488,7 @@ static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, int16_t memcpy(ptr, value, varDataTLen(value)); kvRowLen(row) += varDataTLen(value); } else { - if (offset == 0) { + if (*offset == 0) { ASSERT(type == TSDB_DATA_TYPE_TIMESTAMP); TKEY tvalue = tdGetTKEY(*(TSKEY *)value); memcpy(ptr, (void *)(&tvalue), TYPE_BYTES[type]); @@ -420,9 +497,27 @@ static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, int16_t } kvRowLen(row) += TYPE_BYTES[type]; } + *offset += sizeof(SColIdx); return 0; } +// NOTE: offset here including the header size +static FORCE_INLINE void *tdGetKvRowDataOfCol(void *row, int32_t offset) { return POINTER_SHIFT(row, offset); } + +static FORCE_INLINE void *tdGetKVRowValOfColEx(SKVRow row, int16_t colId, int32_t *nIdx) { + while (*nIdx < kvRowNCols(row)) { + SColIdx *pColIdx = kvRowColIdxAt(row, *nIdx); + if (pColIdx->colId == colId) { + ++(*nIdx); + return tdGetKvRowDataOfCol(row, pColIdx->offset); + } else if (pColIdx->colId > colId) { + return NULL; + } else { + ++(*nIdx); + } + } + return NULL; +} // ----------------- K-V data row builder typedef struct { @@ -495,7 +590,7 @@ typedef void *SMemRow; #define TD_MEM_ROW_KV_VER_SIZE sizeof(int16_t) #define TD_MEM_ROW_KV_TYPE_VER_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE) #define TD_MEM_ROW_DATA_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_DATA_ROW_HEAD_SIZE) -// #define TD_MEM_ROW_KV_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE + TD_KV_ROW_HEAD_SIZE) +#define TD_MEM_ROW_KV_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE + TD_KV_ROW_HEAD_SIZE) #define SMEM_ROW_DATA 0U // SDataRow #define SMEM_ROW_KV 1U // SKVRow @@ -538,27 +633,80 @@ typedef void *SMemRow; #define memRowSetType(r, t) (memRowType(r) = (t)) #define memRowSetLen(r, l) (isDataRow(r) ? memRowDataLen(r) = (l) : memRowKvLen(r) = (l)) -#define memRowSetVersion(r, v) (isDataRow(r) ? dataRowSetVersion(memRowDataBody(r), v) : memRowKvSetVersion(r, v)) +#define memRowSetVersion(r, v) (isDataRow(r) ? dataRowSetVersion(memRowDataBody(r), v) : memRowSetKvVersion(r, v)) #define memRowCpy(dst, r) memcpy((dst), (r), memRowTLen(r)) #define memRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_MEM_ROW_DATA_HEAD_SIZE) #define memRowDeleted(r) TKEY_IS_DELETED(memRowTKey(r)) SMemRow tdMemRowDup(SMemRow row); -void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols); +void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull); + // NOTE: offset here including the header size -static FORCE_INLINE void *tdGetKvRowDataOfCol(void *row, int32_t offset) { return POINTER_SHIFT(row, offset); } -// NOTE: offset here including the header size -static FORCE_INLINE void *tdGetMemRowDataOfCol(void *row, int8_t type, int32_t offset) { +static FORCE_INLINE void *tdGetMemRowDataOfCol(void *row, int16_t colId, int8_t colType, uint16_t offset) { if (isDataRow(row)) { - return tdGetRowDataOfCol(row, type, offset); - } else if (isKvRow(row)) { - return tdGetKvRowDataOfCol(row, offset); + return tdGetRowDataOfCol(memRowDataBody(row), colType, offset); } else { - ASSERT(0); + return tdGetKVRowValOfCol(memRowKvBody(row), colId); } - return NULL; } +/** + * NOTE: + * 1. Applicable to scan columns one by one + * 2. offset here including the header size + */ +static FORCE_INLINE void *tdGetMemRowDataOfColEx(void *row, int16_t colId, int8_t colType, int32_t offset, + int32_t *kvNIdx) { + if (isDataRow(row)) { + return tdGetRowDataOfCol(memRowDataBody(row), colType, offset); + } else { + return tdGetKVRowValOfColEx(memRowKvBody(row), colId, kvNIdx); + } +} + +static FORCE_INLINE int tdAppendMemColVal(SMemRow row, const void *value, int16_t colId, int8_t type, int32_t offset, + int32_t *kvOffset) { + if (isDataRow(row)) { + tdAppendColVal(memRowDataBody(row), value, type, offset); + } else { + tdAppendKvColVal(memRowKvBody(row), value, colId, type, kvOffset); + } + return 0; +} + +// make sure schema->flen appended for SDataRow +static FORCE_INLINE int32_t tdGetColAppendLen(uint8_t rowType, const void *value, int8_t colType) { + int32_t len = 0; + if (IS_VAR_DATA_TYPE(colType)) { + len += varDataTLen(value); + if (rowType == SMEM_ROW_KV) { + len += sizeof(SColIdx); + } + } else { + if (rowType == SMEM_ROW_KV) { + len += TYPE_BYTES[colType]; + len += sizeof(SColIdx); + } + } + return len; +} + + +typedef struct { + int16_t colId; + uint8_t colType; + char* colVal; +} SColInfo; + +static FORCE_INLINE void setSColInfo(SColInfo* colInfo, int16_t colId, uint8_t colType, char* colVal) { + colInfo->colId = colId; + colInfo->colType = colType; + colInfo->colVal = colVal; +} + +SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2); + + // ----------------- Raw payload structure for row: /* |<------------ Head ------------->|<----------- body of column data tuple ------------------->| * | |<----------------- flen ------------->|<--- value part --->| @@ -608,4 +756,4 @@ static FORCE_INLINE char *payloadNextCol(char *pCol) { return (char *)POINTER_SH } #endif -#endif // _TD_DATA_FORMAT_H_ \ No newline at end of file +#endif // _TD_DATA_FORMAT_H_ diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 5a63588611..8ef3d083c7 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -17,9 +17,10 @@ #include "talgo.h" #include "tcoding.h" #include "wchar.h" +#include "tarray.h" static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, - int limit2, int tRows); + int limit2, int tRows, bool forceSetNull); /** * Duplicate the schema and return a new object @@ -418,7 +419,8 @@ void tdResetDataCols(SDataCols *pCols) { } } } -static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) { + +static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) { ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < dataRowKey(row)); int rcol = 0; @@ -452,8 +454,10 @@ static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols } else if (pRowCol->colId < pDataCol->colId) { rcol++; } else { - // dataColSetNullAt(pDataCol, pCols->numOfRows); - dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + if(forceSetNull) { + //dataColSetNullAt(pDataCol, pCols->numOfRows); + dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + } dcol++; } } @@ -461,7 +465,7 @@ static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols pCols->numOfRows++; } -static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCols) { +static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) { ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < kvRowKey(row)); int rcol = 0; @@ -498,8 +502,10 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo } else if (colIdx->colId < pDataCol->colId) { ++rcol; } else { - // dataColSetNullAt(pDataCol, pCols->numOfRows); - dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + if (forceSetNull) { + // dataColSetNullAt(pDataCol, pCols->numOfRows); + dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + } ++dcol; } } @@ -507,17 +513,17 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo pCols->numOfRows++; } -void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols) { +void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) { if (isDataRow(row)) { - tdAppendDataRowToDataCol(memRowDataBody(row), pSchema, pCols); + tdAppendDataRowToDataCol(memRowDataBody(row), pSchema, pCols, forceSetNull); } else if (isKvRow(row)) { - tdAppendKvRowToDataCol(memRowKvBody(row), pSchema, pCols); + tdAppendKvRowToDataCol(memRowKvBody(row), pSchema, pCols, forceSetNull); } else { ASSERT(0); } } -int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset) { +int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool forceSetNull) { ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows); ASSERT(target->numOfCols == source->numOfCols); int offset = 0; @@ -546,7 +552,7 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int * int iter1 = 0; tdMergeTwoDataCols(target, pTarget, &iter1, pTarget->numOfRows, source, pOffset, source->numOfRows, - pTarget->numOfRows + rowsToMerge); + pTarget->numOfRows + rowsToMerge, forceSetNull); } tdFreeDataCols(pTarget); @@ -559,7 +565,7 @@ _err: // src2 data has more priority than src1 static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, - int limit2, int tRows) { + int limit2, int tRows, bool forceSetNull) { tdResetDataCols(target); ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows); @@ -588,7 +594,7 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i if ((key1 > key2) || (key1 == key2 && !TKEY_IS_DELETED(tkey2))) { for (int i = 0; i < src2->numOfCols; i++) { ASSERT(target->cols[i].type == src2->cols[i].type); - if (src2->cols[i].len > 0) { + if (src2->cols[i].len > 0 && (forceSetNull || (!forceSetNull && !isNull(src2->cols[i].pData, src2->cols[i].type)))) { dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows, target->maxPoints); } @@ -761,4 +767,97 @@ SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) { memcpy(kvRowValues(row), pBuilder->buf, pBuilder->size); return row; -} \ No newline at end of file +} + +SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2) { +#if 0 + ASSERT(memRowKey(row1) == memRowKey(row2)); + ASSERT(schemaVersion(pSchema1) == memRowVersion(row1)); + ASSERT(schemaVersion(pSchema2) == memRowVersion(row2)); + ASSERT(schemaVersion(pSchema1) >= schemaVersion(pSchema2)); +#endif + + SArray *stashRow = taosArrayInit(pSchema1->numOfCols, sizeof(SColInfo)); + if (stashRow == NULL) { + return NULL; + } + + SMemRow pRow = buffer; + SDataRow dataRow = memRowDataBody(pRow); + memRowSetType(pRow, SMEM_ROW_DATA); + dataRowSetVersion(dataRow, schemaVersion(pSchema1)); // use latest schema version + dataRowSetLen(dataRow, (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + pSchema1->flen)); + + TDRowTLenT dataLen = 0, kvLen = TD_MEM_ROW_KV_HEAD_SIZE; + + int32_t i = 0; // row1 + int32_t j = 0; // row2 + int32_t nCols1 = schemaNCols(pSchema1); + int32_t nCols2 = schemaNCols(pSchema2); + SColInfo colInfo = {0}; + int32_t kvIdx1 = 0, kvIdx2 = 0; + + while (i < nCols1) { + STColumn *pCol = schemaColAt(pSchema1, i); + void * val1 = tdGetMemRowDataOfColEx(row1, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx1); + // if val1 != NULL, use val1; + if (val1 != NULL && !isNull(val1, pCol->type)) { + tdAppendColVal(dataRow, val1, pCol->type, pCol->offset); + kvLen += tdGetColAppendLen(SMEM_ROW_KV, val1, pCol->type); + setSColInfo(&colInfo, pCol->colId, pCol->type, val1); + taosArrayPush(stashRow, &colInfo); + ++i; // next col + continue; + } + + void *val2 = NULL; + while (j < nCols2) { + STColumn *tCol = schemaColAt(pSchema2, j); + if (tCol->colId < pCol->colId) { + ++j; + continue; + } + if (tCol->colId == pCol->colId) { + val2 = tdGetMemRowDataOfColEx(row2, tCol->colId, tCol->type, TD_DATA_ROW_HEAD_SIZE + tCol->offset, &kvIdx2); + } else if (tCol->colId > pCol->colId) { + // set NULL + } + break; + } // end of while(jtype); + } + tdAppendColVal(dataRow, val2, pCol->type, pCol->offset); + if (!isNull(val2, pCol->type)) { + kvLen += tdGetColAppendLen(SMEM_ROW_KV, val2, pCol->type); + setSColInfo(&colInfo, pCol->colId, pCol->type, val2); + taosArrayPush(stashRow, &colInfo); + } + + ++i; // next col + } + + dataLen = memRowTLen(pRow); + + if (kvLen < dataLen) { + // scan stashRow and generate SKVRow + memset(buffer, 0, sizeof(dataLen)); + SMemRow tRow = buffer; + memRowSetType(tRow, SMEM_ROW_KV); + SKVRow kvRow = (SKVRow)memRowKvBody(tRow); + int16_t nKvNCols = (int16_t) taosArrayGetSize(stashRow); + kvRowSetLen(kvRow, (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nKvNCols)); + kvRowSetNCols(kvRow, nKvNCols); + memRowSetKvVersion(tRow, pSchema1->version); + + int32_t toffset = 0; + int16_t k; + for (k = 0; k < nKvNCols; ++k) { + SColInfo *pColInfo = taosArrayGet(stashRow, k); + tdAppendKvColVal(kvRow, pColInfo->colVal, pColInfo->colId, pColInfo->colType, &toffset); + } + ASSERT(kvLen == memRowTLen(tRow)); + } + taosArrayDestroy(stashRow); + return buffer; +} diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c index 13bcfe6480..eeffe49adc 100644 --- a/src/common/src/ttypes.c +++ b/src/common/src/ttypes.c @@ -409,7 +409,7 @@ bool isValidDataType(int32_t type) { return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_UBIGINT; } -void setVardataNull(char* val, int32_t type) { +void setVardataNull(void* val, int32_t type) { if (type == TSDB_DATA_TYPE_BINARY) { varDataSetLen(val, sizeof(int8_t)); *(uint8_t*) varDataVal(val) = TSDB_DATA_BINARY_NULL; @@ -421,75 +421,75 @@ void setVardataNull(char* val, int32_t type) { } } -void setNull(char *val, int32_t type, int32_t bytes) { setNullN(val, type, bytes, 1); } +void setNull(void *val, int32_t type, int32_t bytes) { setNullN(val, type, bytes, 1); } -void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems) { +void setNullN(void *val, int32_t type, int32_t bytes, int32_t numOfElems) { switch (type) { case TSDB_DATA_TYPE_BOOL: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint8_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_BOOL_NULL; + *(uint8_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_BOOL_NULL; } break; case TSDB_DATA_TYPE_TINYINT: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint8_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_TINYINT_NULL; + *(uint8_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_TINYINT_NULL; } break; case TSDB_DATA_TYPE_SMALLINT: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint16_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_SMALLINT_NULL; + *(uint16_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_SMALLINT_NULL; } break; case TSDB_DATA_TYPE_INT: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint32_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_INT_NULL; + *(uint32_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_INT_NULL; } break; case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint64_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_BIGINT_NULL; + *(uint64_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_BIGINT_NULL; } break; case TSDB_DATA_TYPE_UTINYINT: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint8_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_UTINYINT_NULL; + *(uint8_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_UTINYINT_NULL; } break; case TSDB_DATA_TYPE_USMALLINT: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint16_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_USMALLINT_NULL; + *(uint16_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_USMALLINT_NULL; } break; case TSDB_DATA_TYPE_UINT: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint32_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_UINT_NULL; + *(uint32_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_UINT_NULL; } break; case TSDB_DATA_TYPE_UBIGINT: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint64_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_UBIGINT_NULL; + *(uint64_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_UBIGINT_NULL; } break; case TSDB_DATA_TYPE_FLOAT: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint32_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_FLOAT_NULL; + *(uint32_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_FLOAT_NULL; } break; case TSDB_DATA_TYPE_DOUBLE: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint64_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_DOUBLE_NULL; + *(uint64_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_DOUBLE_NULL; } break; case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_BINARY: for (int32_t i = 0; i < numOfElems; ++i) { - setVardataNull(val + i * bytes, type); + setVardataNull(POINTER_SHIFT(val, i * bytes), type); } break; default: { for (int32_t i = 0; i < numOfElems; ++i) { - *(uint32_t *)(val + i * tDataTypes[TSDB_DATA_TYPE_INT].bytes) = TSDB_DATA_INT_NULL; + *(uint32_t *)(POINTER_SHIFT(val, i * tDataTypes[TSDB_DATA_TYPE_INT].bytes)) = TSDB_DATA_INT_NULL; } break; } diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index ca8ad3cc09..4a5497795f 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -307,7 +307,7 @@ do { \ #define TSDB_DEFAULT_WAL_LEVEL 1 #define TSDB_MIN_DB_UPDATE 0 -#define TSDB_MAX_DB_UPDATE 1 +#define TSDB_MAX_DB_UPDATE 2 #define TSDB_DEFAULT_DB_UPDATE_OPTION 0 #define TSDB_MIN_DB_CACHE_LAST_ROW 0 @@ -435,6 +435,12 @@ typedef enum { TSDB_CHECK_ITEM_MAX } ECheckItemType; +typedef enum { + TD_ROW_DISCARD_UPDATE = 0, + TD_ROW_OVERWRITE_UPDATE = 1, + TD_ROW_PARTIAL_UPDATE = 2 +} TDUpdateConfig; + extern char *qtypeStr[]; #ifdef __cplusplus diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 79d9029dbc..7fd6d8c10c 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -111,7 +111,7 @@ typedef struct { uint64_t superUid; STSchema * schema; STSchema * tagSchema; - SDataRow tagValues; + SKVRow tagValues; char * sql; } STableCfg; diff --git a/src/inc/ttype.h b/src/inc/ttype.h index fcd2651c70..44e666106a 100644 --- a/src/inc/ttype.h +++ b/src/inc/ttype.h @@ -141,7 +141,7 @@ typedef struct { #define IS_VALID_FLOAT(_t) ((_t) >= -FLT_MAX && (_t) <= FLT_MAX) #define IS_VALID_DOUBLE(_t) ((_t) >= -DBL_MAX && (_t) <= DBL_MAX) -static FORCE_INLINE bool isNull(const char *val, int32_t type) { +static FORCE_INLINE bool isNull(const void *val, int32_t type) { switch (type) { case TSDB_DATA_TYPE_BOOL: return *(uint8_t *)val == TSDB_DATA_BOOL_NULL; @@ -193,9 +193,9 @@ extern tDataTypeDescriptor tDataTypes[15]; bool isValidDataType(int32_t type); -void setVardataNull(char* val, int32_t type); -void setNull(char *val, int32_t type, int32_t bytes); -void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems); +void setVardataNull(void* val, int32_t type); +void setNull(void *val, int32_t type, int32_t bytes); +void setNullN(void *val, int32_t type, int32_t bytes, int32_t numOfElems); const void *getNullValue(int32_t type); void assignVal(char *val, const char *src, int32_t len, int32_t type); diff --git a/src/tsdb/inc/tsdbBuffer.h b/src/tsdb/inc/tsdbBuffer.h index 4e18ac711a..ec6b057aef 100644 --- a/src/tsdb/inc/tsdbBuffer.h +++ b/src/tsdb/inc/tsdbBuffer.h @@ -40,7 +40,7 @@ void tsdbFreeBufPool(STsdbBufPool* pBufPool); int tsdbOpenBufPool(STsdbRepo* pRepo); void tsdbCloseBufPool(STsdbRepo* pRepo); SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo); -int tsdbExpendPool(STsdbRepo* pRepo, int32_t oldTotalBlocks); +int tsdbExpandPool(STsdbRepo* pRepo, int32_t oldTotalBlocks); void tsdbRecycleBufferBlock(STsdbBufPool* pPool, SListNode *pNode); -#endif /* _TD_TSDB_BUFFER_H_ */ \ No newline at end of file +#endif /* _TD_TSDB_BUFFER_H_ */ diff --git a/src/tsdb/inc/tsdbReadImpl.h b/src/tsdb/inc/tsdbReadImpl.h index e7840d9723..814c4d1305 100644 --- a/src/tsdb/inc/tsdbReadImpl.h +++ b/src/tsdb/inc/tsdbReadImpl.h @@ -16,6 +16,13 @@ #ifndef _TD_TSDB_READ_IMPL_H_ #define _TD_TSDB_READ_IMPL_H_ +#include "tfs.h" +#include "tsdb.h" +#include "os.h" +#include "tsdbFile.h" +#include "tskiplist.h" +#include "tsdbMeta.h" + typedef struct SReadH SReadH; typedef struct { @@ -143,4 +150,4 @@ static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) { return 0; } -#endif /*_TD_TSDB_READ_IMPL_H_*/ \ No newline at end of file +#endif /*_TD_TSDB_READ_IMPL_H_*/ diff --git a/src/tsdb/inc/tsdbRowMergeBuf.h b/src/tsdb/inc/tsdbRowMergeBuf.h new file mode 100644 index 0000000000..302bf25750 --- /dev/null +++ b/src/tsdb/inc/tsdbRowMergeBuf.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TSDB_ROW_MERGE_BUF_H +#define TSDB_ROW_MERGE_BUF_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tsdb.h" +#include "tchecksum.h" +#include "tsdbReadImpl.h" + +typedef void* SMergeBuf; + +SDataRow tsdbMergeTwoRows(SMergeBuf *pBuf, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2); + +static FORCE_INLINE int tsdbMergeBufMakeSureRoom(SMergeBuf *pBuf, STSchema* pSchema1, STSchema* pSchema2) { + return tsdbMakeRoom(pBuf, MAX(dataRowMaxBytesFromSchema(pSchema1), dataRowMaxBytesFromSchema(pSchema2))); +} + +static FORCE_INLINE void tsdbFreeMergeBuf(SMergeBuf buf) { + taosTZfree(buf); +} + +#ifdef __cplusplus +} +#endif + +#endif /* ifndef TSDB_ROW_MERGE_BUF_H */ diff --git a/src/tsdb/inc/tsdbint.h b/src/tsdb/inc/tsdbint.h index dd43e39310..757a0951e8 100644 --- a/src/tsdb/inc/tsdbint.h +++ b/src/tsdb/inc/tsdbint.h @@ -68,6 +68,8 @@ extern "C" { #include "tsdbCompact.h" // Commit Queue #include "tsdbCommitQueue.h" + +#include "tsdbRowMergeBuf.h" // Main definitions struct STsdbRepo { uint8_t state; @@ -92,6 +94,7 @@ struct STsdbRepo { pthread_mutex_t mutex; bool repoLocked; int32_t code; // Commit code + SMergeBuf mergeBuf; //used when update=2 bool inCompact; // is in compact process? }; @@ -139,4 +142,4 @@ static FORCE_INLINE int tsdbGetNextMaxTables(int tid) { } #endif -#endif /* _TD_TSDB_INT_H_ */ \ No newline at end of file +#endif /* _TD_TSDB_INT_H_ */ diff --git a/src/tsdb/src/tsdbBuffer.c b/src/tsdb/src/tsdbBuffer.c index 429ea8e0ce..e675bf6f9d 100644 --- a/src/tsdb/src/tsdbBuffer.c +++ b/src/tsdb/src/tsdbBuffer.c @@ -159,7 +159,7 @@ _err: static void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { tfree(pBufBlock); } -int tsdbExpendPool(STsdbRepo* pRepo, int32_t oldTotalBlocks) { +int tsdbExpandPool(STsdbRepo* pRepo, int32_t oldTotalBlocks) { if (oldTotalBlocks == pRepo->config.totalBlocks) { return TSDB_CODE_SUCCESS; } @@ -199,4 +199,4 @@ void tsdbRecycleBufferBlock(STsdbBufPool* pPool, SListNode *pNode) { tsdbFreeBufBlock(pBufBlock); free(pNode); pPool->nBufBlocks--; -} \ No newline at end of file +} diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 6330da6058..6c98283189 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -1290,7 +1290,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt ASSERT(pSchema != NULL); } - tdAppendMemRowToDataCol(row, pSchema, pTarget); + tdAppendMemRowToDataCol(row, pSchema, pTarget, true); } tSkipListIterNext(pCommitIter->pIter); @@ -1302,7 +1302,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt ASSERT(pSchema != NULL); } - tdAppendMemRowToDataCol(row, pSchema, pTarget); + tdAppendMemRowToDataCol(row, pSchema, pTarget, update == TD_ROW_OVERWRITE_UPDATE); } } else { ASSERT(!isRowDel); diff --git a/src/tsdb/src/tsdbCommitQueue.c b/src/tsdb/src/tsdbCommitQueue.c index e45ac05e97..59fb4f334d 100644 --- a/src/tsdb/src/tsdbCommitQueue.c +++ b/src/tsdb/src/tsdbCommitQueue.c @@ -138,7 +138,7 @@ static void tsdbApplyRepoConfig(STsdbRepo *pRepo) { pSaveCfg->compression, pSaveCfg->keep,pSaveCfg->keep1, pSaveCfg->keep2, pSaveCfg->totalBlocks, oldCfg.cacheLastRow, pSaveCfg->cacheLastRow, oldTotalBlocks, pSaveCfg->totalBlocks); - int err = tsdbExpendPool(pRepo, oldTotalBlocks); + int err = tsdbExpandPool(pRepo, oldTotalBlocks); if (!TAOS_SUCCEEDED(err)) { tsdbError("vgId:%d expand pool from %d to %d fail,reason:%s", REPO_ID(pRepo), oldTotalBlocks, pSaveCfg->totalBlocks, tstrerror(err)); diff --git a/src/tsdb/src/tsdbCompact.c b/src/tsdb/src/tsdbCompact.c index 5211ee3c61..0490f26b5e 100644 --- a/src/tsdb/src/tsdbCompact.c +++ b/src/tsdb/src/tsdbCompact.c @@ -455,7 +455,7 @@ static int tsdbCompactMeta(STsdbRepo *pRepo) { if (pReadh->pDCols[0]->numOfRows - ridx == 0) break; int rowsToMerge = MIN(pReadh->pDCols[0]->numOfRows - ridx, defaultRows - pComph->pDataCols->numOfRows); - tdMergeDataCols(pComph->pDataCols, pReadh->pDCols[0], rowsToMerge, &ridx); + tdMergeDataCols(pComph->pDataCols, pReadh->pDCols[0], rowsToMerge, &ridx, pCfg->update != TD_ROW_PARTIAL_UPDATE); if (pComph->pDataCols->numOfRows < defaultRows) { break; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index f3a7c4b7ee..c877bfc7af 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -14,6 +14,7 @@ */ // no test file errors here +#include "taosdef.h" #include "tsdbint.h" #define IS_VALID_PRECISION(precision) \ @@ -106,6 +107,8 @@ STsdbRepo *tsdbOpenRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { return NULL; } + pRepo->mergeBuf = NULL; + tsdbStartStream(pRepo); tsdbDebug("vgId:%d, TSDB repository opened", REPO_ID(pRepo)); @@ -518,7 +521,8 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { } // update check - if (pCfg->update != 0) pCfg->update = 1; + if (pCfg->update < TD_ROW_DISCARD_UPDATE || pCfg->update > TD_ROW_PARTIAL_UPDATE) + pCfg->update = TD_ROW_DISCARD_UPDATE; // update cacheLastRow if (pCfg->cacheLastRow != 0) { @@ -597,6 +601,7 @@ static void tsdbFreeRepo(STsdbRepo *pRepo) { tsdbFreeFS(pRepo->fs); tsdbFreeBufPool(pRepo->pPool); tsdbFreeMeta(pRepo->tsdbMeta); + tsdbFreeMergeBuf(pRepo->mergeBuf); // tsdbFreeMemTable(pRepo->mem); // tsdbFreeMemTable(pRepo->imem); tsem_destroy(&(pRepo->readyToCommit)); diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index ee9dadd9b7..8bb2d1c44e 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -13,7 +13,11 @@ * along with this program. If not, see . */ +#include "tdataformat.h" +#include "tfunctional.h" #include "tsdbint.h" +#include "tskiplist.h" +#include "tsdbRowMergeBuf.h" #define TSDB_DATA_SKIPLIST_LEVEL 5 #define TSDB_MAX_INSERT_BATCH 512 @@ -30,24 +34,22 @@ typedef struct { void * pMsg; } SSubmitMsgIter; -static SMemTable * tsdbNewMemTable(STsdbRepo *pRepo); -static void tsdbFreeMemTable(SMemTable *pMemTable); -static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable); -static void tsdbFreeTableData(STableData *pTableData); -static char * tsdbGetTsTupleKey(const void *data); +static SMemTable * tsdbNewMemTable(STsdbRepo *pRepo); +static void tsdbFreeMemTable(SMemTable *pMemTable); +static STableData* tsdbNewTableData(STsdbCfg *pCfg, STable *pTable); +static void tsdbFreeTableData(STableData *pTableData); +static char * tsdbGetTsTupleKey(const void *data); static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables); -static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SMemRow row); +static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SMemRow row); static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter); -static SMemRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter); +static SMemRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter); static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg); static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *affectedrows); -static int tsdbCopyRowToMem(STsdbRepo *pRepo, SMemRow row, STable *pTable, void **ppRow); static int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter); static int tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock); static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable); -static int tsdbInsertDataToTableImpl(STsdbRepo *pRepo, STable *pTable, void **rows, int rowCounter); -static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter); -static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SMemRow row); +static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SMemRow row); + static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SMemRow row, TSKEY minKey, TSKEY maxKey, TSKEY now); @@ -342,7 +344,7 @@ int tsdbSyncCommit(STsdbRepo *repo) { * 3. rowsIncreased = rowsInserted - rowsDeleteSucceed >= maxRowsToRead * 4. operations in pCols not exceeds its max capacity if pCols is given * - * The function tries to procceed AS MUSH AS POSSIBLE. + * The function tries to procceed AS MUCH AS POSSIBLE. */ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) { @@ -523,9 +525,15 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) { pTableData->keyLast = 0; pTableData->numOfRows = 0; + uint8_t skipListCreateFlags; + if(pCfg->update == TD_ROW_DISCARD_UPDATE) + skipListCreateFlags = SL_DISCARD_DUP_KEY; + else + skipListCreateFlags = SL_UPDATE_DUP_KEY; + pTableData->pData = tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], - tkeyComparFn, pCfg->update ? SL_UPDATE_DUP_KEY : SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); + tkeyComparFn, skipListCreateFlags, tsdbGetTsTupleKey); if (pTableData->pData == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; free(pTableData); @@ -581,7 +589,7 @@ static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema * } } - tdAppendMemRowToDataCol(row, *ppSchema, pCols); + tdAppendMemRowToDataCol(row, *ppSchema, pCols, true); } return 0; @@ -693,45 +701,152 @@ static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg) { return 0; } -static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *affectedrows) { - STsdbMeta * pMeta = pRepo->tsdbMeta; - int64_t points = 0; - STable * pTable = NULL; - SSubmitBlkIter blkIter = {0}; - SMemRow row = NULL; - void * rows[TSDB_MAX_INSERT_BATCH] = {0}; - int rowCounter = 0; +//row1 has higher priority +static SMemRow tsdbInsertDupKeyMerge(SMemRow row1, SMemRow row2, STsdbRepo* pRepo, STSchema **ppSchema1, STSchema **ppSchema2, STable* pTable, int32_t* pAffectedRows, int64_t* pPoints, SMemRow* pLastRow) { + + //for compatiblity, duplicate key inserted when update=0 should be also calculated as affected rows! + if(row1 == NULL && row2 == NULL && pRepo->config.update == TD_ROW_DISCARD_UPDATE) { + (*pAffectedRows)++; + (*pPoints)++; + return NULL; + } - ASSERT(pBlock->tid < pMeta->maxTables); - pTable = pMeta->tables[pBlock->tid]; - ASSERT(pTable != NULL && TABLE_UID(pTable) == pBlock->uid); + if(row2 == NULL || pRepo->config.update != TD_ROW_PARTIAL_UPDATE) { + void* pMem = tsdbAllocBytes(pRepo, memRowTLen(row1)); + if(pMem == NULL) return NULL; + memRowCpy(pMem, row1); + (*pAffectedRows)++; + (*pPoints)++; + *pLastRow = pMem; + return pMem; + } - tsdbInitSubmitBlkIter(pBlock, &blkIter); - while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) { - if (tsdbCopyRowToMem(pRepo, row, pTable, &(rows[rowCounter])) < 0) { - tsdbFreeRows(pRepo, rows, rowCounter); - goto _err; + STSchema *pSchema1 = *ppSchema1; + STSchema *pSchema2 = *ppSchema2; + SMergeBuf * pBuf = &pRepo->mergeBuf; + int dv1 = memRowVersion(row1); + int dv2 = memRowVersion(row2); + if(pSchema1 == NULL || schemaVersion(pSchema1) != dv1) { + if(pSchema2 != NULL && schemaVersion(pSchema2) == dv1) { + *ppSchema1 = pSchema2; + } else { + *ppSchema1 = tsdbGetTableSchemaImpl(pTable, false, false, memRowVersion(row1)); } + pSchema1 = *ppSchema1; + } - (*affectedrows)++; - points++; - - if (rows[rowCounter] != NULL) { - rowCounter++; - } - - if (rowCounter == TSDB_MAX_INSERT_BATCH) { - if (tsdbInsertDataToTableImpl(pRepo, pTable, rows, rowCounter) < 0) { - goto _err; - } - - rowCounter = 0; - memset(rows, 0, sizeof(rows)); + if(pSchema2 == NULL || schemaVersion(pSchema2) != dv2) { + if(schemaVersion(pSchema1) == dv2) { + pSchema2 = pSchema1; + } else { + *ppSchema2 = tsdbGetTableSchemaImpl(pTable, false, false, memRowVersion(row2)); + pSchema2 = *ppSchema2; } } - if (rowCounter > 0 && tsdbInsertDataToTableImpl(pRepo, pTable, rows, rowCounter) < 0) { - goto _err; + SMemRow tmp = tsdbMergeTwoRows(pBuf, row1, row2, pSchema1, pSchema2); + + void* pMem = tsdbAllocBytes(pRepo, memRowTLen(tmp)); + if(pMem == NULL) return NULL; + memRowCpy(pMem, tmp); + + (*pAffectedRows)++; + (*pPoints)++; + + *pLastRow = pMem; + return pMem; +} + +static void* tsdbInsertDupKeyMergePacked(void** args) { + return tsdbInsertDupKeyMerge(args[0], args[1], args[2], (STSchema**)&args[3], (STSchema**)&args[4], args[5], args[6], args[7], args[8]); +} + +static void tsdbSetupSkipListHookFns(SSkipList* pSkipList, STsdbRepo *pRepo, STable *pTable, int32_t* pAffectedRows, int64_t* pPoints, SMemRow* pLastRow) { + + if(pSkipList->insertHandleFn == NULL) { + tGenericSavedFunc *dupHandleSavedFunc = genericSavedFuncInit((GenericVaFunc)&tsdbInsertDupKeyMergePacked, 9); + dupHandleSavedFunc->args[2] = pRepo; + dupHandleSavedFunc->args[3] = NULL; + dupHandleSavedFunc->args[4] = NULL; + dupHandleSavedFunc->args[5] = pTable; + dupHandleSavedFunc->args[6] = pAffectedRows; + dupHandleSavedFunc->args[7] = pPoints; + dupHandleSavedFunc->args[8] = pLastRow; + pSkipList->insertHandleFn = dupHandleSavedFunc; + } +} + +static int tsdbInsertDataToTable(STsdbRepo* pRepo, SSubmitBlk* pBlock, int32_t *pAffectedRows) { + + STsdbMeta *pMeta = pRepo->tsdbMeta; + int64_t points = 0; + STable *pTable = NULL; + SSubmitBlkIter blkIter = {0}; + SMemTable *pMemTable = NULL; + STableData *pTableData = NULL; + STsdbCfg *pCfg = &(pRepo->config); + + tsdbInitSubmitBlkIter(pBlock, &blkIter); + if(blkIter.row == NULL) return 0; + TSKEY firstRowKey = memRowKey(blkIter.row); + + tsdbAllocBytes(pRepo, 0); + pMemTable = pRepo->mem; + + ASSERT(pMemTable != NULL); + ASSERT(pBlock->tid < pMeta->maxTables); + + pTable = pMeta->tables[pBlock->tid]; + + ASSERT(pTable != NULL && TABLE_UID(pTable) == pBlock->uid); + + + if (TABLE_TID(pTable) >= pMemTable->maxTables) { + if (tsdbAdjustMemMaxTables(pMemTable, pMeta->maxTables) < 0) { + return -1; + } + } + pTableData = pMemTable->tData[TABLE_TID(pTable)]; + + if (pTableData == NULL || pTableData->uid != TABLE_UID(pTable)) { + if (pTableData != NULL) { + taosWLockLatch(&(pMemTable->latch)); + pMemTable->tData[TABLE_TID(pTable)] = NULL; + tsdbFreeTableData(pTableData); + taosWUnLockLatch(&(pMemTable->latch)); + } + + pTableData = tsdbNewTableData(pCfg, pTable); + if (pTableData == NULL) { + tsdbError("vgId:%d failed to insert data to table %s uid %" PRId64 " tid %d since %s", REPO_ID(pRepo), + TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable), tstrerror(terrno)); + return -1; + } + + pRepo->mem->tData[TABLE_TID(pTable)] = pTableData; + } + + ASSERT((pTableData != NULL) && pTableData->uid == TABLE_UID(pTable)); + + SMemRow lastRow = NULL; + int64_t osize = SL_SIZE(pTableData->pData); + tsdbSetupSkipListHookFns(pTableData->pData, pRepo, pTable, pAffectedRows, &points, &lastRow); + tSkipListPutBatchByIter(pTableData->pData, &blkIter, (iter_next_fn_t)tsdbGetSubmitBlkNext); + int64_t dsize = SL_SIZE(pTableData->pData) - osize; + + + if(lastRow != NULL) { + TSKEY lastRowKey = memRowKey(lastRow); + if (pMemTable->keyFirst > firstRowKey) pMemTable->keyFirst = firstRowKey; + pMemTable->numOfRows += dsize; + + if (pTableData->keyFirst > firstRowKey) pTableData->keyFirst = firstRowKey; + pTableData->numOfRows += dsize; + if (pMemTable->keyLast < lastRowKey) pMemTable->keyLast = lastRowKey; + if (pTableData->keyLast < lastRowKey) pTableData->keyLast = lastRowKey; + if (tsdbUpdateTableLatestInfo(pRepo, pTable, lastRow) < 0) { + return -1; + } } STSchema *pSchema = tsdbGetTableSchemaByVersion(pTable, pBlock->sversion); @@ -739,48 +854,8 @@ static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t * pRepo->stat.totalStorage += points * schemaVLen(pSchema); return 0; - -_err: - return -1; } -static int tsdbCopyRowToMem(STsdbRepo *pRepo, SMemRow row, STable *pTable, void **ppRow) { - STsdbCfg * pCfg = &pRepo->config; - TKEY tkey = memRowTKey(row); - TSKEY key = memRowKey(row); - bool isRowDelete = TKEY_IS_DELETED(tkey); - - if (isRowDelete) { - if (!pCfg->update) { - tsdbWarn("vgId:%d vnode is not allowed to update but try to delete a data row", REPO_ID(pRepo)); - terrno = TSDB_CODE_TDB_INVALID_ACTION; - return -1; - } - - TSKEY lastKey = tsdbGetTableLastKeyImpl(pTable); - if (key > lastKey) { - tsdbTrace("vgId:%d skip to delete row key %" PRId64 " which is larger than table lastKey %" PRId64, - REPO_ID(pRepo), key, lastKey); - return 0; - } - } - - void *pRow = tsdbAllocBytes(pRepo, memRowTLen(row)); - if (pRow == NULL) { - tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %" PRIu32 " bytes since %s", - REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), memRowTLen(row), tstrerror(terrno)); - return -1; - } - - memRowCpy(pRow, row); - ppRow[0] = pRow; // save the memory address of data rows - - tsdbTrace("vgId:%d a row is %s table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo), - isRowDelete ? "deleted from" : "updated in", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), - key); - - return 0; -} static int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) { if (pMsg == NULL) { @@ -889,106 +964,6 @@ static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pT return 0; } -static int tsdbInsertDataToTableImpl(STsdbRepo *pRepo, STable *pTable, void **rows, int rowCounter) { - if (rowCounter < 1) return 0; - - SMemTable * pMemTable = NULL; - STableData *pTableData = NULL; - STsdbMeta * pMeta = pRepo->tsdbMeta; - STsdbCfg * pCfg = &(pRepo->config); - - ASSERT(pRepo->mem != NULL); - pMemTable = pRepo->mem; - - if (TABLE_TID(pTable) >= pMemTable->maxTables) { - if (tsdbAdjustMemMaxTables(pMemTable, pMeta->maxTables) < 0) { - tsdbFreeRows(pRepo, rows, rowCounter); - return -1; - } - } - pTableData = pMemTable->tData[TABLE_TID(pTable)]; - - if (pTableData == NULL || pTableData->uid != TABLE_UID(pTable)) { - if (pTableData != NULL) { - taosWLockLatch(&(pMemTable->latch)); - pMemTable->tData[TABLE_TID(pTable)] = NULL; - tsdbFreeTableData(pTableData); - taosWUnLockLatch(&(pMemTable->latch)); - } - - pTableData = tsdbNewTableData(pCfg, pTable); - if (pTableData == NULL) { - tsdbError("vgId:%d failed to insert data to table %s uid %" PRId64 " tid %d since %s", REPO_ID(pRepo), - TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable), tstrerror(terrno)); - tsdbFreeRows(pRepo, rows, rowCounter); - return -1; - } - - pRepo->mem->tData[TABLE_TID(pTable)] = pTableData; - } - - ASSERT((pTableData != NULL) && pTableData->uid == TABLE_UID(pTable)); - - int64_t osize = SL_SIZE(pTableData->pData); - tSkipListPutBatch(pTableData->pData, rows, rowCounter); - int64_t dsize = SL_SIZE(pTableData->pData) - osize; - TSKEY keyFirstRow = memRowKey(rows[0]); - TSKEY keyLastRow = memRowKey(rows[rowCounter - 1]); - - if (pMemTable->keyFirst > keyFirstRow) pMemTable->keyFirst = keyFirstRow; - if (pMemTable->keyLast < keyLastRow) pMemTable->keyLast = keyLastRow; - pMemTable->numOfRows += dsize; - - if (pTableData->keyFirst > keyFirstRow) pTableData->keyFirst = keyFirstRow; - if (pTableData->keyLast < keyLastRow) pTableData->keyLast = keyLastRow; - pTableData->numOfRows += dsize; - - // update table latest info - if (tsdbUpdateTableLatestInfo(pRepo, pTable, rows[rowCounter - 1]) < 0) { - return -1; - } - - return 0; -} - -static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter) { - ASSERT(pRepo->mem != NULL); - STsdbBufPool *pBufPool = pRepo->pPool; - - for (int i = rowCounter - 1; i >= 0; --i) { - SMemRow row = (SMemRow)rows[i]; - int bytes = (int)memRowTLen(row); - - if (pRepo->mem->extraBuffList == NULL) { - STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo); - ASSERT(pBufBlock != NULL && pBufBlock->offset >= bytes); - - pBufBlock->offset -= bytes; - pBufBlock->remain += bytes; - ASSERT(row == POINTER_SHIFT(pBufBlock->data, pBufBlock->offset)); - tsdbTrace("vgId:%d free %d bytes to TSDB buffer pool, nBlocks %d offset %d remain %d", REPO_ID(pRepo), bytes, - listNEles(pRepo->mem->bufBlockList), pBufBlock->offset, pBufBlock->remain); - - if (pBufBlock->offset == 0) { // return the block to buffer pool - if (tsdbLockRepo(pRepo) < 0) return; - SListNode *pNode = tdListPopTail(pRepo->mem->bufBlockList); - tdListPrependNode(pBufPool->bufBlockList, pNode); - if (tsdbUnlockRepo(pRepo) < 0) return; - } - } else { - ASSERT(listNEles(pRepo->mem->extraBuffList) > 0); - SListNode *pNode = tdListPopTail(pRepo->mem->extraBuffList); - ASSERT(row == pNode->data); - free(pNode); - tsdbTrace("vgId:%d free %d bytes to SYSTEM buffer pool", REPO_ID(pRepo), bytes); - - if (listNEles(pRepo->mem->extraBuffList) == 0) { - tdListFree(pRepo->mem->extraBuffList); - pRepo->mem->extraBuffList = NULL; - } - } - } -} static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SMemRow row) { tsdbDebug("vgId:%d updateTableLatestColumn, %s row version:%d", REPO_ID(pRepo), pTable->name->data, @@ -1005,8 +980,8 @@ static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SMemRow ro } SDataCol *pLatestCols = pTable->lastCols; + int32_t kvIdx = 0; - bool isDataRow = isDataRow(row); for (int16_t j = 0; j < schemaNCols(pSchema); j++) { STColumn *pTCol = schemaColAt(pSchema, j); // ignore not exist colId @@ -1017,16 +992,8 @@ static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SMemRow ro void *value = NULL; - if (isDataRow) { - value = tdGetRowDataOfCol(memRowDataBody(row), (int8_t)pTCol->type, - TD_DATA_ROW_HEAD_SIZE + pTCol->offset); - } else { - // SKVRow - SColIdx *pColIdx = tdGetKVRowIdxOfCol(memRowKvBody(row), pTCol->colId); - if (pColIdx) { - value = tdGetKvRowDataOfCol(memRowKvBody(row), pColIdx->offset); - } - } + value = tdGetMemRowDataOfColEx(row, pTCol->colId, (int8_t)pTCol->type, + TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset, &kvIdx); if ((value == NULL) || isNull(value, pTCol->type)) { continue; @@ -1055,13 +1022,14 @@ static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SMemRow r // if cacheLastRow config has been reset, free the lastRow if (!pCfg->cacheLastRow && pTable->lastRow != NULL) { - taosTZfree(pTable->lastRow); + SMemRow cachedLastRow = pTable->lastRow; TSDB_WLOCK_TABLE(pTable); pTable->lastRow = NULL; TSDB_WUNLOCK_TABLE(pTable); + taosTZfree(cachedLastRow); } - if (tsdbGetTableLastKeyImpl(pTable) < memRowKey(row)) { + if (tsdbGetTableLastKeyImpl(pTable) <= memRowKey(row)) { if (CACHE_LAST_ROW(pCfg) || pTable->lastRow != NULL) { SMemRow nrow = pTable->lastRow; if (taosTSizeof(nrow) < memRowTLen(row)) { diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 7a595da39a..f5c01d86e7 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -14,11 +14,14 @@ */ #include "os.h" +#include "tdataformat.h" +#include "tskiplist.h" #include "tulog.h" #include "talgo.h" #include "tcompare.h" #include "exception.h" +#include "taosdef.h" #include "tlosertree.h" #include "tsdbint.h" #include "texpr.h" @@ -68,6 +71,12 @@ typedef struct SLoadCompBlockInfo { int32_t fileId; } SLoadCompBlockInfo; +enum { + CHECKINFO_CHOSEN_MEM = 0, + CHECKINFO_CHOSEN_IMEM = 1, + CHECKINFO_CHOSEN_BOTH = 2 //for update=2(merge case) +}; + typedef struct STableCheckInfo { STableId tableId; @@ -76,7 +85,7 @@ typedef struct STableCheckInfo { SBlockInfo* pCompInfo; int32_t compSize; int32_t numOfBlocks:29; // number of qualified data blocks not the original blocks - int8_t chosen:2; // indicate which iterator should move forward + uint8_t chosen:2; // indicate which iterator should move forward bool initBuf; // whether to initialize the in-memory skip list iterator or not SSkipListIterator* iter; // mem buffer skip list iterator SSkipListIterator* iiter; // imem buffer skip list iterator @@ -781,7 +790,62 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) { tSkipListDestroyIter(pCheckInfo->iiter); } -static SMemRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update) { +static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, int32_t update) { + SMemRow rmem = NULL, rimem = NULL; + if (pCheckInfo->iter) { + SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); + if (node != NULL) { + rmem = (SMemRow)SL_GET_NODE_DATA(node); + } + } + + if (pCheckInfo->iiter) { + SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter); + if (node != NULL) { + rimem = (SMemRow)SL_GET_NODE_DATA(node); + } + } + + if (rmem == NULL && rimem == NULL) { + return TSKEY_INITIAL_VAL; + } + + if (rmem != NULL && rimem == NULL) { + pCheckInfo->chosen = CHECKINFO_CHOSEN_MEM; + return memRowKey(rmem); + } + + if (rmem == NULL && rimem != NULL) { + pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM; + return memRowKey(rimem); + } + + TSKEY r1 = memRowKey(rmem); + TSKEY r2 = memRowKey(rimem); + + if (r1 == r2) { + if(update == TD_ROW_DISCARD_UPDATE){ + pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM; + tSkipListIterNext(pCheckInfo->iter); + } + else if(update == TD_ROW_OVERWRITE_UPDATE) { + pCheckInfo->chosen = CHECKINFO_CHOSEN_MEM; + tSkipListIterNext(pCheckInfo->iiter); + } else { + pCheckInfo->chosen = CHECKINFO_CHOSEN_BOTH; + } + return r1; + } else if (r1 < r2 && ASCENDING_TRAVERSE(order)) { + pCheckInfo->chosen = CHECKINFO_CHOSEN_MEM; + return r1; + } + else { + pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM; + return r2; + } +} + +static SMemRow getSMemRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, SMemRow* extraRow) { SMemRow rmem = NULL, rimem = NULL; if (pCheckInfo->iter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); @@ -814,31 +878,35 @@ static SMemRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, TSKEY r1 = memRowKey(rmem); TSKEY r2 = memRowKey(rimem); - if (r1 == r2) { // data ts are duplicated, ignore the data in mem - if (!update) { + if (r1 == r2) { + if (update == TD_ROW_DISCARD_UPDATE) { tSkipListIterNext(pCheckInfo->iter); - pCheckInfo->chosen = 1; + pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM; return rimem; - } else { + } else if(update == TD_ROW_OVERWRITE_UPDATE){ tSkipListIterNext(pCheckInfo->iiter); - pCheckInfo->chosen = 0; + pCheckInfo->chosen = CHECKINFO_CHOSEN_MEM; + return rmem; + } else { + pCheckInfo->chosen = CHECKINFO_CHOSEN_BOTH; + extraRow = rimem; return rmem; } } else { if (ASCENDING_TRAVERSE(order)) { if (r1 < r2) { - pCheckInfo->chosen = 0; + pCheckInfo->chosen = CHECKINFO_CHOSEN_MEM; return rmem; } else { - pCheckInfo->chosen = 1; + pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM; return rimem; } } else { if (r1 < r2) { - pCheckInfo->chosen = 1; + pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM; return rimem; } else { - pCheckInfo->chosen = 0; + pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM; return rmem; } } @@ -847,7 +915,7 @@ static SMemRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) { bool hasNext = false; - if (pCheckInfo->chosen == 0) { + if (pCheckInfo->chosen == CHECKINFO_CHOSEN_MEM) { if (pCheckInfo->iter != NULL) { hasNext = tSkipListIterNext(pCheckInfo->iter); } @@ -859,7 +927,7 @@ static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) { if (pCheckInfo->iiter != NULL) { return tSkipListIterGet(pCheckInfo->iiter) != NULL; } - } else { //pCheckInfo->chosen == 1 + } else if (pCheckInfo->chosen == CHECKINFO_CHOSEN_IMEM){ if (pCheckInfo->iiter != NULL) { hasNext = tSkipListIterNext(pCheckInfo->iiter); } @@ -871,6 +939,13 @@ static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) { if (pCheckInfo->iter != NULL) { return tSkipListIterGet(pCheckInfo->iter) != NULL; } + } else { + if (pCheckInfo->iter != NULL) { + hasNext = tSkipListIterNext(pCheckInfo->iter); + } + if (pCheckInfo->iiter != NULL) { + hasNext = tSkipListIterNext(pCheckInfo->iiter) || hasNext; + } } return hasNext; @@ -891,7 +966,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { initTableMemIterator(pHandle, pCheckInfo); } - SMemRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order, pCfg->update); + SMemRow row = getSMemRowInTableMem(pCheckInfo, pHandle->order, pCfg->update, NULL); if (row == NULL) { return false; } @@ -1147,25 +1222,28 @@ static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t c static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, int32_t numOfCols); static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle); static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos); +static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, int32_t update); static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo){ SQueryFilePos* cur = &pQueryHandle->cur; STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); + TSKEY key; int32_t code = TSDB_CODE_SUCCESS; /*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo); - SMemRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); assert(cur->pos >= 0 && cur->pos <= binfo.rows); - TSKEY key = (row != NULL) ? memRowKey(row) : TSKEY_INITIAL_VAL; + key = extractFirstTraverseKey(pCheckInfo, pQueryHandle->order, pCfg->update); + if (key != TSKEY_INITIAL_VAL) { tsdbDebug("%p key in mem:%"PRId64", 0x%"PRIx64, pQueryHandle, key, pQueryHandle->qId); } else { tsdbDebug("%p no data in mem, 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); } + if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) || (!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) { @@ -1190,6 +1268,7 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p return code; } + // return error, add test cases if ((code = doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { return code; @@ -1452,186 +1531,204 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity return numOfRows + num; } -static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, SMemRow row, - int32_t numOfCols, STable* pTable, STSchema* pSchema) { +// Note: row1 always has high priority +static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, + SMemRow row1, SMemRow row2, int32_t numOfCols, STable* pTable, + STSchema* pSchema1, STSchema* pSchema2, bool forceSetNull) { char* pData = NULL; + STSchema* pSchema; + SMemRow row; + int16_t colId; + int16_t offset; - // the schema version info is embedded in SDataRow, and use latest schema version for SKVRow - int32_t numOfRowCols = 0; - if (pSchema == NULL) { - pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row)); - numOfRowCols = schemaNCols(pSchema); + bool isRow1DataRow = isDataRow(row1); + bool isRow2DataRow; + bool isChosenRowDataRow; + int32_t chosen_itr; + void *value; + + // the schema version info is embeded in SDataRow + int32_t numOfColsOfRow1 = 0; + + if (pSchema1 == NULL) { + pSchema1 = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row1)); + } + if(isRow1DataRow) { + numOfColsOfRow1 = schemaNCols(pSchema1); } else { - numOfRowCols = schemaNCols(pSchema); + numOfColsOfRow1 = kvRowNCols(memRowKvBody(row1)); } - int32_t i = 0; - - if (isDataRow(row)) { - SDataRow dataRow = memRowDataBody(row); - int32_t j = 0; - while (i < numOfCols && j < numOfRowCols) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - if (pSchema->columns[j].colId < pColInfo->info.colId) { - j++; - continue; - } - - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; - } else { - pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; - } - - if (pSchema->columns[j].colId == pColInfo->info.colId) { - void* value = - tdGetRowDataOfCol(dataRow, (int8_t)pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset); - switch (pColInfo->info.type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - memcpy(pData, value, varDataTLen(value)); - break; - case TSDB_DATA_TYPE_NULL: - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - *(uint8_t*)pData = *(uint8_t*)value; - break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - *(uint16_t*)pData = *(uint16_t*)value; - break; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - *(uint32_t*)pData = *(uint32_t*)value; - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - *(uint64_t*)pData = *(uint64_t*)value; - break; - case TSDB_DATA_TYPE_FLOAT: - SET_FLOAT_PTR(pData, value); - break; - case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_PTR(pData, value); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - *(TSKEY*)pData = tdGetKey(*(TKEY*)value); - } else { - *(TSKEY*)pData = *(TSKEY*)value; - } - break; - default: - memcpy(pData, value, pColInfo->info.bytes); - } - - j++; - i++; - } else { // pColInfo->info.colId < pSchema->columns[j].colId, it is a NULL data - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(pData, pColInfo->info.type); - } else { - setNull(pData, pColInfo->info.type, pColInfo->info.bytes); - } - i++; - } + int32_t numOfColsOfRow2 = 0; + if(row2) { + isRow2DataRow = isDataRow(row2); + if (pSchema2 == NULL) { + pSchema2 = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row2)); } - } else if (isKvRow(row)) { - SKVRow kvRow = memRowKvBody(row); - int32_t k = 0; - int32_t nKvRowCols = kvRowNCols(kvRow); - - while (i < numOfCols && k < nKvRowCols) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - SColIdx* pColIdx = kvRowColIdxAt(kvRow, k); - - if (pColIdx->colId < pColInfo->info.colId) { - ++k; - continue; - } - - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; - } else { - pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; - } - - if (pColIdx->colId == pColInfo->info.colId) { - // offset of pColIdx for SKVRow including the TD_KV_ROW_HEAD_SIZE - void* value = tdGetKvRowDataOfCol(kvRow, pColIdx->offset); - switch (pColInfo->info.type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - memcpy(pData, value, varDataTLen(value)); - break; - case TSDB_DATA_TYPE_NULL: - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - *(uint8_t*)pData = *(uint8_t*)value; - break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - *(uint16_t*)pData = *(uint16_t*)value; - break; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - *(uint32_t*)pData = *(uint32_t*)value; - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - *(uint64_t*)pData = *(uint64_t*)value; - break; - case TSDB_DATA_TYPE_FLOAT: - SET_FLOAT_PTR(pData, value); - break; - case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_PTR(pData, value); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - *(TSKEY*)pData = tdGetKey(*(TKEY*)value); - } else { - *(TSKEY*)pData = *(TSKEY*)value; - } - break; - default: - memcpy(pData, value, pColInfo->info.bytes); - } - ++k; - ++i; - continue; - } - // If (pColInfo->info.colId < pColIdx->colId), it is NULL data - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(pData, pColInfo->info.type); - } else { - setNull(pData, pColInfo->info.type, pColInfo->info.bytes); - } - ++i; + if(isRow2DataRow) { + numOfColsOfRow2 = schemaNCols(pSchema2); + } else { + numOfColsOfRow2 = kvRowNCols(memRowKvBody(row2)); } - } else { - ASSERT(0); } - while (i < numOfCols) { // the remain columns are all null data + + int32_t i = 0, j = 0, k = 0; + while(i < numOfCols && (j < numOfColsOfRow1 || k < numOfColsOfRow2)) { SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + if (ASCENDING_TRAVERSE(pQueryHandle->order)) { pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; } else { pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; } - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(pData, pColInfo->info.type); + int32_t colIdOfRow1; + if(j >= numOfColsOfRow1) { + colIdOfRow1 = INT32_MAX; + } else if(isRow1DataRow) { + colIdOfRow1 = pSchema1->columns[j].colId; } else { - setNull(pData, pColInfo->info.type, pColInfo->info.bytes); + void *rowBody = memRowKvBody(row1); + SColIdx *pColIdx = kvRowColIdxAt(rowBody, j); + colIdOfRow1 = pColIdx->colId; } - i++; + int32_t colIdOfRow2; + if(k >= numOfColsOfRow2) { + colIdOfRow2 = INT32_MAX; + } else if(isRow2DataRow) { + colIdOfRow2 = pSchema2->columns[k].colId; + } else { + void *rowBody = memRowKvBody(row2); + SColIdx *pColIdx = kvRowColIdxAt(rowBody, k); + colIdOfRow2 = pColIdx->colId; + } + + if(colIdOfRow1 == colIdOfRow2) { + if(colIdOfRow1 < pColInfo->info.colId) { + j++; + k++; + continue; + } + row = row1; + pSchema = pSchema1; + isChosenRowDataRow = isRow1DataRow; + chosen_itr = j; + } else if(colIdOfRow1 < colIdOfRow2) { + if(colIdOfRow1 < pColInfo->info.colId) { + j++; + continue; + } + row = row1; + pSchema = pSchema1; + isChosenRowDataRow = isRow1DataRow; + chosen_itr = j; + } else { + if(colIdOfRow2 < pColInfo->info.colId) { + k++; + continue; + } + row = row2; + pSchema = pSchema2; + chosen_itr = k; + isChosenRowDataRow = isRow2DataRow; + } + if(isChosenRowDataRow) { + colId = pSchema->columns[chosen_itr].colId; + offset = pSchema->columns[chosen_itr].offset; + void *rowBody = memRowDataBody(row); + value = tdGetRowDataOfCol(rowBody, (int8_t)pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + offset); + } else { + void *rowBody = memRowKvBody(row); + SColIdx *pColIdx = kvRowColIdxAt(rowBody, chosen_itr); + colId = pColIdx->colId; + offset = pColIdx->offset; + value = tdGetKvRowDataOfCol(rowBody, pColIdx->offset); + } + + + if (colId == pColInfo->info.colId) { + if(forceSetNull || (!isNull(value, (int8_t)pColInfo->info.type))) { + switch (pColInfo->info.type) { + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + memcpy(pData, value, varDataTLen(value)); + break; + case TSDB_DATA_TYPE_NULL: + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + *(uint8_t *)pData = *(uint8_t *)value; + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + *(uint16_t *)pData = *(uint16_t *)value; + break; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + *(uint32_t *)pData = *(uint32_t *)value; + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + *(uint64_t *)pData = *(uint64_t *)value; + break; + case TSDB_DATA_TYPE_FLOAT: + SET_FLOAT_PTR(pData, value); + break; + case TSDB_DATA_TYPE_DOUBLE: + SET_DOUBLE_PTR(pData, value); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + *(TSKEY *)pData = tdGetKey(*(TKEY *)value); + } else { + *(TSKEY *)pData = *(TSKEY *)value; + } + break; + default: + memcpy(pData, value, pColInfo->info.bytes); + } + } + i++; + + if(row == row1) { + j++; + } else { + k++; + } + } else { + if(forceSetNull) { + if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { + setVardataNull(pData, pColInfo->info.type); + } else { + setNull(pData, pColInfo->info.type, pColInfo->info.bytes); + } + } + i++; + } + } + + if(forceSetNull) { + while (i < numOfCols) { // the remain columns are all null data + SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; + } else { + pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; + } + + if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { + setVardataNull(pData, pColInfo->info.type); + } else { + setNull(pData, pColInfo->info.type, pColInfo->info.bytes); + } + + i++; + } } } + static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, int32_t numOfCols) { if (numOfRows == 0 || ASCENDING_TRAVERSE(pQueryHandle->order)) { return; @@ -1798,8 +1895,10 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* // compared with the data from in-memory buffer, to generate the correct timestamp array list int32_t numOfRows = 0; - int16_t rv = -1; - STSchema* pSchema = NULL; + int16_t rv1 = -1; + int16_t rv2 = -1; + STSchema* pSchema1 = NULL; + STSchema* pSchema2 = NULL; int32_t pos = cur->pos; cur->win = TSWINDOW_INITIALIZER; @@ -1811,12 +1910,13 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) { SSkipListNode* node = NULL; do { - SMemRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); - if (row == NULL) { + SMemRow row2 = NULL; + SMemRow row1 = getSMemRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update, &row2); + if (row1 == NULL) { break; } - TSKEY key = memRowKey(row); + TSKEY key = memRowKey(row1); if ((key > pQueryHandle->window.ekey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < pQueryHandle->window.ekey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { break; @@ -1829,12 +1929,16 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key > tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { - if (rv != memRowVersion(row)) { - pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row)); - rv = memRowVersion(row); + if (rv1 != memRowVersion(row1)) { + pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); + rv1 = memRowVersion(row1); } - - copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable, pSchema); + if(row2 && rv2 != memRowVersion(row2)) { + pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); + rv2 = memRowVersion(row2); + } + + mergeTwoRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row1, row2, numOfCols, pTable, pSchema1, pSchema2, true); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -1847,12 +1951,20 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* moveToNextRowInMem(pCheckInfo); } else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it if (pCfg->update) { - if (rv != memRowVersion(row)) { - pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row)); - rv = memRowVersion(row); + if(pCfg->update == TD_ROW_PARTIAL_UPDATE) { + doCopyRowsFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, pos, pos); } - - copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable, pSchema); + if (rv1 != memRowVersion(row1)) { + pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); + rv1 = memRowVersion(row1); + } + if(row2 && rv2 != memRowVersion(row2)) { + pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); + rv2 = memRowVersion(row2); + } + + bool forceSetNull = pCfg->update != TD_ROW_PARTIAL_UPDATE; + mergeTwoRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row1, row2, numOfCols, pTable, pSchema1, pSchema2, forceSetNull); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -1877,7 +1989,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* assert(end != -1); if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it - if (!pCfg->update) { + if (pCfg->update == TD_ROW_DISCARD_UPDATE) { moveToNextRowInMem(pCheckInfo); } else { end -= step; @@ -2490,7 +2602,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int STSchema* pSchema = NULL; do { - SMemRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); + SMemRow row = getSMemRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update, NULL); if (row == NULL) { break; } @@ -2512,7 +2624,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row)); rv = memRowVersion(row); } - copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, numOfCols, pTable, pSchema); + mergeTwoRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, NULL, numOfCols, pTable, pSchema, NULL, true); if (++numOfRows >= maxRowsToRead) { moveToNextRowInMem(pCheckInfo); @@ -2637,7 +2749,7 @@ static bool loadCachedLastRow(STsdbQueryHandle* pQueryHandle) { if (ret != TSDB_CODE_SUCCESS) { return false; } - copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, numOfCols, pCheckInfo->pTableObj, NULL); + mergeTwoRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, NULL, numOfCols, pCheckInfo->pTableObj, NULL, NULL, true); tfree(pRow); // update the last key value @@ -2920,7 +3032,7 @@ static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SM } } - SArray* row = (type == TSDB_PREV_ROW)? pQueryHandle->prev:pQueryHandle->next; + SArray* row = (type == TSDB_PREV_ROW)? pQueryHandle->prev : pQueryHandle->next; for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); @@ -3787,10 +3899,6 @@ static void* doFreeColumnInfoData(SArray* pColumnInfoData) { } static void* destroyTableCheckInfo(SArray* pTableCheckInfo) { - if (pTableCheckInfo == NULL) { - return NULL; - } - size_t size = taosArrayGetSize(pTableCheckInfo); for (int32_t i = 0; i < size; ++i) { STableCheckInfo* p = taosArrayGet(pTableCheckInfo, i); @@ -3834,6 +3942,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); SIOCostSummary* pCost = &pQueryHandle->cost; + tsdbDebug("%p :io-cost summary: head-file read cnt:%"PRIu64", head-file time:%"PRIu64" us, statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, 0x%"PRIx64, pQueryHandle, pCost->headFileLoad, pCost->headFileLoadTime, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qId); diff --git a/src/tsdb/src/tsdbReadImpl.c b/src/tsdb/src/tsdbReadImpl.c index dd14dc700f..666a2d3571 100644 --- a/src/tsdb/src/tsdbReadImpl.c +++ b/src/tsdb/src/tsdbReadImpl.c @@ -244,6 +244,7 @@ int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) { int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { ASSERT(pBlock->numOfSubBlocks > 0); + int8_t update = pReadh->pRepo->config.update; SBlock *iBlock = pBlock; if (pBlock->numOfSubBlocks > 1) { @@ -258,7 +259,7 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { for (int i = 1; i < pBlock->numOfSubBlocks; i++) { iBlock++; if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[1]) < 0) return -1; - if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL) < 0) return -1; + if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL, update != TD_ROW_PARTIAL_UPDATE) < 0) return -1; } ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows); @@ -270,6 +271,7 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, int16_t *colIds, int numOfColsIds) { ASSERT(pBlock->numOfSubBlocks > 0); + int8_t update = pReadh->pRepo->config.update; SBlock *iBlock = pBlock; if (pBlock->numOfSubBlocks > 1) { @@ -284,7 +286,7 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, for (int i = 1; i < pBlock->numOfSubBlocks; i++) { iBlock++; if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds) < 0) return -1; - if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL) < 0) return -1; + if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL, update != TD_ROW_PARTIAL_UPDATE) < 0) return -1; } ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows); @@ -657,4 +659,4 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc } return 0; -} \ No newline at end of file +} diff --git a/src/tsdb/src/tsdbRowMergeBuf.c b/src/tsdb/src/tsdbRowMergeBuf.c new file mode 100644 index 0000000000..5ce580f70f --- /dev/null +++ b/src/tsdb/src/tsdbRowMergeBuf.c @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbRowMergeBuf.h" +#include "tdataformat.h" + +// row1 has higher priority +SMemRow tsdbMergeTwoRows(SMergeBuf *pBuf, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2) { + if(row2 == NULL) return row1; + if(row1 == NULL) return row2; + ASSERT(pSchema1->version == memRowVersion(row1)); + ASSERT(pSchema2->version == memRowVersion(row2)); + + if(tsdbMergeBufMakeSureRoom(pBuf, pSchema1, pSchema2) < 0) { + return NULL; + } + return mergeTwoMemRows(*pBuf, row1, row2, pSchema1, pSchema2); +} diff --git a/src/util/inc/tfunctional.h b/src/util/inc/tfunctional.h new file mode 100644 index 0000000000..70f54e921d --- /dev/null +++ b/src/util/inc/tfunctional.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef TD_TFUNCTIONAL_H +#define TD_TFUNCTIONAL_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "os.h" + +//TODO: hard to use, trying to rewrite it using va_list + +typedef void* (*GenericVaFunc)(void* args[]); +typedef int32_t (*I32VaFunc) (void* args[]); +typedef void (*VoidVaFunc) (void* args[]); + +typedef struct GenericSavedFunc { + GenericVaFunc func; + void * args[]; +} tGenericSavedFunc; + +typedef struct I32SavedFunc { + I32VaFunc func; + void * args[]; +} tI32SavedFunc; + +typedef struct VoidSavedFunc { + VoidVaFunc func; + void * args[]; +} tVoidSavedFunc; + +tGenericSavedFunc* genericSavedFuncInit(GenericVaFunc func, int numOfArgs); +tI32SavedFunc* i32SavedFuncInit(I32VaFunc func, int numOfArgs); +tVoidSavedFunc* voidSavedFuncInit(VoidVaFunc func, int numOfArgs); +void* genericInvoke(tGenericSavedFunc* const pSavedFunc); +int32_t i32Invoke(tI32SavedFunc* const pSavedFunc); +void voidInvoke(tVoidSavedFunc* const pSavedFunc); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/util/inc/tskiplist.h b/src/util/inc/tskiplist.h index 17f5940b49..d9dc001ccd 100644 --- a/src/util/inc/tskiplist.h +++ b/src/util/inc/tskiplist.h @@ -23,6 +23,7 @@ extern "C" { #include "os.h" #include "taosdef.h" #include "tarray.h" +#include "tfunctional.h" #define MAX_SKIP_LIST_LEVEL 15 #define SKIP_LIST_RECORD_PERFORMANCE 0 @@ -30,13 +31,17 @@ extern "C" { // For key property setting #define SL_ALLOW_DUP_KEY (uint8_t)0x0 // Allow duplicate key exists (for tag index usage) #define SL_DISCARD_DUP_KEY (uint8_t)0x1 // Discard duplicate key (for data update=0 case) -#define SL_UPDATE_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert (for data update=1 case) +#define SL_UPDATE_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert (for data update!=0 case) + // For thread safety setting #define SL_THREAD_SAFE (uint8_t)0x4 typedef char *SSkipListKey; typedef char *(*__sl_key_fn_t)(const void *); +typedef void (*sl_patch_row_fn_t)(void * pDst, const void * pSrc); +typedef void* (*iter_next_fn_t)(void *iter); + typedef struct SSkipListNode { uint8_t level; void * pData; @@ -95,6 +100,12 @@ typedef struct tSkipListState { uint64_t nTotalElapsedTimeForInsert; } tSkipListState; +typedef enum { + SSkipListPutSuccess = 0, + SSkipListPutEarlyStop = 1, + SSkipListPutSkipOne = 2 +} SSkipListPutStatus; + typedef struct SSkipList { unsigned int seed; __compar_fn_t comparFn; @@ -111,6 +122,7 @@ typedef struct SSkipList { #if SKIP_LIST_RECORD_PERFORMANCE tSkipListState state; // skiplist state #endif + tGenericSavedFunc* insertHandleFn; } SSkipList; typedef struct SSkipListIterator { @@ -118,7 +130,7 @@ typedef struct SSkipListIterator { SSkipListNode *cur; int32_t step; // the number of nodes that have been checked already int32_t order; // order of the iterator - SSkipListNode *next; // next points to the true qualified node in skip list + SSkipListNode *next; // next points to the true qualified node in skiplist } SSkipListIterator; #define SL_IS_THREAD_SAFE(s) (((s)->flags) & SL_THREAD_SAFE) @@ -132,7 +144,7 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, _ __sl_key_fn_t fn); void tSkipListDestroy(SSkipList *pSkipList); SSkipListNode * tSkipListPut(SSkipList *pSkipList, void *pData); -void tSkipListPutBatch(SSkipList *pSkipList, void **ppData, int ndata); +void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t iterate); SArray * tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey); void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel); SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList); diff --git a/src/util/src/tfunctional.c b/src/util/src/tfunctional.c new file mode 100644 index 0000000000..c470a2b8ae --- /dev/null +++ b/src/util/src/tfunctional.c @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tfunctional.h" +#include "tarray.h" + + +tGenericSavedFunc* genericSavedFuncInit(GenericVaFunc func, int numOfArgs) { + tGenericSavedFunc* pSavedFunc = malloc(sizeof(tGenericSavedFunc) + numOfArgs * (sizeof(void*))); + pSavedFunc->func = func; + return pSavedFunc; +} + +tI32SavedFunc* i32SavedFuncInit(I32VaFunc func, int numOfArgs) { + tI32SavedFunc* pSavedFunc = malloc(sizeof(tI32SavedFunc) + numOfArgs * sizeof(void *)); + pSavedFunc->func = func; + return pSavedFunc; +} + +tVoidSavedFunc* voidSavedFuncInit(VoidVaFunc func, int numOfArgs) { + tVoidSavedFunc* pSavedFunc = malloc(sizeof(tVoidSavedFunc) + numOfArgs * sizeof(void*)); + pSavedFunc->func = func; + return pSavedFunc; +} + +FORCE_INLINE void* genericInvoke(tGenericSavedFunc* const pSavedFunc) { + return pSavedFunc->func(pSavedFunc->args); +} + +FORCE_INLINE int32_t i32Invoke(tI32SavedFunc* const pSavedFunc) { + return pSavedFunc->func(pSavedFunc->args); +} + +FORCE_INLINE void voidInvoke(tVoidSavedFunc* const pSavedFunc) { + if(pSavedFunc) pSavedFunc->func(pSavedFunc->args); +} diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 082b454bb5..b464519ba6 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -16,6 +16,7 @@ #include "tskiplist.h" #include "os.h" #include "tcompare.h" +#include "tdataformat.h" #include "tulog.h" #include "tutil.h" @@ -31,6 +32,7 @@ static SSkipListNode *tSkipListNewNode(uint8_t level); static SSkipListNode *tSkipListPutImpl(SSkipList *pSkipList, void *pData, SSkipListNode **direction, bool isForward, bool hasDup); + static FORCE_INLINE int tSkipListWLock(SSkipList *pSkipList); static FORCE_INLINE int tSkipListRLock(SSkipList *pSkipList); static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList); @@ -80,6 +82,7 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, _ #if SKIP_LIST_RECORD_PERFORMANCE pSkipList->state.nTotalMemSize += sizeof(SSkipList); #endif + pSkipList->insertHandleFn = NULL; return pSkipList; } @@ -97,6 +100,8 @@ void tSkipListDestroy(SSkipList *pSkipList) { tSkipListFreeNode(pTemp); } + tfree(pSkipList->insertHandleFn); + tSkipListUnlock(pSkipList); if (pSkipList->lock != NULL) { pthread_rwlock_destroy(pSkipList->lock); @@ -124,8 +129,7 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData) { return pNode; } -// Put a batch of data into skiplist. The batch of data must be in ascending order -void tSkipListPutBatch(SSkipList *pSkipList, void **ppData, int ndata) { +void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t iterate) { SSkipListNode *backward[MAX_SKIP_LIST_LEVEL] = {0}; SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; bool hasDup = false; @@ -135,17 +139,21 @@ void tSkipListPutBatch(SSkipList *pSkipList, void **ppData, int ndata) { tSkipListWLock(pSkipList); + void* pData = iterate(iter); + if(pData == NULL) return; + // backward to put the first data - hasDup = tSkipListGetPosToPut(pSkipList, backward, ppData[0]); - tSkipListPutImpl(pSkipList, ppData[0], backward, false, hasDup); + hasDup = tSkipListGetPosToPut(pSkipList, backward, pData); + + tSkipListPutImpl(pSkipList, pData, backward, false, hasDup); for (int level = 0; level < pSkipList->maxLevel; level++) { forward[level] = SL_NODE_GET_BACKWARD_POINTER(backward[level], level); } // forward to put the rest of data - for (int idata = 1; idata < ndata; idata++) { - pDataKey = pSkipList->keyFn(ppData[idata]); + while ((pData = iterate(iter)) != NULL) { + pDataKey = pSkipList->keyFn(pData); hasDup = false; // Compare max key @@ -186,9 +194,8 @@ void tSkipListPutBatch(SSkipList *pSkipList, void **ppData, int ndata) { } } - tSkipListPutImpl(pSkipList, ppData[idata], forward, true, hasDup); + tSkipListPutImpl(pSkipList, pData, forward, true, hasDup); } - tSkipListUnlock(pSkipList); } @@ -661,18 +668,40 @@ static SSkipListNode *tSkipListPutImpl(SSkipList *pSkipList, void *pData, SSkipL uint8_t dupMode = SL_DUP_MODE(pSkipList); SSkipListNode *pNode = NULL; - if (hasDup && (dupMode == SL_DISCARD_DUP_KEY || dupMode == SL_UPDATE_DUP_KEY)) { + if (hasDup && (dupMode != SL_ALLOW_DUP_KEY)) { if (dupMode == SL_UPDATE_DUP_KEY) { if (isForward) { pNode = SL_NODE_GET_FORWARD_POINTER(direction[0], 0); } else { pNode = SL_NODE_GET_BACKWARD_POINTER(direction[0], 0); } - atomic_store_ptr(&(pNode->pData), pData); + if (pSkipList->insertHandleFn) { + pSkipList->insertHandleFn->args[0] = pData; + pSkipList->insertHandleFn->args[1] = pNode->pData; + pData = genericInvoke(pSkipList->insertHandleFn); + } + if(pData) { + atomic_store_ptr(&(pNode->pData), pData); + } + } else { + //for compatiblity, duplicate key inserted when update=0 should be also calculated as affected rows! + if(pSkipList->insertHandleFn) { + pSkipList->insertHandleFn->args[0] = NULL; + pSkipList->insertHandleFn->args[1] = NULL; + genericInvoke(pSkipList->insertHandleFn); + } } } else { pNode = tSkipListNewNode(getSkipListRandLevel(pSkipList)); if (pNode != NULL) { + // insertHandleFn will be assigned only for timeseries data, + // in which case, pData is pointed to an memory to be freed later; + // while for metadata, the mem alloc will not be called. + if (pSkipList->insertHandleFn) { + pSkipList->insertHandleFn->args[0] = pData; + pSkipList->insertHandleFn->args[1] = NULL; + pData = genericInvoke(pSkipList->insertHandleFn); + } pNode->pData = pData; tSkipListDoInsert(pSkipList, direction, pNode, isForward); From d0b7e937ac0ecdcd1254075356d32076010ed820 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 29 Jul 2021 14:11:09 +0800 Subject: [PATCH 083/106] make CI happy --- tests/pytest/update/merge_commit_data2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pytest/update/merge_commit_data2.py b/tests/pytest/update/merge_commit_data2.py index 3f0fc718ad..5fc58e3b4f 100644 --- a/tests/pytest/update/merge_commit_data2.py +++ b/tests/pytest/update/merge_commit_data2.py @@ -28,6 +28,7 @@ class TDTestCase: def restart_taosd(self,db): tdDnodes.stop(1) tdDnodes.startWithoutSleep(1) + tdsql.sleep(2) tdSql.execute("use %s;" % db) def date_to_timestamp_microseconds(self, date): From 9fc4f7466f14527ebec5e7a3631b70f370327a48 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 29 Jul 2021 14:11:09 +0800 Subject: [PATCH 084/106] make CI happy --- tests/pytest/update/merge_commit_data2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pytest/update/merge_commit_data2.py b/tests/pytest/update/merge_commit_data2.py index 3f0fc718ad..5fc58e3b4f 100644 --- a/tests/pytest/update/merge_commit_data2.py +++ b/tests/pytest/update/merge_commit_data2.py @@ -28,6 +28,7 @@ class TDTestCase: def restart_taosd(self,db): tdDnodes.stop(1) tdDnodes.startWithoutSleep(1) + tdsql.sleep(2) tdSql.execute("use %s;" % db) def date_to_timestamp_microseconds(self, date): From ab885b6b8c57d68b41bf4234d4b4507549cc7727 Mon Sep 17 00:00:00 2001 From: tickduan <417921451@qq.com> Date: Thu, 29 Jul 2021 14:52:33 +0800 Subject: [PATCH 085/106] fixed BUG TD-5596 --- src/util/src/tcompression.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/util/src/tcompression.c b/src/util/src/tcompression.c index 4472633304..b6b63f46c8 100644 --- a/src/util/src/tcompression.c +++ b/src/util/src/tcompression.c @@ -480,6 +480,10 @@ int tsCompressTimestampImp(const char *const input, const int nelements, char *c int64_t *istream = (int64_t *)input; int64_t prev_value = istream[0]; + if(prev_value >= 0x8000000000000000) { + uWarn("compression timestamp is over signed long long range. ts = 0x%"PRIx64" \n", prev_value); + goto _exit_over; + } int64_t prev_delta = -prev_value; uint8_t flags = 0, flag1 = 0, flag2 = 0; uint64_t dd1 = 0, dd2 = 0; From 8ad74438229b6597e7b8f071ad06decc2768e48e Mon Sep 17 00:00:00 2001 From: xiaolei li <85657333+xleili@users.noreply.github.com> Date: Thu, 29 Jul 2021 16:37:58 +0800 Subject: [PATCH 086/106] [TD-4944]:c# connector nanosecond support (#7041) * [TD-4944]:c# connector nanosecond support * [TD-4944]:c# connector nanosecond support * [TD-4944]:c# connector nanosecond support fix indent --- src/connector/C#/TDengineDriver.cs | 3 + tests/examples/C#/C#checker/C#checker.cs | 55 +++- tests/examples/C#/C#checker/TDengineDriver.cs | 267 ++++++++++-------- tests/examples/C#/TDengineDriver.cs | 4 + tests/examples/C#/taosdemo/TDengineDriver.cs | 3 + 5 files changed, 196 insertions(+), 136 deletions(-) diff --git a/src/connector/C#/TDengineDriver.cs b/src/connector/C#/TDengineDriver.cs index 2c150341f6..e6c3a598ad 100644 --- a/src/connector/C#/TDengineDriver.cs +++ b/src/connector/C#/TDengineDriver.cs @@ -163,5 +163,8 @@ namespace TDengineDriver [DllImport("taos", EntryPoint = "taos_close", CallingConvention = CallingConvention.Cdecl)] static extern public int Close(IntPtr taos); + //get precisionŁ¬in parameter restultset + [DllImport("taos", EntryPoint = "taos_result_precision", CallingConvention = CallingConvention.Cdecl)] + static extern public int ResultPrecision(IntPtr taos); } } diff --git a/tests/examples/C#/C#checker/C#checker.cs b/tests/examples/C#/C#checker/C#checker.cs index 80fa3b8386..29ad290343 100644 --- a/tests/examples/C#/C#checker/C#checker.cs +++ b/tests/examples/C#/C#checker/C#checker.cs @@ -33,7 +33,7 @@ namespace TDengineDriver //sql parameters private string dbName; private string tbName; - + private string precision; private bool isInsertData; private bool isQueryData; @@ -61,9 +61,9 @@ namespace TDengineDriver tester.checkInsert(); tester.checkSelect(); tester.checkDropTable(); - + tester.dropDatabase(); tester.CloseConnection(); - + tester.cleanup(); } @@ -156,7 +156,9 @@ namespace TDengineDriver Console.WriteLine("{0:G}{1:G}{2:G}", indent, indent, "How many rows to insert, default is 100"); Console.WriteLine("{0:G}{1:G}", indent, "-c"); Console.WriteLine("{0:G}{1:G}{2:G}", indent, indent, "Configuration directory"); - + // + Console.WriteLine("{0:G}{1:G}", indent, "-ps"); + Console.WriteLine("{0:G}{1:G}{2:G}", indent, indent, "Configurate db precision,default millisecond"); ExitProgram(); } } @@ -168,9 +170,9 @@ namespace TDengineDriver host = this.GetArgumentAsString(argv, "-h", "127.0.0.1"); user = this.GetArgumentAsString(argv, "-u", "root"); password = this.GetArgumentAsString(argv, "-p", "taosdata"); - dbName = this.GetArgumentAsString(argv, "-db", "test"); + dbName = this.GetArgumentAsString(argv, "-d", "test"); tbName = this.GetArgumentAsString(argv, "-s", "weather"); - + precision = this.GetArgumentAsString(argv, "-ps", "ms"); isInsertData = this.GetArgumentAsLong(argv, "-w", 0, 1, 1) != 0; isQueryData = this.GetArgumentAsLong(argv, "-r", 0, 1, 1) != 0; tableCount = this.GetArgumentAsLong(argv, "-n", 1, 10000, 10); @@ -183,6 +185,7 @@ namespace TDengineDriver { TDengine.Options((int)TDengineInitOption.TDDB_OPTION_CONFIGDIR, this.configDir); TDengine.Options((int)TDengineInitOption.TDDB_OPTION_SHELL_ACTIVITY_TIMER, "60"); + Console.WriteLine("init..."); TDengine.Init(); Console.WriteLine("get connection starting..."); } @@ -204,7 +207,7 @@ namespace TDengineDriver public void createDatabase() { StringBuilder sql = new StringBuilder(); - sql.Append("create database if not exists ").Append(this.dbName); + sql.Append("create database if not exists ").Append(this.dbName).Append(" precision '").Append(this.precision).Append("'"); execute(sql.ToString()); } public void useDatabase() @@ -216,8 +219,8 @@ namespace TDengineDriver public void checkSelect() { StringBuilder sql = new StringBuilder(); - sql.Append("select * from test.weather"); - execute(sql.ToString()); + sql.Append("select * from ").Append(this.dbName).Append(".").Append(this.tbName); + ExecuteQuery(sql.ToString()); } public void createTable() { @@ -228,7 +231,7 @@ namespace TDengineDriver public void checkInsert() { StringBuilder sql = new StringBuilder(); - sql.Append("insert into test.weather (ts, temperature, humidity) values(now, 20.5, 34)"); + sql.Append("insert into ").Append(this.dbName).Append(".").Append(this.tbName).Append("(ts, temperature, humidity) values(now, 20.5, 34)"); execute(sql.ToString()); } public void checkDropTable() @@ -237,6 +240,12 @@ namespace TDengineDriver sql.Append("drop table if exists ").Append(this.dbName).Append(".").Append(this.tbName).Append(""); execute(sql.ToString()); } + public void dropDatabase() + { + StringBuilder sql = new StringBuilder(); + sql.Append("drop database if exists ").Append(this.dbName); + execute(sql.ToString()); + } public void execute(string sql) { DateTime dt1 = DateTime.Now; @@ -266,6 +275,7 @@ namespace TDengineDriver DateTime dt1 = DateTime.Now; long queryRows = 0; IntPtr res = TDengine.Query(conn, sql); + getPrecision(res); if ((res == IntPtr.Zero) || (TDengine.ErrorNo(res) != 0)) { Console.Write(sql.ToString() + " failure, "); @@ -379,8 +389,31 @@ namespace TDengineDriver static void ExitProgram() { - TDengine.Cleanup(); System.Environment.Exit(0); } + + public void cleanup() + { + Console.WriteLine("clean up..."); + System.Environment.Exit(0); + } + // method to get db precision + public void getPrecision(IntPtr res) + { + int psc=TDengine.ResultPrecision(res); + switch(psc) + { + case 0: + Console.WriteLine("db：[{0:G}]'s precision is {1:G}",this.dbName,"millisecond"); + break; + case 1: + Console.WriteLine("db：[{0:G}]'s precision is {1:G}",this.dbName,"microsecond"); + break; + case 2: + Console.WriteLine("db：[{0:G}]'s precision is {1:G}",this.dbName,"nanosecond"); + break; + } + + } } } diff --git a/tests/examples/C#/C#checker/TDengineDriver.cs b/tests/examples/C#/C#checker/TDengineDriver.cs index b6f143e181..2864b7bcdd 100644 --- a/tests/examples/C#/C#checker/TDengineDriver.cs +++ b/tests/examples/C#/C#checker/TDengineDriver.cs @@ -19,136 +19,153 @@ using System.Runtime.InteropServices; namespace TDengineDriver { - enum TDengineDataType { - TSDB_DATA_TYPE_NULL = 0, // 1 bytes - TSDB_DATA_TYPE_BOOL = 1, // 1 bytes - TSDB_DATA_TYPE_TINYINT = 2, // 1 bytes - TSDB_DATA_TYPE_SMALLINT = 3, // 2 bytes - TSDB_DATA_TYPE_INT = 4, // 4 bytes - TSDB_DATA_TYPE_BIGINT = 5, // 8 bytes - TSDB_DATA_TYPE_FLOAT = 6, // 4 bytes - TSDB_DATA_TYPE_DOUBLE = 7, // 8 bytes - TSDB_DATA_TYPE_BINARY = 8, // string - TSDB_DATA_TYPE_TIMESTAMP = 9,// 8 bytes - TSDB_DATA_TYPE_NCHAR = 10 // unicode string - } - - enum TDengineInitOption - { - TSDB_OPTION_LOCALE = 0, - TSDB_OPTION_CHARSET = 1, - TSDB_OPTION_TIMEZONE = 2, - TDDB_OPTION_CONFIGDIR = 3, - TDDB_OPTION_SHELL_ACTIVITY_TIMER = 4 - } - - class TDengineMeta - { - public string name; - public short size; - public byte type; - public string TypeName() + enum TDengineDataType { - switch ((TDengineDataType)type) - { - case TDengineDataType.TSDB_DATA_TYPE_BOOL: - return "BOOLEAN"; - case TDengineDataType.TSDB_DATA_TYPE_TINYINT: - return "BYTE"; - case TDengineDataType.TSDB_DATA_TYPE_SMALLINT: - return "SHORT"; - case TDengineDataType.TSDB_DATA_TYPE_INT: - return "INT"; - case TDengineDataType.TSDB_DATA_TYPE_BIGINT: - return "LONG"; - case TDengineDataType.TSDB_DATA_TYPE_FLOAT: - return "FLOAT"; - case TDengineDataType.TSDB_DATA_TYPE_DOUBLE: - return "DOUBLE"; - case TDengineDataType.TSDB_DATA_TYPE_BINARY: - return "STRING"; - case TDengineDataType.TSDB_DATA_TYPE_TIMESTAMP: - return "TIMESTAMP"; - case TDengineDataType.TSDB_DATA_TYPE_NCHAR: - return "NCHAR"; - default: - return "undefine"; - } - } - } - - class TDengine - { - public const int TSDB_CODE_SUCCESS = 0; - - [DllImport("taos.dll", EntryPoint = "taos_init", CallingConvention = CallingConvention.Cdecl)] - static extern public void Init(); - - [DllImport("taos.dll", EntryPoint = "taos_cleanup", CallingConvention = CallingConvention.Cdecl)] - static extern public void Cleanup(); - - [DllImport("taos.dll", EntryPoint = "taos_options", CallingConvention = CallingConvention.Cdecl)] - static extern public void Options(int option, string value); - - [DllImport("taos.dll", EntryPoint = "taos_connect", CallingConvention = CallingConvention.Cdecl)] - static extern public IntPtr Connect(string ip, string user, string password, string db, short port); - - [DllImport("taos.dll", EntryPoint = "taos_errstr", CallingConvention = CallingConvention.Cdecl)] - static extern private IntPtr taos_errstr(IntPtr res); - static public string Error(IntPtr res) - { - IntPtr errPtr = taos_errstr(res); - return Marshal.PtrToStringAnsi(errPtr); + TSDB_DATA_TYPE_NULL = 0, // 1 bytes + TSDB_DATA_TYPE_BOOL = 1, // 1 bytes + TSDB_DATA_TYPE_TINYINT = 2, // 1 bytes + TSDB_DATA_TYPE_SMALLINT = 3, // 2 bytes + TSDB_DATA_TYPE_INT = 4, // 4 bytes + TSDB_DATA_TYPE_BIGINT = 5, // 8 bytes + TSDB_DATA_TYPE_FLOAT = 6, // 4 bytes + TSDB_DATA_TYPE_DOUBLE = 7, // 8 bytes + TSDB_DATA_TYPE_BINARY = 8, // string + TSDB_DATA_TYPE_TIMESTAMP = 9,// 8 bytes + TSDB_DATA_TYPE_NCHAR = 10, // unicode string + TSDB_DATA_TYPE_UTINYINT = 11,// 1 byte + TSDB_DATA_TYPE_USMALLINT= 12,// 2 bytes + TSDB_DATA_TYPE_UINT = 13, // 4 bytes + TSDB_DATA_TYPE_UBIGINT= 14 // 8 bytes } - [DllImport("taos.dll", EntryPoint = "taos_errno", CallingConvention = CallingConvention.Cdecl)] - static extern public int ErrorNo(IntPtr res); - - [DllImport("taos.dll", EntryPoint = "taos_query", CallingConvention = CallingConvention.Cdecl)] - static extern public IntPtr Query(IntPtr conn, string sqlstr); - - [DllImport("taos.dll", EntryPoint = "taos_affected_rows", CallingConvention = CallingConvention.Cdecl)] - static extern public int AffectRows(IntPtr res); - - [DllImport("taos.dll", EntryPoint = "taos_field_count", CallingConvention = CallingConvention.Cdecl)] - static extern public int FieldCount(IntPtr res); - - [DllImport("taos.dll", EntryPoint = "taos_fetch_fields", CallingConvention = CallingConvention.Cdecl)] - static extern private IntPtr taos_fetch_fields(IntPtr res); - static public List FetchFields(IntPtr res) + enum TDengineInitOption { - const int fieldSize = 68; - - List metas = new List(); - if (res == IntPtr.Zero) - { - return metas; - } - - int fieldCount = FieldCount(res); - IntPtr fieldsPtr = taos_fetch_fields(res); - - for (int i = 0; i < fieldCount; ++i) - { - int offset = i * fieldSize; - - TDengineMeta meta = new TDengineMeta(); - meta.name = Marshal.PtrToStringAnsi(fieldsPtr + offset); - meta.type = Marshal.ReadByte(fieldsPtr + offset + 65); - meta.size = Marshal.ReadInt16(fieldsPtr + offset + 66); - metas.Add(meta); - } - - return metas; + TSDB_OPTION_LOCALE = 0, + TSDB_OPTION_CHARSET = 1, + TSDB_OPTION_TIMEZONE = 2, + TDDB_OPTION_CONFIGDIR = 3, + TDDB_OPTION_SHELL_ACTIVITY_TIMER = 4 } - [DllImport("taos.dll", EntryPoint = "taos_fetch_row", CallingConvention = CallingConvention.Cdecl)] - static extern public IntPtr FetchRows(IntPtr res); + class TDengineMeta + { + public string name; + public short size; + public byte type; + public string TypeName() + { + switch ((TDengineDataType)type) + { + case TDengineDataType.TSDB_DATA_TYPE_BOOL: + return "BOOL"; + case TDengineDataType.TSDB_DATA_TYPE_TINYINT: + return "TINYINT"; + case TDengineDataType.TSDB_DATA_TYPE_SMALLINT: + return "SMALLINT"; + case TDengineDataType.TSDB_DATA_TYPE_INT: + return "INT"; + case TDengineDataType.TSDB_DATA_TYPE_BIGINT: + return "BIGINT"; + case TDengineDataType.TSDB_DATA_TYPE_UTINYINT: + return "TINYINT UNSIGNED"; + case TDengineDataType.TSDB_DATA_TYPE_USMALLINT: + return "SMALLINT UNSIGNED"; + case TDengineDataType.TSDB_DATA_TYPE_UINT: + return "INT UNSIGNED"; + case TDengineDataType.TSDB_DATA_TYPE_UBIGINT: + return "BIGINT UNSIGNED"; + case TDengineDataType.TSDB_DATA_TYPE_FLOAT: + return "FLOAT"; + case TDengineDataType.TSDB_DATA_TYPE_DOUBLE: + return "DOUBLE"; + case TDengineDataType.TSDB_DATA_TYPE_BINARY: + return "STRING"; + case TDengineDataType.TSDB_DATA_TYPE_TIMESTAMP: + return "TIMESTAMP"; + case TDengineDataType.TSDB_DATA_TYPE_NCHAR: + return "NCHAR"; + default: + return "undefine"; + } + } + } - [DllImport("taos.dll", EntryPoint = "taos_free_result", CallingConvention = CallingConvention.Cdecl)] - static extern public IntPtr FreeResult(IntPtr res); + class TDengine + { + public const int TSDB_CODE_SUCCESS = 0; - [DllImport("taos.dll", EntryPoint = "taos_close", CallingConvention = CallingConvention.Cdecl)] - static extern public int Close(IntPtr taos); - } -} \ No newline at end of file + [DllImport("taos", EntryPoint = "taos_init", CallingConvention = CallingConvention.Cdecl)] + static extern public void Init(); + + [DllImport("taos", EntryPoint = "taos_cleanup", CallingConvention = CallingConvention.Cdecl)] + static extern public void Cleanup(); + + [DllImport("taos", EntryPoint = "taos_options", CallingConvention = CallingConvention.Cdecl)] + static extern public void Options(int option, string value); + + [DllImport("taos", EntryPoint = "taos_connect", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr Connect(string ip, string user, string password, string db, short port); + + [DllImport("taos", EntryPoint = "taos_errstr", CallingConvention = CallingConvention.Cdecl)] + static extern private IntPtr taos_errstr(IntPtr res); + static public string Error(IntPtr res) + { + IntPtr errPtr = taos_errstr(res); + return Marshal.PtrToStringAnsi(errPtr); + } + + [DllImport("taos", EntryPoint = "taos_errno", CallingConvention = CallingConvention.Cdecl)] + static extern public int ErrorNo(IntPtr res); + + [DllImport("taos", EntryPoint = "taos_query", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr Query(IntPtr conn, string sqlstr); + + [DllImport("taos", EntryPoint = "taos_affected_rows", CallingConvention = CallingConvention.Cdecl)] + static extern public int AffectRows(IntPtr res); + + [DllImport("taos", EntryPoint = "taos_field_count", CallingConvention = CallingConvention.Cdecl)] + static extern public int FieldCount(IntPtr res); + + [DllImport("taos", EntryPoint = "taos_fetch_fields", CallingConvention = CallingConvention.Cdecl)] + static extern private IntPtr taos_fetch_fields(IntPtr res); + static public List FetchFields(IntPtr res) + { + const int fieldSize = 68; + + List metas = new List(); + if (res == IntPtr.Zero) + { + return metas; + } + + int fieldCount = FieldCount(res); + IntPtr fieldsPtr = taos_fetch_fields(res); + + for (int i = 0; i < fieldCount; ++i) + { + int offset = i * fieldSize; + + TDengineMeta meta = new TDengineMeta(); + meta.name = Marshal.PtrToStringAnsi(fieldsPtr + offset); + meta.type = Marshal.ReadByte(fieldsPtr + offset + 65); + meta.size = Marshal.ReadInt16(fieldsPtr + offset + 66); + metas.Add(meta); + } + + return metas; + } + + [DllImport("taos", EntryPoint = "taos_fetch_row", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr FetchRows(IntPtr res); + + [DllImport("taos", EntryPoint = "taos_free_result", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr FreeResult(IntPtr res); + + [DllImport("taos", EntryPoint = "taos_close", CallingConvention = CallingConvention.Cdecl)] + static extern public int Close(IntPtr taos); + + //get precisionŁ¬in parameter restultset + [DllImport("taos", EntryPoint = "taos_result_precision", CallingConvention = CallingConvention.Cdecl)] + static extern public int ResultPrecision(IntPtr taos); + } +} diff --git a/tests/examples/C#/TDengineDriver.cs b/tests/examples/C#/TDengineDriver.cs index 2c150341f6..2864b7bcdd 100644 --- a/tests/examples/C#/TDengineDriver.cs +++ b/tests/examples/C#/TDengineDriver.cs @@ -163,5 +163,9 @@ namespace TDengineDriver [DllImport("taos", EntryPoint = "taos_close", CallingConvention = CallingConvention.Cdecl)] static extern public int Close(IntPtr taos); + + //get precisionŁ¬in parameter restultset + [DllImport("taos", EntryPoint = "taos_result_precision", CallingConvention = CallingConvention.Cdecl)] + static extern public int ResultPrecision(IntPtr taos); } } diff --git a/tests/examples/C#/taosdemo/TDengineDriver.cs b/tests/examples/C#/taosdemo/TDengineDriver.cs index 2c150341f6..e6c3a598ad 100644 --- a/tests/examples/C#/taosdemo/TDengineDriver.cs +++ b/tests/examples/C#/taosdemo/TDengineDriver.cs @@ -163,5 +163,8 @@ namespace TDengineDriver [DllImport("taos", EntryPoint = "taos_close", CallingConvention = CallingConvention.Cdecl)] static extern public int Close(IntPtr taos); + //get precisionŁ¬in parameter restultset + [DllImport("taos", EntryPoint = "taos_result_precision", CallingConvention = CallingConvention.Cdecl)] + static extern public int ResultPrecision(IntPtr taos); } } From 7ae5f04d3e95e1265ea0541adba48357bfa40ff3 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 29 Jul 2021 16:41:09 +0800 Subject: [PATCH 087/106] Empty Commit From f12d252116ba2a2d2a23993fc40d9bca6497ebce Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 29 Jul 2021 18:08:02 +0800 Subject: [PATCH 088/106] [td-5175] --- src/client/src/tscAsync.c | 18 +++--- src/client/src/tscServer.c | 11 ++-- src/client/src/tscSubquery.c | 8 ++- src/client/src/tscUtil.c | 108 ++++++++++++++++++++++++++++++----- 4 files changed, 115 insertions(+), 30 deletions(-) diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index c8c9fe85e3..64f259ecaa 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -211,27 +211,27 @@ void taos_fetch_rows_a(TAOS_RES *tres, __async_cb_func_t fp, void *param) { pSql->fp = tscAsyncFetchRowsProxy; pSql->param = param; - if (pRes->qId == 0) { - tscError("qhandle is invalid"); - pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE; - tscAsyncResultOnError(pSql); - return; - } - tscResetForNextRetrieve(pRes); // handle outer query based on the already retrieved nest query results. SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); if (pQueryInfo->pUpstream != NULL && taosArrayGetSize(pQueryInfo->pUpstream) > 0) { SSchedMsg schedMsg = {0}; - schedMsg.fp = doRetrieveSubqueryData; + schedMsg.fp = doRetrieveSubqueryData; schedMsg.ahandle = (void *)pSql; schedMsg.thandle = (void *)1; - schedMsg.msg = 0; + schedMsg.msg = 0; taosScheduleTask(tscQhandle, &schedMsg); return; } + if (pRes->qId == 0) { + tscError("qhandle is invalid"); + pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE; + tscAsyncResultOnError(pSql); + return; + } + if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) { tscFetchDatablockForSubquery(pSql); } else if (pRes->completed) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index eaf397529b..a069e7639c 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -392,14 +392,17 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { // single table query error need to be handled here. if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_UPDATE_TAGS_VAL) && - (((rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || // change the retry procedure + (((rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID)) || - rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || // change the retry procedure + rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || rpcMsg->code == TSDB_CODE_APP_NOT_READY)) { - if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, (TSDB_QUERY_TYPE_STABLE_SUBQUERY | TSDB_QUERY_TYPE_SUBQUERY | + // 1. super table subquery + // 2. nest queries are all not updated the tablemeta and retry parse the sql after cleanup local tablemeta/vgroup id buffer + if ((TSDB_QUERY_HAS_TYPE(pQueryInfo->type, (TSDB_QUERY_TYPE_STABLE_SUBQUERY | TSDB_QUERY_TYPE_SUBQUERY | TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) && - !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY)) { + !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY)) || + (pQueryInfo->pUpstream != NULL)) { // do nothing in case of super table subquery } else { pSql->retry += 1; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index af7a18ca7a..8c20aed350 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2705,7 +2705,6 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO // release allocated resource tscDestroyGlobalMergerEnv(trsupport->pExtMemBuffer, trsupport->pOrderDescriptor, pState->numOfSub); - tscFreeRetrieveSup(pSql); // in case of second stage join subquery, invoke its callback function instead of regular QueueAsyncRes @@ -2716,10 +2715,13 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO int32_t code = pParentSql->res.code; if ((code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) && pParentSql->retry < pParentSql->maxRetry) { // remove the cached tableMeta and vgroup id list, and then parse the sql again - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pParentSql->cmd, 0); + SSqlCmd* pParentCmd = &pParentSql->cmd; + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pParentCmd, 0); tscRemoveTableMetaBuf(pTableMetaInfo, pParentSql->self); - tscResetSqlCmd(&pParentSql->cmd, true); + pParentCmd->pTableMetaMap = tscCleanupTableMetaMap(pParentCmd->pTableMetaMap); + pParentCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + pParentSql->res.code = TSDB_CODE_SUCCESS; pParentSql->retry++; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index b9a9333fd7..d4425f7ad2 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -3829,13 +3829,64 @@ static void tscSubqueryRetrieveCallback(void* param, TAOS_RES* tres, int code) { } } -// todo handle the failure static void tscSubqueryCompleteCallback(void* param, TAOS_RES* tres, int code) { + SSqlObj* pSql = tres; + SRetrieveSupport* ps = param; + + if (pSql->res.code != TSDB_CODE_SUCCESS) { + SSqlObj* pParentSql = ps->pParentSql; + + int32_t index = ps->subqueryIndex; + bool ret = subAndCheckDone(pSql, pParentSql, index); + + tfree(ps); + pSql->param = NULL; + + if (!ret) { + tscDebug("0x%"PRIx64" sub:0x%"PRIx64" orderOfSub:%d completed, not all subquery finished", pParentSql->self, pSql->self, index); + return; + } + + // todo refactor + tscDebug("0x%"PRIx64" all subquery response received, retry", pParentSql->self); + + SSqlCmd* pParentCmd = &pParentSql->cmd; + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pParentCmd, 0); + tscRemoveTableMetaBuf(pTableMetaInfo, pParentSql->self); + + pParentCmd->pTableMetaMap = tscCleanupTableMetaMap(pParentCmd->pTableMetaMap); + pParentCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + + pParentSql->res.code = TSDB_CODE_SUCCESS; + pParentSql->retry++; + + tscDebug("0x%"PRIx64" retry parse sql and send query, prev error: %s, retry:%d", pParentSql->self, + tstrerror(code), pParentSql->retry); + + code = tsParseSql(pParentSql, true); + if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + return; + } + + if (code != TSDB_CODE_SUCCESS) { + pParentSql->res.code = code; + tscAsyncResultOnError(pParentSql); + return; + } + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pParentCmd); + executeQuery(pParentSql, pQueryInfo); + return; + } + taos_fetch_rows_a(tres, tscSubqueryRetrieveCallback, param); } // do execute the query according to the query execution plan void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t numOfInit = 0; + if (pSql->cmd.command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) { (*pSql->fp)(pSql->param, pSql, 0); return; @@ -3850,7 +3901,12 @@ void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo) { pSql->pSubs = calloc(pSql->subState.numOfSub, POINTER_BYTES); pSql->subState.states = calloc(pSql->subState.numOfSub, sizeof(int8_t)); - pthread_mutex_init(&pSql->subState.mutex, NULL); + code = pthread_mutex_init(&pSql->subState.mutex, NULL); + + if (pSql->pSubs == NULL || pSql->subState.states == NULL || code != TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { SQueryInfo* pSub = taosArrayGetP(pQueryInfo->pUpstream, i); @@ -3858,45 +3914,69 @@ void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo) { pSql->cmd.active = pSub; pSql->cmd.command = TSDB_SQL_SELECT; - // TODO handle memory failure SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); if (pNew == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - // return NULL; + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; } - pNew->pTscObj = pSql->pTscObj; + pNew->pTscObj = pSql->pTscObj; pNew->signature = pNew; - pNew->sqlstr = strdup(pSql->sqlstr); // todo refactor - pNew->fp = tscSubqueryCompleteCallback; + pNew->sqlstr = strdup(pSql->sqlstr); + pNew->fp = tscSubqueryCompleteCallback; + pNew->maxRetry = pSql->maxRetry; tsem_init(&pNew->rspSem, 0, 0); SRetrieveSupport* ps = calloc(1, sizeof(SRetrieveSupport)); // todo use object id + if (ps == NULL) { + tscFreeSqlObj(pNew); + goto _error; + } + ps->pParentSql = pSql; ps->subqueryIndex = i; pNew->param = ps; pSql->pSubs[i] = pNew; - registerSqlObj(pNew); SSqlCmd* pCmd = &pNew->cmd; pCmd->command = TSDB_SQL_SELECT; - if (tscAddQueryInfo(pCmd) != TSDB_CODE_SUCCESS) { + if ((code = tscAddQueryInfo(pCmd)) != TSDB_CODE_SUCCESS) { + goto _error; } SQueryInfo* pNewQueryInfo = tscGetQueryInfo(pCmd); tscQueryInfoCopy(pNewQueryInfo, pSub); - - // create sub query to handle the sub query. - executeQuery(pNew, pNewQueryInfo); + numOfInit++; + } + + for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { + SSqlObj* psub = pSql->pSubs[i]; + registerSqlObj(psub); + + // create sub query to handle the sub query. + SQueryInfo* pq = tscGetQueryInfo(&psub->cmd); + executeQuery(psub, pq); } - // merge sub query result and generate final results return; } pSql->cmd.active = pQueryInfo; doExecuteQuery(pSql, pQueryInfo); + return; + + _error: + for(int32_t i = 0; i < numOfInit; ++i) { + SSqlObj* p = pSql->pSubs[i]; + tscFreeSqlObj(p); + } + + pSql->res.code = code; + pSql->subState.numOfSub = 0; // not initialized sub query object will not be freed + tfree(pSql->subState.states); + tfree(pSql->pSubs); + tscAsyncResultOnError(pSql); } int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid) { From eefa3cf5d1eec63071478fe293be8b7d3347651d Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 29 Jul 2021 18:17:31 +0800 Subject: [PATCH 089/106] change --- tests/pytest/update/merge_commit_data2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/update/merge_commit_data2.py b/tests/pytest/update/merge_commit_data2.py index 5fc58e3b4f..d4dbfec554 100644 --- a/tests/pytest/update/merge_commit_data2.py +++ b/tests/pytest/update/merge_commit_data2.py @@ -28,7 +28,7 @@ class TDTestCase: def restart_taosd(self,db): tdDnodes.stop(1) tdDnodes.startWithoutSleep(1) - tdsql.sleep(2) + tdSql.sleep(2) tdSql.execute("use %s;" % db) def date_to_timestamp_microseconds(self, date): From 3f7239b916d7a349024a009a1f73652a0a02263f Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 29 Jul 2021 18:18:06 +0800 Subject: [PATCH 090/106] change --- tests/pytest/update/merge_commit_data2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/update/merge_commit_data2.py b/tests/pytest/update/merge_commit_data2.py index 5fc58e3b4f..d4dbfec554 100644 --- a/tests/pytest/update/merge_commit_data2.py +++ b/tests/pytest/update/merge_commit_data2.py @@ -28,7 +28,7 @@ class TDTestCase: def restart_taosd(self,db): tdDnodes.stop(1) tdDnodes.startWithoutSleep(1) - tdsql.sleep(2) + tdSql.sleep(2) tdSql.execute("use %s;" % db) def date_to_timestamp_microseconds(self, date): From 6a9b2e64556a828891befe9737a4ee92215eccc5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 29 Jul 2021 23:19:28 +0800 Subject: [PATCH 091/106] [td-225]fix a bug found by regression test. --- src/client/src/tscSQLParser.c | 3 +-- src/client/src/tscServer.c | 2 +- src/client/src/tscUtil.c | 2 ++ src/inc/taosdef.h | 1 + 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index ef4b295990..a55b7ca253 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -898,8 +898,7 @@ int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { for (int32_t i = 0; i < size; ++i) { SSqlNode* pSqlNode = taosArrayGetP(pInfo->list, i); - tscTrace("%p start to parse %dth subclause, total:%"PRIzu, pSql, i, size); - + tscTrace("0x%"PRIx64" start to parse the %dth subclause, total:%"PRIzu, pSql->self, i, size); // normalizeSqlNode(pSqlNode); // normalize the column name in each function if ((code = validateSqlNode(pSql, pSqlNode, pQueryInfo)) != TSDB_CODE_SUCCESS) { return code; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index a069e7639c..0d23d8af20 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -402,7 +402,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { if ((TSDB_QUERY_HAS_TYPE(pQueryInfo->type, (TSDB_QUERY_TYPE_STABLE_SUBQUERY | TSDB_QUERY_TYPE_SUBQUERY | TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) && !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY)) || - (pQueryInfo->pUpstream != NULL)) { + (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_NEST_SUBQUERY))) { // do nothing in case of super table subquery } else { pSql->retry += 1; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index d4425f7ad2..3a7c3e50ed 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -3947,6 +3947,8 @@ void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo) { SQueryInfo* pNewQueryInfo = tscGetQueryInfo(pCmd); tscQueryInfoCopy(pNewQueryInfo, pSub); + + TSDB_QUERY_SET_TYPE(pNewQueryInfo->type, TSDB_QUERY_TYPE_NEST_SUBQUERY); numOfInit++; } diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index ca8ad3cc09..9c4642ec73 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -365,6 +365,7 @@ do { \ #define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x200u #define TSDB_QUERY_TYPE_FILE_INSERT 0x400u // insert data from file #define TSDB_QUERY_TYPE_STMT_INSERT 0x800u // stmt insert type +#define TSDB_QUERY_TYPE_NEST_SUBQUERY 0x1000u // nested sub query #define TSDB_QUERY_HAS_TYPE(x, _type) (((x) & (_type)) != 0) #define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type)) From c0dfdc9c73e7077f9f933de4dc2fca64cb616255 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 29 Jul 2021 23:24:46 +0800 Subject: [PATCH 092/106] [td-225]add check for malloc failure. --- src/tsdb/src/tsdbRead.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 7a595da39a..a3cc02473b 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -477,6 +477,10 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pC TsdbQueryHandleT* tsdbQueryTables(STsdbRepo* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, SMemRef* pRef) { STsdbQueryHandle* pQueryHandle = tsdbQueryTablesImpl(tsdb, pCond, qId, pRef); + if (pQueryHandle == NULL) { + return NULL; + } + if (emptyQueryTimewindow(pQueryHandle)) { return (TsdbQueryHandleT*) pQueryHandle; } @@ -587,6 +591,10 @@ TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STable } STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); + if (pQueryHandle == NULL) { + return NULL; + } + int32_t code = checkForCachedLastRow(pQueryHandle, groupList); if (code != TSDB_CODE_SUCCESS) { // set the numOfTables to be 0 terrno = code; @@ -604,6 +612,10 @@ TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STable TsdbQueryHandleT tsdbQueryCacheLast(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pMemRef) { STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); + if (pQueryHandle == NULL) { + return NULL; + } + int32_t code = checkForCachedLast(pQueryHandle); if (code != TSDB_CODE_SUCCESS) { // set the numOfTables to be 0 terrno = code; From c6f660a88ed6dd1648c463e78258657807979bd5 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 29 Jul 2021 23:28:47 +0800 Subject: [PATCH 093/106] Hotfix/sangshuduo/td 3973 use jemalloc for master (#7074) * [TD-3973]: add jemalloc as submodule. * [TD-3973]: add jemalloc as submodule. * [TD-3973]: use jemalloc. build works as following instructions: cmake .. -DJEMALLOC_ENABLED=true make * fix jemalloc at tag 5.2.1 * fix conflicts * make install works. * fix conflicts. * release script works. * fix typo * [TD-3937]: support jemalloc add install funtion to all scripts. * adjust install_jemalloc() position for check compatibility. * fix position bug. * add ldconfig for jemalloc library cache refresh. * cherry pick from develop branch. * cherry pick from develop branch. * fix install_jemalloc() in install_client.sh. * fix makeclient.sh. * fix install script typo mistake. Co-authored-by: Shuduo Sang --- packaging/tools/install.sh | 2 +- packaging/tools/install_power.sh | 2 +- packaging/tools/makeclient.sh | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 57467e4b72..a9c3a085db 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -777,7 +777,7 @@ function is_version_compatible() { if [ -f ${script_dir}/driver/vercomp.txt ]; then min_compatible_version=`cat ${script_dir}/driver/vercomp.txt` else - min_compatible_version=$(${script_dir}/bin/tqd -V | head -1 | cut -d ' ' -f 5) + min_compatible_version=$(${script_dir}/bin/taosd -V | head -1 | cut -d ' ' -f 5) fi vercomp $curr_version $min_compatible_version diff --git a/packaging/tools/install_power.sh b/packaging/tools/install_power.sh index d0220cca25..05eb09d8f3 100755 --- a/packaging/tools/install_power.sh +++ b/packaging/tools/install_power.sh @@ -746,7 +746,7 @@ function is_version_compatible() { if [ -f ${script_dir}/driver/vercomp.txt ]; then min_compatible_version=`cat ${script_dir}/driver/vercomp.txt` else - min_compatible_version=$(${script_dir}/bin/tqd -V | head -1 | cut -d ' ' -f 5) + min_compatible_version=$(${script_dir}/bin/powerd -V | head -1 | cut -d ' ' -f 5) fi vercomp $curr_version $min_compatible_version diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 2c36e85970..97e80088be 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -41,10 +41,10 @@ fi if [ "$osType" != "Darwin" ]; then if [ "$pagMode" == "lite" ]; then - #strip ${build_dir}/bin/taosd + #strip ${build_dir}/bin/taosd strip ${build_dir}/bin/taos bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh" - else + else bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo \ ${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh ${script_dir}/taosd-dump-cfg.gdb" fi @@ -139,7 +139,7 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then cp -r ${examples_dir}/C# ${install_dir}/examples fi # Copy driver -mkdir -p ${install_dir}/driver +mkdir -p ${install_dir}/driver cp ${lib_files} ${install_dir}/driver # Copy connector @@ -168,7 +168,7 @@ fi # exit 1 -cd ${release_dir} +cd ${release_dir} # install_dir has been distinguishes cluster from edege, so comments this code pkg_name=${install_dir}-${osType}-${cpuType} From b9db84586cd29cfbfbed767bec46f624c6555c36 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 30 Jul 2021 02:17:10 +0800 Subject: [PATCH 094/106] [TD-5534]:fix high risk strncpy to tstrncpy --- src/client/src/tscSQLParser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 8be38be504..0a018c863f 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -8415,7 +8415,7 @@ static int32_t doValidateSubquery(SSqlNode* pSqlNode, int32_t index, SSqlObj* pS return invalidOperationMsg(msgBuf, "subquery alias name too long"); } - tstrncpy(pTableMetaInfo1->aliasName, subInfo->aliasName.z, sizeof(pTableMetaInfo1->aliasName)); + tstrncpy(pTableMetaInfo1->aliasName, subInfo->aliasName.z, subInfo->aliasName.n + 1); } taosArrayPush(pQueryInfo->pUpstream, &pSub); From 72a05d1e809b6e28ea71737a0c080dfb9cb50129 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 30 Jul 2021 09:32:13 +0800 Subject: [PATCH 095/106] fix --- tests/pytest/update/merge_commit_data2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/update/merge_commit_data2.py b/tests/pytest/update/merge_commit_data2.py index d4dbfec554..a334f39e86 100644 --- a/tests/pytest/update/merge_commit_data2.py +++ b/tests/pytest/update/merge_commit_data2.py @@ -28,7 +28,7 @@ class TDTestCase: def restart_taosd(self,db): tdDnodes.stop(1) tdDnodes.startWithoutSleep(1) - tdSql.sleep(2) + tdLog.sleep(2) tdSql.execute("use %s;" % db) def date_to_timestamp_microseconds(self, date): From cd744f5fa71f68f842b8482695f3f5ac0f893ad3 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 30 Jul 2021 09:32:13 +0800 Subject: [PATCH 096/106] fix --- tests/pytest/update/merge_commit_data2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/update/merge_commit_data2.py b/tests/pytest/update/merge_commit_data2.py index d4dbfec554..a334f39e86 100644 --- a/tests/pytest/update/merge_commit_data2.py +++ b/tests/pytest/update/merge_commit_data2.py @@ -28,7 +28,7 @@ class TDTestCase: def restart_taosd(self,db): tdDnodes.stop(1) tdDnodes.startWithoutSleep(1) - tdSql.sleep(2) + tdLog.sleep(2) tdSql.execute("use %s;" % db) def date_to_timestamp_microseconds(self, date): From 5f37d8a2a5038dcf600432a12bd83ada29a3fc4a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 30 Jul 2021 11:27:34 +0800 Subject: [PATCH 097/106] [TD-5534]:fix test case out of date error --- tests/script/general/parser/function.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/general/parser/function.sim b/tests/script/general/parser/function.sim index a3470b1763..5edadad3a6 100644 --- a/tests/script/general/parser/function.sim +++ b/tests/script/general/parser/function.sim @@ -783,7 +783,7 @@ endi sql create stable st1 (ts timestamp, f1 int, f2 int) tags (id int); sql create table tb1 using st1 tags(1); -sql insert into tb1 values (now, 1, 1); +sql insert into tb1 values ('2021-07-02 00:00:00', 1, 1); sql select stddev(f1) from st1 group by f1; From fae99ad04de19748d75ea1afdeb9a534df936b08 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 30 Jul 2021 13:13:38 +0800 Subject: [PATCH 098/106] [TD-5628]: prompt --force-keep-file if upgrade lower version with 2.0.16.0 (#7080) --- packaging/tools/install.sh | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index a9c3a085db..e116d72d26 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -54,6 +54,7 @@ if command -v sudo > /dev/null; then fi update_flag=0 +prompt_force=0 initd_mod=0 service_mod=2 @@ -780,7 +781,18 @@ function is_version_compatible() { min_compatible_version=$(${script_dir}/bin/taosd -V | head -1 | cut -d ' ' -f 5) fi + # [TD-5628] prompt to execute taosd --force-keep-file if upgrade from lower version within 2.0.16.0 + exist_version=$(/usr/local/taos/bin/taosd -V | head -1 | cut -d ' ' -f 3) + vercomp $exist_version "2.0.16.0" + case $? in + 2) + prompt_force=1 + ;; + esac + vercomp $curr_version $min_compatible_version + echo "" # avoid $? value not update + case $? in 0) return 0;; 1) return 0;; @@ -789,6 +801,12 @@ function is_version_compatible() { } function update_TDengine() { + # Check if version compatible + if ! is_version_compatible; then + echo -e "${RED}Version incompatible${NC}" + return 1 + fi + # Start to update if [ ! -e taos.tar.gz ]; then echo "File taos.tar.gz does not exist" @@ -797,12 +815,6 @@ function update_TDengine() { tar -zxf taos.tar.gz install_jemalloc - # Check if version compatible - if ! is_version_compatible; then - echo -e "${RED}Version incompatible${NC}" - return 1 - fi - echo -e "${GREEN}Start to update TDengine...${NC}" # Stop the service if running if pidof taosd &> /dev/null; then @@ -875,6 +887,10 @@ function update_TDengine() { echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos -h $serverFqdn${NC} in shell${NC}" fi + if ((${prompt_force}==1)); then + echo "" + echo -e "${RED}Please run 'taosd --force-keep-file' at first time for the exist TDengine $exist_version!${NC}" + fi echo echo -e "\033[44;32;1mTDengine is updated successfully!${NC}" else From 6188c139e1aa83c0620a30192bc9e40d91299b8b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 30 Jul 2021 13:55:33 +0800 Subject: [PATCH 099/106] change version number --- cmake/version.inc | 2 +- snap/snapcraft.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/version.inc b/cmake/version.inc index 20be64147c..f576627908 100755 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.0.20.10") + SET(TD_VER_NUMBER "2.1.5.0") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index b06864fe5d..2bb0cd3d2f 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: tdengine base: core18 -version: '2.0.20.10' +version: '2.1.5.0' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | From 7fac670e591e8702fa26851c1de2d998d3680724 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 30 Jul 2021 13:58:09 +0800 Subject: [PATCH 100/106] change version number --- snap/snapcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 2bb0cd3d2f..aef706311d 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -72,7 +72,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.0.20.10 + - usr/lib/libtaos.so.2.1.5.0 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so From ef8bd46b4981ce3a96b2102c1eba140dc250a5c1 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 31 Jul 2021 01:07:24 +0800 Subject: [PATCH 101/106] Feature/sangshuduo/td 3973 use jemalloc (#7071) * [TD-3973]: add jemalloc as submodule. * add macro definitions in cmake. * [TD-3973]: use jemalloc. build works as following instructions: cmake .. -DJEMALLOC_ENABLED=true make * fix jemalloc at tag 5.2.1 * link jemalloc works. * make install works. * support jemalloc in release.sh. * release script works. * fix a typo. * [TD-3937]: support jemalloc add install funtion to all scripts. * adjust install_jemalloc() position for update check compatiblity. * fix position bug. * add ldconfig for jemalloc library cache refresh. * add /etc/ld.so.conf.d/jemalloc.conf for centos * check ver comp by file. * fix makeclient.sh * fix install script typo mistake. * change minimal compatible version back to 2.0.0.0 Co-authored-by: Shuduo Sang --- cmake/version.inc | 2 +- packaging/tools/install.sh | 2 +- packaging/tools/install_power.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/version.inc b/cmake/version.inc index f576627908..7c0a824c9c 100755 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -10,7 +10,7 @@ ENDIF () IF (DEFINED VERCOMPATIBLE) SET(TD_VER_COMPATIBLE ${VERCOMPATIBLE}) ELSE () - SET(TD_VER_COMPATIBLE "1.0.0.0") + SET(TD_VER_COMPATIBLE "2.0.0.0") ENDIF () find_program(HAVE_GIT NAMES git) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 57467e4b72..a9c3a085db 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -777,7 +777,7 @@ function is_version_compatible() { if [ -f ${script_dir}/driver/vercomp.txt ]; then min_compatible_version=`cat ${script_dir}/driver/vercomp.txt` else - min_compatible_version=$(${script_dir}/bin/tqd -V | head -1 | cut -d ' ' -f 5) + min_compatible_version=$(${script_dir}/bin/taosd -V | head -1 | cut -d ' ' -f 5) fi vercomp $curr_version $min_compatible_version diff --git a/packaging/tools/install_power.sh b/packaging/tools/install_power.sh index d0220cca25..05eb09d8f3 100755 --- a/packaging/tools/install_power.sh +++ b/packaging/tools/install_power.sh @@ -746,7 +746,7 @@ function is_version_compatible() { if [ -f ${script_dir}/driver/vercomp.txt ]; then min_compatible_version=`cat ${script_dir}/driver/vercomp.txt` else - min_compatible_version=$(${script_dir}/bin/tqd -V | head -1 | cut -d ' ' -f 5) + min_compatible_version=$(${script_dir}/bin/powerd -V | head -1 | cut -d ' ' -f 5) fi vercomp $curr_version $min_compatible_version From 929a7ce4b60669f109d10c7a6f296fd4be57aa01 Mon Sep 17 00:00:00 2001 From: Zhiqiang Wang <1296468573@qq.com> Date: Sat, 31 Jul 2021 08:05:22 +0800 Subject: [PATCH 102/106] [TD-5585]: arm32 python fromtimestamp error. (#7090) --- src/connector/python/taos/cinterface.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/connector/python/taos/cinterface.py b/src/connector/python/taos/cinterface.py index 6d8ceb7a29..660707bfcd 100644 --- a/src/connector/python/taos/cinterface.py +++ b/src/connector/python/taos/cinterface.py @@ -7,11 +7,11 @@ import platform def _convert_millisecond_to_datetime(milli): - return datetime.datetime.fromtimestamp(milli / 1000.0) + return datetime.datetime.fromtimestamp(0) + datetime.timedelta(seconds=milli/1000.0) def _convert_microsecond_to_datetime(micro): - return datetime.datetime.fromtimestamp(micro / 1000000.0) + return datetime.datetime.fromtimestamp(0) + datetime.timedelta(seconds=micro / 1000000.0) def _convert_nanosecond_to_datetime(nanosec): From da3e7847212cbc3699f98e23f4e464df6a31cd5f Mon Sep 17 00:00:00 2001 From: xiaolei li <85657333+xleili@users.noreply.github.com> Date: Sat, 31 Jul 2021 08:07:25 +0800 Subject: [PATCH 103/106] [TD-4852] fix host,error diplay (#6929) --- tests/examples/JDBC/JDBCDemo/pom.xml | 2 +- .../main/java/com/taosdata/example/JdbcDemo.java | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/examples/JDBC/JDBCDemo/pom.xml b/tests/examples/JDBC/JDBCDemo/pom.xml index ca8cd24030..fed00c147b 100644 --- a/tests/examples/JDBC/JDBCDemo/pom.xml +++ b/tests/examples/JDBC/JDBCDemo/pom.xml @@ -17,7 +17,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.30 + 2.0.31 diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcDemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcDemo.java index f256668dc6..d4ea5f919d 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcDemo.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcDemo.java @@ -43,6 +43,7 @@ public class JdbcDemo { if (connection != null) System.out.println("[ OK ] Connection established."); } catch (ClassNotFoundException | SQLException e) { + System.out.println("[ ERROR! ] Connection establish failed."); e.printStackTrace(); } } @@ -68,7 +69,7 @@ public class JdbcDemo { } private void insert() { - final String sql = "insert into " + dbName + "." + tbName + " (ts, temperature, humidity) values(now, 20.5, 34)"; + final String sql = "insert into " + dbName + "." + tbName + " (ts, temperature, humidity) values(now, 20.5, 34)"; exuete(sql); } @@ -91,13 +92,15 @@ public class JdbcDemo { /************************************************************************/ private void executeQuery(String sql) { + long start = System.currentTimeMillis(); try (Statement statement = connection.createStatement()) { - long start = System.currentTimeMillis(); ResultSet resultSet = statement.executeQuery(sql); long end = System.currentTimeMillis(); printSql(sql, true, (end - start)); printResult(resultSet); } catch (SQLException e) { + long end = System.currentTimeMillis(); + printSql(sql, false, (end - start)); e.printStackTrace(); } } @@ -120,12 +123,14 @@ public class JdbcDemo { } private void exuete(String sql) { + long start = System.currentTimeMillis(); try (Statement statement = connection.createStatement()) { - long start = System.currentTimeMillis(); boolean execute = statement.execute(sql); long end = System.currentTimeMillis(); - printSql(sql, execute, (end - start)); + printSql(sql, true, (end - start)); } catch (SQLException e) { + long end = System.currentTimeMillis(); + printSql(sql, false, (end - start)); e.printStackTrace(); } @@ -137,4 +142,4 @@ public class JdbcDemo { } -} +} \ No newline at end of file From 877f20cc7c13da8d9567dac261e14a0b27a10253 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 1 Aug 2021 09:15:24 +0800 Subject: [PATCH 104/106] [TD-5625]: taosdemo don't use printf for data generation. (#7093) --- src/kit/taosdemo/taosdemo.c | 397 ++++++++++++++++++++++++++---------- 1 file changed, 284 insertions(+), 113 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 32bad230cf..14ad597a15 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -82,9 +82,18 @@ extern char configDir[]; #define MAX_TB_NAME_SIZE 64 #define MAX_DATA_SIZE (16*TSDB_MAX_COLUMNS)+20 // max record len: 16*MAX_COLUMNS, timestamp string and ,('') need extra space #define OPT_ABORT 1 /* –abort */ -#define MAX_PREPARED_RAND 1000000 #define MAX_FILE_NAME_LEN 256 // max file name length on linux is 255. +#define MAX_PREPARED_RAND 1000000 +#define INT_BUFF_LEN 11 +#define BIGINT_BUFF_LEN 21 +#define SMALLINT_BUFF_LEN 6 +#define TINYINT_BUFF_LEN 4 +#define BOOL_BUFF_LEN 6 +#define FLOAT_BUFF_LEN 22 +#define DOUBLE_BUFF_LEN 42 +#define TIMESTAMP_BUFF_LEN 21 + #define MAX_SAMPLES_ONCE_FROM_FILE 10000 #define MAX_NUM_COLUMNS (TSDB_MAX_COLUMNS - 1) // exclude first column timestamp @@ -561,11 +570,23 @@ static void init_rand_data(); /* ************ Global variables ************ */ -int32_t randint[MAX_PREPARED_RAND]; -int64_t randbigint[MAX_PREPARED_RAND]; -float randfloat[MAX_PREPARED_RAND]; -double randdouble[MAX_PREPARED_RAND]; -char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)", +int32_t g_randint[MAX_PREPARED_RAND]; +int64_t g_randbigint[MAX_PREPARED_RAND]; +float g_randfloat[MAX_PREPARED_RAND]; +double g_randdouble[MAX_PREPARED_RAND]; + +char *g_randbool_buff = NULL; +char *g_randint_buff = NULL; +char *g_rand_voltage_buff = NULL; +char *g_randbigint_buff = NULL; +char *g_randsmallint_buff = NULL; +char *g_randtinyint_buff = NULL; +char *g_randfloat_buff = NULL; +char *g_rand_current_buff = NULL; +char *g_rand_phase_buff = NULL; +char *g_randdouble_buff = NULL; + +char *g_aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)", "max(col0)", "min(col0)", "first(col0)", "last(col0)"}; #define DEFAULT_DATATYPE_NUM 3 @@ -1307,67 +1328,144 @@ static void selectAndGetResult( } } +static char *rand_bool_str(){ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randbool_buff + (cursor * BOOL_BUFF_LEN); +} + static int32_t rand_bool(){ static int cursor; cursor++; cursor = cursor % MAX_PREPARED_RAND; - return randint[cursor] % 2; + return g_randint[cursor] % 2; } -static int32_t rand_tinyint(){ +static char *rand_tinyint_str() +{ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randtinyint_buff + (cursor * TINYINT_BUFF_LEN); +} + +static int32_t rand_tinyint() +{ static int cursor; cursor++; cursor = cursor % MAX_PREPARED_RAND; - return randint[cursor] % 128; + return g_randint[cursor] % 128; } -static int32_t rand_smallint(){ +static char *rand_smallint_str() +{ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randsmallint_buff + (cursor * SMALLINT_BUFF_LEN); +} + +static int32_t rand_smallint() +{ static int cursor; cursor++; cursor = cursor % MAX_PREPARED_RAND; - return randint[cursor] % 32767; + return g_randint[cursor] % 32767; } -static int32_t rand_int(){ +static char *rand_int_str() +{ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randint_buff + (cursor * INT_BUFF_LEN); +} + +static int32_t rand_int() +{ static int cursor; cursor++; cursor = cursor % MAX_PREPARED_RAND; - return randint[cursor]; + return g_randint[cursor]; } -static int64_t rand_bigint(){ +static char *rand_bigint_str() +{ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randbigint_buff + (cursor * BIGINT_BUFF_LEN); +} + +static int64_t rand_bigint() +{ static int cursor; cursor++; cursor = cursor % MAX_PREPARED_RAND; - return randbigint[cursor]; + return g_randbigint[cursor]; } -static float rand_float(){ +static char *rand_float_str() +{ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randfloat_buff + (cursor * FLOAT_BUFF_LEN); +} + +static float rand_float() +{ static int cursor; cursor++; cursor = cursor % MAX_PREPARED_RAND; - return randfloat[cursor]; + return g_randfloat[cursor]; } -static float demo_current_float(){ +static char *demo_current_float_str() +{ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_rand_current_buff + (cursor * FLOAT_BUFF_LEN); +} + +static float UNUSED_FUNC demo_current_float() +{ static int cursor; cursor++; cursor = cursor % MAX_PREPARED_RAND; - return (float)(9.8 + 0.04 * (randint[cursor] % 10) + randfloat[cursor]/1000000000); + return (float)(9.8 + 0.04 * (g_randint[cursor] % 10) + g_randfloat[cursor]/1000000000); } -static int32_t demo_voltage_int(){ +static char *demo_voltage_int_str() +{ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_rand_voltage_buff + (cursor * INT_BUFF_LEN); +} + +static int32_t UNUSED_FUNC demo_voltage_int() +{ static int cursor; cursor++; cursor = cursor % MAX_PREPARED_RAND; - return 215 + randint[cursor] % 10; + return 215 + g_randint[cursor] % 10; } -static float demo_phase_float(){ +static char *demo_phase_float_str() { + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_rand_phase_buff + (cursor * FLOAT_BUFF_LEN); +} + +static float UNUSED_FUNC demo_phase_float(){ static int cursor; cursor++; cursor = cursor % MAX_PREPARED_RAND; - return (float)((115 + randint[cursor] % 10 + randfloat[cursor]/1000000000)/360); + return (float)((115 + g_randint[cursor] % 10 + g_randfloat[cursor]/1000000000)/360); } #if 0 @@ -1402,19 +1500,76 @@ static void rand_string(char *str, int size) { } } -static double rand_double() { +static char *rand_double_str() +{ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randdouble_buff + (cursor * DOUBLE_BUFF_LEN); +} + +static double rand_double() +{ static int cursor; cursor++; cursor = cursor % MAX_PREPARED_RAND; - return randdouble[cursor]; + return g_randdouble[cursor]; } static void init_rand_data() { + + g_randint_buff = calloc(1, INT_BUFF_LEN * MAX_PREPARED_RAND); + assert(g_randint_buff); + g_rand_voltage_buff = calloc(1, INT_BUFF_LEN * MAX_PREPARED_RAND); + assert(g_rand_voltage_buff); + g_randbigint_buff = calloc(1, BIGINT_BUFF_LEN * MAX_PREPARED_RAND); + assert(g_randbigint_buff); + g_randsmallint_buff = calloc(1, SMALLINT_BUFF_LEN * MAX_PREPARED_RAND); + assert(g_randsmallint_buff); + g_randtinyint_buff = calloc(1, TINYINT_BUFF_LEN * MAX_PREPARED_RAND); + assert(g_randtinyint_buff); + g_randbool_buff = calloc(1, BOOL_BUFF_LEN * MAX_PREPARED_RAND); + assert(g_randbool_buff); + g_randfloat_buff = calloc(1, FLOAT_BUFF_LEN * MAX_PREPARED_RAND); + assert(g_randfloat_buff); + g_rand_current_buff = calloc(1, FLOAT_BUFF_LEN * MAX_PREPARED_RAND); + assert(g_rand_current_buff); + g_rand_phase_buff = calloc(1, FLOAT_BUFF_LEN * MAX_PREPARED_RAND); + assert(g_rand_phase_buff); + g_randdouble_buff = calloc(1, DOUBLE_BUFF_LEN * MAX_PREPARED_RAND); + assert(g_randdouble_buff); + for (int i = 0; i < MAX_PREPARED_RAND; i++){ - randint[i] = (int)(taosRandom() % 65535); - randbigint[i] = (int64_t)(taosRandom() % 2147483648); - randfloat[i] = (float)(taosRandom() / 1000.0); - randdouble[i] = (double)(taosRandom() / 1000000.0); + g_randint[i] = (int)(taosRandom() % 65535); + sprintf(g_randint_buff + i * INT_BUFF_LEN, "%d", + g_randint[i]); + sprintf(g_rand_voltage_buff + i * INT_BUFF_LEN, "%d", + 215 + g_randint[i] % 10); + + sprintf(g_randbool_buff + i * BOOL_BUFF_LEN, "%s", + ((g_randint[i] % 2) & 1)?"true":"false"); + sprintf(g_randsmallint_buff + i * SMALLINT_BUFF_LEN, "%d", + g_randint[i] % 32767); + sprintf(g_randtinyint_buff + i * TINYINT_BUFF_LEN, "%d", + g_randint[i] % 128); + + g_randbigint[i] = (int64_t)(taosRandom() % 2147483648); + sprintf(g_randbigint_buff + i * BIGINT_BUFF_LEN, "%"PRId64"", + g_randbigint[i]); + + g_randfloat[i] = (float)(taosRandom() / 1000.0); + sprintf(g_randfloat_buff + i * FLOAT_BUFF_LEN, "%f", + g_randfloat[i]); + sprintf(g_rand_current_buff + i * FLOAT_BUFF_LEN, "%f", + (float)(9.8 + 0.04 * (g_randint[i] % 10) + + g_randfloat[i]/1000000000)); + sprintf(g_rand_phase_buff + i * FLOAT_BUFF_LEN, "%f", + (float)((115 + g_randint[i] % 10 + + g_randfloat[i]/1000000000)/360)); + + g_randdouble[i] = (double)(taosRandom() / 1000000.0); + sprintf(g_randdouble_buff + i * DOUBLE_BUFF_LEN, "%f", + g_randdouble[i]); } } @@ -2496,21 +2651,21 @@ static int calcRowLen(SSuperTable* superTbls) { } else if (strcasecmp(dataType, "NCHAR") == 0) { lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; } else if (strcasecmp(dataType, "INT") == 0) { - lenOfOneRow += 11; + lenOfOneRow += INT_BUFF_LEN; } else if (strcasecmp(dataType, "BIGINT") == 0) { - lenOfOneRow += 21; + lenOfOneRow += BIGINT_BUFF_LEN; } else if (strcasecmp(dataType, "SMALLINT") == 0) { - lenOfOneRow += 6; + lenOfOneRow += SMALLINT_BUFF_LEN; } else if (strcasecmp(dataType, "TINYINT") == 0) { - lenOfOneRow += 4; + lenOfOneRow += TINYINT_BUFF_LEN; } else if (strcasecmp(dataType, "BOOL") == 0) { - lenOfOneRow += 6; + lenOfOneRow += BOOL_BUFF_LEN; } else if (strcasecmp(dataType, "FLOAT") == 0) { - lenOfOneRow += 22; + lenOfOneRow += FLOAT_BUFF_LEN; } else if (strcasecmp(dataType, "DOUBLE") == 0) { - lenOfOneRow += 42; + lenOfOneRow += DOUBLE_BUFF_LEN; } else if (strcasecmp(dataType, "TIMESTAMP") == 0) { - lenOfOneRow += 21; + lenOfOneRow += TIMESTAMP_BUFF_LEN; } else { printf("get error data type : %s\n", dataType); exit(-1); @@ -2529,19 +2684,19 @@ static int calcRowLen(SSuperTable* superTbls) { } else if (strcasecmp(dataType, "NCHAR") == 0) { lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; } else if (strcasecmp(dataType, "INT") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11; + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + INT_BUFF_LEN; } else if (strcasecmp(dataType, "BIGINT") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21; + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + BIGINT_BUFF_LEN; } else if (strcasecmp(dataType, "SMALLINT") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + SMALLINT_BUFF_LEN; } else if (strcasecmp(dataType, "TINYINT") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4; + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + TINYINT_BUFF_LEN; } else if (strcasecmp(dataType, "BOOL") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + BOOL_BUFF_LEN; } else if (strcasecmp(dataType, "FLOAT") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22; + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + FLOAT_BUFF_LEN; } else if (strcasecmp(dataType, "DOUBLE") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42; + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + DOUBLE_BUFF_LEN; } else { printf("get error tag type : %s\n", dataType); exit(-1); @@ -2770,21 +2925,21 @@ static int createSuperTable( } else { len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "INT"); } - lenOfOneRow += 11; + lenOfOneRow += INT_BUFF_LEN; } else if (strcasecmp(dataType, "BIGINT") == 0) { len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "BIGINT"); - lenOfOneRow += 21; + lenOfOneRow += BIGINT_BUFF_LEN; } else if (strcasecmp(dataType, "SMALLINT") == 0) { len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "SMALLINT"); - lenOfOneRow += 6; + lenOfOneRow += SMALLINT_BUFF_LEN; } else if (strcasecmp(dataType, "TINYINT") == 0) { len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "TINYINT"); - lenOfOneRow += 4; + lenOfOneRow += TINYINT_BUFF_LEN; } else if (strcasecmp(dataType, "BOOL") == 0) { len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "BOOL"); - lenOfOneRow += 6; + lenOfOneRow += BOOL_BUFF_LEN; } else if (strcasecmp(dataType, "FLOAT") == 0) { if (g_args.demo_mode) { if (colIndex == 0) { @@ -2796,15 +2951,15 @@ static int createSuperTable( len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "FLOAT"); } - lenOfOneRow += 22; + lenOfOneRow += FLOAT_BUFF_LEN; } else if (strcasecmp(dataType, "DOUBLE") == 0) { len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "DOUBLE"); - lenOfOneRow += 42; + lenOfOneRow += DOUBLE_BUFF_LEN; } else if (strcasecmp(dataType, "TIMESTAMP") == 0) { len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "TIMESTAMP"); - lenOfOneRow += 21; + lenOfOneRow += TIMESTAMP_BUFF_LEN; } else { taos_close(taos); errorPrint("%s() LN%d, config error data type : %s\n", @@ -2867,31 +3022,31 @@ static int createSuperTable( len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, "T%d %s,", tagIndex, "INT"); } - lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 11; + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + INT_BUFF_LEN; } else if (strcasecmp(dataType, "BIGINT") == 0) { len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, "T%d %s,", tagIndex, "BIGINT"); - lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 21; + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + BIGINT_BUFF_LEN; } else if (strcasecmp(dataType, "SMALLINT") == 0) { len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, "T%d %s,", tagIndex, "SMALLINT"); - lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6; + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + SMALLINT_BUFF_LEN; } else if (strcasecmp(dataType, "TINYINT") == 0) { len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, "T%d %s,", tagIndex, "TINYINT"); - lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 4; + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + TINYINT_BUFF_LEN; } else if (strcasecmp(dataType, "BOOL") == 0) { len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, "T%d %s,", tagIndex, "BOOL"); - lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6; + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + BOOL_BUFF_LEN; } else if (strcasecmp(dataType, "FLOAT") == 0) { len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, "T%d %s,", tagIndex, "FLOAT"); - lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 22; + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + FLOAT_BUFF_LEN; } else if (strcasecmp(dataType, "DOUBLE") == 0) { len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, "T%d %s,", tagIndex, "DOUBLE"); - lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 42; + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + DOUBLE_BUFF_LEN; } else { taos_close(taos); errorPrint("%s() LN%d, config error tag type : %s\n", @@ -4894,6 +5049,17 @@ static void postFreeResource() { } } } + + tmfree(g_randbool_buff); + tmfree(g_randint_buff); + tmfree(g_rand_voltage_buff); + tmfree(g_randbigint_buff); + tmfree(g_randsmallint_buff); + tmfree(g_randtinyint_buff); + tmfree(g_randfloat_buff); + tmfree(g_rand_current_buff); + tmfree(g_rand_phase_buff); + tmfree(g_randdouble_buff); } static int getRowDataFromSample( @@ -4956,56 +5122,61 @@ static int64_t generateStbRowData( rand_string(buf, stbInfo->columns[i].dataLen); dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "\'%s\',", buf); tmfree(buf); - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, - "INT", strlen("INT"))) { - if ((g_args.demo_mode) && (i == 1)) { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, - "%d,", demo_voltage_int()); - } else { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, - "%d,", rand_int()); - } - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, - "BIGINT", strlen("BIGINT"))) { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, - "%"PRId64",", rand_bigint()); - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, - "FLOAT", strlen("FLOAT"))) { - if (g_args.demo_mode) { - if (i == 0) { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, - "%f,", demo_current_float()); + } else { + char *tmp; + + if (0 == strncasecmp(stbInfo->columns[i].dataType, + "INT", strlen("INT"))) { + if ((g_args.demo_mode) && (i == 1)) { + tmp = demo_voltage_int_str(); } else { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, - "%f,", demo_phase_float()); + tmp = rand_int_str(); } - } else { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, - "%f,", rand_float()); + tstrncpy(pstr + dataLen, tmp, INT_BUFF_LEN); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, + "BIGINT", strlen("BIGINT"))) { + tmp = rand_bigint_str(); + tstrncpy(pstr + dataLen, tmp, BIGINT_BUFF_LEN); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, + "FLOAT", strlen("FLOAT"))) { + if (g_args.demo_mode) { + if (i == 0) { + tmp = demo_current_float_str(); + } else { + tmp = demo_phase_float_str(); + } + } else { + tmp = rand_float_str(); + } + tstrncpy(pstr + dataLen, tmp, FLOAT_BUFF_LEN); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, + "DOUBLE", strlen("DOUBLE"))) { + tmp = rand_double_str(); + tstrncpy(pstr + dataLen, tmp, DOUBLE_BUFF_LEN); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, + "SMALLINT", strlen("SMALLINT"))) { + tmp = rand_smallint_str(); + tstrncpy(pstr + dataLen, tmp, SMALLINT_BUFF_LEN); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, + "TINYINT", strlen("TINYINT"))) { + tmp = rand_tinyint_str(); + tstrncpy(pstr + dataLen, tmp, TINYINT_BUFF_LEN); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, + "BOOL", strlen("BOOL"))) { + tmp = rand_bool_str(); + tstrncpy(pstr + dataLen, tmp, BOOL_BUFF_LEN); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, + "TIMESTAMP", strlen("TIMESTAMP"))) { + tmp = rand_int_str(); + tstrncpy(pstr + dataLen, tmp, INT_BUFF_LEN); + } else { + errorPrint( "Not support data type: %s\n", stbInfo->columns[i].dataType); + return -1; } - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, - "DOUBLE", strlen("DOUBLE"))) { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, - "%f,", rand_double()); - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, - "SMALLINT", strlen("SMALLINT"))) { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, - "%d,", rand_smallint()); - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, - "TINYINT", strlen("TINYINT"))) { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, - "%d,", rand_tinyint()); - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, - "BOOL", strlen("BOOL"))) { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, - "%d,", rand_bool()); - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, - "TIMESTAMP", strlen("TIMESTAMP"))) { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, - "%"PRId64",", rand_bigint()); - } else { - errorPrint( "Not support data type: %s\n", stbInfo->columns[i].dataType); - return -1; + + dataLen += strlen(tmp); + tstrncpy(pstr + dataLen, ",", 2); + dataLen += 1; } } @@ -7105,7 +7276,7 @@ static void *readTable(void *sarg) { int64_t totalData = num_of_DPT * num_of_tables; bool do_aggreFunc = g_Dbs.do_aggreFunc; - int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; + int n = do_aggreFunc ? (sizeof(g_aggreFunc) / sizeof(g_aggreFunc[0])) : 2; if (!do_aggreFunc) { printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n"); } @@ -7117,7 +7288,7 @@ static void *readTable(void *sarg) { uint64_t count = 0; for (int64_t i = 0; i < num_of_tables; i++) { sprintf(command, "select %s from %s%"PRId64" where ts>= %" PRIu64, - aggreFunc[j], tb_prefix, i, sTime); + g_aggreFunc[j], tb_prefix, i, sTime); double t = taosGetTimestampMs(); TAOS_RES *pSql = taos_query(taos, command); @@ -7142,9 +7313,9 @@ static void *readTable(void *sarg) { } fprintf(fp, "|%10s | %"PRId64" | %12.2f | %10.2f |\n", - aggreFunc[j][0] == '*' ? " * " : aggreFunc[j], totalData, + g_aggreFunc[j][0] == '*' ? " * " : g_aggreFunc[j], totalData, (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000); - printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT * 1000); + printf("select %10s took %.6f second(s)\n", g_aggreFunc[j], totalT * 1000); } fprintf(fp, "\n"); fclose(fp); @@ -7169,7 +7340,7 @@ static void *readMetric(void *sarg) { int64_t totalData = num_of_DPT * num_of_tables; bool do_aggreFunc = g_Dbs.do_aggreFunc; - int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; + int n = do_aggreFunc ? (sizeof(g_aggreFunc) / sizeof(g_aggreFunc[0])) : 2; if (!do_aggreFunc) { printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n"); } @@ -7190,7 +7361,7 @@ static void *readMetric(void *sarg) { } strncat(condition, tempS, COND_BUF_LEN - 1); - sprintf(command, "select %s from meters where %s", aggreFunc[j], condition); + sprintf(command, "select %s from meters where %s", g_aggreFunc[j], condition); printf("Where condition: %s\n", condition); fprintf(fp, "%s\n", command); @@ -7215,7 +7386,7 @@ static void *readMetric(void *sarg) { fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n", num_of_tables * num_of_DPT / (t * 1000.0), t); - printf("select %10s took %.6f second(s)\n\n", aggreFunc[j], t * 1000.0); + printf("select %10s took %.6f second(s)\n\n", g_aggreFunc[j], t * 1000.0); taos_free_result(pSql); } From 84ef79fed11d411d633485c97a92156ce30bf67e Mon Sep 17 00:00:00 2001 From: Zhiqiang Wang <1296468573@qq.com> Date: Sun, 1 Aug 2021 10:34:25 +0800 Subject: [PATCH 105/106] [TD-5656]: arm32 socket post result \0 error. (#7092) --- src/plugins/http/src/httpJson.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/http/src/httpJson.c b/src/plugins/http/src/httpJson.c index 4bd66a17a3..3c72b795ee 100644 --- a/src/plugins/http/src/httpJson.c +++ b/src/plugins/http/src/httpJson.c @@ -268,7 +268,7 @@ void httpJsonTimestamp(JsonBuf* buf, int64_t t, int32_t timePrecision) { int32_t fractionLen; char* format = NULL; time_t quot = 0; - long mod = 0; + int64_t mod = 0; switch (timePrecision) { case TSDB_TIME_PRECISION_MILLI: { From dd5a627bc2a93abc04d679e5b04d69006890a529 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 1 Aug 2021 13:06:27 +0800 Subject: [PATCH 106/106] Hotfix/sangshuduo/td 3197 fix taosdemo coverity scan (#7098) * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix taosdemo coverity scan issue. fix subscribeTest pids uninitialized. * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix coverity scan issues. check super tbl info pointer. * [TD-3197] : fix coverity scan issues. move sub tbl query thread join into loop * [TD-3197] : fix coverity scan issues. remove unused variable * [TD-3197] : fix coverity scan issues. use more secure random library * [TD-3197] : fix coverity scan issues. use strncpy for more safe * [TD-3197] : fix taosdemo coverity scan issue. replace arc4random with rand(). * [TD-3197] : fix coverity scan issues. check stb info pointer for start time * [TD-3197] : fix coverity scan issues. fix strcpy vulnerability * [TD-3197] : fix taosdemo coverity scan issue. modify taosdemoTest2. try to check database continously. * [TD-3197] : taosdemo coverity scan issues. * [TD-3197] : fix memory leak when parsing arguments. * [TD-3197] : fix cmake strip arguments. * [TD-3197] : taosdemo coverity scan. fix cmake string manipulation. * [TD-3197]: taosdemo coverity scan issue. configDir buffer overwrite. * [TD-3197]: coverity scan issue. taosdump argument validation. * [TD-3197]: taosdemo and taosdump coverity scan issues. * [TD-3197]: taosdemo coverity scan. append result buf to file. for develop branch. * exit if read sample file failed. * fix converity scan issue. * fix coverity scan issue. * fix coverity scan memory leak. * fix resource leak reported by coverity scan. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 40c8ae559d..5ac85f87f1 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -6162,6 +6162,7 @@ static int32_t prepareStbStmt( if (0 != ret) { errorPrint("%s() LN%d, stmt_bind_param() failed! reason: %s\n", __func__, __LINE__, taos_stmt_errstr(stmt)); + free(bindArray); return -1; } // if msg > 3MB, break @@ -6169,6 +6170,7 @@ static int32_t prepareStbStmt( if (0 != ret) { errorPrint("%s() LN%d, stmt_add_batch() failed! reason: %s\n", __func__, __LINE__, taos_stmt_errstr(stmt)); + free(bindArray); return -1; }