From 6816c59b2b3dbaa91ea16ff7812347bac478be5e Mon Sep 17 00:00:00 2001 From: tomchon Date: Mon, 2 Aug 2021 23:39:53 +0800 Subject: [PATCH 01/19] change version number --- cmake/version.inc | 2 +- snap/snapcraft.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/version.inc b/cmake/version.inc index 3bb06bfcf2..d34080aa43 100755 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.0.20.10") + SET(TD_VER_NUMBER "2.0.20.12") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index b06864fe5d..7538765d2c 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: tdengine base: core18 -version: '2.0.20.10' +version: '2.0.20.12' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | @@ -72,7 +72,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.0.20.10 + - usr/lib/libtaos.so.2.0.20.12 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so From 891978882a3a171b08315684a592a71b22b87487 Mon Sep 17 00:00:00 2001 From: Yiqing Liu Date: Wed, 4 Aug 2021 18:18:34 +0800 Subject: [PATCH 02/19] [TD-5783]add test for 2.0 branch (#7153) * [TD-5783]add test for 2.0 branch * fix * fix --- .drone.yml | 8 ++++++++ Jenkinsfile | 56 ++++++++++++++++++++++++++++++++++------------------- 2 files changed, 44 insertions(+), 20 deletions(-) diff --git a/.drone.yml b/.drone.yml index f7ee4e976f..3f6c622598 100644 --- a/.drone.yml +++ b/.drone.yml @@ -23,6 +23,7 @@ steps: branch: - develop - master + - 2.0 --- kind: pipeline name: test_arm64 @@ -48,6 +49,7 @@ steps: branch: - develop - master + - 2.0 --- kind: pipeline name: test_arm @@ -73,6 +75,7 @@ steps: branch: - develop - master + - 2.0 --- kind: pipeline @@ -100,6 +103,7 @@ steps: branch: - develop - master + - 2.0 --- kind: pipeline name: build_xenial @@ -125,6 +129,7 @@ steps: branch: - develop - master + - 2.0 --- kind: pipeline @@ -150,6 +155,7 @@ steps: branch: - develop - master + - 2.0 --- kind: pipeline name: build_centos7 @@ -173,6 +179,7 @@ steps: branch: - develop - master + - 2.0 --- kind: pipeline @@ -191,6 +198,7 @@ steps: branch: - develop - master + - 2.0 depends_on: diff --git a/Jenkinsfile b/Jenkinsfile index d685df3d0c..4838850722 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -41,6 +41,7 @@ def pre_test(){ sh ''' killall -9 taosd ||echo "no taosd running" killall -9 gdb || echo "no gdb running" + killall -9 python3.8 || echo "no python program running" cd ${WKC} git reset --hard HEAD~10 >/dev/null ''' @@ -51,12 +52,18 @@ def pre_test(){ git checkout master ''' } - else { + else if(env.CHANGE_TARGET == '2.0'){ + sh ''' + cd ${WKC} + git checkout 2.0 + ''' + } + else{ sh ''' cd ${WKC} git checkout develop ''' - } + } } sh''' cd ${WKC} @@ -74,7 +81,13 @@ def pre_test(){ git checkout master ''' } - else { + else if(env.CHANGE_TARGET == '2.0'){ + sh ''' + cd ${WK} + git checkout 2.0 + ''' + } + else{ sh ''' cd ${WK} git checkout develop @@ -223,24 +236,27 @@ pipeline { steps { pre_test() catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - cd ${WKC}/tests/pytest - ./crash_gen.sh -a -p -t 4 -s 2000 - ''' + timeout(time: 60, unit: 'MINUTES'){ + sh ''' + cd ${WKC}/tests/pytest + ./crash_gen.sh -a -p -t 4 -s 2000 + ''' + } + } + timeout(time: 60, unit: 'MINUTES'){ + sh ''' + cd ${WKC}/tests/pytest + rm -rf /var/lib/taos/* + rm -rf /var/log/taos/* + ./handle_crash_gen_val_log.sh + ''' + sh ''' + cd ${WKC}/tests/pytest + rm -rf /var/lib/taos/* + rm -rf /var/log/taos/* + ./handle_taosd_val_log.sh + ''' } - - sh ''' - cd ${WKC}/tests/pytest - rm -rf /var/lib/taos/* - rm -rf /var/log/taos/* - ./handle_crash_gen_val_log.sh - ''' - sh ''' - cd ${WKC}/tests/pytest - rm -rf /var/lib/taos/* - rm -rf /var/log/taos/* - ./handle_taosd_val_log.sh - ''' timeout(time: 45, unit: 'MINUTES'){ sh ''' date From d3a6db53b3cb832448919438ebfcb336361870a3 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Wed, 4 Aug 2021 19:57:18 +0800 Subject: [PATCH 03/19] Update version.inc --- cmake/version.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/version.inc b/cmake/version.inc index 3bb06bfcf2..d34080aa43 100755 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.0.20.10") + SET(TD_VER_NUMBER "2.0.20.12") ENDIF () IF (DEFINED VERCOMPATIBLE) From 3e62ea504108980d19062ef4fde68ce2614de02e Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 5 Aug 2021 09:54:31 +0800 Subject: [PATCH 04/19] [TD-5790]: taosdemo memory corrupted. (#7170) --- src/kit/taosdemo/taosdemo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index f9fe66f466..ba98fa395b 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -5182,7 +5182,7 @@ static int64_t generateStbRowData( dataLen += 1; } - if (dataLen > remainderBufLen) + if (dataLen > (remainderBufLen - (DOUBLE_BUFF_LEN + 1))) return 0; } From 8b4f05d43a60a423dc7b730778c4d68c4b190b14 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 5 Aug 2021 18:55:59 +0800 Subject: [PATCH 05/19] [TD-5811]: taosdemo use us to count delay. (#7183) to avoid very large number if the delay is 0ms. --- src/kit/taosdemo/taosdemo.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index ba98fa395b..091fde1d2f 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -6317,8 +6317,8 @@ static void printStatPerThread(threadInfo *pThreadInfo) pThreadInfo->threadID, pThreadInfo->totalInsertRows, pThreadInfo->totalAffectedRows, - (pThreadInfo->totalDelay/1000.0)? - (double)(pThreadInfo->totalAffectedRows/(pThreadInfo->totalDelay/1000.0)): + (pThreadInfo->totalDelay)? + (double)(pThreadInfo->totalAffectedRows/(pThreadInfo->totalDelay/1000000.0)): FLT_MAX); } @@ -6714,11 +6714,11 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { start_time += generated * timeStampStep; pThreadInfo->totalInsertRows += generated; - startTs = taosGetTimestampMs(); + startTs = taosGetTimestampUs(); int32_t affectedRows = execInsert(pThreadInfo, generated); - endTs = taosGetTimestampMs(); + endTs = taosGetTimestampUs(); uint64_t delay = endTs - startTs; performancePrint("%s() LN%d, insert execution time is %"PRId64"ms\n", __func__, __LINE__, delay); @@ -7255,11 +7255,15 @@ static void startMultiThreadInsertData(int threads, char* db_name, } } - fprintf(stderr, "insert delay, avg: %10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n", - avgDelay, maxDelay, minDelay); + fprintf(stderr, "insert delay, avg: %10.2fms, max: %10.2fms, min: %10.2fms\n\n", + (double)avgDelay/1000.0, + (double)maxDelay/1000.0, + (double)minDelay/1000.0); if (g_fpOfInsertResult) { - fprintf(g_fpOfInsertResult, "insert delay, avg:%10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n", - avgDelay, maxDelay, minDelay); + fprintf(g_fpOfInsertResult, "insert delay, avg:%10.2fms, max: %10.2fms, min: %10.2fms\n\n", + (double)avgDelay/1000.0, + (double)maxDelay/1000.0, + (double)minDelay/1000.0); } //taos_close(taos); From 988f3f72fc59db94e0a8fe8f3f64868a06cbf9ba Mon Sep 17 00:00:00 2001 From: tomchon Date: Thu, 5 Aug 2021 20:52:39 +0800 Subject: [PATCH 06/19] modify dockerManifest.sh --- packaging/docker/dockerManifest.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packaging/docker/dockerManifest.sh b/packaging/docker/dockerManifest.sh index 9c5a618f34..98abe4e099 100755 --- a/packaging/docker/dockerManifest.sh +++ b/packaging/docker/dockerManifest.sh @@ -44,7 +44,8 @@ echo "version=${version}" #docker manifest rm tdengine/tdengine #docker manifest rm tdengine/tdengine:${version} if [ "$verType" == "beta" ]; then - docker manifest rm tdengine/tdengine:latest + docker manifest inspect tdengine/tdengine-beta:latest + docker manifest rm tdengine/tdengine-beta:latest docker manifest create -a tdengine/tdengine-beta:${version} tdengine/tdengine-amd64-beta:${version} tdengine/tdengine-aarch64-beta:${version} tdengine/tdengine-aarch32-beta:${version} docker manifest create -a tdengine/tdengine-beta:latest tdengine/tdengine-amd64-beta:latest tdengine/tdengine-aarch64-beta:latest tdengine/tdengine-aarch32-beta:latest docker login -u tdengine -p ${passWord} #replace the docker registry username and password @@ -52,6 +53,7 @@ if [ "$verType" == "beta" ]; then docker manifest push tdengine/tdengine-beta:${version} elif [ "$verType" == "stable" ]; then + docker manifest inspect tdengine/tdengine:latest docker manifest rm tdengine/tdengine:latest docker manifest create -a tdengine/tdengine:${version} tdengine/tdengine-amd64:${version} tdengine/tdengine-aarch64:${version} tdengine/tdengine-aarch32:${version} docker manifest create -a tdengine/tdengine:latest tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest From 1c7fe38384a5963159214350b05d1f60827dd8ad Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 6 Aug 2021 00:38:16 +0800 Subject: [PATCH 07/19] Hotfix/sangshuduo/td 5725 taosdump help timestamp for 2.0 (#7192) * [TD-5725]: taosdump --help, regarding acceptable timestamp. * make example date time same. --- src/kit/taosdump/taosdump.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 165268e231..6f7a99b2df 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -231,8 +231,8 @@ static struct argp_option options[] = { {"schemaonly", 's', 0, 0, "Only dump schema.", 2}, {"without-property", 'N', 0, 0, "Dump schema without properties.", 2}, {"avro", 'V', 0, 0, "Dump apache avro format data file. By default, dump sql command sequence.", 2}, - {"start-time", 'S', "START_TIME", 0, "Start time to dump. Either epoch or ISO8601/RFC3339 format is acceptable. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 4}, - {"end-time", 'E', "END_TIME", 0, "End time to dump. Either epoch or ISO8601/RFC3339 format is acceptable. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 5}, + {"start-time", 'S', "START_TIME", 0, "Start time to dump. Either epoch or ISO8601/RFC3339 format is acceptable. ISO8601 format example: 2017-10-01T00:00:00.000+0800 or 2017-10-0100:00:00:000+0800 or '2017-10-01 00:00:00.000+0800'", 4}, + {"end-time", 'E', "END_TIME", 0, "End time to dump. Either epoch or ISO8601/RFC3339 format is acceptable. ISO8601 format example: 2017-10-01T00:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 5}, #if TSDB_SUPPORT_NANOSECOND == 1 {"precision", 'C', "PRECISION", 0, "Specify precision for converting human-readable time to epoch. Valid value is one of ms, us, and ns. Default is ms.", 6}, #else From c6474ce1bc106c1c44ab672c6fa337250b7be4ff Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 6 Aug 2021 09:17:27 +0800 Subject: [PATCH 08/19] Hotfix/sangshuduo/td 5811 taosdemo delay us for 2.0 (#7205) * [TD-5811]: taosdemo use us to count delay. to avoid very large number if the delay is 0ms. * fix interlace delay unit. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 091fde1d2f..dec26ae92d 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -6318,7 +6318,7 @@ static void printStatPerThread(threadInfo *pThreadInfo) pThreadInfo->totalInsertRows, pThreadInfo->totalAffectedRows, (pThreadInfo->totalDelay)? - (double)(pThreadInfo->totalAffectedRows/(pThreadInfo->totalDelay/1000000.0)): + (double)(pThreadInfo->totalAffectedRows/((double)pThreadInfo->totalDelay/1000000.0)): FLT_MAX); } @@ -6538,7 +6538,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { verbosePrint("[%d] %s() LN%d, buffer=%s\n", pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->buffer); - startTs = taosGetTimestampMs(); + startTs = taosGetTimestampUs(); if (recOfBatch == 0) { errorPrint("[%d] %s() LN%d Failed to insert records of batch %d\n", @@ -6554,10 +6554,10 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { } int64_t affectedRows = execInsert(pThreadInfo, recOfBatch); - endTs = taosGetTimestampMs(); + endTs = taosGetTimestampUs(); uint64_t delay = endTs - startTs; - performancePrint("%s() LN%d, insert execution time is %"PRIu64"ms\n", - __func__, __LINE__, delay); + performancePrint("%s() LN%d, insert execution time is %10.2f ms\n", + __func__, __LINE__, delay / 1000.0); verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n", pThreadInfo->threadID, __func__, __LINE__, affectedRows); @@ -6720,8 +6720,8 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { endTs = taosGetTimestampUs(); uint64_t delay = endTs - startTs; - performancePrint("%s() LN%d, insert execution time is %"PRId64"ms\n", - __func__, __LINE__, delay); + performancePrint("%s() LN%d, insert execution time is %10.f ms\n", + __func__, __LINE__, delay/1000.0); verbosePrint("[%d] %s() LN%d affectedRows=%d\n", pThreadInfo->threadID, __func__, __LINE__, affectedRows); From 90ffa91b2e0ff9c66f38b2c8d3cb5cef683c0cfb Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 6 Aug 2021 13:45:34 +0800 Subject: [PATCH 09/19] [TD-5773]add some arm env in Drone --- .drone.yml | 114 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 80 insertions(+), 34 deletions(-) diff --git a/.drone.yml b/.drone.yml index 3f6c622598..b520f308ba 100644 --- a/.drone.yml +++ b/.drone.yml @@ -23,18 +23,16 @@ steps: branch: - develop - master - - 2.0 --- kind: pipeline -name: test_arm64 +name: test_arm64_bionic platform: os: linux arch: arm64 - steps: - name: build - image: gcc + image: arm64v8/ubuntu:bionic commands: - apt-get update - apt-get install -y cmake build-essential @@ -52,7 +50,84 @@ steps: - 2.0 --- kind: pipeline -name: test_arm +name: test_arm64_focal + +platform: + os: linux + arch: arm64 + +steps: +- name: build + image: arm64v8/ubuntu:focal + commands: + - echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections + - apt-get update + - apt-get install -y -qq cmake build-essential + - mkdir debug + - cd debug + - cmake .. -DCPUTYPE=aarch64 > /dev/null + - make + trigger: + event: + - pull_request + when: + branch: + - develop + - master + - 2.0 +--- +kind: pipeline +name: test_arm64_centos7 + +platform: + os: linux + arch: arm64 + +steps: +- name: build + image: arm64v8/centos:7 + commands: + - yum install -y gcc gcc-c++ make cmake git + - mkdir debug + - cd debug + - cmake .. -DCPUTYPE=aarch64 > /dev/null + - make + trigger: + event: + - pull_request + when: + branch: + - develop + - master + - 2.0 +--- +kind: pipeline +name: test_arm64_centos8 + +platform: + os: linux + arch: arm64 + +steps: +- name: build + image: arm64v8/centos:8 + commands: + - dnf install -y gcc gcc-c++ make cmake epel-release git libarchive + - mkdir debug + - cd debug + - cmake .. -DCPUTYPE=aarch64 > /dev/null + - make + trigger: + event: + - pull_request + when: + branch: + - develop + - master + - 2.0 +--- +kind: pipeline +name: test_arm_bionic platform: os: linux @@ -75,8 +150,6 @@ steps: branch: - develop - master - - 2.0 - --- kind: pipeline name: build_trusty @@ -103,7 +176,6 @@ steps: branch: - develop - master - - 2.0 --- kind: pipeline name: build_xenial @@ -129,7 +201,6 @@ steps: branch: - develop - master - - 2.0 --- kind: pipeline @@ -155,7 +226,6 @@ steps: branch: - develop - master - - 2.0 --- kind: pipeline name: build_centos7 @@ -179,28 +249,4 @@ steps: branch: - develop - master - - 2.0 ---- -kind: pipeline -name: goodbye - -platform: - os: linux - arch: amd64 - -steps: -- name: 64-bit - image: alpine - commands: - - echo 64-bit is good. - when: - branch: - - develop - - master - - 2.0 - - -depends_on: -- test_arm64 -- test_amd64 \ No newline at end of file From 14ab3747e6b31ca03e86cb72ad6d75748a76d8ef Mon Sep 17 00:00:00 2001 From: Linhe Huo Date: Fri, 6 Aug 2021 14:57:38 +0800 Subject: [PATCH 10/19] [TD-5860]: fix grafanaplugin error in grafana 6.x(6.2+) (#7224) --- src/connector/grafanaplugin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin index 4a4d79099b..a44ec1ca49 160000 --- a/src/connector/grafanaplugin +++ b/src/connector/grafanaplugin @@ -1 +1 @@ -Subproject commit 4a4d79099b076b8ff12d5b4fdbcba54049a6866d +Subproject commit a44ec1ca493ad01b2bf825b6418f69e11f548206 From a93326c9799def7472e07ec6bcdcb3db40055697 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 7 Aug 2021 14:05:57 +0800 Subject: [PATCH 11/19] [TD-5872]: taosdemo stmt performance. (#7236) --- src/kit/taosdemo/taosdemo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index dec26ae92d..c895545b81 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -5982,7 +5982,7 @@ static int32_t prepareStbStmtBind( int64_t startTime, int32_t recSeq, bool isColumn) { - char *bindBuffer = calloc(1, g_args.len_of_binary); + char *bindBuffer = calloc(1, DOUBLE_BUFF_LEN); // g_args.len_of_binary); if (bindBuffer == NULL) { errorPrint("%s() LN%d, Failed to allocate %d bind buffer\n", __func__, __LINE__, g_args.len_of_binary); From a419f091253bc83314d1e51c4b9596629b1de6b6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 9 Aug 2021 10:14:31 +0800 Subject: [PATCH 12/19] change version number --- snap/snapcraft.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index aef706311d..c04fa3298b 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: tdengine base: core18 -version: '2.1.5.0' +version: '2.1.6.0' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | @@ -72,7 +72,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.1.5.0 + - usr/lib/libtaos.so.2.1.6.0 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so From e554181cfb0f2641bcfcdbf4dee02ff8968099f1 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 10 Aug 2021 14:13:00 +0800 Subject: [PATCH 13/19] [TD-5921]:fix case problem in 'tbname in' syntax --- src/tsdb/src/tsdbRead.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index e1d40aa7d0..260c3d67dc 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -3468,6 +3468,7 @@ void filterPrepare(void* expr, void* param) { SArray *arr = (SArray *)(pCond->arr); for (size_t i = 0; i < taosArrayGetSize(arr); i++) { char* p = taosArrayGetP(arr, i); + strtolower(varDataVal(p), varDataVal(p)); taosHashPut(pObj, varDataVal(p),varDataLen(p), &dummy, sizeof(dummy)); } } else { From fbf76a3705bcb6014a8f413d0a18a553aa4f5a2e Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 10 Aug 2021 18:46:55 +0800 Subject: [PATCH 14/19] [TD-5922]: add unitest cases --- tests/pytest/query/tbname.py | 3 +++ .../script/general/parser/tbnameIn_query.sim | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/tests/pytest/query/tbname.py b/tests/pytest/query/tbname.py index 08416ba3ed..30d90b1f9d 100644 --- a/tests/pytest/query/tbname.py +++ b/tests/pytest/query/tbname.py @@ -53,6 +53,9 @@ class TDTestCase: "select * from cars where id=0 and tbname in ('carzero', 'cartwo')") tdSql.checkRows(1) + tdSql.query("select * from cars where tbname in ('carZero', 'CARONE')") + tdSql.checkRows(2) + """ tdSql.query("select * from cars where tbname like 'car%'") tdSql.checkRows(2) diff --git a/tests/script/general/parser/tbnameIn_query.sim b/tests/script/general/parser/tbnameIn_query.sim index 65bb89d549..db27886bbf 100644 --- a/tests/script/general/parser/tbnameIn_query.sim +++ b/tests/script/general/parser/tbnameIn_query.sim @@ -101,6 +101,30 @@ if $data11 != 2 then return -1 endi +## tbname in can accpet Upper case table name +sql select count(*) from $stb where tbname in ('ti_tb0', 'TI_tb1', 'TI_TB2') group by t1 order by t1 +if $rows != 3 then + return -1 +endi +if $data00 != 10 then + return -1 +endi +if $data01 != 0 then + return -1 +endi +if $data10 != 10 then + return -1 +endi +if $data11 != 1 then + return -1 +endi +if $data20 != 10 then + return -1 +endi +if $data21 != 2 then + return -1 +endi + # multiple tbname in is not allowed NOW sql_error select count(*) from $stb where tbname in ('ti_tb1', 'ti_tb300') and tbname in ('ti_tb5', 'ti_tb1000') group by t1 order by t1 asc #if $rows != 4 then From ce5b43c17c701083ab94af3045bce41d0e587f9f Mon Sep 17 00:00:00 2001 From: wpan Date: Wed, 11 Aug 2021 08:46:59 +0800 Subject: [PATCH 15/19] fix top/bottom use wrong expr issue --- src/query/inc/qUtil.h | 4 ++-- src/query/src/qExecutor.c | 10 +++++----- src/query/src/qUtil.c | 12 ++++++++++++ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index d2802d9fe0..ce607f0fe2 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -39,7 +39,6 @@ #define GET_QID(_r) (((SQInfo*)((_r)->qinfo))->qId) #define curTimeWindowIndex(_winres) ((_winres)->curIndex) -#define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!(sq)))? (_q)->pExpr1[1].base.param[0].i64:1) int32_t getOutputInterResultBufSize(SQueryAttr* pQueryAttr); @@ -60,6 +59,7 @@ SResultRowCellInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t void* destroyQueryFuncExpr(SExprInfo* pExprInfo, int32_t numOfExpr); void* freeColumnInfo(SColumnInfo* pColumnInfo, int32_t numOfCols); +int32_t getRowNumForMultioutput(SQueryAttr* pQueryAttr, bool topBottomQuery, bool stable); static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) { assert(pResultRowInfo != NULL && slot >= 0 && slot < pResultRowInfo->size); @@ -70,7 +70,7 @@ static FORCE_INLINE char* getPosInResultPage(SQueryAttr* pQueryAttr, tFilePage* int32_t offset) { assert(rowOffset >= 0 && pQueryAttr != NULL); - int32_t numOfRows = (int32_t)GET_ROW_PARAM_FOR_MULTIOUTPUT(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery); + int32_t numOfRows = (int32_t)getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery); return ((char *)page->data) + rowOffset + offset * numOfRows; } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 52e5dbb7f3..e31792398d 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2706,7 +2706,7 @@ static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, i SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t MIN_ROWS_PER_PAGE = 4; - *rowsize = (int32_t)(pQueryAttr->resultRowSize * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); + *rowsize = (int32_t)(pQueryAttr->resultRowSize * getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); int32_t overhead = sizeof(tFilePage); // one page contains at least two rows @@ -3630,7 +3630,7 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOf // re-estabilish output buffer pointer. int32_t functionId = pBInfo->pCtx[i].functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { - pBInfo->pCtx[i].ptsOutputBuf = pBInfo->pCtx[0].pOutput; + pBInfo->pCtx[i].ptsOutputBuf = pBInfo->pCtx[i-1].pOutput; } } } @@ -5298,7 +5298,7 @@ SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SMultiwayMergeInfo* pInfo = calloc(1, sizeof(SMultiwayMergeInfo)); pInfo->resultRowFactor = - (int32_t)(GET_ROW_PARAM_FOR_MULTIOUTPUT(pRuntimeEnv->pQueryAttr, pRuntimeEnv->pQueryAttr->topBotQuery, false)); + (int32_t)(getRowNumForMultioutput(pRuntimeEnv->pQueryAttr, pRuntimeEnv->pQueryAttr->topBotQuery, false)); pRuntimeEnv->scanFlag = MERGE_STAGE; // TODO init when creating pCtx @@ -6327,7 +6327,7 @@ SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOpera SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t numOfRows = (int32_t)(GET_ROW_PARAM_FOR_MULTIOUTPUT(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); + int32_t numOfRows = (int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, numOfRows); pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); @@ -6701,7 +6701,7 @@ SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperato SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; pQueryAttr->resultRowSize = (pQueryAttr->resultRowSize * - (int32_t)(GET_ROW_PARAM_FOR_MULTIOUTPUT(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery))); + (int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery))); pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index a3d2e424d2..f017c4db1f 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -30,6 +30,18 @@ typedef struct SCompSupporter { int32_t order; } SCompSupporter; +int32_t getRowNumForMultioutput(SQueryAttr* pQueryAttr, bool topBottomQuery, bool stable) { + if (pQueryAttr && (!stable)) { + for (int16_t i = 0; i < pQueryAttr->numOfOutput; ++i) { + if (pQueryAttr->pExpr1[i].base.functionId == TSDB_FUNC_TOP || pQueryAttr->pExpr1[i].base.functionId == TSDB_FUNC_BOTTOM) { + return pQueryAttr->pExpr1[i].base.param[0].i64; + } + } + } + + return 1; +} + int32_t getOutputInterResultBufSize(SQueryAttr* pQueryAttr) { int32_t size = 0; From 37b43a40067d512d5e76fd1fbf4561434a7c4085 Mon Sep 17 00:00:00 2001 From: wpan Date: Wed, 11 Aug 2021 09:16:13 +0800 Subject: [PATCH 16/19] fix windows compile error --- src/query/src/qUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index f017c4db1f..4caf351799 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -34,7 +34,7 @@ int32_t getRowNumForMultioutput(SQueryAttr* pQueryAttr, bool topBottomQuery, boo if (pQueryAttr && (!stable)) { for (int16_t i = 0; i < pQueryAttr->numOfOutput; ++i) { if (pQueryAttr->pExpr1[i].base.functionId == TSDB_FUNC_TOP || pQueryAttr->pExpr1[i].base.functionId == TSDB_FUNC_BOTTOM) { - return pQueryAttr->pExpr1[i].base.param[0].i64; + return (int32_t)pQueryAttr->pExpr1[i].base.param[0].i64; } } } From e130f9f2f9bde5f2b6649ac1f2fd8c01eceb6af9 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 8 Aug 2021 16:59:04 +0800 Subject: [PATCH 17/19] [TD-5875]: taosdemo show progress --- src/kit/taosdemo/taosdemo.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 8db149a559..883dfd95f2 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -246,7 +246,6 @@ typedef struct SArguments_S { uint32_t disorderRatio; // 0: no disorder, >0: x% int disorderRange; // ms, us or ns. accordig to database precision uint32_t method_of_delete; - char ** arg_list; uint64_t totalInsertRows; uint64_t totalAffectedRows; bool demo_mode; // use default column name and semi-random data @@ -638,7 +637,6 @@ SArguments g_args = { 0, // disorderRatio 1000, // disorderRange 1, // method_of_delete - NULL, // arg_list 0, // totalInsertRows; 0, // totalAffectedRows; true, // demo_mode; @@ -6401,6 +6399,9 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { bool flagSleep = true; uint64_t sleepTimeTotal = 0; + int percentComplete = 0; + int64_t totalRows = insertRows * pThreadInfo->ntables; + while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) { if ((flagSleep) && (insert_interval)) { st = taosGetTimestampMs(); @@ -6577,6 +6578,11 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pThreadInfo->totalAffectedRows += affectedRows; + int currentPercent = pThreadInfo->totalAffectedRows * 100 / totalRows; + if (currentPercent > percentComplete ) { + printf("[%d]:%d%%\n", pThreadInfo->threadID, currentPercent); + percentComplete = currentPercent; + } int64_t currentPrintTime = taosGetTimestampMs(); if (currentPrintTime - lastPrintTime > 30*1000) { printf("thread[%d] has currently inserted rows: %"PRIu64 ", affected rows: %"PRIu64 "\n", @@ -6598,6 +6604,8 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { } } } + if (percentComplete < 100) + printf("[%d]:%d%%\n", pThreadInfo->threadID, percentComplete); free_of_interlace: tmfree(pThreadInfo->buffer); @@ -6635,6 +6643,9 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { pThreadInfo->samplePos = 0; + int percentComplete = 0; + int64_t totalRows = insertRows * pThreadInfo->ntables; + for (uint64_t tableSeq = pThreadInfo->start_table_from; tableSeq <= pThreadInfo->end_table_to; tableSeq ++) { @@ -6740,6 +6751,11 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { pThreadInfo->totalAffectedRows += affectedRows; + int currentPercent = pThreadInfo->totalAffectedRows * 100 / totalRows; + if (currentPercent > percentComplete ) { + printf("[%d]:%d%%\n", pThreadInfo->threadID, currentPercent); + percentComplete = currentPercent; + } int64_t currentPrintTime = taosGetTimestampMs(); if (currentPrintTime - lastPrintTime > 30*1000) { printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n", @@ -6762,6 +6778,8 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { __func__, __LINE__, pThreadInfo->samplePos); } } // tableSeq + if (percentComplete < 100) + printf("[%d]:%d%%\n", pThreadInfo->threadID, percentComplete); free_of_progressive: tmfree(pThreadInfo->buffer); From 52848c4de5ac5d6415267e8a08fb7860acb8cc43 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Wed, 11 Aug 2021 14:55:45 +0800 Subject: [PATCH 18/19] [TD-5923]finish test--->where tbname in UPPER/lower/mIXed add query/queryTbnameUpperLower.py add util/common.py for adding common function in case of duplicate use --- tests/pytest/fulltest.sh | 1 + tests/pytest/query/queryTbnameUpperLower.py | 78 +++++++++++++++++++++ tests/pytest/util/common.py | 53 ++++++++++++++ 3 files changed, 132 insertions(+) create mode 100644 tests/pytest/query/queryTbnameUpperLower.py create mode 100644 tests/pytest/util/common.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index fdcd0a3c42..17c2fc6915 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -381,6 +381,7 @@ python3 test.py -f alter/alter_create_exception.py python3 ./test.py -f insert/flushwhiledrop.py python3 ./test.py -f insert/schemalessInsert.py python3 ./test.py -f alter/alterColMultiTimes.py +python3 ./test.py -f query/queryTbnameUpperLower.py #======================p4-end=============== diff --git a/tests/pytest/query/queryTbnameUpperLower.py b/tests/pytest/query/queryTbnameUpperLower.py new file mode 100644 index 0000000000..bd4e85c5ca --- /dev/null +++ b/tests/pytest/query/queryTbnameUpperLower.py @@ -0,0 +1,78 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.common import tdCom + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def checkStbWhereIn(self): + ''' + where in ---> upper lower mixed + ''' + tdCom.cleanTb() + table_name = tdCom.getLongName(8, "letters_mixed") + table_name_sub = f'{table_name}_sub' + tb_name_lower = table_name_sub.lower() + tb_name_upper = table_name_sub.upper() + + ## create stb and tb + tdSql.execute(f'CREATE TABLE {table_name} (ts timestamp, id int, bi1 binary(20)) tags (si1 binary(20))') + tdSql.execute(f'create table {table_name_sub}1 using {table_name} tags ("{table_name_sub}1")') + tdSql.execute(f'create table {tb_name_lower}2 using {table_name} tags ("{tb_name_lower}2")') + tdSql.execute(f'create table {tb_name_upper}3 using {table_name} tags ("{tb_name_upper}3")') + + ## insert values + tdSql.execute(f'insert into {table_name_sub}1 values (now-1s, 1, "{table_name_sub}1")') + tdSql.execute(f'insert into {tb_name_lower}2 values (now-2s, 2, "{tb_name_lower}21")') + tdSql.execute(f'insert into {tb_name_lower}2 values (now-3s, 3, "{tb_name_lower}22")') + tdSql.execute(f'insert into {tb_name_upper}3 values (now-4s, 4, "{tb_name_upper}31")') + tdSql.execute(f'insert into {tb_name_upper}3 values (now-5s, 5, "{tb_name_upper}32")') + tdSql.execute(f'insert into {tb_name_upper}3 values (now-6s, 6, "{tb_name_upper}33")') + + ## query where tbname in single + tdSql.query(f'select * from {table_name} where tbname in ("{table_name_sub}1")') + tdSql.checkRows(1) + tdSql.query(f'select * from {table_name} where tbname in ("{table_name_sub.upper()}1")') + tdSql.checkRows(1) + tdSql.query(f'select * from {table_name} where tbname in ("{table_name_sub.lower()}1")') + tdSql.checkRows(1) + tdSql.query(f'select * from {table_name} where tbname in ("{tb_name_lower}2")') + tdSql.checkRows(2) + tdSql.query(f'select * from {table_name} where tbname in ("{tb_name_lower.upper()}2")') + tdSql.checkRows(2) + tdSql.query(f'select * from {table_name} where tbname in ("{tb_name_upper}3")') + tdSql.checkRows(3) + tdSql.query(f'select * from {table_name} where tbname in ("{tb_name_upper.lower()}3")') + tdSql.checkRows(3) + + ## query where tbname in multi + tdSql.query(f'select * from {table_name} where id=5 and tbname in ("{table_name_sub}1", "{tb_name_lower.upper()}2", "{tb_name_upper.lower()}3")') + tdSql.checkRows(1) + tdSql.query(f'select * from {table_name} where tbname in ("{table_name_sub}1", "{tb_name_lower.upper()}2", "{tb_name_upper.lower()}3")') + tdSql.checkRows(6) + + def run(self): + tdSql.prepare() + self.checkStbWhereIn() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py new file mode 100644 index 0000000000..1c7d94a8a4 --- /dev/null +++ b/tests/pytest/util/common.py @@ -0,0 +1,53 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string +from util.sql import tdSql + +class TDCom: + def init(self, conn, logSql): + tdSql.init(conn.cursor(), logSql) + + def cleanTb(self): + query_sql = "show stables" + res_row_list = tdSql.query(query_sql, True) + stb_list = map(lambda x: x[0], res_row_list) + for stb in stb_list: + tdSql.execute(f'drop table if exists {stb}') + + query_sql = "show tables" + res_row_list = tdSql.query(query_sql, True) + tb_list = map(lambda x: x[0], res_row_list) + for tb in tb_list: + tdSql.execute(f'drop table if exists {tb}') + + def getLongName(self, len, mode = "mixed"): + """ + generate long name + mode could be numbers/letters/letters_mixed/mixed + """ + if mode == "numbers": + chars = ''.join(random.choice(string.digits) for i in range(len)) + elif mode == "letters": + chars = ''.join(random.choice(string.ascii_letters.lower()) for i in range(len)) + elif mode == "letters_mixed": + chars = ''.join(random.choice(string.ascii_letters.upper() + string.ascii_letters.lower()) for i in range(len)) + else: + chars = ''.join(random.choice(string.ascii_letters.lower() + string.digits) for i in range(len)) + return chars + + def close(self): + self.cursor.close() + +tdCom = TDCom() \ No newline at end of file From 2dd5f45b544c050a2d6535ecc620a4bf9e7cfba1 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 11 Aug 2021 18:43:42 +0800 Subject: [PATCH 19/19] [TD-5835]: update performance test script --- tests/pytest/tools/taosdemoPerformance.py | 62 +++++++++++++++++++---- 1 file changed, 53 insertions(+), 9 deletions(-) diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py index 6b5681dfbc..c28f94b3db 100644 --- a/tests/pytest/tools/taosdemoPerformance.py +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -19,11 +19,16 @@ import json import sys class taosdemoPerformace: - def __init__(self, commitID, dbName, branch, type): + def __init__(self, commitID, dbName, branch, type, numOfTables, numOfRows, numOfInt, numOfDouble, numOfBinary): self.commitID = commitID self.dbName = dbName self.branch = branch self.type = type + self.numOfTables = numOfTables + self.numOfRows = numOfRows + self.numOfInt = numOfInt + self.numOfDouble = numOfDouble + self.numOfBinary = numOfBinary self.host = "127.0.0.1" self.user = "root" self.password = "taosdata" @@ -51,13 +56,13 @@ class taosdemoPerformace: stb = { "name": "meters", "child_table_exists": "no", - "childtable_count": 10000, + "childtable_count": self.numOfTables, "childtable_prefix": "stb_", "auto_create_table": "no", "data_source": "rand", "batch_create_tbl_num": 10, "insert_mode": "taosc", - "insert_rows": 100000, + "insert_rows": self.numOfRows, "interlace_rows": 100, "max_sql_len": 1024000, "disorder_ratio": 0, @@ -68,7 +73,9 @@ class taosdemoPerformace: "sample_file": "./sample.csv", "tags_file": "", "columns": [ - {"type": "INT", "count": 4} + {"type": "INT", "count": self.numOfInt}, + {"type": "DOUBLE", "count": self.numOfDouble}, + {"type": "BINARY", "len": 128, "count": self.numOfBinary} ], "tags": [ {"type": "INT", "count": 1}, @@ -76,6 +83,7 @@ class taosdemoPerformace: ] } + stables = [] stables.append(stb) @@ -163,7 +171,7 @@ class taosdemoPerformace: cursor.execute("create database if not exists %s" % self.dbName) cursor.execute("use %s" % self.dbName) - cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50), avg_delay float, max_delay float, min_delay float, branch binary(50), type binary(20))") + cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50), avg_delay float, max_delay float, min_delay float, branch binary(50), type binary(20), numoftables int, numofrows int, numofint int, numofdouble int, numofbinary int)") print("==================== taosdemo performance ====================") print("create tables time: %f" % float(self.createTableTime)) print("insert records time: %f" % float(self.insertRecordsTime)) @@ -171,13 +179,14 @@ class taosdemoPerformace: print("avg delay: %f" % float(self.avgDelay)) print("max delay: %f" % float(self.maxDelay)) print("min delay: %f" % float(self.minDelay)) - cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f, '%s', '%s')" % + cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f, '%s', '%s', %d, %d, %d, %d, %d)" % (float(self.createTableTime), float(self.insertRecordsTime), float(self.recordsPerSecond), - self.commitID, float(self.avgDelay), float(self.maxDelay), float(self.minDelay), self.branch, self.type)) + self.commitID, float(self.avgDelay), float(self.maxDelay), float(self.minDelay), self.branch, + self.type, self.numOfTables, self.numOfRows, self.numOfInt, self.numOfDouble, self.numOfBinary)) cursor.close() cursor1 = self.conn.cursor() - cursor1.execute("drop database if exists %s" % self.insertDB) + # cursor1.execute("drop database if exists %s" % self.insertDB) cursor1.close() if __name__ == '__main__': @@ -209,8 +218,43 @@ if __name__ == '__main__': default='glibc', type=str, help='build type (default: glibc)') + parser.add_argument( + '-i', + '--num-of-int', + action='store', + default=4, + type=int, + help='num of int columns (default: 4)') + parser.add_argument( + '-D', + '--num-of-double', + action='store', + default=0, + type=int, + help='num of double columns (default: 4)') + parser.add_argument( + '-B', + '--num-of-binary', + action='store', + default=0, + type=int, + help='num of binary columns (default: 4)') + parser.add_argument( + '-t', + '--num-of-tables', + action='store', + default=10000, + type=int, + help='num of tables (default: 10000)') + parser.add_argument( + '-r', + '--num-of-rows', + action='store', + default=100000, + type=int, + help='num of rows (default: 100000)') args = parser.parse_args() - perftest = taosdemoPerformace(args.commit_id, args.database_name, args.git_branch, args.build_type) + perftest = taosdemoPerformace(args.commit_id, args.database_name, args.git_branch, args.build_type, args.num_of_tables, args.num_of_rows, args.num_of_int, args.num_of_double, args.num_of_binary) perftest.insertData() perftest.createTablesAndStoreData()