From c5cc49331247a16e474d3dfdd7144d596b8b5580 Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 29 Dec 2020 16:09:19 +0800 Subject: [PATCH 01/10] [TD-2302]: fix nodejs exception when query a null value --- src/connector/nodejs/test/testNull.js | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 src/connector/nodejs/test/testNull.js diff --git a/src/connector/nodejs/test/testNull.js b/src/connector/nodejs/test/testNull.js new file mode 100644 index 0000000000..ae3938a634 --- /dev/null +++ b/src/connector/nodejs/test/testNull.js @@ -0,0 +1,10 @@ +const taos = require('../tdengine'); +var conn = taos.connect({host: "127.0.0.1", user: "root", password: "taosdata", config: "/etc/taos", port: 6030}); +var c1 = conn.cursor(); + + +c1.query('select * from test.weather', true).then(function (result) { + result.pretty(); +}); + +conn.close(); \ No newline at end of file From f9dd3a97bb09af7a45aeb94da7ed854f5a372c77 Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 29 Dec 2020 16:54:41 +0800 Subject: [PATCH 02/10] change --- src/connector/nodejs/nodetaos/cinterface.js | 34 +++++++++++++-------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/src/connector/nodejs/nodetaos/cinterface.js b/src/connector/nodejs/nodetaos/cinterface.js index 7e58f4eb02..995babdb2b 100644 --- a/src/connector/nodejs/nodetaos/cinterface.js +++ b/src/connector/nodejs/nodetaos/cinterface.js @@ -367,11 +367,15 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { let offset = 0; for (let i = 0; i < fields.length; i++) { pdata = ref.reinterpret(pblock,8,i*8); - pdata = ref.ref(pdata.readPointer()); - if (!convertFunctions[fields[i]['type']] ) { - throw new errors.DatabaseError("Invalid data type returned from database"); - } - blocks[i] = convertFunctions[fields[i]['type']](pdata, 1, fieldlens[i], offset, isMicro); + if(ref.isNull(pdata.readPointer())){ + blocks[i] = new Array(); + }else{ + pdata = ref.ref(pdata.readPointer()); + if (!convertFunctions[fields[i]['type']] ) { + throw new errors.DatabaseError("Invalid data type returned from database"); + } + blocks[i] = convertFunctions[fields[i]['type']](pdata, 1, fieldlens[i], offset, isMicro); + } } return {blocks: blocks, num_of_rows:Math.abs(num_of_rows)} } @@ -437,14 +441,18 @@ CTaosInterface.prototype.fetch_rows_a = function fetch_rows_a(result, callback, } if (numOfRows2 > 0){ for (let i = 0; i < fields.length; i++) { - if (!convertFunctions[fields[i]['type']] ) { - throw new errors.DatabaseError("Invalid data type returned from database"); - } - let prow = ref.reinterpret(row,8,i*8); - prow = prow.readPointer(); - prow = ref.ref(prow); - blocks[i] = convertFunctions[fields[i]['type']](prow, 1, fieldlens[i], offset, isMicro); - //offset += fields[i]['bytes'] * numOfRows2; + if(ref.isNull(pdata.readPointer())){ + blocks[i] = new Array(); + }else{ + if (!convertFunctions[fields[i]['type']] ) { + throw new errors.DatabaseError("Invalid data type returned from database"); + } + let prow = ref.reinterpret(row,8,i*8); + prow = prow.readPointer(); + prow = ref.ref(prow); + blocks[i] = convertFunctions[fields[i]['type']](prow, 1, fieldlens[i], offset, isMicro); + //offset += fields[i]['bytes'] * numOfRows2; + } } } callback(param2, result2, numOfRows2, blocks); From b71885472716c8ea7a84e0b22afbdcf85c186b6a Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 29 Dec 2020 17:12:17 +0800 Subject: [PATCH 03/10] change --- src/connector/nodejs/package.json | 2 +- src/connector/nodejs/test/test.js | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/connector/nodejs/package.json b/src/connector/nodejs/package.json index 3f0600a09c..2d5cf45e1d 100644 --- a/src/connector/nodejs/package.json +++ b/src/connector/nodejs/package.json @@ -1,6 +1,6 @@ { "name": "td2.0-connector", - "version": "2.0.1", + "version": "2.0.4", "description": "A Node.js connector for TDengine.", "main": "tdengine.js", "scripts": { diff --git a/src/connector/nodejs/test/test.js b/src/connector/nodejs/test/test.js index 27c35bb481..bf4bb2c541 100644 --- a/src/connector/nodejs/test/test.js +++ b/src/connector/nodejs/test/test.js @@ -84,10 +84,19 @@ q.execute().then(function(r) { r.pretty(); }); + +// test query null value +c1.execute("create table if not exists td_connector_test.weather(ts timestamp, temperature float, humidity int) tags(location nchar(64))"); +c1.execute("insert into t1 using weather tags('北京') values(now, 11.11, 11)"); +c1.execute("insert into t1(ts, temperature) values(now, 22.22)"); +c1.execute("insert into t1(ts, humidity) values(now, 33)"); +c1.query('select * from test.t1', true).then(function (result) { + result.pretty(); +}); + var q = c1.query('select * from td_connector_test.weather'); console.log(q.query); q.execute().then(function(r) { - //console.log(r); r.pretty(); }); From 8af97223ed7c0f3b6887b2cd2106c6a667569f95 Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 29 Dec 2020 17:12:50 +0800 Subject: [PATCH 04/10] change --- src/connector/nodejs/test/testNull.js | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 src/connector/nodejs/test/testNull.js diff --git a/src/connector/nodejs/test/testNull.js b/src/connector/nodejs/test/testNull.js deleted file mode 100644 index ae3938a634..0000000000 --- a/src/connector/nodejs/test/testNull.js +++ /dev/null @@ -1,10 +0,0 @@ -const taos = require('../tdengine'); -var conn = taos.connect({host: "127.0.0.1", user: "root", password: "taosdata", config: "/etc/taos", port: 6030}); -var c1 = conn.cursor(); - - -c1.query('select * from test.weather', true).then(function (result) { - result.pretty(); -}); - -conn.close(); \ No newline at end of file From 58e23b5295e40b05e37476ca08a8233b9ce5d1df Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 29 Dec 2020 18:39:45 +0800 Subject: [PATCH 05/10] [TD-2551]: get insert delay values for taosdemo performance --- tests/perftest-scripts/perftest-query.sh | 8 ++++- tests/pytest/tools/taosdemoPerformance.py | 36 +++++++++++++++++++---- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/tests/perftest-scripts/perftest-query.sh b/tests/perftest-scripts/perftest-query.sh index b96daa5464..8498094181 100755 --- a/tests/perftest-scripts/perftest-query.sh +++ b/tests/perftest-scripts/perftest-query.sh @@ -74,8 +74,14 @@ function runQueryPerfTest { CREATETABLETIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'` INSERTRECORDSTIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'` REQUESTSPERSECOND=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $13}'` + delay=`grep 'delay' taosdemoperf.txt | awk '{print $4}'` + AVGDELAY=`echo ${delay:0:${#delay}-3}` + delay=`grep 'delay' taosdemoperf.txt | awk '{print $6}'` + MAXDELAY=`echo ${delay:0:${#delay}-3}` + delay=`grep 'delay' taosdemoperf.txt | awk '{print $8}'` + MINDELAY=`echo ${delay:0:${#delay}-2}` - python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT -t $CREATETABLETIME -i $INSERTRECORDSTIME -r $REQUESTSPERSECOND | tee -a $PERFORMANCE_TEST_REPORT + python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT -t $CREATETABLETIME -i $INSERTRECORDSTIME -r $REQUESTSPERSECOND -avg $AVGDELAY -max $MAXDELAY -min $MINDELAY | tee -a $PERFORMANCE_TEST_REPORT [ -f taosdemoperf.txt ] && rm taosdemoperf.txt } diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py index 6b6296e61a..28f451b6a0 100644 --- a/tests/pytest/tools/taosdemoPerformance.py +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -22,12 +22,15 @@ import argparse import os.path class taosdemoPerformace: - def __init__(self, commitID, dbName, createTableTime, insertRecordsTime, recordsPerSecond): + def __init__(self, commitID, dbName, createTableTime, insertRecordsTime, recordsPerSecond, avgDelay, maxDelay, minDelay): self.commitID = commitID self.dbName = dbName self.createTableTime = createTableTime self.insertRecordsTime = insertRecordsTime - self.recordsPerSecond = recordsPerSecond + self.recordsPerSecond = recordsPerSecond + self.avgDelay = avgDelay + self.maxDelay = maxDelay + self.minDelay = minDelay self.host = "127.0.0.1" self.user = "root" self.password = "taosdata" @@ -43,12 +46,15 @@ class taosdemoPerformace: cursor.execute("create database if not exists %s" % self.dbName) cursor.execute("use %s" % self.dbName) - cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50))") + cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50), avg_delay float, max_delay float, min_delay float)") print("==================== taosdemo performance ====================") print("create tables time: %f" % self.createTableTime) print("insert records time: %f" % self.insertRecordsTime) print("records per second: %f" % self.recordsPerSecond) - cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s')" % (self.createTableTime, self.insertRecordsTime, self.recordsPerSecond, self.commitID)) + print("avg delay: %f" % self.avgDelay) + print("max delay: %f" % self.maxDelay) + print("min delay: %f" % self.minDelay) + cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f)" % (self.createTableTime, self.insertRecordsTime, self.recordsPerSecond, self.commitID, self.avgDelay, self.maxDelay, self.minDelay)) cursor.execute("drop database if exists taosdemo_insert_test") cursor.close() @@ -86,8 +92,28 @@ if __name__ == '__main__': action='store', type=float, help='records per request') + parser.add_argument( + '-avg', + '---avg-delay', + action='store', + type=float, + help='avg delay') + parser.add_argument( + '-max', + '---max-delay', + action='store', + type=float, + help='max delay') + parser.add_argument( + '-min', + '---min-delay', + action='store', + type=float, + help='min delay') + args = parser.parse_args() - perftest = taosdemoPerformace(args.commit_id, args.database_name, args.create_table, args.insert_records, args.records_per_second) + perftest = taosdemoPerformace(args.commit_id, args.database_name, args.create_table, args.insert_records, args.records_per_second, + args.avg_delay, args.max_delay, args.min_delay) perftest.createTablesAndStoreData() \ No newline at end of file From 502c92770ad19556dabf4d08fc0d798e19aaf231 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 30 Dec 2020 09:17:29 +0800 Subject: [PATCH 06/10] [TD-2606]: double TSDB_MAX_WAL_SIZE from 1M to 2M --- src/inc/taosdef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 1ae37fa28a..7f1ed40815 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -436,7 +436,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf #define TSDB_PORT_HTTP 11 #define TSDB_PORT_ARBITRATOR 12 -#define TSDB_MAX_WAL_SIZE (1024*1024) +#define TSDB_MAX_WAL_SIZE (1024*1024*2) typedef enum { TAOS_QTYPE_RPC = 0, From 856ada1b410258dc289560e49620659976ae4162 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 30 Dec 2020 13:18:38 +0800 Subject: [PATCH 07/10] [TD-2457]bottom+interval+order by --- tests/pytest/functions/function_bottom.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/pytest/functions/function_bottom.py b/tests/pytest/functions/function_bottom.py index 4074166f92..5e570dc8e0 100644 --- a/tests/pytest/functions/function_bottom.py +++ b/tests/pytest/functions/function_bottom.py @@ -85,6 +85,9 @@ class TDTestCase: tdSql.checkData(0, 1, 0.1) tdSql.checkData(1, 1, 1.1) + #TD-2457 bottom + interval + order by + tdSql.error('select top(col2,1) from test interval(1y) order by col2;') + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From f175b1cd08172934decd7665a389253c38cc8509 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 30 Dec 2020 13:52:49 +0800 Subject: [PATCH 08/10] [TD-2560] add test case for show tables --- tests/pytest/table/create.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/pytest/table/create.py b/tests/pytest/table/create.py index 8fedd4e920..a0991d674a 100644 --- a/tests/pytest/table/create.py +++ b/tests/pytest/table/create.py @@ -39,6 +39,23 @@ class TDTestCase: except Exception as e: tdLog.exit(e) + # case for defect: https://jira.taosdata.com:18080/browse/TD-2560 + tdSql.execute("create table db.tb02 using st tags(2)") + tdSql.execute("create table db.tb03 using st tags(3)") + tdSql.execute("create table db.tb04 using st tags(4)") + + tdSql.query("show tables like 'tb%' ") + tdSql.checkRows(4) + + tdSql.query("show tables like 'tb0%' ") + tdSql.checkRows(3) + + tdSql.execute("create table db.st0 (ts timestamp, i int) tags(j int)") + tdSql.execute("create table db.st1 (ts timestamp, i int, c2 int) tags(j int, loc nchar(20))") + + tdSql.query("show stables like 'st%' ") + tdSql.checkRows(3) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From 305d3c10c5dced347cfbbca86515365d405c2920 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 30 Dec 2020 14:05:59 +0800 Subject: [PATCH 09/10] [TD-2563]bottom/top+interval --- tests/pytest/functions/function_bottom.py | 13 +++++++++++++ tests/pytest/functions/function_top.py | 14 ++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/tests/pytest/functions/function_bottom.py b/tests/pytest/functions/function_bottom.py index 5e570dc8e0..3cc3892218 100644 --- a/tests/pytest/functions/function_bottom.py +++ b/tests/pytest/functions/function_bottom.py @@ -88,6 +88,19 @@ class TDTestCase: #TD-2457 bottom + interval + order by tdSql.error('select top(col2,1) from test interval(1y) order by col2;') + #TD-2563 top + super_table + interval + tdSql.execute("create table meters(ts timestamp, c int) tags (d int)") + tdSql.execute("create table t1 using meters tags (1)") + sql = 'insert into t1 values ' + for i in range(20000): + sql = sql + '(%d, %d)' % (self.ts + i , i % 47) + if i % 2000 == 0: + tdSql.execute(sql) + sql = 'insert into t1 values ' + tdSql.execute(sql) + tdSql.query('select bottom(c,1) from meters interval(10a)') + tdSql.checkData(0,1,0) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/functions/function_top.py b/tests/pytest/functions/function_top.py index e24ff1cc53..a98f31eea0 100644 --- a/tests/pytest/functions/function_top.py +++ b/tests/pytest/functions/function_top.py @@ -89,6 +89,20 @@ class TDTestCase: tdSql.checkRows(2) tdSql.checkData(0, 1, 8.1) tdSql.checkData(1, 1, 9.1) + + #TD-2563 top + super_table + interval + tdSql.execute("create table meters(ts timestamp, c int) tags (d int)") + tdSql.execute("create table t1 using meters tags (1)") + sql = 'insert into t1 values ' + for i in range(20000): + sql = sql + '(%d, %d)' % (self.ts + i , i % 47) + if i % 2000 == 0: + tdSql.execute(sql) + sql = 'insert into t1 values ' + tdSql.execute(sql) + tdSql.query('select top(c,1) from meters interval(10a)') + tdSql.checkData(0,1,9) + def stop(self): tdSql.close() From 2cab4f2bf17f7f772e95ef3ea6e8c10faaea3c64 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 30 Dec 2020 15:33:14 +0800 Subject: [PATCH 10/10] [TD-2617]restore log of failed case --- tests/test-all.sh | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/test-all.sh b/tests/test-all.sh index a85ac39b44..19d7803255 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -6,7 +6,16 @@ GREEN='\033[1;32m' GREEN_DARK='\033[0;32m' GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' - +function git_branch { + branch="`git branch 2>/dev/null | grep "^\*" | sed -e "s/^\*\ //"`" + if [ "${branch}" != "" ];then + if [ "${branch}" = "(no branch)" ];then + branch="(`git rev-parse --short HEAD`...)" + fi + branch=(${branch////_}) + echo "$branch" + fi +} function runSimCaseOneByOne { while read -r line; do if [[ $line =~ ^./test.sh* ]] || [[ $line =~ ^run* ]]; then @@ -44,6 +53,11 @@ function runSimCaseOneByOnefq { out_log=`tail -1 out.log ` if [[ $out_log =~ 'failed' ]];then + if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + cp -r ../../../sim ~/sim_$(git_branch)_`date "+%Y_%m_%d_%H:%M:%S"` + else + cp -r ../../sim ~/sim_$(git_branch)_`date "+%Y_%m_%d_%H:%M:%S" ` + fi exit 8 fi end_time=`date +%s` @@ -95,6 +109,7 @@ function runPyCaseOneByOnefq { end_time=`date +%s` out_log=`tail -1 pytest-out.log ` if [[ $out_log =~ 'failed' ]];then + cp -r ../../sim ~/sim_$(git_branch)_`date "+%Y_%m_%d_%H:%M:%S" ` exit 8 fi echo execution time of $case was `expr $end_time - $start_time`s. | tee -a pytest-out.log