From 6bc7ea256323d0173948a088a297c963bd59850f Mon Sep 17 00:00:00 2001 From: zyyang Date: Thu, 24 Dec 2020 17:49:07 +0800 Subject: [PATCH 1/8] [TD-2550]: fix nodejs connector of td2.0-connector cannot use in 2.0.8.0 or later --- src/connector/nodejs/nodetaos/cinterface.js | 25 +++++++-------------- src/connector/nodejs/nodetaos/taosresult.js | 4 ++-- src/connector/nodejs/test/test.js | 23 ++++++++++++++++--- 3 files changed, 30 insertions(+), 22 deletions(-) diff --git a/src/connector/nodejs/nodetaos/cinterface.js b/src/connector/nodejs/nodetaos/cinterface.js index b0908d2bd1..7e58f4eb02 100644 --- a/src/connector/nodejs/nodetaos/cinterface.js +++ b/src/connector/nodejs/nodetaos/cinterface.js @@ -144,18 +144,9 @@ function convertBinary(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; - let currOffset = 0; - // every 4 bytes, a character is encoded; - while (currOffset < data.length) { - let dataEntry = data.slice(currOffset, currOffset + nbytes); //one entry in a row under a column; - if (dataEntry.readInt64LE(0) == FieldTypes.C_NCHAR_NULL) { - res.push(null); - } - else { - res.push(dataEntry.toString("utf16le").replace(/\u0000/g, "")); - } - currOffset += nbytes; - } + let dataEntry = data.slice(0, nbytes); //one entry in a row under a column; + //TODO: should use the correct character encoding + res.push(dataEntry.toString("utf-8")); return res; } @@ -351,7 +342,8 @@ CTaosInterface.prototype.useResult = function useResult(result) { CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { //let pblock = ref.ref(ref.ref(ref.NULL)); // equal to our raw data let pblock = this.libtaos.taos_fetch_row(result); - if (pblock == null) { + let num_of_rows = 1; + if (ref.isNull(pblock) == true) { return {block:null, num_of_rows:0}; } @@ -363,17 +355,16 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { if (ref.isNull(fieldL) == false) { for (let i = 0; i < fields.length; i ++) { - let plen = ref.reinterpret(fieldL, 4, i*4); + let plen = ref.reinterpret(fieldL, 4, i*4); let len = plen.readInt32LE(0); - fieldlens.push(len); + fieldlens.push(len); } } let blocks = new Array(fields.length); blocks.fill(null); - num_of_rows = Math.abs(num_of_rows); + //num_of_rows = Math.abs(num_of_rows); let offset = 0; - pblock = pblock.deref(); for (let i = 0; i < fields.length; i++) { pdata = ref.reinterpret(pblock,8,i*8); pdata = ref.ref(pdata.readPointer()); diff --git a/src/connector/nodejs/nodetaos/taosresult.js b/src/connector/nodejs/nodetaos/taosresult.js index fd82f4e236..4138ebbec6 100644 --- a/src/connector/nodejs/nodetaos/taosresult.js +++ b/src/connector/nodejs/nodetaos/taosresult.js @@ -25,6 +25,7 @@ function TaosResult(data, fields) { * @function pretty * @since 1.0.6 */ + TaosResult.prototype.pretty = function pretty() { let fieldsStr = ""; let sizing = []; @@ -46,8 +47,7 @@ TaosResult.prototype.pretty = function pretty() { row.data.forEach((entry, i) => { if (this.fields[i]._field.type == 9) { entry = entry.toTaosString(); - } - else { + } else { entry = entry == null ? 'null' : entry.toString(); } rowStr += entry diff --git a/src/connector/nodejs/test/test.js b/src/connector/nodejs/test/test.js index 73dac8b26c..27c35bb481 100644 --- a/src/connector/nodejs/test/test.js +++ b/src/connector/nodejs/test/test.js @@ -48,6 +48,7 @@ for (let i = 0; i < 10000; i++) { // Select console.log('select * from td_connector_test.all_types limit 3 offset 100;'); c1.execute('select * from td_connector_test.all_types limit 2 offset 100;'); + var d = c1.fetchall(); console.log(c1.fields); console.log(d); @@ -77,13 +78,24 @@ c1.query('select stddev(_double), stddev(_bigint), stddev(_float) from all_types }) // Binding arguments, and then using promise -var q = c1.query('select * from td_connector_test.all_types where ts >= ? and _int > ? limit 100 offset 40;').bind(new Date(1231), 100) +var q = c1.query('select _nchar from td_connector_test.all_types where ts >= ? and _int > ? limit 100 offset 40;').bind(new Date(1231), 100) console.log(q.query); q.execute().then(function(r) { r.pretty(); }); +var q = c1.query('select * from td_connector_test.weather'); +console.log(q.query); +q.execute().then(function(r) { + //console.log(r); + r.pretty(); +}); +function sleep(sleepTime) { + for(var start = +new Date; +new Date - start <= sleepTime; ) { } +} + +sleep(10000); // Raw Async Testing (Callbacks, not promises) function cb2(param, result, rowCount, rd) { @@ -129,16 +141,21 @@ setTimeout(function(){ c1.fetchall_a(thisRes, cb4, param); },100); + // Async through promises var aq = c1.query('select count(*) from td_connector_test.all_types;',false); aq.execute_a().then(function(data) { data.pretty(); }); -c1.query('describe td_connector_test.stabletest;').execute_a().then(r=> r.pretty()); + +c1.query('describe td_connector_test.stabletest').execute_a().then(function(r){ + r.pretty() +}); + setTimeout(function(){ c1.query('drop database td_connector_test;'); },200); + setTimeout(function(){ conn.close(); },2000); - From 9585974ab36c86d41254247376f0afff3aa1d83d Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Mon, 28 Dec 2020 16:18:41 +0800 Subject: [PATCH 2/8] handle taosd route --- tests/pytest/handle_crash_gen_val_log.sh | 27 +++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/tests/pytest/handle_crash_gen_val_log.sh b/tests/pytest/handle_crash_gen_val_log.sh index f00fe40c69..6721ba49e5 100755 --- a/tests/pytest/handle_crash_gen_val_log.sh +++ b/tests/pytest/handle_crash_gen_val_log.sh @@ -5,9 +5,30 @@ GREEN='\033[1;32m' GREEN_DARK='\033[0;32m' GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' -nohup /var/lib/jenkins/workspace/TDinternal/debug/build/bin/taosd -c /var/lib/jenkins/workspace/TDinternal/community/sim/dnode1/cfg >/dev/null & +#nohup /var/lib/jenkins/workspace/TDinternal/debug/build/bin/taosd -c /var/lib/jenkins/workspace/TDinternal/community/sim/dnode1/cfg >/dev/null & + +#nohup /root/TDinternal/debug/build/bin/taosd -c /root/TDinternal/community/sim/dnode1/cfg >/dev/null & + +IN_TDINTERNAL="community" +TDIR=`pwd` +if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + cd ../.. +else + cd ../../.. +fi + +TOP_DIR=`pwd` +TAOS_DIR=`find . -name "taosd"|head -n1` +echo $TAOS_DIR +if [[ "$TAOSLIB_DIR" == *"$IN_TDINTERNAL"* ]]; then + LIB_DIR=`find . -name "libtaos.so"|grep -w lib|head -n1|cut -d '/' --fields=2,3,4,5` +else + LIB_DIR=`find . -name "libtaos.so"|grep -w lib|head -n1|cut -d '/' --fields=2,3,4` +fi +nohup $TAOS_DIR >/dev/null & +cd - ./crash_gen.sh --valgrind -p -t 10 -s 250 -b 4 -pidof taosd|xargs kill +pidof taosd|xargs kill -9 grep 'start to execute\|ERROR SUMMARY' valgrind.err|grep -v 'grep'|uniq|tee crash_gen_mem_err.log for memError in `grep 'ERROR SUMMARY' crash_gen_mem_err.log | awk '{print $4}'` @@ -31,4 +52,4 @@ if [ -n "$defiMemError" ]; then exit 8 fi fi -done \ No newline at end of file +done From 1f45dc623b999e848d2a91f752815a97e306a1c3 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Mon, 28 Dec 2020 19:06:36 +0800 Subject: [PATCH 3/8] [TD-2589] fix Integer expression error --- tests/pytest/handle_crash_gen_val_log.sh | 25 +++--------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/tests/pytest/handle_crash_gen_val_log.sh b/tests/pytest/handle_crash_gen_val_log.sh index 6721ba49e5..3388eca3a9 100755 --- a/tests/pytest/handle_crash_gen_val_log.sh +++ b/tests/pytest/handle_crash_gen_val_log.sh @@ -6,33 +6,14 @@ GREEN_DARK='\033[0;32m' GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' #nohup /var/lib/jenkins/workspace/TDinternal/debug/build/bin/taosd -c /var/lib/jenkins/workspace/TDinternal/community/sim/dnode1/cfg >/dev/null & - -#nohup /root/TDinternal/debug/build/bin/taosd -c /root/TDinternal/community/sim/dnode1/cfg >/dev/null & - -IN_TDINTERNAL="community" -TDIR=`pwd` -if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then - cd ../.. -else - cd ../../.. -fi - -TOP_DIR=`pwd` -TAOS_DIR=`find . -name "taosd"|head -n1` -echo $TAOS_DIR -if [[ "$TAOSLIB_DIR" == *"$IN_TDINTERNAL"* ]]; then - LIB_DIR=`find . -name "libtaos.so"|grep -w lib|head -n1|cut -d '/' --fields=2,3,4,5` -else - LIB_DIR=`find . -name "libtaos.so"|grep -w lib|head -n1|cut -d '/' --fields=2,3,4` -fi -nohup $TAOS_DIR >/dev/null & -cd - +nohup /root/TDinternal/debug/build/bin/taosd -c /root/TDinternal/community/sim/dnode1/cfg >/dev/null & ./crash_gen.sh --valgrind -p -t 10 -s 250 -b 4 pidof taosd|xargs kill -9 grep 'start to execute\|ERROR SUMMARY' valgrind.err|grep -v 'grep'|uniq|tee crash_gen_mem_err.log for memError in `grep 'ERROR SUMMARY' crash_gen_mem_err.log | awk '{print $4}'` do +memError=(${memError//,/}) if [ -n "$memError" ]; then if [ "$memError" -gt 12 ]; then echo -e "${RED} ## Memory errors number valgrind reports is $memError.\ @@ -44,7 +25,7 @@ done grep 'start to execute\|definitely lost:' valgrind.err|grep -v 'grep'|uniq|tee crash_gen-definitely-lost-out.log for defiMemError in `grep 'definitely lost:' crash_gen-definitely-lost-out.log | awk '{print $7}'` do - +defiMemError=(${defiMemError//,/}) if [ -n "$defiMemError" ]; then if [ "$defiMemError" -gt 3 ]; then echo -e "${RED} ## Memory errors number valgrind reports \ From 4abeb0cebcd3b5a2af7f1afdae255fd90da42d10 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 29 Dec 2020 10:13:35 +0800 Subject: [PATCH 4/8] auto find taosd --- tests/pytest/handle_crash_gen_val_log.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/pytest/handle_crash_gen_val_log.sh b/tests/pytest/handle_crash_gen_val_log.sh index 3388eca3a9..8d131b938e 100755 --- a/tests/pytest/handle_crash_gen_val_log.sh +++ b/tests/pytest/handle_crash_gen_val_log.sh @@ -6,7 +6,18 @@ GREEN_DARK='\033[0;32m' GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' #nohup /var/lib/jenkins/workspace/TDinternal/debug/build/bin/taosd -c /var/lib/jenkins/workspace/TDinternal/community/sim/dnode1/cfg >/dev/null & -nohup /root/TDinternal/debug/build/bin/taosd -c /root/TDinternal/community/sim/dnode1/cfg >/dev/null & +IN_TDINTERNAL="community" +TDIR=`pwd` +if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + cd ../.. +else + cd ../../.. +fi + +TOP_DIR=`pwd` +TAOSD_DIR=`find . -name "taosd"|grep -v community|head -n1` +nohup $TAOSD_DIR >/dev/null & +cd - ./crash_gen.sh --valgrind -p -t 10 -s 250 -b 4 pidof taosd|xargs kill -9 grep 'start to execute\|ERROR SUMMARY' valgrind.err|grep -v 'grep'|uniq|tee crash_gen_mem_err.log From 4d5a6dc971cdd1e799b8cf311b2a8961d457424c Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 29 Dec 2020 10:20:10 +0800 Subject: [PATCH 5/8] [TD-2596]clean python sql --- Jenkinsfile | 2 ++ tests/Jenkinsfile | 1 + 2 files changed, 3 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index ef26773534..b46c68e2b9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -87,6 +87,7 @@ pipeline { pre_test() sh ''' cd ${WKC}/tests + find pytest -name '*'sql|xargs rm -rf ./test-all.sh p1 date''' } @@ -98,6 +99,7 @@ pipeline { pre_test() sh ''' cd ${WKC}/tests + find pytest -name '*'sql|xargs rm -rf ./test-all.sh p2 date''' } diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile index e7d8a0b70f..09547710c6 100644 --- a/tests/Jenkinsfile +++ b/tests/Jenkinsfile @@ -42,6 +42,7 @@ pipeline { pre_test() sh ''' cd ${WKC}/tests + find pytest -name '*'sql|xargs rm -rf ./test-all.sh pytest date''' } From 3d207a7d03bcb88a5538f07be0f20847a8342ea4 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 29 Dec 2020 11:29:38 +0800 Subject: [PATCH 6/8] [TD-2558]add test case for twa --- tests/pytest/functions/function_twa_test2.py | 13 +++++++++++++ tests/pytest/pytest_1.sh | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/pytest/functions/function_twa_test2.py b/tests/pytest/functions/function_twa_test2.py index 1ffaa3c628..2a09ae3fc3 100644 --- a/tests/pytest/functions/function_twa_test2.py +++ b/tests/pytest/functions/function_twa_test2.py @@ -120,6 +120,19 @@ class TDTestCase: tdSql.checkData(2, 1, -1.5) tdSql.checkData(3, 1, -2) + #TD-2533 twa+interval with large records + tdSql.execute("create table t4(ts timestamp, c int)") + sql = 'insert into t4 values ' + for i in range(20000): + sql = sql + '(%d, %d)' % (self.ts + i * 500, i + 1) + if i % 2000 == 0: + tdSql.execute(sql) + sql = 'insert into t4 values ' + tdSql.execute(sql) + tdSql.query('select twa(c) from t4 interval(10s)') + tdSql.checkData(0,1,10.999) + + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/pytest_1.sh b/tests/pytest/pytest_1.sh index 645d89899e..b4860344d8 100755 --- a/tests/pytest/pytest_1.sh +++ b/tests/pytest/pytest_1.sh @@ -231,6 +231,6 @@ python3 test.py -f tools/taosdemoTest2.py # subscribe python3 test.py -f subscribe/singlemeter.py -#python3 test.py -f subscribe/stability.py +#python3 test.py -f subscribe/stability.py python3 test.py -f subscribe/supertable.py From 2c0b4c2a575411bbe604c85eb122d91cfc05eae8 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 29 Dec 2020 11:37:14 +0800 Subject: [PATCH 7/8] update --- tests/pytest/pytest_1.sh | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/pytest/pytest_1.sh b/tests/pytest/pytest_1.sh index bb7dc8a3e3..79815cb6c6 100755 --- a/tests/pytest/pytest_1.sh +++ b/tests/pytest/pytest_1.sh @@ -230,13 +230,7 @@ python3 test.py -f tools/lowaTest.py python3 test.py -f tools/taosdemoTest2.py # subscribe -<<<<<<< HEAD python3 test.py -f subscribe/singlemeter.py -#python3 test.py -f subscribe/stability.py +#python3 test.py -f subscribe/stability.py python3 test.py -f subscribe/supertable.py -======= -#python3 test.py -f subscribe/singlemeter.py -#python3 test.py -f subscribe/stability.py -#python3 test.py -f subscribe/supertable.py ->>>>>>> develop From b449a32f44de2b8b44c4f2bc4ca2f815a21d63d0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Dec 2020 10:14:45 +0000 Subject: [PATCH 8/8] [TD-2604]: fix coredump --- src/tsdb/src/tsdbMemTable.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 87bc7b9443..07f001f68a 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -207,10 +207,10 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { int tsdbAsyncCommit(STsdbRepo *pRepo) { if (pRepo->mem == NULL) return 0; - ASSERT(pRepo->imem == NULL); - sem_wait(&(pRepo->readyToCommit)); + ASSERT(pRepo->imem == NULL); + if (pRepo->code != TSDB_CODE_SUCCESS) { tsdbWarn("vgId:%d try to commit when TSDB not in good state: %s", REPO_ID(pRepo), tstrerror(terrno)); }