From bcfaa9f3353e35c76fe797dc8c153becade4bfe8 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 9 May 2022 14:59:23 +0800 Subject: [PATCH 1/9] add testcase --- .../1-insert/insertWithMoreVgroup.py | 6 +- tests/system-test/1-insert/manyVgroups.json | 2 +- tests/system-test/insert.json | 76 +++++++++++++++++++ 3 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 tests/system-test/insert.json diff --git a/tests/system-test/1-insert/insertWithMoreVgroup.py b/tests/system-test/1-insert/insertWithMoreVgroup.py index b583ee93e8..ae450ae69a 100644 --- a/tests/system-test/1-insert/insertWithMoreVgroup.py +++ b/tests/system-test/1-insert/insertWithMoreVgroup.py @@ -219,7 +219,7 @@ class TDTestCase: for i in range(threadNumbers): jsonfile="1-insert/Vgroups%d%d.json"%(vgroups,i) os.system("cp -f 1-insert/manyVgroups.json %s"%(jsonfile)) - os.system("sed -i 's/\"name\": \"db\",/\"name\": \"%s%d\",/g' %s"%(dbname,i,jsonfile)) + os.system("sed -i 's/\"name\": \"db\",/\"name\": \"%s\",/g' %s"%(dbname,jsonfile)) os.system("sed -i 's/\"childtable_count\": 300000,/\"childtable_count\": %d,/g' %s "%(count,jsonfile)) os.system("sed -i 's/\"name\": \"stb1\",/\"name\": \"%s%d\",/g' %s "%(stbname,i,jsonfile)) os.system("sed -i 's/\"childtable_prefix\": \"stb1_\",/\"childtable_prefix\": \"%s%d_\",/g' %s "%(stbname,i,jsonfile)) @@ -337,9 +337,11 @@ class TDTestCase: return def test_case3(self): - self.taosBenchCreate("db1", "stb1", 1, 2, 1*50000) + # self.taosBenchCreate("db1", "stb1", 4, 5, 100*10000) + # self.taosBenchCreate("db1", "stb1", 1, 5, 100*10000) + return # diff --git a/tests/system-test/1-insert/manyVgroups.json b/tests/system-test/1-insert/manyVgroups.json index df6f1163e8..6ce78af334 100644 --- a/tests/system-test/1-insert/manyVgroups.json +++ b/tests/system-test/1-insert/manyVgroups.json @@ -16,7 +16,7 @@ { "dbinfo": { "name": "db", - "drop": "yes", + "drop": "no", "vgroups": 1 }, "super_tables": [ diff --git a/tests/system-test/insert.json b/tests/system-test/insert.json new file mode 100644 index 0000000000..5dea9eabfe --- /dev/null +++ b/tests/system-test/insert.json @@ -0,0 +1,76 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 16, + "create_table_thread_count": 1, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 0, + "num_of_records_per_req": 10000, + "prepared_rand": 10000, + "chinese": "no", + "databases": [ + { + "dbinfo": { + "name": "db", + "drop": "yes", + "vgroups":4, + "replica": 1, + "precision": "ms" + }, + "super_tables": [ + { + "name": "stb", + "child_table_exists": "no", + "childtable_count": 1000, + "childtable_prefix": "stb_", + "escape_character": "no", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "non_stop_mode": "no", + "line_protocol": "line", + "insert_rows": 100000, + "interlace_rows": 0, + "insert_interval": 0, + "disorder_ratio": 0, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "use_sample_ts": "no", + "tags_file": "", + "columns": [ + { + "type": "FLOAT", + "name": "current", + "count": 4, + "max": 12, + "min": 8 + }, + { "type": "INT", "name": "voltage", "max": 225, "min": 215 }, + { "type": "FLOAT", "name": "phase", "max": 1, "min": 0 } + ], + "tags": [ + { + "type": "TINYINT", + "name": "groupid", + "max": 10, + "min": 1 + }, + { + "name": "location", + "type": "BINARY", + "len": 16, + "values": ["beijing", "shanghai"] + } + ] + } + ] + } + ] +} From 0b86cc594fa4235559e08e7aa9f3f2c206e92612 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Mon, 9 May 2022 21:46:40 +0800 Subject: [PATCH 2/9] test: add cases to ci --- tests/system-test/fulltest.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index f713f707cb..34c3484df4 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -48,4 +48,5 @@ python3 ./test.py -f 2-query/cos.py python3 ./test.py -f 2-query/tan.py python3 ./test.py -f 2-query/arcsin.py python3 ./test.py -f 2-query/arccos.py -python3 ./test.py -f 2-query/arctan.py \ No newline at end of file +python3 ./test.py -f 2-query/arctan.py +python3 ./test.py -f 2-query/query_cols_tags_and_or.py From 5588aadc27e0fcb8a7a507f0893414638542a382 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 9 May 2022 23:10:25 +0800 Subject: [PATCH 3/9] modify testcase that using taosBenchmark to test multi-process table building --- .../1-insert/insertWithMoreVgroup.py | 22 ++++++++++++++----- tests/system-test/1-insert/manyVgroups.json | 4 ++-- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/tests/system-test/1-insert/insertWithMoreVgroup.py b/tests/system-test/1-insert/insertWithMoreVgroup.py index ae450ae69a..d8050c53c5 100644 --- a/tests/system-test/1-insert/insertWithMoreVgroup.py +++ b/tests/system-test/1-insert/insertWithMoreVgroup.py @@ -199,7 +199,8 @@ class TDTestCase: os.system("%s -f %s -y " %(taosBenchbin,jsonFile)) return - def taosBenchCreate(self,dbname,stbname,vgroups,threadNumbers,count): + def taosBenchCreate(self,host,dropdb,dbname,stbname,vgroups,threadNumbers,count): + # count=50000 buildPath = self.getBuildPath() if (buildPath == ""): @@ -207,12 +208,18 @@ class TDTestCase: else: tdLog.info("taosd found in %s" % buildPath) taosBenchbin = buildPath+ "/build/bin/taosBenchmark" + buildPath = self.getBuildPath() + config = buildPath+ "../sim/dnode1/cfg/" + tsql=self.newcur(host,config) # insert: create one or mutiple tables per sql and insert multiple rows per sql - tdSql.execute("drop database if exists %s"%dbname) + tsql.execute("drop database if exists %s"%dbname) - tdSql.execute("create database %s vgroups %d"%(dbname,vgroups)) - tdSql.execute("use %s" %dbname) + tsql.execute("create database %s vgroups %d"%(dbname,vgroups)) + print("db has been created") + # tsql.getResult("show databases") + # print(tdSql.queryResult) + tsql.execute("use %s" %dbname) threads = [] # threadNumbers=2 @@ -220,7 +227,9 @@ class TDTestCase: jsonfile="1-insert/Vgroups%d%d.json"%(vgroups,i) os.system("cp -f 1-insert/manyVgroups.json %s"%(jsonfile)) os.system("sed -i 's/\"name\": \"db\",/\"name\": \"%s\",/g' %s"%(dbname,jsonfile)) - os.system("sed -i 's/\"childtable_count\": 300000,/\"childtable_count\": %d,/g' %s "%(count,jsonfile)) + os.system("sed -i 's/\"drop\": \"no\",/\"drop\": \"%s\",/g' %s"%(dropdb,jsonfile)) + os.system("sed -i 's/\"host\": \"127.0.0.1\",/\"host\": \"%s\",/g' %s"%(host,jsonfile)) + os.system("sed -i 's/\"childtable_count\": 10000,/\"childtable_count\": %d,/g' %s "%(count,jsonfile)) os.system("sed -i 's/\"name\": \"stb1\",/\"name\": \"%s%d\",/g' %s "%(stbname,i,jsonfile)) os.system("sed -i 's/\"childtable_prefix\": \"stb1_\",/\"childtable_prefix\": \"%s%d_\",/g' %s "%(stbname,i,jsonfile)) threads.append(mp.Process(target=self.taosBench, args=("%s"%jsonfile,))) @@ -337,7 +346,8 @@ class TDTestCase: return def test_case3(self): - self.taosBenchCreate("db1", "stb1", 1, 2, 1*50000) + # self.taosBenchCreate("chenhaoran02","no","db1", "stb1", 1, 8, 1*10000) + self.taosBenchCreate("chenhaoran02","no","db1", "stb1", 1, 8, 1*1000) # self.taosBenchCreate("db1", "stb1", 4, 5, 100*10000) # self.taosBenchCreate("db1", "stb1", 1, 5, 100*10000) diff --git a/tests/system-test/1-insert/manyVgroups.json b/tests/system-test/1-insert/manyVgroups.json index 6ce78af334..5487dff708 100644 --- a/tests/system-test/1-insert/manyVgroups.json +++ b/tests/system-test/1-insert/manyVgroups.json @@ -1,7 +1,7 @@ { "filetype": "insert", "cfgdir": "/etc/taos/", - "host": "test216", + "host": "127.0.0.1", "port": 6030, "user": "root", "password": "taosdata", @@ -23,7 +23,7 @@ { "name": "stb1", "child_table_exists": "no", - "childtable_count": 300000, + "childtable_count": 10000, "childtable_prefix": "stb1_", "auto_create_table": "no", "batch_create_tbl_num": 50000, From 20ce1b79362d4faa06031aab55b02b38a3008ec1 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 10 May 2022 00:03:44 +0800 Subject: [PATCH 4/9] feat: code refactor for rollup --- source/dnode/vnode/src/tsdb/tsdbRead.c | 35 ++++++++++---------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 9294718550..80e4b3167f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -351,35 +351,26 @@ static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, SQueryTableData pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr); } } -#if 1 -int nQUERY = 0; -#endif + static STsdb* getTsdbByRetentions(SVnode* pVnode, STsdbReadHandle* pReadHandle, TSKEY winSKey, SRetention* retentions) { if (vnodeIsRollup(pVnode)) { - int level = 0; -#if 0 + int level = 0; int64_t now = taosGetTimestamp(pVnode->config.tsdbCfg.precision); + for (int i = 0; i < TSDB_RETENTION_MAX; ++i) { - SRetention* pRetention = retentions + i; - if (pRetention->keep <= 0 || (now - pRetention->keep) >= winSKey) { + SRetention* pRetention = retentions + level; + if (pRetention->keep <= 0) { + if (level > 0) { + --level; + } + break; + } + if ((now - pRetention->keep) <= winSKey) { break; } ++level; } -#endif -#if 1 - switch ((nQUERY++) % 3) { - case 0: - level = 0; - break; - case 1: - level = 1; - break; - default: - level = 2; - break; - } -#endif + if (level == TSDB_RETENTION_L0) { tsdbDebug("%p rsma level %d is selected to query\n", pReadHandle, level); return VND_RSMA0(pVnode); @@ -391,7 +382,7 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, STsdbReadHandle* pReadHandle, return VND_RSMA2(pVnode); } } - return pVnode->pTsdb; + return VND_TSDB(pVnode); } static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond* pCond, uint64_t qId, uint64_t taskId) { From baddf3207659ea9645e47349173a456ca74779c4 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Tue, 10 May 2022 10:15:16 +0800 Subject: [PATCH 5/9] update test case --- tests/system-test/2-query/diff.py | 49 ++++++++----------------------- 1 file changed, 12 insertions(+), 37 deletions(-) diff --git a/tests/system-test/2-query/diff.py b/tests/system-test/2-query/diff.py index 1a6a2e7cf1..03b3899dc6 100644 --- a/tests/system-test/2-query/diff.py +++ b/tests/system-test/2-query/diff.py @@ -83,17 +83,20 @@ class TDTestCase: tdSql.query("select diff(col6) from stb_1") tdSql.checkRows(0) + tdSql.query("select diff(col7) from stb_1") + tdSql.checkRows(0) + for i in range(self.rowNum): tdSql.execute("insert into stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) - tdSql.error("select diff(ts) from stb") + # tdSql.error("select diff(ts) from stb") tdSql.error("select diff(ts) from stb_1") - tdSql.error("select diff(col7) from stb") - tdSql.error("select diff(col7) from stb_1") - tdSql.error("select diff(col8) from stb") + # tdSql.error("select diff(col7) from stb") + + # tdSql.error("select diff(col8) from stb") tdSql.error("select diff(col8) from stb_1") - tdSql.error("select diff(col9) from stb") + # tdSql.error("select diff(col9) from stb") tdSql.error("select diff(col9) from stb_1") tdSql.error("select diff(col11) from stb_1") tdSql.error("select diff(col12) from stb_1") @@ -101,31 +104,12 @@ class TDTestCase: tdSql.error("select diff(col14) from stb_1") tdSql.query("select ts,diff(col1),ts from stb_1") - tdSql.checkRows(10) + tdSql.checkRows(11) tdSql.checkData(0, 0, "2018-09-17 09:00:00.000") - tdSql.checkData(0, 1, "2018-09-17 09:00:00.000") - tdSql.checkData(0, 3, "2018-09-17 09:00:00.000") + tdSql.checkData(1, 0, "2018-09-17 09:00:00.000") + tdSql.checkData(1, 2, "2018-09-17 09:00:00.000") tdSql.checkData(9, 0, "2018-09-17 09:00:00.009") - tdSql.checkData(9, 1, "2018-09-17 09:00:00.009") - tdSql.checkData(9, 3, "2018-09-17 09:00:00.009") - - # tdSql.query("select ts,diff(col1),ts from stb group by tbname") - # tdSql.checkRows(10) - # tdSql.checkData(0, 0, "2018-09-17 09:00:00.000") - # tdSql.checkData(0, 1, "2018-09-17 09:00:00.000") - # tdSql.checkData(0, 3, "2018-09-17 09:00:00.000") - # tdSql.checkData(9, 0, "2018-09-17 09:00:00.009") - # tdSql.checkData(9, 1, "2018-09-17 09:00:00.009") - # tdSql.checkData(9, 3, "2018-09-17 09:00:00.009") - - tdSql.query("select ts,diff(col1),ts from stb_1") - tdSql.checkRows(10) - tdSql.checkData(0, 0, "2018-09-17 09:00:00.000") - tdSql.checkData(0, 1, "2018-09-17 09:00:00.000") - tdSql.checkData(0, 3, "2018-09-17 09:00:00.000") - tdSql.checkData(9, 0, "2018-09-17 09:00:00.009") - tdSql.checkData(9, 1, "2018-09-17 09:00:00.009") - tdSql.checkData(9, 3, "2018-09-17 09:00:00.009") + tdSql.checkData(9, 2, "2018-09-17 09:00:00.009") # tdSql.query("select ts,diff(col1),ts from stb group by tbname") # tdSql.checkRows(10) @@ -153,15 +137,6 @@ class TDTestCase: tdSql.query("select diff(col6) from stb_1") tdSql.checkRows(10) - - self.insertData() - - tdSql.query("select diff(col) from st group by tbname") - tdSql.checkRows(185) - - tdSql.error("select diff(col) from st group by dev") - - tdSql.error("select diff(col) from st group by col") def stop(self): tdSql.close() From 728d63c6646923f3feed8f5dcf47a70156280c50 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 10 May 2022 12:19:32 +0800 Subject: [PATCH 6/9] ehn: remove debug log --- include/common/taosdef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 5384082da3..e1f8832edf 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -89,7 +89,7 @@ extern char *qtypeStr[]; #define TSDB_PORT_HTTP 11 -#define TD_DEBUG_PRINT_ROW +#undef TD_DEBUG_PRINT_ROW #ifdef __cplusplus } From 8c0d600cdf2dff21c9bd5fea17d5e44d4c1ec0a2 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 10 May 2022 12:20:13 +0800 Subject: [PATCH 7/9] enh(sync): add error log --- source/libs/sync/src/syncRaftLog.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 031722ab3c..a422ddf4db 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -60,7 +60,7 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { if (code != 0) { int32_t err = terrno; const char *errStr = tstrerror(err); - sError("walWriteWithSyncInfo error, err:%d, msg:%s", err, errStr); + sError("walWriteWithSyncInfo error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno)); ASSERT(0); } //assert(code == 0); @@ -79,7 +79,7 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { if (code != 0) { int32_t err = terrno; const char *errStr = tstrerror(err); - sError("walReadWithHandle error, err:%d, msg:%s", err, errStr); + sError("walWriteWithSyncInfo error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno)); ASSERT(0); } //assert(walReadWithHandle(pWalHandle, index) == 0); @@ -108,7 +108,14 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - assert(walRollback(pWal, fromIndex) == 0); + //assert(walRollback(pWal, fromIndex) == 0); + int32_t code = walRollback(pWal, fromIndex); + if (code != 0) { + int32_t err = terrno; + const char *errStr = tstrerror(err); + sError("walRollback error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno)); + ASSERT(0); + } return 0; // to avoid compiler error } @@ -132,7 +139,14 @@ SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore) { int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - assert(walCommit(pWal, index) == 0); + //assert(walCommit(pWal, index) == 0); + int32_t code = walCommit(pWal, index); + if (code != 0) { + int32_t err = terrno; + const char *errStr = tstrerror(err); + sError("walCommit error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno)); + ASSERT(0); + } return 0; // to avoid compiler error } From 5a8497fe4568b3f4282b2f25163bc1612c40d5ed Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 10 May 2022 12:21:47 +0800 Subject: [PATCH 8/9] fix: undef debug flag --- include/common/taosdef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 5384082da3..e1f8832edf 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -89,7 +89,7 @@ extern char *qtypeStr[]; #define TSDB_PORT_HTTP 11 -#define TD_DEBUG_PRINT_ROW +#undef TD_DEBUG_PRINT_ROW #ifdef __cplusplus } From 503de649cbed621b4585b4926741535bfd2b4f62 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 10 May 2022 12:22:57 +0800 Subject: [PATCH 9/9] enh(sync): add error log --- source/libs/sync/src/syncRaftLog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index a422ddf4db..66c85b9cbb 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -79,7 +79,7 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { if (code != 0) { int32_t err = terrno; const char *errStr = tstrerror(err); - sError("walWriteWithSyncInfo error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno)); + sError("walReadWithHandle error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno)); ASSERT(0); } //assert(walReadWithHandle(pWalHandle, index) == 0);