From 9eb020f900f18befe5d6c3b181944c13a11e0d36 Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Tue, 1 Jun 2021 18:02:32 +0800 Subject: [PATCH 01/10] [TD-4475] adding draft test case --- tests/pytest/manualTest/manual_alter_block.py | 70 +++++++++++++++++++ .../insert_5Mrows_small_cache.json | 60 ++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 tests/pytest/manualTest/manual_alter_block.py create mode 100644 tests/pytest/tools/taosdemoAllTest/insert_5Mrows_small_cache.json diff --git a/tests/pytest/manualTest/manual_alter_block.py b/tests/pytest/manualTest/manual_alter_block.py new file mode 100644 index 0000000000..a06e1bba6f --- /dev/null +++ b/tests/pytest/manualTest/manual_alter_block.py @@ -0,0 +1,70 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + tdSql.prepare() + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + tdSql.execute('alter database db blocks 8') + tdSql.query('show databases') + tdSql.checkData(0,9,3) + + os.system("%staosdemo -f tools/taosdemoAllTest/insert_5Mrows_small_cache.json" % binPath) + + input("please check memory usage for taosd. After checking, press enter") + + tdSql.execute('alter database db blocks 8') + + os.system("%staosdemo -f tools/taosdemoAllTest/insert_5Mrows_small_cache.json" % binPath) + + input("please check memory usage for taosd. After checking, press enter") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdemoAllTest/insert_5Mrows_small_cache.json b/tests/pytest/tools/taosdemoAllTest/insert_5Mrows_small_cache.json new file mode 100644 index 0000000000..d437735ebc --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/insert_5Mrows_small_cache.json @@ -0,0 +1,60 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 100, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "no", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 3, + "precision": "ms", + "keep": 3650, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb", + "child_table_exists":"no", + "childtable_count": 500, + "childtable_prefix": "stb_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 10000, + "childtable_limit": 10, + "childtable_offset":100, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 10, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}], + "tags": [{"type": "TINYINT", "count":2}] + }] + }] +} \ No newline at end of file From 08f33005b6d1b5d7048ce046e1475558627e832a Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Tue, 1 Jun 2021 19:02:34 +0800 Subject: [PATCH 02/10] [TD-4475] update test case --- tests/pytest/manualTest/manual_alter_block.py | 8 ++- .../taosdemoAllTest/insert_5Mrows_hasTB.json | 60 +++++++++++++++++++ ...all_cache.json => insert_5Mrows_noTB.json} | 0 3 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 tests/pytest/tools/taosdemoAllTest/insert_5Mrows_hasTB.json rename tests/pytest/tools/taosdemoAllTest/{insert_5Mrows_small_cache.json => insert_5Mrows_noTB.json} (100%) diff --git a/tests/pytest/manualTest/manual_alter_block.py b/tests/pytest/manualTest/manual_alter_block.py index a06e1bba6f..6d80006d7d 100644 --- a/tests/pytest/manualTest/manual_alter_block.py +++ b/tests/pytest/manualTest/manual_alter_block.py @@ -47,17 +47,19 @@ class TDTestCase: tdLog.info("taosd found in %s" % buildPath) binPath = buildPath+ "/build/bin/" - tdSql.execute('alter database db blocks 8') + tdSql.execute('alter database db blocks 3') tdSql.query('show databases') tdSql.checkData(0,9,3) - os.system("%staosdemo -f tools/taosdemoAllTest/insert_5Mrows_small_cache.json" % binPath) + os.system("%staosdemo -f tools/taosdemoAllTest/insert_5Mrows_noTB.json" % binPath) input("please check memory usage for taosd. After checking, press enter") tdSql.execute('alter database db blocks 8') + tdSql.query('show databases') + tdSql.checkData(0,9,8) - os.system("%staosdemo -f tools/taosdemoAllTest/insert_5Mrows_small_cache.json" % binPath) + os.system("%staosdemo -f tools/taosdemoAllTest/insert_5Mrows_hasTB.json" % binPath) input("please check memory usage for taosd. After checking, press enter") diff --git a/tests/pytest/tools/taosdemoAllTest/insert_5Mrows_hasTB.json b/tests/pytest/tools/taosdemoAllTest/insert_5Mrows_hasTB.json new file mode 100644 index 0000000000..9d22dbaabe --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/insert_5Mrows_hasTB.json @@ -0,0 +1,60 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 100, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "no", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 3, + "precision": "ms", + "keep": 3650, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb", + "child_table_exists":"yes", + "childtable_count": 500, + "childtable_prefix": "stb_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 10000, + "childtable_limit": 500, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 10, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}], + "tags": [{"type": "TINYINT", "count":2}] + }] + }] +} \ No newline at end of file diff --git a/tests/pytest/tools/taosdemoAllTest/insert_5Mrows_small_cache.json b/tests/pytest/tools/taosdemoAllTest/insert_5Mrows_noTB.json similarity index 100% rename from tests/pytest/tools/taosdemoAllTest/insert_5Mrows_small_cache.json rename to tests/pytest/tools/taosdemoAllTest/insert_5Mrows_noTB.json From 6439e9addca8be102bbf2353fc191e7dd4e650b5 Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Wed, 2 Jun 2021 11:55:16 +0800 Subject: [PATCH 03/10] [TD-4475] modifying the test case --- tests/pytest/manualTest/manual_alter_block.py | 3 +++ .../pytest/tools/taosdemoAllTest/insert_5Mrows_hasTB.json | 8 ++++---- .../pytest/tools/taosdemoAllTest/insert_5Mrows_noTB.json | 8 ++++---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/pytest/manualTest/manual_alter_block.py b/tests/pytest/manualTest/manual_alter_block.py index 6d80006d7d..0db74e160a 100644 --- a/tests/pytest/manualTest/manual_alter_block.py +++ b/tests/pytest/manualTest/manual_alter_block.py @@ -15,6 +15,7 @@ import sys from util.log import * from util.cases import * from util.sql import * +from util.dnodes import tdDnodes class TDTestCase: @@ -58,6 +59,8 @@ class TDTestCase: tdSql.execute('alter database db blocks 8') tdSql.query('show databases') tdSql.checkData(0,9,8) + tdDnodes.stop(1) + tdDnodes.start(1) os.system("%staosdemo -f tools/taosdemoAllTest/insert_5Mrows_hasTB.json" % binPath) diff --git a/tests/pytest/tools/taosdemoAllTest/insert_5Mrows_hasTB.json b/tests/pytest/tools/taosdemoAllTest/insert_5Mrows_hasTB.json index 9d22dbaabe..2731e0505c 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert_5Mrows_hasTB.json +++ b/tests/pytest/tools/taosdemoAllTest/insert_5Mrows_hasTB.json @@ -11,7 +11,7 @@ "confirm_parameter_prompt": "no", "insert_interval": 0, "interlace_rows": 100, - "num_of_records_per_req": 100, + "num_of_records_per_req": 32766, "databases": [{ "dbinfo": { "name": "db", @@ -40,7 +40,7 @@ "batch_create_tbl_num": 20, "data_source": "rand", "insert_mode": "taosc", - "insert_rows": 10000, + "insert_rows": 100000, "childtable_limit": 500, "childtable_offset":0, "interlace_rows": 0, @@ -48,8 +48,8 @@ "max_sql_len": 1024000, "disorder_ratio": 0, "disorder_range": 1000, - "timestamp_step": 10, - "start_timestamp": "2020-10-01 00:00:00.000", + "timestamp_step": 1, + "start_timestamp": "now", "sample_format": "csv", "sample_file": "./sample.csv", "tags_file": "", diff --git a/tests/pytest/tools/taosdemoAllTest/insert_5Mrows_noTB.json b/tests/pytest/tools/taosdemoAllTest/insert_5Mrows_noTB.json index d437735ebc..8583b9a897 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert_5Mrows_noTB.json +++ b/tests/pytest/tools/taosdemoAllTest/insert_5Mrows_noTB.json @@ -11,7 +11,7 @@ "confirm_parameter_prompt": "no", "insert_interval": 0, "interlace_rows": 100, - "num_of_records_per_req": 100, + "num_of_records_per_req": 32766, "databases": [{ "dbinfo": { "name": "db", @@ -40,7 +40,7 @@ "batch_create_tbl_num": 20, "data_source": "rand", "insert_mode": "taosc", - "insert_rows": 10000, + "insert_rows": 100000, "childtable_limit": 10, "childtable_offset":100, "interlace_rows": 0, @@ -48,8 +48,8 @@ "max_sql_len": 1024000, "disorder_ratio": 0, "disorder_range": 1000, - "timestamp_step": 10, - "start_timestamp": "2020-10-01 00:00:00.000", + "timestamp_step": 1, + "start_timestamp": "2019-10-01 00:00:00.000", "sample_format": "csv", "sample_file": "./sample.csv", "tags_file": "", From 29ca859f1160f20ae1514d8fba1578d043ce4a7d Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Wed, 2 Jun 2021 13:37:36 +0800 Subject: [PATCH 04/10] [TD-4475] add manual test case for hot alter block --- tests/pytest/manualTest/manual_alter_block.py | 23 ++++++++++++------- ...rows_noTB.json => manual_block1_comp.json} | 8 +++---- ...t_5Mrows_hasTB.json => manual_block2.json} | 2 +- 3 files changed, 20 insertions(+), 13 deletions(-) rename tests/pytest/tools/taosdemoAllTest/{insert_5Mrows_noTB.json => manual_block1_comp.json} (84%) rename tests/pytest/tools/taosdemoAllTest/{insert_5Mrows_hasTB.json => manual_block2.json} (98%) diff --git a/tests/pytest/manualTest/manual_alter_block.py b/tests/pytest/manualTest/manual_alter_block.py index 0db74e160a..ccd98b1421 100644 --- a/tests/pytest/manualTest/manual_alter_block.py +++ b/tests/pytest/manualTest/manual_alter_block.py @@ -17,7 +17,8 @@ from util.cases import * from util.sql import * from util.dnodes import tdDnodes - +##TODO: auto test version is currently unsupported, need to come up with +# an auto test version in the future class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) @@ -48,27 +49,33 @@ class TDTestCase: tdLog.info("taosd found in %s" % buildPath) binPath = buildPath+ "/build/bin/" + #alter cache block to 3, then check alter tdSql.execute('alter database db blocks 3') tdSql.query('show databases') tdSql.checkData(0,9,3) - os.system("%staosdemo -f tools/taosdemoAllTest/insert_5Mrows_noTB.json" % binPath) - + #run taosdemo to occupy all cache, need to manually check memory consumption + os.system("%staosdemo -f tools/taosdemoAllTest/manual_block1_comp.json" % binPath) input("please check memory usage for taosd. After checking, press enter") + #alter cache block to 8, then check alter tdSql.execute('alter database db blocks 8') tdSql.query('show databases') tdSql.checkData(0,9,8) - tdDnodes.stop(1) - tdDnodes.start(1) - - os.system("%staosdemo -f tools/taosdemoAllTest/insert_5Mrows_hasTB.json" % binPath) + #run taosdemo to occupy all cache, need to manually check memory consumption + os.system("%staosdemo -f tools/taosdemoAllTest/manual_block2.json" % binPath) input("please check memory usage for taosd. After checking, press enter") + ##expected result the peak memory consumption should increase by around 80MB = 5 blocks of cache + + ##test results + #2021/06/02 before:2621700K after: 2703640K memory usage increased by 80MB = 5 block + # confirm with the change in block. Baosheng Chang + def stop(self): tdSql.close() - tdLog.success("%s successfully executed" % __file__) + tdLog.debug("%s alter block manual check finish" % __file__) tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdemoAllTest/insert_5Mrows_noTB.json b/tests/pytest/tools/taosdemoAllTest/manual_block1_comp.json similarity index 84% rename from tests/pytest/tools/taosdemoAllTest/insert_5Mrows_noTB.json rename to tests/pytest/tools/taosdemoAllTest/manual_block1_comp.json index 8583b9a897..3ced8272b2 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert_5Mrows_noTB.json +++ b/tests/pytest/tools/taosdemoAllTest/manual_block1_comp.json @@ -22,7 +22,7 @@ "blocks": 3, "precision": "ms", "keep": 3650, - "minRows": 100, + "minRows": 1000, "maxRows": 4096, "comp":2, "walLevel":1, @@ -40,7 +40,7 @@ "batch_create_tbl_num": 20, "data_source": "rand", "insert_mode": "taosc", - "insert_rows": 100000, + "insert_rows": 10000, "childtable_limit": 10, "childtable_offset":100, "interlace_rows": 0, @@ -53,8 +53,8 @@ "sample_format": "csv", "sample_file": "./sample.csv", "tags_file": "", - "columns": [{"type": "INT"}], - "tags": [{"type": "TINYINT", "count":2}] + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] }] }] } \ No newline at end of file diff --git a/tests/pytest/tools/taosdemoAllTest/insert_5Mrows_hasTB.json b/tests/pytest/tools/taosdemoAllTest/manual_block2.json similarity index 98% rename from tests/pytest/tools/taosdemoAllTest/insert_5Mrows_hasTB.json rename to tests/pytest/tools/taosdemoAllTest/manual_block2.json index 2731e0505c..434159159b 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert_5Mrows_hasTB.json +++ b/tests/pytest/tools/taosdemoAllTest/manual_block2.json @@ -19,7 +19,7 @@ "replica": 1, "days": 10, "cache": 16, - "blocks": 3, + "blocks": 8, "precision": "ms", "keep": 3650, "minRows": 100, From 701ca6419de59d27af7ed5dc22a2c2d99c5c8b91 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Wed, 2 Jun 2021 14:22:59 +0800 Subject: [PATCH 05/10] fix bug --- src/client/src/tscUtil.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 8042f032c8..b5dfc956dd 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1119,6 +1119,8 @@ void handleDownstreamOperator(SSqlObj** pSqlObjList, int32_t numOfUpstream, SQue SOperatorInfo* pSourceOperator = createDummyInputOperator(pSqlObjList[0], pSchema, numOfCol1, pFilterInfo, numOfFilterCols); + pOutput->precision = pSqlObjList[0]->res.precision; + SSchema* schema = NULL; if (px->numOfTables > 1) { SOperatorInfo** p = calloc(px->numOfTables, POINTER_BYTES); @@ -4477,4 +4479,4 @@ SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg) { } return info; -} \ No newline at end of file +} From 3b867c876de1da897a04e908b5fef33c20e12a48 Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Wed, 2 Jun 2021 15:07:17 +0800 Subject: [PATCH 06/10] [TD-4476] add manual test case --- .../taosdemoAllTest/testAutoCreateTemp.py | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 tests/pytest/tools/taosdemoAllTest/testAutoCreateTemp.py diff --git a/tests/pytest/tools/taosdemoAllTest/testAutoCreateTemp.py b/tests/pytest/tools/taosdemoAllTest/testAutoCreateTemp.py new file mode 100644 index 0000000000..2d98b1c463 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/testAutoCreateTemp.py @@ -0,0 +1,135 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + # insert: auto_create + os.system("%staosdemo -f tools/taosdemoAllTest/insert-drop-exist-auto-YYY.json -y " % binPath) # drop = yes, exist = yes, auto_create = yes + tdSql.execute('use db') + tdSql.query('show tables') + tdSql.checkRows(20) + + os.system("%staosdemo -f tools/taosdemoAllTest/insert-drop-exist-auto-YYN.json -y " % binPath) # drop = yes, exist = yes, auto_create = no + tdSql.execute('use db') + tdSql.query('show tables') + tdSql.checkRows(20) + + os.system("%staosdemo -f tools/taosdemoAllTest/insert-drop-exist-auto-YNY.json -y " % binPath) # drop = yes, exist = no, auto_create = yes + tdSql.execute('use db') + tdSql.query('show tables') + tdSql.checkRows(20) + + os.system("%staosdemo -f tools/taosdemoAllTest/insert-drop-exist-auto-YNN.json -y " % binPath) # drop = yes, exist = no, auto_create = no + tdSql.execute('use db') + tdSql.query('show tables') + tdSql.checkRows(20) + + tdSql.execute('drop database db') + tdSql.execute('create database db') + tdSql.execute('use db') + os.system("%staosdemo -f tools/taosdemoAllTest/insert-drop-exist-auto-NYY.json -y " % binPath) # drop = no, exist = yes, auto_create = yes + tdSql.query('show tables') + tdSql.checkRows(0) + + tdSql.execute('drop database db') + tdSql.execute('create database db') + tdSql.execute('use db') + os.system("%staosdemo -f tools/taosdemoAllTest/insert-drop-exist-auto-NYN.json -y " % binPath) # drop = no, exist = yes, auto_create = no + tdSql.execute('use db') + tdSql.query('show tables') + tdSql.checkRows(0) + + tdSql.execute('drop database db') + tdSql.execute('create database db') + tdSql.execute('use db') + os.system("%staosdemo -f tools/taosdemoAllTest/insert-drop-exist-auto-NNY.json -y " % binPath) # drop = no, exist = no, auto_create = yes + tdSql.execute('use db') + tdSql.query('show tables') + tdSql.checkRows(20) + + tdSql.execute('drop database db') + tdSql.execute('create database db') + tdSql.execute('use db') + os.system("%staosdemo -f tools/taosdemoAllTest/insert-drop-exist-auto-NNN.json -y " % binPath) # drop = no, exist = no, auto_create = no + tdSql.execute('use db') + tdSql.query('show tables') + tdSql.checkRows(20) + + #the following four test cases are for the exception cases for param auto_create_table + + tdSql.execute('drop database db') + tdSql.execute('create database db') + tdSql.execute('use db') + os.system("%staosdemo -f tools/taosdemoAllTest/insert-drop-exist-auto-NN123.json -y " % binPath) # drop = no, exist = no, auto_create = 123 + tdSql.execute('use db') + tdSql.query('show tables') + tdSql.checkRows(20) + + tdSql.execute('drop database db') + tdSql.execute('create database db') + tdSql.execute('use db') + os.system("%staosdemo -f tools/taosdemoAllTest/insert-drop-exist-auto-NY123.json -y " % binPath) # drop = no, exist = yes, auto_create = 123 + tdSql.execute('use db') + tdSql.query('show tables') + tdSql.checkRows(0) + + os.system("%staosdemo -f tools/taosdemoAllTest/insert-drop-exist-auto-YN123.json -y " % binPath) # drop = yes, exist = no, auto_create = 123 + tdSql.execute('use db') + tdSql.query('show tables') + tdSql.checkRows(20) + + os.system("%staosdemo -f tools/taosdemoAllTest/insert-drop-exist-auto-YY123.json -y " % binPath) # drop = yes, exist = yes, auto_create = 123 + tdSql.execute('use db') + tdSql.query('show tables') + tdSql.checkRows(20) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 97e9070a21f83d071d320c55f3fddaef0fe9df22 Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Wed, 2 Jun 2021 16:34:41 +0800 Subject: [PATCH 07/10] [TD-4476] test case finished --- tests/pytest/manualTest/manual_alter_comp.py | 106 ++++++++++++++++++ .../taosdemoAllTest/manual_block1_comp.json | 8 +- 2 files changed, 110 insertions(+), 4 deletions(-) create mode 100644 tests/pytest/manualTest/manual_alter_comp.py diff --git a/tests/pytest/manualTest/manual_alter_comp.py b/tests/pytest/manualTest/manual_alter_comp.py new file mode 100644 index 0000000000..1b6d1d1f81 --- /dev/null +++ b/tests/pytest/manualTest/manual_alter_comp.py @@ -0,0 +1,106 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes + +##TODO: auto test version is currently unsupported, need to come up with +# an auto test version in the future +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getRootPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + print(selfPath) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + print(projPath) + else: + projPath = selfPath[:selfPath.find("tests")] + print("test" + projPath) + + for root, dirs, files in os.walk(projPath): + if ('data' in dirs and 'sim' in root): + rootPath = root + + return rootPath + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + + + def run(self): + dnodePath = self.getRootPath() + os.system(f'rm -rf {dnodePath}/data/* {dnodePath}/log/*') + tdSql.prepare() + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + #comp is at 14 + tdSql.query('show databases') + tdSql.checkData(0,14,2) + os.system("%staosdemo -f tools/taosdemoAllTest/manual_block1_comp.json" % binPath) + tdDnodes.stop(1) + print("default location is at /home/bryan/Documents/Github/TDengine/sim/dnode1/data/vnode") + input("please check disk usage for taosd. After checking, press enter") + + os.system(f'sudo rm -rf {dnodePath}/data/* {dnodePath}/log/*') + #print(f'rm -rf {dnodePath}/data/* {dnodePath}/log/*') #for showing the command ran + input("please check if the pervious data is being deleted. Then, press enter") + tdDnodes.start(1) + tdSql.prepare() + tdSql.query('show databases') + tdSql.checkData(0,14,2) + tdSql.execute('alter database db comp 0') + tdSql.query('show databases') + tdSql.checkData(0,14,0) + tdDnodes.stop(1) + tdDnodes.start(1) + os.system("%staosdemo -f tools/taosdemoAllTest/manual_block1_comp.json" % binPath) + print("default location is at /home/bryan/Documents/Github/TDengine/sim/dnode1/data") + input("please check disk usage for taosd. After checking, press enter") + + ##test result + # 2021/06/02 comp=2: file size = 6.4M comp=0 file size=399M. Test past + # each row entered is identical Tester - Baosheng Chang + + def stop(self): + tdSql.close() + tdLog.debug("%s alter block manual check finish" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdemoAllTest/manual_block1_comp.json b/tests/pytest/tools/taosdemoAllTest/manual_block1_comp.json index 3ced8272b2..a6ac674dd7 100644 --- a/tests/pytest/tools/taosdemoAllTest/manual_block1_comp.json +++ b/tests/pytest/tools/taosdemoAllTest/manual_block1_comp.json @@ -38,7 +38,7 @@ "childtable_prefix": "stb_", "auto_create_table": "no", "batch_create_tbl_num": 20, - "data_source": "rand", + "data_source": "sample", "insert_mode": "taosc", "insert_rows": 10000, "childtable_limit": 10, @@ -51,9 +51,9 @@ "timestamp_step": 1, "start_timestamp": "2019-10-01 00:00:00.000", "sample_format": "csv", - "sample_file": "./sample.csv", - "tags_file": "", - "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "sample_file": "./tools/taosdemoAllTest/sample.csv", + "tags_file": "", + "columns": [{"type": "INT", "count":3}, {"type": "DOUBLE", "count":3}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}, {"type": "BOOL"}], "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] }] }] From 3a12c502c7763bd231f2ff1f23dade5b2e45f5ef Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Wed, 2 Jun 2021 16:38:57 +0800 Subject: [PATCH 08/10] [TD-4475] remove temp file --- .../taosdemoAllTest/testAutoCreateTemp.py | 135 ------------------ 1 file changed, 135 deletions(-) delete mode 100644 tests/pytest/tools/taosdemoAllTest/testAutoCreateTemp.py diff --git a/tests/pytest/tools/taosdemoAllTest/testAutoCreateTemp.py b/tests/pytest/tools/taosdemoAllTest/testAutoCreateTemp.py deleted file mode 100644 index 2d98b1c463..0000000000 --- a/tests/pytest/tools/taosdemoAllTest/testAutoCreateTemp.py +++ /dev/null @@ -1,135 +0,0 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- - -import sys -from util.log import * -from util.cases import * -from util.sql import * - - -class TDTestCase: - def init(self, conn, logSql): - tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), logSql) - - def getBuildPath(self): - selfPath = os.path.dirname(os.path.realpath(__file__)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - - for root, dirs, files in os.walk(projPath): - if ("taosd" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] - break - return buildPath - - def run(self): - buildPath = self.getBuildPath() - if (buildPath == ""): - tdLog.exit("taosd not found!") - else: - tdLog.info("taosd found in %s" % buildPath) - binPath = buildPath+ "/build/bin/" - - # insert: auto_create - os.system("%staosdemo -f tools/taosdemoAllTest/insert-drop-exist-auto-YYY.json -y " % binPath) # drop = yes, exist = yes, auto_create = yes - tdSql.execute('use db') - tdSql.query('show tables') - tdSql.checkRows(20) - - os.system("%staosdemo -f tools/taosdemoAllTest/insert-drop-exist-auto-YYN.json -y " % binPath) # drop = yes, exist = yes, auto_create = no - tdSql.execute('use db') - tdSql.query('show tables') - tdSql.checkRows(20) - - os.system("%staosdemo -f tools/taosdemoAllTest/insert-drop-exist-auto-YNY.json -y " % binPath) # drop = yes, exist = no, auto_create = yes - tdSql.execute('use db') - tdSql.query('show tables') - tdSql.checkRows(20) - - os.system("%staosdemo -f tools/taosdemoAllTest/insert-drop-exist-auto-YNN.json -y " % binPath) # drop = yes, exist = no, auto_create = no - tdSql.execute('use db') - tdSql.query('show tables') - tdSql.checkRows(20) - - tdSql.execute('drop database db') - tdSql.execute('create database db') - tdSql.execute('use db') - os.system("%staosdemo -f tools/taosdemoAllTest/insert-drop-exist-auto-NYY.json -y " % binPath) # drop = no, exist = yes, auto_create = yes - tdSql.query('show tables') - tdSql.checkRows(0) - - tdSql.execute('drop database db') - tdSql.execute('create database db') - tdSql.execute('use db') - os.system("%staosdemo -f tools/taosdemoAllTest/insert-drop-exist-auto-NYN.json -y " % binPath) # drop = no, exist = yes, auto_create = no - tdSql.execute('use db') - tdSql.query('show tables') - tdSql.checkRows(0) - - tdSql.execute('drop database db') - tdSql.execute('create database db') - tdSql.execute('use db') - os.system("%staosdemo -f tools/taosdemoAllTest/insert-drop-exist-auto-NNY.json -y " % binPath) # drop = no, exist = no, auto_create = yes - tdSql.execute('use db') - tdSql.query('show tables') - tdSql.checkRows(20) - - tdSql.execute('drop database db') - tdSql.execute('create database db') - tdSql.execute('use db') - os.system("%staosdemo -f tools/taosdemoAllTest/insert-drop-exist-auto-NNN.json -y " % binPath) # drop = no, exist = no, auto_create = no - tdSql.execute('use db') - tdSql.query('show tables') - tdSql.checkRows(20) - - #the following four test cases are for the exception cases for param auto_create_table - - tdSql.execute('drop database db') - tdSql.execute('create database db') - tdSql.execute('use db') - os.system("%staosdemo -f tools/taosdemoAllTest/insert-drop-exist-auto-NN123.json -y " % binPath) # drop = no, exist = no, auto_create = 123 - tdSql.execute('use db') - tdSql.query('show tables') - tdSql.checkRows(20) - - tdSql.execute('drop database db') - tdSql.execute('create database db') - tdSql.execute('use db') - os.system("%staosdemo -f tools/taosdemoAllTest/insert-drop-exist-auto-NY123.json -y " % binPath) # drop = no, exist = yes, auto_create = 123 - tdSql.execute('use db') - tdSql.query('show tables') - tdSql.checkRows(0) - - os.system("%staosdemo -f tools/taosdemoAllTest/insert-drop-exist-auto-YN123.json -y " % binPath) # drop = yes, exist = no, auto_create = 123 - tdSql.execute('use db') - tdSql.query('show tables') - tdSql.checkRows(20) - - os.system("%staosdemo -f tools/taosdemoAllTest/insert-drop-exist-auto-YY123.json -y " % binPath) # drop = yes, exist = yes, auto_create = 123 - tdSql.execute('use db') - tdSql.query('show tables') - tdSql.checkRows(20) - - def stop(self): - tdSql.close() - tdLog.success("%s successfully executed" % __file__) - - -tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) From e39c9eed5de45a0bd01c2402596af5ce5882289e Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Wed, 2 Jun 2021 17:03:55 +0800 Subject: [PATCH 09/10] [TD-4476] modified test case --- tests/pytest/manualTest/manual_alter_comp.py | 32 ++++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/tests/pytest/manualTest/manual_alter_comp.py b/tests/pytest/manualTest/manual_alter_comp.py index 1b6d1d1f81..6c3e0fc296 100644 --- a/tests/pytest/manualTest/manual_alter_comp.py +++ b/tests/pytest/manualTest/manual_alter_comp.py @@ -70,31 +70,51 @@ class TDTestCase: binPath = buildPath+ "/build/bin/" #comp is at 14 + #check disk usage when comp=2 tdSql.query('show databases') + tdSql.execute('alter database db blocks 3') # minimize the data in cache tdSql.checkData(0,14,2) os.system("%staosdemo -f tools/taosdemoAllTest/manual_block1_comp.json" % binPath) - tdDnodes.stop(1) print("default location is at /home/bryan/Documents/Github/TDengine/sim/dnode1/data/vnode") + print('comp = 2') input("please check disk usage for taosd. After checking, press enter") - + + #removing all data file os.system(f'sudo rm -rf {dnodePath}/data/* {dnodePath}/log/*') #print(f'rm -rf {dnodePath}/data/* {dnodePath}/log/*') #for showing the command ran input("please check if the pervious data is being deleted. Then, press enter") - tdDnodes.start(1) + + #check disk usage when comp=0 tdSql.prepare() tdSql.query('show databases') tdSql.checkData(0,14,2) tdSql.execute('alter database db comp 0') tdSql.query('show databases') tdSql.checkData(0,14,0) - tdDnodes.stop(1) - tdDnodes.start(1) os.system("%staosdemo -f tools/taosdemoAllTest/manual_block1_comp.json" % binPath) print("default location is at /home/bryan/Documents/Github/TDengine/sim/dnode1/data") + print('comp = 0') + input("please check disk usage for taosd. After checking, press enter") + + #removing all data file + os.system(f'sudo rm -rf {dnodePath}/data/* {dnodePath}/log/*') + #print(f'rm -rf {dnodePath}/data/* {dnodePath}/log/*') #for showing the command ran + input("please check if the pervious data is being deleted. Then, press enter") + + #check disk usage when comp=1 + tdSql.prepare() + tdSql.query('show databases') + tdSql.checkData(0,14,2) + tdSql.execute('alter database db comp 1') + tdSql.query('show databases') + tdSql.checkData(0,14,1) + os.system("%staosdemo -f tools/taosdemoAllTest/manual_block1_comp.json" % binPath) + print("default location is at /home/bryan/Documents/Github/TDengine/sim/dnode1/data") + print('comp = 1') input("please check disk usage for taosd. After checking, press enter") ##test result - # 2021/06/02 comp=2: file size = 6.4M comp=0 file size=399M. Test past + # 2021/06/02 comp=2:13M comp=1:57M comp=0:399M. Test past # each row entered is identical Tester - Baosheng Chang def stop(self): From dff270c96e5f99863077b5a4a7f6f36f2448dc0f Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 2 Jun 2021 17:52:05 +0800 Subject: [PATCH 10/10] [TD-4435]: add test case --- .../jdbc/TSDBPreparedStatementTest.java | 127 +++++++++++++++++- 1 file changed, 123 insertions(+), 4 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java index 52858e7f88..277ca447f5 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java @@ -243,15 +243,15 @@ public class TSDBPreparedStatementTest { s.setNString(1, s2, 4); random = 10 + r.nextInt(5); - ArrayList s5 = new ArrayList(); + ArrayList s3 = new ArrayList(); for(int i = 0; i < numOfRows; i++) { if(i % random == 0) { - s5.add(null); + s3.add(null); }else{ - s5.add("test" + i % 10); + s3.add("test" + i % 10); } } - s.setString(2, s5, 10); + s.setString(2, s3, 10); s.columnDataAddBatch(); s.columnDataExecuteBatch(); @@ -268,7 +268,126 @@ public class TSDBPreparedStatementTest { } } + @Test + public void bindDataWithSingleTagTest() throws SQLException { + Statement stmt = conn.createStatement(); + String types[] = new String[] {"tinyint", "smallint", "int", "bigint", "bool", "float", "double", "binary(10)", "nchar(10)"}; + + for (String type : types) { + stmt.execute("drop table if exists weather_test"); + stmt.execute("create table weather_test(ts timestamp, f1 nchar(10), f2 binary(10)) tags (t " + type + ")"); + + int numOfRows = 1; + + TSDBPreparedStatement s = (TSDBPreparedStatement) conn.prepareStatement("insert into ? using weather_test tags(?) values(?, ?, ?)"); + Random r = new Random(); + s.setTableName("w1"); + + switch(type) { + case "tinyint": + case "smallint": + case "int": + case "bigint": + s.setTagInt(0, 1); + break; + case "float": + s.setTagFloat(0, 1.23f); + break; + case "double": + s.setTagDouble(0, 3.14159265); + break; + case "bool": + s.setTagBoolean(0, true); + break; + case "binary(10)": + s.setTagString(0, "test"); + break; + case "nchar(10)": + s.setTagNString(0, "test"); + break; + default: + break; + } + + + ArrayList ts = new ArrayList(); + for(int i = 0; i < numOfRows; i++) { + ts.add(System.currentTimeMillis() + i); + } + s.setTimestamp(0, ts); + + int random = 10 + r.nextInt(5); + ArrayList s2 = new ArrayList(); + for(int i = 0; i < numOfRows; i++) { + s2.add("分支" + i % 4); + } + s.setNString(1, s2, 10); + + random = 10 + r.nextInt(5); + ArrayList s3 = new ArrayList(); + for(int i = 0; i < numOfRows; i++) { + s3.add("test" + i % 4); + } + s.setString(2, s3, 10); + + s.columnDataAddBatch(); + s.columnDataExecuteBatch(); + s.columnDataCloseBatch(); + + String sql = "select * from weather_test"; + PreparedStatement statement = conn.prepareStatement(sql); + ResultSet rs = statement.executeQuery(); + int rows = 0; + while(rs.next()) { + rows++; + } + Assert.assertEquals(numOfRows, rows); + } + } + + + @Test + public void bindDataWithMultipleTagsTest() throws SQLException { + Statement stmt = conn.createStatement(); + + stmt.execute("drop table if exists weather_test"); + stmt.execute("create table weather_test(ts timestamp, f1 nchar(10), f2 binary(10)) tags (t1 int, t2 binary(10))"); + + int numOfRows = 1; + + TSDBPreparedStatement s = (TSDBPreparedStatement) conn.prepareStatement("insert into ? using weather_test tags(?,?) (ts, f2) values(?, ?)"); + s.setTableName("w2"); + s.setTagInt(0, 1); + s.setTagString(1, "test"); + + + ArrayList ts = new ArrayList(); + for(int i = 0; i < numOfRows; i++) { + ts.add(System.currentTimeMillis() + i); + } + s.setTimestamp(0, ts); + + ArrayList s2 = new ArrayList(); + for(int i = 0; i < numOfRows; i++) { + s2.add("test" + i % 4); + } + s.setString(1, s2, 10); + + s.columnDataAddBatch(); + s.columnDataExecuteBatch(); + s.columnDataCloseBatch(); + + String sql = "select * from weather_test"; + PreparedStatement statement = conn.prepareStatement(sql); + ResultSet rs = statement.executeQuery(); + int rows = 0; + while(rs.next()) { + rows++; + } + Assert.assertEquals(numOfRows, rows); + + } @Test public void setBoolean() throws SQLException {