From c21292c7a69efc175afe864f8fc5f3373f35a9d4 Mon Sep 17 00:00:00 2001 From: tomchon Date: Wed, 21 Apr 2021 20:04:03 +0800 Subject: [PATCH 01/27] [TD-3414]: add insert function with json files of taodemo-testcase -repeat --- tests/pytest/client/thousandsofClient.py | 55 +++++ tests/pytest/cluster/TD-3693/how-to-use | 9 + tests/pytest/cluster/TD-3693/insert1Data.json | 88 +++++++ tests/pytest/cluster/TD-3693/insert2Data.json | 88 +++++++ tests/pytest/cluster/TD-3693/multClient.py | 74 ++++++ tests/pytest/cluster/TD-3693/multQuery.py | 72 ++++++ tests/pytest/cluster/TD-3693/queryCount.json | 15 ++ tests/pytest/fulltest.sh | 4 + .../functions/function_count_last_stab.py | 70 ++++++ .../TD-3453/query-interrupt.json | 62 +++++ .../TD-3453/query-interrupt.py | 88 +++++++ .../taosdemoAllTest/TD-3453/queryall.json | 14 ++ .../tools/taosdemoAllTest/convertResFile.py | 35 +++ .../taosdemoAllTest/insert-1s1tnt1r.json | 88 +++++++ .../taosdemoAllTest/insert-1s1tntmr.json | 88 +++++++ .../taosdemoAllTest/insert-disorder.json | 88 +++++++ .../insert-illegal-columns-count-0.json | 88 +++++++ .../insert-illegal-columns-lmax.json | 88 +++++++ .../insert-illegal-columns.json | 88 +++++++ .../insert-illegal-tags-count129.json | 88 +++++++ .../taosdemoAllTest/insert-interlace-row.json | 62 +++++ .../insert-interval-speed.json | 88 +++++++ .../tools/taosdemoAllTest/insert-newdb.json | 166 +++++++++++++ .../taosdemoAllTest/insert-newtable.json | 166 +++++++++++++ .../taosdemoAllTest/insert-nodbnodrop.json | 62 +++++ .../tools/taosdemoAllTest/insert-offset.json | 166 +++++++++++++ .../tools/taosdemoAllTest/insert-renewdb.json | 166 +++++++++++++ .../tools/taosdemoAllTest/insert-sample.json | 88 +++++++ .../taosdemoAllTest/insert-timestep.json | 88 +++++++ .../taosdemoAllTest/moredemo-insert-offset.py | 72 ++++++ .../moredemo-offset-limit1.json | 62 +++++ .../moredemo-offset-limit5.json | 62 +++++ .../moredemo-offset-limit94.json | 62 +++++ .../moredemo-offset-newdb.json | 61 +++++ tests/pytest/tools/taosdemoAllTest/sample.csv | 3 + .../tools/taosdemoAllTest/speciQuery.json | 36 +++ .../taosdemoAllTest/speciQueryInsertdata.json | 86 +++++++ tests/pytest/tools/taosdemoAllTest/tags.csv | 2 + .../taosdemoTestInsertWithJson.py | 229 ++++++++++++++++++ .../taosdemoTestQueryWithJson.py | 91 +++++++ 40 files changed, 3108 insertions(+) create mode 100644 tests/pytest/client/thousandsofClient.py create mode 100644 tests/pytest/cluster/TD-3693/how-to-use create mode 100644 tests/pytest/cluster/TD-3693/insert1Data.json create mode 100644 tests/pytest/cluster/TD-3693/insert2Data.json create mode 100644 tests/pytest/cluster/TD-3693/multClient.py create mode 100644 tests/pytest/cluster/TD-3693/multQuery.py create mode 100644 tests/pytest/cluster/TD-3693/queryCount.json create mode 100644 tests/pytest/functions/function_count_last_stab.py create mode 100644 tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.json create mode 100644 tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.py create mode 100644 tests/pytest/tools/taosdemoAllTest/TD-3453/queryall.json create mode 100644 tests/pytest/tools/taosdemoAllTest/convertResFile.py create mode 100644 tests/pytest/tools/taosdemoAllTest/insert-1s1tnt1r.json create mode 100644 tests/pytest/tools/taosdemoAllTest/insert-1s1tntmr.json create mode 100644 tests/pytest/tools/taosdemoAllTest/insert-disorder.json create mode 100644 tests/pytest/tools/taosdemoAllTest/insert-illegal-columns-count-0.json create mode 100644 tests/pytest/tools/taosdemoAllTest/insert-illegal-columns-lmax.json create mode 100644 tests/pytest/tools/taosdemoAllTest/insert-illegal-columns.json create mode 100644 tests/pytest/tools/taosdemoAllTest/insert-illegal-tags-count129.json create mode 100644 tests/pytest/tools/taosdemoAllTest/insert-interlace-row.json create mode 100644 tests/pytest/tools/taosdemoAllTest/insert-interval-speed.json create mode 100644 tests/pytest/tools/taosdemoAllTest/insert-newdb.json create mode 100644 tests/pytest/tools/taosdemoAllTest/insert-newtable.json create mode 100644 tests/pytest/tools/taosdemoAllTest/insert-nodbnodrop.json create mode 100644 tests/pytest/tools/taosdemoAllTest/insert-offset.json create mode 100644 tests/pytest/tools/taosdemoAllTest/insert-renewdb.json create mode 100644 tests/pytest/tools/taosdemoAllTest/insert-sample.json create mode 100644 tests/pytest/tools/taosdemoAllTest/insert-timestep.json create mode 100644 tests/pytest/tools/taosdemoAllTest/moredemo-insert-offset.py create mode 100644 tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit1.json create mode 100644 tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit5.json create mode 100644 tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit94.json create mode 100644 tests/pytest/tools/taosdemoAllTest/moredemo-offset-newdb.json create mode 100644 tests/pytest/tools/taosdemoAllTest/sample.csv create mode 100644 tests/pytest/tools/taosdemoAllTest/speciQuery.json create mode 100644 tests/pytest/tools/taosdemoAllTest/speciQueryInsertdata.json create mode 100644 tests/pytest/tools/taosdemoAllTest/tags.csv create mode 100644 tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJson.py create mode 100644 tests/pytest/tools/taosdemoAllTest/taosdemoTestQueryWithJson.py diff --git a/tests/pytest/client/thousandsofClient.py b/tests/pytest/client/thousandsofClient.py new file mode 100644 index 0000000000..36c816aa5b --- /dev/null +++ b/tests/pytest/client/thousandsofClient.py @@ -0,0 +1,55 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import os +import sys +sys.path.insert(0, os.getcwd()) +from util.log import * +from util.sql import * +from util.dnodes import * +import taos +import threading + + +class TwoClients: + def initConnection(self): + self.host = "127.0.0.1" + self.user = "root" + self.password = "taosdata" + self.config = "/home/chr/taosdata/TDengine/sim/dnode1/cfg " + + def newCloseCon(times): + newConList = [] + for times in range(0,times) : + newConList.append(taos.connect(self.host, self.user, self.password, self.config)) + for times in range(0,times) : + newConList[times].close() + + def run(self): + tdDnodes.init("") + tdDnodes.setTestCluster(False) + tdDnodes.setValgrind(False) + + tdDnodes.stopAll() + tdDnodes.deploy(1) + tdDnodes.start(1) + + # multiple new and cloes connection + for m in range(1,101) : + t= threading.Thread(target=newCloseCon,args=(10,)) + t.start() + + +clients = TwoClients() +clients.initConnection() +clients.run() \ No newline at end of file diff --git a/tests/pytest/cluster/TD-3693/how-to-use b/tests/pytest/cluster/TD-3693/how-to-use new file mode 100644 index 0000000000..05a16a8534 --- /dev/null +++ b/tests/pytest/cluster/TD-3693/how-to-use @@ -0,0 +1,9 @@ +execute: +cd TDengine/tests/pytest && python3 ./test.py -f cluster/TD-3693/multClient.py && python3 cluster/TD-3693/multQuery.py + +1. 使用测试的集群，三个节点fc1、fct2、fct4。 +2. 用taosdemo建两个库db1和db2，副本数目为1，插入一定数据。 +3. db1在mnode的master上（fct2），db2在mnode的slave上（fct4）。 +4. 珲哥修改taosdemo，变成多线程查询，修改后的软件我命名成taosdemoMul，然后做持续多线程查询db2上的数据，建立多个连接 +5. 4中查询过程放到后台，同时再次在db2执行建表、插入，查询操作。循环执行查询10次，每次间隔91s。 +6. 然后查询taosd的log日志，看是否还存在上述问题“send auth msg to mnodes”。 \ No newline at end of file diff --git a/tests/pytest/cluster/TD-3693/insert1Data.json b/tests/pytest/cluster/TD-3693/insert1Data.json new file mode 100644 index 0000000000..3ac289a63a --- /dev/null +++ b/tests/pytest/cluster/TD-3693/insert1Data.json @@ -0,0 +1,88 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "192.168.1.104", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 10, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db1", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 3650, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 10, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 10000, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 20, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 20000, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/cluster/TD-3693/insert2Data.json b/tests/pytest/cluster/TD-3693/insert2Data.json new file mode 100644 index 0000000000..25717df4c7 --- /dev/null +++ b/tests/pytest/cluster/TD-3693/insert2Data.json @@ -0,0 +1,88 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "192.168.1.104", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 10, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db2", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 3650, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 10, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 10000, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 20, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 20000, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/cluster/TD-3693/multClient.py b/tests/pytest/cluster/TD-3693/multClient.py new file mode 100644 index 0000000000..24c27d9de9 --- /dev/null +++ b/tests/pytest/cluster/TD-3693/multClient.py @@ -0,0 +1,74 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + self.rowNum = 100000 + self.ts = 1537146000000 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + # insert data to cluster'db + os.system("%staosdemo -f cluster/TD-3693/insert1Data.json -y " % binPath) + # multiple new and cloes connection with query data + os.system("%staosdemo -f cluster/TD-3693/insert2Data.json -y " % binPath) + os.system("nohup %staosdemoMul -f cluster/TD-3693/queryCount.json -y & " % binPath) + + + + # delete useless files + os.system("rm -rf ./insert_res.txt") + os.system("rm -rf ./querySystemInfo*") + os.system("rm -rf cluster/TD-3693/multClient.py.sql") + os.system("rm -rf ./querySystemInfo*") + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/cluster/TD-3693/multQuery.py b/tests/pytest/cluster/TD-3693/multQuery.py new file mode 100644 index 0000000000..70061a27f2 --- /dev/null +++ b/tests/pytest/cluster/TD-3693/multQuery.py @@ -0,0 +1,72 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import os +import sys +sys.path.insert(0, os.getcwd()) +from util.log import * +from util.sql import * +from util.dnodes import * +import taos +import threading + + +class TwoClients: + def initConnection(self): + self.host = "fct4" + self.user = "root" + self.password = "taosdata" + self.config = "/etc/taos/" + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + # query data from cluster'db + conn = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config) + cur = conn.cursor() + tdSql.init(cur, True) + tdSql.execute("use db2") + cur.execute("select count (tbname) from stb0") + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 10) + tdSql.query("select count (tbname) from stb1") + tdSql.checkData(0, 0, 20) + tdSql.query("select count(*) from stb00_0") + tdSql.checkData(0, 0, 10000) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 100000) + tdSql.query("select count(*) from stb01_0") + tdSql.checkData(0, 0, 20000) + tdSql.query("select count(*) from stb1") + tdSql.checkData(0, 0, 400000) + tdSql.execute("drop table if exists squerytest") + tdSql.execute("drop table if exists querytest") + tdSql.execute('''create stable squerytest(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') + tdSql.execute("create table querytest using squerytest tags('beijing')") + tdSql.execute("insert into querytest(ts) values(%d)" % (self.ts - 1)) + for i in range(self.rowNum): + tdSql.execute("insert into querytest values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" % (self.ts + i, i + 1, 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + for j in range(10): + tdSql.execute("use db2") + tdSql.query("select count(*),last(*) from querytest group by col1") + tdSql.checkRows(10) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 2, 2) + tdSql.checkData(1, 3, 1) + sleep(88) + tdSql.execute("drop table if exists squerytest") + tdSql.execute("drop table if exists querytest") + +clients = TwoClients() +clients.initConnection() +clients.run() \ No newline at end of file diff --git a/tests/pytest/cluster/TD-3693/queryCount.json b/tests/pytest/cluster/TD-3693/queryCount.json new file mode 100644 index 0000000000..089ae42aab --- /dev/null +++ b/tests/pytest/cluster/TD-3693/queryCount.json @@ -0,0 +1,15 @@ +{ + "filetype":"query", + "cfgdir": "/etc/taos", + "host": "192.168.1.104", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "db2", + "query_times": 1000000, + "specified_table_query": + {"query_interval":1, "concurrent":100, + "sqls": [{"sql": "select count(*) from db.stb0", "result": ""}] + } +} \ No newline at end of file diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index c26dc8ae2f..d306f665ac 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -151,6 +151,9 @@ python3 test.py -f tools/taosdemoTestTblAlt.py python3 test.py -f tools/taosdemoTestSampleData.py python3 test.py -f tools/taosdemoTestInterlace.py python3 test.py -f tools/taosdemoTestQuery.py +python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertWithJson.py +python3 test.py -f tools/taosdemoAllTest/taosdemoTestQueryWithJson.py + # update python3 ./test.py -f update/allow_update.py @@ -278,6 +281,7 @@ python3 ./test.py -f functions/all_null_value.py python3 ./test.py -f functions/function_avg.py -r 1 python3 ./test.py -f functions/function_bottom.py -r 1 python3 ./test.py -f functions/function_count.py -r 1 +python3 ./test.py -f functions/function_count_last_stab.py python3 ./test.py -f functions/function_diff.py -r 1 python3 ./test.py -f functions/function_first.py -r 1 python3 ./test.py -f functions/function_last.py -r 1 diff --git a/tests/pytest/functions/function_count_last_stab.py b/tests/pytest/functions/function_count_last_stab.py new file mode 100644 index 0000000000..1d777c6bd3 --- /dev/null +++ b/tests/pytest/functions/function_count_last_stab.py @@ -0,0 +1,70 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + tdSql.execute('''create stable stest(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') + tdSql.execute("create table test1 using stest tags('beijing')") + tdSql.execute("insert into test1(ts) values(%d)" % (self.ts - 1)) + + + # last verifacation + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + + tdSql.query("select count(*),last(*) from stest group by col1") + tdSql.checkRows(10) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 2, 2) + tdSql.checkData(1, 3, 1) + + tdSql.query("select count(*),last(*) from stest group by col2") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.checkData(0, 2, 10) + tdSql.checkData(0, 3, 1) + + tdSql.query("select count(*),last(ts,stest.*) from stest group by col1") + tdSql.checkRows(10) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 2, "2018-09-17 09:00:00") + tdSql.checkData(1, 4, 1) + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) + diff --git a/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.json b/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.json new file mode 100644 index 0000000000..433d279825 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.json @@ -0,0 +1,62 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 150000, + "childtable_limit": -1, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 1000, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.py b/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.py new file mode 100644 index 0000000000..270bfd8b60 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.py @@ -0,0 +1,88 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import subprocess +import time +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + tdSql.prepare() + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + # # insert 1000w rows in stb0 + os.system("%staosdemo -f tools/taosdemoAllTest/query-interrupt.json -y " % binPath) + tdSql.execute("use db") + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0,60) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 6000000) + os.system('%staosdemo -f tools/taosdemoAllTest/queryall.json -y & ' % binPath) + time.sleep(2) + query_pid = int(subprocess.getstatusoutput('ps aux|grep "taosdemoAllTest/queryall.json" |grep -v "grep"|awk \'{print $2}\'')[1]) + taosd_cpu_load_1 = float(subprocess.getstatusoutput('top -n 1 -b -p $(ps aux|grep "bin/taosd -c"|grep -v "grep" |awk \'{print $2}\')|awk \'END{print}\' |awk \'{print $9}\'')[1]) + if taosd_cpu_load_1 > 10.0 : + os.system("kill -9 %d" % query_pid) + time.sleep(5) + taosd_cpu_load_2 = float(subprocess.getstatusoutput('top -n 1 -b -p $(ps aux|grep "bin/taosd -c"|grep -v "grep" |awk \'{print $2}\')|awk \'END{print}\' |awk \'{print $9}\'')[1]) + if taosd_cpu_load_2 < 10.0 : + suc_kill = 60 + else: + suc_kill = 10 + print("taosd_cpu_load is higher than 10%") + else: + suc_kill = 20 + print("taosd_cpu_load is still less than 10%") + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, "%d" % suc_kill) + os.system("rm -rf querySystemInfo*") + os.system("rm -rf insert_res.txt") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdemoAllTest/TD-3453/queryall.json b/tests/pytest/tools/taosdemoAllTest/TD-3453/queryall.json new file mode 100644 index 0000000000..bbc3b9717c --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/TD-3453/queryall.json @@ -0,0 +1,14 @@ +{ + "filetype":"query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "db", + "specified_table_query": + {"query_interval":1, "concurrent":1, + "sqls": [{"sql": "select * from stb0", "result": ""}] + } +} \ No newline at end of file diff --git a/tests/pytest/tools/taosdemoAllTest/convertResFile.py b/tests/pytest/tools/taosdemoAllTest/convertResFile.py new file mode 100644 index 0000000000..52bb8f40d0 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/convertResFile.py @@ -0,0 +1,35 @@ +from datetime import datetime +import time +import os + +os.system("awk -v OFS=',' '{$1=$1;print$0}' ./all_query_res0.txt > ./new_query_res0.txt") +with open('./new_query_res0.txt','r+') as f0: + contents = f0.readlines() + if os.path.exists('./test_query_res0.txt'): + os.system("rm -rf ./test_query_res0.txt") + for i in range(len(contents)): + content = contents[i].rstrip('\n') + stimestamp = content.split(',')[0] + timestamp = int(stimestamp) + d = datetime.fromtimestamp(timestamp/1000) + str0 = d.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] + ts = "'"+str0+"'" + str1 = "'"+content.split(',')[1]+"'" + str2 = "'"+content.split(',')[2]+"'" + content = ts + "," + str1 + "," + str2 + "," + content.split(',',3)[3] + contents[i] = content + "\n" + with open('./test_query_res0.txt','a') as fi: + fi.write(contents[i]) + +os.system("rm -rf ./new_query_res0.txt") + + + + + + + +# timestamp = 1604160000099 +# d = datetime.fromtimestamp(timestamp/1000) +# str1 = d.strftime("%Y-%m-%d %H:%M:%S.%f") +# print(str1[:-3]) diff --git a/tests/pytest/tools/taosdemoAllTest/insert-1s1tnt1r.json b/tests/pytest/tools/taosdemoAllTest/insert-1s1tnt1r.json new file mode 100644 index 0000000000..8e40ad812d --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/insert-1s1tnt1r.json @@ -0,0 +1,88 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 10, + "num_of_records_per_req": 1, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 1000, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 1, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 1000, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 200, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":4}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/insert-1s1tntmr.json b/tests/pytest/tools/taosdemoAllTest/insert-1s1tntmr.json new file mode 100644 index 0000000000..e741fd5c05 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/insert-1s1tntmr.json @@ -0,0 +1,88 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 10, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 10, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 10000, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 20, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 20000, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/insert-disorder.json b/tests/pytest/tools/taosdemoAllTest/insert-disorder.json new file mode 100644 index 0000000000..fddaa4b4b9 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/insert-disorder.json @@ -0,0 +1,88 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "": 4, + "result_file":"./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 10, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 1, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 10, + "childtable_limit": -1, + "childtable_offset": 0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 1, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 10, + "disorder_range": 100, + "timestamp_step": 1000, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count":1, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 1, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 10, + "childtable_limit": -1, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 100, + "disorder_range": 1, + "timestamp_step": 1000, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/insert-illegal-columns-count-0.json b/tests/pytest/tools/taosdemoAllTest/insert-illegal-columns-count-0.json new file mode 100644 index 0000000000..f6a103f001 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/insert-illegal-columns-count-0.json @@ -0,0 +1,88 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 10, + "num_of_records_per_req": 100, + "max_sql_len": 10240000000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 10, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 1000, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":0}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":7}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 20, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 12, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 2000, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/insert-illegal-columns-lmax.json b/tests/pytest/tools/taosdemoAllTest/insert-illegal-columns-lmax.json new file mode 100644 index 0000000000..17050278c8 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/insert-illegal-columns-lmax.json @@ -0,0 +1,88 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 10, + "num_of_records_per_req": 100, + "max_sql_len": 10240000000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 10, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 1000, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1024}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":7}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 20, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 12, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 2000, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1004}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":7}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/insert-illegal-columns.json b/tests/pytest/tools/taosdemoAllTest/insert-illegal-columns.json new file mode 100644 index 0000000000..53735dc413 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/insert-illegal-columns.json @@ -0,0 +1,88 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 10, + "num_of_records_per_req": 100, + "max_sql_len": 10240000000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 10, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 1000, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1005}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":7}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 20, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 12, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 2000, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/insert-illegal-tags-count129.json b/tests/pytest/tools/taosdemoAllTest/insert-illegal-tags-count129.json new file mode 100644 index 0000000000..115c42b502 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/insert-illegal-tags-count129.json @@ -0,0 +1,88 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 10, + "num_of_records_per_req": 100, + "max_sql_len": 10240000000, + "databases": [{ + "dbinfo": { + "name": "db1", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 10, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 1000, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BIGINT", "count":1}, {"type": "float", "count":1}, {"type": "double", "count":1}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":127}, {"type": "BINARY", "len": 16, "count":2}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 20, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 12, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 2000, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/insert-interlace-row.json b/tests/pytest/tools/taosdemoAllTest/insert-interlace-row.json new file mode 100644 index 0000000000..26e8b7e88d --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/insert-interlace-row.json @@ -0,0 +1,62 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 150, + "childtable_limit": -1, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 151, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/insert-interval-speed.json b/tests/pytest/tools/taosdemoAllTest/insert-interval-speed.json new file mode 100644 index 0000000000..c7c5150a06 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/insert-interval-speed.json @@ -0,0 +1,88 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 100, + "interlace_rows": 0, + "num_of_records_per_req": 2000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 20000, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 1000, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 20000, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 1000, + "insert_interval": 2000, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":9}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/insert-newdb.json b/tests/pytest/tools/taosdemoAllTest/insert-newdb.json new file mode 100644 index 0000000000..72e380a66c --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/insert-newdb.json @@ -0,0 +1,166 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 0, + "num_of_records_per_req": 3000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 1 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"yes", + "childtable_count": 5, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 10, + "childtable_limit": -1, + "childtable_offset": 0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 6, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 20, + "childtable_limit": -1, + "childtable_offset": 0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb2", + "child_table_exists":"no", + "childtable_count": 7, + "childtable_prefix": "stb02_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 20, + "childtable_limit": 4, + "childtable_offset": 0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb3", + "child_table_exists":"no", + "childtable_count": 8, + "childtable_prefix": "stb03_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 20, + "childtable_limit": 2, + "childtable_offset": 7, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb4", + "child_table_exists":"no", + "childtable_count": 8, + "childtable_prefix": "stb04_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 20, + "childtable_limit": 0, + "childtable_offset": 7, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/insert-newtable.json b/tests/pytest/tools/taosdemoAllTest/insert-newtable.json new file mode 100644 index 0000000000..3115c9ba72 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/insert-newtable.json @@ -0,0 +1,166 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 0, + "num_of_records_per_req": 3000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "no", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 1 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"yes", + "childtable_count": 5, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 20, + "childtable_limit": -1, + "childtable_offset": 0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-12-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 6, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 20, + "childtable_limit": -1, + "childtable_offset": 0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-12-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb2", + "child_table_exists":"no", + "childtable_count": 7, + "childtable_prefix": "stb02_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 20, + "childtable_limit": 4, + "childtable_offset": 0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-12-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb3", + "child_table_exists":"no", + "childtable_count": 8, + "childtable_prefix": "stb03_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 20, + "childtable_limit": 2, + "childtable_offset": 7, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-12-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb4", + "child_table_exists":"no", + "childtable_count": 8, + "childtable_prefix": "stb04_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 30, + "childtable_limit": 0, + "childtable_offset": 7, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-12-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/insert-nodbnodrop.json b/tests/pytest/tools/taosdemoAllTest/insert-nodbnodrop.json new file mode 100644 index 0000000000..7fdba4add1 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/insert-nodbnodrop.json @@ -0,0 +1,62 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 0, + "num_of_records_per_req": 3000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "dbno", + "drop": "no", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 1 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 5, + "childtable_prefix": "stb00_", + "auto_create_table": "yes", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 10, + "childtable_limit": -1, + "childtable_offset": 0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/insert-offset.json b/tests/pytest/tools/taosdemoAllTest/insert-offset.json new file mode 100644 index 0000000000..611b4a8989 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/insert-offset.json @@ -0,0 +1,166 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 0, + "num_of_records_per_req": 3000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "no", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"yes", + "childtable_count": 5, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 20, + "childtable_limit": 0, + "childtable_offset": 0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-11-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb1", + "child_table_exists":"yes", + "childtable_count": 6, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 20, + "childtable_limit": -1, + "childtable_offset": 0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-11-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb2", + "child_table_exists":"yes", + "childtable_count": 7, + "childtable_prefix": "stb02_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 20, + "childtable_limit": 4, + "childtable_offset": 0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-11-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb3", + "child_table_exists":"yes", + "childtable_count": 8, + "childtable_prefix": "stb03_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 20, + "childtable_limit": 2, + "childtable_offset":7, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-11-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb4", + "child_table_exists":"yes", + "childtable_count": 8, + "childtable_prefix": "stb04_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 20, + "childtable_limit": 0, + "childtable_offset": 7, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-11-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/insert-renewdb.json b/tests/pytest/tools/taosdemoAllTest/insert-renewdb.json new file mode 100644 index 0000000000..72e380a66c --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/insert-renewdb.json @@ -0,0 +1,166 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 0, + "num_of_records_per_req": 3000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 1 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"yes", + "childtable_count": 5, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 10, + "childtable_limit": -1, + "childtable_offset": 0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 6, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 20, + "childtable_limit": -1, + "childtable_offset": 0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb2", + "child_table_exists":"no", + "childtable_count": 7, + "childtable_prefix": "stb02_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 20, + "childtable_limit": 4, + "childtable_offset": 0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb3", + "child_table_exists":"no", + "childtable_count": 8, + "childtable_prefix": "stb03_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 20, + "childtable_limit": 2, + "childtable_offset": 7, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb4", + "child_table_exists":"no", + "childtable_count": 8, + "childtable_prefix": "stb04_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 20, + "childtable_limit": 0, + "childtable_offset": 7, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/insert-sample.json b/tests/pytest/tools/taosdemoAllTest/insert-sample.json new file mode 100644 index 0000000000..015993227e --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/insert-sample.json @@ -0,0 +1,88 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file":"./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 10, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "dbtest123", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 1, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "sample", + "insert_mode": "taosc", + "insert_rows": 10, + "childtable_limit": -1, + "childtable_offset": 0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./tools/taosdemoAllTest/sample.csv", + "tags_file": "", + "columns": [{"type": "INT", "count":3}, {"type": "DOUBLE", "count":3}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}, {"type": "BOOL"}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count":2, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 10, + "childtable_limit": -1, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 10, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "./tools/taosdemoAllTest/tags.csv", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":3}, {"type": "BINARY", "len": 16, "count":2}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/insert-timestep.json b/tests/pytest/tools/taosdemoAllTest/insert-timestep.json new file mode 100644 index 0000000000..01d8ac9098 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/insert-timestep.json @@ -0,0 +1,88 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file":"./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 10, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 10, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 20, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count":20, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 20, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 10, + "start_timestamp": "2020-11-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/moredemo-insert-offset.py b/tests/pytest/tools/taosdemoAllTest/moredemo-insert-offset.py new file mode 100644 index 0000000000..703f755c31 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/moredemo-insert-offset.py @@ -0,0 +1,72 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + # insert: drop and child_table_exists combination test + # insert: using parament "childtable_offset and childtable_limit" to control table'offset point and offset + os.system("%staosdemo -f tools/taosdemoAllTest/moredemo-offset-newdb.json" % binPath) + os.system("%staosdemo -f tools/taosdemoAllTest/moredemo-offset-limit1.json & " % binPath) + os.system("%staosdemo -f tools/taosdemoAllTest/moredemo-offset-limit94.json & " % binPath) + os.system("%staosdemo -f tools/taosdemoAllTest/moredemo-offset-limit5.json & " % binPath) + sleep(15) + tdSql.execute("use db") + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 1000000) + + os.system("rm -rf ./insert_res.txt") + os.system("rm -rf tools/taosdemoAllTest/taosdemoTestWithJson-1.py.sql") + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit1.json b/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit1.json new file mode 100644 index 0000000000..ad6cb8118d --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit1.json @@ -0,0 +1,62 @@ + +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 0, + "num_of_records_per_req": 3000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "no", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"yes", + "childtable_count": 100, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 10000, + "childtable_limit": 1, + "childtable_offset": 99, + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-11-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit5.json b/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit5.json new file mode 100644 index 0000000000..7109dab53f --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit5.json @@ -0,0 +1,62 @@ + +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 0, + "num_of_records_per_req": 3000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "no", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"yes", + "childtable_count": 100, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 10000, + "childtable_limit": 5, + "childtable_offset": 0, + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-11-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit94.json b/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit94.json new file mode 100644 index 0000000000..a98a185b54 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit94.json @@ -0,0 +1,62 @@ + +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 0, + "num_of_records_per_req": 3000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "no", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"yes", + "childtable_count": 100, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 10000, + "childtable_limit": 94, + "childtable_offset": 5, + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-11-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/moredemo-offset-newdb.json b/tests/pytest/tools/taosdemoAllTest/moredemo-offset-newdb.json new file mode 100644 index 0000000000..e2f3fb0379 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/moredemo-offset-newdb.json @@ -0,0 +1,61 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 0, + "num_of_records_per_req": 3000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 0, + "childtable_limit": 0, + "childtable_offset": 0, + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-11-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/sample.csv b/tests/pytest/tools/taosdemoAllTest/sample.csv new file mode 100644 index 0000000000..471118a2ce --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/sample.csv @@ -0,0 +1,3 @@ +1,-1,2147483647,0,2247483647.1,-12.2,'12ac,;\[uer]','23ac,;\[uer23423]123123','true' +0,-1,2147483647,0,2247483647.1,-12.2,'12ac,;\[uer]','23ac,;\[uer23423]123123','true' +0,-1,2147483647,0,2247483647.1,-12.2,'12ac,;\[uer]','23ac,;\[uer23423]123123','false' \ No newline at end of file diff --git a/tests/pytest/tools/taosdemoAllTest/speciQuery.json b/tests/pytest/tools/taosdemoAllTest/speciQuery.json new file mode 100644 index 0000000000..5e99e80108 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/speciQuery.json @@ -0,0 +1,36 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "db", + "query_times": 2, + "specified_table_query": { + "query_interval": 1, + "concurrent": 3, + "sqls": [ + { + "sql": "select last_row(*) from stb0 ", + "result": "./query_res0.txt" + }, + { + "sql": "select count(*) from stb00_1", + "result": "./query_res1.txt" + } + ] + }, + "super_table_query": { + "stblname": "stb1", + "query_interval": 1, + "threads": 3, + "sqls": [ + { + "sql": "select last_row(ts) from xxxx", + "result": "./query_res2.txt" + } + ] + } +} diff --git a/tests/pytest/tools/taosdemoAllTest/speciQueryInsertdata.json b/tests/pytest/tools/taosdemoAllTest/speciQueryInsertdata.json new file mode 100644 index 0000000000..ec9cb5a40d --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/speciQueryInsertdata.json @@ -0,0 +1,86 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 0, + "num_of_records_per_req": 3000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_limit": 0, + "childtable_offset": 0, + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-11-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}, {"type": "INT"}, {"type": "DOUBLE", "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 200, + "childtable_limit": 0, + "childtable_offset": 0, + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-11-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":6}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/tags.csv b/tests/pytest/tools/taosdemoAllTest/tags.csv new file mode 100644 index 0000000000..89bf8e3fb3 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/tags.csv @@ -0,0 +1,2 @@ +1,-127,127,'23ac,;\[uer]3','true' +1,-127,126,'23ac,;\[uer]3','true' diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJson.py b/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJson.py new file mode 100644 index 0000000000..2dd50bf639 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJson.py @@ -0,0 +1,229 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + # insert: create one or mutiple tables per sql and insert multiple rows per sql + os.system("%staosdemo -f tools/taosdemoAllTest/insert-1s1tnt1r.json -y " % binPath) + tdSql.execute("use db") + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 1000) + tdSql.query("select count (tbname) from stb1") + tdSql.checkData(0, 0, 1000) + tdSql.query("select count(*) from stb00_0") + tdSql.checkData(0, 0, 100) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 100000) + tdSql.query("select count(*) from stb01_1") + tdSql.checkData(0, 0, 200) + tdSql.query("select count(*) from stb1") + tdSql.checkData(0, 0, 200000) + + + # insert: create mutiple tables per sql and insert one rows per sql . + os.system("%staosdemo -f tools/taosdemoAllTest/insert-1s1tntmr.json -y " % binPath) + tdSql.execute("use db") + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 10) + tdSql.query("select count (tbname) from stb1") + tdSql.checkData(0, 0, 20) + tdSql.query("select count(*) from stb00_0") + tdSql.checkData(0, 0, 10000) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 100000) + tdSql.query("select count(*) from stb01_0") + tdSql.checkData(0, 0, 20000) + tdSql.query("select count(*) from stb1") + tdSql.checkData(0, 0, 400000) + + # insert: using parament "insert_interval to controls spped of insert. + # but We need to have accurate methods to control the speed, such as getting the speed value, checking the count and so on。 + os.system("%staosdemo -f tools/taosdemoAllTest/insert-interval-speed.json -y" % binPath) + tdSql.execute("use db") + tdSql.query("show stables") + tdSql.checkData(0, 4, 100) + tdSql.query("select count(*) from stb00_0") + tdSql.checkData(0, 0, 20000) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 2000000) + tdSql.query("show stables") + tdSql.checkData(1, 4, 100) + tdSql.query("select count(*) from stb01_0") + tdSql.checkData(0, 0, 20000) + tdSql.query("select count(*) from stb1") + tdSql.checkData(0, 0, 2000000) + + # spend 2min30s for 3 testcases. + # insert: drop and child_table_exists combination test + # insert: using parament "childtable_offset and childtable_limit" to control table'offset point and offset + os.system("%staosdemo -f tools/taosdemoAllTest/insert-nodbnodrop.json -y" % binPath) + tdSql.error("show dbno.stables") + os.system("%staosdemo -f tools/taosdemoAllTest/insert-newdb.json -y" % binPath) + tdSql.execute("use db") + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 5) + tdSql.query("select count (tbname) from stb1") + tdSql.checkData(0, 0, 6) + tdSql.query("select count (tbname) from stb2") + tdSql.checkData(0, 0, 7) + tdSql.query("select count (tbname) from stb3") + tdSql.checkData(0, 0, 8) + tdSql.query("select count (tbname) from stb4") + tdSql.checkData(0, 0, 8) + os.system("%staosdemo -f tools/taosdemoAllTest/insert-offset.json -y" % binPath) + tdSql.execute("use db") + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 50) + tdSql.query("select count(*) from stb1") + tdSql.checkData(0, 0, 240) + tdSql.query("select count(*) from stb2") + tdSql.checkData(0, 0, 220) + tdSql.query("select count(*) from stb3") + tdSql.checkData(0, 0, 180) + tdSql.query("select count(*) from stb4") + tdSql.checkData(0, 0, 160) + os.system("%staosdemo -f tools/taosdemoAllTest/insert-newtable.json -y" % binPath) + tdSql.execute("use db") + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 150) + tdSql.query("select count(*) from stb1") + tdSql.checkData(0, 0, 360) + tdSql.query("select count(*) from stb2") + tdSql.checkData(0, 0, 360) + tdSql.query("select count(*) from stb3") + tdSql.checkData(0, 0, 340) + tdSql.query("select count(*) from stb4") + tdSql.checkData(0, 0, 400) + os.system("%staosdemo -f tools/taosdemoAllTest/insert-renewdb.json -y" % binPath) + tdSql.execute("use db") + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 50) + tdSql.query("select count(*) from stb1") + tdSql.checkData(0, 0, 120) + tdSql.query("select count(*) from stb2") + tdSql.checkData(0, 0, 140) + tdSql.query("select count(*) from stb3") + tdSql.checkData(0, 0, 160) + tdSql.query("select count(*) from stb4") + tdSql.checkData(0, 0, 160) + + + # insert: let parament in json file is illegal ，i need know how to write exception. + tdSql.execute("drop database if exists db") + os.system("%staosdemo -f tools/taosdemoAllTest/insert-illegal-columns.json -y " % binPath) + tdSql.error("use db") + os.system("%staosdemo -f tools/taosdemoAllTest/insert-illegal-columns-lmax.json -y " % binPath) + tdSql.error("select * from db.stb0") + os.system("%staosdemo -f tools/taosdemoAllTest/insert-illegal-columns-count-0.json -y " % binPath) + tdSql.execute("use db") + tdSql.query("select count(*) from db.stb0") + tdSql.checkData(0, 0, 10000) + tdSql.execute("drop database if exists db") + os.system("%staosdemo -f tools/taosdemoAllTest/insert-illegal-tags-count129.json -y " % binPath) + tdSql.error("use db1") + + # insert: timestamp and step + os.system("%staosdemo -f tools/taosdemoAllTest/insert-timestep.json -y " % binPath) + tdSql.execute("use db") + tdSql.query("show stables") + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 10) + tdSql.query("select count (tbname) from stb1") + tdSql.checkData(0, 0, 20) + tdSql.query("select last(ts) from db.stb00_0") + tdSql.checkData(0, 0, "2020-10-01 00:00:00.019000") + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 200) + tdSql.query("select last(ts) from db.stb01_0") + tdSql.checkData(0, 0, "2020-11-01 00:00:00.190000") + tdSql.query("select count(*) from stb1") + tdSql.checkData(0, 0, 400) + + # # insert: disorder_ratio + os.system("%staosdemo -f tools/taosdemoAllTest/insert-disorder.json -g 2>&1 -y " % binPath) + tdSql.execute("use db") + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 1) + tdSql.query("select count (tbname) from stb1") + tdSql.checkData(0, 0, 1) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(*) from stb1") + tdSql.checkData(0, 0, 10) + + # insert: sample json + os.system("%staosdemo -f tools/taosdemoAllTest/insert-sample.json -y " % binPath) + tdSql.execute("use dbtest123") + tdSql.query("select col2 from stb0") + tdSql.checkData(0, 0, 2147483647) + tdSql.query("select t1 from stb1") + tdSql.checkData(0, 0, -127) + tdSql.query("select t2 from stb1") + tdSql.checkData(1, 0, 126) + + # insert: test interlace parament + os.system("%staosdemo -f tools/taosdemoAllTest/insert-interlace-row.json -y " % binPath) + tdSql.execute("use db") + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 100) + tdSql.query("select count (*) from stb0") + tdSql.checkData(0, 0, 15000) + + + os.system("rm -rf ./insert_res.txt") + os.system("rm -rf tools/taosdemoAllTest/taosdemoTestInsertWithJson.py.sql") + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestQueryWithJson.py b/tests/pytest/tools/taosdemoAllTest/taosdemoTestQueryWithJson.py new file mode 100644 index 0000000000..00da2c0874 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestQueryWithJson.py @@ -0,0 +1,91 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import time +from datetime import datetime + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + # insert: drop and child_table_exists combination test + # insert: using parament "childtable_offset and childtable_limit" to control table'offset point and offset + os.system("%staosdemo -f tools/taosdemoAllTest/speciQueryInsertdata.json" % binPath) + os.system("%staosdemo -f tools/taosdemoAllTest/speciQuery.json" % binPath) + os.system("cat query_res0.txt* |sort -u > all_query_res0.txt") + os.system("cat query_res1.txt* |sort -u > all_query_res1.txt") + os.system("cat query_res2.txt* |sort -u > all_query_res2.txt") + tdSql.execute("use db") + tdSql.execute('create table result0 using stb0 tags(121,43,"beijing","beijing","beijing","beijing","beijing")') + os.system("python3 tools/taosdemoAllTest/convertResFile.py") + tdSql.execute("insert into result0 file './test_query_res0.txt-0'") + tdSql.query("select ts from result0") + tdSql.checkData(0, 0, "2020-11-01 00:00:00.099000") + tdSql.query("select count(*) from result0") + tdSql.checkData(0, 0, 1) + with open('./all_query_res1.txt','r+') as f1: + result1 = int(f1.readline()) + tdSql.query("select count(*) from stb00_1") + tdSql.checkData(0, 0, "%d" % result1) + + with open('./all_query_res2.txt','r+') as f2: + result2 = int(f2.readline()) + d2 = datetime.fromtimestamp(result2/1000) + timest = d2.strftime("%Y-%m-%d %H:%M:%S.%f") + tdSql.query("select last_row(ts) from stb1") + tdSql.checkData(0, 0, "%s" % timest) + + os.system("rm -rf ./insert_res.txt") + os.system("rm -rf tools/taosdemoAllTest/taosdemoTestQuerytWithJson.py.sql") + os.system("rm -rf ./querySystemInfo*") + os.system("rm -rf ./query_res*") + os.system("rm -rf ./all_query*") + os.system("rm -rf ./test_query_res0.txt") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 2450c0e7880f8344f6da7dbb5dbe0f1164b59106 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 22 Apr 2021 10:46:56 +0800 Subject: [PATCH 02/27] [TD-3879]add stmt mode for taodemo go --- tests/examples/go/taosdemo.go | 149 +++++++++++++++++++++++++++++++++- 1 file changed, 148 insertions(+), 1 deletion(-) diff --git a/tests/examples/go/taosdemo.go b/tests/examples/go/taosdemo.go index 003f5aeddc..e36eee5629 100644 --- a/tests/examples/go/taosdemo.go +++ b/tests/examples/go/taosdemo.go @@ -18,6 +18,7 @@ import ( "database/sql" "flag" "fmt" + "log" "math/rand" "os" "runtime" @@ -26,7 +27,6 @@ import ( "time" _ "github.com/taosdata/driver-go/taosSql" - //"golang.org/x/sys/unix" ) @@ -48,6 +48,7 @@ type config struct { dbName string supTblName string tablePrefix string + mode string numOftables int numOfRecordsPerTable int numOfRecordsPerReq int @@ -70,6 +71,7 @@ func init() { flag.StringVar(&configPara.password, "P", "taosdata", "The password to use when connecting to the server.") flag.StringVar(&configPara.dbName, "d", "test", "Destination database.") flag.StringVar(&configPara.tablePrefix, "m", "d", "Table prefix name.") + flag.StringVar(&configPara.mode, "M", "r", "mode,r:raw,s:stmt") flag.IntVar(&configPara.numOftables, "t", 2, "The number of tables.") flag.IntVar(&configPara.numOfRecordsPerTable, "n", 10, "The number of records per table.") flag.IntVar(&configPara.numOfRecordsPerReq, "r", 3, "The number of records per request.") @@ -94,6 +96,7 @@ func printAllArgs() { fmt.Printf("usr: %v\n", configPara.user) fmt.Printf("password: %v\n", configPara.password) fmt.Printf("dbName: %v\n", configPara.dbName) + fmt.Printf("mode: %v\n", configPara.mode) fmt.Printf("tablePrefix: %v\n", configPara.tablePrefix) fmt.Printf("numOftables: %v\n", configPara.numOftables) fmt.Printf("numOfRecordsPerTable: %v\n", configPara.numOfRecordsPerTable) @@ -119,6 +122,24 @@ func main() { //defer db.Close() rand.Seed(time.Now().Unix()) + if configPara.mode == "s" { + fmt.Printf("\n======== start stmt mode test ========\n") + db, err := sql.Open("taosSql", url) + if err != nil { + log.Fatalf("Open database error: %s\n", err) + } + defer db.Close() + demodbStmt := configPara.dbName + demotStmt := "demotStmt" + drop_database_stmt(db, demodbStmt) + create_database_stmt(db, demodbStmt) + use_database_stmt(db, demodbStmt) + create_table_stmt(db, demotStmt) + insert_data_stmt(db, demotStmt) + select_data_stmt(db, demotStmt) + return + } + createDatabase(configPara.dbName, configPara.supTblName) fmt.Printf("======== create database success! ========\n\n") @@ -407,6 +428,132 @@ func selectTest(dbName string, tbPrefix string, supTblName string) { checkErr(err, "rows next iteration error") } } +func drop_database_stmt(db *sql.DB, demodb string) { + st := time.Now().Nanosecond() + // drop test db + res, err := db.Exec("drop database if exists " + demodb) + checkErr(err, "drop database "+demodb) + + affectd, err := res.RowsAffected() + checkErr(err, "drop db, res.RowsAffected") + + et := time.Now().Nanosecond() + fmt.Printf("drop database result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1e9) +} + +func create_database_stmt(db *sql.DB, demodb string) { + st := time.Now().Nanosecond() + // create database + //var stmt interface{} + stmt, err := db.Prepare("create database ?") + checkErr(err, "create db, db.Prepare") + + //var res driver.Result + res, err := stmt.Exec(demodb) + checkErr(err, "create db, stmt.Exec") + + //fmt.Printf("Query OK, %d row(s) affected()", res.RowsAffected()) + affectd, err := res.RowsAffected() + checkErr(err, "create db, res.RowsAffected") + + et := time.Now().Nanosecond() + fmt.Printf("create database result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1e9) +} + +func use_database_stmt(db *sql.DB, demodb string) { + st := time.Now().Nanosecond() + // create database + //var stmt interface{} + stmt, err := db.Prepare("use " + demodb) + checkErr(err, "use db, db.Prepare") + + res, err := stmt.Exec() + checkErr(err, "use db, stmt.Exec") + + affectd, err := res.RowsAffected() + checkErr(err, "use db, res.RowsAffected") + + et := time.Now().Nanosecond() + fmt.Printf("use database result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1e9) +} + +func create_table_stmt(db *sql.DB, demot string) { + st := time.Now().Nanosecond() + // create table + // (ts timestamp, id int, name binary(8), len tinyint, flag bool, notes binary(8), fv float, dv double) + stmt, err := db.Prepare("create table ? (? timestamp, ? int, ? binary(10), ? tinyint, ? bool, ? binary(8), ? float, ? double)") + checkErr(err, "create table db.Prepare") + + res, err := stmt.Exec(demot, "ts", "id", "name", "len", "flag", "notes", "fv", "dv") + checkErr(err, "create table stmt.Exec") + + affectd, err := res.RowsAffected() + checkErr(err, "create table res.RowsAffected") + + et := time.Now().Nanosecond() + fmt.Printf("create table result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1e9) +} + +func insert_data_stmt(db *sql.DB, demot string) { + st := time.Now().Nanosecond() + // insert data into table + stmt, err := db.Prepare("insert into ? values(?, ?, ?, ?, ?, ?, ?, ?) (?, ?, ?, ?, ?, ?, ?, ?) (?, ?, ?, ?, ?, ?, ?, ?)") + checkErr(err, "insert db.Prepare") + + res, err := stmt.Exec(demot, "now", 1000, "'haidian'", 6, true, "'AI world'", 6987.654, 321.987, + "now+1s", 1001, "'changyang'", 7, false, "'DeepMode'", 12356.456, 128634.456, + "now+2s", 1002, "'chuangping'", 8, true, "'database'", 3879.456, 65433478.456) + checkErr(err, "insert data, stmt.Exec") + + affectd, err := res.RowsAffected() + checkErr(err, "res.RowsAffected") + + et := time.Now().Nanosecond() + fmt.Printf("insert data result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1e9) +} + +func select_data_stmt(db *sql.DB, demot string) { + st := time.Now().Nanosecond() + + stmt, err := db.Prepare("select ?, ?, ?, ?, ?, ?, ?, ? from ?") // go binary mode + checkErr(err, "db.Prepare") + + rows, err := stmt.Query("ts", "id", "name", "len", "flag", "notes", "fv", "dv", demot) + checkErr(err, "stmt.Query") + + fmt.Printf("%10s%s%8s %5s %8s%s %s %10s%s %7s%s %8s%s %11s%s %14s%s\n", " ", "ts", " ", "id", " ", "name", " ", "len", " ", "flag", " ", "notes", " ", "fv", " ", " ", "dv") + var affectd int + for rows.Next() { + var ts string + var name string + var id int + var len int8 + var flag bool + var notes string + var fv float32 + var dv float64 + + err = rows.Scan(&ts, &id, &name, &len, &flag, ¬es, &fv, &dv) + //fmt.Println("start scan fields from row.rs, &fv:", &fv) + //err = rows.Scan(&fv) + checkErr(err, "rows.Scan") + + fmt.Printf("%s\t", ts) + fmt.Printf("%d\t", id) + fmt.Printf("%10s\t", name) + fmt.Printf("%d\t", len) + fmt.Printf("%t\t", flag) + fmt.Printf("%s\t", notes) + fmt.Printf("%06.3f\t", fv) + fmt.Printf("%09.6f\n", dv) + + affectd++ + + } + + et := time.Now().Nanosecond() + fmt.Printf("insert data result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1e9) +} func checkErr(err error, prompt string) { if err != nil { fmt.Printf("%s\n", prompt) From 7eba44be0823b11102c8fba5544f8b8c51e72ab2 Mon Sep 17 00:00:00 2001 From: tomchon Date: Thu, 22 Apr 2021 13:54:23 +0800 Subject: [PATCH 03/27] [TD-3414]: add insert function with json files of taodemo-testcase --- tests/pytest/tools/taosdemoAllTest/taosdemoTestQueryWithJson.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestQueryWithJson.py b/tests/pytest/tools/taosdemoAllTest/taosdemoTestQueryWithJson.py index 00da2c0874..00b387e398 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestQueryWithJson.py +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestQueryWithJson.py @@ -59,7 +59,7 @@ class TDTestCase: tdSql.execute("use db") tdSql.execute('create table result0 using stb0 tags(121,43,"beijing","beijing","beijing","beijing","beijing")') os.system("python3 tools/taosdemoAllTest/convertResFile.py") - tdSql.execute("insert into result0 file './test_query_res0.txt-0'") + tdSql.execute("insert into result0 file './test_query_res0.txt'") tdSql.query("select ts from result0") tdSql.checkData(0, 0, "2020-11-01 00:00:00.099000") tdSql.query("select count(*) from result0") From 369c9a76f00a83585f53d9a8d485a20758bd6611 Mon Sep 17 00:00:00 2001 From: wu champion Date: Fri, 23 Apr 2021 10:45:22 +0800 Subject: [PATCH 04/27] [TD-3918] add case to verify the bug of TD-3897 --- tests/pytest/fulltest.sh | 1 + tests/pytest/query/queryTscomputWithNow.py | 177 +++++++++++++++++++++ 2 files changed, 178 insertions(+) create mode 100644 tests/pytest/query/queryTscomputWithNow.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index c26dc8ae2f..e716f721c3 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -220,6 +220,7 @@ python3 ./test.py -f query/bug3375.py python3 ./test.py -f query/queryJoin10tables.py python3 ./test.py -f query/queryStddevWithGroupby.py python3 ./test.py -f query/querySecondtscolumnTowherenow.py +python3 ./test.py -f query/queryTscomputWithNow.py diff --git a/tests/pytest/query/queryTscomputWithNow.py b/tests/pytest/query/queryTscomputWithNow.py new file mode 100644 index 0000000000..3b808d551c --- /dev/null +++ b/tests/pytest/query/queryTscomputWithNow.py @@ -0,0 +1,177 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug(f"start to execute {__file__}") + tdSql.init(conn.cursor(), logSql) + + def inertnow(self): + tsp1 = 0 + tsp2 = -28800000 + tsp3 = -946800000000 + + tdSql.execute( + "create table stbts (ts timestamp, ts1 timestamp, c1 int, ts2 timestamp) TAGS(t1 int)" + ) + tdSql.execute("create table tts1 using stbts tags(1)") + + tdSql.execute("insert into tts1 values (now+1d, now+1d, 6, now+1d)") + tdSql.execute("insert into tts1 values (now, now, 5, now)") + tdSql.execute("insert into tts1 values (now-1d, now-1d, 4, now-1d)") + tdSql.execute(f"insert into tts1 values ({tsp1}, {tsp1}, 3, {tsp1})") + tdSql.execute(f"insert into tts1 values ({tsp2}, {tsp2}, 2, {tsp2})") + tdSql.execute(f"insert into tts1 values ({tsp3}, {tsp3}, 1, {tsp3})") + + def querynow(self): + interval_day1 = (datetime.date.today() - datetime.date(1970, 1, 1)).days + interval_day2 = (datetime.date.today() - datetime.date(1940, 1, 1)).days + + tdLog.printNoPrefix("==========step query: execute query operation") + time.sleep(1) + tdSql.execute(" select * from tts1 where ts > now+1d ") + ts_len1 = len(tdSql.cursor.fetchall()) + tdSql.execute(" select * from tts1 where ts < now+1d ") + ts_len2 = len(tdSql.cursor.fetchall()) + tdSql.execute(" select * from tts1 where ts > now-1d ") + ts_len3 = len(tdSql.cursor.fetchall()) + tdSql.execute(" select * from tts1 where ts < now-1d ") + ts_len4 = len(tdSql.cursor.fetchall()) + tdSql.execute(f" select * from tts1 where ts > now-{interval_day1+1}d ") + ts_len5 = len(tdSql.cursor.fetchall()) + tdSql.execute(f" select * from tts1 where ts < now-{interval_day1+1}d ") + ts_len6 = len(tdSql.cursor.fetchall()) + tdSql.execute(f" select * from tts1 where ts > now-{interval_day1-1}d ") + ts_len7 = len(tdSql.cursor.fetchall()) + tdSql.execute(f" select * from tts1 where ts < now-{interval_day1-1}d ") + ts_len8 = len(tdSql.cursor.fetchall()) + tdSql.execute(f" select * from tts1 where ts > now-{interval_day2+1}d ") + ts_len9 = len(tdSql.cursor.fetchall()) + tdSql.execute(f" select * from tts1 where ts < now-{interval_day2+1}d ") + ts_len10 = len(tdSql.cursor.fetchall()) + tdSql.execute(f" select * from tts1 where ts > now-{interval_day2-1}d ") + ts_len11 = len(tdSql.cursor.fetchall()) + tdSql.execute(f" select * from tts1 where ts < now-{interval_day2-1}d ") + ts_len12 = len(tdSql.cursor.fetchall()) + + tdSql.query(" select * from tts1 where ts1 > now+1d ") + tdSql.checkRows(ts_len1) + tdSql.query(" select * from tts1 where ts2 > now+1440m ") + tdSql.checkRows(ts_len1) + + tdSql.query(" select * from tts1 where ts1 < now+1d ") + tdSql.checkRows(ts_len2) + tdSql.query(" select * from tts1 where ts2 < now+1440m ") + tdSql.checkRows(ts_len2) + + tdSql.query(" select * from tts1 where ts1 > now-1d ") + tdSql.checkRows(ts_len3) + tdSql.query(" select * from tts1 where ts2 > now-1440m ") + tdSql.checkRows(ts_len3) + + tdSql.query(" select * from tts1 where ts1 < now-1d ") + tdSql.checkRows(ts_len4) + tdSql.query(" select * from tts1 where ts2 < now-1440m ") + tdSql.checkRows(ts_len4) + + tdSql.query(f" select * from tts1 where ts1 > now-{interval_day1+1}d ") + tdSql.checkRows(ts_len5) + tdSql.query(f" select * from tts1 where ts2 > now-{(interval_day1+1)*1440}m " ) + tdSql.checkRows(ts_len5) + + tdSql.query(f" select * from tts1 where ts1 < now-{interval_day1+1}d ") + tdSql.checkRows(ts_len6) + tdSql.query(f" select * from tts1 where ts2 < now-{(interval_day1+1)*1440}m ") + tdSql.checkRows(ts_len6) + + tdSql.query(f" select * from tts1 where ts1 > now-{interval_day1-1}d ") + tdSql.checkRows(ts_len7) + tdSql.query(f" select * from tts1 where ts2 > now-{(interval_day1-1)*1440}m ") + tdSql.checkRows(ts_len7) + + tdSql.query(f" select * from tts1 where ts1 < now-{interval_day1-1}d ") + tdSql.checkRows(ts_len8) + tdSql.query(f" select * from tts1 where ts2 < now-{(interval_day1-1)*1440}m ") + tdSql.checkRows(ts_len8) + + tdSql.query(f" select * from tts1 where ts1 > now-{interval_day2 + 1}d ") + tdSql.checkRows(ts_len9) + tdSql.query(f" select * from tts1 where ts2 > now-{(interval_day2 + 1)*1440}m ") + tdSql.checkRows(ts_len9) + + tdSql.query(f" select * from tts1 where ts1 < now-{interval_day2 + 1}d ") + tdSql.checkRows(ts_len10) + tdSql.query(f" select * from tts1 where ts2 < now-{(interval_day2 + 1)*1440}m ") + tdSql.checkRows(ts_len10) + + tdSql.query(f" select * from tts1 where ts1 > now-{interval_day2 - 1}d ") + tdSql.checkRows(ts_len11) + tdSql.query(f" select * from tts1 where ts2 > now-{(interval_day2 - 1)*1440}m ") + tdSql.checkRows(ts_len11) + + tdSql.query(f" select * from tts1 where ts1 < now-{interval_day2 - 1}d ") + tdSql.checkRows(ts_len12) + tdSql.query(f" select * from tts1 where ts2 < now-{(interval_day2 - 1)*1440}m ") + tdSql.checkRows(ts_len12) + + + + def run(self): + tdSql.execute("drop database if exists dbms") + tdSql.execute("drop database if exists dbus") + + # timestamp list: + # 0 -> "1970-01-01 08:00:00" | -28800000 -> "1970-01-01 00:00:00" | -946800000000 -> "1940-01-01 00:00:00" + # -631180800000 -> "1950-01-01 00:00:00" + + tdLog.printNoPrefix("==========step1:create table precision ms && insert data && query") + # create databases precision is ms + tdSql.execute("create database if not exists dbms keep 36500") + tdSql.execute("use dbms") + self.inertnow() + self.querynow() + + tdLog.printNoPrefix("==========step2:create table precision us && insert data && query") + # create databases precision is us + tdSql.execute("create database if not exists dbus keep 36500 precision 'us' ") + tdSql.execute("use dbus") + self.inertnow() + self.querynow() + + tdSql.query("show dnodes") + index = tdSql.getData(0, 0) + tdDnodes.stop(index) + tdDnodes.start(index) + + tdLog.printNoPrefix("==========step3:after wal, query table precision ms") + tdSql.execute("use dbus") + self.querynow() + + tdLog.printNoPrefix("==========step4: query table precision us") + tdSql.execute("use dbus") + self.querynow() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 46905c402a11e81b8c81a6f6d5093d7c5c2027e1 Mon Sep 17 00:00:00 2001 From: tomchon Date: Fri, 23 Apr 2021 11:25:08 +0800 Subject: [PATCH 05/27] [TD-3453]:modify filepath in scripts --- .../taosdemoAllTest/TD-3453/query-interrupt.json | 4 ++-- .../taosdemoAllTest/TD-3453/query-interrupt.py | 7 ++++--- .../tools/taosdemoAllTest/TD-3453/queryall.json | 14 ++++++++++---- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.json b/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.json index 433d279825..5e53bd7e7d 100644 --- a/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.json +++ b/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.json @@ -35,13 +35,13 @@ "super_tables": [{ "name": "stb0", "child_table_exists":"no", - "childtable_count": 100, + "childtable_count": 60, "childtable_prefix": "stb00_", "auto_create_table": "no", "batch_create_tbl_num": 20, "data_source": "rand", "insert_mode": "taosc", - "insert_rows": 150000, + "insert_rows": 100000, "childtable_limit": -1, "childtable_offset":0, "multi_thread_write_one_tbl": "no", diff --git a/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.py b/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.py index 270bfd8b60..1401716da9 100644 --- a/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.py +++ b/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.py @@ -52,15 +52,15 @@ class TDTestCase: binPath = buildPath+ "/build/bin/" # # insert 1000w rows in stb0 - os.system("%staosdemo -f tools/taosdemoAllTest/query-interrupt.json -y " % binPath) + os.system("%staosdemo -f tools/taosdemoAllTest/TD-3453/query-interrupt.json -y " % binPath) tdSql.execute("use db") tdSql.query("select count (tbname) from stb0") tdSql.checkData(0, 0,60) tdSql.query("select count(*) from stb0") tdSql.checkData(0, 0, 6000000) - os.system('%staosdemo -f tools/taosdemoAllTest/queryall.json -y & ' % binPath) + os.system('%staosdemo -f tools/taosdemoAllTest/TD-3453/queryall.json -y & ' % binPath) time.sleep(2) - query_pid = int(subprocess.getstatusoutput('ps aux|grep "taosdemoAllTest/queryall.json" |grep -v "grep"|awk \'{print $2}\'')[1]) + query_pid = int(subprocess.getstatusoutput('ps aux|grep "TD-3453/queryall.json" |grep -v "grep"|awk \'{print $2}\'')[1]) taosd_cpu_load_1 = float(subprocess.getstatusoutput('top -n 1 -b -p $(ps aux|grep "bin/taosd -c"|grep -v "grep" |awk \'{print $2}\')|awk \'END{print}\' |awk \'{print $9}\'')[1]) if taosd_cpu_load_1 > 10.0 : os.system("kill -9 %d" % query_pid) @@ -78,6 +78,7 @@ class TDTestCase: tdSql.checkData(0, 0, "%d" % suc_kill) os.system("rm -rf querySystemInfo*") os.system("rm -rf insert_res.txt") + os.system("rm -rf insert_res.txt") def stop(self): tdSql.close() diff --git a/tests/pytest/tools/taosdemoAllTest/TD-3453/queryall.json b/tests/pytest/tools/taosdemoAllTest/TD-3453/queryall.json index bbc3b9717c..a92906fa73 100644 --- a/tests/pytest/tools/taosdemoAllTest/TD-3453/queryall.json +++ b/tests/pytest/tools/taosdemoAllTest/TD-3453/queryall.json @@ -7,8 +7,14 @@ "password": "taosdata", "confirm_parameter_prompt": "no", "databases": "db", - "specified_table_query": - {"query_interval":1, "concurrent":1, - "sqls": [{"sql": "select * from stb0", "result": ""}] - } + "specified_table_query":{ + "query_interval":1, + "concurrent":1, + "sqls":[ + { + "sql": "select * from stb0", + "result": "" + } + ] + } } \ No newline at end of file From bbaf30ed9130285ad72dbfd58e1e4b7adbfd1d21 Mon Sep 17 00:00:00 2001 From: wu champion Date: Fri, 23 Apr 2021 11:28:28 +0800 Subject: [PATCH 06/27] Update fulltest.sh --- tests/pytest/fulltest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index f3e9aa02dd..24af231fc8 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -220,6 +220,7 @@ python3 ./test.py -f query/bug3375.py python3 ./test.py -f query/queryJoin10tables.py python3 ./test.py -f query/queryStddevWithGroupby.py python3 ./test.py -f query/querySecondtscolumnTowherenow.py +python3 ./test.py -f query/queryFilterTswithDateUnit.py python3 ./test.py -f query/queryTscomputWithNow.py From ec63c9fd649e2cba8bcaa20a812dc08bb5c5534c Mon Sep 17 00:00:00 2001 From: wu champion Date: Fri, 23 Apr 2021 16:49:06 +0800 Subject: [PATCH 07/27] [TD-3918] fix the case --- .../pytest/query/queryFilterTswithDateUnit.py | 171 ++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 tests/pytest/query/queryFilterTswithDateUnit.py diff --git a/tests/pytest/query/queryFilterTswithDateUnit.py b/tests/pytest/query/queryFilterTswithDateUnit.py new file mode 100644 index 0000000000..6348acc644 --- /dev/null +++ b/tests/pytest/query/queryFilterTswithDateUnit.py @@ -0,0 +1,171 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug(f"start to execute {__file__}") + tdSql.init(conn.cursor(), logSql) + + def insertnow(self, tsp1, tsp2, tsp3): + + tdSql.execute( + "create table stbts (ts timestamp, ts1 timestamp, c1 int, ts2 timestamp) TAGS(t1 int)" + ) + tdSql.execute("create table tts1 using stbts tags(1)") + + tdSql.execute("insert into tts1 values (now+1d, now+1d, 6, now+1d)") + tdSql.execute("insert into tts1 values (now, now, 5, now)") + tdSql.execute("insert into tts1 values (now-1d, now-1d, 4, now-1d)") + tdSql.execute(f"insert into tts1 values ({tsp1}, {tsp1}, 3, {tsp1})") + tdSql.execute(f"insert into tts1 values ({tsp2}, {tsp2}, 2, {tsp2})") + tdSql.execute(f"insert into tts1 values ({tsp3}, {tsp3}, 1, {tsp3})") + + + def querynow(self): + + tdLog.printNoPrefix("==========step query: execute query operation") + time.sleep(1) + + cols = ["ts", "ts1", "ts2"] + + for col in cols: + tdSql.error(f" select * from tts1 where {col} = 1d ") + tdSql.error(f" select * from tts1 where {col} < -1d ") + tdSql.error(f" select * from tts1 where {col} > 1d ") + tdSql.error(f" select * from tts1 where {col} >= -1d ") + tdSql.error(f" select * from tts1 where {col} <= 1d ") + tdSql.error(f" select * from tts1 where {col} <> 1d ") + + tdSql.error(f" select * from tts1 where {col} = -1m ") + tdSql.error(f" select * from tts1 where {col} < 1m ") + tdSql.error(f" select * from tts1 where {col} > 1m ") + tdSql.error(f" select * from tts1 where {col} >= -1m ") + tdSql.error(f" select * from tts1 where {col} <= 1m ") + tdSql.error(f" select * from tts1 where {col} <> 1m ") + + tdSql.error(f" select * from tts1 where {col} = -1s ") + tdSql.error(f" select * from tts1 where {col} < 1s ") + tdSql.error(f" select * from tts1 where {col} > 1s ") + tdSql.error(f" select * from tts1 where {col} >= -1s ") + tdSql.error(f" select * from tts1 where {col} <= 1s ") + tdSql.error(f" select * from tts1 where {col} <> 1s ") + + tdSql.error(f" select * from tts1 where {col} = -1a ") + tdSql.error(f" select * from tts1 where {col} < 1a ") + tdSql.error(f" select * from tts1 where {col} > 1a ") + tdSql.error(f" select * from tts1 where {col} >= -1a ") + tdSql.error(f" select * from tts1 where {col} <= 1a ") + tdSql.error(f" select * from tts1 where {col} <> 1a ") + + tdSql.error(f" select * from tts1 where {col} = -1h ") + tdSql.error(f" select * from tts1 where {col} < 1h ") + tdSql.error(f" select * from tts1 where {col} > 1h ") + tdSql.error(f" select * from tts1 where {col} >= -1h ") + tdSql.error(f" select * from tts1 where {col} <= 1h ") + tdSql.error(f" select * from tts1 where {col} <> 1h ") + + tdSql.error(f" select * from tts1 where {col} = -1w ") + tdSql.error(f" select * from tts1 where {col} < 1w ") + tdSql.error(f" select * from tts1 where {col} > 1w ") + tdSql.error(f" select * from tts1 where {col} >= -1w ") + tdSql.error(f" select * from tts1 where {col} <= 1w ") + tdSql.error(f" select * from tts1 where {col} <> 1w ") + + tdSql.error(f" select * from tts1 where {col} = -1u ") + tdSql.error(f" select * from tts1 where {col} < 1u ") + tdSql.error(f" select * from tts1 where {col} > 1u ") + tdSql.error(f" select * from tts1 where {col} >= -1u ") + tdSql.error(f" select * from tts1 where {col} <= 1u ") + tdSql.error(f" select * from tts1 where {col} <> u ") + + tdSql.error(f" select * from tts1 where {col} = 0d ") + tdSql.error(f" select * from tts1 where {col} < 0s ") + tdSql.error(f" select * from tts1 where {col} > 0a ") + tdSql.error(f" select * from tts1 where {col} >= 0m ") + tdSql.error(f" select * from tts1 where {col} <= 0h ") + tdSql.error(f" select * from tts1 where {col} <> 0u ") + tdSql.error(f" select * from tts1 where {col} <> 0w ") + + tdSql.error(f" select * from tts1 where {col} = 1m+1h ") + tdSql.error(f" select * from tts1 where {col} < 1w-1d ") + tdSql.error(f" select * from tts1 where {col} > 0a/1u ") + tdSql.error(f" select * from tts1 where {col} >= 1d/0s ") + tdSql.error(f" select * from tts1 where {col} <= 1s*1a ") + tdSql.error(f" select * from tts1 where {col} <> 0w/0d ") + + tdSql.error(f" select * from tts1 where {col} = 1m+1h ") + tdSql.error(f" select * from tts1 where {col} < 1w-1d ") + tdSql.error(f" select * from tts1 where {col} > 0a/1u ") + tdSql.error(f" select * from tts1 where {col} >= 1d/0s ") + tdSql.error(f" select * from tts1 where {col} <= 1s*1a ") + tdSql.error(f" select * from tts1 where {col} <> 0w/0d ") + + tdSql.error(f" select * from tts1 where {col} = 1u+1 ") + tdSql.error(f" select * from tts1 where {col} < 1a-1 ") + tdSql.error(f" select * from tts1 where {col} > 1s*1 ") + tdSql.error(f" select * from tts1 where {col} >= 1m/1 ") + tdSql.error(f" select * from tts1 where {col} <= 1h/0 ") + tdSql.error(f" select * from tts1 where {col} <> 0/1d ") + tdSql.error(f" select * from tts1 where {col} <> 1w+'2010-01-01 00:00:00' ") + + tdSql.error(f" select * from tts1 where {col} = 1-1h ") + tdSql.error(f" select * from tts1 where {col} < 1w-d ") + tdSql.error(f" select * from tts1 where {col} > 0/u ") + tdSql.error(f" select * from tts1 where {col} >= d/s ") + tdSql.error(f" select * from tts1 where {col} <= 1/a ") + tdSql.error(f" select * from tts1 where {col} <> d/1 ") + + + + def run(self): + tdSql.execute("drop database if exists dbms") + tdSql.execute("drop database if exists dbus") + + # timestamp list: + # 0 -> "1970-01-01 08:00:00" | -28800000 -> "1970-01-01 00:00:00" | -946800000000 -> "1940-01-01 00:00:00" + # -631180800000 -> "1950-01-01 00:00:00" + + tdLog.printNoPrefix("==========step1:create table precision ms && insert data && query") + # create databases precision is ms + tdSql.execute("create database if not exists dbms keep 36500") + tdSql.execute("use dbms") + tsp1 = 0 + tsp2 = -28800000 + tsp3 = -946800000000 + self.insertnow(tsp1,tsp2,tsp3) + self.querynow() + + tdLog.printNoPrefix("==========step2:create table precision us && insert data && query") + # create databases precision is us + tdSql.execute("create database if not exists dbus keep 36500 precision 'us' ") + tdSql.execute("use dbus") + tsp2 = tsp2 * 1000 + tsp3 = tsp3 * 1000 + self.insertnow(tsp1,tsp2,tsp3) + self.querynow() + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From ffc88d5f2090b741eed4d335cf195ab774f24a8b Mon Sep 17 00:00:00 2001 From: wu champion Date: Fri, 23 Apr 2021 16:54:47 +0800 Subject: [PATCH 08/27] Update queryFilterTswithDateUnit.py --- .../pytest/query/queryFilterTswithDateUnit.py | 70 +------------------ 1 file changed, 2 insertions(+), 68 deletions(-) diff --git a/tests/pytest/query/queryFilterTswithDateUnit.py b/tests/pytest/query/queryFilterTswithDateUnit.py index e834c1572c..eb9eb02afd 100644 --- a/tests/pytest/query/queryFilterTswithDateUnit.py +++ b/tests/pytest/query/queryFilterTswithDateUnit.py @@ -47,7 +47,6 @@ class TDTestCase: for col in cols: tdSql.error(f" select * from tts1 where {col} = 1d ") -<<<<<<< HEAD tdSql.error(f" select * from tts1 where {col} < -1d ") tdSql.error(f" select * from tts1 where {col} > 1d ") tdSql.error(f" select * from tts1 where {col} >= -1d ") @@ -95,55 +94,6 @@ class TDTestCase: tdSql.error(f" select * from tts1 where {col} >= -1u ") tdSql.error(f" select * from tts1 where {col} <= 1u ") tdSql.error(f" select * from tts1 where {col} <> u ") -======= - tdSql.error(f" select * from tts1 where {col} < 1d ") - tdSql.error(f" select * from tts1 where {col} > 1d ") - tdSql.error(f" select * from tts1 where {col} >= 1d ") - tdSql.error(f" select * from tts1 where {col} <= 1d ") - tdSql.error(f" select * from tts1 where {col} <> 1d ") - - tdSql.error(f" select * from tts1 where {col} = 1m ") - tdSql.error(f" select * from tts1 where {col} < 1m ") - tdSql.error(f" select * from tts1 where {col} > 1m ") - tdSql.error(f" select * from tts1 where {col} >= 1m ") - tdSql.error(f" select * from tts1 where {col} <= 1m ") - tdSql.error(f" select * from tts1 where {col} <> 1m ") - - tdSql.error(f" select * from tts1 where {col} = 1s ") - tdSql.error(f" select * from tts1 where {col} < 1s ") - tdSql.error(f" select * from tts1 where {col} > 1s ") - tdSql.error(f" select * from tts1 where {col} >= 1s ") - tdSql.error(f" select * from tts1 where {col} <= 1s ") - tdSql.error(f" select * from tts1 where {col} <> 1s ") - - tdSql.error(f" select * from tts1 where {col} = 1a ") - tdSql.error(f" select * from tts1 where {col} < 1a ") - tdSql.error(f" select * from tts1 where {col} > 1a ") - tdSql.error(f" select * from tts1 where {col} >= 1a ") - tdSql.error(f" select * from tts1 where {col} <= 1a ") - tdSql.error(f" select * from tts1 where {col} <> 1a ") - - tdSql.error(f" select * from tts1 where {col} = 1h ") - tdSql.error(f" select * from tts1 where {col} < 1h ") - tdSql.error(f" select * from tts1 where {col} > 1h ") - tdSql.error(f" select * from tts1 where {col} >= 1h ") - tdSql.error(f" select * from tts1 where {col} <= 1h ") - tdSql.error(f" select * from tts1 where {col} <> 1h ") - - tdSql.error(f" select * from tts1 where {col} = 1w ") - tdSql.error(f" select * from tts1 where {col} < 1w ") - tdSql.error(f" select * from tts1 where {col} > 1w ") - tdSql.error(f" select * from tts1 where {col} >= 1w ") - tdSql.error(f" select * from tts1 where {col} <= 1w ") - tdSql.error(f" select * from tts1 where {col} <> 1w ") - - tdSql.error(f" select * from tts1 where {col} = 1u ") - tdSql.error(f" select * from tts1 where {col} < 1u ") - tdSql.error(f" select * from tts1 where {col} > 1u ") - tdSql.error(f" select * from tts1 where {col} >= 1u ") - tdSql.error(f" select * from tts1 where {col} <= 1u ") - tdSql.error(f" select * from tts1 where {col} <> 1u ") ->>>>>>> bbaf30ed9130285ad72dbfd58e1e4b7adbfd1d21 tdSql.error(f" select * from tts1 where {col} = 0d ") tdSql.error(f" select * from tts1 where {col} < 0s ") @@ -175,7 +125,6 @@ class TDTestCase: tdSql.error(f" select * from tts1 where {col} <> 0/1d ") tdSql.error(f" select * from tts1 where {col} <> 1w+'2010-01-01 00:00:00' ") -<<<<<<< HEAD tdSql.error(f" select * from tts1 where {col} = 1-1h ") tdSql.error(f" select * from tts1 where {col} < 1w-d ") tdSql.error(f" select * from tts1 where {col} > 0/u ") @@ -183,10 +132,6 @@ class TDTestCase: tdSql.error(f" select * from tts1 where {col} <= 1/a ") tdSql.error(f" select * from tts1 where {col} <> d/1 ") - -======= ->>>>>>> bbaf30ed9130285ad72dbfd58e1e4b7adbfd1d21 - def run(self): tdSql.execute("drop database if exists dbms") tdSql.execute("drop database if exists dbus") @@ -209,27 +154,16 @@ class TDTestCase: # create databases precision is us tdSql.execute("create database if not exists dbus keep 36500 precision 'us' ") tdSql.execute("use dbus") -<<<<<<< HEAD tsp2 = tsp2 * 1000 tsp3 = tsp3 * 1000 -======= - tsp2 = -28800000 * 1000 - tsp3 = -946800000000 * 1000 ->>>>>>> bbaf30ed9130285ad72dbfd58e1e4b7adbfd1d21 self.insertnow(tsp1,tsp2,tsp3) self.querynow() - -<<<<<<< HEAD -======= - - - ->>>>>>> bbaf30ed9130285ad72dbfd58e1e4b7adbfd1d21 + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) From f2e6135967dc55f3218b71da82aa37ed9e80f13d Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Fri, 23 Apr 2021 19:04:01 +0800 Subject: [PATCH 09/27] fix bug --- src/client/src/tscStream.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 3ec2c5336a..b751aea364 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -203,6 +203,14 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf tNameExtractFullName(&pTableMetaInfo->name, name); taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + + tfree(pTableMetaInfo->pTableMeta); + + tscFreeSqlResult(pStream->pSql); + tscFreeSubobj(pStream->pSql); + tfree(pStream->pSql->pSubs); + pStream->pSql->subState.numOfSub = 0; + pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); tscSetRetryTimer(pStream, pStream->pSql, retryDelay); From 68b3629efa1c6b545f5ca01cf1b5b0d8c2ef65f6 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sun, 25 Apr 2021 11:05:31 +0800 Subject: [PATCH 10/27] [TD-3850]: fix vnode write enqueue flowctrl UAF & no response --- src/vnode/src/vnodeWrite.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index aab685e678..def9cf3b32 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -347,9 +347,11 @@ static void vnodeFlowCtrlMsgToWQueue(void *param, void *tmrId) { vDebug("vgId:%d, msg:%p, write into vwqueue after flowctrl, retry:%d", pVnode->vgId, pWrite, pWrite->processedCount); pWrite->processedCount = 0; + void *handle = pWrite->rpcMsg.handle; code = vnodeWriteToWQueueImp(pWrite); - if (code != 0) { - dnodeSendRpcVWriteRsp(pWrite->pVnode, pWrite, code); + if (code != TSDB_CODE_SUCCESS) { + SRpcMsg rpcRsp = {.handle = handle, .code = code}; + rpcSendResponse(&rpcRsp); } } } From 183e6ff05e02cbcbd66664e5a7317fa09635b493 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 25 Apr 2021 13:23:59 +0800 Subject: [PATCH 11/27] Hotfix/sangshuduo/td 3401 query statistic (#5907) * [TD-3401]: taosdemo query statistic. refactor func name. * [TD-3401]: taosdemo query statistic. refactor func name 2. * [TD-3401]: taosdemo support query statistic. implementation. * cleanup * [TD-3401]: taosdemo query statistic. change us to ms. * [TD-3401]: taosdemo query statistic. increase sql buffer for query. * [TD-3401]: taosdemo query statistic more accurate result. * [TD-3401]: taosdemo query statistic. modify last time logic. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 3be8141bfe..8fb7d79d70 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -5992,8 +5992,8 @@ static void *specifiedTableQuery(void *sarg) { pThreadInfo->threadID, totalQueried, (double)(totalQueried/((endTs-startTs)/1000.0))); + lastPrintTime = currentPrintTime; } - lastPrintTime = currentPrintTime; } return NULL; } @@ -6078,8 +6078,8 @@ static void *superTableQuery(void *sarg) { pThreadInfo->threadID, totalQueried, (double)(totalQueried/((endTs-startTs)/1000.0))); + lastPrintTime = currentPrintTime; } - lastPrintTime = currentPrintTime; } } et = taosGetTimestampMs(); From 07cded1fad23c8f8dc31284c10be675d0d789b68 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Sun, 25 Apr 2021 15:16:39 +0800 Subject: [PATCH 12/27] improve coverage of operations --- tests/pytest/functions/function_operations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytest/functions/function_operations.py b/tests/pytest/functions/function_operations.py index 162aa3eb65..859cd78a3d 100644 --- a/tests/pytest/functions/function_operations.py +++ b/tests/pytest/functions/function_operations.py @@ -82,14 +82,14 @@ class TDTestCase: self.ts = self.ts + self.rowNum + 10 - tdSql.execute("insert into test1 values(%d, 1, 1, 1, 1, 1.1, 1.1, 1, NULL, '涛思数据3', 1, 1, 1, 1)" % ( self.ts + self.rowNum + 1 )) + tdSql.execute("insert into test1 values(%d, 1, 1, 1, 1, 1.1, 1.1, 1, NULL, '涛思数据3', 254, 65534, 4294967294, 18446744073709551614)" % ( self.ts + self.rowNum + 1 )) tdSql.execute("insert into test1 values(%d, 1, 1, 1, 1, 1.1, 1.1, 1, 'taosdata', NULL, 1, 1, 1, 1)" % ( self.ts + self.rowNum + 2 )) tdSql.execute("insert into test1 values(%d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)" % ( self.ts + self.rowNum + 3 )) tdSql.execute("insert into test1 values(%d, 1, 1, 1, 1, NULL, 1.1, 1, NULL, '涛思数据3', 1, 1, 1, 1)" % ( self.ts + self.rowNum + 4 )) tdSql.execute("insert into test1 values(%d, 1, 1, 1, 1, 1.1, NULL, 1, 'taosdata', NULL, 1, 1, 1, 1)" % ( self.ts + self.rowNum + 5 )) self.rowNum = self.rowNum + 5 - col_list = [ 'col1' , 'col2' , 'col3' , 'col4' , 'col5' , 'col6' , 'col7' , 'col8' , 'col9' , 'col11' , 'col12' , 'col13' , 'col14' , '1' , '1.1' , 'NULL' ] + col_list = [ 'col1' , 'col2' , 'col3' , 'col4' , 'col5' , 'col6' , 'col7' , 'col8' , 'col9' , 'col11' , 'col12' , 'col13' , 'col14' , '1' , '1.1' , 'NULL' , '18446744073709551614' ] op_list = [ '+' , '-' , '*' , '/' , '%' ] err_list = [ 'col7' , 'col8' , 'col9' , 'NULL' ] order_lsit = [ ' order by ts ', ' order by ts desc ', ' order by ts asc '] From c8ea62a676b2125d4464f373ef73d958133fe35e Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 25 Apr 2021 18:16:48 +0800 Subject: [PATCH 13/27] [TD-3944]: make default offline threshold to 10 days. (#5912) Co-authored-by: Shuduo Sang --- packaging/cfg/taos.cfg | 2 +- src/common/src/tglobal.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 83b70ed9f8..d3bd7510a3 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -64,7 +64,7 @@ # monitorInterval 30 # number of seconds allowed for a dnode to be offline, for cluster only -# offlineThreshold 8640000 +# offlineThreshold 864000 # RPC re-try timer, millisecond # rpcTimer 300 diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 69b01e6c08..c3c159ee45 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -139,7 +139,7 @@ int32_t tsTableIncStepPerVnode = TSDB_TABLES_STEP; int8_t tsEnableBalance = 1; int8_t tsAlternativeRole = 0; int32_t tsBalanceInterval = 300; // seconds -int32_t tsOfflineThreshold = 86400 * 100; // seconds 100 days +int32_t tsOfflineThreshold = 86400 * 10; // seconds of 10 days int32_t tsMnodeEqualVnodeNum = 4; int8_t tsEnableFlowCtrl = 1; int8_t tsEnableSlaveQuery = 1; From f52cf50c5c8007a8fc663b2ee4dcabd37af65280 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sun, 25 Apr 2021 14:22:09 +0800 Subject: [PATCH 14/27] [TD-3572]: response out of dnodes if #dnodes <= maxDbReplica --- src/mnode/inc/mnodeDb.h | 1 + src/mnode/src/mnodeDb.c | 18 ++++++++++++++++++ src/mnode/src/mnodeDnode.c | 9 +++++++++ tests/script/unique/cluster/balance2.sim | 23 +++++++++++++++++++---- tests/script/unique/dnode/remove1.sim | 4 ++-- tests/script/unique/dnode/remove2.sim | 21 ++++++++++++++++++++- 6 files changed, 69 insertions(+), 7 deletions(-) diff --git a/src/mnode/inc/mnodeDb.h b/src/mnode/inc/mnodeDb.h index d03ba8d717..da0865833d 100644 --- a/src/mnode/inc/mnodeDb.h +++ b/src/mnode/inc/mnodeDb.h @@ -31,6 +31,7 @@ enum _TSDB_DB_STATUS { int32_t mnodeInitDbs(); void mnodeCleanupDbs(); int64_t mnodeGetDbNum(); +int32_t mnodeGetDbMaxReplica(); SDbObj *mnodeGetDb(char *db); SDbObj *mnodeGetDbByTableName(char *db); void * mnodeGetNextDb(void *pIter, SDbObj **pDb); diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 909ca7cac6..8af20aa862 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -74,6 +74,24 @@ int64_t mnodeGetDbNum() { return sdbGetNumOfRows(tsDbSdb); } +int32_t mnodeGetDbMaxReplica() { + int32_t maxReplica = 0; + SDbObj *pDb = NULL; + void *pIter = NULL; + + while (1) { + pIter = mnodeGetNextDb(pIter, &pDb); + if (pDb == NULL) break; + + if (pDb->cfg.replications > maxReplica) + maxReplica = pDb->cfg.replications; + + mnodeDecDbRef(pDb); + } + + return maxReplica; +} + static int32_t mnodeDbActionInsert(SSdbRow *pRow) { SDbObj *pDb = pRow->pObj; SAcctObj *pAcct = mnodeGetAcct(pDb->acct); diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 85d9f94b88..b513da29f4 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -29,6 +29,7 @@ #include "mnodeDef.h" #include "mnodeInt.h" #include "mnodeDnode.h" +#include "mnodeDb.h" #include "mnodeMnode.h" #include "mnodeSdb.h" #include "mnodeShow.h" @@ -745,6 +746,14 @@ static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) { return TSDB_CODE_MND_NO_REMOVE_MASTER; } + int32_t maxReplica = mnodeGetDbMaxReplica(); + int32_t dnodesNum = mnodeGetDnodesNum(); + if (dnodesNum <= maxReplica) { + mError("dnode:%d, can't drop dnode:%s, #dnodes: %d, replia: %d", pDnode->dnodeId, ep, dnodesNum, maxReplica); + mnodeDecDnodeRef(pDnode); + return TSDB_CODE_MND_NO_ENOUGH_DNODES; + } + mInfo("dnode:%d, start to drop it", pDnode->dnodeId); int32_t code = bnDropDnode(pDnode); diff --git a/tests/script/unique/cluster/balance2.sim b/tests/script/unique/cluster/balance2.sim index 026678af7c..0b80acbe6c 100644 --- a/tests/script/unique/cluster/balance2.sim +++ b/tests/script/unique/cluster/balance2.sim @@ -338,10 +338,6 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT print stop dnode1 and sleep 3000 sleep 3000 -sql drop dnode $hostname1 -print drop dnode1 and sleep 9000 -sleep 9000 - sql show mnodes $dnode1Role = $data2_1 $dnode4Role = $data2_4 @@ -357,6 +353,25 @@ endi print ============================== step6.1 system sh/exec.sh -n dnode1 -s start +$x = 0 +step6.1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 + +if $data4_1 != ready then + goto step6.1 +endi + +sql drop dnode $hostname1 +print drop dnode1 and sleep 9000 +sleep 9000 + $x = 0 show6: $x = $x + 1 diff --git a/tests/script/unique/dnode/remove1.sim b/tests/script/unique/dnode/remove1.sim index 6f830d2cf8..25e0846129 100644 --- a/tests/script/unique/dnode/remove1.sim +++ b/tests/script/unique/dnode/remove1.sim @@ -97,7 +97,6 @@ if $data2_2 != 3 then endi print ========== step3 -sql drop dnode $hostname2 $x = 0 show3: @@ -114,6 +113,7 @@ print dnode2 openVnodes $data2_2 print ========== step4 sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start +sql drop dnode $hostname2 $x = 0 show4: @@ -224,4 +224,4 @@ system sh/exec.sh -n dnode4 -s stop -x SIGINT system sh/exec.sh -n dnode5 -s stop -x SIGINT system sh/exec.sh -n dnode6 -s stop -x SIGINT system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode8 -s stop -x SIGINT diff --git a/tests/script/unique/dnode/remove2.sim b/tests/script/unique/dnode/remove2.sim index ff92ff7eb0..1d707bc4a3 100644 --- a/tests/script/unique/dnode/remove2.sim +++ b/tests/script/unique/dnode/remove2.sim @@ -98,7 +98,6 @@ endi print ========== step3 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sql drop dnode $hostname2 sql show dnodes print dnode1 openVnodes $data2_1 @@ -128,6 +127,26 @@ endi print ============ step 4.1 system sh/exec.sh -n dnode2 -s start +$x = 0 +step4.1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_2 != ready then + goto step4.1 +endi + +sql drop dnode $hostname2 + $x = 0 show4: $x = $x + 1 From 97c983bb5ad3b15303b4611969c9644ae133764a Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Sun, 25 Apr 2021 17:16:56 +0800 Subject: [PATCH 15/27] [TD-3937]: add taosdemo performance test compare scripts --- .../perftest-taosdemo-compare.sh | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100755 tests/perftest-scripts/perftest-taosdemo-compare.sh diff --git a/tests/perftest-scripts/perftest-taosdemo-compare.sh b/tests/perftest-scripts/perftest-taosdemo-compare.sh new file mode 100755 index 0000000000..f0ce4d6741 --- /dev/null +++ b/tests/perftest-scripts/perftest-taosdemo-compare.sh @@ -0,0 +1,147 @@ +#!/bin/bash + +WORK_DIR=/home/ubuntu/pxiao +TDENGINE_DIR=/home/ubuntu/pxiao/TDengine +NUM_OF_VERSIONS=5 +CURRENT_VERSION=0 +today=`date +"%Y%m%d"` +TAOSDEMO_COMPARE_TEST_REPORT=$TDENGINE_DIR/tests/taosdemo-compare-test-report-$today.log + +# Coloured Echoes +function red_echo { echo -e "\033[31m$@\033[0m"; } +function green_echo { echo -e "\033[32m$@\033[0m"; } +function yellow_echo { echo -e "\033[33m$@\033[0m"; } +function white_echo { echo -e "\033[1;37m$@\033[0m"; } +# Coloured Printfs +function red_printf { printf "\033[31m$@\033[0m"; } +function green_printf { printf "\033[32m$@\033[0m"; } +function yellow_printf { printf "\033[33m$@\033[0m"; } +function white_printf { printf "\033[1;37m$@\033[0m"; } +# Debugging Outputs +function white_brackets { local args="$@"; white_printf "["; printf "${args}"; white_printf "]"; } +function echoInfo { local args="$@"; white_brackets $(green_printf "INFO") && echo " ${args}"; } +function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf "WARN")" && echo " ${args}";)" 1>&2; } +function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } + +function getCurrentVersion { + echoInfo "Build TDengine" + cd $WORK_DIR/TDengine + + git remote update > /dev/null + git reset --hard HEAD + git checkout master + REMOTE_COMMIT=`git rev-parse --short remotes/origin/master` + LOCAL_COMMIT=`git rev-parse --short @` + + echo " LOCAL: $LOCAL_COMMIT" + echo "REMOTE: $REMOTE_COMMIT" + if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then + echo "repo up-to-date" + else + echo "repo need to pull" + git pull > /dev/null 2>&1 + fi + cd debug + rm -rf * + cmake .. > /dev/null 2>&1 + make > /dev/null 2>&1 + make install > /dev/null 2>&1 + + rm -rf $WORK_DIR/taosdemo + cp -r $TDENGINE_DIR/src/kit/taosdemo $WORK_DIR + CURRENT_VERSION=`taosd -V | grep version | awk '{print $3}' | awk -F. '{print $3}'` +} + +function buildTDengineByVersion() { + echoInfo "build TDengine on branch: $1" + git reset --hard HEAD + git checkout $1 + git pull > /dev/null + + rm -rf $TDENGINE_DIR/src/kit/taosdemo + cp -r $WORK_DIR/taosdemo $TDENGINE_DIR/src/kit + + cd $TDENGINE_DIR/debug + rm -rf * + cmake .. > /dev/null 2>&1 + make > /dev/null 2>&1 + make install > /dev/null 2>&1 +} + +function stopTaosd { + echo "Stop taosd" + systemctl stop taosd + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done +} + +function startTaosd { + echo "Start taosd" + rm -rf /var/lib/perf/* + rm -rf /var/log/perf/* + nohup taosd -c /etc/perf/ > /dev/null 2>&1 & + sleep 10 +} + +function runTaosdemoCompare { + echoInfo "Stop Taosd" + stopTaosd + + getCurrentVersion + release="master" + + [ -f $TAOSDEMO_COMPARE_TEST_REPORT ] && rm $TAOSDEMO_COMPARE_TEST_REPORT + + for((i=0;i<$NUM_OF_VERSIONS;i++)) + do + startTaosd + taos -s "drop database if exists demodb;" + taosdemo -y -d demodb > taosdemoperf.txt + + echoInfo "==================== taosdemo performance for $release ====================" | tee -a $TAOSDEMO_COMPARE_TEST_REPORT + CREATE_TABLE_TIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'` + INSERT_RECORDS_TIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'` + RECORDS_PER_SECOND=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $16}'` + AVG_DELAY=`grep 'delay' taosdemoperf.txt | awk '{print $4}' | awk -Fm '{print $1}'` + MAX_DELAY=`grep 'delay' taosdemoperf.txt | awk '{print $6}' | awk -Fm '{print $1}'` + MIN_DELAY=`grep 'delay' taosdemoperf.txt | awk '{print $8}' | awk -Fm '{print $1}'` + + echo "create table time: $CREATE_TABLE_TIME seconds" | tee -a $TAOSDEMO_COMPARE_TEST_REPORT + echo "insert records time: $INSERT_RECORDS_TIME seconds" | tee -a $TAOSDEMO_COMPARE_TEST_REPORT + echo "records per second: $RECORDS_PER_SECOND records/second" | tee -a $TAOSDEMO_COMPARE_TEST_REPORT + echo "avg delay: $AVG_DELAY ms" | tee -a $TAOSDEMO_COMPARE_TEST_REPORT + echo "max delay: $MAX_DELAY ms" | tee -a $TAOSDEMO_COMPARE_TEST_REPORT + echo "min delay: $MIN_DELAY ms" | tee -a $TAOSDEMO_COMPARE_TEST_REPORT + + [ -f taosdemoperf.txt ] && rm taosdemoperf.txt + + stopTaosd + version=`expr $CURRENT_VERSION - $i` + release="release/s1$version" + buildTDengineByVersion $release + done +} + +function sendReport { + echo "send report" + receiver="develop@taosdata.com" + mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n" + + cd $TDENGINE_DIR + + sed -i 's/\x1b\[[0-9;]*m//g' $TAOSDEMO_COMPARE_TEST_REPORT + BODY_CONTENT=`cat $TAOSDEMO_COMPARE_TEST_REPORT` + echo -e "to: ${receiver}\nsubject: Coverage test report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \ + (cat - && uuencode $TAOSDEMO_COMPARE_TEST_REPORT taosdemo-compare-test-report-$today.log) | \ + ssmtp "${receiver}" && echo "Report Sent!" +} + +runTaosdemoCompare +sendReport + +echoInfo "End of Taosdemo Compare Test" | tee -a $WORK_DIR/cron.log \ No newline at end of file From 4415eb2114749252cd59d3d9881dffa4770ce93a Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Mon, 26 Apr 2021 09:33:56 +0800 Subject: [PATCH 16/27] update script --- tests/perftest-scripts/perftest-taosdemo-compare.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/perftest-scripts/perftest-taosdemo-compare.sh b/tests/perftest-scripts/perftest-taosdemo-compare.sh index f0ce4d6741..60b6d1310d 100755 --- a/tests/perftest-scripts/perftest-taosdemo-compare.sh +++ b/tests/perftest-scripts/perftest-taosdemo-compare.sh @@ -103,7 +103,7 @@ function runTaosdemoCompare { taos -s "drop database if exists demodb;" taosdemo -y -d demodb > taosdemoperf.txt - echoInfo "==================== taosdemo performance for $release ====================" | tee -a $TAOSDEMO_COMPARE_TEST_REPORT + echo "==================== taosdemo performance for $release ====================" | tee -a $TAOSDEMO_COMPARE_TEST_REPORT CREATE_TABLE_TIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'` INSERT_RECORDS_TIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'` RECORDS_PER_SECOND=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $16}'` @@ -136,7 +136,7 @@ function sendReport { sed -i 's/\x1b\[[0-9;]*m//g' $TAOSDEMO_COMPARE_TEST_REPORT BODY_CONTENT=`cat $TAOSDEMO_COMPARE_TEST_REPORT` - echo -e "to: ${receiver}\nsubject: Coverage test report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \ + echo -e "to: ${receiver}\nsubject: taosdemo performance compare test report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \ (cat - && uuencode $TAOSDEMO_COMPARE_TEST_REPORT taosdemo-compare-test-report-$today.log) | \ ssmtp "${receiver}" && echo "Report Sent!" } From f79408eedb64f76469c220a2097f18dff23be87b Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 26 Apr 2021 14:05:45 +0800 Subject: [PATCH 17/27] Hotfix/sangshuduo/td 3317 for master (#5921) * [TD-3317]: taosdemo interlace insertion. patch for master. * [TD-3317]: taosdemo support interlace mode. adjust remainder rows logic. * [TD-3317]: taosdemo support interlace mode. fix global and stable interlace rows logic. * [TD-3317]: taosdemo support interlace mode. fix 'interlaceRows' is used uninitialized Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 45 +++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 8fb7d79d70..68292692fa 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -252,7 +252,7 @@ typedef struct SSuperTable_S { int maxSqlLen; // int insertInterval; // insert interval, will override global insert interval - int64_t insertRows; // 0: no limit + int64_t insertRows; int timeStampStep; char startTimestamp[MAX_TB_NAME_SIZE]; char sampleFormat[MAX_TB_NAME_SIZE]; // csv, json @@ -4868,6 +4868,8 @@ static int generateInterlaceDataBuffer( pstr += dataLen; *pRemainderBufLen -= dataLen; } else { + debugPrint("%s() LN%d, generated data tail: %d, not equal batch per table: %d\n", + __func__, __LINE__, k, batchPerTbl); pstr -= headLen; pstr[0] = '\0'; k = 0; @@ -4925,13 +4927,27 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { debugPrint("[%d] %s() LN%d: ### interlace write\n", pThreadInfo->threadID, __func__, __LINE__); + int64_t insertRows; + int interlaceRows; + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; - int interlaceRows = superTblInfo?superTblInfo->interlaceRows:g_args.interlace_rows; + if (superTblInfo) { + insertRows = superTblInfo->insertRows; + + if ((superTblInfo->interlaceRows == 0) + && (g_args.interlace_rows > 0)) { + interlaceRows = g_args.interlace_rows; + } else { + interlaceRows = superTblInfo->interlaceRows; + } + } else { + insertRows = g_args.num_of_DPT; + interlaceRows = g_args.interlace_rows; + } if (interlaceRows > insertRows) - interlaceRows = insertRows; + interlaceRows = insertRows; if (interlaceRows > g_args.num_of_RPR) interlaceRows = g_args.num_of_RPR; @@ -5063,15 +5079,15 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { if (generatedRecPerTbl >= insertRows) break; + int remainRows = insertRows - generatedRecPerTbl; + if ((remainRows > 0) && (batchPerTbl > remainRows)) + batchPerTbl = remainRows; + if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR) break; } } - int remainRows = insertRows - generatedRecPerTbl; - if ((remainRows > 0) && (batchPerTbl > remainRows)) - batchPerTbl = remainRows; - verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%d insertRows=%"PRId64"\n", pThreadInfo->threadID, __func__, __LINE__, generatedRecPerTbl, insertRows); @@ -5294,7 +5310,18 @@ static void* syncWrite(void *sarg) { threadInfo *pThreadInfo = (threadInfo *)sarg; SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - int interlaceRows = superTblInfo?superTblInfo->interlaceRows:g_args.interlace_rows; + int interlaceRows; + + if (superTblInfo) { + if ((superTblInfo->interlaceRows == 0) + && (g_args.interlace_rows > 0)) { + interlaceRows = g_args.interlace_rows; + } else { + interlaceRows = superTblInfo->interlaceRows; + } + } else { + interlaceRows = g_args.interlace_rows; + } if (interlaceRows > 0) { // interlace mode From a10fa312679bffb159babc398c506de4a002a3f0 Mon Sep 17 00:00:00 2001 From: wu champion Date: Mon, 26 Apr 2021 14:19:06 +0800 Subject: [PATCH 18/27] [TD-3920] add case to fix the bug of TD-3920 --- tests/pytest/fulltest.sh | 1 + tests/pytest/stream/cqSupportBefore1970.py | 93 ++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 tests/pytest/stream/cqSupportBefore1970.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index d47eca13e5..d2d706213d 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -239,6 +239,7 @@ python3 ./test.py -f stream/history.py python3 ./test.py -f stream/sys.py python3 ./test.py -f stream/table_1.py python3 ./test.py -f stream/table_n.py +python3 ./test.py -f stream/cqSupportBefore1970.py #alter table python3 ./test.py -f alter/alter_table_crash.py diff --git a/tests/pytest/stream/cqSupportBefore1970.py b/tests/pytest/stream/cqSupportBefore1970.py new file mode 100644 index 0000000000..75587d1743 --- /dev/null +++ b/tests/pytest/stream/cqSupportBefore1970.py @@ -0,0 +1,93 @@ +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug(f"start to execute {__file__}") + tdSql.init(conn.cursor(), logSql) + + def insertnow(self): + + # timestamp list: + # 0 -> "1970-01-01 08:00:00" | -28800000 -> "1970-01-01 00:00:00" | -946800000000 -> "1940-01-01 00:00:00" + # -631180800000 -> "1950-01-01 00:00:00" + + tsp1 = 0 + tsp2 = -28800000 + tsp3 = -946800000000 + tsp4 = "1969-01-01 00:00:00.000" + + tdSql.execute("insert into tcq1 values (now-11d, 5)") + tdSql.execute(f"insert into tcq1 values ({tsp1}, 4)") + tdSql.execute(f"insert into tcq1 values ({tsp2}, 3)") + tdSql.execute(f"insert into tcq1 values ('{tsp4}', 2)") + tdSql.execute(f"insert into tcq1 values ({tsp3}, 1)") + + def waitedQuery(self, sql, expectRows, timeout): + tdLog.info(f"sql: {sql}, try to retrieve {expectRows} rows in {timeout} seconds") + try: + for i in range(timeout): + tdSql.cursor.execute(sql) + self.queryResult = tdSql.cursor.fetchall() + self.queryRows = len(self.queryResult) + self.queryCols = len(tdSql.cursor.description) + # tdLog.info("sql: %s, try to retrieve %d rows,get %d rows" % (sql, expectRows, self.queryRows)) + if self.queryRows >= expectRows: + return (self.queryRows, i) + time.sleep(1) + except Exception as e: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + tdLog.notice(f"{caller.filename}({caller.lineno}) failed: sql:{sql}, {repr(e)}") + raise Exception(repr(e)) + return (self.queryRows, timeout) + + def cq(self): + tdSql.execute( + "create table cq1 as select avg(c1) from tcq1 where ts > -946800000000 interval(10d) sliding(1d)" + ) + self.waitedQuery("select * from cq1", 1, 120) + + def querycq(self): + tdSql.query("select * from cq1") + tdSql.checkData(0, 1, 1.0) + tdSql.checkData(10, 1, 2.0) + + def run(self): + tdSql.execute("drop database if exists dbcq") + tdSql.execute("create database if not exists dbcq keep 36500") + tdSql.execute("use dbcq") + + tdSql.execute("create table stbcq (ts timestamp, c1 int ) TAGS(t1 int)") + tdSql.execute("create table tcq1 using stbcq tags(1)") + + self.insertnow() + self.cq() + self.querycq() + + # after wal and sync, check again + tdSql.query("show dnodes") + index = tdSql.getData(0, 0) + tdDnodes.stop(index) + tdDnodes.start(index) + + self.querycq() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 9b7aeff6021f188c14e56ac750293c9224e3b0d6 Mon Sep 17 00:00:00 2001 From: wu champion Date: Mon, 26 Apr 2021 15:34:00 +0800 Subject: [PATCH 19/27] [TD-3921] add case to resolv show streams return erro msg --- tests/pytest/fulltest.sh | 1 + .../pytest/stream/showStreamExecTimeisNull.py | 97 +++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 tests/pytest/stream/showStreamExecTimeisNull.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index d47eca13e5..41b4043f64 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -239,6 +239,7 @@ python3 ./test.py -f stream/history.py python3 ./test.py -f stream/sys.py python3 ./test.py -f stream/table_1.py python3 ./test.py -f stream/table_n.py +python3 ./test.py -f stream/showStreamExecTimeisNull.py #alter table python3 ./test.py -f alter/alter_table_crash.py diff --git a/tests/pytest/stream/showStreamExecTimeisNull.py b/tests/pytest/stream/showStreamExecTimeisNull.py new file mode 100644 index 0000000000..39b0259018 --- /dev/null +++ b/tests/pytest/stream/showStreamExecTimeisNull.py @@ -0,0 +1,97 @@ +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug(f"start to execute {__file__}") + tdSql.init(conn.cursor(), logSql) + + def insertnow(self): + + # timestamp list: + # 0 -> "1970-01-01 08:00:00" | -28800000 -> "1970-01-01 00:00:00" | -946800000000 -> "1940-01-01 00:00:00" + # -631180800000 -> "1950-01-01 00:00:00" + + tsp1 = 0 + tsp2 = -28800000 + tsp3 = -946800000000 + tsp4 = "1969-01-01 00:00:00.000" + + tdSql.execute("insert into tcq1 values (now-11d, 5)") + tdSql.execute(f"insert into tcq1 values ({tsp1}, 4)") + tdSql.execute(f"insert into tcq1 values ({tsp2}, 3)") + tdSql.execute(f"insert into tcq1 values ('{tsp4}', 2)") + tdSql.execute(f"insert into tcq1 values ({tsp3}, 1)") + + def waitedQuery(self, sql, expectRows, timeout): + tdLog.info(f"sql: {sql}, try to retrieve {expectRows} rows in {timeout} seconds") + try: + for i in range(timeout): + tdSql.cursor.execute(sql) + self.queryResult = tdSql.cursor.fetchall() + self.queryRows = len(self.queryResult) + self.queryCols = len(tdSql.cursor.description) + # tdLog.info("sql: %s, try to retrieve %d rows,get %d rows" % (sql, expectRows, self.queryRows)) + if self.queryRows >= expectRows: + return (self.queryRows, i) + time.sleep(1) + except Exception as e: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + tdLog.notice(f"{caller.filename}({caller.lineno}) failed: sql:{sql}, {repr(e)}") + raise Exception(repr(e)) + return (self.queryRows, timeout) + + def showstream(self): + tdSql.execute( + "create table cq1 as select avg(c1) from tcq1 interval(10d) sliding(1d)" + ) + sql = "show streams" + timeout = 30 + exception = "ValueError('year -292275055 is out of range')" + try: + for i in range(timeout): + tdSql.cursor.execute(sql) + self.queryResult = tdSql.cursor.fetchall() + self.queryRows = len(self.queryResult) + self.queryCols = len(tdSql.cursor.description) + # tdLog.info("sql: %s, try to retrieve %d rows,get %d rows" % (sql, expectRows, self.queryRows)) + if self.queryRows >= timeout: + return (self.queryRows, i) + time.sleep(1) + except Exception as e: + tdLog.info(f"sql: {sql} except raise {exception}, actually raise {repr(e)} ") + else: + tdLog.exit(f"sql: {sql} except raise {exception}, actually not") + + + def run(self): + tdSql.execute("drop database if exists dbcq") + tdSql.execute("create database if not exists dbcq keep 36500") + tdSql.execute("use dbcq") + + tdSql.execute("create table stbcq (ts timestamp, c1 int ) TAGS(t1 int)") + tdSql.execute("create table tcq1 using stbcq tags(1)") + + self.insertnow() + self.showstream() + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 19fa9097a12267a04b33c549d28b9de7420749e7 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 26 Apr 2021 15:34:39 +0800 Subject: [PATCH 20/27] [TD-3968]: taosdemo data length should be 16*1024 (#5926) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 68292692fa..83ecd145a0 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -81,7 +81,7 @@ enum QUERY_MODE { #define MAX_DB_NAME_SIZE 64 #define MAX_HOSTNAME_SIZE 64 #define MAX_TB_NAME_SIZE 64 -#define MAX_DATA_SIZE 16000 +#define MAX_DATA_SIZE (16*1024) #define MAX_NUM_DATATYPE 10 #define OPT_ABORT 1 /* –abort */ #define STRING_LEN 60000 From 1779c952fd086c9382ccd198b8131122929f8d4d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 26 Apr 2021 17:45:54 +0800 Subject: [PATCH 21/27] [td-3951]: check the input value of big int which is overflowed. --- src/common/src/tvariant.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 7798deaa60..c872d8731b 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -48,6 +48,13 @@ void tVariantCreate(tVariant *pVar, SStrToken *token) { case TSDB_DATA_TYPE_INT:{ ret = tStrToInteger(token->z, token->type, token->n, &pVar->i64, true); if (ret != 0) { + SStrToken t = {0}; + tSQLGetToken(token->z, &t.type); + if (t.type == TK_MINUS) { // it is a signed number which is greater than INT64_MAX or less than INT64_MIN + pVar->nType = -1; // -1 means error type + return; + } + // data overflow, try unsigned parse the input number ret = tStrToInteger(token->z, token->type, token->n, &pVar->i64, false); if (ret != 0) { From 6ac7b467e43ff7d3c8f20431f1b65cc615d547fc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 26 Apr 2021 17:52:22 +0800 Subject: [PATCH 22/27] [td-225]update the log output --- src/client/src/tscParseInsert.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 00960b9fc0..d21a4f7643 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1089,7 +1089,7 @@ int tsParseInsertSql(SSqlObj *pSql) { str = pCmd->curSql; } - tscDebug("0x%"PRIx64" create data block list hashList:%p", pSql->self, pCmd->pTableBlockHashList); + tscDebug("0x%lx create data block list hashList:%p", pSql->self, pCmd->pTableBlockHashList); while (1) { int32_t index = 0; From 48b630a1169f977815ce05db2ae7b707f1a0d6b6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 26 Apr 2021 18:14:04 +0800 Subject: [PATCH 23/27] [td-225]fix compiler error. --- src/client/src/tscParseInsert.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index d21a4f7643..00960b9fc0 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1089,7 +1089,7 @@ int tsParseInsertSql(SSqlObj *pSql) { str = pCmd->curSql; } - tscDebug("0x%lx create data block list hashList:%p", pSql->self, pCmd->pTableBlockHashList); + tscDebug("0x%"PRIx64" create data block list hashList:%p", pSql->self, pCmd->pTableBlockHashList); while (1) { int32_t index = 0; From 127dd1e86870f51e746d460328b9d6fd7d4f08fe Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 26 Apr 2021 22:23:29 +0800 Subject: [PATCH 24/27] [TD-3969]: taosdemo re-enable batch-create-tbl-num (#5932) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 83ecd145a0..0fc8687b22 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -3657,7 +3657,6 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } - /* cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num"); if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) { g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint; @@ -3667,7 +3666,6 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { printf("ERROR: failed to read json, batch_create_tbl_num not found\n"); goto PARSE_OVER; } - */ cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no if (childTblExists From dceb51d4463c8d20319c490fb1ef92b0f72bb9dc Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Tue, 27 Apr 2021 17:49:48 +0800 Subject: [PATCH 25/27] print rid instead of pointer --- src/client/src/tscAsync.c | 8 ++-- src/client/src/tscLocal.c | 2 +- src/client/src/tscLocalMerge.c | 16 +++---- src/client/src/tscParseInsert.c | 6 +-- src/client/src/tscProfile.c | 2 +- src/client/src/tscSQLParser.c | 2 +- src/client/src/tscServer.c | 78 +++++++++++++++---------------- src/client/src/tscSql.c | 13 +++--- src/client/src/tscStream.c | 24 +++++----- src/client/src/tscSub.c | 4 +- src/client/src/tscSubquery.c | 82 ++++++++++++++++----------------- src/client/src/tscUtil.c | 14 +++--- 12 files changed, 125 insertions(+), 126 deletions(-) diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index df13ca45fd..3f2a54fbdc 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -49,7 +49,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* para pSql->sqlstr = calloc(1, sqlLen + 1); if (pSql->sqlstr == NULL) { - tscError("%p failed to malloc sql string buffer", pSql); + tscError("0x%"PRIx64" failed to malloc sql string buffer", pSql->self); pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; tscAsyncResultOnError(pSql); return; @@ -80,7 +80,7 @@ void taos_query_a(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *pa TAOS_RES * taos_query_ra(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param) { STscObj *pObj = (STscObj *)taos; if (pObj == NULL || pObj->signature != pObj) { - tscError("bug!!! pObj:%p", pObj); + tscError("pObj:%p is NULL or freed", pObj); terrno = TSDB_CODE_TSC_DISCONNECTED; tscQueueAsyncError(fp, param, TSDB_CODE_TSC_DISCONNECTED); return NULL; @@ -288,7 +288,7 @@ static void tscAsyncResultCallback(SSchedMsg *pMsg) { } assert(pSql->res.code != TSDB_CODE_SUCCESS); - tscError("%p invoke user specified function due to error occurred, code:%s", pSql, tstrerror(pSql->res.code)); + tscError("0x%"PRIx64" async result callback, code:%s", pSql->self, tstrerror(pSql->res.code)); SSqlRes *pRes = &pSql->res; if (pSql->fp == NULL || pSql->fetchFp == NULL){ @@ -368,7 +368,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { SSqlObj *sub = (SSqlObj*) res; const char* msg = (sub->cmd.command == TSDB_SQL_STABLEVGROUP)? "vgroup-list":"table-meta"; if (code != TSDB_CODE_SUCCESS) { - tscError("%p get %s failed, code:%s", pSql, msg, tstrerror(code)); + tscError("0x%"PRIx64" get %s failed, code:%s", pSql->self, msg, tstrerror(code)); goto _error; } diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 188ba29a97..f88f6546f9 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -926,7 +926,7 @@ int tscProcessLocalCmd(SSqlObj *pSql) { pRes->code = tscProcessServStatus(pSql); } else { pRes->code = TSDB_CODE_TSC_INVALID_SQL; - tscError("%p not support command:%d", pSql, pCmd->command); + tscError("0x%"PRIx64" not support command:%d", pSql->self, pCmd->command); } // keep the code in local variable in order to avoid invalid read in case of async query diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index bc49153092..97d52cc684 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -177,14 +177,14 @@ void tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrde if (pMemBuffer == NULL) { tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, pFFModel, numOfBuffer); - tscError("%p pMemBuffer is NULL", pMemBuffer); + tscError("pMemBuffer:%p is NULL", pMemBuffer); pRes->code = TSDB_CODE_TSC_APP_ERROR; return; } if (pDesc->pColumnModel == NULL) { tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, pFFModel, numOfBuffer); - tscError("%p no local buffer or intermediate result format model", pSql); + tscError("0x%"PRIx64" no local buffer or intermediate result format model", pSql->self); pRes->code = TSDB_CODE_TSC_APP_ERROR; return; } @@ -208,7 +208,7 @@ void tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrde } if (pDesc->pColumnModel->capacity >= pMemBuffer[0]->pageSize) { - tscError("%p Invalid value of buffer capacity %d and page size %d ", pSql, pDesc->pColumnModel->capacity, + tscError("0x%"PRIx64" Invalid value of buffer capacity %d and page size %d ", pSql->self, pDesc->pColumnModel->capacity, pMemBuffer[0]->pageSize); tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, pFFModel, numOfBuffer); @@ -220,7 +220,7 @@ void tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrde SLocalMerger *pReducer = (SLocalMerger *) calloc(1, size); if (pReducer == NULL) { - tscError("%p failed to create local merge structure, out of memory", pSql); + tscError("0x%"PRIx64" failed to create local merge structure, out of memory", pSql->self); tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, pFFModel, numOfBuffer); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -244,7 +244,7 @@ void tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrde for (int32_t j = 0; j < numOfFlushoutInFile; ++j) { SLocalDataSource *ds = (SLocalDataSource *)malloc(sizeof(SLocalDataSource) + pMemBuffer[0]->pageSize); if (ds == NULL) { - tscError("%p failed to create merge structure", pSql); + tscError("0x%"PRIx64" failed to create merge structure", pSql->self); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; tfree(pReducer); return; @@ -674,7 +674,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * pSql->subState.numOfSub); if (*pMemBuffer == NULL) { - tscError("%p failed to allocate memory", pSql); + tscError("0x%"PRIx64" failed to allocate memory", pSql->self); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; return pRes->code; } @@ -683,7 +683,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr pSchema = (SSchema *)calloc(1, sizeof(SSchema) * size); if (pSchema == NULL) { - tscError("%p failed to allocate memory", pSql); + tscError("0x%"PRIx64" failed to allocate memory", pSql->self); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; return pRes->code; } @@ -1529,7 +1529,7 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { return pRes->code; } - tscError("%p local merge abort due to error occurs, code:%s", pSql, tstrerror(pRes->code)); + tscError("0x%"PRIx64" local merge abort due to error occurs, code:%s", pSql->self, tstrerror(pRes->code)); return pRes->code; } diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 00960b9fc0..ae6e2430b7 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1141,7 +1141,7 @@ int tsParseInsertSql(SSqlObj *pSql) { return code; } - tscError("%p async insert parse error, code:%s", pSql, tstrerror(code)); + tscError("0x%"PRIx64" async insert parse error, code:%s", pSql->self, tstrerror(code)); pCmd->curSql = NULL; goto _clean; } @@ -1409,7 +1409,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRow assert(pSql->res.numOfRows == 0); int32_t ret = fseek(fp, 0, SEEK_SET); if (ret < 0) { - tscError("%p failed to seek SEEK_SET since:%s", pSql, tstrerror(errno)); + tscError("0x%"PRIx64" failed to seek SEEK_SET since:%s", pSql->self, tstrerror(errno)); code = TAOS_SYSTEM_ERROR(errno); goto _error; } @@ -1529,7 +1529,7 @@ void tscImportDataFromFile(SSqlObj *pSql) { FILE *fp = fopen(pCmd->payload, "rb"); if (fp == NULL) { pSql->res.code = TAOS_SYSTEM_ERROR(errno); - tscError("%p failed to open file %s to load data from file, code:%s", pSql, pCmd->payload, tstrerror(pSql->res.code)); + tscError("0x%"PRIx64" failed to open file %s to load data from file, code:%s", pSql->self, pCmd->payload, tstrerror(pSql->res.code)); tfree(pSupporter); taos_free_result(pNew); diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index 5c3c12caba..be304632cc 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -104,7 +104,7 @@ void tscSaveSlowQuery(SSqlObj *pSql) { char *sql = malloc(sqlSize); if (sql == NULL) { - tscError("%p failed to allocate memory to sent slow query to dnode", pSql); + tscError("0x%"PRIx64" failed to allocate memory to sent slow query to dnode", pSql->self); return; } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 106679b18a..751c4bf39f 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5187,7 +5187,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { int32_t size = sizeof(SUpdateTableTagValMsg) + pTagsSchema->bytes + schemaLen + TSDB_EXTRA_PAYLOAD_SIZE; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { - tscError("%p failed to malloc for alter table msg", pSql); + tscError("0x%"PRIx64" failed to malloc for alter table msg", pSql->self); return TSDB_CODE_TSC_OUT_OF_MEMORY; } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 449104de0f..194c7a0eca 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -221,7 +221,7 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { assert(online <= total); if (online < total) { - tscError("HB:%p, total dnode:%d, online dnode:%d", pSql, total, online); + tscError("0x%"PRIx64", HB, total dnode:%d, online dnode:%d", pSql->self, total, online); pSql->res.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; } @@ -273,7 +273,7 @@ void tscProcessActivityTimer(void *handle, void *tmrId) { taosReleaseRef(tscObjRef, pObj->hbrid); if (code != TSDB_CODE_SUCCESS) { - tscError("%p failed to sent HB to server, reason:%s", pHB, tstrerror(code)); + tscError("0x%"PRIx64" failed to sent HB to server, reason:%s", pHB->self, tstrerror(code)); } taosReleaseRef(tscRefId, rid); @@ -285,7 +285,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { char *pMsg = rpcMallocCont(pCmd->payloadLen); if (NULL == pMsg) { - tscError("%p msg:%s malloc failed", pSql, taosMsg[pSql->cmd.msgType]); + tscError("0x%"PRIx64" msg:%s malloc failed", pSql->self, taosMsg[pSql->cmd.msgType]); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -369,11 +369,11 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { rpcMsg->code == TSDB_CODE_APP_NOT_READY)) { pSql->retry++; - tscWarn("%p it shall renew table meta, code:%s, retry:%d", pSql, tstrerror(rpcMsg->code), pSql->retry); + tscWarn("0x%"PRIx64" it shall renew table meta, code:%s, retry:%d", pSql->self, tstrerror(rpcMsg->code), pSql->retry); pSql->res.code = rpcMsg->code; // keep the previous error code if (pSql->retry > pSql->maxRetry) { - tscError("%p max retry %d reached, give up", pSql, pSql->maxRetry); + tscError("0x%"PRIx64" max retry %d reached, give up", pSql->self, pSql->maxRetry); } else { // wait for a little bit moment and then retry // todo do not sleep in rpc callback thread, add this process into queueu to process @@ -664,7 +664,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char assert(index < pTableMetaInfo->vgroupList->numOfVgroups); pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index]; } else { - tscError("%p No vgroup info found", pSql); + tscError("0x%"PRIx64" No vgroup info found", pSql->self); *succeed = 0; return pMsg; @@ -733,7 +733,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t size = tscEstimateQueryMsgSize(pSql, pCmd->clauseIndex); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { - tscError("%p failed to malloc for query msg", pSql); + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); return TSDB_CODE_TSC_INVALID_SQL; // todo add test for this } @@ -743,19 +743,19 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList); if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo) && !tscQueryBlockInfo(pQueryInfo)) { - tscError("%p illegal value of numOfCols in query msg: %" PRIu64 ", table cols:%d", pSql, (uint64_t)numOfSrcCols, + tscError("0x%"PRIx64" illegal value of numOfCols in query msg: %" PRIu64 ", table cols:%d", pSql->self, (uint64_t)numOfSrcCols, tscGetNumOfColumns(pTableMeta)); return TSDB_CODE_TSC_INVALID_SQL; } if (pQueryInfo->interval.interval < 0) { - tscError("%p illegal value of aggregation time interval in query msg: %" PRId64, pSql, (int64_t)pQueryInfo->interval.interval); + tscError("0x%"PRIx64" illegal value of aggregation time interval in query msg: %" PRId64, pSql->self, (int64_t)pQueryInfo->interval.interval); return TSDB_CODE_TSC_INVALID_SQL; } if (pQueryInfo->groupbyExpr.numOfGroupCols < 0) { - tscError("%p illegal value of numOfGroupCols in query msg: %d", pSql, pQueryInfo->groupbyExpr.numOfGroupCols); + tscError("0x%"PRIx64" illegal value of numOfGroupCols in query msg: %d", pSql->self, pQueryInfo->groupbyExpr.numOfGroupCols); return TSDB_CODE_TSC_INVALID_SQL; } @@ -813,8 +813,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { tNameExtractFullName(&pTableMetaInfo->name, n); - tscError("%p tid:%d uid:%" PRIu64" id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s", - pSql, pTableMeta->id.tid, pTableMeta->id.uid, n, tscGetNumOfColumns(pTableMeta), pCol->colIndex.columnIndex, + tscError("0x%"PRIx64" tid:%d uid:%" PRIu64" id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s", + pSql->self, pTableMeta->id.tid, pTableMeta->id.uid, n, tscGetNumOfColumns(pTableMeta), pCol->colIndex.columnIndex, pColSchema->name); return TSDB_CODE_TSC_INVALID_SQL; } @@ -859,12 +859,12 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // the queried table has been removed and a new table with the same name has already been created already // return error msg if (pExpr->uid != pTableMeta->id.uid) { - tscError("%p table has already been destroyed", pSql); + tscError("0x%"PRIx64" table has already been destroyed", pSql->self); return TSDB_CODE_TSC_INVALID_TABLE_NAME; } if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId, pExpr->numOfParams)) { - tscError("%p table schema is not matched with parsed sql", pSql); + tscError("0x%"PRIx64" table schema is not matched with parsed sql", pSql->self); return TSDB_CODE_TSC_INVALID_SQL; } @@ -965,12 +965,12 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // the queried table has been removed and a new table with the same name has already been created already // return error msg if (pExpr->uid != pTableMeta->id.uid) { - tscError("%p table has already been destroyed", pSql); + tscError("0x%"PRIx64" table has already been destroyed", pSql->self); return TSDB_CODE_TSC_INVALID_TABLE_NAME; } if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId, pExpr->numOfParams)) { - tscError("%p table schema is not matched with parsed sql", pSql); + tscError("0x%"PRIx64" table schema is not matched with parsed sql", pSql->self); return TSDB_CODE_TSC_INVALID_SQL; } @@ -1076,8 +1076,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { char n[TSDB_TABLE_FNAME_LEN] = {0}; tNameExtractFullName(&pTableMetaInfo->name, n); - tscError("%p tid:%d uid:%" PRIu64 " id:%s, tag index out of range, totalCols:%d, numOfTags:%d, index:%d, column name:%s", - pSql, pTableMeta->id.tid, pTableMeta->id.uid, n, total, numOfTagColumns, pCol->colIndex.columnIndex, pColSchema->name); + tscError("0x%"PRIx64" tid:%d uid:%" PRIu64 " id:%s, tag index out of range, totalCols:%d, numOfTags:%d, index:%d, column name:%s", + pSql->self, pTableMeta->id.tid, pTableMeta->id.uid, n, total, numOfTagColumns, pCol->colIndex.columnIndex, pColSchema->name); return TSDB_CODE_TSC_INVALID_SQL; } @@ -1170,7 +1170,7 @@ int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; pCmd->payloadLen = sizeof(SCreateDnodeMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("%p failed to malloc for query msg", pSql); + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1188,7 +1188,7 @@ int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; pCmd->payloadLen = sizeof(SCreateAcctMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("%p failed to malloc for query msg", pSql); + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1234,7 +1234,7 @@ int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pCmd->payloadLen = sizeof(SCreateUserMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("%p failed to malloc for query msg", pSql); + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1273,7 +1273,7 @@ int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pCmd->payloadLen = sizeof(SDropDbMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("%p failed to malloc for query msg", pSql); + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1295,7 +1295,7 @@ int32_t tscBuildDropTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pCmd->payloadLen = sizeof(SCMDropTableMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("%p failed to malloc for query msg", pSql); + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1316,7 +1316,7 @@ int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pCmd->payloadLen = sizeof(SDropDnodeMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("%p failed to malloc for query msg", pSql); + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1337,7 +1337,7 @@ int32_t tscBuildDropUserAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pCmd->msgType = (pInfo->type == TSDB_SQL_DROP_USER)? TSDB_MSG_TYPE_CM_DROP_USER:TSDB_MSG_TYPE_CM_DROP_ACCT; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("%p failed to malloc for query msg", pSql); + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1352,7 +1352,7 @@ int32_t tscBuildUseDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pCmd->payloadLen = sizeof(SUseDbMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("%p failed to malloc for query msg", pSql); + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1369,7 +1369,7 @@ int32_t tscBuildSyncDbReplicaMsg(SSqlObj* pSql, SSqlInfo *pInfo) { pCmd->payloadLen = sizeof(SSyncDbMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("%p failed to malloc for query msg", pSql); + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1388,7 +1388,7 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pCmd->payloadLen = sizeof(SShowMsg) + 100; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("%p failed to malloc for query msg", pSql); + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1474,7 +1474,7 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // Reallocate the payload size size = tscEstimateCreateTableMsgLength(pSql, pInfo); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { - tscError("%p failed to malloc for create table msg", pSql); + tscError("0x%"PRIx64" failed to malloc for create table msg", pSql->self); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1573,7 +1573,7 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SAlterTableInfo *pAlterInfo = pInfo->pAlterInfo; int size = tscEstimateAlterTableMsgLength(pCmd); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { - tscError("%p failed to malloc for alter table msg", pSql); + tscError("0x%"PRIx64" failed to malloc for alter table msg", pSql->self); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1647,7 +1647,7 @@ int tscBuildRetrieveFromMgmtMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pCmd->payloadLen = sizeof(SRetrieveTableMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("%p failed to malloc for query msg", pSql); + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1753,7 +1753,7 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pCmd->payloadLen = sizeof(SConnectMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("%p failed to malloc for query msg", pSql); + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1893,7 +1893,7 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int size = numOfQueries * sizeof(SQueryDesc) + numOfStreams * sizeof(SStreamDesc) + sizeof(SHeartBeatMsg) + 100; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { pthread_mutex_unlock(&pObj->mutex); - tscError("%p failed to create heartbeat msg", pSql); + tscError("0x%"PRIx64" failed to create heartbeat msg", pSql->self); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1969,7 +1969,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) { - tscError("%p invalid table meta from mnode, name:%s", pSql, tNameGetTableName(&pTableMetaInfo->name)); + tscError("0x%"PRIx64" invalid table meta from mnode, name:%s", pSql->self, tNameGetTableName(&pTableMetaInfo->name)); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -2163,7 +2163,7 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { pInfo->vgroupList->numOfVgroups = pVgroupMsg->numOfVgroups; if (pInfo->vgroupList->numOfVgroups <= 0) { //tfree(pInfo->vgroupList); - tscError("%p empty vgroup info", pSql); + tscError("0x%"PRIx64" empty vgroup info", pSql->self); } else { for (int32_t j = 0; j < pInfo->vgroupList->numOfVgroups; ++j) { // just init, no need to lock @@ -2475,7 +2475,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code); static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); if (NULL == pNew) { - tscError("%p malloc failed for new sqlobj to get table meta", pSql); + tscError("0x%"PRIx64" malloc failed for new sqlobj to get table meta", pSql->self); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -2489,7 +2489,7 @@ static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaIn pNew->cmd.autoCreated = pSql->cmd.autoCreated; // create table if not exists if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { - tscError("%p malloc failed for payload to get table meta", pSql); + tscError("0x%"PRIx64" malloc failed for payload to get table meta", pSql->self); tscFreeSqlObj(pNew); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -2502,7 +2502,7 @@ static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaIn if (pSql->cmd.autoCreated) { int32_t code = copyTagData(&pNew->cmd.tagData, &pSql->cmd.tagData); if (code != TSDB_CODE_SUCCESS) { - tscError("%p malloc failed for new tag data to get table meta", pSql); + tscError("0x%"PRIx64" malloc failed for new tag data to get table meta", pSql->self); tscFreeSqlObj(pNew); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -2580,7 +2580,7 @@ int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) { char name[TSDB_TABLE_FNAME_LEN] = {0}; int32_t code = tNameExtractFullName(&pTableMetaInfo->name, name); if (code != TSDB_CODE_SUCCESS) { - tscError("%p failed to generate the table full name", pSql); + tscError("0x%"PRIx64" failed to generate the table full name", pSql->self); return TSDB_CODE_TSC_INVALID_SQL; } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index a8284a8fc6..36d5cec06f 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -588,7 +588,7 @@ static bool tscKillQueryInDnode(SSqlObj* pSql) { void taos_free_result(TAOS_RES *res) { SSqlObj* pSql = (SSqlObj*) res; if (pSql == NULL || pSql->signature != pSql) { - tscError("%p already released sqlObj", res); + tscError("0x%"PRIx64" already released sqlObj", pSql ? pSql->self : -1); return; } @@ -881,15 +881,14 @@ int taos_validate_sql(TAOS *taos, const char *sql) { int32_t sqlLen = (int32_t)strlen(sql); if (sqlLen > tsMaxSQLStringLen) { - tscError("%p sql too long", pSql); + tscError("0x%"PRIx64" sql too long", pSql->self); tfree(pSql); return TSDB_CODE_TSC_EXCEED_SQL_LIMIT; } pSql->sqlstr = realloc(pSql->sqlstr, sqlLen + 1); if (pSql->sqlstr == NULL) { - tscError("%p failed to malloc sql string buffer", pSql); - tscDebug("0x%"PRIx64" Valid SQL result:%d, %s pObj:%p", pSql->self, pRes->code, taos_errstr(pSql), pObj); + tscError("0x%"PRIx64" failed to malloc sql string buffer", pSql->self); tfree(pSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -914,7 +913,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) { } if (code != TSDB_CODE_SUCCESS) { - tscDebug("0x%"PRIx64" Valid SQL result:%d, %s pObj:%p", pSql->self, code, taos_errstr(pSql), pObj); + tscError("0x%"PRIx64" invalid SQL result:%d, %s pObj:%p", pSql->self, code, taos_errstr(pSql), pObj); } taos_free_result(pSql); @@ -1031,14 +1030,14 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { int32_t tblListLen = (int32_t)strlen(tableNameList); if (tblListLen > MAX_TABLE_NAME_LENGTH) { - tscError("%p tableNameList too long, length:%d, maximum allowed:%d", pSql, tblListLen, MAX_TABLE_NAME_LENGTH); + tscError("0x%"PRIx64" tableNameList too long, length:%d, maximum allowed:%d", pSql->self, tblListLen, MAX_TABLE_NAME_LENGTH); tscFreeSqlObj(pSql); return TSDB_CODE_TSC_INVALID_SQL; } char *str = calloc(1, tblListLen + 1); if (str == NULL) { - tscError("%p failed to malloc sql string buffer", pSql); + tscError("0x%"PRIx64" failed to malloc sql string buffer", pSql->self); tscFreeSqlObj(pSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index b751aea364..13e311a0b3 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -194,7 +194,7 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf SSqlStream *pStream = (SSqlStream *)param; if (tres == NULL || numOfRows < 0) { int64_t retryDelay = tscGetRetryDelayTime(pStream, pStream->interval.sliding, pStream->precision); - tscError("%p stream:%p, query data failed, code:0x%08x, retry in %" PRId64 "ms", pStream->pSql, pStream, numOfRows, + tscError("0x%"PRIx64" stream:%p, query data failed, code:0x%08x, retry in %" PRId64 "ms", pStream->pSql->self, pStream, numOfRows, retryDelay); STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0, 0); @@ -267,7 +267,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf if (pSql == NULL || numOfRows < 0) { int64_t retryDelayTime = tscGetRetryDelayTime(pStream, pStream->interval.sliding, pStream->precision); - tscError("%p stream:%p, retrieve data failed, code:0x%08x, retry in %" PRId64 "ms", pSql, pStream, numOfRows, retryDelayTime); + tscError("0x%"PRIx64" stream:%p, retrieve data failed, code:0x%08x, retry in %" PRId64 "ms", pSql->self, pStream, numOfRows, retryDelayTime); tscSetRetryTimer(pStream, pStream->pSql, retryDelayTime); return; @@ -300,7 +300,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf /* no resuls in the query range, retry */ // todo set retry dynamic time int32_t retry = tsProjectExecInterval; - tscError("%p stream:%p, retrieve no data, code:0x%08x, retry in %" PRId32 "ms", pSql, pStream, numOfRows, retry); + tscError("0x%"PRIx64" stream:%p, retrieve no data, code:0x%08x, retry in %" PRId32 "ms", pSql->self, pStream, numOfRows, retry); tscSetRetryTimer(pStream, pStream->pSql, retry); return; @@ -448,7 +448,7 @@ static int32_t tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) { } if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit!= 'y' && pQueryInfo->interval.interval < minIntervalTime) { - tscWarn("%p stream:%p, original sample interval:%" PRId64 " too small, reset to:%" PRId64, pSql, pStream, + tscWarn("0x%"PRIx64" stream:%p, original sample interval:%" PRId64 " too small, reset to:%" PRId64, pSql->self, pStream, (int64_t)pQueryInfo->interval.interval, minIntervalTime); pQueryInfo->interval.interval = minIntervalTime; } @@ -465,14 +465,14 @@ static int32_t tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) { (pStream->precision == TSDB_TIME_PRECISION_MICRO) ? tsMinSlidingTime * 1000L : tsMinSlidingTime; if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit!= 'y' && pQueryInfo->interval.sliding < minSlidingTime) { - tscWarn("%p stream:%p, original sliding value:%" PRId64 " too small, reset to:%" PRId64, pSql, pStream, + tscWarn("0x%"PRIx64" stream:%p, original sliding value:%" PRId64 " too small, reset to:%" PRId64, pSql->self, pStream, pQueryInfo->interval.sliding, minSlidingTime); pQueryInfo->interval.sliding = minSlidingTime; } if (pQueryInfo->interval.sliding > pQueryInfo->interval.interval) { - tscWarn("%p stream:%p, sliding value:%" PRId64 " can not be larger than interval range, reset to:%" PRId64, pSql, pStream, + tscWarn("0x%"PRIx64" stream:%p, sliding value:%" PRId64 " can not be larger than interval range, reset to:%" PRId64, pSql->self, pStream, pQueryInfo->interval.sliding, pQueryInfo->interval.interval); pQueryInfo->interval.sliding = pQueryInfo->interval.interval; @@ -515,7 +515,7 @@ static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, in } else { int64_t newStime = taosTimeTruncate(stime, &pStream->interval, pStream->precision); if (newStime != stime) { - tscWarn("%p stream:%p, last timestamp:%" PRId64 ", reset to:%" PRId64, pSql, pStream, stime, newStime); + tscWarn("0x%"PRIx64" stream:%p, last timestamp:%" PRId64 ", reset to:%" PRId64, pSql->self, pStream, stime, newStime); stime = newStime; } } @@ -546,7 +546,7 @@ static void tscCreateStream(void *param, TAOS_RES *res, int code) { if (code != TSDB_CODE_SUCCESS) { pSql->res.code = code; - tscError("%p open stream failed, sql:%s, reason:%s, code:%s", pSql, pSql->sqlstr, pCmd->payload, tstrerror(code)); + tscError("0x%"PRIx64" open stream failed, sql:%s, reason:%s, code:%s", pSql->self, pSql->sqlstr, pCmd->payload, tstrerror(code)); pStream->fp(pStream->param, NULL, NULL); return; @@ -565,7 +565,7 @@ static void tscCreateStream(void *param, TAOS_RES *res, int code) { if (tscSetSlidingWindowInfo(pSql, pStream) != TSDB_CODE_SUCCESS) { pSql->res.code = code; - tscError("%p stream %p open failed, since the interval value is incorrect", pSql, pStream); + tscError("0x%"PRIx64" stream %p open failed, since the interval value is incorrect", pSql->self, pStream); pStream->fp(pStream->param, NULL, NULL); return; } @@ -605,7 +605,7 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p SSqlStream *pStream = (SSqlStream *)calloc(1, sizeof(SSqlStream)); if (pStream == NULL) { - tscError("%p open stream failed, sql:%s, reason:%s, code:0x%08x", pSql, sqlstr, pCmd->payload, pRes->code); + tscError("0x%"PRIx64" open stream failed, sql:%s, reason:%s, code:0x%08x", pSql->self, sqlstr, pCmd->payload, pRes->code); tscFreeSqlObj(pSql); return NULL; } @@ -621,7 +621,7 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p pSql->sqlstr = calloc(1, strlen(sqlstr) + 1); if (pSql->sqlstr == NULL) { - tscError("%p failed to malloc sql string buffer", pSql); + tscError("0x%"PRIx64" failed to malloc sql string buffer", pSql->self); tscFreeSqlObj(pSql); return NULL; } @@ -640,7 +640,7 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p if (code == TSDB_CODE_SUCCESS) { tscCreateStream(pStream, pSql, code); } else if (code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - tscError("%p open stream failed, sql:%s, code:%s", pSql, sqlstr, tstrerror(code)); + tscError("0x%"PRIx64" open stream failed, sql:%s, code:%s", pSql->self, sqlstr, tstrerror(code)); taosReleaseRef(tscObjRef, pSql->self); free(pStream); return NULL; diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 1277a436a1..7da98c7885 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -224,11 +224,11 @@ static SArray* getTableList( SSqlObj* pSql ) { SSqlObj* pNew = taos_query(pSql->pTscObj, sql); if (pNew == NULL) { - tscError("failed to retrieve table id: cannot create new sql object."); + tscError("0x%"PRIx64"failed to retrieve table id: cannot create new sql object.", pSql->self); return NULL; } else if (taos_errno(pNew) != TSDB_CODE_SUCCESS) { - tscError("failed to retrieve table id: %s", tstrerror(taos_errno(pNew))); + tscError("0x%"PRIx64"failed to retrieve table id,error: %s", pSql->self, tstrerror(taos_errno(pNew))); return NULL; } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 3cce8de852..cc4eb751ff 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -673,7 +673,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { //prepare the subqueries object failed, abort if (!success) { pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscError("%p failed to prepare subqueries objs for secondary phase query, numOfSub:%d, code:%d", pSql, + tscError("0x%"PRIx64" failed to prepare subqueries objs for secondary phase query, numOfSub:%d, code:%d", pSql->self, pSql->subState.numOfSub, pSql->res.code); freeJoinSubqueryObj(pSql); @@ -717,7 +717,7 @@ void freeJoinSubqueryObj(SSqlObj* pSql) { static int32_t quitAllSubquery(SSqlObj* pSqlSub, SSqlObj* pSqlObj, SJoinSupporter* pSupporter) { if (subAndCheckDone(pSqlSub, pSqlObj, pSupporter->subqueryIndex)) { - tscError("%p all subquery return and query failed, global code:%s", pSqlObj, tstrerror(pSqlObj->res.code)); + tscError("0x%"PRIx64" all subquery return and query failed, global code:%s", pSqlObj->self, tstrerror(pSqlObj->res.code)); freeJoinSubqueryObj(pSqlObj); return 0; } @@ -801,7 +801,7 @@ void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArr STableIdInfo item = {.uid = tt->uid, .tid = tt->tid, .key = INT64_MIN}; taosArrayPush(vgTables, &item); - tscTrace("%p tid:%d, uid:%"PRIu64",vgId:%d added", pSql, tt->tid, tt->uid, tt->vgId); + tscTrace("0x%"PRIx64" tid:%d, uid:%"PRIu64",vgId:%d added", pSql->self, tt->tid, tt->uid, tt->vgId); prev = tt; } @@ -880,7 +880,7 @@ static bool checkForDuplicateTagVal(SSchema* pColSchema, SJoinSupporter* p1, SSq assert(prev->vgId >= 1 && p->vgId >= 1); if (doCompare(prev->tag, p->tag, pColSchema->type, pColSchema->bytes) == 0) { - tscError("%p join tags have same value for different table, free all sub SqlObj and quit", pPSqlObj); + tscError("0x%"PRIx64" join tags have same value for different table, free all sub SqlObj and quit", pPSqlObj->self); pPSqlObj->res.code = TSDB_CODE_QRY_DUP_JOIN_KEY; return false; } @@ -1116,7 +1116,7 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow assert(TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY)); if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, numOfRows, pParentSql->res.code); + tscError("0x%"PRIx64" abort query due to other subquery failure. code:%d, global code:%d", pSql->self, numOfRows, pParentSql->res.code); if (quitAllSubquery(pSql, pParentSql, pSupporter)) { return; } @@ -1131,7 +1131,7 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow // todo retry if other subqueries are not failed assert(numOfRows < 0 && numOfRows == taos_errno(pSql)); - tscError("%p sub query failed, code:%s, index:%d", pSql, tstrerror(numOfRows), pSupporter->subqueryIndex); + tscError("0x%"PRIx64" sub query failed, code:%s, index:%d", pSql->self, tstrerror(numOfRows), pSupporter->subqueryIndex); pParentSql->res.code = numOfRows; if (quitAllSubquery(pSql, pParentSql, pSupporter)) { @@ -1150,7 +1150,7 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow // todo handle memory error char* tmp = realloc(pSupporter->pIdTagList, length); if (tmp == NULL) { - tscError("%p failed to malloc memory", pSql); + tscError("0x%"PRIx64" failed to malloc memory", pSql->self); pParentSql->res.code = TAOS_SYSTEM_ERROR(errno); if (quitAllSubquery(pSql, pParentSql, pSupporter)) { @@ -1270,7 +1270,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow assert(!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)); if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, numOfRows, pParentSql->res.code); + tscError("0x%"PRIx64" abort query due to other subquery failure. code:%d, global code:%d", pSql->self, numOfRows, pParentSql->res.code); if (quitAllSubquery(pSql, pParentSql, pSupporter)){ return; } @@ -1284,7 +1284,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { // todo retry if other subqueries are not failed yet assert(numOfRows < 0 && numOfRows == taos_errno(pSql)); - tscError("%p sub query failed, code:%s, index:%d", pSql, tstrerror(numOfRows), pSupporter->subqueryIndex); + tscError("0x%"PRIx64" sub query failed, code:%s, index:%d", pSql->self, tstrerror(numOfRows), pSupporter->subqueryIndex); pParentSql->res.code = numOfRows; if (quitAllSubquery(pSql, pParentSql, pSupporter)){ @@ -1300,7 +1300,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow pSupporter->f = fopen(pSupporter->path, "wb"); if (pSupporter->f == NULL) { - tscError("%p failed to create tmp file:%s, reason:%s", pSql, pSupporter->path, strerror(errno)); + tscError("0x%"PRIx64" failed to create tmp file:%s, reason:%s", pSql->self, pSupporter->path, strerror(errno)); pParentSql->res.code = TAOS_SYSTEM_ERROR(errno); @@ -1320,7 +1320,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow STSBuf* pBuf = tsBufCreateFromFile(pSupporter->path, true); if (pBuf == NULL) { // in error process, close the fd - tscError("%p invalid ts comp file from vnode, abort subquery, file size:%d", pSql, numOfRows); + tscError("0x%"PRIx64" invalid ts comp file from vnode, abort subquery, file size:%d", pSql->self, numOfRows); pParentSql->res.code = TAOS_SYSTEM_ERROR(errno); if (quitAllSubquery(pSql, pParentSql, pSupporter)){ @@ -1417,7 +1417,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, numOfRows, pParentSql->res.code); + tscError("0x%"PRIx64" abort query due to other subquery failure. code:%d, global code:%d", pSql->self, numOfRows, pParentSql->res.code); if (quitAllSubquery(pSql, pParentSql, pSupporter)) { return; } @@ -1432,7 +1432,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR assert(numOfRows == taos_errno(pSql)); pParentSql->res.code = numOfRows; - tscError("%p retrieve failed, index:%d, code:%s", pSql, pSupporter->subqueryIndex, tstrerror(numOfRows)); + tscError("0x%"PRIx64" retrieve failed, index:%d, code:%s", pSql->self, pSupporter->subqueryIndex, tstrerror(numOfRows)); tscAsyncResultOnError(pParentSql); return; @@ -1762,7 +1762,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { // retrieve actual query results from vnode during the second stage join subquery if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, code, pParentSql->res.code); + tscError("0x%"PRIx64" abort query due to other subquery failure. code:%d, global code:%d", pSql->self, code, pParentSql->res.code); if (quitAllSubquery(pSql, pParentSql, pSupporter)) { return; } @@ -1776,7 +1776,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { assert(taos_errno(pSql) == code); - tscError("%p abort query, code:%s, global code:%s", pSql, tstrerror(code), tstrerror(pParentSql->res.code)); + tscError("0x%"PRIx64" abort query, code:%s, global code:%s", pSql->self, tstrerror(code), tstrerror(pParentSql->res.code)); pParentSql->res.code = code; if (quitAllSubquery(pSql, pParentSql, pSupporter)) { @@ -2011,7 +2011,7 @@ void tscHandleMasterJoinQuery(SSqlObj* pSql) { SJoinSupporter *pSupporter = tscCreateJoinSupporter(pSql, i); if (pSupporter == NULL) { // failed to create support struct, abort current query - tscError("%p tableIndex:%d, failed to allocate join support object, abort further query", pSql, i); + tscError("0x%"PRIx64" tableIndex:%d, failed to allocate join support object, abort further query", pSql->self, i); code = TSDB_CODE_TSC_OUT_OF_MEMORY; goto _error; } @@ -2498,7 +2498,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { for (; i < pState->numOfSub; ++i) { SRetrieveSupport *trs = (SRetrieveSupport *)calloc(1, sizeof(SRetrieveSupport)); if (trs == NULL) { - tscError("%p failed to malloc buffer for SRetrieveSupport, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); + tscError("0x%"PRIx64" failed to malloc buffer for SRetrieveSupport, orderOfSub:%d, reason:%s", pSql->self, i, strerror(errno)); break; } @@ -2507,7 +2507,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { trs->localBuffer = (tFilePage *)calloc(1, nBufferSize + sizeof(tFilePage)); if (trs->localBuffer == NULL) { - tscError("%p failed to malloc buffer for local buffer, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); + tscError("0x%"PRIx64" failed to malloc buffer for local buffer, orderOfSub:%d, reason:%s", pSql->self, i, strerror(errno)); tfree(trs); break; } @@ -2519,7 +2519,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { SSqlObj *pNew = tscCreateSTableSubquery(pSql, trs, NULL); if (pNew == NULL) { - tscError("%p failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); + tscError("0x%"PRIx64" failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql->self, i, strerror(errno)); tfree(trs->localBuffer); tfree(trs); break; @@ -2536,7 +2536,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { } if (i < pState->numOfSub) { - tscError("%p failed to prepare subquery structure and launch subqueries", pSql); + tscError("0x%"PRIx64" failed to prepare subquery structure and launch subqueries", pSql->self); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pFinalModel, pState->numOfSub); @@ -2580,7 +2580,7 @@ static void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, i static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES *tres, int32_t code) { // set no disk space error info - tscError("sub:%p failed to flush data to disk, reason:%s", tres, tstrerror(code)); + tscError("sub:0x%"PRIx64" failed to flush data to disk, reason:%s", ((SSqlObj *)tres)->self, tstrerror(code)); SSqlObj* pParentSql = trsupport->pParentSql; pParentSql->res.code = code; @@ -2605,7 +2605,7 @@ static int32_t tscReissueSubquery(SRetrieveSupport *oriTrs, SSqlObj *pSql, int32 const uint32_t nBufferSize = (1u << 16u); // 64KB trsupport->localBuffer = (tFilePage *)calloc(1, nBufferSize + sizeof(tFilePage)); if (trsupport->localBuffer == NULL) { - tscError("%p failed to malloc buffer for local buffer, reason:%s", pSql, strerror(errno)); + tscError("0x%"PRIx64" failed to malloc buffer for local buffer, reason:%s", pSql->self, strerror(errno)); tfree(trsupport); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -2620,13 +2620,13 @@ static int32_t tscReissueSubquery(SRetrieveSupport *oriTrs, SSqlObj *pSql, int32 // clear local saved number of results trsupport->localBuffer->num = 0; - tscError("%p sub:%p retrieve/query failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSql, pSql, + tscError("0x%"PRIx64" sub:0x%"PRIx64" retrieve/query failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSql->self, pSql->self, tstrerror(code), subqueryIndex, trsupport->numOfRetry); SSqlObj *pNew = tscCreateSTableSubquery(trsupport->pParentSql, trsupport, pSql); if (pNew == NULL) { - tscError("%p sub:%p failed to create new subquery due to error:%s, abort retry, vgId:%d, orderOfSub:%d", - oriTrs->pParentSql, pSql, tstrerror(terrno), pVgroup->vgId, oriTrs->subqueryIndex); + tscError("0x%"PRIx64" sub:0x%"PRIx64" failed to create new subquery due to error:%s, abort retry, vgId:%d, orderOfSub:%d", + oriTrs->pParentSql->self, pSql->self, tstrerror(terrno), pVgroup->vgId, oriTrs->subqueryIndex); pParentSql->res.code = terrno; oriTrs->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; @@ -2680,7 +2680,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO if (numOfRows >= 0) { // current query is successful, but other sub query failed, still abort current query. tscDebug("0x%"PRIx64" sub:0x%"PRIx64" retrieve numOfRows:%d,orderOfSub:%d", pParentSql->self, pSql->self, numOfRows, subqueryIndex); - tscError("%p sub:%p abort further retrieval due to other queries failure,orderOfSub:%d,code:%s", pParentSql, pSql, + tscError("0x%"PRIx64" sub:0x%"PRIx64" abort further retrieval due to other queries failure,orderOfSub:%d,code:%s", pParentSql->self, pSql->self, subqueryIndex, tstrerror(pParentSql->res.code)); } else { if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY && pParentSql->res.code == TSDB_CODE_SUCCESS) { @@ -2692,7 +2692,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO } } else { // reach the maximum retry count, abort atomic_val_compare_exchange_32(&pParentSql->res.code, TSDB_CODE_SUCCESS, numOfRows); - tscError("%p sub:%p retrieve failed,code:%s,orderOfSub:%d failed.no more retry,set global code:%s", pParentSql, pSql, + tscError("0x%"PRIx64" sub:0x%"PRIx64" retrieve failed,code:%s,orderOfSub:%d failed.no more retry,set global code:%s", pParentSql->self, pSql->self, tstrerror(numOfRows), subqueryIndex, tstrerror(pParentSql->res.code)); } } @@ -2705,7 +2705,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO } // all subqueries are failed - tscError("%p retrieve from %d vnode(s) completed,code:%s.FAILED.", pParentSql, pState->numOfSub, + tscError("0x%"PRIx64" retrieve from %d vnode(s) completed,code:%s.FAILED.", pParentSql->self, pState->numOfSub, tstrerror(pParentSql->res.code)); // release allocated resource @@ -2753,7 +2753,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p #endif if (tsTotalTmpDirGB != 0 && tsAvailTmpDirectorySpace < tsReservedTmpDirectorySpace) { - tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pParentSql, pSql, + tscError("0x%"PRIx64" sub:0x%"PRIx64" client disk space remain %.3f GB, need at least %.3f GB, stop query", pParentSql->self, pSql->self, tsAvailTmpDirectorySpace, tsReservedTmpDirectorySpace); tscAbortFurtherRetryRetrieval(trsupport, pSql, TSDB_CODE_TSC_NO_DISKSPACE); return; @@ -2838,7 +2838,7 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR } if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY) { - tscError("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(numOfRows), trsupport->numOfRetry); + tscError("0x%"PRIx64" sub:0x%"PRIx64" failed code:%s, retry:%d", pParentSql->self, pSql->self, tstrerror(numOfRows), trsupport->numOfRetry); int32_t sent = 0; @@ -2866,8 +2866,8 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR pParentSql->self, pSql, pRes->numOfRows, pState->numOfRetrievedRows, pSql->epSet.fqdn[pSql->epSet.inUse], idx); if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { - tscError("%p sub:%p num of OrderedRes is too many, max allowed:%" PRId32 " , current:%" PRId64, - pParentSql, pSql, tsMaxNumOfOrderedResults, num); + tscError("0x%"PRIx64" sub:0x%"PRIx64" num of OrderedRes is too many, max allowed:%" PRId32 " , current:%" PRId64, + pParentSql->self, pSql->self, tsMaxNumOfOrderedResults, num); tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_SORTED_RES_TOO_MANY); return; } @@ -2882,7 +2882,7 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR // no disk space for tmp directory if (tsTotalTmpDirGB != 0 && tsAvailTmpDirectorySpace < tsReservedTmpDirectorySpace) { - tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pParentSql, pSql, + tscError("0x%"PRIx64" sub:0x%"PRIx64" client disk space remain %.3f GB, need at least %.3f GB, stop query", pParentSql->self, pSql->self, tsAvailTmpDirectorySpace, tsReservedTmpDirectorySpace); tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_NO_DISKSPACE); return; @@ -2951,8 +2951,8 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { // stable query killed or other subquery failed, all query stopped if (pParentSql->res.code != TSDB_CODE_SUCCESS) { trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - tscError("%p query cancelled or failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", - pParentSql, pSql, pVgroup->vgId, trsupport->subqueryIndex, tstrerror(code), tstrerror(pParentSql->res.code)); + tscError("0x%"PRIx64" query cancelled or failed, sub:0x%"PRIx64", vgId:%d, orderOfSub:%d, code:%s, global code:%s", + pParentSql->self, pSql->self, pVgroup->vgId, trsupport->subqueryIndex, tstrerror(code), tstrerror(pParentSql->res.code)); tscHandleSubqueryError(param, tres, code); return; @@ -2969,7 +2969,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { assert(code == taos_errno(pSql)); if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY) { - tscError("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(code), trsupport->numOfRetry); + tscError("0x%"PRIx64" sub:0x%"PRIx64" failed code:%s, retry:%d", pParentSql->self, pSql->self, tstrerror(code), trsupport->numOfRetry); int32_t sent = 0; @@ -2978,7 +2978,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { return; } } else { - tscError("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(code)); + tscError("0x%"PRIx64" sub:0x%"PRIx64" reach the max retry times, set global code:%s", pParentSql->self, pSql->self, tstrerror(code)); atomic_val_compare_exchange_32(&pParentSql->res.code, TSDB_CODE_SUCCESS, code); // set global code and abort } @@ -2998,7 +2998,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { static bool needRetryInsert(SSqlObj* pParentObj, int32_t numOfSub) { if (pParentObj->retry > pParentObj->maxRetry) { - tscError("%p max retry reached, abort the retry effort", pParentObj); + tscError("0x%"PRIx64" max retry reached, abort the retry effort", pParentObj->self); return false; } @@ -3090,7 +3090,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) } } - tscError("%p Async insertion completed, total inserted:%d rows, numOfFailed:%d, numOfTotal:%d", pParentObj, + tscError("0x%"PRIx64" Async insertion completed, total inserted:%d rows, numOfFailed:%d, numOfTotal:%d", pParentObj->self, pParentObj->res.numOfRows, numOfFailed, numOfSub); tscDebug("0x%"PRIx64" cleanup %d tableMeta in hashTable", pParentObj->self, pParentObj->cmd.numOfTables); @@ -3207,7 +3207,7 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { SSqlObj *pNew = createSimpleSubObj(pSql, multiVnodeInsertFinalize, pSupporter, TSDB_SQL_INSERT); if (pNew == NULL) { - tscError("%p failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql, numOfSub, strerror(errno)); + tscError("0x%"PRIx64" failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql->self, numOfSub, strerror(errno)); goto _error; } @@ -3231,7 +3231,7 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { } if (numOfSub < pSql->subState.numOfSub) { - tscError("%p failed to prepare subObj structure and launch sub-insertion", pSql); + tscError("0x%"PRIx64" failed to prepare subObj structure and launch sub-insertion", pSql->self); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; goto _error; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 9bf963876e..fa9e53b0a6 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -913,7 +913,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, bool freeBlockMap) { int32_t ret = tscGetDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0, &pOneTableBlock->tableName, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList); if (ret != TSDB_CODE_SUCCESS) { - tscError("%p failed to prepare the data block buffer for merging table data, code:%d", pSql, ret); + tscError("0x%"PRIx64" failed to prepare the data block buffer for merging table data, code:%d", pSql->self, ret); taosHashCleanup(pVnodeDataBlockHashList); tscDestroyBlockArrayList(pVnodeDataBlockList); return ret; @@ -932,7 +932,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, bool freeBlockMap) { dataBuf->pData = tmp; memset(dataBuf->pData + dataBuf->size, 0, dataBuf->nAllocSize - dataBuf->size); } else { // failed to allocate memory, free already allocated memory and return error code - tscError("%p failed to allocate memory for merging submit block, size:%d", pSql, dataBuf->nAllocSize); + tscError("0x%"PRIx64" failed to allocate memory for merging submit block, size:%d", pSql->self, dataBuf->nAllocSize); taosHashCleanup(pVnodeDataBlockHashList); tscDestroyBlockArrayList(pVnodeDataBlockList); @@ -2115,7 +2115,7 @@ void registerSqlObj(SSqlObj* pSql) { SSqlObj* createSimpleSubObj(SSqlObj* pSql, __async_cb_func_t fp, void* param, int32_t cmd) { SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); if (pNew == NULL) { - tscError("%p new subquery failed, tableIndex:%d", pSql, 0); + tscError("0x%"PRIx64" new subquery failed, tableIndex:%d", pSql->self, 0); return NULL; } @@ -2129,7 +2129,7 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, __async_cb_func_t fp, void* param, in int32_t code = copyTagData(&pNew->cmd.tagData, &pSql->cmd.tagData); if (code != TSDB_CODE_SUCCESS) { - tscError("%p new subquery failed, unable to malloc tag data, tableIndex:%d", pSql, 0); + tscError("0x%"PRIx64" new subquery failed, unable to malloc tag data, tableIndex:%d", pSql->self, 0); free(pNew); return NULL; } @@ -2206,7 +2206,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); if (pNew == NULL) { - tscError("%p new subquery failed, tableIndex:%d", pSql, tableIndex); + tscError("0x%"PRIx64" new subquery failed, tableIndex:%d", pSql->self, tableIndex); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; } @@ -2295,7 +2295,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t } if (tscAllocPayload(pnCmd, TSDB_DEFAULT_PAYLOAD_SIZE) != TSDB_CODE_SUCCESS) { - tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex); + tscError("0x%"PRIx64" new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql->self, tableIndex, pTableMetaInfo->vgroupIndex); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; goto _error; } @@ -2346,7 +2346,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t // this case cannot be happened if (pFinalInfo->pTableMeta == NULL) { - tscError("%p new subquery failed since no tableMeta, name:%s", pSql, tNameGetTableName(&pTableMetaInfo->name)); + tscError("0x%"PRIx64" new subquery failed since no tableMeta, name:%s", pSql->self, tNameGetTableName(&pTableMetaInfo->name)); if (pPrevSql != NULL) { // pass the previous error to client assert(pPrevSql->res.code != TSDB_CODE_SUCCESS); From b2e1361c0fa427fa319a801ec8ecca1d149a4d8f Mon Sep 17 00:00:00 2001 From: Huo Linhe Date: Tue, 27 Apr 2021 18:10:48 +0800 Subject: [PATCH 26/27] [TD-3983] : update driver-go submodule to latest (#5945) The update contains TD-3983 TD-3912 fixes. --- src/connector/go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connector/go b/src/connector/go index d99751356e..7a26c432f8 160000 --- a/src/connector/go +++ b/src/connector/go @@ -1 +1 @@ -Subproject commit d99751356e285696f57bc604304ffafd10287439 +Subproject commit 7a26c432f8b4203e42344ff3290b9b9b01b983d5 From 03d447c5563561f3030bce1d29a27a25fa32a0d6 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 27 Apr 2021 19:37:58 +0800 Subject: [PATCH 27/27] [TD-3985]: taosdemo timestamp step overflow. (#5946) * [TD-3985]: taosdemo timestamp step overflow. * fix specified subscribe test. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 0fc8687b22..8ab4918779 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -253,7 +253,7 @@ typedef struct SSuperTable_S { int insertInterval; // insert interval, will override global insert interval int64_t insertRows; - int timeStampStep; + int64_t timeStampStep; char startTimestamp[MAX_TB_NAME_SIZE]; char sampleFormat[MAX_TB_NAME_SIZE]; // csv, json char sampleFile[MAX_FILE_NAME_LEN+1]; @@ -1369,7 +1369,7 @@ static int printfInsertMeta() { g_Dbs.db[i].superTbls[j].disorderRatio); printf(" maxSqlLen: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].maxSqlLen); - printf(" timeStampStep: \033[33m%d\033[0m\n", + printf(" timeStampStep: \033[33m%"PRId64"\033[0m\n", g_Dbs.db[i].superTbls[j].timeStampStep); printf(" startTimestamp: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].startTimestamp); @@ -1541,7 +1541,7 @@ static void printfInsertMetaToFile(FILE* fp) { fprintf(fp, " disorderRatio: %d\n", g_Dbs.db[i].superTbls[j].disorderRatio); fprintf(fp, " maxSqlLen: %d\n", g_Dbs.db[i].superTbls[j].maxSqlLen); - fprintf(fp, " timeStampStep: %d\n", g_Dbs.db[i].superTbls[j].timeStampStep); + fprintf(fp, " timeStampStep: %"PRId64"\n", g_Dbs.db[i].superTbls[j].timeStampStep); fprintf(fp, " startTimestamp: %s\n", g_Dbs.db[i].superTbls[j].startTimestamp); fprintf(fp, " sampleFormat: %s\n", g_Dbs.db[i].superTbls[j].sampleFormat); fprintf(fp, " sampleFile: %s\n", g_Dbs.db[i].superTbls[j].sampleFile); @@ -4646,7 +4646,7 @@ static void getTableName(char *pTblName, threadInfo* pThreadInfo, int tableSeq) static int generateDataTail( SSuperTable* superTblInfo, int batch, char* buffer, int remainderBufLen, int64_t insertRows, - int64_t startFrom, uint64_t startTime, int *pSamplePos, int *dataLen) { + int64_t startFrom, int64_t startTime, int *pSamplePos, int *dataLen) { int len = 0; int ncols_per_record = 1; // count first col ts @@ -4974,7 +4974,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pThreadInfo->totalInsertRows = 0; pThreadInfo->totalAffectedRows = 0; - int nTimeStampStep = superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP; + int64_t nTimeStampStep = superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP; int insert_interval = superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; @@ -5183,7 +5183,7 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { int64_t startTs = taosGetTimestampMs(); int64_t endTs; - int timeStampStep = + int64_t timeStampStep = superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP; /* int insert_interval = superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; @@ -6448,7 +6448,7 @@ static void *specifiedSubscribe(void *sarg) { } tsub[i] = subscribeImpl(pThreadInfo->taos, g_queryInfo.specifiedQueryInfo.sql[i], topic, tmpFile); - if (NULL == g_queryInfo.specifiedQueryInfo.tsub[i]) { + if (NULL == tsub[i]) { taos_close(pThreadInfo->taos); return NULL; }