From 53a128b2a3a469b06ae28ff013397b8d015433c8 Mon Sep 17 00:00:00 2001 From: wu champion Date: Mon, 22 Mar 2021 17:02:53 +0800 Subject: [PATCH 01/71] fix case,add delete file when processing over --- tests/pytest/query/query1970YearsAf.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/pytest/query/query1970YearsAf.py b/tests/pytest/query/query1970YearsAf.py index d2eead241f..441f835e89 100644 --- a/tests/pytest/query/query1970YearsAf.py +++ b/tests/pytest/query/query1970YearsAf.py @@ -18,7 +18,6 @@ import json import subprocess import datetime - from util.log import * from util.sql import * from util.cases import * @@ -71,7 +70,7 @@ class TDTestCase: "update": 0 } - # 设置创建的超级表格式 + # set stable schema stable1 = { "name": "stb2", "child_table_exists": "no", @@ -83,7 +82,7 @@ class TDTestCase: "insert_rows": 5000, "multi_thread_write_one_tbl": "no", "number_of_tbl_in_one_sql": 0, - "rows_per_tbl": 1000, + "rows_per_tbl": 1, "max_sql_len": 65480, "disorder_ratio": 0, "disorder_range": 1000, @@ -117,7 +116,7 @@ class TDTestCase: ] } - # 创建不同的超级表格式并添加至super_tables + # create different stables like stable1 and add to list super_tables super_tables = [] super_tables.append(stable1) database = { @@ -235,7 +234,7 @@ class TDTestCase: tdLog.info("==========step2:query join") self.sqlsquery() - # 进行数据落盘后检查 + # after wal and sync, check again tdSql.query("show dnodes") index = tdSql.getData(0, 0) tdDnodes.stop(index) @@ -246,8 +245,9 @@ class TDTestCase: def stop(self): tdSql.close() + rm_cmd = f"rm -f /tmp/insert* > /dev/null 2>&1" + _ = subprocess.check_output(rm_cmd, shell=True).decode("utf-8") tdLog.success(f"{__file__} successfully executed") - tdCases.addLinux(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From 7c749e24c8814f03eb147048158efa1ae7b7bfd9 Mon Sep 17 00:00:00 2001 From: wu champion Date: Fri, 19 Mar 2021 18:35:26 +0800 Subject: [PATCH 02/71] [TD-1380] fix the case of query1970YearsAf.py --- tests/pytest/query/query1970YearsAf.py | 253 +++++++++++++++++++++++++ 1 file changed, 253 insertions(+) create mode 100644 tests/pytest/query/query1970YearsAf.py diff --git a/tests/pytest/query/query1970YearsAf.py b/tests/pytest/query/query1970YearsAf.py new file mode 100644 index 0000000000..9902f6908b --- /dev/null +++ b/tests/pytest/query/query1970YearsAf.py @@ -0,0 +1,253 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import taos +import sys +import os +import json +import subprocess +import datetime + + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.dnodes import TDDnode + +class TDTestCase: + + def __init__(self): + self.path = "" + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def getcfgPath(self, path): + binPath = os.path.dirname(os.path.realpath(__file__)) + binPath = binPath + "/../../../debug/" + tdLog.debug(f"binPath {binPath}") + binPath = os.path.realpath(binPath) + tdLog.debug(f"binPath real path {binPath}") + if path == "": + self.path = os.path.abspath(binPath + "../../") + else: + self.path = os.path.realpath(path) + return self.path + + def getCfgDir(self): + self.getcfgPath(self.path) + self.cfgDir = f"{self.path}/sim/psim/cfg" + return self.cfgDir + + def creatcfg(self): + dbinfo = { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 36500, + "minRows": 100, + "maxRows": 4096, + "comp": 2, + "walLevel": 1, + "cachelast": 0, + "quorum": 1, + "fsync": 3000, + "update": 0 + } + + # 设置创建的超级表格式 + stable1 = { + "name": "stb2", + "child_table_exists": "no", + "childtable_count": 10, + "childtable_prefix": "t", + "auto_create_table": "no", + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 5000, + "multi_thread_write_one_tbl": "no", + "number_of_tbl_in_one_sql": 0, + "rows_per_tbl": 1000, + "max_sql_len": 65480, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 20000, + "start_timestamp": "1969-12-31 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [ + {"type": "INT", "count": 2}, + {"type": "DOUBLE", "count": 2}, + {"type": "BIGINT", "count": 2}, + {"type": "FLOAT", "count": 2}, + {"type": "SMALLINT", "count": 2}, + {"type": "TINYINT", "count": 2}, + {"type": "BOOL", "count": 2}, + {"type": "NCHAR", "len": 3, "count": 1}, + {"type": "BINARY", "len": 8, "count": 1} + + ], + "tags": [ + {"type": "INT", "count": 2}, + {"type": "DOUBLE", "count": 2}, + {"type": "BIGINT", "count": 2}, + {"type": "FLOAT", "count": 2}, + {"type": "SMALLINT", "count": 2}, + {"type": "TINYINT", "count": 2}, + {"type": "BOOL", "count": 2}, + {"type": "NCHAR", "len": 3, "count": 1}, + {"type": "BINARY", "len": 8, "count": 1} + ] + } + + # 创建不同的超级表格式并添加至super_tables + super_tables = [] + super_tables.append(stable1) + database = { + "dbinfo": dbinfo, + "super_tables": super_tables + } + + cfgdir = self.getCfgDir() + create_table = { + "filetype": "insert", + "cfgdir": cfgdir, + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "/tmp/insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 100, + "databases": [database] + } + return create_table + + def createinsertfile(self): + create_table = self.creatcfg() + date = datetime.datetime.now().strftime("%Y%m%d%H%M") + file_create_table = f"/tmp/insert_{date}.json" + + with open(file_create_table, 'w') as f: + json.dump(create_table, f) + return file_create_table + + def inserttable(self, filepath): + create_table_cmd = f"taosdemo -f {filepath} > /tmp/1.log 2>&1" + _ = subprocess.check_output(create_table_cmd, shell=True).decode("utf-8") + + def sqlsquery(self): + # stable query + tdSql.query( + "select * from stb2 where stb2.ts < '1970-01-01 00:00:00.000' " + ) + tdSql.checkRows(43200) + + tdSql.query( + "select * from stb2 where stb2.ts >= '1970-01-01 00:00:00.000' " + ) + tdSql.checkRows(6800) + + tdSql.query( + "select * from stb2 where stb2.ts > '1969-12-31 23:00:00.000' and stb2.ts <'1970-01-01 01:00:00.000' " + ) + tdSql.checkRows(3590) + + # child-table query + tdSql.query( + "select * from t0 where t0.ts < '1970-01-01 00:00:00.000' " + ) + tdSql.checkRows(4320) + + tdSql.query( + "select * from t1 where t1.ts >= '1970-01-01 00:00:00.000' " + ) + tdSql.checkRows(680) + + tdSql.query( + "select * from t9 where t9.ts > '1969-12-31 22:00:00.000' and t9.ts <'1970-01-01 02:00:00.000' " + ) + tdSql.checkRows(719) + + tdSql.query( + "select * from t0,t1 where t0.ts=t1.ts and t1.ts >= '1970-01-01 00:00:00.000' " + ) + tdSql.checkRows(680) + + tdSql.query( + "select diff(col1) from t0 where t0.ts >= '1970-01-01 00:00:00.000' " + ) + tdSql.checkRows(679) + + tdSql.query( + "select t0,col1 from stb2 where stb2.ts < '1970-01-01 00:00:00.000' order by ts" + ) + tdSql.checkRows(43200) + + # query with timestamp in 'where ...' + tdSql.query( + "select * from stb2 where stb2.ts > -28800000 " + ) + tdSql.checkRows(6790) + + tdSql.query( + "select * from stb2 where stb2.ts > -28800000 and stb2.ts < '1970-01-01 08:00:00.000' " + ) + tdSql.checkRows(6790) + + tdSql.query( + "select * from stb2 where stb2.ts < -28800000 and stb2.ts > '1969-12-31 22:00:00.000' " + ) + tdSql.checkRows(3590) + + def run(self): + s = 'reset query cache' + tdSql.execute(s) + s = 'create database if not exists db' + tdSql.execute(s) + s = 'use db' + tdSql.execute(s) + + tdLog.info("==========step1:create table stable and child table,then insert data automatically") + insertfile = self.createinsertfile() + self.inserttable(insertfile) + + tdLog.info("==========step2:query join") + self.sqlsquery() + + # 进行数据落盘后检查 + tdSql.query("show dnodes") + index = tdSql.getData(0, 0) + tdDnodes.stop(index) + tdDnodes.start(index) + + tdLog.info("==========step3: query join again") + self.sqlsquery() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From dff5c11f290453a349bdb885e00128586700bece Mon Sep 17 00:00:00 2001 From: wu champion Date: Sat, 20 Mar 2021 16:10:45 +0800 Subject: [PATCH 03/71] [test] add test case and bug case --- tests/pytest/fulltest.sh | 4 ++ tests/pytest/query/bug3351.py | 74 ++++++++++++++++++++++++++ tests/pytest/query/query1970YearsAf.py | 4 +- 3 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 tests/pytest/query/bug3351.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 5037f2c399..aa8e679e81 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -196,6 +196,10 @@ python3 ./test.py -f query/bug2119.py python3 ./test.py -f query/isNullTest.py python3 ./test.py -f query/queryWithTaosdKilled.py python3 ./test.py -f query/floatCompare.py +python3 ./test.py -f query/query1970YearsAf.py +python3 ./test.py -f query/bug3351.py + + #stream python3 ./test.py -f stream/metric_1.py diff --git a/tests/pytest/query/bug3351.py b/tests/pytest/query/bug3351.py new file mode 100644 index 0000000000..288d071a69 --- /dev/null +++ b/tests/pytest/query/bug3351.py @@ -0,0 +1,74 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + tdSql.execute("drop database if exists db") + tdSql.execute("create database if not exists db keep 36500") + tdSql.execute("use db") + tdLog.printNoPrefix("==========step1:create table && insert data") + + tdSql.execute( + "create table stb1 (ts timestamp, c1 int) TAGS(t1 int)" + ) + tdSql.execute("create table t0 using stb1 tags(1)") + tdSql.execute("insert into t0 values (-865000000, 1)") + tdSql.execute("insert into t0 values (-864000000, 2)") + tdSql.execute("insert into t0 values (-863000000, 3)") + tdSql.execute("insert into t0 values (-15230000, 4)") + tdSql.execute("insert into t0 values (-15220000, 5)") + tdSql.execute("insert into t0 values (-15210000, 6)") + + tdLog.printNoPrefix("==========step2:query") + # bug1:when ts > -864000000, return 0 rows; + # bug2:when ts = -15220000, return 0 rows. + tdSql.query('select * from t0 where ts < -864000000') + tdSql.checkRows(1) + tdSql.query('select * from t0 where ts <= -864000000') + tdSql.checkRows(2) + tdSql.query('select * from t0 where ts = -864000000') + tdSql.checkRows(1) + tdSql.query('select * from t0 where ts > -864000000') + tdSql.checkRows(4) + tdSql.query('select * from t0 where ts >= -864000000') + tdSql.checkRows(5) + tdSql.query('select * from t0 where ts < -15220000') + tdSql.checkRows(4) + tdSql.query('select * from t0 where ts <= -15220000') + tdSql.checkRows(5) + tdSql.query('select * from t0 where ts = -15220000') + tdSql.checkRows(1) + tdSql.query('select * from t0 where ts > -15220000') + tdSql.checkRows(1) + tdSql.query('select * from t0 where ts >= -15220000') + tdSql.checkRows(2) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/query/query1970YearsAf.py b/tests/pytest/query/query1970YearsAf.py index 9902f6908b..d2eead241f 100644 --- a/tests/pytest/query/query1970YearsAf.py +++ b/tests/pytest/query/query1970YearsAf.py @@ -88,7 +88,7 @@ class TDTestCase: "disorder_ratio": 0, "disorder_range": 1000, "timestamp_step": 20000, - "start_timestamp": "1969-12-31 00:00:00.000", + "start_timestamp": "1969-12-30 23:59:40.000", "sample_format": "csv", "sample_file": "./sample.csv", "tags_file": "", @@ -153,7 +153,7 @@ class TDTestCase: return file_create_table def inserttable(self, filepath): - create_table_cmd = f"taosdemo -f {filepath} > /tmp/1.log 2>&1" + create_table_cmd = f"taosdemo -f {filepath} > /dev/null 2>&1" _ = subprocess.check_output(create_table_cmd, shell=True).decode("utf-8") def sqlsquery(self): From ef248d09fd248775dfea4a88136354b42dd54f1e Mon Sep 17 00:00:00 2001 From: wu champion Date: Mon, 22 Mar 2021 17:02:53 +0800 Subject: [PATCH 04/71] fix case,add delete file when processing over --- tests/pytest/query/query1970YearsAf.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/pytest/query/query1970YearsAf.py b/tests/pytest/query/query1970YearsAf.py index d2eead241f..441f835e89 100644 --- a/tests/pytest/query/query1970YearsAf.py +++ b/tests/pytest/query/query1970YearsAf.py @@ -18,7 +18,6 @@ import json import subprocess import datetime - from util.log import * from util.sql import * from util.cases import * @@ -71,7 +70,7 @@ class TDTestCase: "update": 0 } - # 设置创建的超级表格式 + # set stable schema stable1 = { "name": "stb2", "child_table_exists": "no", @@ -83,7 +82,7 @@ class TDTestCase: "insert_rows": 5000, "multi_thread_write_one_tbl": "no", "number_of_tbl_in_one_sql": 0, - "rows_per_tbl": 1000, + "rows_per_tbl": 1, "max_sql_len": 65480, "disorder_ratio": 0, "disorder_range": 1000, @@ -117,7 +116,7 @@ class TDTestCase: ] } - # 创建不同的超级表格式并添加至super_tables + # create different stables like stable1 and add to list super_tables super_tables = [] super_tables.append(stable1) database = { @@ -235,7 +234,7 @@ class TDTestCase: tdLog.info("==========step2:query join") self.sqlsquery() - # 进行数据落盘后检查 + # after wal and sync, check again tdSql.query("show dnodes") index = tdSql.getData(0, 0) tdDnodes.stop(index) @@ -246,8 +245,9 @@ class TDTestCase: def stop(self): tdSql.close() + rm_cmd = f"rm -f /tmp/insert* > /dev/null 2>&1" + _ = subprocess.check_output(rm_cmd, shell=True).decode("utf-8") tdLog.success(f"{__file__} successfully executed") - tdCases.addLinux(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From 2de96282490234d89195ad401d001bc71ab71bb5 Mon Sep 17 00:00:00 2001 From: wu champion Date: Tue, 23 Mar 2021 13:40:29 +0800 Subject: [PATCH 05/71] fix the case --- tests/pytest/query/query1970YearsAf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/pytest/query/query1970YearsAf.py b/tests/pytest/query/query1970YearsAf.py index 441f835e89..9cc8948c2f 100644 --- a/tests/pytest/query/query1970YearsAf.py +++ b/tests/pytest/query/query1970YearsAf.py @@ -243,10 +243,11 @@ class TDTestCase: tdLog.info("==========step3: query join again") self.sqlsquery() - def stop(self): - tdSql.close() rm_cmd = f"rm -f /tmp/insert* > /dev/null 2>&1" _ = subprocess.check_output(rm_cmd, shell=True).decode("utf-8") + + def stop(self): + tdSql.close() tdLog.success(f"{__file__} successfully executed") tdCases.addLinux(__file__, TDTestCase()) From 965d553964aea78aeb2e59b3bad9811ea08a86f5 Mon Sep 17 00:00:00 2001 From: wu champion Date: Tue, 23 Mar 2021 13:59:29 +0800 Subject: [PATCH 06/71] fix the case:add delete tmp file step --- tests/pytest/query/query1970YearsAf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/pytest/query/query1970YearsAf.py b/tests/pytest/query/query1970YearsAf.py index 9cc8948c2f..ea83d8669d 100644 --- a/tests/pytest/query/query1970YearsAf.py +++ b/tests/pytest/query/query1970YearsAf.py @@ -87,7 +87,7 @@ class TDTestCase: "disorder_ratio": 0, "disorder_range": 1000, "timestamp_step": 20000, - "start_timestamp": "1969-12-30 23:59:40.000", + "start_timestamp": "1969-12-31 00:00:00.000", "sample_format": "csv", "sample_file": "./sample.csv", "tags_file": "", @@ -172,7 +172,7 @@ class TDTestCase: ) tdSql.checkRows(3590) - # child-table query + # child-tables query tdSql.query( "select * from t0 where t0.ts < '1970-01-01 00:00:00.000' " ) @@ -243,6 +243,7 @@ class TDTestCase: tdLog.info("==========step3: query join again") self.sqlsquery() + # delete temporary file rm_cmd = f"rm -f /tmp/insert* > /dev/null 2>&1" _ = subprocess.check_output(rm_cmd, shell=True).decode("utf-8") From 5f27c38a0b0753d1ff9bcab9fbf64dd378d923ab Mon Sep 17 00:00:00 2001 From: wu champion <72908628+wu-champion@users.noreply.github.com> Date: Tue, 23 Mar 2021 14:10:25 +0800 Subject: [PATCH 07/71] Update query1970YearsAf.py --- tests/pytest/query/query1970YearsAf.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/tests/pytest/query/query1970YearsAf.py b/tests/pytest/query/query1970YearsAf.py index 307180a29c..b3f3ff7a82 100644 --- a/tests/pytest/query/query1970YearsAf.py +++ b/tests/pytest/query/query1970YearsAf.py @@ -87,11 +87,7 @@ class TDTestCase: "disorder_ratio": 0, "disorder_range": 1000, "timestamp_step": 20000, -<<<<<<< HEAD "start_timestamp": "1969-12-31 00:00:00.000", -======= - "start_timestamp": "1969-12-30 23:59:40.000", ->>>>>>> 53a128b2a3a469b06ae28ff013397b8d015433c8 "sample_format": "csv", "sample_file": "./sample.csv", "tags_file": "", @@ -176,11 +172,7 @@ class TDTestCase: ) tdSql.checkRows(3590) -<<<<<<< HEAD # child-tables query -======= - # child-table query ->>>>>>> 53a128b2a3a469b06ae28ff013397b8d015433c8 tdSql.query( "select * from t0 where t0.ts < '1970-01-01 00:00:00.000' " ) @@ -251,20 +243,13 @@ class TDTestCase: tdLog.info("==========step3: query join again") self.sqlsquery() -<<<<<<< HEAD # delete temporary file rm_cmd = f"rm -f /tmp/insert* > /dev/null 2>&1" _ = subprocess.check_output(rm_cmd, shell=True).decode("utf-8") def stop(self): tdSql.close() -======= - def stop(self): - tdSql.close() - rm_cmd = f"rm -f /tmp/insert* > /dev/null 2>&1" - _ = subprocess.check_output(rm_cmd, shell=True).decode("utf-8") ->>>>>>> 53a128b2a3a469b06ae28ff013397b8d015433c8 tdLog.success(f"{__file__} successfully executed") tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase()) From 3ad4453196f8784b2bcae1c82e49beee63569095 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Wed, 24 Mar 2021 13:22:22 +0800 Subject: [PATCH 08/71] fix bug --- src/client/src/tscServer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index b57cc559c3..6f19d49e9a 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2075,7 +2075,8 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { pInfo->vgroupList->numOfVgroups = pVgroupMsg->numOfVgroups; if (pInfo->vgroupList->numOfVgroups <= 0) { - tfree(pInfo->vgroupList); + //tfree(pInfo->vgroupList); + tscError("%p empty vgroup info", pSql); } else { for (int32_t j = 0; j < pInfo->vgroupList->numOfVgroups; ++j) { //just init, no need to lock From e113aa20ba9d7bceb403cc7068ce92a598d28adf Mon Sep 17 00:00:00 2001 From: wu champion Date: Thu, 25 Mar 2021 14:58:18 +0800 Subject: [PATCH 09/71] add case for TD-3433 --- tests/pytest/perfbenchmark/bug3433.py | 245 ++++++++++++++++++++++++++ 1 file changed, 245 insertions(+) create mode 100644 tests/pytest/perfbenchmark/bug3433.py diff --git a/tests/pytest/perfbenchmark/bug3433.py b/tests/pytest/perfbenchmark/bug3433.py new file mode 100644 index 0000000000..2d9c336113 --- /dev/null +++ b/tests/pytest/perfbenchmark/bug3433.py @@ -0,0 +1,245 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import json + +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def __init__(self): + self.path = "" + + def init(self, conn, logSql): + tdLog.debug(f"start to execute {__file__}") + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def getcfgPath(self, path): + binPath = os.path.dirname(os.path.realpath(__file__)) + binPath = binPath + "/../../../debug/" + tdLog.debug(f"binPath {binPath}") + binPath = os.path.realpath(binPath) + tdLog.debug(f"binPath real path {binPath}") + if path == "": + self.path = os.path.abspath(binPath + "../../") + else: + self.path = os.path.realpath(path) + return self.path + + def getCfgDir(self): + self.getcfgPath(self.path) + self.cfgDir = f"{self.path}/sim/psim/cfg" + return self.cfgDir + + def creatcfg(self): + dbinfo = { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 3650, + "minRows": 100, + "maxRows": 4096, + "comp": 2, + "walLevel": 1, + "cachelast": 0, + "quorum": 1, + "fsync": 3000, + "update": 0 + } + + # set stable schema + stable1 = { + "name": "stb1", + "child_table_exists": "no", + "childtable_count": 1000, + "childtable_prefix": "t1", + "auto_create_table": "no", + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 1, + "multi_thread_write_one_tbl": "no", + "number_of_tbl_in_one_sql": 0, + "rows_per_tbl": 1, + "max_sql_len": 65480, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 20000, + "start_timestamp": "2020-12-31 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [ + {"type": "INT", "count": 2}, + {"type": "DOUBLE", "count": 2}, + {"type": "BIGINT", "count": 2}, + {"type": "FLOAT", "count": 2}, + {"type": "SMALLINT", "count": 2}, + {"type": "TINYINT", "count": 2}, + {"type": "BOOL", "count": 2}, + {"type": "NCHAR", "len": 3, "count": 1}, + {"type": "BINARY", "len": 8, "count": 1} + + ], + "tags": [ + {"type": "INT", "count": 2}, + {"type": "DOUBLE", "count": 2}, + {"type": "BIGINT", "count": 2}, + {"type": "FLOAT", "count": 2}, + {"type": "SMALLINT", "count": 2}, + {"type": "TINYINT", "count": 2}, + {"type": "BOOL", "count": 2}, + {"type": "NCHAR", "len": 3, "count": 1}, + {"type": "BINARY", "len": 8, "count": 1} + ] + } + stable2 = { + "name": "stb2", + "child_table_exists": "no", + "childtable_count": 1000, + "childtable_prefix": "t2", + "auto_create_table": "no", + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 1, + "multi_thread_write_one_tbl": "no", + "number_of_tbl_in_one_sql": 0, + "rows_per_tbl": 1, + "max_sql_len": 65480, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 20000, + "start_timestamp": "2020-12-31 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [ + {"type": "INT", "count": 2}, + {"type": "DOUBLE", "count": 2}, + {"type": "BIGINT", "count": 2}, + {"type": "FLOAT", "count": 2}, + {"type": "SMALLINT", "count": 2}, + {"type": "TINYINT", "count": 2}, + {"type": "BOOL", "count": 2}, + {"type": "NCHAR", "len": 3, "count": 1}, + {"type": "BINARY", "len": 8, "count": 1} + + ], + "tags": [ + {"type": "INT", "count": 2}, + {"type": "DOUBLE", "count": 2}, + {"type": "BIGINT", "count": 2}, + {"type": "FLOAT", "count": 2}, + {"type": "SMALLINT", "count": 2}, + {"type": "TINYINT", "count": 2}, + {"type": "BOOL", "count": 2}, + {"type": "NCHAR", "len": 3, "count": 1}, + {"type": "BINARY", "len": 8, "count": 1} + ] + } + + # create different stables like stable1 and add to list super_tables + super_tables = [] + super_tables.append(stable1) + super_tables.append(stable2) + database = { + "dbinfo": dbinfo, + "super_tables": super_tables + } + + cfgdir = self.getCfgDir() + create_table = { + "filetype": "insert", + "cfgdir": cfgdir, + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "/tmp/insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 100, + "databases": [database] + } + return create_table + + def createinsertfile(self): + create_table = self.creatcfg() + date = datetime.datetime.now().strftime("%Y%m%d%H%M") + file_create_table = f"/tmp/insert_{date}.json" + + with open(file_create_table, 'w') as f: + json.dump(create_table, f) + return file_create_table + + def inserttable(self, filepath): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info(f"taosd found in {buildPath}") + binPath = buildPath + "/build/bin/" + + create_table_cmd = f"{binPath}taosdemo -f {filepath} > /dev/null 2>&1" + _ = subprocess.check_output(create_table_cmd, shell=True).decode("utf-8") + + def droptmpfile(self): + drop_file_cmd = "rm -f /tmp/insert_* " + _ = subprocess.check_output(drop_file_cmd, shell=True).decode("utf-8") + + def run(self): + tdLog.printNoPrefix("==========step1:create database and insert records") + file_create_table = self.createinsertfile() + self.inserttable(file_create_table) + + tdLog.printNoPrefix("==========step2:check database and stable records") + tdSql.query("show databases") + tdSql.checkData(0, 2, 2000) + tdSql.execute("use db") + tdSql.query("show stables") + tdSql.checkData(0, 4, 1000) + tdSql.checkData(1, 4, 1000) + + # self.droptmpfile() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From c7f4cd251dfca592b771e2e23db9ae0848c1dde0 Mon Sep 17 00:00:00 2001 From: wu champion Date: Thu, 25 Mar 2021 15:09:25 +0800 Subject: [PATCH 10/71] Update bug3433.py --- tests/pytest/perfbenchmark/bug3433.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytest/perfbenchmark/bug3433.py b/tests/pytest/perfbenchmark/bug3433.py index 2d9c336113..80e9cce0dc 100644 --- a/tests/pytest/perfbenchmark/bug3433.py +++ b/tests/pytest/perfbenchmark/bug3433.py @@ -235,11 +235,11 @@ class TDTestCase: tdSql.checkData(0, 4, 1000) tdSql.checkData(1, 4, 1000) - # self.droptmpfile() + self.droptmpfile() def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase()) From 4af02b782808093c6094017fe851f47d058bec63 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 25 Mar 2021 16:06:25 +0800 Subject: [PATCH 11/71] [TD-3036]: syncdb replica; resync slave vnodes to keep consistent with master vnodes. --- src/dnode/src/dnodePeer.c | 1 + src/dnode/src/dnodeVMgmt.c | 9 +++++++++ src/dnode/src/dnodeVnodes.c | 5 +++-- src/inc/taosmsg.h | 3 ++- src/inc/vnode.h | 3 ++- src/mnode/inc/mnodeVgroup.h | 1 + src/mnode/src/mnodeDb.c | 38 ++++++++++++++++++++++++++++++++++++- src/mnode/src/mnodeVgroup.c | 38 +++++++++++++++++++++++++++++++++++++ src/sync/src/syncMain.c | 2 +- src/vnode/inc/vnodeMain.h | 3 ++- src/vnode/src/vnodeMain.c | 17 +++++++++++++++++ 11 files changed, 113 insertions(+), 7 deletions(-) diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c index 79c60874f9..b8ce1c802b 100644 --- a/src/dnode/src/dnodePeer.c +++ b/src/dnode/src/dnodePeer.c @@ -43,6 +43,7 @@ int32_t dnodeInitServer() { dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeDispatchToVMgmtQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = dnodeDispatchToVMgmtQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_SYNC_VNODE] = dnodeDispatchToVMgmtQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = dnodeDispatchToVMgmtQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeDispatchToVMgmtQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeDispatchToVMgmtQueue; diff --git a/src/dnode/src/dnodeVMgmt.c b/src/dnode/src/dnodeVMgmt.c index 1e428fc8b1..66c94bf675 100644 --- a/src/dnode/src/dnodeVMgmt.c +++ b/src/dnode/src/dnodeVMgmt.c @@ -30,6 +30,7 @@ static taos_queue tsVMgmtQueue = NULL; static void * dnodeProcessMgmtQueue(void *param); static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *pMsg); static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *pMsg); +static int32_t dnodeProcessSyncVnodeMsg(SRpcMsg *pMsg); static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *pMsg); static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg); static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg); @@ -39,6 +40,7 @@ static int32_t (*dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *pMsg); int32_t dnodeInitVMgmt() { dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeProcessCreateVnodeMsg; dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = dnodeProcessAlterVnodeMsg; + dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_SYNC_VNODE] = dnodeProcessSyncVnodeMsg; dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = dnodeProcessDropVnodeMsg; dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeProcessAlterStreamMsg; dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeProcessConfigDnodeMsg; @@ -179,6 +181,13 @@ static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) { } } +static int32_t dnodeProcessSyncVnodeMsg(SRpcMsg *rpcMsg) { + SSyncVnodeMsg *pSyncVnode = rpcMsg->pCont; + pSyncVnode->vgId = htonl(pSyncVnode->vgId); + + return vnodeSync(pSyncVnode->vgId); +} + static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg) { SDropVnodeMsg *pDrop = rpcMsg->pCont; pDrop->vgId = htonl(pDrop->vgId); diff --git a/src/dnode/src/dnodeVnodes.c b/src/dnode/src/dnodeVnodes.c index d00314fcbc..d96251cebe 100644 --- a/src/dnode/src/dnodeVnodes.c +++ b/src/dnode/src/dnodeVnodes.c @@ -202,10 +202,11 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { char clusterId[TSDB_CLUSTER_ID_LEN]; dnodeGetClusterId(clusterId); if (clusterId[0] != '\0') { - dError("exit zombie dropped dnode"); - exit(EXIT_FAILURE); + dError("exit zombie dropped dnode"); + exit(EXIT_FAILURE); } } + taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); return; } diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index b9f45bd0d9..914738338b 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -59,6 +59,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_DROP_STABLE, "drop-stable" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_ALTER_STREAM, "alter-stream" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_CONFIG_DNODE, "config-dnode" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_ALTER_VNODE, "alter-vnode" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_SYNC_VNODE, "sync-vnode" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_CREATE_MNODE, "create-mnode" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY6, "dummy6" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY7, "dummy7" ) @@ -389,7 +390,7 @@ typedef struct { typedef struct { int32_t vgId; -} SDropVnodeMsg; +} SDropVnodeMsg, SSyncVnodeMsg; typedef struct SColIndex { int16_t colId; // column id diff --git a/src/inc/vnode.h b/src/inc/vnode.h index dddec83da8..39bd2f46c3 100644 --- a/src/inc/vnode.h +++ b/src/inc/vnode.h @@ -60,6 +60,7 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg); int32_t vnodeDrop(int32_t vgId); int32_t vnodeOpen(int32_t vgId); int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg); +int32_t vnodeSync(int32_t vgId); int32_t vnodeClose(int32_t vgId); // vnodeMgmt @@ -89,4 +90,4 @@ int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pRead); } #endif -#endif \ No newline at end of file +#endif diff --git a/src/mnode/inc/mnodeVgroup.h b/src/mnode/inc/mnodeVgroup.h index e052cdb83c..7b798c23f8 100644 --- a/src/mnode/inc/mnodeVgroup.h +++ b/src/mnode/inc/mnodeVgroup.h @@ -49,6 +49,7 @@ void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SCTableObj *pTable); void mnodeSendDropVnodeMsg(int32_t vgId, SRpcEpSet *epSet, void *ahandle); void mnodeSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle); void mnodeSendAlterVgroupMsg(SVgObj *pVgroup); +void mnodeSendSyncVgroupMsg(SVgObj *pVgroup); SRpcEpSet mnodeGetEpSetFromVgroup(SVgObj *pVgroup); SRpcEpSet mnodeGetEpSetFromIp(char *ep); diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 0a82b8f27c..909ca7cac6 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -1186,8 +1186,44 @@ static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg) { return mnodeDropDb(pMsg); } +static int32_t mnodeSyncDb(SDbObj *pDb, SMnodeMsg *pMsg) { + void *pIter = NULL; + SVgObj *pVgroup = NULL; + while (1) { + pIter = mnodeGetNextVgroup(pIter, &pVgroup); + if (pVgroup == NULL) break; + if (pVgroup->pDb == pDb) { + mnodeSendSyncVgroupMsg(pVgroup); + } + mnodeDecVgroupRef(pVgroup); + } + + mLInfo("db:%s, is synced by %s", pDb->name, mnodeGetUserFromMsg(pMsg)); + + return TSDB_CODE_SUCCESS; +} + static int32_t mnodeProcessSyncDbMsg(SMnodeMsg *pMsg) { - return 0; + SSyncDbMsg *pSyncDb = pMsg->rpcMsg.pCont; + mDebug("db:%s, syncdb is received from thandle:%p, ignore:%d", pSyncDb->db, pMsg->rpcMsg.handle, pSyncDb->ignoreNotExists); + + if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pSyncDb->db); + if (pMsg->pDb == NULL) { + if (pSyncDb->ignoreNotExists) { + mDebug("db:%s, db is not exist, treat as success", pSyncDb->db); + return TSDB_CODE_SUCCESS; + } else { + mError("db:%s, failed to sync, invalid db", pSyncDb->db); + return TSDB_CODE_MND_INVALID_DB; + } + } + + if (pMsg->pDb->status != TSDB_DB_STATUS_READY) { + mError("db:%s, status:%d, in dropping", pSyncDb->db, pMsg->pDb->status); + return TSDB_CODE_MND_DB_IN_DROPPING; + } + + return mnodeSyncDb(pMsg->pDb, pMsg); } void mnodeDropAllDbs(SAcctObj *pAcct) { diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index fdbf7ae398..7eb3122d83 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -60,6 +60,7 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg); static void mnodeProcessAlterVnodeRsp(SRpcMsg *rpcMsg); +static void mnodeProcessSyncVnodeRsp(SRpcMsg *rpcMsg); static void mnodeProcessDropVnodeRsp(SRpcMsg *rpcMsg); static int32_t mnodeProcessVnodeCfgMsg(SMnodeMsg *pMsg) ; static void mnodeSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle); @@ -236,6 +237,7 @@ int32_t mnodeInitVgroups() { mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_VGROUP, mnodeCancelGetNextVgroup); mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP, mnodeProcessCreateVnodeRsp); mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP, mnodeProcessAlterVnodeRsp); + mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP, mnodeProcessSyncVnodeRsp); mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_DROP_VNODE_RSP, mnodeProcessDropVnodeRsp); mnodeAddPeerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_VNODE, mnodeProcessVnodeCfgMsg); @@ -967,6 +969,38 @@ void mnodeSendAlterVgroupMsg(SVgObj *pVgroup) { } } +static SSyncVnodeMsg *mnodeBuildSyncVnodeMsg(int32_t vgId) { + SSyncVnodeMsg *pSyncVnode = rpcMallocCont(sizeof(SSyncVnodeMsg)); + if (pSyncVnode == NULL) return NULL; + + pSyncVnode->vgId = htonl(vgId); + return pSyncVnode; +} + +static void mnodeSendSyncVnodeMsg(SVgObj *pVgroup, SRpcEpSet *epSet) { + SSyncVnodeMsg *pSyncVnode = mnodeBuildSyncVnodeMsg(pVgroup->vgId); + SRpcMsg rpcMsg = { + .ahandle = NULL, + .pCont = pSyncVnode, + .contLen = pSyncVnode ? sizeof(SSyncVnodeMsg) : 0, + .code = 0, + .msgType = TSDB_MSG_TYPE_MD_SYNC_VNODE + }; + + dnodeSendMsgToDnode(epSet, &rpcMsg); +} + +void mnodeSendSyncVgroupMsg(SVgObj *pVgroup) { + mDebug("vgId:%d, send sync all vnodes msg, numOfVnodes:%d db:%s", pVgroup->vgId, pVgroup->numOfVnodes, + pVgroup->dbName); + for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { + SRpcEpSet epSet = mnodeGetEpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp); + mDebug("vgId:%d, index:%d, send sync vnode msg to dnode %s", pVgroup->vgId, i, + pVgroup->vnodeGid[i].pDnode->dnodeEp); + mnodeSendSyncVnodeMsg(pVgroup, &epSet); + } +} + static void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcEpSet *epSet, void *ahandle) { SCreateVnodeMsg *pCreate = mnodeBuildVnodeMsg(pVgroup); SRpcMsg rpcMsg = { @@ -994,6 +1028,10 @@ static void mnodeProcessAlterVnodeRsp(SRpcMsg *rpcMsg) { mDebug("alter vnode rsp received"); } +static void mnodeProcessSyncVnodeRsp(SRpcMsg *rpcMsg) { + mDebug("sync vnode rsp received"); +} + static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { if (rpcMsg->ahandle == NULL) return; diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 264bbf6b92..76d4379c5f 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -410,7 +410,7 @@ void syncConfirmForward(int64_t rid, uint64_t version, int32_t code, bool force) syncReleaseNode(pNode); } -#if 0 +#if 1 void syncRecover(int64_t rid) { SSyncPeer *pPeer; diff --git a/src/vnode/inc/vnodeMain.h b/src/vnode/inc/vnodeMain.h index 73591bc10d..91a5d632cd 100644 --- a/src/vnode/inc/vnodeMain.h +++ b/src/vnode/inc/vnodeMain.h @@ -25,6 +25,7 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg); int32_t vnodeDrop(int32_t vgId); int32_t vnodeOpen(int32_t vgId); int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg); +int32_t vnodeSync(int32_t vgId); int32_t vnodeClose(int32_t vgId); void vnodeCleanUp(SVnodeObj *pVnode); void vnodeDestroy(SVnodeObj *pVnode); @@ -33,4 +34,4 @@ void vnodeDestroy(SVnodeObj *pVnode); } #endif -#endif \ No newline at end of file +#endif diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index ded39e67cc..441e951250 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -91,6 +91,23 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) { return code; } +int32_t vnodeSync(int32_t vgId) { + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vDebug("vgId:%d, failed to sync, vnode not find", vgId); + return TSDB_CODE_VND_INVALID_VGROUP_ID; + } + + if (pVnode->role != TAOS_SYNC_ROLE_MASTER) { + vInfo("vgId:%d, vnode will sync, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); + syncRecover(pVnode->sync); + } + + vnodeRelease(pVnode); + + return TSDB_CODE_SUCCESS; +} + int32_t vnodeDrop(int32_t vgId) { SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) { From 9887a58e7439f41a3c127580f9ef530116a9c820 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Thu, 25 Mar 2021 16:46:56 +0800 Subject: [PATCH 12/71] [TD-1139]: add c examples into CI --- Jenkinsfile | 5 +++ tests/Jenkinsfile | 5 +++ tests/test-all.sh | 86 ++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 91 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e1bb27bf67..0d1891b5d5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -273,6 +273,11 @@ pipeline { cd ${WKC}/tests ./test-all.sh b7fq date''' + sh ''' + date + cd ${WKC}/tests + ./test-all.sh full example + date''' } } } diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile index 7cdcfb2e24..11bf2f1640 100644 --- a/tests/Jenkinsfile +++ b/tests/Jenkinsfile @@ -162,6 +162,11 @@ pipeline { cd ${WKC}/tests ./test-all.sh b3 date''' + sh ''' + date + cd ${WKC}/tests + ./test-all.sh full example + date''' } } diff --git a/tests/test-all.sh b/tests/test-all.sh index b8f06d8ede..9382232877 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -194,9 +194,10 @@ totalFailed=0 totalPyFailed=0 totalJDBCFailed=0 totalUnitFailed=0 +totalExampleFailed=0 corepath=`grep -oP '.*(?=core_)' /proc/sys/kernel/core_pattern||grep -oP '.*(?=core-)' /proc/sys/kernel/core_pattern` -if [ "$2" != "jdbc" ] && [ "$2" != "python" ] && [ "$2" != "unit" ]; then +if [ "$2" != "jdbc" ] && [ "$2" != "python" ] && [ "$2" != "unit" ] && [ "$2" != "example" ]; then echo "### run TSIM test case ###" cd $tests_dir/script @@ -265,7 +266,7 @@ if [ "$2" != "jdbc" ] && [ "$2" != "python" ] && [ "$2" != "unit" ]; then fi fi -if [ "$2" != "sim" ] && [ "$2" != "jdbc" ] && [ "$2" != "unit" ]; then +if [ "$2" != "sim" ] && [ "$2" != "jdbc" ] && [ "$2" != "unit" ] && [ "$2" != "example" ]; then echo "### run Python test case ###" cd $tests_dir @@ -334,7 +335,7 @@ if [ "$2" != "sim" ] && [ "$2" != "jdbc" ] && [ "$2" != "unit" ]; then fi -if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "unit" ] && [ "$1" == "full" ]; then +if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "unit" ] && [ "$2" != "example" ] && [ "$1" == "full" ]; then echo "### run JDBC test cases ###" cd $tests_dir @@ -378,7 +379,7 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "unit" ] && [ "$1" == dohavecore 1 fi -if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$1" == "full" ]; then +if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != "example" ] && [ "$1" == "full" ]; then echo "### run Unit tests ###" stopTaosd @@ -414,5 +415,80 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$1" == dohavecore 1 fi +if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != "unit" ] && [ "$1" == "full" ]; then + echo "### run Example tests ###" -exit $(($totalFailed + $totalPyFailed + $totalJDBCFailed + $totalUnitFailed)) + stopTaosd + cd $tests_dir + + if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + cd ../../ + else + cd ../ + fi + + pwd + cd debug/build/bin + rm -rf /var/lib/taos/* + nohup ./taosd -c /etc/taos/ > /dev/null 2>&1 & + echo "sleeping for 30 seconds" + #sleep 30 + + cd $tests_dir + echo "current dir: " + pwd + cd examples/c + echo "building applications" + make > /dev/null + totalExamplePass=0 + + echo "Running tests" + ./apitest > /dev/null 2>&1 + if [ $? != "0" ]; then + echo "prepare failed" + totalExampleFailed=`expr $totalExampleFailed + 1` + else + echo "prepare pass" + totalExamplePass=`expr $totalExamplePass + 1` + fi + + ./prepare 127.0.0.1 > /dev/null 2>&1 + if [ $? != "0" ]; then + echo "prepare failed" + totalExampleFailed=`expr $totalExampleFailed + 1` + else + echo "prepare pass" + totalExamplePass=`expr $totalExamplePass + 1` + fi + + ./subscribe -test > /dev/null 2>&1 + if [ $? != "0" ]; then + echo "prepare failed" + totalExampleFailed=`expr $totalExampleFailed + 1` + else + echo "prepare pass" + totalExamplePass=`expr $totalExamplePass + 1` + fi + + yes |./asyncdemo 127.0.0.1 test 1000 10 > /dev/null 2>&1 + if [ $? != "0" ]; then + echo "prepare failed" + totalExampleFailed=`expr $totalExampleFailed + 1` + else + echo "prepare pass" + totalExamplePass=`expr $totalExamplePass + 1` + fi + + if [ "$totalExamplePass" -gt "0" ]; then + echo -e "\n${GREEN} ### Total $totalExamplePass examples succeed! ### ${NC}" + fi + + if [ "$totalExampleFailed" -ne "0" ]; then + echo -e "\n${RED} ### Total $totalExampleFailed examples failed! ### ${NC}" + fi + + dohavecore 1 +fi + + +exit $(($totalFailed + $totalPyFailed + $totalJDBCFailed + $totalUnitFailed + $totalExampleFailed)) From fd0a3a9603562b3c6a1493b8a5787459cc20cc5c Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 26 Mar 2021 01:17:47 +0800 Subject: [PATCH 13/71] change --- Jenkinsfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f0e89c9004..7a4a09ff1c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -215,6 +215,11 @@ pipeline { cd ${WKC}/tests ./test-all.sh b3fq date''' + sh ''' + date + cd ${WKC}/tests + ./test-all.sh full example + date''' } } } @@ -272,12 +277,7 @@ pipeline { date cd ${WKC}/tests ./test-all.sh b7fq - date''' - sh ''' - date - cd ${WKC}/tests - ./test-all.sh full example - date''' + date''' } } } From 9db07982b274464e734b3eb4ce21bcc5eb781042 Mon Sep 17 00:00:00 2001 From: wu champion Date: Fri, 26 Mar 2021 10:50:14 +0800 Subject: [PATCH 14/71] fix --- tests/pytest/fulltest.sh | 4 ++++ tests/pytest/perfbenchmark/bug3433.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index fc86570bd2..a631149c90 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -164,6 +164,10 @@ python3 ./test.py -f user/pass_len.py # stable python3 ./test.py -f stable/query_after_reset.py +# perfbenchmark +python3 ./test.py -f perfbenchmark/bug3433.py + + #query python3 ./test.py -f query/filter.py python3 ./test.py -f query/filterCombo.py diff --git a/tests/pytest/perfbenchmark/bug3433.py b/tests/pytest/perfbenchmark/bug3433.py index 2d9c336113..117b611dc4 100644 --- a/tests/pytest/perfbenchmark/bug3433.py +++ b/tests/pytest/perfbenchmark/bug3433.py @@ -235,7 +235,7 @@ class TDTestCase: tdSql.checkData(0, 4, 1000) tdSql.checkData(1, 4, 1000) - # self.droptmpfile() + self.droptmpfile() def stop(self): tdSql.close() From 6e5a3fe4ad9042b44a4edeeb30383cde46e1ddc0 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 26 Mar 2021 15:06:06 +0800 Subject: [PATCH 15/71] [TD-3185]: default numOfCommitThreads to 4, show in cfg template --- packaging/cfg/taos.cfg | 3 +++ src/common/src/tglobal.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index a1178e2eef..83b70ed9f8 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -29,6 +29,9 @@ # number of threads per CPU core # numOfThreadsPerCore 1.0 +# number of threads to commit cache data +# numOfCommitThreads 4 + # the proportion of total CPU cores available for query processing # 2.0: the query threads will be set to double of the CPU cores. # 1.0: all CPU cores are available for query processing [default]. diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 4fef3844c4..5f4ce046ed 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -51,7 +51,7 @@ int32_t tsMaxShellConns = 50000; int32_t tsMaxConnections = 5000; int32_t tsShellActivityTimer = 3; // second float tsNumOfThreadsPerCore = 1.0f; -int32_t tsNumOfCommitThreads = 1; +int32_t tsNumOfCommitThreads = 4; float tsRatioOfQueryCores = 1.0f; int8_t tsDaylight = 0; char tsTimezone[TSDB_TIMEZONE_LEN] = {0}; From 5bfda528e82f47f2bdee99762e6cb2e368cba4b0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 26 Mar 2021 17:00:57 +0800 Subject: [PATCH 16/71] [TD-3534]: fix TD-3534 --- src/tsdb/src/tsdbCommit.c | 8 ++++++++ src/tsdb/src/tsdbFile.c | 8 ++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 7cbc681149..dd0fd18019 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -452,6 +452,14 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { return -1; } + if (tsdbUpdateDFileSetHeader(&(pCommith->wSet)) < 0) { + tsdbError("vgId:%d failed to update FSET %d header since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); + tsdbCloseCommitFile(pCommith, true); + // revert the file change + tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); + return -1; + } + // Close commit file tsdbCloseCommitFile(pCommith, false); diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index ac33096aae..304ba2f073 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -134,14 +134,14 @@ int tsdbCreateMFile(SMFile *pMFile, bool updateHeader) { return 0; } + pMFile->info.size += TSDB_FILE_HEAD_SIZE; + if (tsdbUpdateMFileHeader(pMFile) < 0) { tsdbCloseMFile(pMFile); tsdbRemoveMFile(pMFile); return -1; } - pMFile->info.size += TSDB_FILE_HEAD_SIZE; - return 0; } @@ -378,14 +378,14 @@ int tsdbCreateDFile(SDFile *pDFile, bool updateHeader) { return 0; } + pDFile->info.size += TSDB_FILE_HEAD_SIZE; + if (tsdbUpdateDFileHeader(pDFile) < 0) { tsdbCloseDFile(pDFile); tsdbRemoveDFile(pDFile); return -1; } - pDFile->info.size += TSDB_FILE_HEAD_SIZE; - return 0; } From 6ff7ca4e1889f505bf21c9d307575a9110234f67 Mon Sep 17 00:00:00 2001 From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Fri, 26 Mar 2021 17:04:04 +0800 Subject: [PATCH 17/71] Update docs.md change jdbcType and SQLType --- documentation20/cn/08.connector/01.java/docs.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/documentation20/cn/08.connector/01.java/docs.md b/documentation20/cn/08.connector/01.java/docs.md index 3ed7343579..30b485f9b6 100644 --- a/documentation20/cn/08.connector/01.java/docs.md +++ b/documentation20/cn/08.connector/01.java/docs.md @@ -471,9 +471,11 @@ TDengine 目前支持时间戳、数字、字符、布尔类型，与 Java 对 | BIGINT | java.lang.Long | | FLOAT | java.lang.Float | | DOUBLE | java.lang.Double | -| SMALLINT, TINYINT | java.lang.Short | +| SMALLINT | java.lang.Short | +| TINYINT | java.lang.Byte | | BOOL | java.lang.Boolean | -| BINARY, NCHAR | java.lang.String | +| BINARY | byte array | +| NCHAR | java.lang.String | From c89ada70fd3869b8abb71e2692135626699aa3e7 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 26 Mar 2021 19:29:17 +0800 Subject: [PATCH 18/71] [TD-3524] : taosdemo query multithreading workaround. (#5587) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 134 ++++++++++++++++++++++++++++-------- 1 file changed, 105 insertions(+), 29 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 1048d7b969..13b4d14f52 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -5418,6 +5418,22 @@ static int insertTestProcess() { static void *superQueryProcess(void *sarg) { threadInfo *winfo = (threadInfo *)sarg; + if (winfo->taos == NULL) { + TAOS * taos = NULL; + taos = taos_connect(g_queryInfo.host, + g_queryInfo.user, + g_queryInfo.password, + NULL, + g_queryInfo.port); + if (taos == NULL) { + errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", + winfo->threadID, taos_errstr(NULL)); + return NULL; + } else { + winfo->taos = taos; + } + } + //char sqlStr[MAX_TB_NAME_SIZE*2]; //sprintf(sqlStr, "use %s", g_queryInfo.dbName); //queryDB(winfo->taos, sqlStr); @@ -5493,6 +5509,23 @@ static void replaceSubTblName(char* inSql, char* outSql, int tblIndex) { static void *subQueryProcess(void *sarg) { char sqlstr[1024]; threadInfo *winfo = (threadInfo *)sarg; + + if (winfo->taos == NULL) { + TAOS * taos = NULL; + taos = taos_connect(g_queryInfo.host, + g_queryInfo.user, + g_queryInfo.password, + NULL, + g_queryInfo.port); + if (taos == NULL) { + errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", + winfo->threadID, taos_errstr(NULL)); + return NULL; + } else { + winfo->taos = taos; + } + } + int64_t st = 0; int64_t et = (int64_t)g_queryInfo.subQueryInfo.rate*1000; int queryTimes = g_args.query_times; @@ -5534,12 +5567,12 @@ static int queryTestProcess() { setupForAnsiEscape(); printfQueryMeta(); resetAfterAnsiEscape(); - - TAOS * taos = NULL; - taos = taos_connect(g_queryInfo.host, - g_queryInfo.user, - g_queryInfo.password, - NULL, + + TAOS * taos = NULL; + taos = taos_connect(g_queryInfo.host, + g_queryInfo.user, + g_queryInfo.password, + NULL, g_queryInfo.port); if (taos == NULL) { errorPrint( "Failed to connect to TDengine, reason:%s\n", @@ -5554,14 +5587,14 @@ static int queryTestProcess() { &g_queryInfo.subQueryInfo.childTblName, &g_queryInfo.subQueryInfo.childTblCount); } - + if (!g_args.answer_yes) { printf("Press enter key to continue\n\n"); (void)getchar(); } printfQuerySystemInfo(taos); - + pthread_t *pids = NULL; threadInfo *infos = NULL; //==== create sub threads for query from specify table @@ -5585,28 +5618,30 @@ static int queryTestProcess() { t_info->threadID = i; if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { - t_info->taos = taos; char sqlStr[MAX_TB_NAME_SIZE*2]; sprintf(sqlStr, "use %s", g_queryInfo.dbName); verbosePrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr); - if (0 != queryDbExec(t_info->taos, sqlStr, NO_INSERT_TYPE, false)) { + if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) { + taos_close(taos); free(infos); free(pids); errorPrint( "use database %s failed!\n\n", g_queryInfo.dbName); return -1; } - } else { - t_info->taos = NULL; } + t_info->taos = NULL;// TODO: workaround to use separate taos connection; + pthread_create(pids + i, NULL, superQueryProcess, t_info); } - }else { + } else { g_queryInfo.superQueryInfo.concurrent = 0; } + taos_close(taos); + pthread_t *pidsOfSub = NULL; threadInfo *infosOfSub = NULL; //==== create sub threads for query from all sub table of the super table @@ -5614,7 +5649,6 @@ static int queryTestProcess() { && (g_queryInfo.subQueryInfo.threadCnt > 0)) { pidsOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(pthread_t)); if (NULL == pidsOfSub) { - taos_close(taos); free(infos); free(pids); @@ -5623,7 +5657,6 @@ static int queryTestProcess() { infosOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(threadInfo)); if (NULL == infosOfSub) { - taos_close(taos); free(pidsOfSub); free(infos); free(pids); @@ -5653,7 +5686,7 @@ static int queryTestProcess() { t_info->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; startFrom = t_info->end_table_to + 1; - t_info->taos = taos; + t_info->taos = NULL; // TODO: workaround to use separate taos connection; pthread_create(pidsOfSub + i, NULL, subQueryProcess, t_info); } @@ -5676,7 +5709,7 @@ static int queryTestProcess() { tmfree((char*)pidsOfSub); tmfree((char*)infosOfSub); - taos_close(taos); +// taos_close(taos);// TODO: workaround to use separate taos connection; return 0; } @@ -5717,10 +5750,27 @@ static void *subSubscribeProcess(void *sarg) { threadInfo *winfo = (threadInfo *)sarg; char subSqlstr[1024]; + if (winfo->taos == NULL) { + TAOS * taos = NULL; + taos = taos_connect(g_queryInfo.host, + g_queryInfo.user, + g_queryInfo.password, + g_queryInfo.dbName, + g_queryInfo.port); + if (taos == NULL) { + errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", + winfo->threadID, taos_errstr(NULL)); + return NULL; + } else { + winfo->taos = taos; + } + } + char sqlStr[MAX_TB_NAME_SIZE*2]; sprintf(sqlStr, "use %s", g_queryInfo.dbName); debugPrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr); - if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE, false)){ + if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE, false)) { + taos_close(winfo->taos); return NULL; } @@ -5742,8 +5792,10 @@ static void *subSubscribeProcess(void *sarg) { if (g_queryInfo.subQueryInfo.result[i][0] != 0) { sprintf(tmpFile, "%s-%d", g_queryInfo.subQueryInfo.result[i], winfo->threadID); } - g_queryInfo.subQueryInfo.tsub[i] = subscribeImpl(winfo->taos, subSqlstr, topic, tmpFile); + g_queryInfo.subQueryInfo.tsub[i] = subscribeImpl( + winfo->taos, subSqlstr, topic, tmpFile); if (NULL == g_queryInfo.subQueryInfo.tsub[i]) { + taos_close(winfo->taos); return NULL; } } @@ -5777,16 +5829,35 @@ static void *subSubscribeProcess(void *sarg) { taos_unsubscribe(g_queryInfo.subQueryInfo.tsub[i], g_queryInfo.subQueryInfo.subscribeKeepProgress); } + + taos_close(winfo->taos); return NULL; } static void *superSubscribeProcess(void *sarg) { threadInfo *winfo = (threadInfo *)sarg; + if (winfo->taos == NULL) { + TAOS * taos = NULL; + taos = taos_connect(g_queryInfo.host, + g_queryInfo.user, + g_queryInfo.password, + g_queryInfo.dbName, + g_queryInfo.port); + if (taos == NULL) { + errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", + winfo->threadID, taos_errstr(NULL)); + return NULL; + } else { + winfo->taos = taos; + } + } + char sqlStr[MAX_TB_NAME_SIZE*2]; sprintf(sqlStr, "use %s", g_queryInfo.dbName); debugPrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr); if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE, false)) { + taos_close(winfo->taos); return NULL; } @@ -5812,6 +5883,7 @@ static void *superSubscribeProcess(void *sarg) { g_queryInfo.superQueryInfo.sql[i], topic, tmpFile); if (NULL == g_queryInfo.superQueryInfo.tsub[i]) { + taos_close(winfo->taos); return NULL; } } @@ -5844,6 +5916,8 @@ static void *superSubscribeProcess(void *sarg) { taos_unsubscribe(g_queryInfo.superQueryInfo.tsub[i], g_queryInfo.superQueryInfo.subscribeKeepProgress); } + + taos_close(winfo->taos); return NULL; } @@ -5851,10 +5925,10 @@ static int subscribeTestProcess() { setupForAnsiEscape(); printfQueryMeta(); resetAfterAnsiEscape(); - + if (!g_args.answer_yes) { printf("Press enter key to continue\n\n"); - (void)getchar(); + (void) getchar(); } TAOS * taos = NULL; @@ -5877,6 +5951,8 @@ static int subscribeTestProcess() { &g_queryInfo.subQueryInfo.childTblCount); } + taos_close(taos); // TODO: workaround to use separate taos connection; + pthread_t *pids = NULL; threadInfo *infos = NULL; //==== create sub threads for query from super table @@ -5892,18 +5968,17 @@ static int subscribeTestProcess() { infos = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(threadInfo)); if ((NULL == pids) || (NULL == infos)) { errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__); - taos_close(taos); exit(-1); } for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { threadInfo *t_info = infos + i; t_info->threadID = i; - t_info->taos = taos; + t_info->taos = NULL; // TODO: workaround to use separate taos connection; pthread_create(pids + i, NULL, superSubscribeProcess, t_info); } - //==== create sub threads for query from sub table + //==== create sub threads for query from sub table pthread_t *pidsOfSub = NULL; threadInfo *infosOfSub = NULL; if ((g_queryInfo.subQueryInfo.sqlCount > 0) @@ -5913,8 +5988,9 @@ static int subscribeTestProcess() { infosOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(threadInfo)); if ((NULL == pidsOfSub) || (NULL == infosOfSub)) { - printf("malloc failed for create threads\n"); - taos_close(taos); + errorPrint("%s() LN%d, malloc failed for create threads\n", + __func__, __LINE__); + // taos_close(taos); exit(-1); } @@ -5941,7 +6017,7 @@ static int subscribeTestProcess() { t_info->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; startFrom = t_info->end_table_to + 1; - t_info->taos = taos; + t_info->taos = NULL; // TODO: workaround to use separate taos connection; pthread_create(pidsOfSub + i, NULL, subSubscribeProcess, t_info); } g_queryInfo.subQueryInfo.threadCnt = threads; @@ -5949,7 +6025,7 @@ static int subscribeTestProcess() { for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { pthread_join(pids[i], NULL); - } + } tmfree((char*)pids); tmfree((char*)infos); @@ -5960,7 +6036,7 @@ static int subscribeTestProcess() { tmfree((char*)pidsOfSub); tmfree((char*)infosOfSub); - taos_close(taos); +// taos_close(taos); return 0; } From 4acaa5dfc031c9833767645d50264ecadd1c64d9 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 26 Mar 2021 20:46:54 +0800 Subject: [PATCH 19/71] Feature/sangshuduo/td 3317 taosdemo interlace (#5593) * [TD-3316] : add testcase for taosdemo limit and offset. check offset 0. * [TD-3316] : add testcase for taosdemo limit and offset. fix sample file import bug. * [TD-3316] : add test case for limit and offset. fix sample data issue. * [TD-3327] : fix taosdemo segfault when import data from sample data file. * [TD-3317] : make taosdemo support interlace mode. json parameter rows_per_tbl support. * [TD-3317] : support interlace mode. refactor * [TD-3317] : support interlace mode. refactor * [TD-3317] : support interlace mode insertion. refactor. * [TD-3317] : support interlace mode insertion. change json file. * [TD-3317] : support interlace mode insertion. fix multithread create table regression. * [TD-3317] : support interlace mode insertion. working but not perfect. * [TD-3317] : support interlace mode insertion. rename lowaTest with taosdemoTestWithJson * [TD-3317] : support interlace mode insertion. perfect * [TD-3317] : support interlace mode insertion. cleanup. * [TD-3317] : support interlace mode insertion. adjust algorithm of loop times. * [TD-3317] : support interlace mode insertion. fix delay time bug. * [TD-3317] : support interlace mode insertion. fix progressive timestamp bug. * [TD-3317] : support interlace mode insertion. add an option for performance print. * [TD-3317] : support interlace mode insertion. change json test case with less table for acceleration. * [TD-3317] : support interlace mode insertion. change progressive mode timestamp step and testcase. * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix taosdemo coverity scan issue. fix subscribeTest pids uninitialized. * [TD-3317] : support interlace mode insertion. add time shift for no sleep time. * [TD-3317] : support interlace insert. rework timestamp. * [TD-3317] : support interlace mode insertion. change rows_per_tbl to interlace_rows. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 13b4d14f52..97b661253a 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -393,6 +393,7 @@ typedef struct SThreadInfo_S { TAOS *taos; int threadID; char db_name[MAX_DB_NAME_SIZE+1]; + uint32_t time_precision; char fp[4096]; char tb_prefix[MAX_TB_NAME_SIZE]; int start_table_from; @@ -1269,7 +1270,6 @@ static void printfInsertMetaToFile(FILE* fp) { fprintf(fp, "resultFile: %s\n", g_Dbs.resultFile); fprintf(fp, "thread num of insert data: %d\n", g_Dbs.threadCount); fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl); - fprintf(fp, "insert interval: %d\n", g_args.insert_interval); fprintf(fp, "number of records per req: %d\n", g_args.num_of_RPR); fprintf(fp, "max sql length: %d\n", g_args.max_sql_len); fprintf(fp, "database count: %d\n", g_Dbs.dbCount); @@ -1355,7 +1355,10 @@ static void printfInsertMetaToFile(FILE* fp) { fprintf(fp, " dataSource: %s\n", g_Dbs.db[i].superTbls[j].dataSource); fprintf(fp, " insertMode: %s\n", g_Dbs.db[i].superTbls[j].insertMode); fprintf(fp, " insertRows: %"PRId64"\n", g_Dbs.db[i].superTbls[j].insertRows); - fprintf(fp, " insert interval: %d\n", g_Dbs.db[i].superTbls[j].insertInterval); + fprintf(fp, " interlace rows: %d\n", g_Dbs.db[i].superTbls[j].interlaceRows); + if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) { + fprintf(fp, " insert interval: %d\n", g_Dbs.db[i].superTbls[j].insertInterval); + } if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { fprintf(fp, " multiThreadWriteOneTbl: no\n"); @@ -4444,7 +4447,7 @@ static int generateSQLHead(char *tableName, int32_t tableSeq, return len; } -static int generateDataBuffer(char *pTblName, +static int generateProgressiveDataBuffer(char *pTblName, int32_t tableSeq, threadInfo *pThreadInfo, char *buffer, int64_t insertRows, @@ -4584,6 +4587,9 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { verbosePrint("[%d] %s() LN%d i=%d batchPerTblTimes=%d batchPerTbl = %d\n", pThreadInfo->threadID, __func__, __LINE__, i, batchPerTblTimes, batchPerTbl); + if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { + startTime = taosGetTimestamp(pThreadInfo->time_precision); + } generateDataTail( tableName, tableSeq, pThreadInfo, superTblInfo, batchPerTbl, pstr, insertRows, 0, @@ -4716,10 +4722,11 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { int timeStampStep = superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP; - int insert_interval = +/* int insert_interval = superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; uint64_t st = 0; uint64_t et = 0xffffffff; + */ pThreadInfo->totalInsertRows = 0; pThreadInfo->totalAffectedRows = 0; @@ -4735,9 +4742,11 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { verbosePrint("%s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, insertRows); for (int64_t i = 0; i < insertRows;) { + /* if (insert_interval) { st = taosGetTimestampUs(); } + */ char tableName[TSDB_TABLE_NAME_LEN]; getTableName(tableName, pThreadInfo, tableSeq); @@ -4745,7 +4754,7 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { __func__, __LINE__, pThreadInfo->threadID, tableSeq, tableName); - int generated = generateDataBuffer( + int generated = generateProgressiveDataBuffer( tableName, tableSeq, pThreadInfo, buffer, insertRows, i, start_time, &(pThreadInfo->samplePos)); @@ -4787,7 +4796,7 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { if (i >= insertRows) break; - +/* if (insert_interval) { et = taosGetTimestampUs(); @@ -4798,6 +4807,7 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { taosMsleep(sleep_time); // ms } } + */ } // num_of_DPT if ((tableSeq == pThreadInfo->ntables - 1) && superTblInfo && @@ -5061,6 +5071,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, threadInfo *t_info = infos + i; t_info->threadID = i; tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE); + t_info->time_precision = timePrec; t_info->superTblInfo = superTblInfo; t_info->start_time = start_time; From c0c282d40cd720cc393394efdd958a689bdbb933 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Fri, 26 Mar 2021 21:51:13 +0800 Subject: [PATCH 20/71] [TD-2639] : fix bold text in table title. --- documentation20/cn/02.getting-started/docs.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation20/cn/02.getting-started/docs.md b/documentation20/cn/02.getting-started/docs.md index c9c82942f3..3953e7b0b8 100644 --- a/documentation20/cn/02.getting-started/docs.md +++ b/documentation20/cn/02.getting-started/docs.md @@ -179,7 +179,7 @@ taos> select avg(f1), max(f2), min(f3) from test.t10 interval(10s); ### TDengine服务器支持的平台列表 -| | **CentOS** **6/7/8** | **Ubuntu** **16/18/20** | **Other Linux** | **统信****UOS** | **银河****/****中标麒麟** | **凝思** **V60/V80** | +| | **CentOS 6/7/8** | **Ubuntu 16/18/20** | **Other Linux** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | | -------------- | --------------------- | ------------------------ | --------------- | --------------- | ------------------------- | --------------------- | | X64 | ● | ● | | ○ | ● | ● | | 树莓派ARM32 | | ● | ● | | | | @@ -203,7 +203,7 @@ taos> select avg(f1), max(f2), min(f3) from test.t10 interval(10s); 对照矩阵如下： -| **CPU** | **X64 64bit** | | | **X86 32bit** | **ARM64** | **ARM32** | **MIPS ** **龙芯** | **Alpha ** **申威** | **X64 ** **海光** | +| **CPU** | **X64 64bit** | | | **X86 32bit** | **ARM64** | **ARM32** | **MIPS 龙芯** | **Alpha 申威** | **X64 海光** | | ----------- | --------------- | --------- | --------- | --------------- | --------- | --------- | ------------------- | -------------------- | ------------------ | | **OS** | **Linux** | **Win64** | **Win32** | **Win32** | **Linux** | **Linux** | **Linux** | **Linux** | **Linux** | | **C/C++** | ● | ● | ● | ○ | ● | ● | ● | ● | ● | From 428258d114a4f95f15b5517b78e550970c91f54a Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Fri, 26 Mar 2021 22:07:52 +0800 Subject: [PATCH 21/71] [TD-2639] : add a little space for better display format. --- documentation20/cn/02.getting-started/docs.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/documentation20/cn/02.getting-started/docs.md b/documentation20/cn/02.getting-started/docs.md index 3953e7b0b8..f86e616c42 100644 --- a/documentation20/cn/02.getting-started/docs.md +++ b/documentation20/cn/02.getting-started/docs.md @@ -182,15 +182,15 @@ taos> select avg(f1), max(f2), min(f3) from test.t10 interval(10s); | | **CentOS 6/7/8** | **Ubuntu 16/18/20** | **Other Linux** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | | -------------- | --------------------- | ------------------------ | --------------- | --------------- | ------------------------- | --------------------- | | X64 | ● | ● | | ○ | ● | ● | -| 树莓派ARM32 | | ● | ● | | | | -| 龙芯MIPS64 | | | ● | | | | -| 鲲鹏 ARM64 | | ○ | ○ | | ● | | -| 申威 Alpha64 | | | ○ | ● | | | -| 飞腾ARM64 | | ○优麒麟 | | | | | -| 海光X64 | ● | ● | ● | ○ | ● | ● | -| 瑞芯微ARM64/32 | | | ○ | | | | -| 全志ARM64/32 | | | ○ | | | | -| 炬力ARM64/32 | | | ○ | | | | +| 树莓派 ARM32 | | ● | ● | | | | +| 龙芯 MIPS64 | | | ● | | | | +| 鲲鹏 ARM64 | | ○ | ○ | | ● | | +| 申威 Alpha64 | | | ○ | ● | | | +| 飞腾 ARM64 | | ○ 优麒麟 | | | | | +| 海光 X64 | ● | ● | ● | ○ | ● | ● | +| 瑞芯微 ARM64/32 | | | ○ | | | | +| 全志 ARM64/32 | | | ○ | | | | +| 炬力 ARM64/32 | | | ○ | | | | | TI ARM32 | | | ○ | | | | 注： ● 表示经过官方测试验证， ○ 表示非官方测试验证。 From 2c4bcbe3061c86fc30a5ec66dc12ca0bf7eb2fe5 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 27 Mar 2021 13:30:14 +0800 Subject: [PATCH 22/71] Feature/sangshuduo/td 3317 taosdemo interlace (#5596) * [TD-3316] : add testcase for taosdemo limit and offset. check offset 0. * [TD-3316] : add testcase for taosdemo limit and offset. fix sample file import bug. * [TD-3316] : add test case for limit and offset. fix sample data issue. * [TD-3327] : fix taosdemo segfault when import data from sample data file. * [TD-3317] : make taosdemo support interlace mode. json parameter rows_per_tbl support. * [TD-3317] : support interlace mode. refactor * [TD-3317] : support interlace mode. refactor * [TD-3317] : support interlace mode insertion. refactor. * [TD-3317] : support interlace mode insertion. change json file. * [TD-3317] : support interlace mode insertion. fix multithread create table regression. * [TD-3317] : support interlace mode insertion. working but not perfect. * [TD-3317] : support interlace mode insertion. rename lowaTest with taosdemoTestWithJson * [TD-3317] : support interlace mode insertion. perfect * [TD-3317] : support interlace mode insertion. cleanup. * [TD-3317] : support interlace mode insertion. adjust algorithm of loop times. * [TD-3317] : support interlace mode insertion. fix delay time bug. * [TD-3317] : support interlace mode insertion. fix progressive timestamp bug. * [TD-3317] : support interlace mode insertion. add an option for performance print. * [TD-3317] : support interlace mode insertion. change json test case with less table for acceleration. * [TD-3317] : support interlace mode insertion. change progressive mode timestamp step and testcase. * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix taosdemo coverity scan issue. fix subscribeTest pids uninitialized. * [TD-3317] : support interlace mode insertion. add time shift for no sleep time. * [TD-3317] : support interlace insert. rework timestamp. * [TD-3317] : support interlace mode insertion. change rows_per_tbl to interlace_rows. * [TD-3317] : taosdemo suppoert interlace mode. remove trailing spaces. * [TD-3317] : taosdemo support interlace insertion. prompt if interlace > num_of_records_per_req Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 522 +++++++++++++++++++----------------- 1 file changed, 273 insertions(+), 249 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 97b661253a..9e8e974096 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -37,10 +37,10 @@ #include #include #include -#else +#else #include #include -#endif +#endif #include #include @@ -100,7 +100,7 @@ typedef enum CREATE_SUB_TALBE_MOD_EN { AUTO_CREATE_SUBTBL, NO_CREATE_SUBTBL } CREATE_SUB_TALBE_MOD_EN; - + typedef enum TALBE_EXISTS_EN { TBL_NO_EXISTS, TBL_ALREADY_EXISTS, @@ -108,7 +108,7 @@ typedef enum TALBE_EXISTS_EN { } TALBE_EXISTS_EN; enum MODE { - SYNC, + SYNC, ASYNC, MODE_BUT }; @@ -131,7 +131,7 @@ enum _show_db_index { TSDB_SHOW_DB_NTABLES_INDEX, TSDB_SHOW_DB_VGROUPS_INDEX, TSDB_SHOW_DB_REPLICA_INDEX, - TSDB_SHOW_DB_QUORUM_INDEX, + TSDB_SHOW_DB_QUORUM_INDEX, TSDB_SHOW_DB_DAYS_INDEX, TSDB_SHOW_DB_KEEP_INDEX, TSDB_SHOW_DB_CACHE_INDEX, @@ -153,10 +153,10 @@ enum _show_stables_index { TSDB_SHOW_STABLES_NAME_INDEX, TSDB_SHOW_STABLES_CREATED_TIME_INDEX, TSDB_SHOW_STABLES_COLUMNS_INDEX, - TSDB_SHOW_STABLES_METRIC_INDEX, - TSDB_SHOW_STABLES_UID_INDEX, + TSDB_SHOW_STABLES_METRIC_INDEX, + TSDB_SHOW_STABLES_UID_INDEX, TSDB_SHOW_STABLES_TID_INDEX, - TSDB_SHOW_STABLES_VGID_INDEX, + TSDB_SHOW_STABLES_VGID_INDEX, TSDB_MAX_SHOW_STABLES }; @@ -220,7 +220,7 @@ typedef struct SColumn_S { char field[TSDB_COL_NAME_LEN + 1]; char dataType[MAX_TB_NAME_SIZE]; int dataLen; - char note[128]; + char note[128]; } StrColumn; typedef struct SSuperTable_S { @@ -269,7 +269,7 @@ typedef struct SSuperTable_S { int tagSampleCount; int tagUsePos; - // statistics + // statistics int64_t totalInsertRows; int64_t totalAffectedRows; } SSuperTable; @@ -278,10 +278,10 @@ typedef struct { char name[TSDB_DB_NAME_LEN + 1]; char create_time[32]; int32_t ntables; - int32_t vgroups; + int32_t vgroups; int16_t replica; int16_t quorum; - int16_t days; + int16_t days; char keeplist[32]; int32_t cache; //MB int32_t blocks; @@ -296,14 +296,14 @@ typedef struct { char status[16]; } SDbInfo; -typedef struct SDbCfg_S { +typedef struct SDbCfg_S { // int maxtablesPerVnode; - int minRows; + int minRows; int maxRows; int comp; int walLevel; int cacheLast; - int fsync; + int fsync; int replica; int update; int keep; @@ -311,7 +311,7 @@ typedef struct SDbCfg_S { int cache; int blocks; int quorum; - char precision[MAX_TB_NAME_SIZE]; + char precision[MAX_TB_NAME_SIZE]; } SDbCfg; typedef struct SDataBase_S { @@ -333,7 +333,7 @@ typedef struct SDbs_S { bool insert_only; bool do_aggreFunc; bool queryMode; - + int threadCount; int threadCountByCreateTbl; int dbCount; @@ -372,7 +372,7 @@ typedef struct SubQueryInfo_S { char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; - + char* childTblName; } SubQueryInfo; @@ -386,7 +386,7 @@ typedef struct SQueryMetaInfo_S { char queryMode[MAX_TB_NAME_SIZE]; // taosc, restful SuperQueryInfo superQueryInfo; - SubQueryInfo subQueryInfo; + SubQueryInfo subQueryInfo; } SQueryMetaInfo; typedef struct SThreadInfo_S { @@ -424,7 +424,7 @@ typedef struct SThreadInfo_S { int64_t avgDelay; int64_t maxDelay; int64_t minDelay; - + } threadInfo; #ifdef WINDOWS @@ -463,12 +463,12 @@ static void setupForAnsiEscape(void) { if(!SetConsoleMode(g_stdoutHandle, mode)) { exit(GetLastError()); - } + } } static void resetAfterAnsiEscape(void) { // Reset colors - printf("\x1b[0m"); + printf("\x1b[0m"); // Reset console mode if(!SetConsoleMode(g_stdoutHandle, g_consoleMode)) { @@ -508,7 +508,7 @@ int32_t randint[MAX_PREPARED_RAND]; int64_t randbigint[MAX_PREPARED_RAND]; float randfloat[MAX_PREPARED_RAND]; double randdouble[MAX_PREPARED_RAND]; -char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)", +char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)", "max(col0)", "min(col0)", "first(col0)", "last(col0)"}; SArguments g_args = { @@ -517,7 +517,7 @@ SArguments g_args = { "127.0.0.1", // host 6030, // port "root", // user - #ifdef _TD_POWER_ + #ifdef _TD_POWER_ "powerdb", // password #else "taosdata", // password @@ -599,7 +599,7 @@ static void ERROR_EXIT(const char *msg) { perror(msg); exit(-1); } #define TD_VERNUMBER "unknown" #endif -#ifndef TAOSDEMO_STATUS +#ifndef TAOSDEMO_STATUS #define TAOSDEMO_STATUS "unknown" #endif @@ -619,62 +619,62 @@ static void printVersion() { static void printHelp() { char indent[10] = " "; - printf("%s%s%s%s\n", indent, "-f", indent, + printf("%s%s%s%s\n", indent, "-f", indent, "The meta file to the execution procedure. Default is './meta.json'."); - printf("%s%s%s%s\n", indent, "-u", indent, + printf("%s%s%s%s\n", indent, "-u", indent, "The TDengine user name to use when connecting to the server. Default is 'root'."); #ifdef _TD_POWER_ - printf("%s%s%s%s\n", indent, "-P", indent, + printf("%s%s%s%s\n", indent, "-P", indent, "The password to use when connecting to the server. Default is 'powerdb'."); - printf("%s%s%s%s\n", indent, "-c", indent, + printf("%s%s%s%s\n", indent, "-c", indent, "Configuration directory. Default is '/etc/power/'."); #else - printf("%s%s%s%s\n", indent, "-P", indent, + printf("%s%s%s%s\n", indent, "-P", indent, "The password to use when connecting to the server. Default is 'taosdata'."); - printf("%s%s%s%s\n", indent, "-c", indent, + printf("%s%s%s%s\n", indent, "-c", indent, "Configuration directory. Default is '/etc/taos/'."); #endif - printf("%s%s%s%s\n", indent, "-h", indent, + printf("%s%s%s%s\n", indent, "-h", indent, "The host to connect to TDengine. Default is localhost."); - printf("%s%s%s%s\n", indent, "-p", indent, + printf("%s%s%s%s\n", indent, "-p", indent, "The TCP/IP port number to use for the connection. Default is 0."); - printf("%s%s%s%s\n", indent, "-d", indent, + printf("%s%s%s%s\n", indent, "-d", indent, "Destination database. Default is 'test'."); - printf("%s%s%s%s\n", indent, "-a", indent, + printf("%s%s%s%s\n", indent, "-a", indent, "Set the replica parameters of the database, Default 1, min: 1, max: 3."); - printf("%s%s%s%s\n", indent, "-m", indent, + printf("%s%s%s%s\n", indent, "-m", indent, "Table prefix name. Default is 't'."); printf("%s%s%s%s\n", indent, "-s", indent, "The select sql file."); printf("%s%s%s%s\n", indent, "-N", indent, "Use normal table flag."); - printf("%s%s%s%s\n", indent, "-o", indent, + printf("%s%s%s%s\n", indent, "-o", indent, "Direct output to the named file. Default is './output.txt'."); - printf("%s%s%s%s\n", indent, "-q", indent, + printf("%s%s%s%s\n", indent, "-q", indent, "Query mode--0: SYNC, 1: ASYNC. Default is SYNC."); - printf("%s%s%s%s\n", indent, "-b", indent, + printf("%s%s%s%s\n", indent, "-b", indent, "The data_type of columns, default: TINYINT,SMALLINT,INT,BIGINT,FLOAT,DOUBLE,BINARY,NCHAR,BOOL,TIMESTAMP."); - printf("%s%s%s%s\n", indent, "-w", indent, + printf("%s%s%s%s\n", indent, "-w", indent, "The length of data_type 'BINARY' or 'NCHAR'. Default is 16"); - printf("%s%s%s%s\n", indent, "-l", indent, + printf("%s%s%s%s\n", indent, "-l", indent, "The number of columns per record. Default is 10."); - printf("%s%s%s%s\n", indent, "-T", indent, + printf("%s%s%s%s\n", indent, "-T", indent, "The number of threads. Default is 10."); - printf("%s%s%s%s\n", indent, "-i", indent, + printf("%s%s%s%s\n", indent, "-i", indent, "The sleep time (ms) between insertion. Default is 0."); - printf("%s%s%s%s\n", indent, "-r", indent, + printf("%s%s%s%s\n", indent, "-r", indent, "The number of records per request. Default is 100."); - printf("%s%s%s%s\n", indent, "-t", indent, + printf("%s%s%s%s\n", indent, "-t", indent, "The number of tables. Default is 10000."); - printf("%s%s%s%s\n", indent, "-n", indent, + printf("%s%s%s%s\n", indent, "-n", indent, "The number of records per table. Default is 10000."); printf("%s%s%s%s\n", indent, "-x", indent, "Not insert only flag."); printf("%s%s%s%s\n", indent, "-y", indent, "Default input yes for prompt."); - printf("%s%s%s%s\n", indent, "-O", indent, + printf("%s%s%s%s\n", indent, "-O", indent, "Insert mode--0: In order, > 0: disorder ratio. Default is in order."); - printf("%s%s%s%s\n", indent, "-R", indent, + printf("%s%s%s%s\n", indent, "-R", indent, "Out of order data's range, ms, default is 1000."); - printf("%s%s%s%s\n", indent, "-g", indent, + printf("%s%s%s%s\n", indent, "-g", indent, "Print debug info."); - printf("%s%s%s%s\n", indent, "-V, --version", indent, + printf("%s%s%s%s\n", indent, "-V, --version", indent, "Print version info."); /* printf("%s%s%s%s\n", indent, "-D", indent, "if elete database if exists. 0: no, 1: yes, default is 1"); @@ -696,6 +696,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]); wordfree(&full_path); + } else if (strcmp(argv[i], "-h") == 0) { arguments->host = argv[++i]; } else if (strcmp(argv[i], "-p") == 0) { @@ -793,7 +794,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { strcpy(configDir, argv[++i]); } else if (strcmp(argv[i], "-O") == 0) { arguments->disorderRatio = atoi(argv[++i]); - if (arguments->disorderRatio > 1 + if (arguments->disorderRatio > 1 || arguments->disorderRatio < 0) { arguments->disorderRatio = 0; } else if (arguments->disorderRatio == 1) { @@ -801,8 +802,8 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } } else if (strcmp(argv[i], "-R") == 0) { arguments->disorderRange = atoi(argv[++i]); - if (arguments->disorderRange == 1 - && (arguments->disorderRange > 50 + if (arguments->disorderRange == 1 + && (arguments->disorderRange > 50 || arguments->disorderRange <= 0)) { arguments->disorderRange = 10; } @@ -835,7 +836,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { || arguments->verbose_print) { printf("###################################################################\n"); printf("# meta file: %s\n", arguments->metaFile); - printf("# Server IP: %s:%hu\n", + printf("# Server IP: %s:%hu\n", arguments->host == NULL ? "localhost" : arguments->host, arguments->port ); printf("# User: %s\n", arguments->user); @@ -862,7 +863,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { if (arguments->disorderRatio) { printf("# Data order: %d\n", arguments->disorderRatio); printf("# Data out of order rate: %d\n", arguments->disorderRange); - + } printf("# Delete method: %d\n", arguments->method_of_delete); printf("# Answer yes when prompt: %d\n", arguments->answer_yes); @@ -930,7 +931,7 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) { return 0; } -static void getResult(TAOS_RES *res, char* resultFileName) { +static void getResult(TAOS_RES *res, char* resultFileName) { TAOS_ROW row = NULL; int num_rows = 0; int num_fields = taos_field_count(res); @@ -940,13 +941,15 @@ static void getResult(TAOS_RES *res, char* resultFileName) { if (resultFileName[0] != 0) { fp = fopen(resultFileName, "at"); if (fp == NULL) { - errorPrint("%s() LN%d, failed to open result file: %s, result will not save to file\n", __func__, __LINE__, resultFileName); + errorPrint("%s() LN%d, failed to open result file: %s, result will not save to file\n", + __func__, __LINE__, resultFileName); } } - + char* databuf = (char*) calloc(1, 100*1024*1024); if (databuf == NULL) { - errorPrint("%s() LN%d, failed to malloc, warning: save result to file slowly!\n", __func__, __LINE__); + errorPrint("%s() LN%d, failed to malloc, warning: save result to file slowly!\n", + __func__, __LINE__); if (fp) fclose(fp); return ; @@ -982,7 +985,7 @@ static void selectAndGetResult(TAOS *taos, char *command, char* resultFileName) taos_free_result(res); return; } - + getResult(res, resultFileName); taos_free_result(res); } @@ -1030,14 +1033,13 @@ static int64_t rand_bigint(){ cursor++; cursor = cursor % MAX_PREPARED_RAND; return randbigint[cursor]; - } static float rand_float(){ static int cursor; cursor++; cursor = cursor % MAX_PREPARED_RAND; - return randfloat[cursor]; + return randfloat[cursor]; } static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"; @@ -1110,9 +1112,9 @@ static int printfInsertMeta() { printf("database[\033[33m%d\033[0m]:\n", i); printf(" database[%d] name: \033[33m%s\033[0m\n", i, g_Dbs.db[i].dbName); if (0 == g_Dbs.db[i].drop) { - printf(" drop: \033[33mno\033[0m\n"); - }else { - printf(" drop: \033[33myes\033[0m\n"); + printf(" drop: \033[33mno\033[0m\n"); + } else { + printf(" drop: \033[33myes\033[0m\n"); } if (g_Dbs.db[i].dbCfg.blocks > 0) { @@ -1165,8 +1167,8 @@ static int printfInsertMeta() { printf(" super table count: \033[33m%d\033[0m\n", g_Dbs.db[i].superTblCount); for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { printf(" super table[\033[33m%d\033[0m]:\n", j); - - printf(" stbName: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].sTblName); + + printf(" stbName: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].sTblName); if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { printf(" autoCreateTable: \033[33m%s\033[0m\n", "no"); @@ -1175,7 +1177,7 @@ static int printfInsertMeta() { } else { printf(" autoCreateTable: \033[33m%s\033[0m\n", "error"); } - + if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { printf(" childTblExists: \033[33m%s\033[0m\n", "no"); } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { @@ -1278,9 +1280,9 @@ static void printfInsertMetaToFile(FILE* fp) { fprintf(fp, "database[%d]:\n", i); fprintf(fp, " database[%d] name: %s\n", i, g_Dbs.db[i].dbName); if (0 == g_Dbs.db[i].drop) { - fprintf(fp, " drop: no\n"); + fprintf(fp, " drop: no\n"); }else { - fprintf(fp, " drop: yes\n"); + fprintf(fp, " drop: yes\n"); } if (g_Dbs.db[i].dbCfg.blocks > 0) { @@ -1331,8 +1333,8 @@ static void printfInsertMetaToFile(FILE* fp) { fprintf(fp, " super table count: %d\n", g_Dbs.db[i].superTblCount); for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { fprintf(fp, " super table[%d]:\n", j); - - fprintf(fp, " stbName: %s\n", g_Dbs.db[i].superTbls[j].sTblName); + + fprintf(fp, " stbName: %s\n", g_Dbs.db[i].superTbls[j].sTblName); if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { fprintf(fp, " autoCreateTable: %s\n", "no"); @@ -1341,7 +1343,7 @@ static void printfInsertMetaToFile(FILE* fp) { } else { fprintf(fp, " autoCreateTable: %s\n", "error"); } - + if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { fprintf(fp, " childTblExists: %s\n", "no"); } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { @@ -1349,33 +1351,33 @@ static void printfInsertMetaToFile(FILE* fp) { } else { fprintf(fp, " childTblExists: %s\n", "error"); } - - fprintf(fp, " childTblCount: %d\n", g_Dbs.db[i].superTbls[j].childTblCount); - fprintf(fp, " childTblPrefix: %s\n", g_Dbs.db[i].superTbls[j].childTblPrefix); - fprintf(fp, " dataSource: %s\n", g_Dbs.db[i].superTbls[j].dataSource); - fprintf(fp, " insertMode: %s\n", g_Dbs.db[i].superTbls[j].insertMode); - fprintf(fp, " insertRows: %"PRId64"\n", g_Dbs.db[i].superTbls[j].insertRows); + + fprintf(fp, " childTblCount: %d\n", g_Dbs.db[i].superTbls[j].childTblCount); + fprintf(fp, " childTblPrefix: %s\n", g_Dbs.db[i].superTbls[j].childTblPrefix); + fprintf(fp, " dataSource: %s\n", g_Dbs.db[i].superTbls[j].dataSource); + fprintf(fp, " insertMode: %s\n", g_Dbs.db[i].superTbls[j].insertMode); + fprintf(fp, " insertRows: %"PRId64"\n", g_Dbs.db[i].superTbls[j].insertRows); fprintf(fp, " interlace rows: %d\n", g_Dbs.db[i].superTbls[j].interlaceRows); if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) { fprintf(fp, " insert interval: %d\n", g_Dbs.db[i].superTbls[j].insertInterval); } if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { - fprintf(fp, " multiThreadWriteOneTbl: no\n"); + fprintf(fp, " multiThreadWriteOneTbl: no\n"); }else { - fprintf(fp, " multiThreadWriteOneTbl: yes\n"); + fprintf(fp, " multiThreadWriteOneTbl: yes\n"); } - fprintf(fp, " interlaceRows: %d\n", g_Dbs.db[i].superTbls[j].interlaceRows); - fprintf(fp, " disorderRange: %d\n", g_Dbs.db[i].superTbls[j].disorderRange); + fprintf(fp, " interlaceRows: %d\n", g_Dbs.db[i].superTbls[j].interlaceRows); + fprintf(fp, " disorderRange: %d\n", g_Dbs.db[i].superTbls[j].disorderRange); fprintf(fp, " disorderRatio: %d\n", g_Dbs.db[i].superTbls[j].disorderRatio); - fprintf(fp, " maxSqlLen: %d\n", g_Dbs.db[i].superTbls[j].maxSqlLen); - - fprintf(fp, " timeStampStep: %d\n", g_Dbs.db[i].superTbls[j].timeStampStep); - fprintf(fp, " startTimestamp: %s\n", g_Dbs.db[i].superTbls[j].startTimestamp); + fprintf(fp, " maxSqlLen: %d\n", g_Dbs.db[i].superTbls[j].maxSqlLen); + + fprintf(fp, " timeStampStep: %d\n", g_Dbs.db[i].superTbls[j].timeStampStep); + fprintf(fp, " startTimestamp: %s\n", g_Dbs.db[i].superTbls[j].startTimestamp); fprintf(fp, " sampleFormat: %s\n", g_Dbs.db[i].superTbls[j].sampleFormat); - fprintf(fp, " sampleFile: %s\n", g_Dbs.db[i].superTbls[j].sampleFile); - fprintf(fp, " tagsFile: %s\n", g_Dbs.db[i].superTbls[j].tagsFile); - + fprintf(fp, " sampleFile: %s\n", g_Dbs.db[i].superTbls[j].sampleFile); + fprintf(fp, " tagsFile: %s\n", g_Dbs.db[i].superTbls[j].tagsFile); + fprintf(fp, " columnCount: %d\n ", g_Dbs.db[i].superTbls[j].columnCount); for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) { //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); @@ -1565,7 +1567,7 @@ static int xDumpResultToFile(const char* fname, TAOS_RES* tres) { fprintf(fp, "%s", fields[col].name); } fputc('\n', fp); - + int numOfRows = 0; do { int32_t* length = taos_fetch_lengths(tres); @@ -1586,14 +1588,14 @@ static int xDumpResultToFile(const char* fname, TAOS_RES* tres) { return numOfRows; } -static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) { - TAOS_RES * res; +static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) { + TAOS_RES * res; TAOS_ROW row = NULL; int count = 0; - - res = taos_query(taos, "show databases;"); + + res = taos_query(taos, "show databases;"); int32_t code = taos_errno(res); - + if (code != 0) { errorPrint( "failed to run , reason: %s\n", taos_errstr(res)); return -1; @@ -1617,13 +1619,13 @@ static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) { *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX], TSDB_TIME_PRECISION_MILLI); dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]); - dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]); + dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]); dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]); dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]); - dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]); + dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]); tstrncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX], - fields[TSDB_SHOW_DB_KEEP_INDEX].bytes); + fields[TSDB_SHOW_DB_KEEP_INDEX].bytes); dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]); dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]); dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]); @@ -1633,16 +1635,16 @@ static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) { dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX])); dbInfos[count]->cachelast = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX])); - tstrncpy(dbInfos[count]->precision, + tstrncpy(dbInfos[count]->precision, (char *)row[TSDB_SHOW_DB_PRECISION_INDEX], - fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes); + fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes); dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]); tstrncpy(dbInfos[count]->status, (char *)row[TSDB_SHOW_DB_STATUS_INDEX], - fields[TSDB_SHOW_DB_STATUS_INDEX].bytes); + fields[TSDB_SHOW_DB_STATUS_INDEX].bytes); count++; if (count > MAX_DATABASE_COUNT) { - errorPrint( "The database count overflow than %d\n", MAX_DATABASE_COUNT); + errorPrint( "The database count overflow than %d\n", MAX_DATABASE_COUNT); break; } } @@ -1664,11 +1666,11 @@ static void printfDbInfoForQueryToFile(char* filename, SDbInfo* dbInfos, int ind fprintf(fp, "name: %s\n", dbInfos->name); fprintf(fp, "created_time: %s\n", dbInfos->create_time); fprintf(fp, "ntables: %d\n", dbInfos->ntables); - fprintf(fp, "vgroups: %d\n", dbInfos->vgroups); + fprintf(fp, "vgroups: %d\n", dbInfos->vgroups); fprintf(fp, "replica: %d\n", dbInfos->replica); fprintf(fp, "quorum: %d\n", dbInfos->quorum); - fprintf(fp, "days: %d\n", dbInfos->days); - fprintf(fp, "keep0,keep1,keep(D): %s\n", dbInfos->keeplist); + fprintf(fp, "days: %d\n", dbInfos->days); + fprintf(fp, "keep0,keep1,keep(D): %s\n", dbInfos->keeplist); fprintf(fp, "cache(MB): %d\n", dbInfos->cache); fprintf(fp, "blocks: %d\n", dbInfos->blocks); fprintf(fp, "minrows: %d\n", dbInfos->minrows); @@ -1676,10 +1678,10 @@ static void printfDbInfoForQueryToFile(char* filename, SDbInfo* dbInfos, int ind fprintf(fp, "wallevel: %d\n", dbInfos->wallevel); fprintf(fp, "fsync: %d\n", dbInfos->fsync); fprintf(fp, "comp: %d\n", dbInfos->comp); - fprintf(fp, "cachelast: %d\n", dbInfos->cachelast); - fprintf(fp, "precision: %s\n", dbInfos->precision); + fprintf(fp, "cachelast: %d\n", dbInfos->cachelast); + fprintf(fp, "precision: %s\n", dbInfos->precision); fprintf(fp, "update: %d\n", dbInfos->update); - fprintf(fp, "status: %s\n", dbInfos->status); + fprintf(fp, "status: %s\n", dbInfos->status); fprintf(fp, "\n"); fclose(fp); @@ -1729,7 +1731,7 @@ static void printfQuerySystemInfo(TAOS * taos) { snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.vgroups;", dbInfos[i]->name); res = taos_query(taos, buffer); xDumpResultToFile(filename, res); - + // show db.stables snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.stables;", dbInfos[i]->name); res = taos_query(taos, buffer); @@ -1739,7 +1741,6 @@ static void printfQuerySystemInfo(TAOS * taos) { } free(dbInfos); - } static int postProceSql(char* host, uint16_t port, char* sqlstr) @@ -1991,17 +1992,17 @@ static char* generateTagVaulesForStb(SSuperTable* stbInfo) { } dataLen -= 2; - dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")"); + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")"); return dataBuf; } -static int calcRowLen(SSuperTable* superTbls) { +static int calcRowLen(SSuperTable* superTbls) { int colIndex; int lenOfOneRow = 0; - + for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) { char* dataType = superTbls->columns[colIndex].dataType; - + if (strcasecmp(dataType, "BINARY") == 0) { lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; } else if (strcasecmp(dataType, "NCHAR") == 0) { @@ -2018,9 +2019,9 @@ static int calcRowLen(SSuperTable* superTbls) { lenOfOneRow += 6; } else if (strcasecmp(dataType, "FLOAT") == 0) { lenOfOneRow += 22; - } else if (strcasecmp(dataType, "DOUBLE") == 0) { + } else if (strcasecmp(dataType, "DOUBLE") == 0) { lenOfOneRow += 42; - } else if (strcasecmp(dataType, "TIMESTAMP") == 0) { + } else if (strcasecmp(dataType, "TIMESTAMP") == 0) { lenOfOneRow += 21; } else { printf("get error data type : %s\n", dataType); @@ -2060,7 +2061,7 @@ static int calcRowLen(SSuperTable* superTbls) { } superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow; - + return 0; } @@ -2072,7 +2073,7 @@ static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, char command[BUFFER_SIZE] = "\0"; char limitBuf[100] = "\0"; - TAOS_RES * res; + TAOS_RES * res; TAOS_ROW row = NULL; char* childTblName = *childTblNameOfSuperTbl; @@ -2081,10 +2082,10 @@ static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, snprintf(limitBuf, 100, " limit %d offset %d", limit, offset); } - //get all child table name use cmd: select tbname from superTblName; + //get all child table name use cmd: select tbname from superTblName; snprintf(command, BUFFER_SIZE, "select tbname from %s.%s %s", dbName, sTblName, limitBuf); - res = taos_query(taos, command); + res = taos_query(taos, command); int32_t code = taos_errno(res); if (code != 0) { taos_free_result(res); @@ -2153,7 +2154,7 @@ static int getSuperTableFromServer(TAOS * taos, char* dbName, SSuperTable* superTbls) { char command[BUFFER_SIZE] = "\0"; - TAOS_RES * res; + TAOS_RES * res; TAOS_ROW row = NULL; int count = 0; @@ -2474,7 +2475,7 @@ static int createDatabases() { if ((ret != 0) || (g_Dbs.db[i].drop)) { ret = createSuperTable(taos, g_Dbs.db[i].dbName, &g_Dbs.db[i].superTbls[j], g_Dbs.use_metric); - + if (0 != ret) { errorPrint("\ncreate super table %d failed!\n\n", j); taos_close(taos); @@ -2618,7 +2619,7 @@ static int startMultiThreadCreateChildTable( int b = 0; b = ntables % threads; - + for (int i = 0; i < threads; i++) { threadInfo *t_info = infos + i; t_info->threadID = i; @@ -2806,7 +2807,7 @@ static int readSampleFromCsvFileToMem( ssize_t readLen = 0; char * line = NULL; int getRows = 0; - + FILE* fp = fopen(superTblInfo->sampleFile, "r"); if (fp == NULL) { errorPrint( "Failed to open sample file: %s, reason:%s\n", @@ -2828,7 +2829,7 @@ static int readSampleFromCsvFileToMem( } continue; } - + if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { line[--readLen] = 0; } @@ -2939,7 +2940,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( debugPrint("%s() LN%d, failed to read json, tags not found\n", __func__, __LINE__); goto PARSE_OVER; } - + int tagSize = cJSON_GetArraySize(tags); if (tagSize > MAX_TAG_COUNT) { debugPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n", __func__, __LINE__, MAX_TAG_COUNT); @@ -3025,7 +3026,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { cJSON* user = cJSON_GetObjectItem(root, "user"); if (user && user->type == cJSON_String && user->valuestring != NULL) { - tstrncpy(g_Dbs.user, user->valuestring, MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.user, user->valuestring, MAX_DB_NAME_SIZE); } else if (!user) { tstrncpy(g_Dbs.user, "root", MAX_DB_NAME_SIZE); } @@ -3052,8 +3053,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else { printf("ERROR: failed to read json, threads not found\n"); goto PARSE_OVER; - } - + } + cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl"); if (threads2 && threads2->type == cJSON_Number) { g_Dbs.threadCountByCreateTbl = threads2->valueint; @@ -3063,7 +3064,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { errorPrint("%s() LN%d, failed to read json, threads2 not found\n", __func__, __LINE__); goto PARSE_OVER; - } + } cJSON* gInsertInterval = cJSON_GetObjectItem(root, "insert_interval"); if (gInsertInterval && gInsertInterval->type == cJSON_Number) { @@ -3088,12 +3089,23 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { cJSON* interlaceRows = cJSON_GetObjectItem(root, "interlace_rows"); if (interlaceRows && interlaceRows->type == cJSON_Number) { g_args.interlace_rows = interlaceRows->valueint; + + // rows per table need be less than insert batch + if (g_args.interlace_rows > g_args.num_of_RPR) { + printf("NOTICE: interlace rows value %d > num_of_records_per_request %d\n\n", + g_args.interlace_rows, g_args.num_of_RPR); + printf(" interlace rows value will be set to num_of_records_per_request %d\n\n", + g_args.num_of_RPR); + printf(" press Enter key to continue or Ctrl+C to stop."); + (void)getchar(); + g_args.interlace_rows = g_args.num_of_RPR; + } } else if (!interlaceRows) { g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req } else { errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n", __func__, __LINE__); goto PARSE_OVER; - } + } cJSON* maxSqlLen = cJSON_GetObjectItem(root, "max_sql_len"); if (maxSqlLen && maxSqlLen->type == cJSON_Number) { @@ -3104,7 +3116,6 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n", __func__, __LINE__); goto PARSE_OVER; } - cJSON* numRecPerReq = cJSON_GetObjectItem(root, "num_of_records_per_req"); if (numRecPerReq && numRecPerReq->type == cJSON_Number) { @@ -3117,7 +3128,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no, - if (answerPrompt + if (answerPrompt && answerPrompt->type == cJSON_String && answerPrompt->valuestring != NULL) { if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) { @@ -3132,7 +3143,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else { printf("ERROR: failed to read json, confirm_parameter_prompt not found\n"); goto PARSE_OVER; - } + } cJSON* dbs = cJSON_GetObjectItem(root, "databases"); if (!dbs || dbs->type != cJSON_Array) { @@ -3159,7 +3170,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { printf("ERROR: failed to read json, dbinfo not found\n"); goto PARSE_OVER; } - + cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name"); if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) { printf("ERROR: failed to read json, db name not found\n"); @@ -3173,7 +3184,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_Dbs.db[i].drop = true; } else { g_Dbs.db[i].drop = false; - } + } } else if (!drop) { g_Dbs.db[i].drop = g_args.drop_database; } else { @@ -3224,7 +3235,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { printf("ERROR: failed to read json, keep not found\n"); goto PARSE_OVER; } - + cJSON* days = cJSON_GetObjectItem(dbinfo, "days"); if (days && days->type == cJSON_Number) { g_Dbs.db[i].dbCfg.days = days->valueint; @@ -3234,7 +3245,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { printf("ERROR: failed to read json, days not found\n"); goto PARSE_OVER; } - + cJSON* cache = cJSON_GetObjectItem(dbinfo, "cache"); if (cache && cache->type == cJSON_Number) { g_Dbs.db[i].dbCfg.cache = cache->valueint; @@ -3244,7 +3255,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { printf("ERROR: failed to read json, cache not found\n"); goto PARSE_OVER; } - + cJSON* blocks= cJSON_GetObjectItem(dbinfo, "blocks"); if (blocks && blocks->type == cJSON_Number) { g_Dbs.db[i].dbCfg.blocks = blocks->valueint; @@ -3333,15 +3344,15 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else { printf("ERROR: failed to read json, fsync not found\n"); goto PARSE_OVER; - } + } - // super_talbes + // super_talbes cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables"); if (!stables || stables->type != cJSON_Array) { printf("ERROR: failed to read json, super_tables not found\n"); goto PARSE_OVER; - } - + } + int stbSize = cJSON_GetArraySize(stables); if (stbSize > MAX_SUPER_TABLE_COUNT) { errorPrint( @@ -3354,15 +3365,15 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { for (int j = 0; j < stbSize; ++j) { cJSON* stbInfo = cJSON_GetArrayItem(stables, j); if (stbInfo == NULL) continue; - - // dbinfo + + // dbinfo cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name"); if (!stbName || stbName->type != cJSON_String || stbName->valuestring == NULL) { printf("ERROR: failed to read json, stb name not found\n"); goto PARSE_OVER; } tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, MAX_TB_NAME_SIZE); - + cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix"); if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) { printf("ERROR: failed to read json, childtable_prefix not found\n"); @@ -3387,7 +3398,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { printf("ERROR: failed to read json, auto_create_table not found\n"); goto PARSE_OVER; } - + cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num"); if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) { g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint; @@ -3396,7 +3407,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else { printf("ERROR: failed to read json, batch_create_tbl_num not found\n"); goto PARSE_OVER; - } + } cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no if (childTblExists @@ -3412,13 +3423,15 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!childTblExists) { g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; } else { - errorPrint("%s() LN%d, failed to read json, child_table_exists not found\n", __func__, __LINE__); + errorPrint("%s() LN%d, failed to read json, child_table_exists not found\n", + __func__, __LINE__); goto PARSE_OVER; } - + cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count"); if (!count || count->type != cJSON_Number || 0 >= count->valueint) { - errorPrint("%s() LN%d, failed to read json, childtable_count not found\n", __func__, __LINE__); + errorPrint("%s() LN%d, failed to read json, childtable_count not found\n", + __func__, __LINE__); goto PARSE_OVER; } g_Dbs.db[i].superTbls[j].childTblCount = count->valueint; @@ -3502,10 +3515,11 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else { printf("ERROR: failed to read json, sample_buf_size not found\n"); goto PARSE_OVER; - } + } cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format"); - if (sampleFormat && sampleFormat->type == cJSON_String && sampleFormat->valuestring != NULL) { + if (sampleFormat && sampleFormat->type + == cJSON_String && sampleFormat->valuestring != NULL) { tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, sampleFormat->valuestring, MAX_DB_NAME_SIZE); } else if (!sampleFormat) { @@ -3513,7 +3527,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else { printf("ERROR: failed to read json, sample_format not found\n"); goto PARSE_OVER; - } + } cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file"); if (sampleFile && sampleFile->type == cJSON_String && sampleFile->valuestring != NULL) { @@ -3524,7 +3538,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else { printf("ERROR: failed to read json, sample_file not found\n"); goto PARSE_OVER; - } + } cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file"); if (tagsFile && tagsFile->type == cJSON_String && tagsFile->valuestring != NULL) { @@ -3550,14 +3564,14 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { len = TSDB_MAX_ALLOWED_SQL_LEN; } else if (len < TSDB_MAX_SQL_LEN) { len = TSDB_MAX_SQL_LEN; - } + } g_Dbs.db[i].superTbls[j].maxSqlLen = len; } else if (!maxSqlLen) { g_Dbs.db[i].superTbls[j].maxSqlLen = TSDB_MAX_SQL_LEN; } else { printf("ERROR: failed to read json, maxSqlLen not found\n"); goto PARSE_OVER; - } + } cJSON *multiThreadWriteOneTbl = cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes @@ -3568,7 +3582,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1; } else { g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0; - } + } } else if (!multiThreadWriteOneTbl) { g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0; } else { @@ -3579,6 +3593,16 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { cJSON* interlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows"); if (interlaceRows && interlaceRows->type == cJSON_Number) { g_Dbs.db[i].superTbls[j].interlaceRows = interlaceRows->valueint; + // rows per table need be less than insert batch + if (g_Dbs.db[i].superTbls[j].interlaceRows > g_args.num_of_RPR) { + printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %d > num_of_records_per_request %d\n\n", + i, j, g_Dbs.db[i].superTbls[j].interlaceRows, g_args.num_of_RPR); + printf(" interlace rows value will be set to num_of_records_per_request %d\n\n", + g_args.num_of_RPR); + printf(" press Enter key to continue or Ctrl+C to stop."); + (void)getchar(); + g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR; + } } else if (!interlaceRows) { g_Dbs.db[i].superTbls[j].interlaceRows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req } else { @@ -3586,7 +3610,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { "%s() LN%d, failed to read json, interlace rows input mistake\n", __func__, __LINE__); goto PARSE_OVER; - } + } cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio"); if (disorderRatio && disorderRatio->type == cJSON_Number) { @@ -3596,7 +3620,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else { printf("ERROR: failed to read json, disorderRatio not found\n"); goto PARSE_OVER; - } + } cJSON* disorderRange = cJSON_GetObjectItem(stbInfo, "disorder_range"); if (disorderRange && disorderRange->type == cJSON_Number) { @@ -3636,8 +3660,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { stbInfo, &g_Dbs.db[i].superTbls[j]); if (false == retVal) { goto PARSE_OVER; - } - } + } + } } ret = true; @@ -3703,7 +3727,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else { printf("ERROR: failed to read json, confirm_parameter_prompt not found\n"); goto PARSE_OVER; - } + } cJSON* dbs = cJSON_GetObjectItem(root, "databases"); if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) { @@ -3722,7 +3746,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { printf("ERROR: failed to read json, query_mode not found\n"); goto PARSE_OVER; } - + // super_table_query cJSON *superQuery = cJSON_GetObjectItem(root, "specified_table_query"); if (!superQuery) { @@ -3731,26 +3755,26 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (superQuery->type != cJSON_Object) { printf("ERROR: failed to read json, super_table_query not found\n"); goto PARSE_OVER; - } else { + } else { cJSON* rate = cJSON_GetObjectItem(superQuery, "query_interval"); if (rate && rate->type == cJSON_Number) { g_queryInfo.superQueryInfo.rate = rate->valueint; } else if (!rate) { g_queryInfo.superQueryInfo.rate = 0; } - + cJSON* concurrent = cJSON_GetObjectItem(superQuery, "concurrent"); if (concurrent && concurrent->type == cJSON_Number) { g_queryInfo.superQueryInfo.concurrent = concurrent->valueint; } else if (!concurrent) { g_queryInfo.superQueryInfo.concurrent = 1; } - + cJSON* mode = cJSON_GetObjectItem(superQuery, "mode"); if (mode && mode->type == cJSON_String && mode->valuestring != NULL) { - if (0 == strcmp("sync", mode->valuestring)) { + if (0 == strcmp("sync", mode->valuestring)) { g_queryInfo.superQueryInfo.subscribeMode = 0; - } else if (0 == strcmp("async", mode->valuestring)) { + } else if (0 == strcmp("async", mode->valuestring)) { g_queryInfo.superQueryInfo.subscribeMode = 1; } else { printf("ERROR: failed to read json, subscribe mod error\n"); @@ -3759,21 +3783,21 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else { g_queryInfo.superQueryInfo.subscribeMode = 0; } - + cJSON* interval = cJSON_GetObjectItem(superQuery, "interval"); if (interval && interval->type == cJSON_Number) { g_queryInfo.superQueryInfo.subscribeInterval = interval->valueint; - } else if (!interval) { + } else if (!interval) { //printf("failed to read json, subscribe interval no found\n"); //goto PARSE_OVER; g_queryInfo.superQueryInfo.subscribeInterval = 10000; } - + cJSON* restart = cJSON_GetObjectItem(superQuery, "restart"); if (restart && restart->type == cJSON_String && restart->valuestring != NULL) { - if (0 == strcmp("yes", restart->valuestring)) { + if (0 == strcmp("yes", restart->valuestring)) { g_queryInfo.superQueryInfo.subscribeRestart = 1; - } else if (0 == strcmp("no", restart->valuestring)) { + } else if (0 == strcmp("no", restart->valuestring)) { g_queryInfo.superQueryInfo.subscribeRestart = 0; } else { printf("ERROR: failed to read json, subscribe restart error\n"); @@ -3782,14 +3806,14 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else { g_queryInfo.superQueryInfo.subscribeRestart = 1; } - + cJSON* keepProgress = cJSON_GetObjectItem(superQuery, "keepProgress"); if (keepProgress && keepProgress->type == cJSON_String && keepProgress->valuestring != NULL) { - if (0 == strcmp("yes", keepProgress->valuestring)) { + if (0 == strcmp("yes", keepProgress->valuestring)) { g_queryInfo.superQueryInfo.subscribeKeepProgress = 1; - } else if (0 == strcmp("no", keepProgress->valuestring)) { + } else if (0 == strcmp("no", keepProgress->valuestring)) { g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; } else { printf("ERROR: failed to read json, subscribe keepProgress error\n"); @@ -3799,14 +3823,14 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; } - // sqls + // sqls cJSON* superSqls = cJSON_GetObjectItem(superQuery, "sqls"); if (!superSqls) { g_queryInfo.superQueryInfo.sqlCount = 0; } else if (superSqls->type != cJSON_Array) { printf("ERROR: failed to read json, super sqls not found\n"); goto PARSE_OVER; - } else { + } else { int superSqlSize = cJSON_GetArraySize(superSqls); if (superSqlSize > MAX_QUERY_SQL_COUNT) { printf("ERROR: failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT); @@ -3817,7 +3841,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { for (int j = 0; j < superSqlSize; ++j) { cJSON* sql = cJSON_GetArrayItem(superSqls, j); if (sql == NULL) continue; - + cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) { printf("ERROR: failed to read json, sql not found\n"); @@ -3833,7 +3857,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else { printf("ERROR: failed to read json, super query result file not found\n"); goto PARSE_OVER; - } + } } } } @@ -3879,9 +3903,9 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* submode = cJSON_GetObjectItem(subQuery, "mode"); if (submode && submode->type == cJSON_String && submode->valuestring != NULL) { - if (0 == strcmp("sync", submode->valuestring)) { + if (0 == strcmp("sync", submode->valuestring)) { g_queryInfo.subQueryInfo.subscribeMode = 0; - } else if (0 == strcmp("async", submode->valuestring)) { + } else if (0 == strcmp("async", submode->valuestring)) { g_queryInfo.subQueryInfo.subscribeMode = 1; } else { printf("ERROR: failed to read json, subscribe mod error\n"); @@ -3899,12 +3923,12 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { //goto PARSE_OVER; g_queryInfo.subQueryInfo.subscribeInterval = 10000; } - + cJSON* subrestart = cJSON_GetObjectItem(subQuery, "restart"); if (subrestart && subrestart->type == cJSON_String && subrestart->valuestring != NULL) { - if (0 == strcmp("yes", subrestart->valuestring)) { + if (0 == strcmp("yes", subrestart->valuestring)) { g_queryInfo.subQueryInfo.subscribeRestart = 1; - } else if (0 == strcmp("no", subrestart->valuestring)) { + } else if (0 == strcmp("no", subrestart->valuestring)) { g_queryInfo.subQueryInfo.subscribeRestart = 0; } else { printf("ERROR: failed to read json, subscribe restart error\n"); @@ -3913,14 +3937,14 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else { g_queryInfo.subQueryInfo.subscribeRestart = 1; } - + cJSON* subkeepProgress = cJSON_GetObjectItem(subQuery, "keepProgress"); if (subkeepProgress && subkeepProgress->type == cJSON_String && subkeepProgress->valuestring != NULL) { - if (0 == strcmp("yes", subkeepProgress->valuestring)) { + if (0 == strcmp("yes", subkeepProgress->valuestring)) { g_queryInfo.subQueryInfo.subscribeKeepProgress = 1; - } else if (0 == strcmp("no", subkeepProgress->valuestring)) { + } else if (0 == strcmp("no", subkeepProgress->valuestring)) { g_queryInfo.subQueryInfo.subscribeKeepProgress = 0; } else { printf("ERROR: failed to read json, subscribe keepProgress error\n"); @@ -3928,27 +3952,27 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } } else { g_queryInfo.subQueryInfo.subscribeKeepProgress = 0; - } + } - // sqls + // sqls cJSON* subsqls = cJSON_GetObjectItem(subQuery, "sqls"); if (!subsqls) { g_queryInfo.subQueryInfo.sqlCount = 0; } else if (subsqls->type != cJSON_Array) { printf("ERROR: failed to read json, super sqls not found\n"); goto PARSE_OVER; - } else { + } else { int superSqlSize = cJSON_GetArraySize(subsqls); if (superSqlSize > MAX_QUERY_SQL_COUNT) { printf("ERROR: failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT); goto PARSE_OVER; } - + g_queryInfo.subQueryInfo.sqlCount = superSqlSize; - for (int j = 0; j < superSqlSize; ++j) { + for (int j = 0; j < superSqlSize; ++j) { cJSON* sql = cJSON_GetArrayItem(subsqls, j); if (sql == NULL) continue; - + cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) { printf("ERROR: failed to read json, sql not found\n"); @@ -3964,7 +3988,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else { printf("ERROR: failed to read json, sub query result file not found\n"); goto PARSE_OVER; - } + } } } } @@ -4033,7 +4057,7 @@ static bool getInfoFromJsonFile(char* file) { } else { errorPrint("%s() LN%d, input json file type error! please input correct file type: insert or query or subscribe\n", __func__, __LINE__); goto PARSE_OVER; - } + } PARSE_OVER: free(content); @@ -4043,7 +4067,7 @@ PARSE_OVER: } static void prepareSampleData() { - for (int i = 0; i < g_Dbs.dbCount; i++) { + for (int i = 0; i < g_Dbs.dbCount; i++) { for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { if (g_Dbs.db[i].superTbls[j].tagsFile[0] != 0) { (void)readTagFromCsvFileToMem(&g_Dbs.db[i].superTbls[j]); @@ -4054,7 +4078,7 @@ static void prepareSampleData() { static void postFreeResource() { tmfclose(g_fpOfInsertResult); - for (int i = 0; i < g_Dbs.dbCount; i++) { + for (int i = 0; i < g_Dbs.dbCount; i++) { for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { if (0 != g_Dbs.db[i].superTbls[j].colsOfCreateChildTable) { free(g_Dbs.db[i].superTbls[j].colsOfCreateChildTable); @@ -4105,7 +4129,7 @@ static int getRowDataFromSample(char* dataBuf, int maxLen, int64_t timestamp, static int generateRowData(char* dataBuf, int maxLen, int64_t timestamp, SSuperTable* stbInfo) { int dataLen = 0; dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "(%" PRId64 ", ", timestamp); - for (int i = 0; i < stbInfo->columnCount; i++) { + for (int i = 0; i < stbInfo->columnCount; i++) { if ((0 == strncasecmp(stbInfo->columns[i].dataType, "binary", 6)) || (0 == strncasecmp(stbInfo->columns[i].dataType, "nchar", 5))) { if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) { @@ -4316,28 +4340,28 @@ static int generateDataTail(char *tableName, int32_t tableSeq, if (0 == strncasecmp(superTblInfo->dataSource, "sample", strlen("sample"))) { retLen = getRowDataFromSample( - buffer + len, - superTblInfo->maxSqlLen - len, + buffer + len, + superTblInfo->maxSqlLen - len, startTime + superTblInfo->timeStampStep * k, - superTblInfo, + superTblInfo, pSamplePos); } else if (0 == strncasecmp(superTblInfo->dataSource, "rand", strlen("rand"))) { int rand_num = rand_tinyint() % 100; - if (0 != superTblInfo->disorderRatio + if (0 != superTblInfo->disorderRatio && rand_num < superTblInfo->disorderRatio) { int64_t d = startTime + superTblInfo->timeStampStep * k - taosRandom() % superTblInfo->disorderRange; retLen = generateRowData( - buffer + len, + buffer + len, superTblInfo->maxSqlLen - len, - d, + d, superTblInfo); } else { retLen = generateRowData( - buffer + len, - superTblInfo->maxSqlLen - len, + buffer + len, + superTblInfo->maxSqlLen - len, startTime + superTblInfo->timeStampStep * k, superTblInfo); } @@ -4361,7 +4385,7 @@ static int generateDataTail(char *tableName, int32_t tableSeq, if ((g_args.disorderRatio != 0) && (rand_num < g_args.disorderRange)) { - + int64_t d = startTime + DEFAULT_TIMESTAMP_STEP * k - taosRandom() % 1000000 + rand_num; len = generateData(data, data_type, @@ -4489,6 +4513,18 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pThreadInfo->threadID, __func__, __LINE__); SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + int interlaceRows = superTblInfo?superTblInfo->interlaceRows:g_args.interlace_rows; + + int insertMode; + + if (interlaceRows > 0) { + insertMode = INTERLACE_INSERT_MODE; + } else { + insertMode = PROGRESSIVE_INSERT_MODE; + } + + // TODO: prompt tbl count multple interlace rows and batch + // char* buffer = calloc(superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len, 1); if (NULL == buffer) { @@ -4497,22 +4533,10 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { strerror(errno)); return NULL; } - - int insertMode; + + char tableName[TSDB_TABLE_NAME_LEN]; - int interlaceRows = superTblInfo?superTblInfo->interlaceRows:g_args.interlace_rows; - - if (interlaceRows > 0) { - insertMode = INTERLACE_INSERT_MODE; - } else { - insertMode = PROGRESSIVE_INSERT_MODE; - } - - // rows per table need be less than insert batch - if (interlaceRows > g_args.num_of_RPR) - interlaceRows = g_args.num_of_RPR; - pThreadInfo->totalInsertRows = 0; pThreadInfo->totalAffectedRows = 0; @@ -4715,7 +4739,7 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { strerror(errno)); return NULL; } - + int64_t lastPrintTime = taosGetTimestampMs(); int64_t startTs = taosGetTimestampUs(); int64_t endTs; @@ -4856,7 +4880,7 @@ static void callBack(void *param, TAOS_RES *res, int code) { taosMsleep(insert_interval - (winfo->et - winfo->st)/1000); // ms } } - + char *buffer = calloc(1, winfo->superTblInfo->maxSqlLen); char *data = calloc(1, MAX_DATA_SIZE); char *pstr = buffer; @@ -4874,7 +4898,7 @@ static void callBack(void *param, TAOS_RES *res, int code) { taos_free_result(res); return; } - + for (int i = 0; i < g_args.num_of_RPR; i++) { int rand_num = taosRandom() % 100; if (0 != winfo->superTblInfo->disorderRatio @@ -4911,7 +4935,7 @@ static void *asyncWrite(void *sarg) { winfo->st = 0; winfo->et = 0; winfo->lastTs = winfo->start_time; - + int insert_interval = superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; if (insert_interval) { @@ -5000,7 +5024,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, } double start = getCurrentTime(); - + int startFrom; if ((superTblInfo) && (superTblInfo->childTblOffset >= 0)) @@ -5192,7 +5216,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, } static void *readTable(void *sarg) { -#if 1 +#if 1 threadInfo *rinfo = (threadInfo *)sarg; TAOS *taos = rinfo->taos; char command[BUFFER_SIZE] = "\0"; @@ -5359,7 +5383,7 @@ static int insertTestProcess() { printf("Press enter key to continue\n\n"); (void)getchar(); } - + init_rand_data(); // create database and super tables @@ -5448,7 +5472,7 @@ static void *superQueryProcess(void *sarg) { //char sqlStr[MAX_TB_NAME_SIZE*2]; //sprintf(sqlStr, "use %s", g_queryInfo.dbName); //queryDB(winfo->taos, sqlStr); - + int64_t st = 0; int64_t et = 0; @@ -5603,7 +5627,7 @@ static int queryTestProcess() { printf("Press enter key to continue\n\n"); (void)getchar(); } - + printfQuerySystemInfo(taos); pthread_t *pids = NULL; @@ -5676,20 +5700,20 @@ static int queryTestProcess() { int ntables = g_queryInfo.subQueryInfo.childTblCount; int threads = g_queryInfo.subQueryInfo.threadCnt; - + int a = ntables / threads; if (a < 1) { threads = ntables; a = 1; } - + int b = 0; if (threads != 0) { b = ntables % threads; } int startFrom = 0; - for (int i = 0; i < threads; i++) { + for (int i = 0; i < threads; i++) { threadInfo *t_info = infosOfSub + i; t_info->threadID = i; @@ -5705,21 +5729,21 @@ static int queryTestProcess() { } else { g_queryInfo.subQueryInfo.threadCnt = 0; } - + for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { pthread_join(pids[i], NULL); } tmfree((char*)pids); tmfree((char*)infos); - + for (int i = 0; i < g_queryInfo.subQueryInfo.threadCnt; i++) { pthread_join(pidsOfSub[i], NULL); } tmfree((char*)pidsOfSub); tmfree((char*)infosOfSub); - + // taos_close(taos);// TODO: workaround to use separate taos connection; return 0; } @@ -5784,7 +5808,7 @@ static void *subSubscribeProcess(void *sarg) { taos_close(winfo->taos); return NULL; } - + //int64_t st = 0; //int64_t et = 0; do { @@ -5871,7 +5895,7 @@ static void *superSubscribeProcess(void *sarg) { taos_close(winfo->taos); return NULL; } - + //int64_t st = 0; //int64_t et = 0; do { @@ -5988,7 +6012,7 @@ static int subscribeTestProcess() { t_info->taos = NULL; // TODO: workaround to use separate taos connection; pthread_create(pids + i, NULL, superSubscribeProcess, t_info); } - + //==== create sub threads for query from sub table pthread_t *pidsOfSub = NULL; threadInfo *infosOfSub = NULL; @@ -6007,18 +6031,18 @@ static int subscribeTestProcess() { int ntables = g_queryInfo.subQueryInfo.childTblCount; int threads = g_queryInfo.subQueryInfo.threadCnt; - + int a = ntables / threads; if (a < 1) { threads = ntables; a = 1; } - + int b = 0; if (threads != 0) { b = ntables % threads; } - + int startFrom = 0; for (int i = 0; i < threads; i++) { threadInfo *t_info = infosOfSub + i; @@ -6033,7 +6057,7 @@ static int subscribeTestProcess() { } g_queryInfo.subQueryInfo.threadCnt = threads; } - + for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { pthread_join(pids[i], NULL); } @@ -6128,7 +6152,7 @@ static void setParaFromArg(){ g_Dbs.threadCount = g_args.num_of_threads; g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; g_Dbs.queryMode = g_args.mode; - + g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL; g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS; g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange; @@ -6149,7 +6173,7 @@ static void setParaFromArg(){ if (data_type[i] == NULL) { break; } - + tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, data_type[i], MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary; @@ -6168,7 +6192,7 @@ static void setParaFromArg(){ tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, "INT", MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0; - + tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, "BINARY", MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary; g_Dbs.db[0].superTbls[0].tagCount = 2; From c01e23925df17aa98715e60f36c4ad3949d6bb75 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 27 Mar 2021 15:23:23 +0800 Subject: [PATCH 23/71] [TD-3570]add taosd valgrind test --- Jenkinsfile | 6 +++ tests/Jenkinsfile | 11 +++++ tests/pytest/handle_taosd_val_log.sh | 69 ++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+) create mode 100755 tests/pytest/handle_taosd_val_log.sh diff --git a/Jenkinsfile b/Jenkinsfile index daae5a5ad0..bd7d5d700a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -185,6 +185,12 @@ pipeline { rm -rf /var/log/taos/* ./handle_crash_gen_val_log.sh ''' + sh ''' + cd ${WKC}/tests/pytest + rm -rf /var/lib/taos/* + rm -rf /var/log/taos/* + ./handle_taosd_val_log.sh + ''' timeout(time: 45, unit: 'MINUTES'){ sh ''' date diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile index 6db6351442..b7b36eae07 100644 --- a/tests/Jenkinsfile +++ b/tests/Jenkinsfile @@ -76,9 +76,20 @@ pipeline { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh ''' cd ${WKC}/tests/pytest + rm -rf /var/lib/taos/* + rm -rf /var/log/taos/* ./handle_crash_gen_val_log.sh ''' } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/pytest + rm -rf /var/lib/taos/* + rm -rf /var/log/taos/* + ./handle_taosd_val_log.sh + ''' + } + sh''' systemctl start taosd sleep 10 diff --git a/tests/pytest/handle_taosd_val_log.sh b/tests/pytest/handle_taosd_val_log.sh new file mode 100755 index 0000000000..221dfc649f --- /dev/null +++ b/tests/pytest/handle_taosd_val_log.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' +IN_TDINTERNAL="community" +TDIR=`pwd` +if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + cd ../.. +else + cd ../../.. +fi +TOP_DIR=`pwd` +TAOSD_DIR=`find . -name "taosd"|grep -v community|grep debug|head -n1` +VALGRIND_OUT=taosd_valgrind.out +VALGRIND_ERR=taosd_valgrind.err +rm -rf /var/lib/taos/* +# nohup valgrind --tool=memcheck --leak-check=yes $TAOSD_DIR > $TDIR/$VALGRIND_OUT 2> $TDIR/$VALGRIND_ERR & +nohup valgrind --leak-check=yes $TAOSD_DIR > $TDIR/$VALGRIND_OUT 2> $TDIR/$VALGRIND_ERR & +sleep 5 +cd - +./crash_gen.sh -p -t 3 -s 200 +ps -ef |grep valgrind|grep -v grep|awk '{print $2}'|xargs kill -term +while true +do + monitoring=` ps -ef|grep valgrind |grep -v grep| wc -l` + if [ $monitoring -eq 0 ] + then + echo "Manipulator is not running " + break + else + sleep 1 + fi +done + +grep 'start to execute\|ERROR SUMMARY' $VALGRIND_ERR | grep -v 'grep' | uniq | tee taosd_mem_err.log + +for memError in `grep 'ERROR SUMMARY' taosd_mem_err.log | awk '{print $4}'` +do +memError=(${memError//,/}) +echo $memError +if [ -n "$memError" ]; then + if [ "$memError" -gt 12 ]; then + echo -e "${RED} ## Memory errors number valgrind reports is $memError.\ + More than our threshold! ## ${NC}" + fi +fi +done + +grep 'start to execute\|definitely lost:' $VALGRIND_ERR|grep -v 'grep'|uniq|tee taosd-definitely-lost-out.log +for defiMemError in `grep 'definitely lost:' taosd-definitely-lost-out.log | awk '{print $7}'` +do +defiMemError=(${defiMemError//,/}) +if [ -n "$defiMemError" ]; then + if [ "$defiMemError" -gt 0 -a "$defiMemError" -lt 1013 ]; then + cat $VALGRIND_ERR + echo -e "${RED} ## Memory errors number valgrind reports \ + Definitely lost is $defiMemError. More than our threshold! ## ${NC}" + exit 8 + elif [ "$defiMemError" -gt 1013 ];then #add for azure + cat $VALGRIND_ERR + echo -e "${RED} ## Memory errors number valgrind reports \ + Definitely lost is $defiMemError. More than our threshold! ## ${NC}" + exit 8 + fi +fi +done From 81e6b7301e42959a474eb117db0da9faa0baf577 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 27 Mar 2021 15:32:10 +0800 Subject: [PATCH 24/71] fix --- tests/pytest/handle_taosd_val_log.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/pytest/handle_taosd_val_log.sh b/tests/pytest/handle_taosd_val_log.sh index 221dfc649f..6b7f669efe 100755 --- a/tests/pytest/handle_taosd_val_log.sh +++ b/tests/pytest/handle_taosd_val_log.sh @@ -19,9 +19,9 @@ VALGRIND_ERR=taosd_valgrind.err rm -rf /var/lib/taos/* # nohup valgrind --tool=memcheck --leak-check=yes $TAOSD_DIR > $TDIR/$VALGRIND_OUT 2> $TDIR/$VALGRIND_ERR & nohup valgrind --leak-check=yes $TAOSD_DIR > $TDIR/$VALGRIND_OUT 2> $TDIR/$VALGRIND_ERR & -sleep 5 +sleep 20 cd - -./crash_gen.sh -p -t 3 -s 200 +./crash_gen.sh -p -t 10 -s 200 ps -ef |grep valgrind|grep -v grep|awk '{print $2}'|xargs kill -term while true do @@ -40,7 +40,6 @@ grep 'start to execute\|ERROR SUMMARY' $VALGRIND_ERR | grep -v 'grep' | uniq | t for memError in `grep 'ERROR SUMMARY' taosd_mem_err.log | awk '{print $4}'` do memError=(${memError//,/}) -echo $memError if [ -n "$memError" ]; then if [ "$memError" -gt 12 ]; then echo -e "${RED} ## Memory errors number valgrind reports is $memError.\ From 1f4bba65332904d14b695898decdf50c2afa3568 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 27 Mar 2021 15:34:40 +0800 Subject: [PATCH 25/71] fix --- Jenkinsfile | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index bd7d5d700a..52236682d7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -185,12 +185,14 @@ pipeline { rm -rf /var/log/taos/* ./handle_crash_gen_val_log.sh ''' - sh ''' - cd ${WKC}/tests/pytest - rm -rf /var/lib/taos/* - rm -rf /var/log/taos/* - ./handle_taosd_val_log.sh - ''' + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/pytest + rm -rf /var/lib/taos/* + rm -rf /var/log/taos/* + ./handle_crash_gen_val_log.sh + ''' + } timeout(time: 45, unit: 'MINUTES'){ sh ''' date From b03c0c20604a30bac3543d746d168b09c35a35b6 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 27 Mar 2021 15:39:03 +0800 Subject: [PATCH 26/71] fix --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 52236682d7..330ac1987b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -190,7 +190,7 @@ pipeline { cd ${WKC}/tests/pytest rm -rf /var/lib/taos/* rm -rf /var/log/taos/* - ./handle_crash_gen_val_log.sh + ./handle_taosd_val_log.sh ''' } timeout(time: 45, unit: 'MINUTES'){ From 868ee10750a7b40bbd81cdea2df4a0786160c552 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 27 Mar 2021 19:56:45 +0800 Subject: [PATCH 27/71] [TD-3546] : fix taosdemo quer crash as index variable mistake. (#5598) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 9e8e974096..c2387fc7d9 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -5578,9 +5578,9 @@ static void *subQueryProcess(void *sarg) { memset(sqlstr,0,sizeof(sqlstr)); replaceSubTblName(g_queryInfo.subQueryInfo.sql[j], sqlstr, i); char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.subQueryInfo.result[i][0] != 0) { + if (g_queryInfo.subQueryInfo.result[j][0] != 0) { sprintf(tmpFile, "%s-%d", - g_queryInfo.subQueryInfo.result[i], + g_queryInfo.subQueryInfo.result[j], winfo->threadID); } selectAndGetResult(winfo->taos, sqlstr, tmpFile); @@ -5825,7 +5825,8 @@ static void *subSubscribeProcess(void *sarg) { replaceSubTblName(g_queryInfo.subQueryInfo.sql[i], subSqlstr, i); char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.subQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", g_queryInfo.subQueryInfo.result[i], winfo->threadID); + sprintf(tmpFile, "%s-%d", + g_queryInfo.subQueryInfo.result[i], winfo->threadID); } g_queryInfo.subQueryInfo.tsub[i] = subscribeImpl( winfo->taos, subSqlstr, topic, tmpFile); From f69557fb6c346cf54212ad286bdf7956f870fa01 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 27 Mar 2021 19:57:12 +0800 Subject: [PATCH 28/71] [TD-3544] : move query times detect to query meta file parse. (#5597) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index c2387fc7d9..3cdc115504 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -3076,16 +3076,6 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } - cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times"); - if (gQueryTimes && gQueryTimes->type == cJSON_Number) { - g_args.query_times = gQueryTimes->valueint; - } else if (!gQueryTimes) { - g_args.query_times = 1; - } else { - errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", __func__, __LINE__); - goto PARSE_OVER; - } - cJSON* interlaceRows = cJSON_GetObjectItem(root, "interlace_rows"); if (interlaceRows && interlaceRows->type == cJSON_Number) { g_args.interlace_rows = interlaceRows->valueint; @@ -3729,6 +3719,16 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { goto PARSE_OVER; } + cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times"); + if (gQueryTimes && gQueryTimes->type == cJSON_Number) { + g_args.query_times = gQueryTimes->valueint; + } else if (!gQueryTimes) { + g_args.query_times = 1; + } else { + errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", __func__, __LINE__); + goto PARSE_OVER; + } + cJSON* dbs = cJSON_GetObjectItem(root, "databases"); if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) { tstrncpy(g_queryInfo.dbName, dbs->valuestring, MAX_DB_NAME_SIZE); @@ -4050,12 +4050,12 @@ static bool getInfoFromJsonFile(char* file) { if (INSERT_TEST == g_args.test_mode) { ret = getMetaFromInsertJsonFile(root); - } else if (QUERY_TEST == g_args.test_mode) { - ret = getMetaFromQueryJsonFile(root); - } else if (SUBSCRIBE_TEST == g_args.test_mode) { + } else if ((QUERY_TEST == g_args.test_mode) + || (SUBSCRIBE_TEST == g_args.test_mode)) { ret = getMetaFromQueryJsonFile(root); } else { - errorPrint("%s() LN%d, input json file type error! please input correct file type: insert or query or subscribe\n", __func__, __LINE__); + errorPrint("%s() LN%d, input json file type error! please input correct file type: insert or query or subscribe\n", + __func__, __LINE__); goto PARSE_OVER; } From 41741a599f354ceb46782fd0cfeb54c3c1dc2319 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 28 Mar 2021 20:23:11 +0800 Subject: [PATCH 29/71] Hotfix/sangshuduo/td 3197 fix taosdemo coverity scan (#5602) * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix taosdemo coverity scan issue. fix subscribeTest pids uninitialized. * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix coverity scan issues. check super tbl info pointer. * [TD-3197] : fix coverity scan issues. move sub tbl query thread join into loop * [TD-3197] : fix coverity scan issues. remove unused variable * [TD-3197] : fix coverity scan issues. use more secure random library * [TD-3197] : fix coverity scan issues. use strncpy for more safe * [TD-3197] : fix taosdemo coverity scan issue. replace arc4random with rand(). * [TD-3197] : fix coverity scan issues. check stb info pointer for start time * [TD-3197] : fix coverity scan issues. fix strcpy vulnerability * [TD-3197] : fix taosdemo coverity scan issue. modify taosdemoTest2. try to check database continously. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 165 ++++++++++++++++------------ tests/pytest/tools/taosdemoTest2.py | 10 +- 2 files changed, 101 insertions(+), 74 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 3cdc115504..ff69c9f81a 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -69,7 +69,10 @@ enum TEST_MODE { #define MAX_SQL_SIZE 65536 #define BUFFER_SIZE (65536*2) +#define MAX_USERNAME_SIZE 64 +#define MAX_PASSWORD_SIZE 64 #define MAX_DB_NAME_SIZE 64 +#define MAX_HOSTNAME_SIZE 64 #define MAX_TB_NAME_SIZE 64 #define MAX_DATA_SIZE 16000 #define MAX_NUM_DATATYPE 10 @@ -324,10 +327,10 @@ typedef struct SDataBase_S { typedef struct SDbs_S { char cfgDir[MAX_FILE_NAME_LEN+1]; - char host[MAX_DB_NAME_SIZE]; + char host[MAX_HOSTNAME_SIZE]; uint16_t port; - char user[MAX_DB_NAME_SIZE]; - char password[MAX_DB_NAME_SIZE]; + char user[MAX_USERNAME_SIZE]; + char password[MAX_PASSWORD_SIZE]; char resultFile[MAX_FILE_NAME_LEN+1]; bool use_metric; bool insert_only; @@ -378,10 +381,10 @@ typedef struct SubQueryInfo_S { typedef struct SQueryMetaInfo_S { char cfgDir[MAX_FILE_NAME_LEN+1]; - char host[MAX_DB_NAME_SIZE]; + char host[MAX_HOSTNAME_SIZE]; uint16_t port; - char user[MAX_DB_NAME_SIZE]; - char password[MAX_DB_NAME_SIZE]; + char user[MAX_USERNAME_SIZE]; + char password[MAX_PASSWORD_SIZE]; char dbName[MAX_DB_NAME_SIZE+1]; char queryMode[MAX_TB_NAME_SIZE]; // taosc, restful @@ -491,9 +494,12 @@ static void resetAfterAnsiEscape(void) { printf("\x1b[0m"); } +#include + static int taosRandom() { - return random(); + srand(time(NULL)); + return rand(); } #endif @@ -2532,39 +2538,46 @@ static void* createTable(void *sarg) g_args.tb_prefix, i, winfo->cols); } else { - if (0 == len) { - batchNum = 0; - memset(buffer, 0, buff_len); - len += snprintf(buffer + len, - buff_len - len, "create table "); - } - - char* tagsValBuf = NULL; - if (0 == superTblInfo->tagSource) { - tagsValBuf = generateTagVaulesForStb(superTblInfo); - } else { - tagsValBuf = getTagValueFromTagSample( - superTblInfo, - i % superTblInfo->tagSampleCount); - } - if (NULL == tagsValBuf) { + if (superTblInfo == NULL) { + errorPrint("%s() LN%d, use metric, but super table info is NULL\n", + __func__, __LINE__); free(buffer); - return NULL; - } - - len += snprintf(buffer + len, - superTblInfo->maxSqlLen - len, - "if not exists %s.%s%d using %s.%s tags %s ", - winfo->db_name, superTblInfo->childTblPrefix, - i, winfo->db_name, - superTblInfo->sTblName, tagsValBuf); - free(tagsValBuf); - batchNum++; - - if ((batchNum < superTblInfo->batchCreateTableNum) - && ((superTblInfo->maxSqlLen - len) - >= (superTblInfo->lenOfTagOfOneRow + 256))) { - continue; + exit(-1); + } else { + if (0 == len) { + batchNum = 0; + memset(buffer, 0, buff_len); + len += snprintf(buffer + len, + buff_len - len, "create table "); + } + + char* tagsValBuf = NULL; + if (0 == superTblInfo->tagSource) { + tagsValBuf = generateTagVaulesForStb(superTblInfo); + } else { + tagsValBuf = getTagValueFromTagSample( + superTblInfo, + i % superTblInfo->tagSampleCount); + } + if (NULL == tagsValBuf) { + free(buffer); + return NULL; + } + + len += snprintf(buffer + len, + superTblInfo->maxSqlLen - len, + "if not exists %s.%s%d using %s.%s tags %s ", + winfo->db_name, superTblInfo->childTblPrefix, + i, winfo->db_name, + superTblInfo->sTblName, tagsValBuf); + free(tagsValBuf); + batchNum++; + + if ((batchNum < superTblInfo->batchCreateTableNum) + && ((superTblInfo->maxSqlLen - len) + >= (superTblInfo->lenOfTagOfOneRow + 256))) { + continue; + } } } @@ -2699,10 +2712,10 @@ static void createChildTables() { if ((strncasecmp(g_args.datatype[j], "BINARY", strlen("BINARY")) == 0) || (strncasecmp(g_args.datatype[j], "NCHAR", strlen("NCHAR")) == 0)) { - len = snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, + snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ", COL%d %s(60)", j, g_args.datatype[j]); } else { - len = snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, + snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ", COL%d %s", j, g_args.datatype[j]); } len = strlen(tblColsBuf); @@ -3009,9 +3022,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { cJSON* host = cJSON_GetObjectItem(root, "host"); if (host && host->type == cJSON_String && host->valuestring != NULL) { - tstrncpy(g_Dbs.host, host->valuestring, MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.host, host->valuestring, MAX_HOSTNAME_SIZE); } else if (!host) { - tstrncpy(g_Dbs.host, "127.0.0.1", MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE); } else { printf("ERROR: failed to read json, host not found\n"); goto PARSE_OVER; @@ -3026,16 +3039,16 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { cJSON* user = cJSON_GetObjectItem(root, "user"); if (user && user->type == cJSON_String && user->valuestring != NULL) { - tstrncpy(g_Dbs.user, user->valuestring, MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.user, user->valuestring, MAX_USERNAME_SIZE); } else if (!user) { - tstrncpy(g_Dbs.user, "root", MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.user, "root", MAX_USERNAME_SIZE); } cJSON* password = cJSON_GetObjectItem(root, "password"); if (password && password->type == cJSON_String && password->valuestring != NULL) { - tstrncpy(g_Dbs.password, password->valuestring, MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.password, password->valuestring, MAX_PASSWORD_SIZE); } else if (!password) { - tstrncpy(g_Dbs.password, "taosdata", MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.password, "taosdata", MAX_PASSWORD_SIZE); } cJSON* resultfile = cJSON_GetObjectItem(root, "result_file"); @@ -3673,9 +3686,9 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* host = cJSON_GetObjectItem(root, "host"); if (host && host->type == cJSON_String && host->valuestring != NULL) { - tstrncpy(g_queryInfo.host, host->valuestring, MAX_DB_NAME_SIZE); + tstrncpy(g_queryInfo.host, host->valuestring, MAX_HOSTNAME_SIZE); } else if (!host) { - tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_DB_NAME_SIZE); + tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE); } else { printf("ERROR: failed to read json, host not found\n"); goto PARSE_OVER; @@ -3690,16 +3703,16 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* user = cJSON_GetObjectItem(root, "user"); if (user && user->type == cJSON_String && user->valuestring != NULL) { - tstrncpy(g_queryInfo.user, user->valuestring, MAX_DB_NAME_SIZE); + tstrncpy(g_queryInfo.user, user->valuestring, MAX_USERNAME_SIZE); } else if (!user) { - tstrncpy(g_queryInfo.user, "root", MAX_DB_NAME_SIZE); ; + tstrncpy(g_queryInfo.user, "root", MAX_USERNAME_SIZE); ; } cJSON* password = cJSON_GetObjectItem(root, "password"); if (password && password->type == cJSON_String && password->valuestring != NULL) { - tstrncpy(g_queryInfo.password, password->valuestring, MAX_DB_NAME_SIZE); + tstrncpy(g_queryInfo.password, password->valuestring, MAX_PASSWORD_SIZE); } else if (!password) { - tstrncpy(g_queryInfo.password, "taosdata", MAX_DB_NAME_SIZE);; + tstrncpy(g_queryInfo.password, "taosdata", MAX_PASSWORD_SIZE);; } cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no, @@ -4611,8 +4624,13 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { verbosePrint("[%d] %s() LN%d i=%d batchPerTblTimes=%d batchPerTbl = %d\n", pThreadInfo->threadID, __func__, __LINE__, i, batchPerTblTimes, batchPerTbl); - if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { - startTime = taosGetTimestamp(pThreadInfo->time_precision); + + if (superTblInfo) { + if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { + startTime = taosGetTimestamp(pThreadInfo->time_precision); + } + } else { + startTime = 1500000000000; } generateDataTail( tableName, tableSeq, pThreadInfo, superTblInfo, @@ -6056,7 +6074,12 @@ static int subscribeTestProcess() { t_info->taos = NULL; // TODO: workaround to use separate taos connection; pthread_create(pidsOfSub + i, NULL, subSubscribeProcess, t_info); } + g_queryInfo.subQueryInfo.threadCnt = threads; + + for (int i = 0; i < g_queryInfo.subQueryInfo.threadCnt; i++) { + pthread_join(pidsOfSub[i], NULL); + } } for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { @@ -6066,10 +6089,6 @@ static int subscribeTestProcess() { tmfree((char*)pids); tmfree((char*)infos); - for (int i = 0; i < g_queryInfo.subQueryInfo.threadCnt; i++) { - pthread_join(pidsOfSub[i], NULL); - } - tmfree((char*)pidsOfSub); tmfree((char*)infosOfSub); // taos_close(taos); @@ -6080,10 +6099,10 @@ static void initOfInsertMeta() { memset(&g_Dbs, 0, sizeof(SDbs)); // set default values - tstrncpy(g_Dbs.host, "127.0.0.1", MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE); g_Dbs.port = 6030; - tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_DB_NAME_SIZE); - tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE); + tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE); g_Dbs.threadCount = 2; g_Dbs.use_metric = g_args.use_metric; @@ -6093,25 +6112,25 @@ static void initOfQueryMeta() { memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo)); // set default values - tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_DB_NAME_SIZE); + tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE); g_queryInfo.port = 6030; - tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_DB_NAME_SIZE); - tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_DB_NAME_SIZE); + tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE); + tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE); } static void setParaFromArg(){ if (g_args.host) { - strcpy(g_Dbs.host, g_args.host); + tstrncpy(g_Dbs.host, g_args.host, MAX_HOSTNAME_SIZE); } else { - tstrncpy(g_Dbs.host, "127.0.0.1", MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE); } if (g_args.user) { - strcpy(g_Dbs.user, g_args.user); + tstrncpy(g_Dbs.user, g_args.user, MAX_USERNAME_SIZE); } if (g_args.password) { - strcpy(g_Dbs.password, g_args.password); + tstrncpy(g_Dbs.password, g_args.password, MAX_PASSWORD_SIZE); } if (g_args.port) { @@ -6331,12 +6350,12 @@ static void queryResult() { rInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount; rInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1; rInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; - strcpy(rInfo->tb_prefix, - g_Dbs.db[0].superTbls[0].childTblPrefix); + tstrncpy(rInfo->tb_prefix, + g_Dbs.db[0].superTbls[0].childTblPrefix, MAX_TB_NAME_SIZE); } else { rInfo->ntables = g_args.num_of_tables; rInfo->end_table_to = g_args.num_of_tables -1; - strcpy(rInfo->tb_prefix, g_args.tb_prefix); + tstrncpy(rInfo->tb_prefix, g_args.tb_prefix, MAX_TB_NAME_SIZE); } rInfo->taos = taos_connect( @@ -6352,7 +6371,7 @@ static void queryResult() { exit(-1); } - strcpy(rInfo->fp, g_Dbs.resultFile); + tstrncpy(rInfo->fp, g_Dbs.resultFile, MAX_FILE_NAME_LEN); if (!g_Dbs.use_metric) { pthread_create(&read_id, NULL, readTable, rInfo); diff --git a/tests/pytest/tools/taosdemoTest2.py b/tests/pytest/tools/taosdemoTest2.py index 92f6fa1265..74b05faf8b 100644 --- a/tests/pytest/tools/taosdemoTest2.py +++ b/tests/pytest/tools/taosdemoTest2.py @@ -36,7 +36,15 @@ class TDTestCase: if(threadID == 1): time.sleep(2) print("use test") - tdSql.execute("use test") + while True: + try: + tdSql.execute("use test") + break + except Exception as e: + tdLog.info("use database test failed") + time.sleep(1) + continue + # check if all the tables have heen created while True: tdSql.query("show tables") From 4508bb998f96bcf3a1079a59146507145c1e2887 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Mon, 29 Mar 2021 10:15:41 +0800 Subject: [PATCH 30/71] [TD-3474]: add test case --- tests/pytest/insert/metadataUpdate.py | 55 +++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/pytest/insert/metadataUpdate.py b/tests/pytest/insert/metadataUpdate.py index 76b9a3ae8f..4c32b0e39a 100644 --- a/tests/pytest/insert/metadataUpdate.py +++ b/tests/pytest/insert/metadataUpdate.py @@ -25,6 +25,8 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) self.ts = 1538548685000 + self.tables = 10 + self.rows = 1000 def updateMetadata(self): self.host = "127.0.0.1" @@ -37,6 +39,29 @@ class TDTestCase: self.cursor.execute("alter table db.tb add column col2 int") print("alter table done") + def deleteTableAndRecreate(self): + self.host = "127.0.0.1" + self.user = "root" + self.password = "taosdata" + self.config = tdDnodes.getSimCfgPath() + + self.conn = taos.connect(host = self.host, user = self.user, password = self.password, config = self.config) + self.cursor = self.conn.cursor() + + self.cursor.execute("use test") + print("drop table stb") + self.cursor.execute("drop table stb") + + print("create table stb") + self.cursor.execute("create table if not exists stb (ts timestamp, col1 int) tags(areaid int, city nchar(20))") + print("insert data") + for i in range(self.tables): + city = "beijing" if i % 2 == 0 else "shanghai" + self.cursor.execute("create table tb%d using stb tags(%d, '%s')" % (i, i, city)) + for j in range(self.rows): + self.cursor.execute("insert into tb%d values(%d, %d)" % (i, self.ts + j, j * 100000)) + + def run(self): tdSql.prepare() @@ -59,6 +84,36 @@ class TDTestCase: tdSql.query("select * from tb") tdSql.checkRows(2) + # Add test case: https://jira.taosdata.com:18080/browse/TD-3474 + + print("==============step1") + tdSql.execute("create database test") + tdSql.execute("use test") + tdSql.execute("create table if not exists stb (ts timestamp, col1 int) tags(areaid int, city nchar(20))") + + for i in range(self.tables): + city = "beijing" if i % 2 == 0 else "shanghai" + tdSql.execute("create table tb%d using stb tags(%d, '%s')" % (i, i, city)) + for j in range(self.rows): + tdSql.execute("insert into tb%d values(%d, %d)" % (i, self.ts + j, j * 100000)) + + tdSql.query("select count(*) from stb") + tdSql.checkData(0, 0, 10000) + + tdSql.query("select count(*) from tb1") + tdSql.checkData(0, 0, 1000) + + p = Process(target=self.deleteTableAndRecreate, args=()) + p.start() + p.join() + p.terminate() + + tdSql.query("select count(*) from stb") + tdSql.checkData(0, 0, 10000) + + tdSql.query("select count(*) from tb1") + tdSql.checkData(0, 0, 1000) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From 4b69ffbfdbcedbd6567cdc1d4ad6b0a88fa23bb3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 29 Mar 2021 03:02:38 +0000 Subject: [PATCH 31/71] [TD-3574]: Drop database operation is not completed when restarting, may cause taosd crash --- src/dnode/src/dnodeMain.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index c24eac84cf..952f220c57 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -63,13 +63,16 @@ static SStep tsDnodeSteps[] = { {"dnode-vread", dnodeInitVRead, dnodeCleanupVRead}, {"dnode-vwrite", dnodeInitVWrite, dnodeCleanupVWrite}, {"dnode-vmgmt", dnodeInitVMgmt, dnodeCleanupVMgmt}, - {"dnode-mread", dnodeInitMRead, dnodeCleanupMRead}, - {"dnode-mwrite", dnodeInitMWrite, dnodeCleanupMWrite}, - {"dnode-mpeer", dnodeInitMPeer, dnodeCleanupMPeer}, + {"dnode-mread", dnodeInitMRead, NULL}, + {"dnode-mwrite", dnodeInitMWrite, NULL}, + {"dnode-mpeer", dnodeInitMPeer, NULL}, {"dnode-client", dnodeInitClient, dnodeCleanupClient}, {"dnode-server", dnodeInitServer, dnodeCleanupServer}, {"dnode-vnodes", dnodeInitVnodes, dnodeCleanupVnodes}, {"dnode-modules", dnodeInitModules, dnodeCleanupModules}, + {"dnode-mread", NULL, dnodeCleanupMRead}, + {"dnode-mwrite", NULL, dnodeCleanupMWrite}, + {"dnode-mpeer", NULL, dnodeCleanupMPeer}, {"dnode-shell", dnodeInitShell, dnodeCleanupShell}, {"dnode-statustmr", dnodeInitStatusTimer,dnodeCleanupStatusTimer}, {"dnode-telemetry", dnodeInitTelemetry, dnodeCleanupTelemetry}, From 5be1a1d7368d67427f2ab9ded003264dc224670f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 29 Mar 2021 03:14:51 +0000 Subject: [PATCH 32/71] [TD-3449]: fix drop if not exists error --- tests/script/general/db/topic1.sim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/script/general/db/topic1.sim b/tests/script/general/db/topic1.sim index e17a4996f5..42613405af 100644 --- a/tests/script/general/db/topic1.sim +++ b/tests/script/general/db/topic1.sim @@ -878,5 +878,7 @@ if $rows != 0 then endi sql drop topic t2 +sql_error drop topic abc +sql drop topic if exists abc; system sh/exec.sh -n dnode1 -s stop -x SIGINT From 92179b81087efddb847f6f73f4d17a1283e7175b Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 29 Mar 2021 13:55:40 +0800 Subject: [PATCH 33/71] Hotfix/sangshuduo/td 3575 support stb query times (#5605) * [TD-3575] : support query times in specified and super query. refactor code. * [TD-3575] : support query times in specified and super query. refactor code 2. * [TD-3575] : support query times in specified and super query. refactor code 3. * [TD-3575] : support query times in specified and super query. add query times parse in sub sections. * [TD-3575] : support query times in specified and super query. replace query times with specified query or super query. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 405 +++++++++++++++++++----------------- 1 file changed, 214 insertions(+), 191 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index ff69c9f81a..fd4023964a 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -348,20 +348,21 @@ typedef struct SDbs_S { } SDbs; -typedef struct SuperQueryInfo_S { +typedef struct SpecifiedQueryInfo_S { int rate; // 0: unlimit > 0 loop/s int concurrent; int sqlCount; int subscribeMode; // 0: sync, 1: async int subscribeInterval; // ms + int queryTimes; int subscribeRestart; int subscribeKeepProgress; char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; -} SuperQueryInfo; +} SpecifiedQueryInfo; -typedef struct SubQueryInfo_S { +typedef struct SuperQueryInfo_S { char sTblName[MAX_TB_NAME_SIZE+1]; int rate; // 0: unlimit > 0 loop/s int threadCnt; @@ -369,6 +370,7 @@ typedef struct SubQueryInfo_S { int subscribeInterval; // ms int subscribeRestart; int subscribeKeepProgress; + int queryTimes; int childTblCount; char childTblPrefix[MAX_TB_NAME_SIZE]; int sqlCount; @@ -377,7 +379,7 @@ typedef struct SubQueryInfo_S { TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; char* childTblName; -} SubQueryInfo; +} SuperQueryInfo; typedef struct SQueryMetaInfo_S { char cfgDir[MAX_FILE_NAME_LEN+1]; @@ -388,8 +390,8 @@ typedef struct SQueryMetaInfo_S { char dbName[MAX_DB_NAME_SIZE+1]; char queryMode[MAX_TB_NAME_SIZE]; // taosc, restful - SuperQueryInfo superQueryInfo; - SubQueryInfo subQueryInfo; + SpecifiedQueryInfo specifiedQueryInfo; + SuperQueryInfo superQueryInfo; } SQueryMetaInfo; typedef struct SThreadInfo_S { @@ -1434,10 +1436,27 @@ static void printfQueryMeta() { printf("\n"); printf("specified table query info: \n"); - printf("query interval: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.rate); + printf("query interval: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.rate); printf("query times: \033[33m%d\033[0m\n", g_args.query_times); - printf("concurrent: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.concurrent); - printf("sqlCount: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.sqlCount); + printf("concurrent: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.concurrent); + printf("sqlCount: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.sqlCount); + + if (SUBSCRIBE_TEST == g_args.test_mode) { + printf("mod: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.subscribeMode); + printf("interval: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.subscribeInterval); + printf("restart: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.subscribeRestart); + printf("keepProgress: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); + } + + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { + printf(" sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.specifiedQueryInfo.sql[i]); + } + printf("\n"); + printf("super table query info: \n"); + printf("query interval: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.rate); + printf("threadCnt: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.threadCnt); + printf("childTblCount: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.childTblCount); + printf("stable name: \033[33m%s\033[0m\n", g_queryInfo.superQueryInfo.sTblName); if (SUBSCRIBE_TEST == g_args.test_mode) { printf("mod: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeMode); @@ -1446,28 +1465,11 @@ static void printfQueryMeta() { printf("keepProgress: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeKeepProgress); } + printf("sqlCount: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.sqlCount); for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { printf(" sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.superQueryInfo.sql[i]); } printf("\n"); - printf("super table query info: \n"); - printf("query interval: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.rate); - printf("threadCnt: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.threadCnt); - printf("childTblCount: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.childTblCount); - printf("stable name: \033[33m%s\033[0m\n", g_queryInfo.subQueryInfo.sTblName); - - if (SUBSCRIBE_TEST == g_args.test_mode) { - printf("mod: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeMode); - printf("interval: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeInterval); - printf("restart: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeRestart); - printf("keepProgress: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeKeepProgress); - } - - printf("sqlCount: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.sqlCount); - for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) { - printf(" sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.subQueryInfo.sql[i]); - } - printf("\n"); SHOW_PARSE_RESULT_END(); } @@ -3761,85 +3763,95 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } // super_table_query - cJSON *superQuery = cJSON_GetObjectItem(root, "specified_table_query"); - if (!superQuery) { - g_queryInfo.superQueryInfo.concurrent = 0; - g_queryInfo.superQueryInfo.sqlCount = 0; - } else if (superQuery->type != cJSON_Object) { + cJSON *specifiedQuery = cJSON_GetObjectItem(root, "specified_table_query"); + if (!specifiedQuery) { + g_queryInfo.specifiedQueryInfo.concurrent = 0; + g_queryInfo.specifiedQueryInfo.sqlCount = 0; + } else if (specifiedQuery->type != cJSON_Object) { printf("ERROR: failed to read json, super_table_query not found\n"); goto PARSE_OVER; } else { - cJSON* rate = cJSON_GetObjectItem(superQuery, "query_interval"); + cJSON* rate = cJSON_GetObjectItem(specifiedQuery, "query_interval"); if (rate && rate->type == cJSON_Number) { - g_queryInfo.superQueryInfo.rate = rate->valueint; + g_queryInfo.specifiedQueryInfo.rate = rate->valueint; } else if (!rate) { - g_queryInfo.superQueryInfo.rate = 0; + g_queryInfo.specifiedQueryInfo.rate = 0; } - cJSON* concurrent = cJSON_GetObjectItem(superQuery, "concurrent"); + cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery, "query_times"); + if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) { + g_queryInfo.specifiedQueryInfo.queryTimes = specifiedQueryTimes->valueint; + } else if (!specifiedQueryTimes) { + g_queryInfo.specifiedQueryInfo.queryTimes = g_args.query_times; + } else { + errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON* concurrent = cJSON_GetObjectItem(specifiedQuery, "concurrent"); if (concurrent && concurrent->type == cJSON_Number) { - g_queryInfo.superQueryInfo.concurrent = concurrent->valueint; + g_queryInfo.specifiedQueryInfo.concurrent = concurrent->valueint; } else if (!concurrent) { - g_queryInfo.superQueryInfo.concurrent = 1; + g_queryInfo.specifiedQueryInfo.concurrent = 1; } - cJSON* mode = cJSON_GetObjectItem(superQuery, "mode"); + cJSON* mode = cJSON_GetObjectItem(specifiedQuery, "mode"); if (mode && mode->type == cJSON_String && mode->valuestring != NULL) { if (0 == strcmp("sync", mode->valuestring)) { - g_queryInfo.superQueryInfo.subscribeMode = 0; + g_queryInfo.specifiedQueryInfo.subscribeMode = 0; } else if (0 == strcmp("async", mode->valuestring)) { - g_queryInfo.superQueryInfo.subscribeMode = 1; + g_queryInfo.specifiedQueryInfo.subscribeMode = 1; } else { printf("ERROR: failed to read json, subscribe mod error\n"); goto PARSE_OVER; } } else { - g_queryInfo.superQueryInfo.subscribeMode = 0; + g_queryInfo.specifiedQueryInfo.subscribeMode = 0; } - cJSON* interval = cJSON_GetObjectItem(superQuery, "interval"); + cJSON* interval = cJSON_GetObjectItem(specifiedQuery, "interval"); if (interval && interval->type == cJSON_Number) { - g_queryInfo.superQueryInfo.subscribeInterval = interval->valueint; + g_queryInfo.specifiedQueryInfo.subscribeInterval = interval->valueint; } else if (!interval) { //printf("failed to read json, subscribe interval no found\n"); //goto PARSE_OVER; - g_queryInfo.superQueryInfo.subscribeInterval = 10000; + g_queryInfo.specifiedQueryInfo.subscribeInterval = 10000; } - cJSON* restart = cJSON_GetObjectItem(superQuery, "restart"); + cJSON* restart = cJSON_GetObjectItem(specifiedQuery, "restart"); if (restart && restart->type == cJSON_String && restart->valuestring != NULL) { if (0 == strcmp("yes", restart->valuestring)) { - g_queryInfo.superQueryInfo.subscribeRestart = 1; + g_queryInfo.specifiedQueryInfo.subscribeRestart = 1; } else if (0 == strcmp("no", restart->valuestring)) { - g_queryInfo.superQueryInfo.subscribeRestart = 0; + g_queryInfo.specifiedQueryInfo.subscribeRestart = 0; } else { printf("ERROR: failed to read json, subscribe restart error\n"); goto PARSE_OVER; } } else { - g_queryInfo.superQueryInfo.subscribeRestart = 1; + g_queryInfo.specifiedQueryInfo.subscribeRestart = 1; } - cJSON* keepProgress = cJSON_GetObjectItem(superQuery, "keepProgress"); + cJSON* keepProgress = cJSON_GetObjectItem(specifiedQuery, "keepProgress"); if (keepProgress && keepProgress->type == cJSON_String && keepProgress->valuestring != NULL) { if (0 == strcmp("yes", keepProgress->valuestring)) { - g_queryInfo.superQueryInfo.subscribeKeepProgress = 1; + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 1; } else if (0 == strcmp("no", keepProgress->valuestring)) { - g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0; } else { printf("ERROR: failed to read json, subscribe keepProgress error\n"); goto PARSE_OVER; } } else { - g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0; } // sqls - cJSON* superSqls = cJSON_GetObjectItem(superQuery, "sqls"); + cJSON* superSqls = cJSON_GetObjectItem(specifiedQuery, "sqls"); if (!superSqls) { - g_queryInfo.superQueryInfo.sqlCount = 0; + g_queryInfo.specifiedQueryInfo.sqlCount = 0; } else if (superSqls->type != cJSON_Array) { printf("ERROR: failed to read json, super sqls not found\n"); goto PARSE_OVER; @@ -3850,7 +3862,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { goto PARSE_OVER; } - g_queryInfo.superQueryInfo.sqlCount = superSqlSize; + g_queryInfo.specifiedQueryInfo.sqlCount = superSqlSize; for (int j = 0; j < superSqlSize; ++j) { cJSON* sql = cJSON_GetArrayItem(superSqls, j); if (sql == NULL) continue; @@ -3860,13 +3872,13 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { printf("ERROR: failed to read json, sql not found\n"); goto PARSE_OVER; } - tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); + tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); cJSON *result = cJSON_GetObjectItem(sql, "result"); if (NULL != result && result->type == cJSON_String && result->valuestring != NULL) { - tstrncpy(g_queryInfo.superQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN); + tstrncpy(g_queryInfo.specifiedQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN); } else if (NULL == result) { - memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN); + memset(g_queryInfo.specifiedQueryInfo.result[j], 0, MAX_FILE_NAME_LEN); } else { printf("ERROR: failed to read json, super query result file not found\n"); goto PARSE_OVER; @@ -3876,101 +3888,111 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } // sub_table_query - cJSON *subQuery = cJSON_GetObjectItem(root, "super_table_query"); - if (!subQuery) { - g_queryInfo.subQueryInfo.threadCnt = 0; - g_queryInfo.subQueryInfo.sqlCount = 0; - } else if (subQuery->type != cJSON_Object) { + cJSON *superQuery = cJSON_GetObjectItem(root, "super_table_query"); + if (!superQuery) { + g_queryInfo.superQueryInfo.threadCnt = 0; + g_queryInfo.superQueryInfo.sqlCount = 0; + } else if (superQuery->type != cJSON_Object) { printf("ERROR: failed to read json, sub_table_query not found\n"); ret = true; goto PARSE_OVER; } else { - cJSON* subrate = cJSON_GetObjectItem(subQuery, "query_interval"); + cJSON* subrate = cJSON_GetObjectItem(superQuery, "query_interval"); if (subrate && subrate->type == cJSON_Number) { - g_queryInfo.subQueryInfo.rate = subrate->valueint; + g_queryInfo.superQueryInfo.rate = subrate->valueint; } else if (!subrate) { - g_queryInfo.subQueryInfo.rate = 0; + g_queryInfo.superQueryInfo.rate = 0; } - cJSON* threads = cJSON_GetObjectItem(subQuery, "threads"); + cJSON* superQueryTimes = cJSON_GetObjectItem(superQuery, "query_times"); + if (superQueryTimes && superQueryTimes->type == cJSON_Number) { + g_queryInfo.superQueryInfo.queryTimes = superQueryTimes->valueint; + } else if (!superQueryTimes) { + g_queryInfo.superQueryInfo.queryTimes = g_args.query_times; + } else { + errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON* threads = cJSON_GetObjectItem(superQuery, "threads"); if (threads && threads->type == cJSON_Number) { - g_queryInfo.subQueryInfo.threadCnt = threads->valueint; + g_queryInfo.superQueryInfo.threadCnt = threads->valueint; } else if (!threads) { - g_queryInfo.subQueryInfo.threadCnt = 1; + g_queryInfo.superQueryInfo.threadCnt = 1; } - //cJSON* subTblCnt = cJSON_GetObjectItem(subQuery, "childtable_count"); + //cJSON* subTblCnt = cJSON_GetObjectItem(superQuery, "childtable_count"); //if (subTblCnt && subTblCnt->type == cJSON_Number) { - // g_queryInfo.subQueryInfo.childTblCount = subTblCnt->valueint; + // g_queryInfo.superQueryInfo.childTblCount = subTblCnt->valueint; //} else if (!subTblCnt) { - // g_queryInfo.subQueryInfo.childTblCount = 0; + // g_queryInfo.superQueryInfo.childTblCount = 0; //} - cJSON* stblname = cJSON_GetObjectItem(subQuery, "stblname"); + cJSON* stblname = cJSON_GetObjectItem(superQuery, "stblname"); if (stblname && stblname->type == cJSON_String && stblname->valuestring != NULL) { - tstrncpy(g_queryInfo.subQueryInfo.sTblName, stblname->valuestring, MAX_TB_NAME_SIZE); + tstrncpy(g_queryInfo.superQueryInfo.sTblName, stblname->valuestring, MAX_TB_NAME_SIZE); } else { printf("ERROR: failed to read json, super table name not found\n"); goto PARSE_OVER; } - cJSON* submode = cJSON_GetObjectItem(subQuery, "mode"); + cJSON* submode = cJSON_GetObjectItem(superQuery, "mode"); if (submode && submode->type == cJSON_String && submode->valuestring != NULL) { if (0 == strcmp("sync", submode->valuestring)) { - g_queryInfo.subQueryInfo.subscribeMode = 0; + g_queryInfo.superQueryInfo.subscribeMode = 0; } else if (0 == strcmp("async", submode->valuestring)) { - g_queryInfo.subQueryInfo.subscribeMode = 1; + g_queryInfo.superQueryInfo.subscribeMode = 1; } else { printf("ERROR: failed to read json, subscribe mod error\n"); goto PARSE_OVER; } } else { - g_queryInfo.subQueryInfo.subscribeMode = 0; + g_queryInfo.superQueryInfo.subscribeMode = 0; } - cJSON* subinterval = cJSON_GetObjectItem(subQuery, "interval"); + cJSON* subinterval = cJSON_GetObjectItem(superQuery, "interval"); if (subinterval && subinterval->type == cJSON_Number) { - g_queryInfo.subQueryInfo.subscribeInterval = subinterval->valueint; + g_queryInfo.superQueryInfo.subscribeInterval = subinterval->valueint; } else if (!subinterval) { //printf("failed to read json, subscribe interval no found\n"); //goto PARSE_OVER; - g_queryInfo.subQueryInfo.subscribeInterval = 10000; + g_queryInfo.superQueryInfo.subscribeInterval = 10000; } - cJSON* subrestart = cJSON_GetObjectItem(subQuery, "restart"); + cJSON* subrestart = cJSON_GetObjectItem(superQuery, "restart"); if (subrestart && subrestart->type == cJSON_String && subrestart->valuestring != NULL) { if (0 == strcmp("yes", subrestart->valuestring)) { - g_queryInfo.subQueryInfo.subscribeRestart = 1; + g_queryInfo.superQueryInfo.subscribeRestart = 1; } else if (0 == strcmp("no", subrestart->valuestring)) { - g_queryInfo.subQueryInfo.subscribeRestart = 0; + g_queryInfo.superQueryInfo.subscribeRestart = 0; } else { printf("ERROR: failed to read json, subscribe restart error\n"); goto PARSE_OVER; } } else { - g_queryInfo.subQueryInfo.subscribeRestart = 1; + g_queryInfo.superQueryInfo.subscribeRestart = 1; } - cJSON* subkeepProgress = cJSON_GetObjectItem(subQuery, "keepProgress"); + cJSON* subkeepProgress = cJSON_GetObjectItem(superQuery, "keepProgress"); if (subkeepProgress && subkeepProgress->type == cJSON_String && subkeepProgress->valuestring != NULL) { if (0 == strcmp("yes", subkeepProgress->valuestring)) { - g_queryInfo.subQueryInfo.subscribeKeepProgress = 1; + g_queryInfo.superQueryInfo.subscribeKeepProgress = 1; } else if (0 == strcmp("no", subkeepProgress->valuestring)) { - g_queryInfo.subQueryInfo.subscribeKeepProgress = 0; + g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; } else { printf("ERROR: failed to read json, subscribe keepProgress error\n"); goto PARSE_OVER; } } else { - g_queryInfo.subQueryInfo.subscribeKeepProgress = 0; + g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; } // sqls - cJSON* subsqls = cJSON_GetObjectItem(subQuery, "sqls"); + cJSON* subsqls = cJSON_GetObjectItem(superQuery, "sqls"); if (!subsqls) { - g_queryInfo.subQueryInfo.sqlCount = 0; + g_queryInfo.superQueryInfo.sqlCount = 0; } else if (subsqls->type != cJSON_Array) { printf("ERROR: failed to read json, super sqls not found\n"); goto PARSE_OVER; @@ -3981,7 +4003,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { goto PARSE_OVER; } - g_queryInfo.subQueryInfo.sqlCount = superSqlSize; + g_queryInfo.superQueryInfo.sqlCount = superSqlSize; for (int j = 0; j < superSqlSize; ++j) { cJSON* sql = cJSON_GetArrayItem(subsqls, j); if (sql == NULL) continue; @@ -3991,13 +4013,13 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { printf("ERROR: failed to read json, sql not found\n"); goto PARSE_OVER; } - tstrncpy(g_queryInfo.subQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); + tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); cJSON *result = cJSON_GetObjectItem(sql, "result"); if (result != NULL && result->type == cJSON_String && result->valuestring != NULL){ - tstrncpy(g_queryInfo.subQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN); + tstrncpy(g_queryInfo.superQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN); } else if (NULL == result) { - memset(g_queryInfo.subQueryInfo.result[j], 0, MAX_FILE_NAME_LEN); + memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN); } else { printf("ERROR: failed to read json, sub query result file not found\n"); goto PARSE_OVER; @@ -5494,32 +5516,32 @@ static void *superQueryProcess(void *sarg) { int64_t st = 0; int64_t et = 0; - int queryTimes = g_args.query_times; + int queryTimes = g_queryInfo.specifiedQueryInfo.queryTimes; while(queryTimes --) { - if (g_queryInfo.superQueryInfo.rate && (et - st) < - (int64_t)g_queryInfo.superQueryInfo.rate*1000) { - taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms + if (g_queryInfo.specifiedQueryInfo.rate && (et - st) < + (int64_t)g_queryInfo.specifiedQueryInfo.rate*1000) { + taosMsleep(g_queryInfo.specifiedQueryInfo.rate*1000 - (et - st)); // ms //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to); } st = taosGetTimestampUs(); - for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { int64_t t1 = taosGetTimestampUs(); char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.superQueryInfo.result[i][0] != 0) { + if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { sprintf(tmpFile, "%s-%d", - g_queryInfo.superQueryInfo.result[i], winfo->threadID); + g_queryInfo.specifiedQueryInfo.result[i], winfo->threadID); } - selectAndGetResult(winfo->taos, g_queryInfo.superQueryInfo.sql[i], tmpFile); + selectAndGetResult(winfo->taos, g_queryInfo.specifiedQueryInfo.sql[i], tmpFile); int64_t t2 = taosGetTimestampUs(); printf("=[taosc] thread[%"PRId64"] complete one sql, Spent %f s\n", taosGetSelfPthreadId(), (t2 - t1)/1000000.0); } else { int64_t t1 = taosGetTimestampUs(); int retCode = postProceSql(g_queryInfo.host, - g_queryInfo.port, g_queryInfo.superQueryInfo.sql[i]); + g_queryInfo.port, g_queryInfo.specifiedQueryInfo.sql[i]); int64_t t2 = taosGetTimestampUs(); printf("=[restful] thread[%"PRId64"] complete one sql, Spent %f s\n", taosGetSelfPthreadId(), (t2 - t1)/1000000.0); @@ -5542,7 +5564,7 @@ static void replaceSubTblName(char* inSql, char* outSql, int tblIndex) { char subTblName[MAX_TB_NAME_SIZE*3]; sprintf(subTblName, "%s.%s", g_queryInfo.dbName, - g_queryInfo.subQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN); + g_queryInfo.superQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN); //printf("inSql: %s\n", inSql); @@ -5580,25 +5602,26 @@ static void *subQueryProcess(void *sarg) { } int64_t st = 0; - int64_t et = (int64_t)g_queryInfo.subQueryInfo.rate*1000; - int queryTimes = g_args.query_times; + int64_t et = (int64_t)g_queryInfo.superQueryInfo.rate*1000; + + int queryTimes = g_queryInfo.superQueryInfo.queryTimes; while(queryTimes --) { - if (g_queryInfo.subQueryInfo.rate - && (et - st) < (int64_t)g_queryInfo.subQueryInfo.rate*1000) { - taosMsleep(g_queryInfo.subQueryInfo.rate*1000 - (et - st)); // ms + if (g_queryInfo.superQueryInfo.rate + && (et - st) < (int64_t)g_queryInfo.superQueryInfo.rate*1000) { + taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to); } st = taosGetTimestampUs(); for (int i = winfo->start_table_from; i <= winfo->end_table_to; i++) { - for (int j = 0; j < g_queryInfo.subQueryInfo.sqlCount; j++) { + for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { memset(sqlstr,0,sizeof(sqlstr)); - replaceSubTblName(g_queryInfo.subQueryInfo.sql[j], sqlstr, i); + replaceSubTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i); char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.subQueryInfo.result[j][0] != 0) { + if (g_queryInfo.superQueryInfo.result[j][0] != 0) { sprintf(tmpFile, "%s-%d", - g_queryInfo.subQueryInfo.result[j], + g_queryInfo.superQueryInfo.result[j], winfo->threadID); } selectAndGetResult(winfo->taos, sqlstr, tmpFile); @@ -5633,12 +5656,12 @@ static int queryTestProcess() { exit(-1); } - if (0 != g_queryInfo.subQueryInfo.sqlCount) { + if (0 != g_queryInfo.superQueryInfo.sqlCount) { getAllChildNameOfSuperTable(taos, g_queryInfo.dbName, - g_queryInfo.subQueryInfo.sTblName, - &g_queryInfo.subQueryInfo.childTblName, - &g_queryInfo.subQueryInfo.childTblCount); + g_queryInfo.superQueryInfo.sTblName, + &g_queryInfo.superQueryInfo.childTblName, + &g_queryInfo.superQueryInfo.childTblCount); } if (!g_args.answer_yes) { @@ -5651,22 +5674,22 @@ static int queryTestProcess() { pthread_t *pids = NULL; threadInfo *infos = NULL; //==== create sub threads for query from specify table - if (g_queryInfo.superQueryInfo.sqlCount > 0 - && g_queryInfo.superQueryInfo.concurrent > 0) { + if (g_queryInfo.specifiedQueryInfo.sqlCount > 0 + && g_queryInfo.specifiedQueryInfo.concurrent > 0) { - pids = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(pthread_t)); + pids = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t)); if (NULL == pids) { taos_close(taos); ERROR_EXIT("memory allocation failed\n"); } - infos = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(threadInfo)); + infos = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo)); if (NULL == infos) { taos_close(taos); free(pids); ERROR_EXIT("memory allocation failed for create threads\n"); } - for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) { threadInfo *t_info = infos + i; t_info->threadID = i; @@ -5690,7 +5713,7 @@ static int queryTestProcess() { pthread_create(pids + i, NULL, superQueryProcess, t_info); } } else { - g_queryInfo.superQueryInfo.concurrent = 0; + g_queryInfo.specifiedQueryInfo.concurrent = 0; } taos_close(taos); @@ -5698,9 +5721,9 @@ static int queryTestProcess() { pthread_t *pidsOfSub = NULL; threadInfo *infosOfSub = NULL; //==== create sub threads for query from all sub table of the super table - if ((g_queryInfo.subQueryInfo.sqlCount > 0) - && (g_queryInfo.subQueryInfo.threadCnt > 0)) { - pidsOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(pthread_t)); + if ((g_queryInfo.superQueryInfo.sqlCount > 0) + && (g_queryInfo.superQueryInfo.threadCnt > 0)) { + pidsOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t)); if (NULL == pidsOfSub) { free(infos); free(pids); @@ -5708,7 +5731,7 @@ static int queryTestProcess() { ERROR_EXIT("memory allocation failed for create threads\n"); } - infosOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(threadInfo)); + infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo)); if (NULL == infosOfSub) { free(pidsOfSub); free(infos); @@ -5716,8 +5739,8 @@ static int queryTestProcess() { ERROR_EXIT("memory allocation failed for create threads\n"); } - int ntables = g_queryInfo.subQueryInfo.childTblCount; - int threads = g_queryInfo.subQueryInfo.threadCnt; + int ntables = g_queryInfo.superQueryInfo.childTblCount; + int threads = g_queryInfo.superQueryInfo.threadCnt; int a = ntables / threads; if (a < 1) { @@ -5743,19 +5766,19 @@ static int queryTestProcess() { pthread_create(pidsOfSub + i, NULL, subQueryProcess, t_info); } - g_queryInfo.subQueryInfo.threadCnt = threads; + g_queryInfo.superQueryInfo.threadCnt = threads; } else { - g_queryInfo.subQueryInfo.threadCnt = 0; + g_queryInfo.superQueryInfo.threadCnt = 0; } - for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) { pthread_join(pids[i], NULL); } tmfree((char*)pids); tmfree((char*)infos); - for (int i = 0; i < g_queryInfo.subQueryInfo.threadCnt; i++) { + for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) { pthread_join(pidsOfSub[i], NULL); } @@ -5780,14 +5803,14 @@ static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int c static TAOS_SUB* subscribeImpl(TAOS *taos, char *sql, char* topic, char* resultFileName) { TAOS_SUB* tsub = NULL; - if (g_queryInfo.superQueryInfo.subscribeMode) { + if (g_queryInfo.specifiedQueryInfo.subscribeMode) { tsub = taos_subscribe(taos, - g_queryInfo.superQueryInfo.subscribeRestart, + g_queryInfo.specifiedQueryInfo.subscribeRestart, topic, sql, subscribe_callback, (void*)resultFileName, - g_queryInfo.superQueryInfo.subscribeInterval); + g_queryInfo.specifiedQueryInfo.subscribeInterval); } else { tsub = taos_subscribe(taos, - g_queryInfo.superQueryInfo.subscribeRestart, + g_queryInfo.specifiedQueryInfo.subscribeRestart, topic, sql, NULL, NULL, 0); } @@ -5830,25 +5853,25 @@ static void *subSubscribeProcess(void *sarg) { //int64_t st = 0; //int64_t et = 0; do { - //if (g_queryInfo.superQueryInfo.rate && (et - st) < g_queryInfo.superQueryInfo.rate*1000) { - // taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms + //if (g_queryInfo.specifiedQueryInfo.rate && (et - st) < g_queryInfo.specifiedQueryInfo.rate*1000) { + // taosMsleep(g_queryInfo.specifiedQueryInfo.rate*1000 - (et - st)); // ms // //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to); //} //st = taosGetTimestampMs(); char topic[32] = {0}; - for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) { + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { sprintf(topic, "taosdemo-subscribe-%d", i); memset(subSqlstr,0,sizeof(subSqlstr)); - replaceSubTblName(g_queryInfo.subQueryInfo.sql[i], subSqlstr, i); + replaceSubTblName(g_queryInfo.superQueryInfo.sql[i], subSqlstr, i); char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.subQueryInfo.result[i][0] != 0) { + if (g_queryInfo.superQueryInfo.result[i][0] != 0) { sprintf(tmpFile, "%s-%d", - g_queryInfo.subQueryInfo.result[i], winfo->threadID); + g_queryInfo.superQueryInfo.result[i], winfo->threadID); } - g_queryInfo.subQueryInfo.tsub[i] = subscribeImpl( + g_queryInfo.superQueryInfo.tsub[i] = subscribeImpl( winfo->taos, subSqlstr, topic, tmpFile); - if (NULL == g_queryInfo.subQueryInfo.tsub[i]) { + if (NULL == g_queryInfo.superQueryInfo.tsub[i]) { taos_close(winfo->taos); return NULL; } @@ -5860,17 +5883,17 @@ static void *subSubscribeProcess(void *sarg) { // start loop to consume result TAOS_RES* res = NULL; while (1) { - for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) { - if (1 == g_queryInfo.subQueryInfo.subscribeMode) { + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + if (1 == g_queryInfo.superQueryInfo.subscribeMode) { continue; } - res = taos_consume(g_queryInfo.subQueryInfo.tsub[i]); + res = taos_consume(g_queryInfo.superQueryInfo.tsub[i]); if (res) { char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.subQueryInfo.result[i][0] != 0) { + if (g_queryInfo.superQueryInfo.result[i][0] != 0) { sprintf(tmpFile, "%s-%d", - g_queryInfo.subQueryInfo.result[i], + g_queryInfo.superQueryInfo.result[i], winfo->threadID); } getResult(res, tmpFile); @@ -5879,9 +5902,9 @@ static void *subSubscribeProcess(void *sarg) { } taos_free_result(res); - for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) { - taos_unsubscribe(g_queryInfo.subQueryInfo.tsub[i], - g_queryInfo.subQueryInfo.subscribeKeepProgress); + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + taos_unsubscribe(g_queryInfo.superQueryInfo.tsub[i], + g_queryInfo.superQueryInfo.subscribeKeepProgress); } taos_close(winfo->taos); @@ -5918,25 +5941,25 @@ static void *superSubscribeProcess(void *sarg) { //int64_t st = 0; //int64_t et = 0; do { - //if (g_queryInfo.superQueryInfo.rate && (et - st) < g_queryInfo.superQueryInfo.rate*1000) { - // taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms + //if (g_queryInfo.specifiedQueryInfo.rate && (et - st) < g_queryInfo.specifiedQueryInfo.rate*1000) { + // taosMsleep(g_queryInfo.specifiedQueryInfo.rate*1000 - (et - st)); // ms // //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to); //} //st = taosGetTimestampMs(); char topic[32] = {0}; - for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { sprintf(topic, "taosdemo-subscribe-%d", i); char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.subQueryInfo.result[i][0] != 0) { + if (g_queryInfo.superQueryInfo.result[i][0] != 0) { sprintf(tmpFile, "%s-%d", - g_queryInfo.superQueryInfo.result[i], winfo->threadID); + g_queryInfo.specifiedQueryInfo.result[i], winfo->threadID); } - g_queryInfo.superQueryInfo.tsub[i] = + g_queryInfo.specifiedQueryInfo.tsub[i] = subscribeImpl(winfo->taos, - g_queryInfo.superQueryInfo.sql[i], + g_queryInfo.specifiedQueryInfo.sql[i], topic, tmpFile); - if (NULL == g_queryInfo.superQueryInfo.tsub[i]) { + if (NULL == g_queryInfo.specifiedQueryInfo.tsub[i]) { taos_close(winfo->taos); return NULL; } @@ -5948,17 +5971,17 @@ static void *superSubscribeProcess(void *sarg) { // start loop to consume result TAOS_RES* res = NULL; while (1) { - for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - if (1 == g_queryInfo.superQueryInfo.subscribeMode) { + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { + if (1 == g_queryInfo.specifiedQueryInfo.subscribeMode) { continue; } - res = taos_consume(g_queryInfo.superQueryInfo.tsub[i]); + res = taos_consume(g_queryInfo.specifiedQueryInfo.tsub[i]); if (res) { char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.superQueryInfo.result[i][0] != 0) { + if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { sprintf(tmpFile, "%s-%d", - g_queryInfo.superQueryInfo.result[i], winfo->threadID); + g_queryInfo.specifiedQueryInfo.result[i], winfo->threadID); } getResult(res, tmpFile); } @@ -5966,9 +5989,9 @@ static void *superSubscribeProcess(void *sarg) { } taos_free_result(res); - for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - taos_unsubscribe(g_queryInfo.superQueryInfo.tsub[i], - g_queryInfo.superQueryInfo.subscribeKeepProgress); + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { + taos_unsubscribe(g_queryInfo.specifiedQueryInfo.tsub[i], + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); } taos_close(winfo->taos); @@ -5997,12 +6020,12 @@ static int subscribeTestProcess() { exit(-1); } - if (0 != g_queryInfo.subQueryInfo.sqlCount) { + if (0 != g_queryInfo.superQueryInfo.sqlCount) { getAllChildNameOfSuperTable(taos, g_queryInfo.dbName, - g_queryInfo.subQueryInfo.sTblName, - &g_queryInfo.subQueryInfo.childTblName, - &g_queryInfo.subQueryInfo.childTblCount); + g_queryInfo.superQueryInfo.sTblName, + &g_queryInfo.superQueryInfo.childTblName, + &g_queryInfo.superQueryInfo.childTblCount); } taos_close(taos); // TODO: workaround to use separate taos connection; @@ -6010,22 +6033,22 @@ static int subscribeTestProcess() { pthread_t *pids = NULL; threadInfo *infos = NULL; //==== create sub threads for query from super table - if ((g_queryInfo.superQueryInfo.sqlCount <= 0) || - (g_queryInfo.superQueryInfo.concurrent <= 0)) { + if ((g_queryInfo.specifiedQueryInfo.sqlCount <= 0) || + (g_queryInfo.specifiedQueryInfo.concurrent <= 0)) { errorPrint("%s() LN%d, query sqlCount %d or concurrent %d is not correct.\n", - __func__, __LINE__, g_queryInfo.superQueryInfo.sqlCount, - g_queryInfo.superQueryInfo.concurrent); + __func__, __LINE__, g_queryInfo.specifiedQueryInfo.sqlCount, + g_queryInfo.specifiedQueryInfo.concurrent); exit(-1); } - pids = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(pthread_t)); - infos = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(threadInfo)); + pids = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t)); + infos = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo)); if ((NULL == pids) || (NULL == infos)) { errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__); exit(-1); } - for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) { threadInfo *t_info = infos + i; t_info->threadID = i; t_info->taos = NULL; // TODO: workaround to use separate taos connection; @@ -6035,11 +6058,11 @@ static int subscribeTestProcess() { //==== create sub threads for query from sub table pthread_t *pidsOfSub = NULL; threadInfo *infosOfSub = NULL; - if ((g_queryInfo.subQueryInfo.sqlCount > 0) - && (g_queryInfo.subQueryInfo.threadCnt > 0)) { - pidsOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * + if ((g_queryInfo.superQueryInfo.sqlCount > 0) + && (g_queryInfo.superQueryInfo.threadCnt > 0)) { + pidsOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t)); - infosOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * + infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo)); if ((NULL == pidsOfSub) || (NULL == infosOfSub)) { errorPrint("%s() LN%d, malloc failed for create threads\n", @@ -6048,8 +6071,8 @@ static int subscribeTestProcess() { exit(-1); } - int ntables = g_queryInfo.subQueryInfo.childTblCount; - int threads = g_queryInfo.subQueryInfo.threadCnt; + int ntables = g_queryInfo.superQueryInfo.childTblCount; + int threads = g_queryInfo.superQueryInfo.threadCnt; int a = ntables / threads; if (a < 1) { @@ -6075,14 +6098,14 @@ static int subscribeTestProcess() { pthread_create(pidsOfSub + i, NULL, subSubscribeProcess, t_info); } - g_queryInfo.subQueryInfo.threadCnt = threads; + g_queryInfo.superQueryInfo.threadCnt = threads; - for (int i = 0; i < g_queryInfo.subQueryInfo.threadCnt; i++) { + for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) { pthread_join(pidsOfSub[i], NULL); } } - for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) { pthread_join(pids[i], NULL); } From d2ec29715aac63a884e22907483d78ce6c0e2bf1 Mon Sep 17 00:00:00 2001 From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Mon, 29 Mar 2021 14:42:49 +0800 Subject: [PATCH 34/71] [TD-3410]: support nutz framework (#5600) * [TD-3410]feature: adapted to nutz * change * change * change * change * [TD-3447]: JDBC-RESTful and JDBC-JNI support setObject * change * change * change * change * change * change * change * change * change * change * change * change * [TD-3548]: support ParameterMetaData in JDBC-RESTful and JDBC-JNI * change * change * [TD-3446]: JDBC-JNI and JDBC-RESTful do not need invoke Class.forName any more * change jdbc version * change * change * change --- cmake/install.inc | 2 +- src/connector/jdbc/CMakeLists.txt | 2 +- src/connector/jdbc/deploy-pom.xml | 24 +- src/connector/jdbc/pom.xml | 9 +- .../com/taosdata/jdbc/AbstractConnection.java | 4 +- .../jdbc/AbstractParameterMetaData.java | 138 ++++ .../com/taosdata/jdbc/AbstractResultSet.java | 80 +-- .../com/taosdata/jdbc/AbstractStatement.java | 2 + .../taosdata/jdbc/SavedPreparedStatement.java | 452 ------------ .../com/taosdata/jdbc/TSDBConnection.java | 21 +- .../taosdata/jdbc/TSDBParameterMetaData.java | 8 + .../taosdata/jdbc/TSDBPreparedStatement.java | 230 +++--- .../java/com/taosdata/jdbc/TSDBResultSet.java | 199 +++-- .../taosdata/jdbc/TSDBResultSetMetaData.java | 248 ++++--- .../taosdata/jdbc/TSDBResultSetRowData.java | 24 +- .../taosdata/jdbc/bean/TSDBPreparedParam.java | 62 -- .../taosdata/jdbc/rs/RestfulConnection.java | 11 +- .../com/taosdata/jdbc/rs/RestfulDriver.java | 2 +- .../jdbc/rs/RestfulParameterMetaData.java | 10 + .../jdbc/rs/RestfulPreparedStatement.java | 27 +- .../taosdata/jdbc/rs/RestfulResultSet.java | 197 +++-- .../META-INF/services/java.sql.Driver | 1 + .../java/com/taosdata/jdbc/ResultSetTest.java | 266 ------- .../com/taosdata/jdbc/TSDBConnectionTest.java | 2 +- .../jdbc/TSDBParameterMetaDataTest.java | 194 +++++ .../jdbc/TSDBPreparedStatementTest.java | 203 ++++-- .../com/taosdata/jdbc/TSDBResultSetTest.java | 678 ++++++++++++++++++ .../jdbc/cases/DriverAutoloadTest.java | 42 ++ .../jdbc/cases/InsertDbwithoutUseDbTest.java | 1 - .../jdbc/cases/UnsignedNumberRestfulTest.java | 2 +- .../taosdata/jdbc/rs/AuthenticationTest.java | 1 - .../jdbc/rs/RestfulConnectionTest.java | 1 - .../jdbc/rs/RestfulDatabaseMetaDataTest.java | 1 - .../com/taosdata/jdbc/rs/RestfulJDBCTest.java | 1 - .../jdbc/rs/RestfulParameterMetaDataTest.java | 194 +++++ .../jdbc/rs/RestfulPreparedStatementTest.java | 183 ++++- .../jdbc/rs/RestfulResultSetMetaDataTest.java | 1 - .../jdbc/rs/RestfulResultSetTest.java | 121 +++- .../jdbc/rs/RestfulStatementTest.java | 1 - .../java/com/taosdata/jdbc/rs/SQLTest.java | 1 - tests/examples/JDBC/JDBCDemo/pom.xml | 2 +- 41 files changed, 2281 insertions(+), 1367 deletions(-) create mode 100644 src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java delete mode 100644 src/connector/jdbc/src/main/java/com/taosdata/jdbc/SavedPreparedStatement.java create mode 100644 src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBParameterMetaData.java delete mode 100644 src/connector/jdbc/src/main/java/com/taosdata/jdbc/bean/TSDBPreparedParam.java create mode 100644 src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulParameterMetaData.java delete mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBParameterMetaDataTest.java create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulParameterMetaDataTest.java diff --git a/cmake/install.inc b/cmake/install.inc index 1d50ca292d..01d3c8a4df 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS) #INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS shell RUNTIME DESTINATION .) IF (TD_MVN_INSTALLED) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.22-dist.jar DESTINATION connector/jdbc) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.24-dist.jar DESTINATION connector/jdbc) ENDIF () ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") diff --git a/src/connector/jdbc/CMakeLists.txt b/src/connector/jdbc/CMakeLists.txt index 86ddfcb022..540dc8eb58 100644 --- a/src/connector/jdbc/CMakeLists.txt +++ b/src/connector/jdbc/CMakeLists.txt @@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED) ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME} POST_BUILD COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.22-dist.jar ${LIBRARY_OUTPUT_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.24-dist.jar ${LIBRARY_OUTPUT_PATH} COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMENT "build jdbc driver") ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME}) diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index fe93e54c69..935b3f7e4a 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -5,7 +5,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.22 + 2.0.24 jar JDBCDriver @@ -37,17 +37,6 @@ - - commons-logging - commons-logging - 1.2 - - - * - * - - - junit junit @@ -61,21 +50,20 @@ httpclient 4.5.8 - - org.apache.commons - commons-lang3 - 3.9 - com.alibaba fastjson 1.2.58 + + com.google.guava + guava + 29.0-jre + - org.apache.maven.plugins maven-assembly-plugin diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index 51cb0b3808..5d7c89e2d2 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.taosdata.jdbc taos-jdbcdriver - 2.0.22 + 2.0.24 jar JDBCDriver https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc @@ -43,7 +43,6 @@ 4.13 test - org.apache.httpcomponents @@ -55,7 +54,11 @@ fastjson 1.2.58 - + + com.google.guava + guava + 29.0-jre + diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java index a2496b0099..976078da95 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java @@ -30,9 +30,12 @@ public abstract class AbstractConnection extends WrapperImpl implements Connecti if (isClosed()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); // do nothing + return sql; } + + @Override public void setAutoCommit(boolean autoCommit) throws SQLException { if (isClosed()) @@ -448,7 +451,6 @@ public abstract class AbstractConnection extends WrapperImpl implements Connecti if (isClosed) throw (SQLClientInfoException) TSDBError.createSQLException(TSDBErrorNumbers.ERROR_SQLCLIENT_EXCEPTION_ON_CONNECTION_CLOSED); - for (Enumeration enumer = properties.keys(); enumer.hasMoreElements(); ) { String name = (String) enumer.nextElement(); clientInfoProps.put(name, properties.getProperty(name)); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java new file mode 100644 index 0000000000..999df06fc7 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java @@ -0,0 +1,138 @@ +package com.taosdata.jdbc; + +import java.sql.ParameterMetaData; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; + +public abstract class AbstractParameterMetaData extends WrapperImpl implements ParameterMetaData { + + private final Object[] parameters; + + public AbstractParameterMetaData(Object[] parameters) { + this.parameters = parameters; + } + + @Override + public int getParameterCount() throws SQLException { + return parameters.length; + } + + @Override + public int isNullable(int param) throws SQLException { + return ParameterMetaData.parameterNullableUnknown; + } + + @Override + public boolean isSigned(int param) throws SQLException { + if (param < 1 && param >= parameters.length) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + + if (parameters[param - 1] instanceof Byte) + return true; + if (parameters[param - 1] instanceof Short) + return true; + if (parameters[param - 1] instanceof Integer) + return true; + if (parameters[param - 1] instanceof Long) + return true; + if (parameters[param - 1] instanceof Float) + return true; + if (parameters[param - 1] instanceof Double) + return true; + + return false; + } + + @Override + public int getPrecision(int param) throws SQLException { + if (param < 1 && param >= parameters.length) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + + if (parameters[param - 1] instanceof String) + return ((String) parameters[param - 1]).length(); + if (parameters[param - 1] instanceof byte[]) + return ((byte[]) parameters[param - 1]).length; + return 0; + } + + @Override + public int getScale(int param) throws SQLException { + if (param < 1 && param >= parameters.length) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + return 0; + } + + @Override + public int getParameterType(int param) throws SQLException { + if (param < 1 && param >= parameters.length) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + + if (parameters[param - 1] instanceof Timestamp) + return Types.TIMESTAMP; + if (parameters[param - 1] instanceof Byte) + return Types.TINYINT; + if (parameters[param - 1] instanceof Short) + return Types.SMALLINT; + if (parameters[param - 1] instanceof Integer) + return Types.INTEGER; + if (parameters[param - 1] instanceof Long) + return Types.BIGINT; + if (parameters[param - 1] instanceof Float) + return Types.FLOAT; + if (parameters[param - 1] instanceof Double) + return Types.DOUBLE; + if (parameters[param - 1] instanceof String) + return Types.NCHAR; + if (parameters[param - 1] instanceof byte[]) + return Types.BINARY; + if (parameters[param - 1] instanceof Boolean) + return Types.BOOLEAN; + return Types.OTHER; + } + + @Override + public String getParameterTypeName(int param) throws SQLException { + if (param < 1 && param >= parameters.length) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + + if (parameters[param - 1] instanceof Timestamp) + return TSDBConstants.jdbcType2TaosTypeName(Types.TIMESTAMP); + if (parameters[param - 1] instanceof Byte) + return TSDBConstants.jdbcType2TaosTypeName(Types.TINYINT); + if (parameters[param - 1] instanceof Short) + return TSDBConstants.jdbcType2TaosTypeName(Types.SMALLINT); + if (parameters[param - 1] instanceof Integer) + return TSDBConstants.jdbcType2TaosTypeName(Types.INTEGER); + if (parameters[param - 1] instanceof Long) + return TSDBConstants.jdbcType2TaosTypeName(Types.BIGINT); + if (parameters[param - 1] instanceof Float) + return TSDBConstants.jdbcType2TaosTypeName(Types.FLOAT); + if (parameters[param - 1] instanceof Double) + return TSDBConstants.jdbcType2TaosTypeName(Types.DOUBLE); + if (parameters[param - 1] instanceof String) + return TSDBConstants.jdbcType2TaosTypeName(Types.NCHAR); + if (parameters[param - 1] instanceof byte[]) + return TSDBConstants.jdbcType2TaosTypeName(Types.BINARY); + if (parameters[param - 1] instanceof Boolean) + return TSDBConstants.jdbcType2TaosTypeName(Types.BOOLEAN); + + return parameters[param - 1].getClass().getName(); + } + + @Override + public String getParameterClassName(int param) throws SQLException { + if (param < 1 && param >= parameters.length) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + + return parameters[param - 1].getClass().getName(); + } + + @Override + public int getParameterMode(int param) throws SQLException { + if (param < 1 && param >= parameters.length) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + + return ParameterMetaData.parameterModeUnknown; + } +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java index 238d18039d..abd348e68c 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java @@ -11,6 +11,15 @@ import java.util.Map; public abstract class AbstractResultSet extends WrapperImpl implements ResultSet { private int fetchSize; + protected void checkAvailability(int columnIndex, int bounds) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + if (columnIndex < 1) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE, "Column Index out of range, " + columnIndex + " < 1"); + if (columnIndex > bounds) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE, "Column Index out of range, " + columnIndex + " > " + bounds); + } + @Override public abstract boolean next() throws SQLException; @@ -46,38 +55,20 @@ public abstract class AbstractResultSet extends WrapperImpl implements ResultSet @Override public abstract double getDouble(int columnIndex) throws SQLException; + @Deprecated @Override public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + return getBigDecimal(columnIndex); } @Override - public byte[] getBytes(int columnIndex) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); - } + public abstract byte[] getBytes(int columnIndex) throws SQLException; @Override - public Date getDate(int columnIndex) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); - - } + public abstract Date getDate(int columnIndex) throws SQLException; @Override - public Time getTime(int columnIndex) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); - } + public abstract Time getTime(int columnIndex) throws SQLException; @Override public abstract Timestamp getTimestamp(int columnIndex) throws SQLException; @@ -147,9 +138,10 @@ public abstract class AbstractResultSet extends WrapperImpl implements ResultSet return getDouble(findColumn(columnLabel)); } + @Deprecated @Override public BigDecimal getBigDecimal(String columnLabel, int scale) throws SQLException { - return getBigDecimal(findColumn(columnLabel)); + return getBigDecimal(findColumn(columnLabel), scale); } @Override @@ -214,12 +206,7 @@ public abstract class AbstractResultSet extends WrapperImpl implements ResultSet public abstract ResultSetMetaData getMetaData() throws SQLException; @Override - public Object getObject(int columnIndex) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); - } + public abstract Object getObject(int columnIndex) throws SQLException; @Override public Object getObject(String columnLabel) throws SQLException { @@ -243,12 +230,7 @@ public abstract class AbstractResultSet extends WrapperImpl implements ResultSet } @Override - public BigDecimal getBigDecimal(int columnIndex) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); - } + public abstract BigDecimal getBigDecimal(int columnIndex) throws SQLException; @Override public BigDecimal getBigDecimal(String columnLabel) throws SQLException { @@ -718,9 +700,7 @@ public abstract class AbstractResultSet extends WrapperImpl implements ResultSet @Override public Object getObject(String columnLabel, Map> map) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + return getObject(findColumn(columnLabel), map); } @Override @@ -760,9 +740,7 @@ public abstract class AbstractResultSet extends WrapperImpl implements ResultSet @Override public Date getDate(String columnLabel, Calendar cal) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + return getDate(findColumn(columnLabel), cal); } @Override @@ -774,23 +752,15 @@ public abstract class AbstractResultSet extends WrapperImpl implements ResultSet @Override public Time getTime(String columnLabel, Calendar cal) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + return getTime(findColumn(columnLabel), cal); } @Override - public Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); - } + public abstract Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException; @Override public Timestamp getTimestamp(String columnLabel, Calendar cal) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + return getTimestamp(findColumn(columnLabel), cal); } @Override @@ -1198,9 +1168,7 @@ public abstract class AbstractResultSet extends WrapperImpl implements ResultSet @Override public T getObject(String columnLabel, Class type) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + return getObject(findColumn(columnLabel), type); } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java index 8b6c074d1b..9dc559339a 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java @@ -9,6 +9,8 @@ public abstract class AbstractStatement extends WrapperImpl implements Statement protected List batchedArgs; private int fetchSize; + + @Override public abstract ResultSet executeQuery(String sql) throws SQLException; diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/SavedPreparedStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/SavedPreparedStatement.java deleted file mode 100644 index 512fcd26b8..0000000000 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/SavedPreparedStatement.java +++ /dev/null @@ -1,452 +0,0 @@ -package com.taosdata.jdbc; - -import com.taosdata.jdbc.bean.TSDBPreparedParam; - -import java.sql.SQLException; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * this class is used to precompile the sql of tdengine insert or import ops - */ -public class SavedPreparedStatement { - - private TSDBPreparedStatement tsdbPreparedStatement; - - /** - * sql param List - */ - private List sqlParamList; - - /** - * init param according the sql - */ - private TSDBPreparedParam initPreparedParam; - - /** - * is table name dynamic in the prepared sql - */ - private boolean isTableNameDynamic; - - /** - * insert or import sql template pattern, the template are the following: - *

- * insert/import into tableName [(field1, field2, ...)] [using stables tags(?, ?, ...) ] values(?, ?, ...) (?, ?, ...) - *

- * we split it to three part: - * 1. prefix, insert/import - * 2. middle, tableName [(field1, field2, ...)] [using stables tags(?, ?, ...) ] - * 3. valueList, the content after values, for example (?, ?, ...) (?, ?, ...) - */ - private Pattern sqlPattern = Pattern.compile("(?s)(?i)^\\s*(INSERT|IMPORT)\\s+INTO\\s+((?\\S+)\\s*(\\(.*\\))?\\s+(USING\\s+(?\\S+)\\s+TAGS\\s*\\((?.+)\\))?)\\s*VALUES\\s*(?\\(.*\\)).*"); - - /** - * the raw sql template - */ - private String sql; - - /** - * the prefix part of sql - */ - private String prefix; - - /** - * the middle part of sql - */ - private String middle; - - private int middleParamSize; - - /** - * the valueList part of sql - */ - private String valueList; - - private int valueListSize; - - /** - * default param value - */ - private static final String DEFAULT_VALUE = "NULL"; - - private static final String PLACEHOLDER = "?"; - - private String tableName; - - /** - * is the parameter add to batch list - */ - private boolean isAddBatch; - - public SavedPreparedStatement(String sql, TSDBPreparedStatement tsdbPreparedStatement) throws SQLException { - this.sql = sql; - this.tsdbPreparedStatement = tsdbPreparedStatement; - this.sqlParamList = new ArrayList<>(); - - parsePreparedParam(this.sql); - } - - /** - * parse the init param according the sql param - * - * @param sql - */ - private void parsePreparedParam(String sql) throws SQLException { - - Matcher matcher = sqlPattern.matcher(sql); - - if (matcher.find()) { - - tableName = matcher.group("tablename"); - - if (tableName != null && PLACEHOLDER.equals(tableName)) { - // the table name is dynamic - this.isTableNameDynamic = true; - } - - prefix = matcher.group(1); - middle = matcher.group(2); - valueList = matcher.group("valueList"); - - if (middle != null && !"".equals(middle)) { - middleParamSize = parsePlaceholder(middle); - } - - if (valueList != null && !"".equals(valueList)) { - valueListSize = parsePlaceholder(valueList); - } - - initPreparedParam = initDefaultParam(tableName, middleParamSize, valueListSize); - - } else { - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_SQL); - } - - } - - private TSDBPreparedParam initDefaultParam(String tableName, int middleParamSize, int valueListSize) { - - TSDBPreparedParam tsdbPreparedParam = new TSDBPreparedParam(tableName); - - tsdbPreparedParam.setMiddleParamList(getDefaultParamList(middleParamSize)); - - tsdbPreparedParam.setValueList(getDefaultParamList(valueListSize)); - - return tsdbPreparedParam; - } - - /** - * generate the default param value list - * - * @param paramSize - * @return - */ - private List getDefaultParamList(int paramSize) { - - List paramList = new ArrayList<>(paramSize); - if (paramSize > 0) { - for (int i = 0; i < paramSize; i++) { - paramList.add(i, DEFAULT_VALUE); - } - } - - return paramList; - } - - /** - * calculate the placeholder num - * - * @param value - * @return - */ - private int parsePlaceholder(String value) { - - Pattern pattern = Pattern.compile("[?]"); - - Matcher matcher = pattern.matcher(value); - - int result = 0; - while (matcher.find()) { - result++; - } - return result; - } - - /** - * set current row params - * - * @param parameterIndex the first parameter is 1, the second is 2, ... - * @param x the parameter value - */ - public void setParam(int parameterIndex, Object x) throws SQLException { - - int paramSize = this.middleParamSize + this.valueListSize; - - String errorMsg = String.format("the parameterIndex %s out of the range [1, %s]", parameterIndex, paramSize); - - if (parameterIndex < 1 || parameterIndex > paramSize) { - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE,errorMsg); - } - - this.isAddBatch = false; //set isAddBatch to false - - if (x == null) { - x = DEFAULT_VALUE; // set default null string - } - - parameterIndex = parameterIndex - 1; // start from 0 in param list - - if (this.middleParamSize > 0 && parameterIndex >= 0 && parameterIndex < this.middleParamSize) { - - this.initPreparedParam.setMiddleParam(parameterIndex, x); - return; - } - - if (this.valueListSize > 0 && parameterIndex >= this.middleParamSize && parameterIndex < paramSize) { - - this.initPreparedParam.setValueParam(parameterIndex - this.middleParamSize, x); - return; - } - - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE,errorMsg); - } - - public void addBatch() { - - addCurrentRowParamToList(); - this.initPreparedParam = initDefaultParam(tableName, middleParamSize, valueListSize); - } - - /** - * add current param to batch list - */ - private void addCurrentRowParamToList() { - - if (initPreparedParam != null && (this.middleParamSize > 0 || this.valueListSize > 0)) { - this.sqlParamList.add(initPreparedParam); // add current param to batch list - } - this.isAddBatch = true; - } - - - /** - * execute the sql with batch sql - * - * @return - * @throws SQLException - */ - public int[] executeBatch() throws SQLException { - - int result = executeBatchInternal(); - - return new int[]{result}; - } - - - public int executeBatchInternal() throws SQLException { - - if (!isAddBatch) { - addCurrentRowParamToList(); // add current param to batch list - } - - //1. generate batch sql - String sql = generateExecuteSql(); - //2. execute batch sql - int result = executeSql(sql); - - //3. clear batch param list - this.sqlParamList.clear(); - - return result; - } - - /** - * generate the batch sql - * - * @return - */ - private String generateExecuteSql() { - - StringBuilder stringBuilder = new StringBuilder(); - - stringBuilder.append(prefix); - stringBuilder.append(" into "); - - if (!isTableNameDynamic) { - // tablename will not need to be replaced - String middleValue = replaceMiddleListParam(middle, sqlParamList); - stringBuilder.append(middleValue); - stringBuilder.append(" values"); - - stringBuilder.append(replaceValueListParam(valueList, sqlParamList)); - - } else { - // need to replace tablename - - if (sqlParamList.size() > 0) { - - TSDBPreparedParam firstPreparedParam = sqlParamList.get(0); - - //replace middle part and value part of first row - String firstRow = replaceMiddleAndValuePart(firstPreparedParam); - stringBuilder.append(firstRow); - - //the first param in the middleParamList is the tableName - String lastTableName = firstPreparedParam.getMiddleParamList().get(0).toString(); - - if (sqlParamList.size() > 1) { - - for (int i = 1; i < sqlParamList.size(); i++) { - TSDBPreparedParam currentParam = sqlParamList.get(i); - String currentTableName = currentParam.getMiddleParamList().get(0).toString(); - if (lastTableName.equalsIgnoreCase(currentTableName)) { - // tablename is same with the last row ,so only need to append the part of value - - String values = replaceTemplateParam(valueList, currentParam.getValueList()); - stringBuilder.append(values); - - } else { - // tablename difference with the last row - //need to replace middle part and value part - String row = replaceMiddleAndValuePart(currentParam); - stringBuilder.append(row); - lastTableName = currentTableName; - } - } - } - - } else { - - stringBuilder.append(middle); - stringBuilder.append(" values"); - stringBuilder.append(valueList); - } - - } - - return stringBuilder.toString(); - } - - /** - * replace the middle and value part - * - * @param tsdbPreparedParam - * @return - */ - private String replaceMiddleAndValuePart(TSDBPreparedParam tsdbPreparedParam) { - - StringBuilder stringBuilder = new StringBuilder(" "); - - String middlePart = replaceTemplateParam(middle, tsdbPreparedParam.getMiddleParamList()); - - stringBuilder.append(middlePart); - stringBuilder.append(" values "); - - String valuePart = replaceTemplateParam(valueList, tsdbPreparedParam.getValueList()); - stringBuilder.append(valuePart); - stringBuilder.append(" "); - - return stringBuilder.toString(); - } - - /** - * replace the placeholder of the middle part of sql template with TSDBPreparedParam list - * - * @param template - * @param sqlParamList - * @return - */ - private String replaceMiddleListParam(String template, List sqlParamList) { - - if (sqlParamList.size() > 0) { - - //becase once the subTableName is static then will be ignore the tag which after the first setTag - return replaceTemplateParam(template, sqlParamList.get(0).getMiddleParamList()); - - } - - return template; - } - - - /** - * replace the placeholder of the template with TSDBPreparedParam list - * - * @param template - * @param sqlParamList - * @return - */ - private String replaceValueListParam(String template, List sqlParamList) { - - StringBuilder stringBuilder = new StringBuilder(); - - if (sqlParamList.size() > 0) { - - for (TSDBPreparedParam tsdbPreparedParam : sqlParamList) { - - String tmp = replaceTemplateParam(template, tsdbPreparedParam.getValueList()); - - stringBuilder.append(tmp); - } - - } else { - stringBuilder.append(template); - } - - return stringBuilder.toString(); - } - - /** - * replace the placeholder of the template with paramList - * - * @param template - * @param paramList - * @return - */ - private String replaceTemplateParam(String template, List paramList) { - - if (paramList.size() > 0) { - - String tmp = template; - - for (int i = 0; i < paramList.size(); ++i) { - - String paraStr = getParamString(paramList.get(i)); - - tmp = tmp.replaceFirst("[" + PLACEHOLDER + "]", paraStr); - - } - - return tmp; - - } else { - return template; - } - - } - - /** - * get the string of param object - * - * @param paramObj - * @return - */ - private String getParamString(Object paramObj) { - - String paraStr = paramObj.toString(); - if (paramObj instanceof Timestamp || (paramObj instanceof String && !DEFAULT_VALUE.equalsIgnoreCase(paraStr))) { - paraStr = "'" + paraStr + "'"; - } - return paraStr; - } - - - private int executeSql(String sql) throws SQLException { - - return tsdbPreparedStatement.executeUpdate(sql); - } - -} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java index a8653eb172..5b4615485d 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java @@ -27,10 +27,6 @@ public class TSDBConnection extends AbstractConnection { return this.batchFetch; } - public void setBatchFetch(Boolean batchFetch) { - this.batchFetch = batchFetch; - } - public TSDBConnection(Properties info, TSDBDatabaseMetaData meta) throws SQLException { this.databaseMetaData = meta; connect(info.getProperty(TSDBDriver.PROPERTY_KEY_HOST), @@ -61,9 +57,7 @@ public class TSDBConnection extends AbstractConnection { throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); } - TSDBStatement statement = new TSDBStatement(this, this.connector); - statement.setConnection(this); - return statement; + return new TSDBStatement(this, this.connector); } public TSDBSubscribe subscribe(String topic, String sql, boolean restart) throws SQLException { @@ -79,10 +73,8 @@ public class TSDBConnection extends AbstractConnection { } public PreparedStatement prepareStatement(String sql) throws SQLException { - if (isClosed()) { + if (isClosed()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); - } - return new TSDBPreparedStatement(this, this.connector, sql); } @@ -104,11 +96,4 @@ public class TSDBConnection extends AbstractConnection { return this.databaseMetaData; } - public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { - if (isClosed()) { - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); - } - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); - } - -} +} \ No newline at end of file diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBParameterMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBParameterMetaData.java new file mode 100644 index 0000000000..9ee23e3a26 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBParameterMetaData.java @@ -0,0 +1,8 @@ +package com.taosdata.jdbc; + +public class TSDBParameterMetaData extends AbstractParameterMetaData { + + public TSDBParameterMetaData(Object[] parameters) { + super(parameters); + } +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java index d294c89745..728b537f71 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java @@ -18,6 +18,7 @@ import java.io.InputStream; import java.io.Reader; import java.math.BigDecimal; import java.net.URL; +import java.nio.charset.Charset; import java.sql.*; import java.util.ArrayList; import java.util.Calendar; @@ -30,36 +31,49 @@ import java.util.regex.Pattern; * compatibility needs. */ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStatement { - protected String rawSql; - protected String sql; - protected ArrayList parameters = new ArrayList<>(); + + private String rawSql; + private String sql; + // private ArrayList parameters = new ArrayList<>(); + private Object[] parameters; + private boolean isPrepared; //start with insert or import and is case-insensitive private static Pattern savePattern = Pattern.compile("(?i)^\\s*(insert|import)"); - // is insert or import private boolean isSaved; - private SavedPreparedStatement savedPreparedStatement; - private ParameterMetaData parameterMetaData; + // private SavedPreparedStatement savedPreparedStatement; + private volatile TSDBParameterMetaData parameterMetaData; TSDBPreparedStatement(TSDBConnection connection, TSDBJNIConnector connecter, String sql) { super(connection, connecter); init(sql); + + if (sql.contains("?")) { + int parameterCnt = 0; + for (int i = 0; i < sql.length(); i++) { + if ('?' == sql.charAt(i)) { + parameterCnt++; + } + } + parameters = new Object[parameterCnt]; + this.isPrepared = true; + } } private void init(String sql) { this.rawSql = sql; preprocessSql(); +// this.isSaved = isSavedSql(this.rawSql); +// if (this.isSaved) { +// try { +// this.savedPreparedStatement = new SavedPreparedStatement(this.rawSql, this); +// } catch (SQLException e) { +// e.printStackTrace(); +// } +// } - this.isSaved = isSavedSql(this.rawSql); - if (this.isSaved) { - try { - this.savedPreparedStatement = new SavedPreparedStatement(this.rawSql, this); - } catch (SQLException e) { - e.printStackTrace(); - } - } } /** @@ -75,19 +89,11 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat @Override public int[] executeBatch() throws SQLException { - if (isSaved) { - return this.savedPreparedStatement.executeBatch(); - } else { - return super.executeBatch(); - } - } - - public ArrayList getParameters() { - return parameters; - } - - public void setParameters(ArrayList parameters) { - this.parameters = parameters; +// if (isSaved) { +// return this.savedPreparedStatement.executeBatch(); +// } else { + return super.executeBatch(); +// } } /* @@ -151,41 +157,73 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat * * @return a string of the native sql statement for TSDB */ - private String getNativeSql() { - this.sql = this.rawSql; - for (int i = 0; i < parameters.size(); ++i) { - Object para = parameters.get(i); +// private String getNativeSql(String rawSql) { +// for (int i = 0; i < parameters.length; i++) { +// Object para = parameters[i]; +// if (para != null) { +// String paraStr = para.toString(); +// if (para instanceof Timestamp || para instanceof String) { +// paraStr = "'" + paraStr + "'"; +// } +// this.sql = this.sql.replaceFirst("[?]", paraStr); +// } else { +// this.sql = this.sql.replaceFirst("[?]", "NULL"); +// } +// } +// parameters = new Object[parameters.length]; +// return sql; +// } + + private String getNativeSql(String rawSql) throws SQLException { + String sql = rawSql; + for (int i = 0; i < parameters.length; ++i) { + Object para = parameters[i]; if (para != null) { - String paraStr = para.toString(); - if (para instanceof Timestamp || para instanceof String) { + String paraStr; + if (para instanceof byte[]) { + paraStr = new String((byte[]) para, Charset.forName("UTF-8")); + } else { + paraStr = para.toString(); + } + // if para is timestamp or String or byte[] need to translate ' character + if (para instanceof Timestamp || para instanceof String || para instanceof byte[]) { + paraStr = paraStr.replaceAll("'", "\\\\\\\\'"); paraStr = "'" + paraStr + "'"; } - this.sql = this.sql.replaceFirst("[?]", paraStr); + sql = sql.replaceFirst("[?]", paraStr); } else { - this.sql = this.sql.replaceFirst("[?]", "NULL"); + sql = sql.replaceFirst("[?]", "NULL"); } } - parameters.clear(); + clearParameters(); return sql; } @Override public ResultSet executeQuery() throws SQLException { - if (isSaved) { - this.savedPreparedStatement.executeBatchInternal(); - return null; - } else { - return super.executeQuery(getNativeSql()); - } +// if (isSaved) { +// this.savedPreparedStatement.executeBatchInternal(); +// return null; +// } else { + + if (!isPrepared) + return executeQuery(this.rawSql); + + final String sql = getNativeSql(this.rawSql); + return executeQuery(sql); +// } } @Override public int executeUpdate() throws SQLException { - if (isSaved) { - return this.savedPreparedStatement.executeBatchInternal(); - } else { - return super.executeUpdate(getNativeSql()); - } +// if (isSaved) { +// return this.savedPreparedStatement.executeBatchInternal(); +// } else { + if (!isPrepared) + return executeUpdate(this.rawSql); + String sql = getNativeSql(this.rawSql); + return executeUpdate(sql); +// } } private boolean isSupportedSQLType(int sqlType) { @@ -201,35 +239,6 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat case Types.BINARY: case Types.NCHAR: return true; - case Types.ARRAY: - case Types.BIT: - case Types.BLOB: - case Types.CHAR: - case Types.CLOB: - case Types.DATALINK: - case Types.DATE: - case Types.DECIMAL: - case Types.DISTINCT: - case Types.JAVA_OBJECT: - case Types.LONGNVARCHAR: - case Types.LONGVARBINARY: - case Types.LONGVARCHAR: - case Types.NCLOB: - case Types.NULL: - case Types.NUMERIC: - case Types.NVARCHAR: - case Types.OTHER: - case Types.REAL: - case Types.REF: - case Types.REF_CURSOR: - case Types.ROWID: - case Types.SQLXML: - case Types.STRUCT: - case Types.TIME: - case Types.TIME_WITH_TIMEZONE: - case Types.TIMESTAMP_WITH_TIMEZONE: - case Types.VARBINARY: - case Types.VARCHAR: default: return false; } @@ -259,7 +268,7 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat public void setByte(int parameterIndex, byte x) throws SQLException { if (isClosed()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + setObject(parameterIndex,x); } @Override @@ -315,7 +324,8 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat public void setBytes(int parameterIndex, byte[] x) throws SQLException { if (isClosed()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + + setObject(parameterIndex,x); } @Override @@ -365,45 +375,63 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat public void clearParameters() throws SQLException { if (isClosed()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - parameters.clear(); + +// parameters.clear(); + parameters = new Object[parameters.length]; } @Override public void setObject(int parameterIndex, Object x, int targetSqlType) throws SQLException { if (isClosed()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setObject(int parameterIndex, Object x) throws SQLException { - if (isSaved) { - this.savedPreparedStatement.setParam(parameterIndex, x); - } else { - parameters.add(x); - } +// if (isSaved) { +// this.savedPreparedStatement.setParam(parameterIndex, x); +// } else { + if (parameterIndex < 1 && parameterIndex >= parameters.length) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + + parameters[parameterIndex - 1] = x; +// parameters.add(x); +// } } @Override public boolean execute() throws SQLException { - if (isSaved) { - int result = this.savedPreparedStatement.executeBatchInternal(); - return result > 0; - } else { - return super.execute(getNativeSql()); - } +// if (isSaved) { +// int result = this.savedPreparedStatement.executeBatchInternal(); +// return result > 0; +// } else { + if (!isPrepared) + return execute(this.rawSql); + + final String sql = getNativeSql(this.rawSql); + + return execute(sql); +// } } @Override public void addBatch() throws SQLException { - if (isSaved) { - this.savedPreparedStatement.addBatch(); - } else { - if (this.batchedArgs == null) { - batchedArgs = new ArrayList<>(); - } - super.addBatch(getNativeSql()); +// if (isSaved) { +// this.savedPreparedStatement.addBatch(); +// } else { + if (this.batchedArgs == null) { + batchedArgs = new ArrayList<>(); } + + if (!isPrepared) { + addBatch(this.rawSql); + } else { + String sql = this.getConnection().nativeSQL(this.rawSql); + addBatch(sql); + } +// } } @Override @@ -491,9 +519,11 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat public ParameterMetaData getParameterMetaData() throws SQLException { if (isClosed()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - //TODO: parameterMetaData not supported -// return null; - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + + if (parameterMetaData == null) { + this.parameterMetaData = new TSDBParameterMetaData(parameters); + } + return this.parameterMetaData; } @Override diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java index 7c2940fdc2..a20ddaa836 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java @@ -14,9 +14,14 @@ *****************************************************************************/ package com.taosdata.jdbc; +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import com.google.common.primitives.Shorts; + import java.math.BigDecimal; import java.sql.*; import java.util.ArrayList; +import java.util.Calendar; import java.util.List; public class TSDBResultSet extends AbstractResultSet implements ResultSet { @@ -120,158 +125,189 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet { } public String getString(int columnIndex) throws SQLException { + checkAvailability(columnIndex, this.columnMetaDataList.size()); + String res = null; - int colIndex = getTrueColumnIndex(columnIndex); - if (this.getBatchFetch()) - return this.blockData.getString(colIndex); + return this.blockData.getString(columnIndex - 1); - this.lastWasNull = this.rowData.wasNull(colIndex); + this.lastWasNull = this.rowData.wasNull(columnIndex - 1); if (!lastWasNull) { - res = this.rowData.getString(colIndex, this.columnMetaDataList.get(colIndex).getColType()); + res = this.rowData.getString(columnIndex - 1, this.columnMetaDataList.get(columnIndex - 1).getColType()); } return res; } public boolean getBoolean(int columnIndex) throws SQLException { + checkAvailability(columnIndex, this.columnMetaDataList.size()); + boolean res = false; - int colIndex = getTrueColumnIndex(columnIndex); - if (this.getBatchFetch()) - return this.blockData.getBoolean(colIndex); + return this.blockData.getBoolean(columnIndex - 1); - this.lastWasNull = this.rowData.wasNull(colIndex); + this.lastWasNull = this.rowData.wasNull(columnIndex - 1); if (!lastWasNull) { - res = this.rowData.getBoolean(colIndex, this.columnMetaDataList.get(colIndex).getColType()); + res = this.rowData.getBoolean(columnIndex - 1, this.columnMetaDataList.get(columnIndex - 1).getColType()); } return res; } public byte getByte(int columnIndex) throws SQLException { + checkAvailability(columnIndex, this.columnMetaDataList.size()); + byte res = 0; - int colIndex = getTrueColumnIndex(columnIndex); - if (this.getBatchFetch()) - return (byte) this.blockData.getInt(colIndex); + return (byte) this.blockData.getInt(columnIndex - 1); - this.lastWasNull = this.rowData.wasNull(colIndex); + this.lastWasNull = this.rowData.wasNull(columnIndex - 1); if (!lastWasNull) { - res = (byte) this.rowData.getInt(colIndex, this.columnMetaDataList.get(colIndex).getColType()); + res = (byte) this.rowData.getInt(columnIndex - 1, this.columnMetaDataList.get(columnIndex - 1).getColType()); } return res; } public short getShort(int columnIndex) throws SQLException { + checkAvailability(columnIndex, this.columnMetaDataList.size()); + short res = 0; - int colIndex = getTrueColumnIndex(columnIndex); - if (this.getBatchFetch()) - return (short) this.blockData.getInt(colIndex); + return (short) this.blockData.getInt(columnIndex - 1); - this.lastWasNull = this.rowData.wasNull(colIndex); + this.lastWasNull = this.rowData.wasNull(columnIndex - 1); if (!lastWasNull) { - res = (short) this.rowData.getInt(colIndex, this.columnMetaDataList.get(colIndex).getColType()); + res = (short) this.rowData.getInt(columnIndex - 1, this.columnMetaDataList.get(columnIndex - 1).getColType()); } return res; } public int getInt(int columnIndex) throws SQLException { + checkAvailability(columnIndex, this.columnMetaDataList.size()); + int res = 0; - int colIndex = getTrueColumnIndex(columnIndex); - if (this.getBatchFetch()) - return this.blockData.getInt(colIndex); + return this.blockData.getInt(columnIndex - 1); - this.lastWasNull = this.rowData.wasNull(colIndex); + this.lastWasNull = this.rowData.wasNull(columnIndex - 1); if (!lastWasNull) { - res = this.rowData.getInt(colIndex, this.columnMetaDataList.get(colIndex).getColType()); + res = this.rowData.getInt(columnIndex - 1, this.columnMetaDataList.get(columnIndex - 1).getColType()); } return res; } public long getLong(int columnIndex) throws SQLException { + checkAvailability(columnIndex, this.columnMetaDataList.size()); + long res = 0L; - int colIndex = getTrueColumnIndex(columnIndex); - if (this.getBatchFetch()) - return this.blockData.getLong(colIndex); + return this.blockData.getLong(columnIndex - 1); - this.lastWasNull = this.rowData.wasNull(colIndex); + this.lastWasNull = this.rowData.wasNull(columnIndex - 1); if (!lastWasNull) { - res = this.rowData.getLong(colIndex, this.columnMetaDataList.get(colIndex).getColType()); + res = this.rowData.getLong(columnIndex - 1, this.columnMetaDataList.get(columnIndex - 1).getColType()); } return res; } public float getFloat(int columnIndex) throws SQLException { + checkAvailability(columnIndex, this.columnMetaDataList.size()); + float res = 0; - int colIndex = getTrueColumnIndex(columnIndex); - if (this.getBatchFetch()) - return (float) this.blockData.getDouble(colIndex); + return (float) this.blockData.getDouble(columnIndex - 1); - this.lastWasNull = this.rowData.wasNull(colIndex); + this.lastWasNull = this.rowData.wasNull(columnIndex - 1); if (!lastWasNull) - res = this.rowData.getFloat(colIndex, this.columnMetaDataList.get(colIndex).getColType()); + res = this.rowData.getFloat(columnIndex - 1, this.columnMetaDataList.get(columnIndex - 1).getColType()); return res; } public double getDouble(int columnIndex) throws SQLException { + checkAvailability(columnIndex, this.columnMetaDataList.size()); + double res = 0; - int colIndex = getTrueColumnIndex(columnIndex); - if (this.getBatchFetch()) - return this.blockData.getDouble(colIndex); + return this.blockData.getDouble(columnIndex - 1); - this.lastWasNull = this.rowData.wasNull(colIndex); + this.lastWasNull = this.rowData.wasNull(columnIndex - 1); if (!lastWasNull) { - res = this.rowData.getDouble(colIndex, this.columnMetaDataList.get(colIndex).getColType()); + res = this.rowData.getDouble(columnIndex - 1, this.columnMetaDataList.get(columnIndex - 1).getColType()); } return res; } - @Deprecated - public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException { - return new BigDecimal(getLong(columnIndex)); + public byte[] getBytes(int columnIndex) throws SQLException { + checkAvailability(columnIndex, this.columnMetaDataList.size()); + + Object value = this.rowData.get(columnIndex - 1); + if (value == null) + return null; + + int colType = this.columnMetaDataList.get(columnIndex - 1).getColType(); + switch (colType) { + case TSDBConstants.TSDB_DATA_TYPE_BIGINT: + return Longs.toByteArray((Long) value); + case TSDBConstants.TSDB_DATA_TYPE_INT: + return Ints.toByteArray((int) value); + case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: + return Shorts.toByteArray((Short) value); + case TSDBConstants.TSDB_DATA_TYPE_TINYINT: + return new byte[]{(byte) value}; + } + return value.toString().getBytes(); } - public byte[] getBytes(int columnIndex) throws SQLException { - return getString(columnIndex).getBytes(); + @Override + public Date getDate(int columnIndex) throws SQLException { + Timestamp timestamp = getTimestamp(columnIndex); + return timestamp == null ? null : new Date(timestamp.getTime()); + } + + @Override + public Time getTime(int columnIndex) throws SQLException { + Timestamp timestamp = getTimestamp(columnIndex); + return timestamp == null ? null : new Time(timestamp.getTime()); } public Timestamp getTimestamp(int columnIndex) throws SQLException { + checkAvailability(columnIndex, this.columnMetaDataList.size()); + Timestamp res = null; - int colIndex = getTrueColumnIndex(columnIndex); if (this.getBatchFetch()) - return this.blockData.getTimestamp(columnIndex); + return this.blockData.getTimestamp(columnIndex - 1); - this.lastWasNull = this.rowData.wasNull(colIndex); + this.lastWasNull = this.rowData.wasNull(columnIndex - 1); if (!lastWasNull) { - res = this.rowData.getTimestamp(colIndex); + res = this.rowData.getTimestamp(columnIndex - 1); } return res; } public ResultSetMetaData getMetaData() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + return new TSDBResultSetMetaData(this.columnMetaDataList); } @Override public Object getObject(int columnIndex) throws SQLException { - int colIndex = getTrueColumnIndex(columnIndex); + checkAvailability(columnIndex, this.columnMetaDataList.size()); + Object res = null; if (this.getBatchFetch()) - return this.blockData.get(colIndex); + return this.blockData.get(columnIndex - 1); - this.lastWasNull = this.rowData.wasNull(colIndex); - return this.rowData.get(colIndex); - } - - @Override - public Object getObject(String columnLabel) throws SQLException { - return this.getObject(this.findColumn(columnLabel)); + this.lastWasNull = this.rowData.wasNull(columnIndex - 1); + if (!lastWasNull) { + int colType = this.columnMetaDataList.get(columnIndex - 1).getColType(); + if (colType == TSDBConstants.TSDB_DATA_TYPE_BINARY) + res = ((String) this.rowData.get(columnIndex - 1)).getBytes(); + else + res = this.rowData.get(columnIndex - 1); + } + return res; } public int findColumn(String columnLabel) throws SQLException { @@ -285,14 +321,31 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet { @Override public BigDecimal getBigDecimal(int columnIndex) throws SQLException { - int colIndex = getTrueColumnIndex(columnIndex); + if (this.getBatchFetch()) + return new BigDecimal(this.blockData.getLong(columnIndex - 1)); - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - return new BigDecimal(this.rowData.getLong(colIndex, this.columnMetaDataList.get(colIndex).getColType())); - } else { - return new BigDecimal(this.blockData.getLong(colIndex)); + this.lastWasNull = this.rowData.wasNull(columnIndex - 1); + BigDecimal res = null; + if (!lastWasNull) { + int colType = this.columnMetaDataList.get(columnIndex - 1).getColType(); + switch (colType) { + case TSDBConstants.TSDB_DATA_TYPE_TINYINT: + case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: + case TSDBConstants.TSDB_DATA_TYPE_INT: + case TSDBConstants.TSDB_DATA_TYPE_BIGINT: + res = new BigDecimal(Long.valueOf(this.rowData.get(columnIndex - 1).toString())); + break; + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: + res = new BigDecimal(Double.valueOf(this.rowData.get(columnIndex - 1).toString())); + break; + case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: + return new BigDecimal(((Timestamp) this.rowData.get(columnIndex - 1)).getTime()); + default: + res = new BigDecimal(this.rowData.get(columnIndex - 1).toString()); + } } + return res; } @Override @@ -398,6 +451,12 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet { return this.statement; } + @Override + public Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException { + //TODO：did not use the specified timezone in cal + return getTimestamp(columnIndex); + } + public boolean isClosed() throws SQLException { if (isClosed) return true; @@ -408,17 +467,7 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet { } public String getNString(int columnIndex) throws SQLException { - int colIndex = getTrueColumnIndex(columnIndex); - return (String) rowData.get(colIndex); + return getString(columnIndex); } - private int getTrueColumnIndex(int columnIndex) throws SQLException { - if (columnIndex < 1) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE, "columnIndex(" + columnIndex + "): < 1"); - - int numOfCols = this.columnMetaDataList.size(); - if (columnIndex > numOfCols) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE, "columnIndex: " + columnIndex); - return columnIndex - 1; - } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java index 7abd997459..e4ac5303d0 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java @@ -22,136 +22,160 @@ import java.util.List; public class TSDBResultSetMetaData extends WrapperImpl implements ResultSetMetaData { - List colMetaDataList = null; + List colMetaDataList; - public TSDBResultSetMetaData(List metaDataList) { - this.colMetaDataList = metaDataList; - } + public TSDBResultSetMetaData(List metaDataList) { + this.colMetaDataList = metaDataList; + } - public int getColumnCount() throws SQLException { - return colMetaDataList.size(); - } + public int getColumnCount() throws SQLException { + return colMetaDataList.size(); + } - public boolean isAutoIncrement(int column) throws SQLException { - return false; - } + public boolean isAutoIncrement(int column) throws SQLException { + return false; + } - public boolean isCaseSensitive(int column) throws SQLException { - return false; - } + public boolean isCaseSensitive(int column) throws SQLException { + return false; + } - public boolean isSearchable(int column) throws SQLException { - if (column == 1) { - return true; - } - return false; - } + public boolean isSearchable(int column) throws SQLException { + if (column == 1) { + return true; + } + return false; + } - public boolean isCurrency(int column) throws SQLException { - return false; - } + public boolean isCurrency(int column) throws SQLException { + return false; + } - public int isNullable(int column) throws SQLException { - if (column == 1) { - return columnNoNulls; - } - return columnNullable; - } + public int isNullable(int column) throws SQLException { + if (column < 1 && column >= colMetaDataList.size()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); - public boolean isSigned(int column) throws SQLException { - ColumnMetaData meta = this.colMetaDataList.get(column - 1); - switch (meta.getColType()) { - case TSDBConstants.TSDB_DATA_TYPE_TINYINT: - case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: - case TSDBConstants.TSDB_DATA_TYPE_INT: - case TSDBConstants.TSDB_DATA_TYPE_BIGINT: - case TSDBConstants.TSDB_DATA_TYPE_FLOAT: - case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: - return true; - default: - return false; - } - } + if (column == 1) { + return columnNoNulls; + } + return columnNullable; + } - public int getColumnDisplaySize(int column) throws SQLException { - return colMetaDataList.get(column - 1).getColSize(); - } + public boolean isSigned(int column) throws SQLException { + if (column < 1 && column >= colMetaDataList.size()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); - public String getColumnLabel(int column) throws SQLException { - return colMetaDataList.get(column - 1).getColName(); - } + ColumnMetaData meta = this.colMetaDataList.get(column - 1); + switch (meta.getColType()) { + case TSDBConstants.TSDB_DATA_TYPE_TINYINT: + case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: + case TSDBConstants.TSDB_DATA_TYPE_INT: + case TSDBConstants.TSDB_DATA_TYPE_BIGINT: + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: + return true; + default: + return false; + } + } - public String getColumnName(int column) throws SQLException { - return colMetaDataList.get(column - 1).getColName(); - } + public int getColumnDisplaySize(int column) throws SQLException { + if (column < 1 && column >= colMetaDataList.size()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); - public String getSchemaName(int column) throws SQLException { - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); - } + return colMetaDataList.get(column - 1).getColSize(); + } - public int getPrecision(int column) throws SQLException { - ColumnMetaData columnMetaData = this.colMetaDataList.get(column - 1); - switch (columnMetaData.getColType()) { - case TSDBConstants.TSDB_DATA_TYPE_FLOAT: - return 5; - case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: - return 9; - case TSDBConstants.TSDB_DATA_TYPE_BINARY: - case TSDBConstants.TSDB_DATA_TYPE_NCHAR: - return columnMetaData.getColSize(); - default: - return 0; - } - } + public String getColumnLabel(int column) throws SQLException { + if (column < 1 && column >= colMetaDataList.size()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); - public int getScale(int column) throws SQLException { - ColumnMetaData meta = this.colMetaDataList.get(column - 1); - switch (meta.getColType()) { - case TSDBConstants.TSDB_DATA_TYPE_FLOAT: - return 5; - case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: - return 9; - default: - return 0; - } - } + return colMetaDataList.get(column - 1).getColName(); + } - public String getTableName(int column) throws SQLException { - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); - } + public String getColumnName(int column) throws SQLException { + if (column < 1 && column >= colMetaDataList.size()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); - public String getCatalogName(int column) throws SQLException { - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); - } + return colMetaDataList.get(column - 1).getColName(); + } - public int getColumnType(int column) throws SQLException { - ColumnMetaData meta = this.colMetaDataList.get(column - 1); - return TSDBConstants.taosType2JdbcType(meta.getColType()); - } + public String getSchemaName(int column) throws SQLException { + if (column < 1 && column >= colMetaDataList.size()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); - public String getColumnTypeName(int column) throws SQLException { - ColumnMetaData meta = this.colMetaDataList.get(column - 1); - return TSDBConstants.taosType2JdbcTypeName(meta.getColType()); - } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } - public boolean isReadOnly(int column) throws SQLException { - return true; - } + public int getPrecision(int column) throws SQLException { + if (column < 1 && column >= colMetaDataList.size()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); - public boolean isWritable(int column) throws SQLException { - return false; - } + ColumnMetaData columnMetaData = this.colMetaDataList.get(column - 1); + switch (columnMetaData.getColType()) { + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: + return 5; + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: + return 9; + case TSDBConstants.TSDB_DATA_TYPE_BINARY: + case TSDBConstants.TSDB_DATA_TYPE_NCHAR: + return columnMetaData.getColSize(); + default: + return 0; + } + } - public boolean isDefinitelyWritable(int column) throws SQLException { - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); - } + public int getScale(int column) throws SQLException { + if (column < 1 && column >= colMetaDataList.size()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); - public String getColumnClassName(int column) throws SQLException { - int columnType = getColumnType(column); - String columnClassName = ""; - switch (columnType) { - case Types.TIMESTAMP: - columnClassName = Timestamp.class.getName(); + ColumnMetaData meta = this.colMetaDataList.get(column - 1); + switch (meta.getColType()) { + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: + return 5; + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: + return 9; + default: + return 0; + } + } + + public String getTableName(int column) throws SQLException { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + public String getCatalogName(int column) throws SQLException { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + public int getColumnType(int column) throws SQLException { + ColumnMetaData meta = this.colMetaDataList.get(column - 1); + return TSDBConstants.taosType2JdbcType(meta.getColType()); + } + + public String getColumnTypeName(int column) throws SQLException { + ColumnMetaData meta = this.colMetaDataList.get(column - 1); + return TSDBConstants.taosType2JdbcTypeName(meta.getColType()); + } + + public boolean isReadOnly(int column) throws SQLException { + return true; + } + + public boolean isWritable(int column) throws SQLException { + return false; + } + + public boolean isDefinitelyWritable(int column) throws SQLException { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + public String getColumnClassName(int column) throws SQLException { + int columnType = getColumnType(column); + String columnClassName = ""; + switch (columnType) { + case Types.TIMESTAMP: + columnClassName = Timestamp.class.getName(); break; case Types.CHAR: columnClassName = String.class.getName(); @@ -168,8 +192,8 @@ public class TSDBResultSetMetaData extends WrapperImpl implements ResultSetMetaD case Types.INTEGER: columnClassName = Integer.class.getName(); break; - case Types.SMALLINT: - columnClassName = Short.class.getName(); + case Types.SMALLINT: + columnClassName = Short.class.getName(); break; case Types.TINYINT: columnClassName = Byte.class.getName(); @@ -177,7 +201,7 @@ public class TSDBResultSetMetaData extends WrapperImpl implements ResultSetMetaD case Types.BIT: columnClassName = Boolean.class.getName(); break; - } + } return columnClassName; - } + } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java index 44d760897b..7cf5f0d79a 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java @@ -21,18 +21,13 @@ import java.util.ArrayList; import java.util.Collections; public class TSDBResultSetRowData { - private ArrayList data = null; + private ArrayList data; private int colSize = 0; public TSDBResultSetRowData(int colSize) { this.setColSize(colSize); } - public TSDBResultSetRowData() { - this.data = new ArrayList<>(); - this.setColSize(0); - } - public void clear() { if (this.data != null) { this.data.clear(); @@ -71,9 +66,9 @@ public class TSDBResultSetRowData { case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: case TSDBConstants.TSDB_DATA_TYPE_BIGINT: return ((Long) obj) == 1L ? Boolean.TRUE : Boolean.FALSE; + default: + return false; } - - return Boolean.TRUE; } public void setByte(int col, byte value) { @@ -198,7 +193,7 @@ public class TSDBResultSetRowData { data.set(col, value); } - public float getFloat(int col, int srcType) throws SQLException { + public float getFloat(int col, int srcType) { Object obj = data.get(col); switch (srcType) { @@ -226,7 +221,7 @@ public class TSDBResultSetRowData { data.set(col, value); } - public double getDouble(int col, int srcType) throws SQLException { + public double getDouble(int col, int srcType) { Object obj = data.get(col); switch (srcType) { @@ -267,9 +262,8 @@ public class TSDBResultSetRowData { * * @param col column index * @return - * @throws SQLException */ - public String getString(int col, int srcType) throws SQLException { + public String getString(int col, int srcType) { switch (srcType) { case TSDBConstants.TSDB_DATA_TYPE_BINARY: case TSDBConstants.TSDB_DATA_TYPE_NCHAR: @@ -305,11 +299,11 @@ public class TSDBResultSetRowData { } public void setTimestamp(int col, long ts) { - data.set(col, ts); + data.set(col, new Timestamp(ts)); } public Timestamp getTimestamp(int col) { - return new Timestamp((Long) data.get(col)); + return (Timestamp) data.get(col); } public Object get(int col) { @@ -320,7 +314,7 @@ public class TSDBResultSetRowData { return colSize; } - public void setColSize(int colSize) { + private void setColSize(int colSize) { this.colSize = colSize; this.clear(); } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/bean/TSDBPreparedParam.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/bean/TSDBPreparedParam.java deleted file mode 100644 index ef78292de6..0000000000 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/bean/TSDBPreparedParam.java +++ /dev/null @@ -1,62 +0,0 @@ -package com.taosdata.jdbc.bean; - -import java.util.List; - -/** - * tdengine batch insert or import param object - */ -public class TSDBPreparedParam { - - /** - * tableName, if sTable Name is not null, and this is sub table name. - */ - private String tableName; - - /** - * sub middle param list - */ - private List middleParamList; - - /** - * value list - */ - private List valueList; - - public TSDBPreparedParam(String tableName) { - this.tableName = tableName; - } - - public String getTableName() { - return tableName; - } - - public void setTableName(String tableName) { - this.tableName = tableName; - } - - public List getMiddleParamList() { - return middleParamList; - } - - public void setMiddleParamList(List middleParamList) { - this.middleParamList = middleParamList; - } - - public void setMiddleParam(int parameterIndex, Object x) { - this.middleParamList.set(parameterIndex, x); - } - - public List getValueList() { - return valueList; - } - - public void setValueList(List valueList) { - this.valueList = valueList; - } - - - public void setValueParam(int parameterIndex, Object x) { - this.valueList.set(parameterIndex, x); - } - -} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java index c1834a7b80..1f3ed2d144 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java @@ -1,8 +1,14 @@ package com.taosdata.jdbc.rs; -import com.taosdata.jdbc.*; +import com.taosdata.jdbc.AbstractConnection; +import com.taosdata.jdbc.TSDBDriver; +import com.taosdata.jdbc.TSDBError; +import com.taosdata.jdbc.TSDBErrorNumbers; -import java.sql.*; +import java.sql.DatabaseMetaData; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; import java.util.Properties; public class RestfulConnection extends AbstractConnection { @@ -55,7 +61,6 @@ public class RestfulConnection extends AbstractConnection { public DatabaseMetaData getMetaData() throws SQLException { if (isClosed()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); - ; return this.metadata; } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java index a94cfa6e07..6efe13561d 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java @@ -17,7 +17,7 @@ public class RestfulDriver extends AbstractDriver { static { try { - DriverManager.registerDriver(new RestfulDriver()); + java.sql.DriverManager.registerDriver(new RestfulDriver()); } catch (SQLException e) { throw TSDBError.createRuntimeException(TSDBErrorNumbers.ERROR_URL_NOT_SET, e); } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulParameterMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulParameterMetaData.java new file mode 100644 index 0000000000..7a130eb72b --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulParameterMetaData.java @@ -0,0 +1,10 @@ +package com.taosdata.jdbc.rs; + +import com.taosdata.jdbc.AbstractParameterMetaData; + +public class RestfulParameterMetaData extends AbstractParameterMetaData { + + RestfulParameterMetaData(Object[] parameters) { + super(parameters); + } +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java index 3a0ff56dd7..f82955ca9d 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java @@ -7,6 +7,7 @@ import java.io.InputStream; import java.io.Reader; import java.math.BigDecimal; import java.net.URL; +import java.nio.charset.Charset; import java.sql.*; import java.util.Calendar; @@ -30,7 +31,9 @@ public class RestfulPreparedStatement extends RestfulStatement implements Prepar parameters = new Object[parameterCnt]; this.isPrepared = true; } - //TODO: build parameterMetaData + + // build parameterMetaData + this.parameterMetaData = new RestfulParameterMetaData(parameters); } @Override @@ -60,8 +63,15 @@ public class RestfulPreparedStatement extends RestfulStatement implements Prepar for (int i = 0; i < parameters.length; ++i) { Object para = parameters[i]; if (para != null) { - String paraStr = para.toString(); - if (para instanceof Timestamp || para instanceof String) { + String paraStr; + if (para instanceof byte[]) { + paraStr = new String((byte[]) para, Charset.forName("UTF-8")); + } else { + paraStr = para.toString(); + } + // if para is timestamp or String or byte[] need to translate ' character + if (para instanceof Timestamp || para instanceof String || para instanceof byte[]) { + paraStr = paraStr.replaceAll("'", "\\\\\\\\'"); paraStr = "'" + paraStr + "'"; } sql = sql.replaceFirst("[?]", paraStr); @@ -92,7 +102,7 @@ public class RestfulPreparedStatement extends RestfulStatement implements Prepar public void setByte(int parameterIndex, byte x) throws SQLException { if (isClosed()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + setObject(parameterIndex, x); } @Override @@ -153,7 +163,7 @@ public class RestfulPreparedStatement extends RestfulStatement implements Prepar public void setBytes(int parameterIndex, byte[] x) throws SQLException { if (isClosed()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + setObject(parameterIndex, x); } @Override @@ -210,19 +220,16 @@ public class RestfulPreparedStatement extends RestfulStatement implements Prepar public void setObject(int parameterIndex, Object x, int targetSqlType) throws SQLException { if (isClosed()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + + setObject(parameterIndex,x); } @Override public void setObject(int parameterIndex, Object x) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - if (parameterIndex < 1 && parameterIndex >= parameters.length) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); parameters[parameterIndex - 1] = x; - } @Override diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java index 0cbb40dbb6..856f5257bf 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java @@ -2,13 +2,18 @@ package com.taosdata.jdbc.rs; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import com.google.common.primitives.Shorts; import com.taosdata.jdbc.AbstractResultSet; import com.taosdata.jdbc.TSDBConstants; import com.taosdata.jdbc.TSDBError; import com.taosdata.jdbc.TSDBErrorNumbers; +import java.math.BigDecimal; import java.sql.*; import java.util.ArrayList; +import java.util.Calendar; public class RestfulResultSet extends AbstractResultSet implements ResultSet { private volatile boolean isClosed; @@ -16,7 +21,6 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { private final String database; private final Statement statement; -// private final JSONObject resultJson; // data private final ArrayList> resultSet; // meta @@ -32,7 +36,6 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { public RestfulResultSet(String database, Statement statement, JSONObject resultJson) throws SQLException { this.database = database; this.statement = statement; -// this.resultJson = resultJson; // column metadata JSONArray columnMeta = resultJson.getJSONArray("column_meta"); @@ -73,7 +76,7 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { case TSDBConstants.TSDB_DATA_TYPE_INT: return row.getInteger(colIndex); case TSDBConstants.TSDB_DATA_TYPE_BIGINT: - return row.getBigInteger(colIndex); + return row.getLong(colIndex); case TSDBConstants.TSDB_DATA_TYPE_FLOAT: return row.getFloat(colIndex); case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: @@ -81,9 +84,11 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: return new Timestamp(row.getDate(colIndex).getTime()); case TSDBConstants.TSDB_DATA_TYPE_BINARY: + return row.getString(colIndex).getBytes(); case TSDBConstants.TSDB_DATA_TYPE_NCHAR: - default: return row.getString(colIndex); + default: + return row.get(colIndex); } } @@ -130,37 +135,33 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { @Override public String getString(int columnIndex) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - if (columnIndex > resultSet.get(pos).size()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE, "Column Index out of range, " + columnIndex + " > " + resultSet.get(pos).size()); + checkAvailability(columnIndex, resultSet.get(pos).size()); - columnIndex = getTrueColumnIndex(columnIndex); - Object value = resultSet.get(pos).get(columnIndex); - return value == null ? null : value.toString(); + Object value = resultSet.get(pos).get(columnIndex - 1); + if (value == null) + return null; + if (value instanceof byte[]) + return new String((byte[]) value); + return value.toString(); } @Override public boolean getBoolean(int columnIndex) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - if (columnIndex > resultSet.get(pos).size()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE, "Column Index out of range, " + columnIndex + " > " + resultSet.get(pos).size()); + checkAvailability(columnIndex, resultSet.get(pos).size()); - columnIndex = getTrueColumnIndex(columnIndex); - int result = getInt(columnIndex); - return result == 0 ? false : true; + Object value = resultSet.get(pos).get(columnIndex - 1); + if (value == null) + return false; + if (value instanceof Boolean) + return (boolean) value; + return Boolean.valueOf(value.toString()); } @Override public byte getByte(int columnIndex) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - if (columnIndex > resultSet.get(pos).size()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE, "Column Index out of range, " + columnIndex + " > " + resultSet.get(pos).size()); + checkAvailability(columnIndex, resultSet.get(pos).size()); - columnIndex = getTrueColumnIndex(columnIndex); - Object value = resultSet.get(pos).get(columnIndex); + Object value = resultSet.get(pos).get(columnIndex - 1); if (value == null) return 0; long valueAsLong = Long.parseLong(value.toString()); @@ -179,13 +180,9 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { @Override public short getShort(int columnIndex) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - if (columnIndex > resultSet.get(pos).size()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE, "Column Index out of range, " + columnIndex + " > " + resultSet.get(pos).size()); + checkAvailability(columnIndex, resultSet.get(pos).size()); - columnIndex = getTrueColumnIndex(columnIndex); - Object value = resultSet.get(pos).get(columnIndex); + Object value = resultSet.get(pos).get(columnIndex - 1); if (value == null) return 0; long valueAsLong = Long.parseLong(value.toString()); @@ -198,13 +195,9 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { @Override public int getInt(int columnIndex) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - if (columnIndex > resultSet.get(pos).size()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE, "Column Index out of range, " + columnIndex + " > " + resultSet.get(pos).size()); + checkAvailability(columnIndex, resultSet.get(pos).size()); - columnIndex = getTrueColumnIndex(columnIndex); - Object value = resultSet.get(pos).get(columnIndex); + Object value = resultSet.get(pos).get(columnIndex - 1); if (value == null) return 0; long valueAsLong = Long.parseLong(value.toString()); @@ -217,13 +210,9 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { @Override public long getLong(int columnIndex) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - if (columnIndex > resultSet.get(pos).size()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE, "Column Index out of range, " + columnIndex + " > " + resultSet.get(pos).size()); + checkAvailability(columnIndex, resultSet.get(pos).size()); - columnIndex = getTrueColumnIndex(columnIndex); - Object value = resultSet.get(pos).get(columnIndex); + Object value = resultSet.get(pos).get(columnIndex - 1); if (value == null) return 0; @@ -240,64 +229,99 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { @Override public float getFloat(int columnIndex) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - if (columnIndex > resultSet.get(pos).size()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE, "Column Index out of range, " + columnIndex + " > " + resultSet.get(pos).size()); + checkAvailability(columnIndex, resultSet.get(pos).size()); - columnIndex = getTrueColumnIndex(columnIndex); - return Float.parseFloat(resultSet.get(pos).get(columnIndex).toString()); + Object value = resultSet.get(pos).get(columnIndex - 1); + if (value == null) + return 0; + if (value instanceof Float || value instanceof Double) + return (float) value; + return Float.parseFloat(value.toString()); } @Override public double getDouble(int columnIndex) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - if (columnIndex > resultSet.get(pos).size()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE, "Column Index out of range, " + columnIndex + " > " + resultSet.get(pos).size()); + checkAvailability(columnIndex, resultSet.get(pos).size()); - columnIndex = getTrueColumnIndex(columnIndex); - return Double.parseDouble(resultSet.get(pos).get(columnIndex).toString()); + Object value = resultSet.get(pos).get(columnIndex - 1); + if (value == null) + return 0; + if (value instanceof Double || value instanceof Float) + return (double) value; + return Double.parseDouble(value.toString()); } - private int getTrueColumnIndex(int columnIndex) throws SQLException { - if (columnIndex < 1) { - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE - , "Column Index out of range, " + columnIndex + " < 1"); - } + @Override + public byte[] getBytes(int columnIndex) throws SQLException { + checkAvailability(columnIndex, resultSet.get(pos).size()); - int numOfCols = resultSet.get(pos).size(); - if (columnIndex > numOfCols) { - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE - , "Column Index out of range, " + columnIndex + " > " + numOfCols); - } + Object value = resultSet.get(pos).get(columnIndex - 1); + if (value == null) + return null; + if (value instanceof byte[]) + return (byte[]) value; + if (value instanceof String) + return ((String) value).getBytes(); + if (value instanceof Long) + return Longs.toByteArray((long) value); + if (value instanceof Integer) + return Ints.toByteArray((int) value); + if (value instanceof Short) + return Shorts.toByteArray((short) value); + if (value instanceof Byte) + return new byte[]{(byte) value}; - return columnIndex - 1; + return value.toString().getBytes(); + } + + @Override + public Date getDate(int columnIndex) throws SQLException { + checkAvailability(columnIndex, resultSet.get(pos).size()); + + Object value = resultSet.get(pos).get(columnIndex - 1); + if (value == null) + return null; + if (value instanceof Timestamp) + return new Date(((Timestamp) value).getTime()); + return Date.valueOf(value.toString()); + } + + @Override + public Time getTime(int columnIndex) throws SQLException { + checkAvailability(columnIndex, resultSet.get(pos).size()); + + Object value = resultSet.get(pos).get(columnIndex - 1); + if (value == null) + return null; + if (value instanceof Timestamp) + return new Time(((Timestamp) value).getTime()); + return Time.valueOf(value.toString()); } @Override public Timestamp getTimestamp(int columnIndex) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + checkAvailability(columnIndex, resultSet.get(pos).size()); - columnIndex = getTrueColumnIndex(columnIndex); - String strDate = resultSet.get(pos).get(columnIndex).toString(); -// strDate = strDate.substring(1, strDate.length() - 1); - return Timestamp.valueOf(strDate); + Object value = resultSet.get(pos).get(columnIndex - 1); + if (value == null) + return null; + if (value instanceof Timestamp) + return (Timestamp) value; + return Timestamp.valueOf(value.toString()); } - /*************************************************************************************************************/ @Override public ResultSetMetaData getMetaData() throws SQLException { if (isClosed()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - return this.metaData; } @Override - public Object getObject(String columnLabel) throws SQLException { - return getObject(findColumn(columnLabel)); + public Object getObject(int columnIndex) throws SQLException { + checkAvailability(columnIndex, resultSet.get(pos).size()); + + return resultSet.get(pos).get(columnIndex - 1); } @Override @@ -311,6 +335,23 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { return columnIndex + 1; } + @Override + public BigDecimal getBigDecimal(int columnIndex) throws SQLException { + checkAvailability(columnIndex, resultSet.get(pos).size()); + + Object value = resultSet.get(pos).get(columnIndex - 1); + if (value == null) + return null; + + if (value instanceof Long || value instanceof Integer || value instanceof Short || value instanceof Byte) + return new BigDecimal(Long.valueOf(value.toString())); + if (value instanceof Double || value instanceof Float) + return new BigDecimal(Double.valueOf(value.toString())); + if (value instanceof Timestamp) + return new BigDecimal(((Timestamp) value).getTime()); + return new BigDecimal(value.toString()); + } + @Override public boolean isBeforeFirst() throws SQLException { if (isClosed()) @@ -471,6 +512,12 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { return this.statement; } + @Override + public Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException { + //TODO：did not use the specified timezone in cal + return getTimestamp(columnIndex); + } + @Override public boolean isClosed() throws SQLException { return isClosed; diff --git a/src/connector/jdbc/src/main/resources/META-INF/services/java.sql.Driver b/src/connector/jdbc/src/main/resources/META-INF/services/java.sql.Driver index e65b4e3b22..893f7cdf34 100644 --- a/src/connector/jdbc/src/main/resources/META-INF/services/java.sql.Driver +++ b/src/connector/jdbc/src/main/resources/META-INF/services/java.sql.Driver @@ -1 +1,2 @@ com.taosdata.jdbc.TSDBDriver +com.taosdata.jdbc.rs.RestfulDriver diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java deleted file mode 100644 index 87348f9026..0000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java +++ /dev/null @@ -1,266 +0,0 @@ -package com.taosdata.jdbc; - -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -import javax.sql.rowset.serial.SerialBlob; -import javax.sql.rowset.serial.SerialClob; -import java.io.UnsupportedEncodingException; -import java.sql.*; -import java.util.HashMap; -import java.util.Properties; - -import static org.junit.Assert.assertEquals; - -public class ResultSetTest { - static Connection connection; - static Statement statement; - static String dbName = "test"; - static String tName = "t0"; - static String host = "localhost"; - static ResultSet resSet; - - @BeforeClass - public static void createDatabaseAndTable() { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - statement = connection.createStatement(); - statement.executeUpdate("drop database if exists " + dbName); - statement.executeUpdate("create database if not exists " + dbName); - statement.execute("use " + dbName); - statement.executeUpdate("create table if not exists " + dbName + "." + tName + " (ts timestamp, k1 int, k2 bigint, k3 float, k4 double, k5 binary(30), k6 smallint, k7 bool, k8 nchar(20))"); - } catch (ClassNotFoundException | SQLException e) { - return; - } - } - - @Test - public void testResultSet() { - String sql; - long ts = 1496732686000l; - int v1 = 2147483600; - long v2 = ts + 1000; - float v3 = 3.1415926f; - double v4 = 3.1415926535897; - String v5 = "涛思数据，强~！"; - short v6 = 12; - boolean v7 = false; - String v8 = "TDengine is powerful"; - sql = "insert into " + dbName + "." + tName + " values (" + ts + "," + v1 + "," + v2 + "," + v3 + "," + v4 - + ",\"" + v5 + "\"," + v6 + "," + v7 + ",\"" + v8 + "\")"; - try { - statement.executeUpdate(sql); - assertEquals(1, statement.getUpdateCount()); - } catch (SQLException e) { - assert false : "insert error " + e.getMessage(); - } - try { - statement.execute("select * from " + dbName + "." + tName + " where ts = " + ts); - resSet = statement.getResultSet(); - System.out.println(((TSDBResultSet) resSet).getRowData()); - while (resSet.next()) { - assertEquals(ts, resSet.getLong(1)); - assertEquals(ts, resSet.getLong("ts")); - System.out.println(resSet.getTimestamp(1)); - assertEquals(v1, resSet.getInt(2)); - assertEquals(v1, resSet.getInt("k1")); - assertEquals(v2, resSet.getLong(3)); - assertEquals(v2, resSet.getLong("k2")); - assertEquals(v3, resSet.getFloat(4), 7); - assertEquals(v3, resSet.getFloat("k3"), 7); - assertEquals(v4, resSet.getDouble(5), 13); - assertEquals(v4, resSet.getDouble("k4"), 13); - assertEquals(v5, resSet.getString(6)); - assertEquals(v5, resSet.getString("k5")); - assertEquals(v6, resSet.getShort(7)); - assertEquals(v6, resSet.getShort("k6")); - assertEquals(v7, resSet.getBoolean(8)); - assertEquals(v7, resSet.getBoolean("k7")); - assertEquals(v8, resSet.getString(9)); - assertEquals(v8, resSet.getString("k8")); - resSet.getBytes(9); - resSet.getObject(6); - resSet.getObject("k8"); - } - if (!resSet.isClosed()) { - resSet.close(); - } - } catch (SQLException e) { - assert false : "insert error " + e.getMessage(); - } - } - - @Test(expected = SQLException.class) - public void testUnsupport() throws SQLException, UnsupportedEncodingException { - statement.execute("show databases"); - resSet = statement.getResultSet(); - Assert.assertNotNull(resSet.unwrap(TSDBResultSet.class)); - Assert.assertTrue(resSet.isWrapperFor(TSDBResultSet.class)); - resSet.getUnicodeStream(null); - resSet.getBinaryStream(null); - resSet.getAsciiStream(""); - resSet.getUnicodeStream(null); - resSet.getBinaryStream(null); - resSet.getWarnings(); - resSet.clearWarnings(); - resSet.getCursorName(); - resSet.getCharacterStream(null); - resSet.getCharacterStream(null); - resSet.isBeforeFirst(); - resSet.isAfterLast(); - resSet.isFirst(); - resSet.isLast(); - resSet.beforeFirst(); - resSet.afterLast(); - resSet.first(); - resSet.last(); - resSet.getRow(); - resSet.absolute(1); - resSet.relative(1); - resSet.previous(); - resSet.setFetchDirection(0); - resSet.getFetchDirection(); - resSet.setFetchSize(0); - resSet.getFetchSize(); - resSet.getConcurrency(); - resSet.rowUpdated(); - resSet.rowInserted(); - resSet.rowDeleted(); - resSet.updateNull(null); - resSet.updateBoolean(0, true); - resSet.updateByte(0, (byte) 2); - resSet.updateShort(0, (short) 1); - resSet.updateInt(0, 0); - resSet.updateLong(0, 0l); - resSet.updateFloat(0, 3.14f); - resSet.updateDouble(0, 3.1415); - resSet.updateBigDecimal(null, null); - resSet.updateString(null, null); - resSet.updateBytes(null, null); - resSet.updateDate(null, null); - resSet.updateTime(null, null); - resSet.updateTimestamp(null, null); - resSet.updateAsciiStream(null, null); - resSet.updateBinaryStream(null, null); - resSet.updateCharacterStream(null, null); - resSet.updateObject(null, null); - resSet.updateObject(null, null); - resSet.updateNull(null); - resSet.updateBoolean("", false); - resSet.updateByte("", (byte) 1); - resSet.updateShort("", (short) 1); - resSet.updateInt("", 0); - resSet.updateLong("", 0l); - resSet.updateFloat("", 3.14f); - resSet.updateDouble("", 3.1415); - resSet.updateBigDecimal(null, null); - resSet.updateString(null, null); - resSet.updateBytes(null, null); - resSet.updateDate(null, null); - resSet.updateTime(null, null); - resSet.updateTimestamp(null, null); - resSet.updateAsciiStream(null, null); - resSet.updateBinaryStream(null, null); - resSet.updateCharacterStream(null, null); - resSet.updateObject(null, null); - resSet.updateObject(null, null); - resSet.insertRow(); - resSet.updateRow(); - resSet.deleteRow(); - resSet.refreshRow(); - resSet.cancelRowUpdates(); - resSet.moveToInsertRow(); - resSet.moveToCurrentRow(); - resSet.getStatement(); - resSet.getObject(0, new HashMap<>()); - resSet.getRef(null); - resSet.getBlob(null); - resSet.getClob(null); - resSet.getArray(null); - resSet.getObject("", new HashMap<>()); - resSet.getRef(null); - resSet.getBlob(null); - resSet.getClob(null); - resSet.getArray(null); - resSet.getDate(null, null); - resSet.getDate(null, null); - resSet.getTime(null, null); - resSet.getTime(null, null); - resSet.getTimestamp(null, null); - resSet.getTimestamp(null, null); - resSet.getURL(null); - resSet.getURL(null); - resSet.updateRef(null, null); - resSet.updateRef(null, null); - resSet.updateBlob(0, new SerialBlob("".getBytes("UTF8"))); - resSet.updateBlob("", new SerialBlob("".getBytes("UTF8"))); - resSet.updateClob("", new SerialClob("".toCharArray())); - resSet.updateClob(0, new SerialClob("".toCharArray())); - resSet.updateArray(null, null); - resSet.updateArray(null, null); - resSet.getRowId(null); - resSet.getRowId(null); - resSet.updateRowId(null, null); - resSet.updateRowId(null, null); - resSet.getHoldability(); - resSet.updateNString(null, null); - resSet.updateNString(null, null); - resSet.getNClob(null); - resSet.getNClob(null); - resSet.getSQLXML(null); - resSet.getSQLXML(null); - resSet.updateSQLXML(null, null); - resSet.updateSQLXML(null, null); - resSet.getNCharacterStream(null); - resSet.getNCharacterStream(null); - resSet.updateNCharacterStream(null, null); - resSet.updateNCharacterStream(null, null); - resSet.updateAsciiStream(null, null); - resSet.updateBinaryStream(null, null); - resSet.updateCharacterStream(null, null); - resSet.updateAsciiStream(null, null); - resSet.updateBinaryStream(null, null); - resSet.updateCharacterStream(null, null); - resSet.updateNCharacterStream(null, null); - resSet.updateNCharacterStream(null, null); - resSet.updateAsciiStream(null, null); - resSet.updateBinaryStream(null, null); - resSet.updateCharacterStream(null, null); - resSet.updateAsciiStream(null, null); - resSet.updateBinaryStream(null, null); - resSet.updateCharacterStream(null, null); - } - - @Test - public void testBatch() throws SQLException { - String[] sqls = new String[]{"insert into test.t0 values (1496732686001,2147483600,1496732687000,3.1415925,3.1415926535897," + - "'涛思数据，强~',12,0,'TDengine is powerful')", "insert into test.t0 values (1496732686002,2147483600,1496732687000,3.1415925,3.1415926535897," + - "'涛思数据，强~',12,1,'TDengine is powerful')"}; - for (String sql : sqls) { - statement.addBatch(sql); - } - int[] res = statement.executeBatch(); - assertEquals(res.length, 2); - statement.clearBatch(); - } - - @AfterClass - public static void close() { - try { - statement.executeUpdate("drop database " + dbName); - if (statement != null) - statement.close(); - if (connection != null) - connection.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - } -} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBConnectionTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBConnectionTest.java index 0a4ecb739c..ca7251bb0e 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBConnectionTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBConnectionTest.java @@ -233,7 +233,7 @@ public class TSDBConnectionTest { int status = rs.getInt("server_status()"); Assert.assertEquals(1, status); - conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); } @Test(expected = SQLFeatureNotSupportedException.class) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBParameterMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBParameterMetaDataTest.java new file mode 100644 index 0000000000..12bcc43917 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBParameterMetaDataTest.java @@ -0,0 +1,194 @@ +package com.taosdata.jdbc; + +import com.taosdata.jdbc.rs.RestfulParameterMetaData; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.*; + +public class TSDBParameterMetaDataTest { + + private static final String host = "127.0.0.1"; + private static Connection conn; + private static final String sql_insert = "insert into t1 values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + private static PreparedStatement pstmt_insert; + private static final String sql_select = "select * from t1 where ts > ? and ts <= ? and f1 >= ?"; + private static PreparedStatement pstmt_select; + private static ParameterMetaData parameterMetaData_insert; + private static ParameterMetaData parameterMetaData_select; + + @Test + public void getParameterCount() throws SQLException { + Assert.assertEquals(10, parameterMetaData_insert.getParameterCount()); + } + + @Test + public void isNullable() throws SQLException { + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(1)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(2)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(3)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(4)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(5)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(6)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(7)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(8)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(9)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(10)); + } + + @Test + public void isSigned() throws SQLException { + Assert.assertEquals(false, parameterMetaData_insert.isSigned(1)); + Assert.assertEquals(true, parameterMetaData_insert.isSigned(2)); + Assert.assertEquals(true, parameterMetaData_insert.isSigned(3)); + Assert.assertEquals(true, parameterMetaData_insert.isSigned(4)); + Assert.assertEquals(true, parameterMetaData_insert.isSigned(5)); + Assert.assertEquals(true, parameterMetaData_insert.isSigned(6)); + Assert.assertEquals(true, parameterMetaData_insert.isSigned(7)); + Assert.assertEquals(false, parameterMetaData_insert.isSigned(8)); + Assert.assertEquals(false, parameterMetaData_insert.isSigned(9)); + Assert.assertEquals(false, parameterMetaData_insert.isSigned(10)); + } + + @Test + public void getPrecision() throws SQLException { + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(1)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(2)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(3)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(4)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(5)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(6)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(7)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(8)); + Assert.assertEquals(5, parameterMetaData_insert.getPrecision(9)); + Assert.assertEquals(5, parameterMetaData_insert.getPrecision(10)); + } + + @Test + public void getScale() throws SQLException { + Assert.assertEquals(0, parameterMetaData_insert.getScale(1)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(2)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(3)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(4)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(5)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(6)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(7)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(8)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(9)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(10)); + } + + @Test + public void getParameterType() throws SQLException { + Assert.assertEquals(Types.TIMESTAMP, parameterMetaData_insert.getParameterType(1)); + Assert.assertEquals(Types.INTEGER, parameterMetaData_insert.getParameterType(2)); + Assert.assertEquals(Types.BIGINT, parameterMetaData_insert.getParameterType(3)); + Assert.assertEquals(Types.FLOAT, parameterMetaData_insert.getParameterType(4)); + Assert.assertEquals(Types.DOUBLE, parameterMetaData_insert.getParameterType(5)); + Assert.assertEquals(Types.SMALLINT, parameterMetaData_insert.getParameterType(6)); + Assert.assertEquals(Types.TINYINT, parameterMetaData_insert.getParameterType(7)); + Assert.assertEquals(Types.BOOLEAN, parameterMetaData_insert.getParameterType(8)); + Assert.assertEquals(Types.BINARY, parameterMetaData_insert.getParameterType(9)); + Assert.assertEquals(Types.NCHAR, parameterMetaData_insert.getParameterType(10)); + } + + @Test + public void getParameterTypeName() throws SQLException { + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.TIMESTAMP), parameterMetaData_insert.getParameterTypeName(1)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.INTEGER), parameterMetaData_insert.getParameterTypeName(2)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.BIGINT), parameterMetaData_insert.getParameterTypeName(3)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.FLOAT), parameterMetaData_insert.getParameterTypeName(4)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.DOUBLE), parameterMetaData_insert.getParameterTypeName(5)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.SMALLINT), parameterMetaData_insert.getParameterTypeName(6)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.TINYINT), parameterMetaData_insert.getParameterTypeName(7)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.BOOLEAN), parameterMetaData_insert.getParameterTypeName(8)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.BINARY), parameterMetaData_insert.getParameterTypeName(9)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.NCHAR), parameterMetaData_insert.getParameterTypeName(10)); + } + + @Test + public void getParameterClassName() throws SQLException { + Assert.assertEquals(Timestamp.class.getName(), parameterMetaData_insert.getParameterClassName(1)); + Assert.assertEquals(Integer.class.getName(), parameterMetaData_insert.getParameterClassName(2)); + Assert.assertEquals(Long.class.getName(), parameterMetaData_insert.getParameterClassName(3)); + Assert.assertEquals(Float.class.getName(), parameterMetaData_insert.getParameterClassName(4)); + Assert.assertEquals(Double.class.getName(), parameterMetaData_insert.getParameterClassName(5)); + Assert.assertEquals(Short.class.getName(), parameterMetaData_insert.getParameterClassName(6)); + Assert.assertEquals(Byte.class.getName(), parameterMetaData_insert.getParameterClassName(7)); + Assert.assertEquals(Boolean.class.getName(), parameterMetaData_insert.getParameterClassName(8)); + Assert.assertEquals(byte[].class.getName(), parameterMetaData_insert.getParameterClassName(9)); + Assert.assertEquals(String.class.getName(), parameterMetaData_insert.getParameterClassName(10)); + } + + @Test + public void getParameterMode() throws SQLException { + for (int i = 1; i <= parameterMetaData_insert.getParameterCount(); i++) { + int parameterMode = parameterMetaData_insert.getParameterMode(i); + Assert.assertEquals(ParameterMetaData.parameterModeUnknown, parameterMode); + } + } + + @Test + public void unwrap() throws SQLException { + TSDBParameterMetaData unwrap = parameterMetaData_insert.unwrap(TSDBParameterMetaData.class); + Assert.assertNotNull(unwrap); + } + + @Test + public void isWrapperFor() throws SQLException { + Assert.assertTrue(parameterMetaData_insert.isWrapperFor(TSDBParameterMetaData.class)); + } + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"); + try (Statement stmt = conn.createStatement()) { + stmt.execute("drop database if exists test_pstmt"); + stmt.execute("create database if not exists test_pstmt"); + stmt.execute("use test_pstmt"); + stmt.execute("create table weather(ts timestamp, f1 int, f2 bigint, f3 float, f4 double, f5 smallint, f6 tinyint, f7 bool, f8 binary(64), f9 nchar(64)) tags(loc nchar(64))"); + stmt.execute("create table t1 using weather tags('beijing')"); + } + pstmt_insert = conn.prepareStatement(sql_insert); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(2, 111); + pstmt_insert.setObject(3, Long.MAX_VALUE); + pstmt_insert.setObject(4, 3.14159265354f); + pstmt_insert.setObject(5, Double.MAX_VALUE); + pstmt_insert.setObject(6, Short.MAX_VALUE); + pstmt_insert.setObject(7, Byte.MAX_VALUE); + pstmt_insert.setObject(8, true); + pstmt_insert.setObject(9, "hello".getBytes()); + pstmt_insert.setObject(10, "Hello"); + parameterMetaData_insert = pstmt_insert.getParameterMetaData(); + + pstmt_select = conn.prepareStatement(sql_select); + pstmt_select.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_select.setTimestamp(2, new Timestamp(System.currentTimeMillis() + 10000)); + pstmt_select.setInt(3, 0); + parameterMetaData_select = pstmt_select.getParameterMetaData(); + + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (pstmt_insert != null) + pstmt_insert.close(); + if (pstmt_select != null) + pstmt_select.close(); + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java index 475cfef060..434095efa2 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java @@ -5,14 +5,16 @@ import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import java.io.IOException; +import java.io.Serializable; import java.sql.*; public class TSDBPreparedStatementTest { private static final String host = "127.0.0.1"; private static Connection conn; - private static final String sql_insert = "insert into t1 values(?, ?)"; + private static final String sql_insert = "insert into t1 values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; private static PreparedStatement pstmt_insert; - private static final String sql_select = "select * from t1 where ts > ? and ts <= ? and temperature >= ?"; + private static final String sql_select = "select * from t1 where ts > ? and ts <= ? and f1 >= ?"; private static PreparedStatement pstmt_select; @Test @@ -21,7 +23,7 @@ public class TSDBPreparedStatementTest { long start = end - 1000 * 60 * 60; pstmt_select.setTimestamp(1, new Timestamp(start)); pstmt_select.setTimestamp(2, new Timestamp(end)); - pstmt_select.setFloat(3, 0); + pstmt_select.setInt(3, 0); ResultSet rs = pstmt_select.executeQuery(); Assert.assertNotNull(rs); @@ -37,48 +39,73 @@ public class TSDBPreparedStatementTest { @Test public void executeUpdate() throws SQLException { pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); - pstmt_insert.setFloat(2, 3.14f); + pstmt_insert.setFloat(4, 3.14f); int result = pstmt_insert.executeUpdate(); Assert.assertEquals(1, result); } @Test public void setNull() throws SQLException { - pstmt_insert.setNull(2, Types.FLOAT); + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setNull(2, Types.INTEGER); + int result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); } @Test public void setBoolean() throws SQLException { - pstmt_insert.setBoolean(2, true); + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setBoolean(8, true); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); } - @Test(expected = SQLFeatureNotSupportedException.class) + @Test public void setByte() throws SQLException { - pstmt_insert.setByte(1, (byte) 0x001); + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setByte(7, (byte) 0x001); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); } @Test - public void setShort() { - + public void setShort() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setShort(6, (short) 2); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); } @Test - public void setInt() { - + public void setInt() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setInt(2, 10086); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); } @Test - public void setLong() { - + public void setLong() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setLong(3, Long.MAX_VALUE); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); } @Test - public void setFloat() { - + public void setFloat() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setFloat(4, 3.14f); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); } @Test - public void setDouble() { + public void setDouble() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setDouble(5, 3.14444); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); } @Test(expected = SQLFeatureNotSupportedException.class) @@ -87,12 +114,56 @@ public class TSDBPreparedStatementTest { } @Test - public void setString() { + public void setString() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setString(10, "aaaa"); + boolean execute = pstmt_insert.execute(); + Assert.assertFalse(execute); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setString(10, new Person("john", 33, true).toString()); + Assert.assertFalse(pstmt_insert.execute()); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setString(10, new Person("john", 33, true).toString().replaceAll("'", "\"")); + Assert.assertFalse(pstmt_insert.execute()); } - @Test(expected = SQLFeatureNotSupportedException.class) - public void setBytes() throws SQLException { - pstmt_insert.setBytes(1, new byte[]{}); + class Person implements Serializable { + String name; + int age; + boolean sex; + + public Person(String name, int age, boolean sex) { + this.name = name; + this.age = age; + this.sex = sex; + } + + @Override + public String toString() { + return "Person{" + + "name='" + name + '\'' + + ", age=" + age + + ", sex=" + sex + + '}'; + } + } + + @Test + public void setBytes() throws SQLException, IOException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + +// ByteArrayOutputStream baos = new ByteArrayOutputStream(); +// ObjectOutputStream oos = new ObjectOutputStream(baos); +// oos.writeObject(new Person("john", 33, true)); +// oos.flush(); +// byte[] bytes = baos.toByteArray(); +// pstmt_insert.setBytes(9, bytes); + + pstmt_insert.setBytes(9, new Person("john", 33, true).toString().getBytes()); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); } @Test(expected = SQLFeatureNotSupportedException.class) @@ -106,8 +177,10 @@ public class TSDBPreparedStatementTest { } @Test - public void setTimestamp() { - //TODO + public void setTimestamp() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); } @Test(expected = SQLFeatureNotSupportedException.class) @@ -121,24 +194,69 @@ public class TSDBPreparedStatementTest { } @Test - public void clearParameters() { - //TODO + public void clearParameters() throws SQLException { + pstmt_insert.clearParameters(); } @Test public void setObject() throws SQLException { - pstmt_insert.setObject(1, System.currentTimeMillis()); - //TODO + pstmt_insert.setObject(1, new Timestamp(System.currentTimeMillis())); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(2, 111); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(3, Long.MAX_VALUE); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(4, 3.14159265354f); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(5, Double.MAX_VALUE); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(6, Short.MAX_VALUE); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(7, Byte.MAX_VALUE); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(8, true); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(9, "hello".getBytes()); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(10, "Hello"); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); } @Test - public void execute() { - //TODO - } + public void execute() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); - @Test - public void addBatch() { - //TODO: + executeQuery(); } @Test(expected = SQLFeatureNotSupportedException.class) @@ -176,11 +294,11 @@ public class TSDBPreparedStatementTest { pstmt_insert.setURL(1, null); } - @Test(expected = SQLFeatureNotSupportedException.class) + @Test public void getParameterMetaData() throws SQLException { ParameterMetaData parameterMetaData = pstmt_insert.getParameterMetaData(); -// Assert.assertNotNull(parameterMetaData); - //TODO: + Assert.assertNotNull(parameterMetaData); + //TODO: modify the test case } @Test(expected = SQLFeatureNotSupportedException.class) @@ -215,10 +333,10 @@ public class TSDBPreparedStatementTest { Class.forName("com.taosdata.jdbc.TSDBDriver"); conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"); try (Statement stmt = conn.createStatement()) { - stmt.execute("drop database if exists test_pstmt"); - stmt.execute("create database if not exists test_pstmt"); - stmt.execute("use test_pstmt"); - stmt.execute("create table weather(ts timestamp, temperature float) tags(loc nchar(64))"); + stmt.execute("drop database if exists test_pstmt_jni"); + stmt.execute("create database if not exists test_pstmt_jni"); + stmt.execute("use test_pstmt_jni"); + stmt.execute("create table weather(ts timestamp, f1 int, f2 bigint, f3 float, f4 double, f5 smallint, f6 tinyint, f7 bool, f8 binary(64), f9 nchar(64)) tags(loc nchar(64))"); stmt.execute("create table t1 using weather tags('beijing')"); } pstmt_insert = conn.prepareStatement(sql_insert); @@ -231,7 +349,10 @@ public class TSDBPreparedStatementTest { @AfterClass public static void afterClass() { try { - + if (pstmt_insert != null) + pstmt_insert.close(); + if (pstmt_select != null) + pstmt_select.close(); if (conn != null) conn.close(); } catch (SQLException e) { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java new file mode 100644 index 0000000000..374b1335fc --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java @@ -0,0 +1,678 @@ +package com.taosdata.jdbc; + +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import com.google.common.primitives.Shorts; +import com.taosdata.jdbc.rs.RestfulResultSet; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.InputStream; +import java.io.Reader; +import java.math.BigDecimal; +import java.sql.*; +import java.text.ParseException; +import java.text.SimpleDateFormat; + +public class TSDBResultSetTest { + + private static final String host = "127.0.0.1"; + private static Connection conn; + private static Statement stmt; + private static ResultSet rs; + + @Test + public void wasNull() throws SQLException { + Assert.assertFalse(rs.wasNull()); + } + + @Test + public void getString() throws SQLException { + String f10 = rs.getString("f10"); + Assert.assertEquals("涛思数据", f10); + f10 = rs.getString(10); + Assert.assertEquals("涛思数据", f10); + } + + @Test + public void getBoolean() throws SQLException { + Boolean f9 = rs.getBoolean("f9"); + Assert.assertEquals(true, f9); + f9 = rs.getBoolean(9); + Assert.assertEquals(true, f9); + } + + @Test + public void getByte() throws SQLException { + byte f8 = rs.getByte("f8"); + Assert.assertEquals(10, f8); + f8 = rs.getByte(8); + Assert.assertEquals(10, f8); + } + + @Test + public void getShort() throws SQLException { + short f7 = rs.getShort("f7"); + Assert.assertEquals(10, f7); + f7 = rs.getShort(7); + Assert.assertEquals(10, f7); + } + + @Test + public void getInt() throws SQLException { + int f2 = rs.getInt("f2"); + Assert.assertEquals(1, f2); + f2 = rs.getInt(2); + Assert.assertEquals(1, f2); + } + + @Test + public void getLong() throws SQLException { + long f3 = rs.getLong("f3"); + Assert.assertEquals(100, f3); + f3 = rs.getLong(3); + Assert.assertEquals(100, f3); + } + + @Test + public void getFloat() throws SQLException { + float f4 = rs.getFloat("f4"); + Assert.assertEquals(3.1415f, f4, 0f); + f4 = rs.getFloat(4); + Assert.assertEquals(3.1415f, f4, 0f); + } + + @Test + public void getDouble() throws SQLException { + double f5 = rs.getDouble("f5"); + Assert.assertEquals(3.1415926, f5, 0.0); + f5 = rs.getDouble(5); + Assert.assertEquals(3.1415926, f5, 0.0); + } + + @Test + public void getBigDecimal() throws SQLException { + BigDecimal f1 = rs.getBigDecimal("f1"); + Assert.assertEquals(1609430400000l, f1.longValue()); + + BigDecimal f2 = rs.getBigDecimal("f2"); + Assert.assertEquals(1, f2.intValue()); + + BigDecimal f3 = rs.getBigDecimal("f3"); + Assert.assertEquals(100l, f3.longValue()); + + BigDecimal f4 = rs.getBigDecimal("f4"); + Assert.assertEquals(3.1415f, f4.floatValue(), 0.00000f); + + BigDecimal f5 = rs.getBigDecimal("f5"); + Assert.assertEquals(3.1415926, f5.doubleValue(), 0.0000000); + + BigDecimal f7 = rs.getBigDecimal("f7"); + Assert.assertEquals(10, f7.intValue()); + + BigDecimal f8 = rs.getBigDecimal("f8"); + Assert.assertEquals(10, f8.intValue()); + } + + @Test + public void getBytes() throws SQLException { + byte[] f1 = rs.getBytes("f1"); + Assert.assertEquals("2021-01-01 00:00:00.0", new String(f1)); + + byte[] f2 = rs.getBytes("f2"); + Assert.assertEquals(1, Ints.fromByteArray(f2)); + + byte[] f3 = rs.getBytes("f3"); + Assert.assertEquals(100l, Longs.fromByteArray(f3)); + + byte[] f4 = rs.getBytes("f4"); + Assert.assertEquals(3.1415f, Float.valueOf(new String(f4)), 0.000000f); + + byte[] f5 = rs.getBytes("f5"); + Assert.assertEquals(3.1415926, Double.valueOf(new String(f5)), 0.000000f); + + byte[] f6 = rs.getBytes("f6"); + Assert.assertEquals("abc", new String(f6)); + + byte[] f7 = rs.getBytes("f7"); + Assert.assertEquals((short) 10, Shorts.fromByteArray(f7)); + + byte[] f8 = rs.getBytes("f8"); + Assert.assertEquals(1, f8.length); + Assert.assertEquals((byte) 10, f8[0]); + + byte[] f9 = rs.getBytes("f9"); + Assert.assertEquals("true", new String(f9)); + + byte[] f10 = rs.getBytes("f10"); + Assert.assertEquals("涛思数据", new String(f10)); + } + + @Test + public void getDate() throws SQLException, ParseException { + Date f1 = rs.getDate("f1"); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + Assert.assertEquals(sdf.parse("2021-01-01"), f1); + } + + @Test + public void getTime() throws SQLException { + Time f1 = rs.getTime("f1"); + Assert.assertEquals("00:00:00", f1.toString()); + } + + @Test + public void getTimestamp() throws SQLException { + Timestamp f1 = rs.getTimestamp("f1"); + Assert.assertEquals("2021-01-01 00:00:00.0", f1.toString()); + f1 = rs.getTimestamp(1); + Assert.assertEquals("2021-01-01 00:00:00.0", f1.toString()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getAsciiStream() throws SQLException { + rs.getAsciiStream("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getUnicodeStream() throws SQLException { + rs.getUnicodeStream("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getBinaryStream() throws SQLException { + rs.getBinaryStream("f1"); + } + + @Test + public void getWarnings() throws SQLException { + Assert.assertNull(rs.getWarnings()); + } + + @Test + public void clearWarnings() throws SQLException { + rs.clearWarnings(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getCursorName() throws SQLException { + rs.getCursorName(); + } + + @Test + public void getMetaData() throws SQLException { + ResultSetMetaData meta = rs.getMetaData(); + Assert.assertNotNull(meta); + } + + @Test + public void getObject() throws SQLException, ParseException { + Object f1 = rs.getObject("f1"); + Assert.assertEquals(Timestamp.class, f1.getClass()); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss.sss"); + java.util.Date date = sdf.parse("2021-01-01 00:00:00.000"); + Assert.assertEquals(new Timestamp(date.getTime()), f1); + + Object f2 = rs.getObject("f2"); + Assert.assertEquals(Integer.class, f2.getClass()); + Assert.assertEquals(1, f2); + + Object f3 = rs.getObject("f3"); + Assert.assertEquals(Long.class, f3.getClass()); + Assert.assertEquals(100l, f3); + + Object f4 = rs.getObject("f4"); + Assert.assertEquals(Float.class, f4.getClass()); + Assert.assertEquals(3.1415f, f4); + + Object f5 = rs.getObject("f5"); + Assert.assertEquals(Double.class, f5.getClass()); + Assert.assertEquals(3.1415926, f5); + + Object f6 = rs.getObject("f6"); + Assert.assertEquals(byte[].class, f6.getClass()); + Assert.assertEquals("abc", new String((byte[]) f6)); + + Object f7 = rs.getObject("f7"); + Assert.assertEquals(Short.class, f7.getClass()); + Assert.assertEquals((short) 10, f7); + + Object f8 = rs.getObject("f8"); + Assert.assertEquals(Byte.class, f8.getClass()); + Assert.assertEquals((byte) 10, f8); + + Object f9 = rs.getObject("f9"); + Assert.assertEquals(Boolean.class, f9.getClass()); + Assert.assertEquals(true, f9); + + Object f10 = rs.getObject("f10"); + Assert.assertEquals(String.class, f10.getClass()); + Assert.assertEquals("涛思数据", f10); + } + + @Test(expected = SQLException.class) + public void findColumn() throws SQLException { + int columnIndex = rs.findColumn("f1"); + Assert.assertEquals(1, columnIndex); + columnIndex = rs.findColumn("f2"); + Assert.assertEquals(2, columnIndex); + columnIndex = rs.findColumn("f3"); + Assert.assertEquals(3, columnIndex); + columnIndex = rs.findColumn("f4"); + Assert.assertEquals(4, columnIndex); + columnIndex = rs.findColumn("f5"); + Assert.assertEquals(5, columnIndex); + columnIndex = rs.findColumn("f6"); + Assert.assertEquals(6, columnIndex); + columnIndex = rs.findColumn("f7"); + Assert.assertEquals(7, columnIndex); + columnIndex = rs.findColumn("f8"); + Assert.assertEquals(8, columnIndex); + columnIndex = rs.findColumn("f9"); + Assert.assertEquals(9, columnIndex); + columnIndex = rs.findColumn("f10"); + Assert.assertEquals(10, columnIndex); + + rs.findColumn("f11"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getCharacterStream() throws SQLException { + rs.getCharacterStream(1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void isBeforeFirst() throws SQLException { + rs.isBeforeFirst(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void isAfterLast() throws SQLException { + rs.isAfterLast(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void isFirst() throws SQLException { + rs.isFirst(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void isLast() throws SQLException { + rs.isLast(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void beforeFirst() throws SQLException { + rs.beforeFirst(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void afterLast() throws SQLException { + rs.afterLast(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void first() throws SQLException { + rs.first(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void last() throws SQLException { + rs.last(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getRow() throws SQLException { + int row = rs.getRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void absolute() throws SQLException { + rs.absolute(-1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void relative() throws SQLException { + rs.relative(-1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void previous() throws SQLException { + rs.previous(); + } + + @Test + public void setFetchDirection() throws SQLException { + rs.setFetchDirection(ResultSet.FETCH_FORWARD); + Assert.assertEquals(ResultSet.FETCH_FORWARD, rs.getFetchDirection()); + rs.setFetchDirection(ResultSet.FETCH_UNKNOWN); + Assert.assertEquals(ResultSet.FETCH_FORWARD, rs.getFetchDirection()); + } + + @Test + public void getFetchDirection() throws SQLException { + Assert.assertEquals(ResultSet.FETCH_FORWARD, rs.getFetchDirection()); + } + + @Test + public void setFetchSize() throws SQLException { + rs.setFetchSize(0); + Assert.assertEquals(0, rs.getFetchSize()); + } + + @Test + public void getFetchSize() throws SQLException { + Assert.assertEquals(0, rs.getFetchSize()); + } + + @Test + public void getType() throws SQLException { + Assert.assertEquals(ResultSet.TYPE_FORWARD_ONLY, rs.getType()); + } + + @Test + public void getConcurrency() throws SQLException { + Assert.assertEquals(ResultSet.CONCUR_READ_ONLY, rs.getConcurrency()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void rowUpdated() throws SQLException { + rs.rowUpdated(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void rowInserted() throws SQLException { + rs.rowInserted(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void rowDeleted() throws SQLException { + rs.rowDeleted(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateNull() throws SQLException { + rs.updateNull("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateBoolean() throws SQLException { + rs.updateBoolean(1, false); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateByte() throws SQLException { + rs.updateByte(1, new Byte("0")); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateShort() throws SQLException { + rs.updateShort(1, new Short("0")); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateInt() throws SQLException { + rs.updateInt(1, 1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateLong() throws SQLException { + rs.updateLong(1, 1l); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateFloat() throws SQLException { + rs.updateFloat(1, 1f); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateDouble() throws SQLException { + rs.updateDouble(1, 1.0); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateBigDecimal() throws SQLException { + rs.updateBigDecimal(1, new BigDecimal(1)); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateString() throws SQLException { + rs.updateString(1, "abc"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateBytes() throws SQLException { + rs.updateBytes(1, new byte[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateDate() throws SQLException { + rs.updateDate(1, new Date(System.currentTimeMillis())); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateTime() throws SQLException { + rs.updateTime(1, new Time(System.currentTimeMillis())); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateTimestamp() throws SQLException { + rs.updateTimestamp(1, new Timestamp(System.currentTimeMillis())); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateAsciiStream() throws SQLException { + rs.updateAsciiStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateBinaryStream() throws SQLException { + rs.updateBinaryStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateCharacterStream() throws SQLException { + rs.updateCharacterStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateObject() throws SQLException { + rs.updateObject(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void insertRow() throws SQLException { + rs.insertRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateRow() throws SQLException { + rs.updateRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void deleteRow() throws SQLException { + rs.deleteRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void refreshRow() throws SQLException { + rs.refreshRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void cancelRowUpdates() throws SQLException { + rs.cancelRowUpdates(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void moveToInsertRow() throws SQLException { + rs.moveToInsertRow(); + } + + @Test + public void getStatement() throws SQLException { + Statement stmt = rs.getStatement(); + Assert.assertNotNull(stmt); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void moveToCurrentRow() throws SQLException { + rs.moveToCurrentRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getRef() throws SQLException { + rs.getRef(1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getBlob() throws SQLException { + rs.getBlob("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getClob() throws SQLException { + rs.getClob("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getArray() throws SQLException { + rs.getArray("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getURL() throws SQLException { + rs.getURL("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateRef() throws SQLException { + rs.updateRef("f1", null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateBlob() throws SQLException { + rs.updateBlob(1, (InputStream) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateClob() throws SQLException { + rs.updateClob(1, (Reader) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateArray() throws SQLException { + rs.updateArray(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getRowId() throws SQLException { + rs.getRowId("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateRowId() throws SQLException { + rs.updateRowId(1, null); + } + + @Test + public void getHoldability() throws SQLException { + Assert.assertEquals(ResultSet.HOLD_CURSORS_OVER_COMMIT, rs.getHoldability()); + } + + @Test + public void isClosed() throws SQLException { + Assert.assertFalse(rs.isClosed()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateNString() throws SQLException { + rs.updateNString(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateNClob() throws SQLException { + rs.updateNClob(1, (Reader) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getNClob() throws SQLException { + rs.getNClob("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getSQLXML() throws SQLException { + rs.getSQLXML("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateSQLXML() throws SQLException { + rs.updateSQLXML(1, null); + } + + @Test + public void getNString() throws SQLException { + String f10 = rs.getNString("f10"); + Assert.assertEquals("涛思数据", f10); + f10 = rs.getNString(10); + Assert.assertEquals("涛思数据", f10); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getNCharacterStream() throws SQLException { + rs.getNCharacterStream("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateNCharacterStream() throws SQLException { + rs.updateNCharacterStream(1, null); + } + + @Test + public void unwrap() throws SQLException { + TSDBResultSet unwrap = rs.unwrap(TSDBResultSet.class); + Assert.assertNotNull(unwrap); + } + + @Test + public void isWrapperFor() throws SQLException { + Assert.assertTrue(rs.isWrapperFor(TSDBResultSet.class)); + } + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"); + stmt = conn.createStatement(); + stmt.execute("create database if not exists restful_test"); + stmt.execute("use restful_test"); + stmt.execute("drop table if exists weather"); + stmt.execute("create table if not exists weather(f1 timestamp, f2 int, f3 bigint, f4 float, f5 double, f6 binary(64), f7 smallint, f8 tinyint, f9 bool, f10 nchar(64))"); + stmt.execute("insert into restful_test.weather values('2021-01-01 00:00:00.000', 1, 100, 3.1415, 3.1415926, 'abc', 10, 10, true, '涛思数据')"); + rs = stmt.executeQuery("select * from restful_test.weather"); + rs.next(); + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + + } + + @AfterClass + public static void afterClass() { + try { + if (rs != null) + rs.close(); + if (stmt != null) + stmt.close(); + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java new file mode 100644 index 0000000000..580b2ac1b5 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java @@ -0,0 +1,42 @@ +package com.taosdata.jdbc.cases; + +import com.taosdata.jdbc.TSDBDriver; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Properties; + +public class DriverAutoloadTest { + + private Properties properties; + private String host = "127.0.0.1"; + + @Test + public void testRestful() throws SQLException { +// Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + final String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; + Connection conn = DriverManager.getConnection(url, properties); + Assert.assertNotNull(conn); + } + + @Test + public void testJni() throws SQLException { + final String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; + Connection conn = DriverManager.getConnection(url, properties); + Assert.assertNotNull(conn); + } + + + @Before + public void before() { + properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + } + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java index 54ee8fd6eb..84149775c3 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java @@ -14,7 +14,6 @@ import java.util.Random; public class InsertDbwithoutUseDbTest { private static String host = "127.0.0.1"; - // private static String host = "master"; private static Properties properties; private static Random random = new Random(System.currentTimeMillis()); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java index d0e6306704..4ae2f36fe9 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java @@ -10,8 +10,8 @@ import java.util.Properties; @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class UnsignedNumberRestfulTest { + private static final String host = "127.0.0.1"; -// private static final String host = "master"; private static Connection conn; @Test diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/AuthenticationTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/AuthenticationTest.java index b0666989ba..a6fb6cfda0 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/AuthenticationTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/AuthenticationTest.java @@ -8,7 +8,6 @@ import java.sql.*; public class AuthenticationTest { private static final String host = "127.0.0.1"; - // private static final String host = "master"; private static final String user = "root"; private static final String password = "taos?data"; private Connection conn; diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java index 4e005d1291..abd60f5b63 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java @@ -12,7 +12,6 @@ import java.util.Properties; public class RestfulConnectionTest { private static final String host = "127.0.0.1"; - // private static final String host = "master"; private static Connection conn; diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaDataTest.java index 1991c17065..a052fbbdcb 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaDataTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaDataTest.java @@ -10,7 +10,6 @@ import java.sql.*; import java.util.Properties; public class RestfulDatabaseMetaDataTest { - // private static final String host = "master"; private static final String host = "127.0.0.1"; private static final String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; private static Connection connection; diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java index 451f5d4916..c8bb69d827 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java @@ -10,7 +10,6 @@ import java.util.Random; public class RestfulJDBCTest { private static final String host = "127.0.0.1"; -// private static final String host = "master"; private static Connection connection; private Random random = new Random(System.currentTimeMillis()); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulParameterMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulParameterMetaDataTest.java new file mode 100644 index 0000000000..8bb2532ce8 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulParameterMetaDataTest.java @@ -0,0 +1,194 @@ +package com.taosdata.jdbc.rs; + +import com.taosdata.jdbc.TSDBConstants; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.*; + +public class RestfulParameterMetaDataTest { + + private static final String host = "127.0.0.1"; + private static Connection conn; + private static final String sql_insert = "insert into t1 values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + private static PreparedStatement pstmt_insert; + private static final String sql_select = "select * from t1 where ts > ? and ts <= ? and f1 >= ?"; + private static PreparedStatement pstmt_select; + private static ParameterMetaData parameterMetaData_insert; + private static ParameterMetaData parameterMetaData_select; + + @Test + public void getParameterCount() throws SQLException { + Assert.assertEquals(10, parameterMetaData_insert.getParameterCount()); + } + + @Test + public void isNullable() throws SQLException { + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(1)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(2)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(3)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(4)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(5)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(6)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(7)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(8)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(9)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(10)); + } + + @Test + public void isSigned() throws SQLException { + Assert.assertEquals(false, parameterMetaData_insert.isSigned(1)); + Assert.assertEquals(true, parameterMetaData_insert.isSigned(2)); + Assert.assertEquals(true, parameterMetaData_insert.isSigned(3)); + Assert.assertEquals(true, parameterMetaData_insert.isSigned(4)); + Assert.assertEquals(true, parameterMetaData_insert.isSigned(5)); + Assert.assertEquals(true, parameterMetaData_insert.isSigned(6)); + Assert.assertEquals(true, parameterMetaData_insert.isSigned(7)); + Assert.assertEquals(false, parameterMetaData_insert.isSigned(8)); + Assert.assertEquals(false, parameterMetaData_insert.isSigned(9)); + Assert.assertEquals(false, parameterMetaData_insert.isSigned(10)); + } + + @Test + public void getPrecision() throws SQLException { + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(1)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(2)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(3)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(4)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(5)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(6)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(7)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(8)); + Assert.assertEquals(5, parameterMetaData_insert.getPrecision(9)); + Assert.assertEquals(5, parameterMetaData_insert.getPrecision(10)); + } + + @Test + public void getScale() throws SQLException { + Assert.assertEquals(0, parameterMetaData_insert.getScale(1)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(2)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(3)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(4)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(5)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(6)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(7)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(8)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(9)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(10)); + } + + @Test + public void getParameterType() throws SQLException { + Assert.assertEquals(Types.TIMESTAMP, parameterMetaData_insert.getParameterType(1)); + Assert.assertEquals(Types.INTEGER, parameterMetaData_insert.getParameterType(2)); + Assert.assertEquals(Types.BIGINT, parameterMetaData_insert.getParameterType(3)); + Assert.assertEquals(Types.FLOAT, parameterMetaData_insert.getParameterType(4)); + Assert.assertEquals(Types.DOUBLE, parameterMetaData_insert.getParameterType(5)); + Assert.assertEquals(Types.SMALLINT, parameterMetaData_insert.getParameterType(6)); + Assert.assertEquals(Types.TINYINT, parameterMetaData_insert.getParameterType(7)); + Assert.assertEquals(Types.BOOLEAN, parameterMetaData_insert.getParameterType(8)); + Assert.assertEquals(Types.BINARY, parameterMetaData_insert.getParameterType(9)); + Assert.assertEquals(Types.NCHAR, parameterMetaData_insert.getParameterType(10)); + } + + @Test + public void getParameterTypeName() throws SQLException { + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.TIMESTAMP), parameterMetaData_insert.getParameterTypeName(1)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.INTEGER), parameterMetaData_insert.getParameterTypeName(2)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.BIGINT), parameterMetaData_insert.getParameterTypeName(3)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.FLOAT), parameterMetaData_insert.getParameterTypeName(4)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.DOUBLE), parameterMetaData_insert.getParameterTypeName(5)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.SMALLINT), parameterMetaData_insert.getParameterTypeName(6)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.TINYINT), parameterMetaData_insert.getParameterTypeName(7)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.BOOLEAN), parameterMetaData_insert.getParameterTypeName(8)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.BINARY), parameterMetaData_insert.getParameterTypeName(9)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.NCHAR), parameterMetaData_insert.getParameterTypeName(10)); + } + + @Test + public void getParameterClassName() throws SQLException { + Assert.assertEquals(Timestamp.class.getName(), parameterMetaData_insert.getParameterClassName(1)); + Assert.assertEquals(Integer.class.getName(), parameterMetaData_insert.getParameterClassName(2)); + Assert.assertEquals(Long.class.getName(), parameterMetaData_insert.getParameterClassName(3)); + Assert.assertEquals(Float.class.getName(), parameterMetaData_insert.getParameterClassName(4)); + Assert.assertEquals(Double.class.getName(), parameterMetaData_insert.getParameterClassName(5)); + Assert.assertEquals(Short.class.getName(), parameterMetaData_insert.getParameterClassName(6)); + Assert.assertEquals(Byte.class.getName(), parameterMetaData_insert.getParameterClassName(7)); + Assert.assertEquals(Boolean.class.getName(), parameterMetaData_insert.getParameterClassName(8)); + Assert.assertEquals(byte[].class.getName(), parameterMetaData_insert.getParameterClassName(9)); + Assert.assertEquals(String.class.getName(), parameterMetaData_insert.getParameterClassName(10)); + } + + @Test + public void getParameterMode() throws SQLException { + for (int i = 1; i <= parameterMetaData_insert.getParameterCount(); i++) { + int parameterMode = parameterMetaData_insert.getParameterMode(i); + Assert.assertEquals(ParameterMetaData.parameterModeUnknown, parameterMode); + } + } + + @Test + public void unwrap() throws SQLException { + RestfulParameterMetaData unwrap = parameterMetaData_insert.unwrap(RestfulParameterMetaData.class); + Assert.assertNotNull(unwrap); + } + + @Test + public void isWrapperFor() throws SQLException { + Assert.assertTrue(parameterMetaData_insert.isWrapperFor(RestfulParameterMetaData.class)); + } + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"); + try (Statement stmt = conn.createStatement()) { + stmt.execute("drop database if exists test_pstmt"); + stmt.execute("create database if not exists test_pstmt"); + stmt.execute("use test_pstmt"); + stmt.execute("create table weather(ts timestamp, f1 int, f2 bigint, f3 float, f4 double, f5 smallint, f6 tinyint, f7 bool, f8 binary(64), f9 nchar(64)) tags(loc nchar(64))"); + stmt.execute("create table t1 using weather tags('beijing')"); + } + pstmt_insert = conn.prepareStatement(sql_insert); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(2, 111); + pstmt_insert.setObject(3, Long.MAX_VALUE); + pstmt_insert.setObject(4, 3.14159265354f); + pstmt_insert.setObject(5, Double.MAX_VALUE); + pstmt_insert.setObject(6, Short.MAX_VALUE); + pstmt_insert.setObject(7, Byte.MAX_VALUE); + pstmt_insert.setObject(8, true); + pstmt_insert.setObject(9, "hello".getBytes()); + pstmt_insert.setObject(10, "Hello"); + parameterMetaData_insert = pstmt_insert.getParameterMetaData(); + + pstmt_select = conn.prepareStatement(sql_select); + pstmt_select.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_select.setTimestamp(2, new Timestamp(System.currentTimeMillis() + 10000)); + pstmt_select.setInt(3, 0); + parameterMetaData_select = pstmt_select.getParameterMetaData(); + + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (pstmt_insert != null) + pstmt_insert.close(); + if (pstmt_select != null) + pstmt_select.close(); + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java index a3867c1b6e..094dff8c8d 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java @@ -5,11 +5,12 @@ import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import java.io.IOException; +import java.io.Serializable; import java.sql.*; public class RestfulPreparedStatementTest { private static final String host = "127.0.0.1"; - // private static final String host = "master"; private static Connection conn; private static final String sql_insert = "insert into t1 values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; private static PreparedStatement pstmt_insert; @@ -38,48 +39,73 @@ public class RestfulPreparedStatementTest { @Test public void executeUpdate() throws SQLException { pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); - pstmt_insert.setFloat(2, 3.14f); + pstmt_insert.setFloat(4, 3.14f); int result = pstmt_insert.executeUpdate(); Assert.assertEquals(1, result); } @Test public void setNull() throws SQLException { - pstmt_insert.setNull(2, Types.FLOAT); + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setNull(2, Types.INTEGER); + int result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); } @Test public void setBoolean() throws SQLException { - pstmt_insert.setBoolean(2, true); + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setBoolean(8, true); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); } - @Test(expected = SQLFeatureNotSupportedException.class) + @Test public void setByte() throws SQLException { - pstmt_insert.setByte(1, (byte) 0x001); + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setByte(7, (byte) 0x001); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); } @Test - public void setShort() { - + public void setShort() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setShort(6, (short) 2); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); } @Test - public void setInt() { - + public void setInt() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setInt(2, 10086); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); } @Test - public void setLong() { - + public void setLong() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setLong(3, Long.MAX_VALUE); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); } @Test - public void setFloat() { - + public void setFloat() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setFloat(4, 3.14f); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); } @Test - public void setDouble() { + public void setDouble() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setDouble(5, 3.14444); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); } @Test(expected = SQLFeatureNotSupportedException.class) @@ -88,12 +114,56 @@ public class RestfulPreparedStatementTest { } @Test - public void setString() { + public void setString() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setString(10, "aaaa"); + boolean execute = pstmt_insert.execute(); + Assert.assertFalse(execute); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setString(10, new Person("john", 33, true).toString()); + Assert.assertFalse(pstmt_insert.execute()); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setString(10, new Person("john", 33, true).toString().replaceAll("'", "\"")); + Assert.assertFalse(pstmt_insert.execute()); } - @Test(expected = SQLFeatureNotSupportedException.class) - public void setBytes() throws SQLException { - pstmt_insert.setBytes(1, new byte[]{}); + class Person implements Serializable { + String name; + int age; + boolean sex; + + public Person(String name, int age, boolean sex) { + this.name = name; + this.age = age; + this.sex = sex; + } + + @Override + public String toString() { + return "Person{" + + "name='" + name + '\'' + + ", age=" + age + + ", sex=" + sex + + '}'; + } + } + + @Test + public void setBytes() throws SQLException, IOException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + +// ByteArrayOutputStream baos = new ByteArrayOutputStream(); +// ObjectOutputStream oos = new ObjectOutputStream(baos); +// oos.writeObject(new Person("john", 33, true)); +// oos.flush(); +// byte[] bytes = baos.toByteArray(); +// pstmt_insert.setBytes(9, bytes); + + pstmt_insert.setBytes(9, new Person("john", 33, true).toString().getBytes()); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); } @Test(expected = SQLFeatureNotSupportedException.class) @@ -107,8 +177,10 @@ public class RestfulPreparedStatementTest { } @Test - public void setTimestamp() { - //TODO + public void setTimestamp() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); } @Test(expected = SQLFeatureNotSupportedException.class) @@ -122,24 +194,69 @@ public class RestfulPreparedStatementTest { } @Test - public void clearParameters() { - //TODO + public void clearParameters() throws SQLException { + pstmt_insert.clearParameters(); } @Test public void setObject() throws SQLException { - pstmt_insert.setObject(1, System.currentTimeMillis()); - //TODO + pstmt_insert.setObject(1, new Timestamp(System.currentTimeMillis())); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(2, 111); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(3, Long.MAX_VALUE); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(4, 3.14159265354f); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(5, Double.MAX_VALUE); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(6, Short.MAX_VALUE); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(7, Byte.MAX_VALUE); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(8, true); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(9, "hello".getBytes()); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(10, "Hello"); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); } @Test - public void execute() { - //TODO - } + public void execute() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); - @Test - public void addBatch() { - //TODO: + executeQuery(); } @Test(expected = SQLFeatureNotSupportedException.class) @@ -180,8 +297,8 @@ public class RestfulPreparedStatementTest { @Test public void getParameterMetaData() throws SQLException { ParameterMetaData parameterMetaData = pstmt_insert.getParameterMetaData(); - Assert.assertNull(parameterMetaData); - //TODO: + Assert.assertNotNull(parameterMetaData); + //TODO: modify the test case } @Test(expected = SQLFeatureNotSupportedException.class) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java index 13973d8b6b..c7fc812972 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java @@ -10,7 +10,6 @@ import java.sql.*; public class RestfulResultSetMetaDataTest { private static final String host = "127.0.0.1"; -// private static final String host = "master"; private static Connection conn; private static Statement stmt; diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java index a15b1964ea..d6b2a58127 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java @@ -1,5 +1,8 @@ package com.taosdata.jdbc.rs; +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import com.google.common.primitives.Shorts; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -9,11 +12,12 @@ import java.io.InputStream; import java.io.Reader; import java.math.BigDecimal; import java.sql.*; +import java.text.ParseException; +import java.text.SimpleDateFormat; public class RestfulResultSetTest { private static final String host = "127.0.0.1"; -// private static final String host = "master"; private static Connection conn; private static Statement stmt; @@ -88,24 +92,75 @@ public class RestfulResultSetTest { Assert.assertEquals(3.1415926, f5, 0.0); } - @Test(expected = SQLFeatureNotSupportedException.class) + @Test public void getBigDecimal() throws SQLException { - rs.getBigDecimal("f1"); + BigDecimal f1 = rs.getBigDecimal("f1"); + Assert.assertEquals(1609430400000l, f1.longValue()); + + BigDecimal f2 = rs.getBigDecimal("f2"); + Assert.assertEquals(1, f2.intValue()); + + BigDecimal f3 = rs.getBigDecimal("f3"); + Assert.assertEquals(100l, f3.longValue()); + + BigDecimal f4 = rs.getBigDecimal("f4"); + Assert.assertEquals(3.1415f, f4.floatValue(), 0.00000f); + + BigDecimal f5 = rs.getBigDecimal("f5"); + Assert.assertEquals(3.1415926, f5.doubleValue(), 0.0000000); + + BigDecimal f7 = rs.getBigDecimal("f7"); + Assert.assertEquals(10, f7.intValue()); + + BigDecimal f8 = rs.getBigDecimal("f8"); + Assert.assertEquals(10, f8.intValue()); } - @Test(expected = SQLFeatureNotSupportedException.class) + @Test public void getBytes() throws SQLException { - rs.getBytes("f1"); + byte[] f1 = rs.getBytes("f1"); + Assert.assertEquals("2021-01-01 00:00:00.0", new String(f1)); + + byte[] f2 = rs.getBytes("f2"); + Assert.assertEquals(1, Ints.fromByteArray(f2)); + + byte[] f3 = rs.getBytes("f3"); + Assert.assertEquals(100l, Longs.fromByteArray(f3)); + + byte[] f4 = rs.getBytes("f4"); + Assert.assertEquals(3.1415f, Float.valueOf(new String(f4)), 0.000000f); + + byte[] f5 = rs.getBytes("f5"); + Assert.assertEquals(3.1415926, Double.valueOf(new String(f5)), 0.000000f); + + byte[] f6 = rs.getBytes("f6"); + Assert.assertEquals("abc", new String(f6)); + + byte[] f7 = rs.getBytes("f7"); + Assert.assertEquals((short) 10, Shorts.fromByteArray(f7)); + + byte[] f8 = rs.getBytes("f8"); + Assert.assertEquals(1, f8.length); + Assert.assertEquals((byte) 10, f8[0]); + + byte[] f9 = rs.getBytes("f9"); + Assert.assertEquals("true", new String(f9)); + + byte[] f10 = rs.getBytes("f10"); + Assert.assertEquals("涛思数据", new String(f10)); } - @Test(expected = SQLFeatureNotSupportedException.class) - public void getDate() throws SQLException { - rs.getDate("f1"); + @Test + public void getDate() throws SQLException, ParseException { + Date f1 = rs.getDate("f1"); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + Assert.assertEquals(sdf.parse("2021-01-01"), f1); } - @Test(expected = SQLFeatureNotSupportedException.class) + @Test public void getTime() throws SQLException { - rs.getTime("f1"); + Time f1 = rs.getTime("f1"); + Assert.assertEquals("00:00:00", f1.toString()); } @Test @@ -152,9 +207,49 @@ public class RestfulResultSetTest { Assert.assertNotNull(meta); } - @Test(expected = SQLFeatureNotSupportedException.class) - public void getObject() throws SQLException { - rs.getObject("f1"); + @Test + public void getObject() throws SQLException, ParseException { + Object f1 = rs.getObject("f1"); + Assert.assertEquals(Timestamp.class, f1.getClass()); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss.sss"); + java.util.Date date = sdf.parse("2021-01-01 00:00:00.000"); + Assert.assertEquals(new Timestamp(date.getTime()), f1); + + Object f2 = rs.getObject("f2"); + Assert.assertEquals(Integer.class, f2.getClass()); + Assert.assertEquals(1, f2); + + Object f3 = rs.getObject("f3"); + Assert.assertEquals(Long.class, f3.getClass()); + Assert.assertEquals(100l, f3); + + Object f4 = rs.getObject("f4"); + Assert.assertEquals(Float.class, f4.getClass()); + Assert.assertEquals(3.1415f, f4); + + Object f5 = rs.getObject("f5"); + Assert.assertEquals(Double.class, f5.getClass()); + Assert.assertEquals(3.1415926, f5); + + Object f6 = rs.getObject("f6"); + Assert.assertEquals(byte[].class, f6.getClass()); + Assert.assertEquals("abc", new String((byte[]) f6)); + + Object f7 = rs.getObject("f7"); + Assert.assertEquals(Short.class, f7.getClass()); + Assert.assertEquals((short) 10, f7); + + Object f8 = rs.getObject("f8"); + Assert.assertEquals(Byte.class, f8.getClass()); + Assert.assertEquals((byte) 10, f8); + + Object f9 = rs.getObject("f9"); + Assert.assertEquals(Boolean.class, f9.getClass()); + Assert.assertEquals(true, f9); + + Object f10 = rs.getObject("f10"); + Assert.assertEquals(String.class, f10.getClass()); + Assert.assertEquals("涛思数据", f10); } @Test(expected = SQLException.class) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulStatementTest.java index 653e480413..1be32b502d 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulStatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulStatementTest.java @@ -12,7 +12,6 @@ import java.util.UUID; public class RestfulStatementTest { private static final String host = "127.0.0.1"; - // private static final String host = "master"; private static Connection conn; private static Statement stmt; diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java index fe4d04775d..4ad9826384 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java @@ -12,7 +12,6 @@ import java.sql.*; @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class SQLTest { private static final String host = "127.0.0.1"; -// private static final String host = "master"; private static Connection connection; @Test diff --git a/tests/examples/JDBC/JDBCDemo/pom.xml b/tests/examples/JDBC/JDBCDemo/pom.xml index 167e7e37ae..66e866a2d3 100644 --- a/tests/examples/JDBC/JDBCDemo/pom.xml +++ b/tests/examples/JDBC/JDBCDemo/pom.xml @@ -13,7 +13,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.20 + 2.0.22 From 8a54938d50b4b40a860565245686771d098af93f Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 29 Mar 2021 15:23:14 +0800 Subject: [PATCH 35/71] [TD-3581]: fix uniq/dnode/remove2.sim step3 --- tests/script/unique/dnode/remove2.sim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/script/unique/dnode/remove2.sim b/tests/script/unique/dnode/remove2.sim index f97e55164b..ff92ff7eb0 100644 --- a/tests/script/unique/dnode/remove2.sim +++ b/tests/script/unique/dnode/remove2.sim @@ -120,7 +120,8 @@ sql show dnodes print dnode1 openVnodes $data2_1 print dnode2 openVnodes $data2_2 print dnode3 openVnodes $data2_3 -if $data2_3 != 0 then +# wait dnode3 create first vgroup in dnode-status msg +if $data2_3 != 1 then goto step4 endi @@ -181,4 +182,4 @@ system sh/exec.sh -n dnode4 -s stop -x SIGINT system sh/exec.sh -n dnode5 -s stop -x SIGINT system sh/exec.sh -n dnode6 -s stop -x SIGINT system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode8 -s stop -x SIGINT From 5e7a7d31cf5cacf15e38406ad051647691e886b3 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 29 Mar 2021 16:09:42 +0800 Subject: [PATCH 36/71] [TD-3576] : taosdemo print query times. (#5609) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index fd4023964a..810138c4e4 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -1437,9 +1437,11 @@ static void printfQueryMeta() { printf("\n"); printf("specified table query info: \n"); printf("query interval: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.rate); - printf("query times: \033[33m%d\033[0m\n", g_args.query_times); + printf("top query times:\033[33m%d\033[0m\n", g_args.query_times); printf("concurrent: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.concurrent); printf("sqlCount: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.sqlCount); + printf("specified tbl query times:\n"); + printf(" \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.queryTimes); if (SUBSCRIBE_TEST == g_args.test_mode) { printf("mod: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.subscribeMode); @@ -1457,6 +1459,7 @@ static void printfQueryMeta() { printf("threadCnt: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.threadCnt); printf("childTblCount: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.childTblCount); printf("stable name: \033[33m%s\033[0m\n", g_queryInfo.superQueryInfo.sTblName); + printf("stb query times:\033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.queryTimes); if (SUBSCRIBE_TEST == g_args.test_mode) { printf("mod: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeMode); From 1ef3c552423e843cfeed3ec549155b1d1e05cd49 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 29 Mar 2021 17:03:40 +0800 Subject: [PATCH 37/71] [TD-3592]: fix dnode ID syntax in drop dnode --- src/mnode/src/mnodeDnode.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 304096f3ae..db03da4fe1 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -722,6 +722,10 @@ int32_t mnodeDropDnode(SDnodeObj *pDnode, void *pMsg) { static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) { SDnodeObj *pDnode = mnodeGetDnodeByEp(ep); if (pDnode == NULL) { + if (strspn(ep, "0123456789 ;") != strlen(ep)) { + return TSDB_CODE_MND_DNODE_NOT_EXIST; + } + int32_t dnodeId = (int32_t)strtol(ep, NULL, 10); pDnode = mnodeGetDnode(dnodeId); if (pDnode == NULL) { From fa6c851cea74661a946c381c3c09b06e638b156b Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 29 Mar 2021 17:31:27 +0800 Subject: [PATCH 38/71] [TD-3577] : taosdemo print stb insert interval. (#5610) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 810138c4e4..00a88a5bfd 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -1110,7 +1110,7 @@ static int printfInsertMeta() { printf("resultFile: \033[33m%s\033[0m\n", g_Dbs.resultFile); printf("thread num of insert data: \033[33m%d\033[0m\n", g_Dbs.threadCount); printf("thread num of create table: \033[33m%d\033[0m\n", g_Dbs.threadCountByCreateTbl); - printf("insert interval: \033[33m%d\033[0m\n", g_args.insert_interval); + printf("top insert interval: \033[33m%d\033[0m\n", g_args.insert_interval); printf("number of records per req: \033[33m%d\033[0m\n", g_args.num_of_RPR); printf("max sql length: \033[33m%d\033[0m\n", g_args.max_sql_len); @@ -1213,6 +1213,12 @@ static int printfInsertMeta() { } printf(" interlaceRows: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].interlaceRows); + + if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) { + printf(" stable insert interval: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].insertInterval); + } + printf(" disorderRange: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].disorderRange); printf(" disorderRatio: \033[33m%d\033[0m\n", @@ -1360,14 +1366,21 @@ static void printfInsertMetaToFile(FILE* fp) { fprintf(fp, " childTblExists: %s\n", "error"); } - fprintf(fp, " childTblCount: %d\n", g_Dbs.db[i].superTbls[j].childTblCount); - fprintf(fp, " childTblPrefix: %s\n", g_Dbs.db[i].superTbls[j].childTblPrefix); - fprintf(fp, " dataSource: %s\n", g_Dbs.db[i].superTbls[j].dataSource); - fprintf(fp, " insertMode: %s\n", g_Dbs.db[i].superTbls[j].insertMode); - fprintf(fp, " insertRows: %"PRId64"\n", g_Dbs.db[i].superTbls[j].insertRows); - fprintf(fp, " interlace rows: %d\n", g_Dbs.db[i].superTbls[j].interlaceRows); + fprintf(fp, " childTblCount: %d\n", + g_Dbs.db[i].superTbls[j].childTblCount); + fprintf(fp, " childTblPrefix: %s\n", + g_Dbs.db[i].superTbls[j].childTblPrefix); + fprintf(fp, " dataSource: %s\n", + g_Dbs.db[i].superTbls[j].dataSource); + fprintf(fp, " insertMode: %s\n", + g_Dbs.db[i].superTbls[j].insertMode); + fprintf(fp, " insertRows: %"PRId64"\n", + g_Dbs.db[i].superTbls[j].insertRows); + fprintf(fp, " interlace rows: %d\n", + g_Dbs.db[i].superTbls[j].interlaceRows); if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) { - fprintf(fp, " insert interval: %d\n", g_Dbs.db[i].superTbls[j].insertInterval); + fprintf(fp, " stable insert interval: %d\n", + g_Dbs.db[i].superTbls[j].insertInterval); } if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { @@ -1375,7 +1388,8 @@ static void printfInsertMetaToFile(FILE* fp) { }else { fprintf(fp, " multiThreadWriteOneTbl: yes\n"); } - fprintf(fp, " interlaceRows: %d\n", g_Dbs.db[i].superTbls[j].interlaceRows); + fprintf(fp, " interlaceRows: %d\n", + g_Dbs.db[i].superTbls[j].interlaceRows); fprintf(fp, " disorderRange: %d\n", g_Dbs.db[i].superTbls[j].disorderRange); fprintf(fp, " disorderRatio: %d\n", g_Dbs.db[i].superTbls[j].disorderRatio); fprintf(fp, " maxSqlLen: %d\n", g_Dbs.db[i].superTbls[j].maxSqlLen); @@ -3655,7 +3669,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (insertInterval && insertInterval->type == cJSON_Number) { g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint; } else if (!insertInterval) { - debugPrint("%s() LN%d: stable insert interval be overrided by global %d.\n", + verbosePrint("%s() LN%d: stable insert interval be overrided by global %d.\n", __func__, __LINE__, g_args.insert_interval); g_Dbs.db[i].superTbls[j].insertInterval = g_args.insert_interval; } else { From 78e67ebe7c5c705b8719a0546784f951abbee6bb Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 30 Mar 2021 00:35:01 +0800 Subject: [PATCH 39/71] [TD-3589] : taosdemo need to use db before query. (#5612) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 00a88a5bfd..fb42b6f6d6 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -5526,9 +5526,14 @@ static void *superQueryProcess(void *sarg) { } } - //char sqlStr[MAX_TB_NAME_SIZE*2]; - //sprintf(sqlStr, "use %s", g_queryInfo.dbName); - //queryDB(winfo->taos, sqlStr); + char sqlStr[MAX_DB_NAME_SIZE + 5]; + sprintf(sqlStr, "use %s", g_queryInfo.dbName); + if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE, false)) { + taos_close(winfo->taos); + errorPrint( "use database %s failed!\n\n", + g_queryInfo.dbName); + return NULL; + } int64_t st = 0; int64_t et = 0; @@ -5861,9 +5866,10 @@ static void *subSubscribeProcess(void *sarg) { char sqlStr[MAX_TB_NAME_SIZE*2]; sprintf(sqlStr, "use %s", g_queryInfo.dbName); - debugPrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr); if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE, false)) { taos_close(winfo->taos); + errorPrint( "use database %s failed!\n\n", + g_queryInfo.dbName); return NULL; } From c13f335fcfbb19a227df2d95073cea695d947abb Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 30 Mar 2021 00:35:34 +0800 Subject: [PATCH 40/71] [TD-3591] : rework limit and offset logic. (#5618) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 91 +++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 45 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index fb42b6f6d6..f4009459a7 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -3118,7 +3118,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_args.interlace_rows, g_args.num_of_RPR); printf(" interlace rows value will be set to num_of_records_per_request %d\n\n", g_args.num_of_RPR); - printf(" press Enter key to continue or Ctrl+C to stop."); + printf(" press Enter key to continue or Ctrl-C to stop."); (void)getchar(); g_args.interlace_rows = g_args.num_of_RPR; } @@ -3621,7 +3621,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { i, j, g_Dbs.db[i].superTbls[j].interlaceRows, g_args.num_of_RPR); printf(" interlace rows value will be set to num_of_records_per_request %d\n\n", g_args.num_of_RPR); - printf(" press Enter key to continue or Ctrl+C to stop."); + printf(" press Enter key to continue or Ctrl-C to stop."); (void)getchar(); g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR; } @@ -5008,39 +5008,14 @@ static void *asyncWrite(void *sarg) { static void startMultiThreadInsertData(int threads, char* db_name, char* precision,SSuperTable* superTblInfo) { - pthread_t *pids = malloc(threads * sizeof(pthread_t)); - assert(pids != NULL); + pthread_t *pids = malloc(threads * sizeof(pthread_t)); + assert(pids != NULL); - threadInfo *infos = malloc(threads * sizeof(threadInfo)); - assert(infos != NULL); + threadInfo *infos = malloc(threads * sizeof(threadInfo)); + assert(infos != NULL); - memset(pids, 0, threads * sizeof(pthread_t)); - memset(infos, 0, threads * sizeof(threadInfo)); - - int ntables = 0; - if (superTblInfo) { - - if ((superTblInfo->childTblOffset >= 0) - && (superTblInfo->childTblLimit > 0)) { - - ntables = superTblInfo->childTblLimit; - } else { - ntables = superTblInfo->childTblCount; - } - } else { - ntables = g_args.num_of_tables; - } - - int a = ntables / threads; - if (a < 1) { - threads = ntables; - a = 1; - } - - int b = 0; - if (threads != 0) { - b = ntables % threads; - } + memset(pids, 0, threads * sizeof(pthread_t)); + memset(infos, 0, threads * sizeof(threadInfo)); //TAOS* taos; //if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { @@ -5082,13 +5057,6 @@ static void startMultiThreadInsertData(int threads, char* db_name, double start = getCurrentTime(); - int startFrom; - - if ((superTblInfo) && (superTblInfo->childTblOffset >= 0)) - startFrom = superTblInfo->childTblOffset; - else - startFrom = 0; - // read sample data from file first if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource, "sample", strlen("sample")))) { @@ -5118,18 +5086,36 @@ static void startMultiThreadInsertData(int threads, char* db_name, exit(-1); } - if (superTblInfo) { + int ntables = 0; + int startFrom; + if (superTblInfo) { int limit, offset; - if (superTblInfo && (superTblInfo->childTblOffset >= 0) - && (superTblInfo->childTblLimit > 0)) { - limit = superTblInfo->childTblLimit; - offset = superTblInfo->childTblOffset; + + if (superTblInfo->childTblOffset >= superTblInfo->childTblCount) { + printf("WARNING: specified offset >= child table count! \n"); + if (!g_args.answer_yes) { + printf(" Press enter key to continue or Ctrl-C to stop\n\n"); + (void)getchar(); + } + } + + if (superTblInfo->childTblOffset >= 0) { + if (superTblInfo->childTblLimit <= 0) { + superTblInfo->childTblLimit = + superTblInfo->childTblCount - superTblInfo->childTblOffset; + } + + offset = superTblInfo->childTblOffset; + limit = superTblInfo->childTblLimit; } else { limit = superTblInfo->childTblCount; offset = 0; } + ntables = limit; + startFrom = offset; + superTblInfo->childTblName = (char*)calloc(1, limit * TSDB_TABLE_NAME_LEN); if (superTblInfo->childTblName == NULL) { @@ -5145,9 +5131,24 @@ static void startMultiThreadInsertData(int threads, char* db_name, &superTblInfo->childTblName, &childTblCount, limit, offset); + } else { + ntables = g_args.num_of_tables; + startFrom = 0; } + taos_close(taos); + int a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } + + int b = 0; + if (threads != 0) { + b = ntables % threads; + } + for (int i = 0; i < threads; i++) { threadInfo *t_info = infos + i; t_info->threadID = i; From 98488bb931590cabb22dd74901e3b9b3643de762 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Tue, 30 Mar 2021 10:37:44 +0800 Subject: [PATCH 41/71] [TD-2639] : add descriptions about database options that can be altered. --- documentation20/cn/11.administrator/docs.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/documentation20/cn/11.administrator/docs.md b/documentation20/cn/11.administrator/docs.md index c2c2927387..cc8689786d 100644 --- a/documentation20/cn/11.administrator/docs.md +++ b/documentation20/cn/11.administrator/docs.md @@ -120,17 +120,17 @@ taosd -C 不同应用场景的数据往往具有不同的数据特征，比如保留天数、副本数、采集频次、记录大小、采集点的数量、压缩等都可完全不同。为获得在存储上的最高效率，TDengine提供如下存储相关的系统配置参数： - days：一个数据文件存储数据的时间跨度，单位为天，默认值：10。 -- keep：数据库中数据保留的天数，单位为天，默认值：3650。 +- keep：数据库中数据保留的天数，单位为天，默认值：3650。（可通过 alter database 修改） - minRows：文件块中记录的最小条数，单位为条，默认值：100。 - maxRows：文件块中记录的最大条数，单位为条，默认值：4096。 -- comp：文件压缩标志位，0：关闭；1：一阶段压缩；2：两阶段压缩。默认值：2。 +- comp：文件压缩标志位，0：关闭；1：一阶段压缩；2：两阶段压缩。默认值：2。（可通过 alter database 修改） - walLevel：WAL级别。1：写wal，但不执行fsync；2：写wal, 而且执行fsync。默认值：1。 - fsync：当wal设置为2时，执行fsync的周期。设置为0，表示每次写入，立即执行fsync。单位为毫秒，默认值：3000。 - cache：内存块的大小，单位为兆字节（MB），默认值：16。 -- blocks：每个VNODE（TSDB）中有多少cache大小的内存块。因此一个VNODE的用的内存大小粗略为（cache * blocks）。单位为块，默认值：4。 -- replica：副本个数，取值范围：1-3。单位为个，默认值：1 -- precision：时间戳精度标识，ms表示毫秒，us表示微秒。默认值：ms -- cacheLast：是否在内存中缓存子表 last_row，0：关闭；1：开启。默认值：0。（从 2.0.11 版本开始支持此参数） +- blocks：每个VNODE（TSDB）中有多少cache大小的内存块。因此一个VNODE的用的内存大小粗略为（cache * blocks）。单位为块，默认值：4。（可通过 alter database 修改） +- replica：副本个数，取值范围：1-3。单位为个，默认值：1。（可通过 alter database 修改） +- precision：时间戳精度标识，ms表示毫秒，us表示微秒。默认值：ms。 +- cacheLast：是否在内存中缓存子表 last_row，0：关闭；1：开启。默认值：0。（可通过 alter database 修改）（从 2.0.11 版本开始支持此参数） 对于一个应用场景，可能有多种数据特征的数据并存，最佳的设计是将具有相同数据特征的表放在一个库里，这样一个应用有多个库，而每个库可以配置不同的存储参数，从而保证系统有最优的性能。TDengine允许应用在创建库时指定上述存储参数，如果指定，该参数就将覆盖对应的系统配置参数。举例，有下述SQL： From 77e367564005b5b7f14cf752010ff4817444b078 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 30 Mar 2021 12:51:02 +0800 Subject: [PATCH 42/71] [TD-3598] : cleanup few code lines. (#5622) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 66 ++++++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 23 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index f4009459a7..3d96004fa7 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -1194,17 +1194,24 @@ static int printfInsertMeta() { printf(" childTblExists: \033[33m%s\033[0m\n", "error"); } - printf(" childTblCount: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].childTblCount); - printf(" childTblPrefix: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].childTblPrefix); - printf(" dataSource: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].dataSource); - printf(" insertMode: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].insertMode); + printf(" childTblCount: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].childTblCount); + printf(" childTblPrefix: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].childTblPrefix); + printf(" dataSource: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].dataSource); + printf(" insertMode: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].insertMode); if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) { - printf(" childTblLimit: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].childTblLimit); + printf(" childTblLimit: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].childTblLimit); } if (g_Dbs.db[i].superTbls[j].childTblOffset >= 0) { - printf(" childTblOffset: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].childTblOffset); + printf(" childTblOffset: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].childTblOffset); } - printf(" insertRows: \033[33m%"PRId64"\033[0m\n", g_Dbs.db[i].superTbls[j].insertRows); + printf(" insertRows: \033[33m%"PRId64"\033[0m\n", + g_Dbs.db[i].superTbls[j].insertRows); if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { printf(" multiThreadWriteOneTbl: \033[33mno\033[0m\n"); @@ -1257,8 +1264,10 @@ static int printfInsertMeta() { g_Dbs.db[i].superTbls[j].tagCount); for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) { //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); - if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "binary", 6)) - || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "nchar", 5))) { + if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, + "binary", strlen("binary"))) + || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, + "nchar", strlen("nchar")))) { printf("tag[%d]:\033[33m%s(%d)\033[0m ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); @@ -1630,7 +1639,10 @@ static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) { while ((row = taos_fetch_row(res)) != NULL) { // sys database name : 'log' - if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log", fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) continue; + if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log", + fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) { + continue; + } dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo)); if (dbInfos[count] == NULL) { @@ -1677,7 +1689,8 @@ static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) { return count; } -static void printfDbInfoForQueryToFile(char* filename, SDbInfo* dbInfos, int index) { +static void printfDbInfoForQueryToFile( + char* filename, SDbInfo* dbInfos, int index) { if (filename[0] == 0) return; @@ -1869,7 +1882,8 @@ static int postProceSql(char* host, uint16_t port, char* sqlstr) for (int l = 0; l < mod_table[userpass_buf_len % 3]; l++) base64_buf[encoded_len - 1 - l] = '='; - debugPrint("%s() LN%d: auth string base64 encoded: %s\n", __func__, __LINE__, base64_buf); + debugPrint("%s() LN%d: auth string base64 encoded: %s\n", + __func__, __LINE__, base64_buf); char *auth = base64_buf; int r = snprintf(request_buf, @@ -2108,7 +2122,8 @@ static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, } //get all child table name use cmd: select tbname from superTblName; - snprintf(command, BUFFER_SIZE, "select tbname from %s.%s %s", dbName, sTblName, limitBuf); + snprintf(command, BUFFER_SIZE, "select tbname from %s.%s %s", + dbName, sTblName, limitBuf); res = taos_query(taos, command); int32_t code = taos_errno(res); @@ -2256,7 +2271,8 @@ static int getSuperTableFromServer(TAOS * taos, char* dbName, return 0; } -static int createSuperTable(TAOS * taos, char* dbName, SSuperTable* superTbls, bool use_metric) { +static int createSuperTable(TAOS * taos, char* dbName, + SSuperTable* superTbls, bool use_metric) { char command[BUFFER_SIZE] = "\0"; char cols[STRING_LEN] = "\0"; @@ -2907,7 +2923,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( superTbls->tagCount = 0; return true; } - + int columnSize = cJSON_GetArraySize(columns); if (columnSize > MAX_COLUMN_COUNT) { errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n", @@ -2963,7 +2979,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( } } superTbls->columnCount = index; - + count = 1; index = 0; // tags @@ -3364,22 +3380,24 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!fsync) { g_Dbs.db[i].dbCfg.fsync = -1; } else { - printf("ERROR: failed to read json, fsync not found\n"); - goto PARSE_OVER; + errorPrint("%s() LN%d, failed to read json, fsync input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; } // super_talbes cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables"); if (!stables || stables->type != cJSON_Array) { - printf("ERROR: failed to read json, super_tables not found\n"); + errorPrint("%s() LN%d, failed to read json, super_tables not found\n", + __func__, __LINE__); goto PARSE_OVER; } int stbSize = cJSON_GetArraySize(stables); if (stbSize > MAX_SUPER_TABLE_COUNT) { errorPrint( - "ERROR: failed to read json, databases size overflow, max database is %d\n", - MAX_SUPER_TABLE_COUNT); + "%s() LN%d, failed to read json, supertable size overflow, max supertable is %d\n", + __func__, __LINE__, MAX_SUPER_TABLE_COUNT); goto PARSE_OVER; } @@ -3391,7 +3409,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { // dbinfo cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name"); if (!stbName || stbName->type != cJSON_String || stbName->valuestring == NULL) { - printf("ERROR: failed to read json, stb name not found\n"); + errorPrint("%s() LN%d, failed to read json, stb name not found\n", + __func__, __LINE__); goto PARSE_OVER; } tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, MAX_TB_NAME_SIZE); @@ -4351,7 +4370,8 @@ static void getTableName(char *pTblName, threadInfo* pThreadInfo, int tableSeq) if ((superTblInfo->childTblOffset >= 0) && (superTblInfo->childTblLimit > 0)) { snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s", - superTblInfo->childTblName + (tableSeq - superTblInfo->childTblOffset) * TSDB_TABLE_NAME_LEN); + superTblInfo->childTblName + + (tableSeq - superTblInfo->childTblOffset) * TSDB_TABLE_NAME_LEN); } else { verbosePrint("[%d] %s() LN%d: from=%d count=%d seq=%d\n", From 189058e7dfc0dbc699f76bf897c39ab7de7ea86e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 30 Mar 2021 14:32:14 +0800 Subject: [PATCH 43/71] update file header on start --- src/tsdb/src/tsdbFS.c | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/tsdb/src/tsdbFS.c b/src/tsdb/src/tsdbFS.c index a9e74cb229..59fdfa1952 100644 --- a/src/tsdb/src/tsdbFS.c +++ b/src/tsdb/src/tsdbFS.c @@ -33,6 +33,7 @@ static int tsdbScanDataDir(STsdbRepo *pRepo); static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf); static int tsdbRestoreCurrent(STsdbRepo *pRepo); static int tsdbComparTFILE(const void *arg1, const void *arg2); +static void tsdbScanAndTryFixDFilesHeader(STsdbRepo *pRepo); // ================== CURRENT file header info static int tsdbEncodeFSHeader(void **buf, SFSHeader *pHeader) { @@ -246,6 +247,8 @@ int tsdbOpenFS(STsdbRepo *pRepo) { tsdbError("vgId:%d failed to open FS since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; } + + tsdbScanAndTryFixDFilesHeader(pRepo); } else { if (tsdbRestoreCurrent(pRepo) < 0) { tsdbError("vgId:%d failed to restore current file since %s", REPO_ID(pRepo), tstrerror(terrno)); @@ -1201,4 +1204,40 @@ static int tsdbComparTFILE(const void *arg1, const void *arg2) { return 0; } } +} + +static void tsdbScanAndTryFixDFilesHeader(STsdbRepo *pRepo) { + STsdbFS * pfs = REPO_FS(pRepo); + SFSStatus *pStatus = pfs->cstatus; + SDFInfo info; + + for (size_t i = 0; i < taosArrayGetSize(pStatus->df); i++) { + SDFileSet fset; + tsdbInitDFileSetEx(&fset, (SDFileSet *)taosArrayGet(pStatus->df, i)); + + tsdbDebug("vgId:%d scan DFileSet %d header", REPO_ID(pRepo), fset.fid); + + if (tsdbOpenDFileSet(&fset, O_RDWR) < 0) { + tsdbError("vgId:%d failed to open DFileSet %d since %s, continue", REPO_ID(pRepo), fset.fid, tstrerror(terrno)); + continue; + } + + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + SDFile *pDFile = TSDB_DFILE_IN_SET(&fset, ftype); + + if (tsdbLoadDFileHeader(pDFile, &info) < 0 || memcmp(&(pDFile->info), &info, sizeof(info)) != 0) { + if (tsdbUpdateDFileHeader(pDFile) < 0) { + tsdbError("vgId:%d failed to update DFile header of %s since %s, continue", REPO_ID(pRepo), + TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno)); + } else { + tsdbInfo("vgId:%d DFile header of %s is updated", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile)); + TSDB_FILE_FSYNC(pDFile); + } + } else { + tsdbDebug("vgId:%d DFile header of %s is correct", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile)); + } + } + + tsdbCloseDFileSet(&fset); + } } \ No newline at end of file From 1a8d62834f01b392d3bf7a8c93fa3606b3864919 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 30 Mar 2021 14:32:45 +0800 Subject: [PATCH 44/71] [TD-3602]: update perf test script --- tests/perftest-scripts/perftest-query.sh | 8 +-- tests/pytest/tools/taosdemoPerformance.py | 65 ++++++++++++++++++++++- 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/tests/perftest-scripts/perftest-query.sh b/tests/perftest-scripts/perftest-query.sh index e43a45ed5e..38cfb3ca19 100755 --- a/tests/perftest-scripts/perftest-query.sh +++ b/tests/perftest-scripts/perftest-query.sh @@ -50,7 +50,7 @@ function buildTDengine { echo "repo up-to-date" else echo "repo need to pull" - git pull > /dev/null + git pull > /dev/null 2>&1 LOCAL_COMMIT=`git rev-parse --short @` cd debug @@ -72,11 +72,11 @@ function runQueryPerfTest { python3 insert/insertFromCSVPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT - yes | taosdemo -c /etc/taosperf/ -d taosdemo_insert_test -x > taosdemoperf.txt + taosdemo -f /home/ubuntu/pxiao/insert.json > taosdemoperf.txt CREATETABLETIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'` INSERTRECORDSTIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'` - REQUESTSPERSECOND=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $13}'` + REQUESTSPERSECOND=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $16}'` delay=`grep 'delay' taosdemoperf.txt | awk '{print $4}'` AVGDELAY=`echo ${delay:0:${#delay}-3}` delay=`grep 'delay' taosdemoperf.txt | awk '{print $6}'` @@ -91,7 +91,7 @@ function runQueryPerfTest { function sendReport { echo "send report" - receiver="pxiao@taosdata.com" + receiver="develop@taosdata.com" mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n" cd $TDENGINE_DIR diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py index 28f451b6a0..a337ced4e6 100644 --- a/tests/pytest/tools/taosdemoPerformance.py +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -39,8 +39,69 @@ class taosdemoPerformace: self.host, self.user, self.password, - self.config) + self.config) + self.insertDB = "insertDB"; + def generateJson(self): + db = { + "name": "%s" % self.insertDB, + "drop": "yes", + "replica": 1 + } + + stb = { + "name": "meters", + "child_table_exists":"no", + "childtable_count": 10000, + "childtable_prefix": "stb_", + "auto_create_table": "no", + "data_source": "rand", + "batch_create_tbl_num": 10, + "insert_mode": "taosc", + "insert_rows": 100000, + "multi_thread_write_one_tbl": "no", + "number_of_tbl_in_one_sql": 0, + "rows_per_tbl": 100, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{ + "columns": [{"type": "INT", "count": 4}], + }], + "tags": [{"type": "INT", "count":1}, {"type": "BINARY", "len": 16}] + } + + stables = [] + stables.append(stb) + + db = { + "dbinfo": db, + "super_tables": stables + } + + insert_data = { + "filetype": "insert", + "cfgdir": "/etc/taosperf", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 10, + "thread_count_create_tbl": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 30000, + "databases": [db] + } + + return insert_data + def createTablesAndStoreData(self): cursor = self.conn.cursor() @@ -55,7 +116,7 @@ class taosdemoPerformace: print("max delay: %f" % self.maxDelay) print("min delay: %f" % self.minDelay) cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f)" % (self.createTableTime, self.insertRecordsTime, self.recordsPerSecond, self.commitID, self.avgDelay, self.maxDelay, self.minDelay)) - cursor.execute("drop database if exists taosdemo_insert_test") + cursor.execute("drop database if exists %s" % self.insertDB) cursor.close() From 9353aed0685c7f8c238a740688ac2f0c7d979d2b Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 30 Mar 2021 15:57:14 +0800 Subject: [PATCH 45/71] [TD-3604]subscribe hanlder overwrite --- src/kit/taosdemo/taosdemo.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 3d96004fa7..7b0668b82d 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -5868,6 +5868,7 @@ static TAOS_SUB* subscribeImpl(TAOS *taos, char *sql, char* topic, char* resultF static void *subSubscribeProcess(void *sarg) { threadInfo *winfo = (threadInfo *)sarg; char subSqlstr[1024]; + TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT] = {0}; if (winfo->taos == NULL) { TAOS * taos = NULL; @@ -5913,9 +5914,8 @@ static void *subSubscribeProcess(void *sarg) { sprintf(tmpFile, "%s-%d", g_queryInfo.superQueryInfo.result[i], winfo->threadID); } - g_queryInfo.superQueryInfo.tsub[i] = subscribeImpl( - winfo->taos, subSqlstr, topic, tmpFile); - if (NULL == g_queryInfo.superQueryInfo.tsub[i]) { + tsub[i] = subscribeImpl(winfo->taos, subSqlstr, topic, tmpFile); + if (NULL == tsub[i]) { taos_close(winfo->taos); return NULL; } @@ -5932,7 +5932,7 @@ static void *subSubscribeProcess(void *sarg) { continue; } - res = taos_consume(g_queryInfo.superQueryInfo.tsub[i]); + res = taos_consume(tsub[i]); if (res) { char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.superQueryInfo.result[i][0] != 0) { @@ -5947,8 +5947,7 @@ static void *subSubscribeProcess(void *sarg) { taos_free_result(res); for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - taos_unsubscribe(g_queryInfo.superQueryInfo.tsub[i], - g_queryInfo.superQueryInfo.subscribeKeepProgress); + taos_unsubscribe(tsub[i], g_queryInfo.superQueryInfo.subscribeKeepProgress); } taos_close(winfo->taos); @@ -5957,6 +5956,7 @@ static void *subSubscribeProcess(void *sarg) { static void *superSubscribeProcess(void *sarg) { threadInfo *winfo = (threadInfo *)sarg; + TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT] = {0}; if (winfo->taos == NULL) { TAOS * taos = NULL; @@ -5999,10 +5999,7 @@ static void *superSubscribeProcess(void *sarg) { sprintf(tmpFile, "%s-%d", g_queryInfo.specifiedQueryInfo.result[i], winfo->threadID); } - g_queryInfo.specifiedQueryInfo.tsub[i] = - subscribeImpl(winfo->taos, - g_queryInfo.specifiedQueryInfo.sql[i], - topic, tmpFile); + tsub[i] = subscribeImpl(winfo->taos, g_queryInfo.specifiedQueryInfo.sql[i], topic, tmpFile); if (NULL == g_queryInfo.specifiedQueryInfo.tsub[i]) { taos_close(winfo->taos); return NULL; @@ -6020,7 +6017,7 @@ static void *superSubscribeProcess(void *sarg) { continue; } - res = taos_consume(g_queryInfo.specifiedQueryInfo.tsub[i]); + res = taos_consume(tsub[i]); if (res) { char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { @@ -6034,8 +6031,7 @@ static void *superSubscribeProcess(void *sarg) { taos_free_result(res); for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { - taos_unsubscribe(g_queryInfo.specifiedQueryInfo.tsub[i], - g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); + taos_unsubscribe(tsub[i], g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); } taos_close(winfo->taos); From 5e82ec59884e5beb0b024017504e932790fda621 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Tue, 30 Mar 2021 15:58:14 +0800 Subject: [PATCH 46/71] fix memleak issue --- src/client/inc/tsclient.h | 2 ++ src/client/src/tscStream.c | 1 + src/client/src/tscUtil.c | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 0e20c66305..f233af35a3 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -439,6 +439,8 @@ void tscFreeSqlResult(SSqlObj *pSql); * @param pObj */ void tscFreeSqlObj(SSqlObj *pSql); +void tscFreeSubobj(SSqlObj* pSql); + void tscFreeRegisteredSqlObj(void *pSql); void tscCloseTscObj(void *pObj); diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index a9cd1965e8..7699e6f459 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -299,6 +299,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf tfree(pTableMetaInfo->pTableMeta); tscFreeSqlResult(pSql); + tscFreeSubobj(pSql); tfree(pSql->pSubs); pSql->subState.numOfSub = 0; pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 8d320c9a83..ad36246461 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -447,7 +447,7 @@ void tscFreeSqlResult(SSqlObj* pSql) { memset(&pSql->res, 0, sizeof(SSqlRes)); } -static void tscFreeSubobj(SSqlObj* pSql) { +void tscFreeSubobj(SSqlObj* pSql) { if (pSql->subState.numOfSub == 0) { return; } From f447556178bd7d76038adf7d591ee0a01bf517c9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 30 Mar 2021 16:01:33 +0800 Subject: [PATCH 47/71] fix more --- src/tsdb/src/tsdbFS.c | 3 +- src/tsdb/src/tsdbFile.c | 1 + src/tsdb/src/tsdbRead.c | 8 ++- src/tsdb/src/tsdbSync.c | 6 +- tests/script/data2.sim | 135 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 147 insertions(+), 6 deletions(-) create mode 100644 tests/script/data2.sim diff --git a/src/tsdb/src/tsdbFS.c b/src/tsdb/src/tsdbFS.c index 59fdfa1952..c9f087a5cf 100644 --- a/src/tsdb/src/tsdbFS.c +++ b/src/tsdb/src/tsdbFS.c @@ -1225,7 +1225,8 @@ static void tsdbScanAndTryFixDFilesHeader(STsdbRepo *pRepo) { for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { SDFile *pDFile = TSDB_DFILE_IN_SET(&fset, ftype); - if (tsdbLoadDFileHeader(pDFile, &info) < 0 || memcmp(&(pDFile->info), &info, sizeof(info)) != 0) { + if ((tsdbLoadDFileHeader(pDFile, &info) < 0) || pDFile->info.size != info.size || + pDFile->info.magic != info.magic) { if (tsdbUpdateDFileHeader(pDFile) < 0) { tsdbError("vgId:%d failed to update DFile header of %s since %s, continue", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno)); diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 304ba2f073..5db993e463 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -567,6 +567,7 @@ void tsdbInitDFileSet(SDFileSet *pSet, SDiskID did, int vid, int fid, uint32_t v } void tsdbInitDFileSetEx(SDFileSet *pSet, SDFileSet *pOSet) { + pSet->fid = pOSet->fid; for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { tsdbInitDFileEx(TSDB_DFILE_IN_SET(pSet, ftype), TSDB_DFILE_IN_SET(pOSet, ftype)); } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 6c0137abf5..ea72760568 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -917,7 +917,9 @@ static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int pCheckInfo->compSize = compIndex->len; } - tsdbLoadBlockInfo(&(pQueryHandle->rhelper), (void*)(pCheckInfo->pCompInfo)); + if (tsdbLoadBlockInfo(&(pQueryHandle->rhelper), (void*)(pCheckInfo->pCompInfo)) < 0) { + return terrno; + } SBlockInfo* pCompInfo = pCheckInfo->pCompInfo; TSKEY s = TSKEY_INITIAL_VAL, e = TSKEY_INITIAL_VAL; @@ -2832,7 +2834,9 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta } int64_t stime = taosGetTimestampUs(); - tsdbLoadBlockStatis(&pHandle->rhelper, pBlockInfo->compBlock); + if (tsdbLoadBlockStatis(&pHandle->rhelper, pBlockInfo->compBlock) < 0) { + return terrno; + } int16_t* colIds = pHandle->defaultLoadColumn->pData; diff --git a/src/tsdb/src/tsdbSync.c b/src/tsdb/src/tsdbSync.c index cef561aebe..5a2756537e 100644 --- a/src/tsdb/src/tsdbSync.c +++ b/src/tsdb/src/tsdbSync.c @@ -194,8 +194,8 @@ static int32_t tsdbSyncRecvMeta(SSyncH *pSynch) { return 0; } - if (pLMFile == NULL || memcmp(&(pSynch->pmf->info), &(pLMFile->info), sizeof(SMFInfo)) != 0 || - TSDB_FILE_IS_BAD(pLMFile)) { + if (pLMFile == NULL || pSynch->pmf->info.size != pLMFile->info.size || + pSynch->pmf->info.magic != pLMFile->info.magic || TSDB_FILE_IS_BAD(pLMFile)) { // Local has no meta file or has a different meta file, need to copy from remote pSynch->mfChanged = true; @@ -536,7 +536,7 @@ static bool tsdbIsTowFSetSame(SDFileSet *pSet1, SDFileSet *pSet2) { SDFile *pDFile1 = TSDB_DFILE_IN_SET(pSet1, ftype); SDFile *pDFile2 = TSDB_DFILE_IN_SET(pSet2, ftype); - if (memcmp((void *)(TSDB_FILE_INFO(pDFile1)), (void *)(TSDB_FILE_INFO(pDFile2)), sizeof(SDFInfo)) != 0) { + if (pDFile1->info.size != pDFile2->info.size || pDFile1->info.magic != pDFile2->info.magic) { return false; } } diff --git a/tests/script/data2.sim b/tests/script/data2.sim new file mode 100644 index 0000000000..357b2bd451 --- /dev/null +++ b/tests/script/data2.sim @@ -0,0 +1,135 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 + +system sh/cfg.sh -n dnode1 -c wallevel -v 2 +system sh/cfg.sh -n dnode2 -c wallevel -v 2 +system sh/cfg.sh -n dnode3 -c wallevel -v 2 +system sh/cfg.sh -n dnode4 -c wallevel -v 2 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect +sleep 2000 + +print ========== step2 +sql create dnode $hostname2 +system sh/exec.sh -n dnode2 -s start +sql create dnode $hostname3 +system sh/exec.sh -n dnode3 -s start + +$x = 0 +show2: + $x = $x + 1 + sleep 2000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +if $data2_1 != 0 then + goto show2 +endi +if $data2_2 != 0 then + goto show2 +endi +if $data2_3 != 0 then + goto show2 +endi + +print ========== step3 +sql create database d1 days 1 replica 2 +sql create table d1.t1 (t timestamp, i int) +sql insert into d1.t1 values(1591160480000, 25) +sql insert into d1.t1 values(1591160480002, 24) +sql insert into d1.t1 values(1591160480003, 23) +sql insert into d1.t1 values(1591160480004, 22) +sql insert into d1.t1 values(1591160480005, 21) + +sql insert into d1.t1 values(1591260480000, 35) +sql insert into d1.t1 values(1591260480002, 34) +sql insert into d1.t1 values(1591260480003, 33) +sql insert into d1.t1 values(1591260480004, 32) +sql insert into d1.t1 values(1591260480005, 31) + +sql insert into d1.t1 values(1591360480000, 45) +sql insert into d1.t1 values(1591360480002, 44) +sql insert into d1.t1 values(1591360480003, 43) +sql insert into d1.t1 values(1591360480004, 42) +sql insert into d1.t1 values(1591360480005, 41) + +sql insert into d1.t1 values(1591460480000, 55) +sql insert into d1.t1 values(1591460480002, 54) +sql insert into d1.t1 values(1591460480003, 53) +sql insert into d1.t1 values(1591460480004, 52) +sql insert into d1.t1 values(1591460480005, 51) + +sql insert into d1.t1 values(1591560480000, 65) +sql insert into d1.t1 values(1591560480002, 64) +sql insert into d1.t1 values(1591560480003, 63) +sql insert into d1.t1 values(1591560480004, 62) +sql insert into d1.t1 values(1591560480005, 61) + +sql insert into d1.t1 values(1591660480000, 75) +sql insert into d1.t1 values(1591660480002, 74) +sql insert into d1.t1 values(1591660480003, 73) +sql insert into d1.t1 values(1591660480004, 72) +sql insert into d1.t1 values(1591660480005, 71) + +$x = 0 +show3: + $x = $x + 1 + sleep 2000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +if $data2_1 != 0 then + goto show3 +endi +if $data2_2 != 1 then + goto show3 +endi +if $data2_3 != 1 then + goto show3 +endi + +sleep 3000 + +print ========== step4 +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT + +print ========== step5 +system_content rm -rf ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/v2f18418.* + +print ========== step6 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start + +return + +# print ========== step7 +# system sh/exec.sh -n dnode1 -s stop -x SIGINT +# system sh/exec.sh -n dnode2 -s stop -x SIGINT +# system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file From 70b87e333ff4965ab011a26ab04fd7df9f0551ce Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 30 Mar 2021 16:02:55 +0800 Subject: [PATCH 48/71] delete useless file --- tests/script/data2.sim | 135 ----------------------------------------- 1 file changed, 135 deletions(-) delete mode 100644 tests/script/data2.sim diff --git a/tests/script/data2.sim b/tests/script/data2.sim deleted file mode 100644 index 357b2bd451..0000000000 --- a/tests/script/data2.sim +++ /dev/null @@ -1,135 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/deploy.sh -n dnode2 -i 2 -system sh/deploy.sh -n dnode3 -i 3 -system sh/deploy.sh -n dnode4 -i 4 - -system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 -system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 -system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 -system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 - -system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 - -system sh/cfg.sh -n dnode1 -c wallevel -v 2 -system sh/cfg.sh -n dnode2 -c wallevel -v 2 -system sh/cfg.sh -n dnode3 -c wallevel -v 2 -system sh/cfg.sh -n dnode4 -c wallevel -v 2 - -print ========== step1 -system sh/exec.sh -n dnode1 -s start -sql connect -sleep 2000 - -print ========== step2 -sql create dnode $hostname2 -system sh/exec.sh -n dnode2 -s start -sql create dnode $hostname3 -system sh/exec.sh -n dnode3 -s start - -$x = 0 -show2: - $x = $x + 1 - sleep 2000 - if $x == 10 then - return -1 - endi - -sql show dnodes -print dnode1 openVnodes $data2_1 -print dnode2 openVnodes $data2_2 -print dnode3 openVnodes $data2_3 -if $data2_1 != 0 then - goto show2 -endi -if $data2_2 != 0 then - goto show2 -endi -if $data2_3 != 0 then - goto show2 -endi - -print ========== step3 -sql create database d1 days 1 replica 2 -sql create table d1.t1 (t timestamp, i int) -sql insert into d1.t1 values(1591160480000, 25) -sql insert into d1.t1 values(1591160480002, 24) -sql insert into d1.t1 values(1591160480003, 23) -sql insert into d1.t1 values(1591160480004, 22) -sql insert into d1.t1 values(1591160480005, 21) - -sql insert into d1.t1 values(1591260480000, 35) -sql insert into d1.t1 values(1591260480002, 34) -sql insert into d1.t1 values(1591260480003, 33) -sql insert into d1.t1 values(1591260480004, 32) -sql insert into d1.t1 values(1591260480005, 31) - -sql insert into d1.t1 values(1591360480000, 45) -sql insert into d1.t1 values(1591360480002, 44) -sql insert into d1.t1 values(1591360480003, 43) -sql insert into d1.t1 values(1591360480004, 42) -sql insert into d1.t1 values(1591360480005, 41) - -sql insert into d1.t1 values(1591460480000, 55) -sql insert into d1.t1 values(1591460480002, 54) -sql insert into d1.t1 values(1591460480003, 53) -sql insert into d1.t1 values(1591460480004, 52) -sql insert into d1.t1 values(1591460480005, 51) - -sql insert into d1.t1 values(1591560480000, 65) -sql insert into d1.t1 values(1591560480002, 64) -sql insert into d1.t1 values(1591560480003, 63) -sql insert into d1.t1 values(1591560480004, 62) -sql insert into d1.t1 values(1591560480005, 61) - -sql insert into d1.t1 values(1591660480000, 75) -sql insert into d1.t1 values(1591660480002, 74) -sql insert into d1.t1 values(1591660480003, 73) -sql insert into d1.t1 values(1591660480004, 72) -sql insert into d1.t1 values(1591660480005, 71) - -$x = 0 -show3: - $x = $x + 1 - sleep 2000 - if $x == 10 then - return -1 - endi - -sql show dnodes -print dnode1 openVnodes $data2_1 -print dnode2 openVnodes $data2_2 -print dnode3 openVnodes $data2_3 -if $data2_1 != 0 then - goto show3 -endi -if $data2_2 != 1 then - goto show3 -endi -if $data2_3 != 1 then - goto show3 -endi - -sleep 3000 - -print ========== step4 -system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT - -print ========== step5 -system_content rm -rf ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/v2f18418.* - -print ========== step6 -system sh/exec.sh -n dnode2 -s start -system sh/exec.sh -n dnode3 -s start - -return - -# print ========== step7 -# system sh/exec.sh -n dnode1 -s stop -x SIGINT -# system sh/exec.sh -n dnode2 -s stop -x SIGINT -# system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file From d8b1d36e6c4af528707d12be23b60833ec4c11a7 Mon Sep 17 00:00:00 2001 From: wu champion Date: Tue, 30 Mar 2021 16:21:26 +0800 Subject: [PATCH 49/71] add TD-3589 test case --- tests/pytest/perfbenchmark/bug3589.py | 156 ++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 tests/pytest/perfbenchmark/bug3589.py diff --git a/tests/pytest/perfbenchmark/bug3589.py b/tests/pytest/perfbenchmark/bug3589.py new file mode 100644 index 0000000000..c54ef8595d --- /dev/null +++ b/tests/pytest/perfbenchmark/bug3589.py @@ -0,0 +1,156 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import json + +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def __init__(self): + self.path = "" + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/debug/build/bin")] + break + return buildPath + + def getCfgDir(self): + return self.getBuildPath() + "/sim/psim/cfg" + + def querycfg(self): + cfgdir = self.getCfgDir() + querycfg={ + "filetype": "query", + "cfgdir": cfgdir, + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "yes", + "databases": "db", + "specified_table_query": { + "query_interval": 0, + "concurrent": 1, + "sqls": [ + { + "sql": "select * from t10, t11 where t10.ts=t11.ts" + } + ] + } + } + + return querycfg + + def querycfgfile(self): + querycfg = self.querycfg() + date = datetime.datetime.now().strftime("%Y%m%d%H%M") + querycfg.get("specified_table_query").get("sqls")[0]["result"] = f"/tmp/query_{date}.log" + file_query_table = f"/tmp/query_{date}.json" + with open(file_query_table, "w") as f: + json.dump(querycfg, f) + + return [file_query_table, querycfg] + + def querytable(self, filepath): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info(f"taosd found in {buildPath}") + binPath = buildPath + "/debug/build/bin/" + + query_table_cmd = f"yes | {binPath}taosdemo -f {filepath}" + _ = subprocess.check_output(query_table_cmd, shell=True).decode("utf-8") + + def checkqueryresult(self, expectrows): + querycfg = self.querycfgfile()[1] + result_file = querycfg.get("specified_table_query").get("sqls")[0].get("result") + "-0" + if result_file: + check_cmd = f"wc -l {result_file}" + check_data_init = subprocess.check_output(check_cmd, shell=True).decode("utf-8") + check_data = int(check_data_init[0]) + if check_data == expectrows: + tdLog.info(f"queryResultRows:{check_data} == expect:{expectrows}") + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, check_data, expectrows) + tdLog.exit(f"{args[0]}({args[1]}) failed: result:{args[2]} != expect:{args[3]}") + + def droptmpfile(self): + drop_file_cmd = "rm -f /tmp/query_* " + _ = subprocess.check_output(drop_file_cmd, shell=True).decode("utf-8") + drop_file_cmd = "rm -f querySystemInfo-*" + _ = subprocess.check_output(drop_file_cmd, shell=True).decode("utf-8") + + def run(self): + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table && insert data") + tdSql.execute("alter database db keep 36500") + tdSql.execute( + "create table stb1 (ts timestamp, c1 int) TAGS(t1 int)" + ) + tdSql.execute("create table t10 using stb1 tags(1)") + tdSql.execute("create table t11 using stb1 tags(2)") + + tdSql.execute("insert into t10 values (-865000000, 1)") + tdSql.execute("insert into t11 values (-865000000, 2)") + tdSql.execute("insert into t10 values ('1969-12-31 23:59:59.000', 2)") + tdSql.execute("insert into t11 values ('1969-12-31 23:59:59.000', 3)") + tdSql.execute("insert into t10 values ('1970-01-01 00:00:00.000', 3)") + tdSql.execute("insert into t11 values ('1970-01-01 00:00:00.000', 4)") + tdSql.execute("insert into t10 values (-15230000, 4)") + tdSql.execute("insert into t11 values (-15230000, 5)") + tdSql.execute("insert into t10 values (-15220000, 5)") + tdSql.execute("insert into t11 values (-15220000, 6)") + tdSql.execute("insert into t10 values (-15210000, 6)") + tdSql.execute("insert into t11 values (-15210000, 7)") + tdSql.execute("insert into t10 values (0, 7)") + tdSql.execute("insert into t11 values (0, 8)") + tdSql.execute("insert into t10 values ('2020-10-01 00:00:00.000', 8)") + tdSql.execute("insert into t11 values ('2020-10-01 00:00:00.000', 9)") + + tdLog.printNoPrefix("==========step2:query") + query_file = self.querycfgfile()[0] + self.querytable(query_file) + self.checkqueryresult(8) + + self.droptmpfile() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From a782a3c5694d99b2ace57200afa9cd6ecc4b490f Mon Sep 17 00:00:00 2001 From: wu champion Date: Tue, 30 Mar 2021 16:22:43 +0800 Subject: [PATCH 50/71] add perfbenchmark/bug3589.py to fulltest.sh --- tests/pytest/fulltest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index a631149c90..c3282ce6f7 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -166,6 +166,7 @@ python3 ./test.py -f stable/query_after_reset.py # perfbenchmark python3 ./test.py -f perfbenchmark/bug3433.py +python3 ./test.py -f perfbenchmark/bug3589.py #query From e2e88b2e950b940b033f1ef2156e6e833be23f7f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 30 Mar 2021 18:05:29 +0800 Subject: [PATCH 51/71] [TD-3601]: fix early 1970 time problem --- src/tsdb/src/tsdbCommit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index dd0fd18019..4351cce51c 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -17,7 +17,7 @@ #define TSDB_MAX_SUBBLOCKS 8 static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) { if (key < 0) { - return (int)((key + 1) / tsMsPerDay[precision] / days + 1); + return (int)((key + 1) / tsMsPerDay[precision] / days - 1); } else { return (int)((key / tsMsPerDay[precision] / days)); } From ffb8c3d645c32e14b1145a473e3c3c2962664dd1 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Tue, 30 Mar 2021 18:09:15 +0800 Subject: [PATCH 52/71] [TD-3573] : add timezone section in faq. --- documentation20/cn/13.faq/docs.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/documentation20/cn/13.faq/docs.md b/documentation20/cn/13.faq/docs.md index e2285b29e2..c01247d345 100644 --- a/documentation20/cn/13.faq/docs.md +++ b/documentation20/cn/13.faq/docs.md @@ -156,3 +156,13 @@ ALTER LOCAL RESETLOG; ``` 其含义是，清空本机所有由客户端生成的日志文件。 + +## 18. 时间戳的时区信息是怎样处理的？ + +TDengine 中时间戳的时区总是由客户端进行处理，而与服务端无关。具体来说，客户端会对 SQL 语句中的时间戳进行时区转换，转为 UTC 时区（即 Unix 时间戳——Unix Timestamp）再交由服务端进行写入和查询；在读取数据时，服务端也是采用 UTC 时区提供原始数据，客户端收到后再根据本地设置，把时间戳转换为本地系统所要求的时区进行显示。 + +客户端在处理时间戳字符串时，会采取如下逻辑： +1. 在未做特殊设置的情况下，客户端默认使用所在操作系统的时区设置。 +2. 如果在 taos.cfg 中设置了 timezone 参数，则客户端会以这个配置文件中的设置为准。 +3. 如果在 C/C++/Java/Python 等各种编程语言的 Connector Driver 中，在建立数据库连接时显式指定了 timezone，那么会以这个指定的时区设置为准。例如 Java Connector 的 JDBC URL 中就有 timezone 参数。 +4. 在书写 SQL 语句时，也可以直接使用 Unix 时间戳（例如 `1554984068000`）或带有时区的时间戳字符串，也即以 RFC 3339 格式（例如 `2013-04-12T15:52:01.123+08:00`）或 ISO-8601 格式（例如 `2013-04-12T15:52:01.123+0800`）来书写时间戳，此时这些时间戳的取值将不再受其他时区设置的影响。 From 97b3cdb0de446f547fc124f19bef8274cbd5bf05 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 30 Mar 2021 18:28:21 +0800 Subject: [PATCH 53/71] [TD-3602]: update perf test script --- tests/perftest-scripts/perftest-query.sh | 15 +-- tests/pytest/tools/taosdemoPerformance.py | 116 +++++++++------------- 2 files changed, 50 insertions(+), 81 deletions(-) diff --git a/tests/perftest-scripts/perftest-query.sh b/tests/perftest-scripts/perftest-query.sh index 38cfb3ca19..8b9fa1a546 100755 --- a/tests/perftest-scripts/perftest-query.sh +++ b/tests/perftest-scripts/perftest-query.sh @@ -71,21 +71,8 @@ function runQueryPerfTest { python3 query/queryPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT python3 insert/insertFromCSVPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT - - taosdemo -f /home/ubuntu/pxiao/insert.json > taosdemoperf.txt - - CREATETABLETIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'` - INSERTRECORDSTIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'` - REQUESTSPERSECOND=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $16}'` - delay=`grep 'delay' taosdemoperf.txt | awk '{print $4}'` - AVGDELAY=`echo ${delay:0:${#delay}-3}` - delay=`grep 'delay' taosdemoperf.txt | awk '{print $6}'` - MAXDELAY=`echo ${delay:0:${#delay}-3}` - delay=`grep 'delay' taosdemoperf.txt | awk '{print $8}'` - MINDELAY=`echo ${delay:0:${#delay}-2}` - python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT -t $CREATETABLETIME -i $INSERTRECORDSTIME -r $REQUESTSPERSECOND -avg $AVGDELAY -max $MAXDELAY -min $MINDELAY | tee -a $PERFORMANCE_TEST_REPORT - [ -f taosdemoperf.txt ] && rm taosdemoperf.txt + python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT } diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py index a337ced4e6..1156cc2484 100644 --- a/tests/pytest/tools/taosdemoPerformance.py +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -11,26 +11,16 @@ # -*- coding: utf-8 -*- -import sys import taos -import time -import datetime -import csv -import random import pandas as pd import argparse import os.path +import json class taosdemoPerformace: - def __init__(self, commitID, dbName, createTableTime, insertRecordsTime, recordsPerSecond, avgDelay, maxDelay, minDelay): + def __init__(self, commitID, dbName): self.commitID = commitID - self.dbName = dbName - self.createTableTime = createTableTime - self.insertRecordsTime = insertRecordsTime - self.recordsPerSecond = recordsPerSecond - self.avgDelay = avgDelay - self.maxDelay = maxDelay - self.minDelay = minDelay + self.dbName = dbName self.host = "127.0.0.1" self.user = "root" self.password = "taosdata" @@ -69,11 +59,14 @@ class taosdemoPerformace: "start_timestamp": "2020-10-01 00:00:00.000", "sample_format": "csv", "sample_file": "./sample.csv", - "tags_file": "", - "columns": [{ - "columns": [{"type": "INT", "count": 4}], - }], - "tags": [{"type": "INT", "count":1}, {"type": "BINARY", "len": 16}] + "tags_file": "", + "columns": [ + {"type": "INT", "count": 4} + ], + "tags": [ + {"type": "INT", "count":1}, + {"type": "BINARY", "len": 16} + ] } stables = [] @@ -97,10 +90,35 @@ class taosdemoPerformace: "confirm_parameter_prompt": "no", "insert_interval": 0, "num_of_records_per_req": 30000, - "databases": [db] + "databases": [db] } - return insert_data + insert_json_file = f"/tmp/insert.json" + + with open(insert_json_file, 'w') as f: + json.dump(insert_data, f) + return insert_json_file + + def getCMDOutput(self, cmd): + cmd = os.popen(cmd) + output = cmd.read() + cmd.close() + return output + + def insertData(self): + os.system("taosdemo -f %s > taosdemoperf.txt" % self.generateJson()) + self.createTableTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'") + self.insertRecordsTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'") + self.recordsPerSecond = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $16}'") + self.commitID = self.getCMDOutput("git rev-parse --short HEAD") + delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $4}'") + self.avgDelay = delay[:-4] + delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $6}'") + self.maxDelay = delay[:-4] + delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $8}'") + self.minDelay = delay[:-3] + + os.system("[ -f taosdemoperf.txt ] && rm taosdemoperf.txt") def createTablesAndStoreData(self): cursor = self.conn.cursor() @@ -109,13 +127,14 @@ class taosdemoPerformace: cursor.execute("use %s" % self.dbName) cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50), avg_delay float, max_delay float, min_delay float)") print("==================== taosdemo performance ====================") - print("create tables time: %f" % self.createTableTime) - print("insert records time: %f" % self.insertRecordsTime) - print("records per second: %f" % self.recordsPerSecond) - print("avg delay: %f" % self.avgDelay) - print("max delay: %f" % self.maxDelay) - print("min delay: %f" % self.minDelay) - cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f)" % (self.createTableTime, self.insertRecordsTime, self.recordsPerSecond, self.commitID, self.avgDelay, self.maxDelay, self.minDelay)) + print("create tables time: %f" % float(self.createTableTime)) + print("insert records time: %f" % float(self.insertRecordsTime)) + print("records per second: %f" % float(self.recordsPerSecond)) + print("avg delay: %f" % float(self.avgDelay)) + print("max delay: %f" % float(self.maxDelay)) + print("min delay: %f" % float(self.minDelay)) + cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f)" % + (float(self.createTableTime), float(self.insertRecordsTime), float(self.recordsPerSecond), self.commitID, float(self.avgDelay), float(self.maxDelay), float(self.minDelay))) cursor.execute("drop database if exists %s" % self.insertDB) cursor.close() @@ -125,7 +144,7 @@ if __name__ == '__main__': parser.add_argument( '-c', '--commit-id', - action='store', + action='store', type=str, help='git commit id (default: null)') parser.add_argument( @@ -135,46 +154,9 @@ if __name__ == '__main__': default='perf', type=str, help='Database name to be created (default: perf)') - parser.add_argument( - '-t', - '--create-table', - action='store', - type=float, - help='create table time') - parser.add_argument( - '-i', - '--insert-records', - action='store', - type=float, - help='insert records time') - parser.add_argument( - '-r', - '---records-per-second', - action='store', - type=float, - help='records per request') - parser.add_argument( - '-avg', - '---avg-delay', - action='store', - type=float, - help='avg delay') - parser.add_argument( - '-max', - '---max-delay', - action='store', - type=float, - help='max delay') - parser.add_argument( - '-min', - '---min-delay', - action='store', - type=float, - help='min delay') - args = parser.parse_args() - perftest = taosdemoPerformace(args.commit_id, args.database_name, args.create_table, args.insert_records, args.records_per_second, - args.avg_delay, args.max_delay, args.min_delay) + perftest = taosdemoPerformace(args.commit_id, args.database_name) + perftest.insertData() perftest.createTablesAndStoreData() \ No newline at end of file From 75faa804df5472b7a724c456b5b0e0c3b060059f Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Tue, 30 Mar 2021 19:14:25 +0800 Subject: [PATCH 54/71] restart travis --- src/client/src/tscServer.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index d6deb7d7e2..7085318e35 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2647,7 +2647,6 @@ void tscInitMsgsFp() { tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_TABLE] = tscProcessShowCreateRsp; tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp; - tscKeepConn[TSDB_SQL_SHOW] = 1; tscKeepConn[TSDB_SQL_RETRIEVE] = 1; tscKeepConn[TSDB_SQL_SELECT] = 1; From 735e1a441a3f28cd6c3c5a6462eadfd9958c311b Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Wed, 31 Mar 2021 09:19:31 +0800 Subject: [PATCH 55/71] add debug info --- tests/test-all.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test-all.sh b/tests/test-all.sh index 1f9c76daf4..b1a331073b 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -25,7 +25,9 @@ function stopTaosd { function dohavecore(){ corefile=`find $corepath -mmin 1` core_file=`echo $corefile|cut -d " " -f2` - echo $core_file + echo "corefile:$core_file" + echo "corepath:$corepath" + ls -l $corepath proc=`echo $corefile|cut -d "_" -f3` if [ -n "$corefile" ];then echo 'taosd or taos has generated core' From c26d882057219fd82962576a359472325dda924d Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 31 Mar 2021 11:17:51 +0800 Subject: [PATCH 56/71] [TD-3284] : taosdemo support fixed value tag. (#5632) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 7b0668b82d..3725712339 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -1962,7 +1962,7 @@ static char* getTagValueFromTagSample(SSuperTable* stbInfo, int tagUsePos) { return dataBuf; } -static char* generateTagVaulesForStb(SSuperTable* stbInfo) { +static char* generateTagVaulesForStb(SSuperTable* stbInfo, int32_t tableSeq) { char* dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1); if (NULL == dataBuf) { printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1); @@ -1981,20 +1981,27 @@ static char* generateTagVaulesForStb(SSuperTable* stbInfo) { return NULL; } - char* buf = (char*)calloc(stbInfo->tags[i].dataLen+1, 1); + int tagBufLen = stbInfo->tags[i].dataLen + 1; + char* buf = (char*)calloc(tagBufLen, 1); if (NULL == buf) { printf("calloc failed! size:%d\n", stbInfo->tags[i].dataLen); tmfree(dataBuf); return NULL; } - rand_string(buf, stbInfo->tags[i].dataLen); + + if (tableSeq % 2) { + tstrncpy(buf, "beijing", tagBufLen); + } else { + tstrncpy(buf, "shanghai", tagBufLen); + } + //rand_string(buf, stbInfo->tags[i].dataLen); dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "\'%s\', ", buf); tmfree(buf); } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "int", strlen("int"))) { dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, - "%d, ", rand_int()); + "%d, ", tableSeq); } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "bigint", strlen("bigint"))) { dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, @@ -2585,10 +2592,10 @@ static void* createTable(void *sarg) len += snprintf(buffer + len, buff_len - len, "create table "); } - + char* tagsValBuf = NULL; if (0 == superTblInfo->tagSource) { - tagsValBuf = generateTagVaulesForStb(superTblInfo); + tagsValBuf = generateTagVaulesForStb(superTblInfo, i); } else { tagsValBuf = getTagValueFromTagSample( superTblInfo, @@ -2598,7 +2605,7 @@ static void* createTable(void *sarg) free(buffer); return NULL; } - + len += snprintf(buffer + len, superTblInfo->maxSqlLen - len, "if not exists %s.%s%d using %s.%s tags %s ", @@ -2607,7 +2614,7 @@ static void* createTable(void *sarg) superTblInfo->sTblName, tagsValBuf); free(tagsValBuf); batchNum++; - + if ((batchNum < superTblInfo->batchCreateTableNum) && ((superTblInfo->maxSqlLen - len) >= (superTblInfo->lenOfTagOfOneRow + 256))) { @@ -4499,7 +4506,7 @@ static int generateSQLHead(char *tableName, int32_t tableSeq, if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { char* tagsValBuf = NULL; if (0 == superTblInfo->tagSource) { - tagsValBuf = generateTagVaulesForStb(superTblInfo); + tagsValBuf = generateTagVaulesForStb(superTblInfo, tableSeq); } else { tagsValBuf = getTagValueFromTagSample( superTblInfo, From d2a0fc5b931459022819c94c6942e72940df4f85 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 30 Mar 2021 16:36:16 +0800 Subject: [PATCH 57/71] [TD-3314]: new staging directory for deleting vnodes --- src/dnode/src/dnodeMain.c | 12 +++ src/vnode/inc/vnodeBackup.h | 32 +++++++ src/vnode/src/vnodeBackup.c | 172 ++++++++++++++++++++++++++++++++++++ src/vnode/src/vnodeMain.c | 15 ++-- src/vnode/src/vnodeMgmt.c | 2 + 5 files changed, 224 insertions(+), 9 deletions(-) create mode 100644 src/vnode/inc/vnodeBackup.h create mode 100644 src/vnode/src/vnodeBackup.c diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 2cd5c637e5..ea0ef4655d 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -237,6 +237,18 @@ static int32_t dnodeInitStorage() { return -1; } + TDIR *tdir = tfsOpendir("vnode_bak/.staging"); + if (tfsReaddir(tdir) != NULL) { + dError("vnode_bak/.staging dir not empty, fix it first."); + tfsClosedir(tdir); + return -1; + } + + if (tfsMkdir("vnode_bak/.staging") < 0) { + dError("failed to create vnode_bak/.staging dir since %s", tstrerror(terrno)); + return -1; + } + dnodeCheckDataDirOpenned(tsDnodeDir); dInfo("dnode storage is initialized at %s", tsDnodeDir); diff --git a/src/vnode/inc/vnodeBackup.h b/src/vnode/inc/vnodeBackup.h new file mode 100644 index 0000000000..0a6b26546c --- /dev/null +++ b/src/vnode/inc/vnodeBackup.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_VNODE_BACKUP_H +#define TDENGINE_VNODE_BACKUP_H + +#ifdef __cplusplus +extern "C" { +#endif +#include "vnodeInt.h" + +int32_t vnodeInitBackup(); +void vnodeCleanupBackup(); +int32_t vnodeBackup(int32_t vgId); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/vnode/src/vnodeBackup.c b/src/vnode/src/vnodeBackup.c new file mode 100644 index 0000000000..a0a975be2b --- /dev/null +++ b/src/vnode/src/vnodeBackup.c @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taoserror.h" +#include "taosmsg.h" +#include "tutil.h" +#include "tqueue.h" +#include "tglobal.h" +#include "tfs.h" +#include "vnodeBackup.h" +#include "vnodeMain.h" + +typedef struct { + int32_t vgId; +} SVBackupMsg; + +typedef struct { + pthread_t thread; + int32_t workerId; +} SVBackupWorker; + +typedef struct { + int32_t num; + SVBackupWorker *worker; +} SVBackupWorkerPool; + +static SVBackupWorkerPool tsVBackupPool; +static taos_qset tsVBackupQset; +static taos_queue tsVBackupQueue; + +static void vnodeProcessBackupMsg(SVBackupMsg *pMsg) { + int32_t vgId = pMsg->vgId; + char newDir[TSDB_FILENAME_LEN] = {0}; + char stagingDir[TSDB_FILENAME_LEN] = {0}; + + sprintf(newDir, "%s/vnode%d", "vnode_bak", vgId); + sprintf(stagingDir, "%s/.staging/vnode%d", "vnode_bak", vgId); + + if (tsEnableVnodeBak) { + tfsRmdir(newDir); + tfsRename(stagingDir, newDir); + } else { + vInfo("vgId:%d, vnode backup not enabled", vgId); + + tfsRmdir(stagingDir); + } +} + +static void *vnodeBackupFunc(void *param) { + while (1) { + SVBackupMsg *pMsg = NULL; + if (taosReadQitemFromQset(tsVBackupQset, NULL, (void **)&pMsg, NULL) == 0) { + vDebug("qset:%p, vbackup got no message from qset, exiting", tsVBackupQset); + break; + } + + vTrace("vgId:%d, will be processed in vbackup queue", pMsg->vgId); + vnodeProcessBackupMsg(pMsg); + + vTrace("vgId:%d, disposed in vbackup worker", pMsg->vgId); + taosFreeQitem(pMsg); + } + + return NULL; +} + +static int32_t vnodeStartBackup() { + tsVBackupQueue = taosOpenQueue(); + if (tsVBackupQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY; + + taosAddIntoQset(tsVBackupQset, tsVBackupQueue, NULL); + + for (int32_t i = 0; i < tsVBackupPool.num; ++i) { + SVBackupWorker *pWorker = tsVBackupPool.worker + i; + pWorker->workerId = i; + + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + + if (pthread_create(&pWorker->thread, &thAttr, vnodeBackupFunc, pWorker) != 0) { + vError("failed to create thread to process vbackup queue, reason:%s", strerror(errno)); + } + + pthread_attr_destroy(&thAttr); + + vDebug("vbackup:%d is launched, total:%d", pWorker->workerId, tsVBackupPool.num); + } + + vDebug("vbackup queue:%p is allocated", tsVBackupQueue); + + return TSDB_CODE_SUCCESS; +} + +static int32_t vnodeWriteIntoBackupWorker(int32_t vgId) { + SVBackupMsg *pMsg = taosAllocateQitem(sizeof(SVBackupMsg)); + if (pMsg == NULL) return TSDB_CODE_VND_OUT_OF_MEMORY; + + pMsg->vgId = vgId; + + int32_t code = taosWriteQitem(tsVBackupQueue, TAOS_QTYPE_RPC, pMsg); + if (code == 0) code = TSDB_CODE_DND_ACTION_IN_PROGRESS; + + return code; +} + +int32_t vnodeBackup(int32_t vgId) { + vTrace("vgId:%d, will backup", vgId); + return vnodeWriteIntoBackupWorker(vgId); +} + +int32_t vnodeInitBackup() { + tsVBackupQset = taosOpenQset(); + + tsVBackupPool.num = 1; + tsVBackupPool.worker = calloc(sizeof(SVBackupWorker), tsVBackupPool.num); + + if (tsVBackupPool.worker == NULL) return -1; + for (int32_t i = 0; i < tsVBackupPool.num; ++i) { + SVBackupWorker *pWorker = tsVBackupPool.worker + i; + pWorker->workerId = i; + vDebug("vbackup:%d is created", i); + } + + vDebug("vbackup is initialized, num:%d qset:%p", tsVBackupPool.num, tsVBackupQset); + + return vnodeStartBackup(); +} + +void vnodeCleanupBackup() { + for (int32_t i = 0; i < tsVBackupPool.num; ++i) { + SVBackupWorker *pWorker = tsVBackupPool.worker + i; + if (taosCheckPthreadValid(pWorker->thread)) { + taosQsetThreadResume(tsVBackupQset); + } + vDebug("vbackup:%d is closed", i); + } + + for (int32_t i = 0; i < tsVBackupPool.num; ++i) { + SVBackupWorker *pWorker = tsVBackupPool.worker + i; + vDebug("vbackup:%d start to join", i); + if (taosCheckPthreadValid(pWorker->thread)) { + pthread_join(pWorker->thread, NULL); + } + vDebug("vbackup:%d join success", i); + } + + vDebug("vbackup is closed, qset:%p", tsVBackupQset); + + taosCloseQset(tsVBackupQset); + tsVBackupQset = NULL; + + tfree(tsVBackupPool.worker); + + vDebug("vbackup queue:%p is freed", tsVBackupQueue); + taosCloseQueue(tsVBackupQueue); + tsVBackupQueue = NULL; +} diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 441e951250..7a14cc7f5b 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -27,6 +27,7 @@ #include "vnodeVersion.h" #include "vnodeMgmt.h" #include "vnodeWorker.h" +#include "vnodeBackup.h" #include "vnodeMain.h" static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno); @@ -448,18 +449,14 @@ void vnodeDestroy(SVnodeObj *pVnode) { if (pVnode->dropped) { char rootDir[TSDB_FILENAME_LEN] = {0}; - char newDir[TSDB_FILENAME_LEN] = {0}; + char stagingDir[TSDB_FILENAME_LEN] = {0}; sprintf(rootDir, "%s/vnode%d", "vnode", vgId); - sprintf(newDir, "%s/vnode%d", "vnode_bak", vgId); + sprintf(stagingDir, "%s/.staging/vnode%d", "vnode_bak", vgId); - if (0 == tsEnableVnodeBak) { - vInfo("vgId:%d, vnode backup not enabled", pVnode->vgId); - } else { - tfsRmdir(newDir); - tfsRename(rootDir, newDir); - } + tfsRename(rootDir, stagingDir); + + vnodeBackup(vgId); - tfsRmdir(rootDir); dnodeSendStatusMsgToMnode(); } diff --git a/src/vnode/src/vnodeMgmt.c b/src/vnode/src/vnodeMgmt.c index 71d9bc07f5..32f9532138 100644 --- a/src/vnode/src/vnodeMgmt.c +++ b/src/vnode/src/vnodeMgmt.c @@ -17,6 +17,7 @@ #include "os.h" #include "dnode.h" #include "vnodeStatus.h" +#include "vnodeBackup.h" #include "vnodeWorker.h" #include "vnodeRead.h" #include "vnodeWrite.h" @@ -29,6 +30,7 @@ static void vnodeCleanupHash(void); static void vnodeIncRef(void *ptNode); static SStep tsVnodeSteps[] = { + {"vnode-backup", vnodeInitBackup, vnodeCleanupBackup}, {"vnode-worker", vnodeInitMWorker, vnodeCleanupMWorker}, {"vnode-write", vnodeInitWrite, vnodeCleanupWrite}, {"vnode-read", vnodeInitRead, vnodeCleanupRead}, From 3631d276f26b226d8902b5d49f342da7933269f7 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 31 Mar 2021 11:35:15 +0800 Subject: [PATCH 58/71] update test script --- tests/test-all.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test-all.sh b/tests/test-all.sh index 1f9c76daf4..adb9fa006e 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -463,10 +463,10 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != echo "Running tests" ./apitest > /dev/null 2>&1 if [ $? != "0" ]; then - echo "prepare failed" + echo "apitest failed" totalExampleFailed=`expr $totalExampleFailed + 1` else - echo "prepare pass" + echo "apitest pass" totalExamplePass=`expr $totalExamplePass + 1` fi @@ -481,19 +481,19 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != ./subscribe -test > /dev/null 2>&1 if [ $? != "0" ]; then - echo "prepare failed" + echo "subscribe failed" totalExampleFailed=`expr $totalExampleFailed + 1` else - echo "prepare pass" + echo "subscribe pass" totalExamplePass=`expr $totalExamplePass + 1` fi yes |./asyncdemo 127.0.0.1 test 1000 10 > /dev/null 2>&1 if [ $? != "0" ]; then - echo "prepare failed" + echo "asyncdemo failed" totalExampleFailed=`expr $totalExampleFailed + 1` else - echo "prepare pass" + echo "asyncdemo pass" totalExamplePass=`expr $totalExamplePass + 1` fi From 61d7d9ab41b52e1327c40987facff6c1f547e91d Mon Sep 17 00:00:00 2001 From: nianhongdong Date: Wed, 31 Mar 2021 13:46:01 +0800 Subject: [PATCH 59/71] Update JDBCDemo.java MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 71和76行测试示例的数据库和表名没有动态拼接，测试用在前面改数据库名和表名，会导致运行报错 --- .../JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java index 07a21e79d1..da865b3ffd 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java @@ -68,12 +68,12 @@ public class JDBCDemo { } private void insert() { - final String sql = "insert into test.weather (ts, temperature, humidity) values(now, 20.5, 34)"; + final String sql = "insert into " + dbName + "." + tbName + " (ts, temperature, humidity) values(now, 20.5, 34)"; exuete(sql); } private void select() { - final String sql = "select * from test.weather"; + final String sql = "select * from "+ dbName + "." + tbName; executeQuery(sql); } From 536a9e9f3fcb7792ee3bc5943bf342d6d03a8cb5 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 31 Mar 2021 16:55:31 +0800 Subject: [PATCH 60/71] remove FResetVersion function --- src/inc/tsync.h | 4 ---- src/sync/inc/syncInt.h | 1 - src/sync/src/syncMain.c | 1 - src/sync/src/syncRestore.c | 1 - src/vnode/inc/vnodeSync.h | 1 - src/vnode/src/vnodeMain.c | 1 - src/vnode/src/vnodeSync.c | 19 ++----------------- 7 files changed, 2 insertions(+), 26 deletions(-) diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 99dfd3a6a3..d1b68e3f5a 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -79,9 +79,6 @@ typedef void (*FStopSyncFile)(int32_t vgId, uint64_t fversion); // get file version typedef int32_t (*FGetVersion)(int32_t vgId, uint64_t *fver, uint64_t *vver); -// reset version -typedef int32_t (*FResetVersion)(int32_t vgId, uint64_t fver); - typedef int32_t (*FSendFile)(void *tsdb, SOCKET socketFd); typedef int32_t (*FRecvFile)(void *tsdb, SOCKET socketFd); @@ -99,7 +96,6 @@ typedef struct { FStartSyncFile startSyncFileFp; FStopSyncFile stopSyncFileFp; FGetVersion getVersionFp; - FResetVersion resetVersionFp; FSendFile sendFileFp; FRecvFile recvFileFp; } SSyncInfo; diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index 2b87938474..91613ae351 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -117,7 +117,6 @@ typedef struct SSyncNode { FStartSyncFile startSyncFileFp; FStopSyncFile stopSyncFileFp; FGetVersion getVersionFp; - FResetVersion resetVersionFp; FSendFile sendFileFp; FRecvFile recvFileFp; pthread_mutex_t mutex; diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 76d4379c5f..d21743d40a 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -182,7 +182,6 @@ int64_t syncStart(const SSyncInfo *pInfo) { pNode->startSyncFileFp = pInfo->startSyncFileFp; pNode->stopSyncFileFp = pInfo->stopSyncFileFp; pNode->getVersionFp = pInfo->getVersionFp; - pNode->resetVersionFp = pInfo->resetVersionFp; pNode->sendFileFp = pInfo->sendFileFp; pNode->recvFileFp = pInfo->recvFileFp; diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 22d0a27581..c0d66316cd 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -238,7 +238,6 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { (*pNode->stopSyncFileFp)(pNode->vgId, fversion); nodeVersion = fversion; - if (pNode->resetVersionFp) (*pNode->resetVersionFp)(pNode->vgId, fversion); sInfo("%s, start to restore wal, fver:%" PRIu64, pPeer->id, nodeVersion); uint64_t wver = 0; diff --git a/src/vnode/inc/vnodeSync.h b/src/vnode/inc/vnodeSync.h index 75d7ffbabd..28fb63dd6a 100644 --- a/src/vnode/inc/vnodeSync.h +++ b/src/vnode/inc/vnodeSync.h @@ -30,7 +30,6 @@ void vnodeStopSyncFile(int32_t vgId, uint64_t fversion); void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code); int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam); int32_t vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver); -int32_t vnodeResetVersion(int32_t vgId, uint64_t fver); void vnodeConfirmForward(void *pVnode, uint64_t version, int32_t code, bool force); diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 441e951250..96a9decf4b 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -364,7 +364,6 @@ int32_t vnodeOpen(int32_t vgId) { syncInfo.startSyncFileFp = vnodeStartSyncFile; syncInfo.stopSyncFileFp = vnodeStopSyncFile; syncInfo.getVersionFp = vnodeGetVersion; - syncInfo.resetVersionFp = vnodeResetVersion; syncInfo.sendFileFp = tsdbSyncSend; syncInfo.recvFileFp = tsdbSyncRecv; syncInfo.pTsdb = pVnode->tsdb; diff --git a/src/vnode/src/vnodeSync.c b/src/vnode/src/vnodeSync.c index 929dd15926..aa4cf0fc15 100644 --- a/src/vnode/src/vnodeSync.c +++ b/src/vnode/src/vnodeSync.c @@ -107,8 +107,9 @@ void vnodeStopSyncFile(int32_t vgId, uint64_t fversion) { pVnode->fversion = fversion; pVnode->version = fversion; vnodeSaveVersion(pVnode); + walResetVersion(pVnode->wal, fversion); - vDebug("vgId:%d, datafile is synced, fver:%" PRIu64 " vver:%" PRIu64, vgId, fversion, fversion); + vInfo("vgId:%d, datafile is synced, fver:%" PRIu64 " vver:%" PRIu64, vgId, fversion, fversion); vnodeSetReadyStatus(pVnode); vnodeRelease(pVnode); @@ -158,22 +159,6 @@ int32_t vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver) { return code; } -int32_t vnodeResetVersion(int32_t vgId, uint64_t fver) { - SVnodeObj *pVnode = vnodeAcquire(vgId); - if (pVnode == NULL) { - vError("vgId:%d, vnode not found while reset version", vgId); - return -1; - } - - pVnode->fversion = fver; - pVnode->version = fver; - walResetVersion(pVnode->wal, fver); - vInfo("vgId:%d, version reset to %" PRIu64, vgId, fver); - - vnodeRelease(pVnode); - return 0; -} - void vnodeConfirmForward(void *vparam, uint64_t version, int32_t code, bool force) { SVnodeObj *pVnode = vparam; syncConfirmForward(pVnode->sync, version, code, force); From 10ea8e143d3ea3fd994b68fe82483c809dd50ec2 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Wed, 31 Mar 2021 17:41:16 +0800 Subject: [PATCH 61/71] [TD-3624] : fix minor typo. --- .../cn/07.advanced-features/docs.md | 136 +++++------------- .../cn/08.connector/01.java/docs.md | 2 +- 2 files changed, 36 insertions(+), 102 deletions(-) diff --git a/documentation20/cn/07.advanced-features/docs.md b/documentation20/cn/07.advanced-features/docs.md index bdf93fdc3d..650a2ca96b 100644 --- a/documentation20/cn/07.advanced-features/docs.md +++ b/documentation20/cn/07.advanced-features/docs.md @@ -1,27 +1,16 @@ # 高级功能 -## 连续查询(Continuous Query) +## 连续查询（Continuous Query） -连续查询是TDengine定期自动执行的查询，采用滑动窗口的方式进行计算，是一种简化的时间驱动的流式计算。 -针对库中的表或超级表，TDengine可提供定期自动执行的连续查询， -用户可让TDengine推送查询的结果，也可以将结果再写回到TDengine中。 -每次执行的查询是一个时间窗口，时间窗口随着时间流动向前滑动。 -在定义连续查询的时候需要指定时间窗口（time window, 参数interval）大小和每次前向增量时间（forward sliding times, 参数sliding）。 +连续查询是TDengine定期自动执行的查询，采用滑动窗口的方式进行计算，是一种简化的时间驱动的流式计算。针对库中的表或超级表，TDengine可提供定期自动执行的连续查询，用户可让TDengine推送查询的结果，也可以将结果再写回到TDengine中。每次执行的查询是一个时间窗口，时间窗口随着时间流动向前滑动。在定义连续查询的时候需要指定时间窗口（time window, 参数interval）大小和每次前向增量时间（forward sliding times, 参数sliding）。 -TDengine的连续查询采用时间驱动模式，可以直接使用TAOS SQL进行定义，不需要额外的操作。 -使用连续查询，可以方便快捷地按照时间窗口生成结果，从而对原始采集数据进行降采样（down sampling）。 -用户通过TAOS SQL定义连续查询以后，TDengine自动在最后的一个完整的时间周期末端拉起查询， -并将计算获得的结果推送给用户或者写回TDengine。 +TDengine的连续查询采用时间驱动模式，可以直接使用TAOS SQL进行定义，不需要额外的操作。使用连续查询，可以方便快捷地按照时间窗口生成结果，从而对原始采集数据进行降采样（down sampling）。用户通过TAOS SQL定义连续查询以后，TDengine自动在最后的一个完整的时间周期末端拉起查询，并将计算获得的结果推送给用户或者写回TDengine。 TDengine提供的连续查询与普通流计算中的时间窗口计算具有以下区别： -- 不同于流计算的实时反馈计算结果，连续查询只在时间窗口关闭以后才开始计算。 -例如时间周期是1天，那么当天的结果只会在23:59:59以后才会生成。 -- 如果有历史记录写入到已经计算完成的时间区间，连续查询并不会重新进行计算， -也不会重新将结果推送给用户。对于写回TDengine的模式，也不会更新已经存在的计算结果。 -- 使用连续查询推送结果的模式，服务端并不缓存客户端计算状态，也不提供Exactly-Once的语意保证。 -如果用户的应用端崩溃，再次拉起的连续查询将只会从再次拉起的时间开始重新计算最近的一个完整的时间窗口。 -如果使用写回模式，TDengine可确保数据写回的有效性和连续性。 +- 不同于流计算的实时反馈计算结果，连续查询只在时间窗口关闭以后才开始计算。例如时间周期是1天，那么当天的结果只会在23:59:59以后才会生成。 +- 如果有历史记录写入到已经计算完成的时间区间，连续查询并不会重新进行计算，也不会重新将结果推送给用户。对于写回TDengine的模式，也不会更新已经存在的计算结果。 +- 使用连续查询推送结果的模式，服务端并不缓存客户端计算状态，也不提供Exactly-Once的语意保证。如果用户的应用端崩溃，再次拉起的连续查询将只会从再次拉起的时间开始重新计算最近的一个完整的时间窗口。如果使用写回模式，TDengine可确保数据写回的有效性和连续性。 ### 使用连续查询 @@ -40,23 +29,19 @@ create table D1002 using meters tags ("Beijing.Haidian", 2); select avg(voltage) from meters interval(1m) sliding(30s); ``` -每次执行这条语句，都会重新计算所有数据。 -如果需要每隔30秒执行一次来增量计算最近一分钟的数据， -可以把上面的语句改进成下面的样子，每次使用不同的 `startTime` 并定期执行： +每次执行这条语句，都会重新计算所有数据。 如果需要每隔30秒执行一次来增量计算最近一分钟的数据，可以把上面的语句改进成下面的样子，每次使用不同的 `startTime` 并定期执行： ```sql select avg(voltage) from meters where ts > {startTime} interval(1m) sliding(30s); ``` -这样做没有问题，但TDengine提供了更简单的方法， -只要在最初的查询语句前面加上 `create table {tableName} as ` 就可以了, 例如： +这样做没有问题，但TDengine提供了更简单的方法，只要在最初的查询语句前面加上 `create table {tableName} as ` 就可以了, 例如： ```sql create table avg_vol as select avg(voltage) from meters interval(1m) sliding(30s); ``` -会自动创建一个名为 `avg_vol` 的新表，然后每隔30秒，TDengine会增量执行 `as` 后面的 SQL 语句， -并将查询结果写入这个表中，用户程序后续只要从 `avg_vol` 中查询数据即可。 例如： +会自动创建一个名为 `avg_vol` 的新表，然后每隔30秒，TDengine会增量执行 `as` 后面的 SQL 语句，并将查询结果写入这个表中，用户程序后续只要从 `avg_vol` 中查询数据即可。 例如： ```mysql taos> select * from avg_vol; @@ -70,43 +55,27 @@ taos> select * from avg_vol; 需要注意，查询时间窗口的最小值是10毫秒，没有时间窗口范围的上限。 -此外，TDengine还支持用户指定连续查询的起止时间。 -如果不输入开始时间，连续查询将从第一条原始数据所在的时间窗口开始； -如果没有输入结束时间，连续查询将永久运行； -如果用户指定了结束时间，连续查询在系统时间达到指定的时间以后停止运行。 -比如使用下面的SQL创建的连续查询将运行一小时，之后会自动停止。 +此外，TDengine还支持用户指定连续查询的起止时间。如果不输入开始时间，连续查询将从第一条原始数据所在的时间窗口开始；如果没有输入结束时间，连续查询将永久运行；如果用户指定了结束时间，连续查询在系统时间达到指定的时间以后停止运行。比如使用下面的SQL创建的连续查询将运行一小时，之后会自动停止。 ```mysql create table avg_vol as select avg(voltage) from meters where ts > now and ts <= now + 1h interval(1m) sliding(30s); ``` -需要说明的是，上面例子中的 `now` 是指创建连续查询的时间，而不是查询执行的时间，否则，查询就无法自动停止了。 -另外，为了尽量避免原始数据延迟写入导致的问题，TDengine中连续查询的计算有一定的延迟。 -也就是说，一个时间窗口过去后，TDengine并不会立即计算这个窗口的数据， -所以要稍等一会（一般不会超过1分钟）才能查到计算结果。 +需要说明的是，上面例子中的 `now` 是指创建连续查询的时间，而不是查询执行的时间，否则，查询就无法自动停止了。另外，为了尽量避免原始数据延迟写入导致的问题，TDengine中连续查询的计算有一定的延迟。也就是说，一个时间窗口过去后，TDengine并不会立即计算这个窗口的数据，所以要稍等一会（一般不会超过1分钟）才能查到计算结果。 ### 管理连续查询 -用户可在控制台中通过 `show streams` 命令来查看系统中全部运行的连续查询， -并可以通过 `kill stream` 命令杀掉对应的连续查询。 -后续版本会提供更细粒度和便捷的连续查询管理命令。 +用户可在控制台中通过 `show streams` 命令来查看系统中全部运行的连续查询，并可以通过 `kill stream` 命令杀掉对应的连续查询。后续版本会提供更细粒度和便捷的连续查询管理命令。 -## 数据订阅(Publisher/Subscriber) +## 数据订阅（Publisher/Subscriber） -基于数据天然的时间序列特性，TDengine的数据写入（insert）与消息系统的数据发布（pub）逻辑上一致， -均可视为系统中插入一条带时间戳的新记录。 -同时，TDengine在内部严格按照数据时间序列单调递增的方式保存数据。 -本质上来说，TDengine中里每一张表均可视为一个标准的消息队列。 +基于数据天然的时间序列特性，TDengine的数据写入（insert）与消息系统的数据发布（pub）逻辑上一致，均可视为系统中插入一条带时间戳的新记录。同时，TDengine在内部严格按照数据时间序列单调递增的方式保存数据。本质上来说，TDengine中里每一张表均可视为一个标准的消息队列。 -TDengine内嵌支持轻量级的消息订阅与推送服务。 -使用系统提供的API，用户可使用普通查询语句订阅数据库中的一张或多张表。 -订阅的逻辑和操作状态的维护均是由客户端完成，客户端定时轮询服务器是否有新的记录到达， -有新的记录到达就会将结果反馈到客户。 +TDengine内嵌支持轻量级的消息订阅与推送服务。使用系统提供的API，用户可使用普通查询语句订阅数据库中的一张或多张表。订阅的逻辑和操作状态的维护均是由客户端完成，客户端定时轮询服务器是否有新的记录到达，有新的记录到达就会将结果反馈到客户。 -TDengine的订阅与推送服务的状态是客户端维持，TDengine服务器并不维持。 -因此如果应用重启，从哪个时间点开始获取最新数据，由应用决定。 +TDengine的订阅与推送服务的状态是客户端维持，TDengine服务器并不维持。因此如果应用重启，从哪个时间点开始获取最新数据，由应用决定。 TDengine的API中，与订阅相关的主要有以下三个： @@ -116,12 +85,9 @@ taos_consume taos_unsubscribe ``` -这些API的文档请见 [C/C++ Connector](https://www.taosdata.com/cn/documentation/connector/)， -下面仍以智能电表场景为例介绍一下它们的具体用法（超级表和子表结构请参考上一节“连续查询”）， -完整的示例代码可以在 [这里](https://github.com/taosdata/TDengine/blob/master/tests/examples/c/subscribe.c) 找到。 +这些API的文档请见 [C/C++ Connector](https://www.taosdata.com/cn/documentation/connector#c-cpp)，下面仍以智能电表场景为例介绍一下它们的具体用法（超级表和子表结构请参考上一节“连续查询”），完整的示例代码可以在 [这里](https://github.com/taosdata/TDengine/blob/master/tests/examples/c/subscribe.c) 找到。 -如果我们希望当某个电表的电流超过一定限制（比如10A）后能得到通知并进行一些处理， 有两种方法： -一是分别对每张子表进行查询，每次查询后记录最后一条数据的时间戳，后续只查询这个时间戳之后的数据： +如果我们希望当某个电表的电流超过一定限制（比如10A）后能得到通知并进行一些处理， 有两种方法：一是分别对每张子表进行查询，每次查询后记录最后一条数据的时间戳，后续只查询这个时间戳之后的数据： ```sql select * from D1001 where ts > {last_timestamp1} and current > 10; @@ -129,8 +95,7 @@ select * from D1002 where ts > {last_timestamp2} and current > 10; ... ``` -这确实可行，但随着电表数量的增加，查询数量也会增加，客户端和服务端的性能都会受到影响， -当电表数增长到一定的程度，系统就无法承受了。 +这确实可行，但随着电表数量的增加，查询数量也会增加，客户端和服务端的性能都会受到影响，当电表数增长到一定的程度，系统就无法承受了。 另一种方法是对超级表进行查询。这样，无论有多少电表，都只需一次查询： @@ -138,12 +103,7 @@ select * from D1002 where ts > {last_timestamp2} and current > 10; select * from meters where ts > {last_timestamp} and current > 10; ``` -但是，如何选择 `last_timestamp` 就成了一个新的问题。 -因为，一方面数据的产生时间（也就是数据时间戳）和数据入库的时间一般并不相同，有时偏差还很大； -另一方面，不同电表的数据到达TDengine的时间也会有差异。 -所以，如果我们在查询中使用最慢的那台电表的数据的时间戳作为 `last_timestamp`， -就可能重复读入其它电表的数据； -如果使用最快的电表的时间戳，其它电表的数据就可能被漏掉。 +但是，如何选择 `last_timestamp` 就成了一个新的问题。因为，一方面数据的产生时间（也就是数据时间戳）和数据入库的时间一般并不相同，有时偏差还很大；另一方面，不同电表的数据到达TDengine的时间也会有差异。所以，如果我们在查询中使用最慢的那台电表的数据的时间戳作为 `last_timestamp`，就可能重复读入其它电表的数据；如果使用最快的电表的时间戳，其它电表的数据就可能被漏掉。 TDengine的订阅功能为上面这个问题提供了一个彻底的解决方案。 @@ -160,47 +120,29 @@ if (async) { } ``` -TDengine中的订阅既可以是同步的，也可以是异步的， -上面的代码会根据从命令行获取的参数`async`的值来决定使用哪种方式。 -这里，同步的意思是用户程序要直接调用`taos_consume`来拉取数据， -而异步则由API在内部的另一个线程中调用`taos_consume`， -然后把拉取到的数据交给回调函数`subscribe_callback`去处理。 +TDengine中的订阅既可以是同步的，也可以是异步的，上面的代码会根据从命令行获取的参数`async`的值来决定使用哪种方式。这里，同步的意思是用户程序要直接调用`taos_consume`来拉取数据，而异步则由API在内部的另一个线程中调用`taos_consume`，然后把拉取到的数据交给回调函数`subscribe_callback`去处理。 -参数`taos`是一个已经建立好的数据库连接，在同步模式下无特殊要求。 -但在异步模式下，需要注意它不会被其它线程使用，否则可能导致不可预计的错误， -因为回调函数在API的内部线程中被调用，而TDengine的部分API不是线程安全的。 +参数`taos`是一个已经建立好的数据库连接，在同步模式下无特殊要求。但在异步模式下，需要注意它不会被其它线程使用，否则可能导致不可预计的错误，因为回调函数在API的内部线程中被调用，而TDengine的部分API不是线程安全的。 -参数`sql`是查询语句，可以在其中使用where子句指定过滤条件。 -在我们的例子中，如果只想订阅电流超过10A时的数据，可以这样写： +参数`sql`是查询语句，可以在其中使用where子句指定过滤条件。在我们的例子中，如果只想订阅电流超过10A时的数据，可以这样写： ```sql select * from meters where current > 10; ``` -注意，这里没有指定起始时间，所以会读到所有时间的数据。 -如果只想从一天前的数据开始订阅，而不需要更早的历史数据，可以再加上一个时间条件： +注意，这里没有指定起始时间，所以会读到所有时间的数据。如果只想从一天前的数据开始订阅，而不需要更早的历史数据，可以再加上一个时间条件： ```sql select * from meters where ts > now - 1d and current > 10; ``` -订阅的`topic`实际上是它的名字，因为订阅功能是在客户端API中实现的， -所以没必要保证它全局唯一，但需要它在一台客户端机器上唯一。 +订阅的`topic`实际上是它的名字，因为订阅功能是在客户端API中实现的，所以没必要保证它全局唯一，但需要它在一台客户端机器上唯一。 -如果名`topic`的订阅不存在，参数`restart`没有意义； -但如果用户程序创建这个订阅后退出，当它再次启动并重新使用这个`topic`时， -`restart`就会被用于决定是从头开始读取数据，还是接续上次的位置进行读取。 -本例中，如果`restart`是 **true**（非零值），用户程序肯定会读到所有数据。 -但如果这个订阅之前就存在了，并且已经读取了一部分数据， -且`restart`是 **false**（**0**），用户程序就不会读到之前已经读取的数据了。 +如果名`topic`的订阅不存在，参数`restart`没有意义；但如果用户程序创建这个订阅后退出，当它再次启动并重新使用这个`topic`时，`restart`就会被用于决定是从头开始读取数据，还是接续上次的位置进行读取。本例中，如果`restart`是 **true**（非零值），用户程序肯定会读到所有数据。但如果这个订阅之前就存在了，并且已经读取了一部分数据，且`restart`是 **false**（**0**），用户程序就不会读到之前已经读取的数据了。 -`taos_subscribe`的最后一个参数是以毫秒为单位的轮询周期。 -在同步模式下，如果前后两次调用`taos_consume`的时间间隔小于此时间， -`taos_consume`会阻塞，直到间隔超过此时间。 -异步模式下，这个时间是两次调用回调函数的最小时间间隔。 +`taos_subscribe`的最后一个参数是以毫秒为单位的轮询周期。在同步模式下，如果前后两次调用`taos_consume`的时间间隔小于此时间，`taos_consume`会阻塞，直到间隔超过此时间。异步模式下，这个时间是两次调用回调函数的最小时间间隔。 -`taos_subscribe`的倒数第二个参数用于用户程序向回调函数传递附加参数， -订阅API不对其做任何处理，只原样传递给回调函数。此参数在同步模式下无意义。 +`taos_subscribe`的倒数第二个参数用于用户程序向回调函数传递附加参数，订阅API不对其做任何处理，只原样传递给回调函数。此参数在同步模式下无意义。 订阅创建以后，就可以消费其数据了，同步模式下，示例代码是下面的 else 部分： @@ -219,9 +161,7 @@ if (async) { } ``` -这里是一个 **while** 循环，用户每按一次回车键就调用一次`taos_consume`， -而`taos_consume`的返回值是查询到的结果集，与`taos_use_result`完全相同， -例子中使用这个结果集的代码是函数`print_result`： +这里是一个 **while** 循环，用户每按一次回车键就调用一次`taos_consume`，而`taos_consume`的返回值是查询到的结果集，与`taos_use_result`完全相同，例子中使用这个结果集的代码是函数`print_result`： ```c void print_result(TAOS_RES* res, int blockFetch) { @@ -247,8 +187,7 @@ void print_result(TAOS_RES* res, int blockFetch) { } ``` -其中的 `taos_print_row` 用于处理订阅到数据，在我们的例子中，它会打印出所有符合条件的记录。 -而异步模式下，消费订阅到的数据则显得更为简单： +其中的 `taos_print_row` 用于处理订阅到数据，在我们的例子中，它会打印出所有符合条件的记录。而异步模式下，消费订阅到的数据则显得更为简单： ```c void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { @@ -262,11 +201,7 @@ void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { taos_unsubscribe(tsub, keep); ``` -其第二个参数，用于决定是否在客户端保留订阅的进度信息。 -如果这个参数是**false**（**0**），那无论下次调用`taos_subscribe`的时的`restart`参数是什么， -订阅都只能重新开始。 -另外，进度信息的保存位置是 *{DataDir}/subscribe/* 这个目录下， -每个订阅有一个与其`topic`同名的文件，删掉某个文件，同样会导致下次创建其对应的订阅时只能重新开始。 +其第二个参数，用于决定是否在客户端保留订阅的进度信息。如果这个参数是**false**（**0**），那无论下次调用`taos_subscribe`时的`restart`参数是什么，订阅都只能重新开始。另外，进度信息的保存位置是 *{DataDir}/subscribe/* 这个目录下，每个订阅有一个与其`topic`同名的文件，删掉某个文件，同样会导致下次创建其对应的订阅时只能重新开始。 代码介绍完毕，我们来看一下实际的运行效果。假设： @@ -289,12 +224,11 @@ $ taos > insert into D1001 values(now, 12, 220, 1); ``` -这时，因为电流超过了10A，您应该可以看到示例程序将它输出到了屏幕上。 -您可以继续插入一些数据观察示例程序的输出。 +这时，因为电流超过了10A，您应该可以看到示例程序将它输出到了屏幕上。您可以继续插入一些数据观察示例程序的输出。 ### Java 使用数据订阅功能 -订阅功能也提供了 Java 开发接口，相关说明请见 [Java Connector](https://www.taosdata.com/cn/documentation/connector/)。需要注意的是，目前 Java 接口没有提供异步订阅模式，但用户程序可以通过创建 `TimerTask` 等方式达到同样的效果。 +订阅功能也提供了 Java 开发接口，相关说明请见 [Java Connector](https://www.taosdata.com/cn/documentation/connector/java#subscribe)。需要注意的是，目前 Java 接口没有提供异步订阅模式，但用户程序可以通过创建 `TimerTask` 等方式达到同样的效果。 下面以一个示例程序介绍其具体使用方法。它所完成的功能与前面介绍的 C 语言示例基本相同，也是订阅数据库中所有电流超过 10A 的记录。 @@ -404,7 +338,7 @@ ts: 1597466400000 current: 12.4 voltage: 220 phase: 1 location: Beijing.Chaoyang ``` -## 缓存(Cache) +## 缓存（Cache） TDengine采用时间驱动缓存管理策略（First-In-First-Out，FIFO），又称为写驱动的缓存管理机制。这种策略有别于读驱动的数据缓存模式（Least-Recent-Use，LRU），直接将最近写入的数据保存在系统的缓存中。当缓存达到临界值的时候，将最早的数据批量写入磁盘。一般意义上来说，对于物联网数据的使用，用户最为关心最近产生的数据，即当前状态。TDengine充分利用了这一特性，将最近到达的（当前状态）数据保存在缓存中。 @@ -423,7 +357,7 @@ select last_row(voltage) from meters where location='Beijing.Chaoyang'; 该SQL语句将获取所有位于北京朝阳区的电表最后记录的电压值。 -## 报警监测(Alert) +## 报警监测（Alert） 在 TDengine 的应用场景中，报警监测是一个常见需求，从概念上说，它要求程序从最近一段时间的数据中筛选出符合一定条件的数据，并基于这些数据根据定义好的公式计算出一个结果，当这个结果符合某个条件且持续一定时间后，以某种形式通知用户。 diff --git a/documentation20/cn/08.connector/01.java/docs.md b/documentation20/cn/08.connector/01.java/docs.md index 30b485f9b6..3442a2248c 100644 --- a/documentation20/cn/08.connector/01.java/docs.md +++ b/documentation20/cn/08.connector/01.java/docs.md @@ -285,7 +285,7 @@ JDBC连接器可能报错的错误码包括3种：JDBC driver本身的报错（ * https://github.com/taosdata/TDengine/blob/develop/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java * https://github.com/taosdata/TDengine/blob/develop/src/inc/taoserror.h -### 订阅 +### 订阅 #### 创建 From 0bbfc5a532d94ed49a35f2164ee84c9cd7aeeaa1 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 31 Mar 2021 18:32:56 +0800 Subject: [PATCH 62/71] Bxiao request changed from 100 to 1000 --- src/inc/taosdef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 252e37a5d9..024bc198df 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -259,7 +259,7 @@ do { \ #define TSDB_MIN_TABLES 4 #define TSDB_MAX_TABLES 10000000 #define TSDB_DEFAULT_TABLES 1000000 -#define TSDB_TABLES_STEP 100 +#define TSDB_TABLES_STEP 1000 #define TSDB_MIN_DAYS_PER_FILE 1 #define TSDB_MAX_DAYS_PER_FILE 3650 From 6cb5af8072ae1d4a196c7d666d459654146aa8e7 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 31 Mar 2021 21:40:40 +0800 Subject: [PATCH 63/71] Hotfix/sangshuduo/td 3607 taosdemo buffer overflow (#5636) * [TD-3607] : fix taosdemo buffer overflow. * [TD-3607] : taosdemo buffer overflow. add tmp buffer. * [TD-3607] : taosdemo buffer overflow. fix data generation. * [TD-3607] : taosdemo buffer overflow. fix normal table writting. * [TD-3607] : taosdemo buffer overflow. remove tail spaces. * [TD-3607] : taosdemo buffer overflow. fix taosdemo alter table test case. * [TD-3607] : taosdemo buffer overflow. fix taosdemo alter table case. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 240 ++++++++++-------- tests/pytest/fulltest.sh | 24 +- ...taosdemoTest2.py => taosdemoTestTblAlt.py} | 50 +++- 3 files changed, 185 insertions(+), 129 deletions(-) rename tests/pytest/tools/{taosdemoTest2.py => taosdemoTestTblAlt.py} (63%) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 3725712339..9f367b41f8 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -124,7 +124,7 @@ typedef enum enum_INSERT_MODE { typedef enum enumQUERY_TYPE { NO_INSERT_TYPE, - INSERT_TYPE, + INSERT_TYPE, QUERY_TYPE_BUT } QUERY_TYPE; @@ -229,7 +229,7 @@ typedef struct SColumn_S { typedef struct SSuperTable_S { char sTblName[MAX_TB_NAME_SIZE+1]; int childTblCount; - bool childTblExists; // 0: no, 1: yes + bool childTblExists; // 0: no, 1: yes int batchCreateTableNum; // 0: no batch, > 0: batch table number in one sql int8_t autoCreateTable; // 0: create sub table, 1: auto create sub table char childTblPrefix[MAX_TB_NAME_SIZE]; @@ -239,15 +239,15 @@ typedef struct SSuperTable_S { int childTblOffset; int multiThreadWriteOneTbl; // 0: no, 1: yes - int interlaceRows; // + int interlaceRows; // int disorderRatio; // 0: no disorder, >0: x% int disorderRange; // ms or us by database precision - int maxSqlLen; // + int maxSqlLen; // int insertInterval; // insert interval, will override global insert interval int64_t insertRows; // 0: no limit int timeStampStep; - char startTimestamp[MAX_TB_NAME_SIZE]; // + char startTimestamp[MAX_TB_NAME_SIZE]; // char sampleFormat[MAX_TB_NAME_SIZE]; // csv, json char sampleFile[MAX_FILE_NAME_LEN+1]; char tagsFile[MAX_FILE_NAME_LEN+1]; @@ -539,7 +539,7 @@ SArguments g_args = { true, // insert_only false, // debug_print false, // verbose_print - false, // performance statistic print + false, // performance statistic print false, // answer_yes; "./output.txt", // output_file 0, // mode : sync or async @@ -641,7 +641,7 @@ static void printHelp() { "The password to use when connecting to the server. Default is 'taosdata'."); printf("%s%s%s%s\n", indent, "-c", indent, "Configuration directory. Default is '/etc/taos/'."); -#endif +#endif printf("%s%s%s%s\n", indent, "-h", indent, "The host to connect to TDengine. Default is localhost."); printf("%s%s%s%s\n", indent, "-p", indent, @@ -684,7 +684,7 @@ static void printHelp() { "Print debug info."); printf("%s%s%s%s\n", indent, "-V, --version", indent, "Print version info."); -/* printf("%s%s%s%s\n", indent, "-D", indent, +/* printf("%s%s%s%s\n", indent, "-D", indent, "if elete database if exists. 0: no, 1: yes, default is 1"); */ } @@ -749,7 +749,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { && strcasecmp(argv[i], "SMALLINT") && strcasecmp(argv[i], "BIGINT") && strcasecmp(argv[i], "DOUBLE") - && strcasecmp(argv[i], "BINARY") + && strcasecmp(argv[i], "BINARY") && strcasecmp(argv[i], "NCHAR")) { printHelp(); ERROR_EXIT( "Invalid data_type!\n"); @@ -1762,7 +1762,7 @@ static void printfQuerySystemInfo(TAOS * taos) { } for (int i = 0; i < dbCount; i++) { - // printf database info + // printf database info printfDbInfoForQueryToFile(filename, dbInfos[i], i); // show db.vgroups @@ -2098,7 +2098,7 @@ static int calcRowLen(SSuperTable* superTbls) { lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; } else if (strcasecmp(dataType, "FLOAT") == 0) { lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22; - } else if (strcasecmp(dataType, "DOUBLE") == 0) { + } else if (strcasecmp(dataType, "DOUBLE") == 0) { lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42; } else { printf("get error tag type : %s\n", dataType); @@ -2262,7 +2262,7 @@ static int getSuperTableFromServer(TAOS * taos, char* dbName, /* if (TBL_ALREADY_EXISTS == superTbls->childTblExists) { - //get all child table name use cmd: select tbname from superTblName; + //get all child table name use cmd: select tbname from superTblName; int childTblCount = 10000; superTbls->childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN); if (superTbls->childTblName == NULL) { @@ -2289,7 +2289,7 @@ static int createSuperTable(TAOS * taos, char* dbName, int lenOfOneRow = 0; for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) { char* dataType = superTbls->columns[colIndex].dataType; - + if (strcasecmp(dataType, "BINARY") == 0) { len += snprintf(cols + len, STRING_LEN - len, ", col%d %s(%d)", colIndex, "BINARY", @@ -2386,7 +2386,7 @@ static int createSuperTable(TAOS * taos, char* dbName, len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "FLOAT"); lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22; - } else if (strcasecmp(dataType, "DOUBLE") == 0) { + } else if (strcasecmp(dataType, "DOUBLE") == 0) { len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "DOUBLE"); lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42; @@ -2638,7 +2638,7 @@ static void* createTable(void *sarg) lastPrintTime = currentPrintTime; } } - + if (0 != len) { verbosePrint("%s() %d buffer: %s\n", __func__, __LINE__, buffer); if (0 != queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE, false)) { @@ -2703,7 +2703,7 @@ static int startMultiThreadCreateChildTable( t_info->minDelay = INT16_MAX; pthread_create(pids + i, NULL, createTable, t_info); } - + for (int i = 0; i < threads; i++) { pthread_join(pids[i], NULL); } @@ -2920,7 +2920,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( cJSON* stbInfo, SSuperTable* superTbls) { bool ret = false; - // columns + // columns cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns"); if (columns && columns->type != cJSON_Array) { printf("ERROR: failed to read json, columns not found\n"); @@ -2958,7 +2958,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( count = 1; } - // column info + // column info memset(&columnCase, 0, sizeof(StrColumn)); cJSON *dataType = cJSON_GetObjectItem(column, "type"); if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) { @@ -2989,7 +2989,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( count = 1; index = 0; - // tags + // tags cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags"); if (!tags || tags->type != cJSON_Array) { debugPrint("%s() LN%d, failed to read json, tags not found\n", __func__, __LINE__); @@ -3018,7 +3018,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( count = 1; } - // column info + // column info memset(&columnCase, 0, sizeof(StrColumn)); cJSON *dataType = cJSON_GetObjectItem(tag, "type"); if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) { @@ -3166,7 +3166,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (numRecPerReq && numRecPerReq->type == cJSON_Number) { g_args.num_of_RPR = numRecPerReq->valueint; } else if (!numRecPerReq) { - g_args.num_of_RPR = 100; + g_args.num_of_RPR = 0xffff; } else { errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n", __func__, __LINE__); goto PARSE_OVER; @@ -3209,7 +3209,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { cJSON* dbinfos = cJSON_GetArrayItem(dbs, i); if (dbinfos == NULL) continue; - // dbinfo + // dbinfo cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo"); if (!dbinfo || dbinfo->type != cJSON_Object) { printf("ERROR: failed to read json, dbinfo not found\n"); @@ -3615,7 +3615,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } g_Dbs.db[i].superTbls[j].maxSqlLen = len; } else if (!maxSqlLen) { - g_Dbs.db[i].superTbls[j].maxSqlLen = TSDB_MAX_SQL_LEN; + g_Dbs.db[i].superTbls[j].maxSqlLen = g_args.max_sql_len; } else { printf("ERROR: failed to read json, maxSqlLen not found\n"); goto PARSE_OVER; @@ -3748,7 +3748,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* user = cJSON_GetObjectItem(root, "user"); if (user && user->type == cJSON_String && user->valuestring != NULL) { - tstrncpy(g_queryInfo.user, user->valuestring, MAX_USERNAME_SIZE); + tstrncpy(g_queryInfo.user, user->valuestring, MAX_USERNAME_SIZE); } else if (!user) { tstrncpy(g_queryInfo.user, "root", MAX_USERNAME_SIZE); ; } @@ -3805,7 +3805,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { goto PARSE_OVER; } - // super_table_query + // super_table_query cJSON *specifiedQuery = cJSON_GetObjectItem(root, "specified_table_query"); if (!specifiedQuery) { g_queryInfo.specifiedQueryInfo.concurrent = 0; @@ -3930,7 +3930,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } } - // sub_table_query + // sub_table_query cJSON *superQuery = cJSON_GetObjectItem(root, "super_table_query"); if (!superQuery) { g_queryInfo.superQueryInfo.threadCnt = 0; @@ -3996,7 +3996,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* subinterval = cJSON_GetObjectItem(superQuery, "interval"); if (subinterval && subinterval->type == cJSON_Number) { g_queryInfo.superQueryInfo.subscribeInterval = subinterval->valueint; - } else if (!subinterval) { + } else if (!subinterval) { //printf("failed to read json, subscribe interval no found\n"); //goto PARSE_OVER; g_queryInfo.superQueryInfo.subscribeInterval = 10000; @@ -4200,71 +4200,77 @@ static int getRowDataFromSample(char* dataBuf, int maxLen, int64_t timestamp, dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")"); (*sampleUsePos)++; - + return dataLen; } -static int generateRowData(char* dataBuf, int maxLen, int64_t timestamp, SSuperTable* stbInfo) { - int dataLen = 0; - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "(%" PRId64 ", ", timestamp); +static int generateRowData(char* recBuf, int64_t timestamp, SSuperTable* stbInfo) { + int dataLen = 0; + char *pstr = recBuf; + int maxLen = MAX_DATA_SIZE; + + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "(%" PRId64 ", ", timestamp); + for (int i = 0; i < stbInfo->columnCount; i++) { if ((0 == strncasecmp(stbInfo->columns[i].dataType, "binary", 6)) || (0 == strncasecmp(stbInfo->columns[i].dataType, "nchar", 5))) { if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) { errorPrint( "binary or nchar length overflow, max size:%u\n", (uint32_t)TSDB_MAX_BINARY_LEN); - return (-1); + return -1; } char* buf = (char*)calloc(stbInfo->columns[i].dataLen+1, 1); if (NULL == buf) { errorPrint( "calloc failed! size:%d\n", stbInfo->columns[i].dataLen); - return (-1); + return -1; } rand_string(buf, stbInfo->columns[i].dataLen); - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "\'%s\', ", buf); + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "\'%s\', ", buf); tmfree(buf); } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "int", 3)) { - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%d, ", rand_int()); } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "bigint", 6)) { - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%"PRId64", ", rand_bigint()); } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "float", 5)) { - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%f, ", rand_float()); } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "double", 6)) { - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%f, ", rand_double()); } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "smallint", 8)) { - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_smallint()); + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%d, ", rand_smallint()); } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "tinyint", 7)) { - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_tinyint()); + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%d, ", rand_tinyint()); } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "bool", 4)) { - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_bool()); + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%d, ", rand_bool()); } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "timestamp", 9)) { - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%"PRId64", ", rand_bigint()); + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%"PRId64", ", rand_bigint()); } else { errorPrint( "No support data type: %s\n", stbInfo->columns[i].dataType); - return (-1); + return -1; } } dataLen -= 2; - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")"); + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, ")"); - return dataLen; + verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf); + + return strlen(recBuf); } -static int32_t generateData(char *res, char **data_type, +static int32_t generateData(char *recBuf, char **data_type, int num_of_cols, int64_t timestamp, int lenOfBinary) { - memset(res, 0, MAX_DATA_SIZE); - char *pstr = res; + memset(recBuf, 0, MAX_DATA_SIZE); + char *pstr = recBuf; pstr += sprintf(pstr, "(%" PRId64, timestamp); int c = 0; @@ -4285,7 +4291,7 @@ static int32_t generateData(char *res, char **data_type, } else if (strcasecmp(data_type[i % c], "smallint") == 0) { pstr += sprintf(pstr, ", %d", rand_smallint()); } else if (strcasecmp(data_type[i % c], "int") == 0) { - pstr += sprintf(pstr, ", %d", rand_int()); + pstr += sprintf(pstr, ", %d", rand_int()); } else if (strcasecmp(data_type[i % c], "bigint") == 0) { pstr += sprintf(pstr, ", %" PRId64, rand_bigint()); } else if (strcasecmp(data_type[i % c], "float") == 0) { @@ -4308,7 +4314,7 @@ static int32_t generateData(char *res, char **data_type, free(s); } - if (pstr - res > MAX_DATA_SIZE) { + if (strlen(recBuf) > MAX_DATA_SIZE) { perror("column length too long, abort"); exit(-1); } @@ -4316,7 +4322,7 @@ static int32_t generateData(char *res, char **data_type, pstr += sprintf(pstr, ")"); - return (int32_t)(pstr - res); + return (int32_t)strlen(recBuf); } static int prepareSampleDataForSTable(SSuperTable *superTblInfo) { @@ -4325,9 +4331,9 @@ static int prepareSampleDataForSTable(SSuperTable *superTblInfo) { sampleDataBuf = calloc( superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1); if (sampleDataBuf == NULL) { - errorPrint("%s() LN%d, Failed to calloc %d Bytes, reason:%s\n", + errorPrint("%s() LN%d, Failed to calloc %d Bytes, reason:%s\n", __func__, __LINE__, - superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, + superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, strerror(errno)); return -1; } @@ -4396,7 +4402,7 @@ static void getTableName(char *pTblName, threadInfo* pThreadInfo, int tableSeq) static int generateDataTail(char *tableName, int32_t tableSeq, threadInfo* pThreadInfo, SSuperTable* superTblInfo, - int batch, char* buffer, int64_t insertRows, + int batch, char* buffer, int remainderBufLen, int64_t insertRows, int64_t startFrom, uint64_t startTime, int *pSamplePos, int *dataLen) { int len = 0; int ncols_per_record = 1; // count first col ts @@ -4413,18 +4419,19 @@ static int generateDataTail(char *tableName, int32_t tableSeq, int k = 0; for (k = 0; k < batch;) { - if (superTblInfo) { - int retLen = 0; + char data[MAX_DATA_SIZE]; + int retLen = 0; - if (0 == strncasecmp(superTblInfo->dataSource, + if (superTblInfo) { + if (0 == strncasecmp(superTblInfo->dataSource, "sample", strlen("sample"))) { retLen = getRowDataFromSample( - buffer + len, - superTblInfo->maxSqlLen - len, + data, + remainderBufLen, startTime + superTblInfo->timeStampStep * k, superTblInfo, pSamplePos); - } else if (0 == strncasecmp(superTblInfo->dataSource, + } else if (0 == strncasecmp(superTblInfo->dataSource, "rand", strlen("rand"))) { int rand_num = rand_tinyint() % 100; if (0 != superTblInfo->disorderRatio @@ -4433,60 +4440,56 @@ static int generateDataTail(char *tableName, int32_t tableSeq, + superTblInfo->timeStampStep * k - taosRandom() % superTblInfo->disorderRange; retLen = generateRowData( - buffer + len, - superTblInfo->maxSqlLen - len, + data, d, superTblInfo); } else { retLen = generateRowData( - buffer + len, - superTblInfo->maxSqlLen - len, + data, startTime + superTblInfo->timeStampStep * k, superTblInfo); - } - } + } + } - if (retLen < 0) { - return -1; - } + if (retLen > remainderBufLen) { + break; + } - len += retLen; - - if (len >= (superTblInfo->maxSqlLen - 256)) { // reserve for overwrite - k++; - break; - } + buffer += sprintf(buffer, " %s", data); + k++; + len += retLen; + remainderBufLen -= retLen; } else { int rand_num = taosRandom() % 100; - char data[MAX_DATA_SIZE]; - char **data_type = g_args.datatype; - int lenOfBinary = g_args.len_of_binary; + char **data_type = g_args.datatype; + int lenOfBinary = g_args.len_of_binary; if ((g_args.disorderRatio != 0) && (rand_num < g_args.disorderRange)) { - int64_t d = startTime + DEFAULT_TIMESTAMP_STEP * k + int64_t d = startTime + DEFAULT_TIMESTAMP_STEP * k - taosRandom() % 1000000 + rand_num; - len = generateData(data, data_type, + retLen = generateData(data, data_type, ncols_per_record, d, lenOfBinary); } else { - len = generateData(data, data_type, + retLen = generateData(data, data_type, ncols_per_record, startTime + DEFAULT_TIMESTAMP_STEP * k, lenOfBinary); } + if (len > remainderBufLen) + break; + buffer += sprintf(buffer, " %s", data); - if (strlen(buffer) >= (g_args.max_sql_len - 256)) { // too long - k++; - break; - } + k++; + len += retLen; + remainderBufLen -= retLen; } verbosePrint("%s() LN%d len=%d k=%d \nbuffer=%s\n", __func__, __LINE__, len, k, buffer); - k++; startFrom ++; if (startFrom >= insertRows) { @@ -4570,20 +4573,25 @@ static int generateProgressiveDataBuffer(char *pTblName, assert(buffer != NULL); - memset(buffer, 0, superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len); + int maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len; + int remainderBufLen = maxSqlLen; + + memset(buffer, 0, maxSqlLen); char *pstr = buffer; int headLen = generateSQLHead(pTblName, tableSeq, pThreadInfo, superTblInfo, buffer); pstr += headLen; + remainderBufLen -= headLen; int k; int dataLen; k = generateDataTail(pTblName, tableSeq, pThreadInfo, superTblInfo, - g_args.num_of_RPR, pstr, insertRows, startFrom, + g_args.num_of_RPR, pstr, remainderBufLen, insertRows, startFrom, startTime, pSamplePos, &dataLen); + return k; } @@ -4656,13 +4664,18 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { int generatedRecPerTbl = 0; bool flagSleep = true; int sleepTimeTotal = 0; + + int maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len; + int remainderBufLen; + while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) { if ((flagSleep) && (insert_interval)) { st = taosGetTimestampUs(); flagSleep = false; } // generate data - memset(buffer, 0, superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len); + memset(buffer, 0, maxSqlLen); + remainderBufLen = maxSqlLen; char *pstr = buffer; int recOfBatch = 0; @@ -4685,6 +4698,8 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pThreadInfo->threadID, __func__, __LINE__, i, buffer); pstr += headLen; + remainderBufLen -= headLen; + int dataLen = 0; verbosePrint("[%d] %s() LN%d i=%d batchPerTblTimes=%d batchPerTbl = %d\n", @@ -4698,13 +4713,20 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { } else { startTime = 1500000000000; } - generateDataTail( + int generated = generateDataTail( tableName, tableSeq, pThreadInfo, superTblInfo, - batchPerTbl, pstr, insertRows, 0, + batchPerTbl, pstr, remainderBufLen, insertRows, 0, startTime, &(pThreadInfo->samplePos), &dataLen); + if (generated < 0) { + debugPrint("[%d] %s() LN%d, generated data is %d\n", + pThreadInfo->threadID, __func__, __LINE__, generated); + goto free_and_statistics_interlace; + } pstr += dataLen; + remainderBufLen -= dataLen; + recOfBatch += batchPerTbl; startTime += batchPerTbl * superTblInfo->timeStampStep; pThreadInfo->totalInsertRows += batchPerTbl; @@ -4796,9 +4818,9 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { free_and_statistics_interlace: tmfree(buffer); - printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n", - pThreadInfo->threadID, - pThreadInfo->totalInsertRows, + printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n", + pThreadInfo->threadID, + pThreadInfo->totalInsertRows, pThreadInfo->totalAffectedRows); return NULL; } @@ -4929,16 +4951,16 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { free_and_statistics_2: tmfree(buffer); - printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n", - pThreadInfo->threadID, - pThreadInfo->totalInsertRows, + printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n", + pThreadInfo->threadID, + pThreadInfo->totalInsertRows, pThreadInfo->totalAffectedRows); return NULL; } static void* syncWrite(void *sarg) { - threadInfo *winfo = (threadInfo *)sarg; + threadInfo *winfo = (threadInfo *)sarg; SSuperTable* superTblInfo = winfo->superTblInfo; int interlaceRows = superTblInfo?superTblInfo->interlaceRows:g_args.interlace_rows; @@ -4953,7 +4975,7 @@ static void* syncWrite(void *sarg) { } static void callBack(void *param, TAOS_RES *res, int code) { - threadInfo* winfo = (threadInfo*)param; + threadInfo* winfo = (threadInfo*)param; SSuperTable* superTblInfo = winfo->superTblInfo; int insert_interval = @@ -4966,7 +4988,7 @@ static void callBack(void *param, TAOS_RES *res, int code) { } char *buffer = calloc(1, winfo->superTblInfo->maxSqlLen); - char *data = calloc(1, MAX_DATA_SIZE); + char data[MAX_DATA_SIZE]; char *pstr = buffer; pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix, winfo->start_table_from); @@ -4978,7 +5000,6 @@ static void callBack(void *param, TAOS_RES *res, int code) { if (winfo->start_table_from > winfo->end_table_to) { tsem_post(&winfo->lock_sem); free(buffer); - free(data); taos_free_result(res); return; } @@ -4988,11 +5009,9 @@ static void callBack(void *param, TAOS_RES *res, int code) { if (0 != winfo->superTblInfo->disorderRatio && rand_num < winfo->superTblInfo->disorderRatio) { int64_t d = winfo->lastTs - taosRandom() % 1000000 + rand_num; - //generateData(data, datatype, ncols_per_record, d, len_of_binary); - generateRowData(data, MAX_DATA_SIZE, d, winfo->superTblInfo); + generateRowData(data, d, winfo->superTblInfo); } else { - //generateData(data, datatype, ncols_per_record, start_time += 1000, len_of_binary); - generateRowData(data, MAX_DATA_SIZE, winfo->lastTs += 1000, winfo->superTblInfo); + generateRowData(data, winfo->lastTs += 1000, winfo->superTblInfo); } pstr += sprintf(pstr, "%s", data); winfo->counter++; @@ -5007,7 +5026,6 @@ static void callBack(void *param, TAOS_RES *res, int code) { } taos_query_a(winfo->taos, buffer, callBack, winfo); free(buffer); - free(data); taos_free_result(res); } @@ -5373,7 +5391,7 @@ static void *readTable(void *sarg) { } static void *readMetric(void *sarg) { -#if 1 +#if 1 threadInfo *rinfo = (threadInfo *)sarg; TAOS *taos = rinfo->taos; char command[BUFFER_SIZE] = "\0"; @@ -5524,7 +5542,7 @@ static int insertTestProcess() { //int64_t totalInsertRows = 0; //int64_t totalAffectedRows = 0; - //for (int i = 0; i < g_Dbs.dbCount; i++) { + //for (int i = 0; i < g_Dbs.dbCount; i++) { // for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { // totalInsertRows+= g_Dbs.db[i].superTbls[j].totalInsertRows; // totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows; @@ -5921,7 +5939,7 @@ static void *subSubscribeProcess(void *sarg) { sprintf(tmpFile, "%s-%d", g_queryInfo.superQueryInfo.result[i], winfo->threadID); } - tsub[i] = subscribeImpl(winfo->taos, subSqlstr, topic, tmpFile); + tsub[i] = subscribeImpl(winfo->taos, subSqlstr, topic, tmpFile); if (NULL == tsub[i]) { taos_close(winfo->taos); return NULL; @@ -6109,7 +6127,7 @@ static int subscribeTestProcess() { && (g_queryInfo.superQueryInfo.threadCnt > 0)) { pidsOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t)); - infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * + infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo)); if ((NULL == pidsOfSub) || (NULL == infosOfSub)) { errorPrint("%s() LN%d, malloc failed for create threads\n", @@ -6256,7 +6274,7 @@ static void setParaFromArg(){ g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP; g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT; - g_Dbs.db[0].superTbls[0].maxSqlLen = TSDB_PAYLOAD_SIZE; + g_Dbs.db[0].superTbls[0].maxSqlLen = g_args.max_sql_len; g_Dbs.db[0].superTbls[0].columnCount = 0; for (int i = 0; i < MAX_NUM_DATATYPE; i++) { diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index c3282ce6f7..8e15d1b5ec 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -139,6 +139,18 @@ python3 ./test.py -f import_merge/importInsertThenImport.py python3 ./test.py -f import_merge/importCSV.py #======================p1-end=============== #======================p2-start=============== +# tools +python3 test.py -f tools/taosdumpTest.py + +python3 test.py -f tools/taosdemoTest.py +python3 test.py -f tools/taosdemoTestWithoutMetric.py +python3 test.py -f tools/taosdemoTestWithJson.py +python3 test.py -f tools/taosdemoTestLimitOffset.py +python3 test.py -f tools/taosdemoTestTblAlt.py +python3 test.py -f tools/taosdemoTestSampleData.py +python3 test.py -f tools/taosdemoTestInterlace.py +python3 test.py -f tools/taosdemoTestQuery.py + # update python3 ./test.py -f update/allow_update.py python3 ./test.py -f update/allow_update-0.py @@ -247,18 +259,6 @@ python3 test.py -f subscribe/supertable.py #======================p3-end=============== #======================p4-start=============== -# tools -python3 test.py -f tools/taosdumpTest.py - -python3 test.py -f tools/taosdemoTest.py -python3 test.py -f tools/taosdemoTestWithoutMetric.py -python3 test.py -f tools/taosdemoTestWithJson.py -python3 test.py -f tools/taosdemoTestLimitOffset.py -python3 test.py -f tools/taosdemoTest2.py -python3 test.py -f tools/taosdemoTestSampleData.py -python3 test.py -f tools/taosdemoTestInterlace.py -python3 test.py -f tools/taosdemoTestQuery.py - python3 ./test.py -f update/merge_commit_data-0.py # wal python3 ./test.py -f wal/addOldWalTest.py diff --git a/tests/pytest/tools/taosdemoTest2.py b/tests/pytest/tools/taosdemoTestTblAlt.py similarity index 63% rename from tests/pytest/tools/taosdemoTest2.py rename to tests/pytest/tools/taosdemoTestTblAlt.py index 74b05faf8b..bb367817cf 100644 --- a/tests/pytest/tools/taosdemoTest2.py +++ b/tests/pytest/tools/taosdemoTestTblAlt.py @@ -29,10 +29,33 @@ class TDTestCase: self.numberOfTables = 10 self.numberOfRecords = 1000000 + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + def insertDataAndAlterTable(self, threadID): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath + "/build/bin/" + if(threadID == 0): - os.system("taosdemo -y -t %d -n %d" % - (self.numberOfTables, self.numberOfRecords)) + os.system("%staosdemo -y -t %d -n %d" % + (binPath, self.numberOfTables, self.numberOfRecords)) if(threadID == 1): time.sleep(2) print("use test") @@ -47,7 +70,13 @@ class TDTestCase: # check if all the tables have heen created while True: - tdSql.query("show tables") + try: + tdSql.query("show tables") + except Exception as e: + tdLog.info("show tables test failed") + time.sleep(1) + continue + rows = tdSql.queryRows print("number of tables: %d" % rows) if(rows == self.numberOfTables): @@ -56,16 +85,23 @@ class TDTestCase: # check if there are any records in the last created table while True: print("query started") - tdSql.query("select * from test.t9") + try: + tdSql.query("select * from test.t9") + except Exception as e: + tdLog.info("select * test failed") + time.sleep(2) + continue + rows = tdSql.queryRows print("number of records: %d" % rows) if(rows > 0): break time.sleep(1) + print("alter table test.meters add column col10 int") tdSql.execute("alter table test.meters add column col10 int") - print("insert into test.t0 values (now, 1, 2, 3, 4, 0.1, 0.01,'test', '测试', TRUE, 1610000000000, 0)") - tdSql.execute("insert into test.t0 values (now, 1, 2, 3, 4, 0.1, 0.01,'test', '测试', TRUE, 1610000000000, 0)") + print("insert into test.t9 values (now, 1, 2, 3, 4, 0.1, 0.01,'test', '测试', TRUE, 1610000000000, 0)") + tdSql.execute("insert into test.t9 values (now, 1, 2, 3, 4, 0.1, 0.01,'test', '测试', TRUE, 1610000000000, 0)") def run(self): tdSql.prepare() @@ -78,6 +114,8 @@ class TDTestCase: t1.join() t2.join() + time.sleep(3) + tdSql.query("select count(*) from test.meters") tdSql.checkData(0, 0, self.numberOfRecords * self.numberOfTables + 1) From ac7baae88b67587184692a148fe97edac6121b4b Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 1 Apr 2021 10:14:25 +0800 Subject: [PATCH 64/71] fix travis output --- Jenkinsfile | 4 +++- tests/test-all.sh | 5 ----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 433b46067a..bf2454c903 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -42,11 +42,12 @@ def pre_test(){ killall -9 taosd ||echo "no taosd running" killall -9 gdb || echo "no gdb running" cd ${WKC} - git checkout develop git reset --hard HEAD~10 >/dev/null + git checkout develop git pull >/dev/null git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD + git clean -dfx find ${WKC}/tests/pytest -name \'*\'.sql -exec rm -rf {} \\; cd ${WK} git reset --hard HEAD~10 @@ -55,6 +56,7 @@ def pre_test(){ cd ${WK} export TZ=Asia/Harbin date + git clean -dfx rm -rf ${WK}/debug mkdir debug cd debug diff --git a/tests/test-all.sh b/tests/test-all.sh index 883c1495e9..7bde698d95 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -25,9 +25,6 @@ function stopTaosd { function dohavecore(){ corefile=`find $corepath -mmin 1` core_file=`echo $corefile|cut -d " " -f2` - echo "corefile:$core_file" - echo "corepath:$corepath" - ls -l $corepath proc=`echo $corefile|cut -d "_" -f3` if [ -n "$corefile" ];then echo 'taosd or taos has generated core' @@ -82,7 +79,6 @@ function runSimCaseOneByOne { # fi end_time=`date +%s` echo execution time of $case was `expr $end_time - $start_time`s. | tee -a out.log - dohavecore 0 fi done < $1 } @@ -159,7 +155,6 @@ function runPyCaseOneByOne { else $line > /dev/null 2>&1 fi - dohavecore 0 fi done < $1 } From 58d76f3d26ec60a8ba6790a2f1fed4beb76bd8e5 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 1 Apr 2021 10:41:37 +0800 Subject: [PATCH 65/71] fix --- tests/test-all.sh | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/tests/test-all.sh b/tests/test-all.sh index 7bde698d95..3c8aed7d18 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -28,23 +28,21 @@ function dohavecore(){ proc=`echo $corefile|cut -d "_" -f3` if [ -n "$corefile" ];then echo 'taosd or taos has generated core' + rm case.log if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]] && [[ $1 == 1 ]]; then cd ../../../ tar -zcPf $corepath'taos_'`date "+%Y_%m_%d_%H_%M_%S"`.tar.gz debug/build/bin/taosd debug/build/bin/tsim debug/build/lib/libtaos*so* if [[ $2 == 1 ]];then cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S"` - rm -rf sim/case.log else cd community cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` - rm -rf sim/case.log fi else cd ../../ if [[ $1 == 1 ]];then tar -zcPf $corepath'taos_'`date "+%Y_%m_%d_%H_%M_%S"`.tar.gz debug/build/bin/taosd debug/build/bin/tsim debug/build/lib/libtaos*so* cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` - rm -rf sim/case.log fi fi if [[ $1 == 1 ]];then @@ -95,26 +93,25 @@ function runSimCaseOneByOnefq { date +%F\ %T | tee -a out.log if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then echo -n $case - ./test.sh -f $case > ../../../sim/case.log 2>&1 && \ + ./test.sh -f $case > case.log 2>&1 && \ ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ ( grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ - ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat ../../../sim/case.log ) + ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat case.log ) else echo -n $case ./test.sh -f $case > ../../sim/case.log 2>&1 && \ ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ ( grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ - ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat ../../sim/case.log ) + ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat case.log ) fi out_log=`tail -1 out.log ` if [[ $out_log =~ 'failed' ]];then + rm case.log if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then cp -r ../../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S"` - rm -rf ../../../sim/case.log else cp -r ../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` - rm -rf ../../sim/case.log fi dohavecore $2 1 if [[ $2 == 1 ]];then @@ -180,7 +177,7 @@ function runPyCaseOneByOnefq() { start_time=`date +%s` date +%F\ %T | tee -a pytest-out.log echo -n $case - $line > ../../sim/case.log 2>&1 && \ + $line > case.log 2>&1 && \ echo -e "${GREEN} success${NC}" | tee -a pytest-out.log || \ echo -e "${RED} failed${NC}" | tee -a pytest-out.log end_time=`date +%s` @@ -188,8 +185,8 @@ function runPyCaseOneByOnefq() { if [[ $out_log =~ 'failed' ]];then cp -r ../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` echo '=====================log===================== ' - cat ../../sim/case.log - rm -rf ../../sim/case.log + cat case.log + rm -rf case.log dohavecore $2 2 if [[ $2 == 1 ]];then exit 8 From 73889873fc6786bf12dda6e3daf3837bf525c797 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 1 Apr 2021 10:54:46 +0800 Subject: [PATCH 66/71] git clean --- Jenkinsfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index bf2454c903..bac0cce33b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -48,7 +48,6 @@ def pre_test(){ git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD git clean -dfx - find ${WKC}/tests/pytest -name \'*\'.sql -exec rm -rf {} \\; cd ${WK} git reset --hard HEAD~10 git checkout develop @@ -57,7 +56,6 @@ def pre_test(){ export TZ=Asia/Harbin date git clean -dfx - rm -rf ${WK}/debug mkdir debug cd debug cmake .. > /dev/null From 3e073149888f54a264d20dae63476b256e1f9419 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Thu, 1 Apr 2021 14:17:44 +0800 Subject: [PATCH 67/71] [TD-2518] : timestamp add support for time before year 1970. --- documentation20/cn/12.taos-sql/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index b4fa2b160a..dfda8997c2 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -43,7 +43,7 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic | | 类型 | Bytes | 说明 | | ---- | :-------: | ------ | ------------------------------------------------------------ | -| 1 | TIMESTAMP | 8 | 时间戳。缺省精度毫秒，可支持微秒。从格林威治时间 1970-01-01 00:00:00.000 (UTC/GMT) 开始，计时不能早于该时间。 | +| 1 | TIMESTAMP | 8 | 时间戳。缺省精度毫秒，可支持微秒。从格林威治时间 1970-01-01 00:00:00.000 (UTC/GMT) 开始，计时不能早于该时间。（从 2.0.18 版本开始，已经去除了这一时间范围限制） | | 2 | INT | 4 | 整型，范围 [-2^31+1, 2^31-1], -2^31 用作 NULL | | 3 | BIGINT | 8 | 长整型，范围 [-2^63+1, 2^63-1], -2^63 用于 NULL | | 4 | FLOAT | 4 | 浮点型，有效位数 6-7，范围 [-3.4E38, 3.4E38] | From af996100a27410d42d047e8cbeb386ab0b68cdc8 Mon Sep 17 00:00:00 2001 From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Thu, 1 Apr 2021 15:15:50 +0800 Subject: [PATCH 68/71] [TD-3417]: remove.sh stop and delete the nginxd.service for enterprise edition (#5617) --- packaging/tools/remove.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index e63889aff1..9241f01efa 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -120,7 +120,7 @@ function clean_service_on_systemd() { if [ "$verMode" == "cluster" ]; then nginx_service_config="${service_config_dir}/${nginx_service_name}.service" - if [ -d ${bin_dir}/web ]; then + if [ -d ${install_nginxd_dir} ]; then if systemctl is-active --quiet ${nginx_service_name}; then echo "Nginx for TDengine is running, stopping it..." ${csudo} systemctl stop ${nginx_service_name} &> /dev/null || echo &> /dev/null From 598b1ce6bffaa97a5abdb15d03b753f52bcaf2d6 Mon Sep 17 00:00:00 2001 From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Thu, 1 Apr 2021 15:16:36 +0800 Subject: [PATCH 69/71] Feature/td 3411 (#5648) * [TD-3410]feature: adapted to nutz * change * change * change * change * [TD-3447]: JDBC-RESTful and JDBC-JNI support setObject * change * change * change * change * change * change * change * change * change * change * change * change * [TD-3548]: support ParameterMetaData in JDBC-RESTful and JDBC-JNI * change * change * [TD-3446]: JDBC-JNI and JDBC-RESTful do not need invoke Class.forName any more * change jdbc version * change * change * change * change * change * change * change * change * change version --- cmake/install.inc | 2 +- src/connector/jdbc/CMakeLists.txt | 2 +- src/connector/jdbc/deploy-pom.xml | 2 +- src/connector/jdbc/pom.xml | 10 ++- .../jdbc/AbstractParameterMetaData.java | 2 +- .../com/taosdata/jdbc/TSDBJNIConnector.java | 46 ++----------- .../taosdata/jdbc/rs/RestfulResultSet.java | 4 +- .../java/com/taosdata/jdbc/utils/OSUtils.java | 17 +++++ .../jdbc/cases/DriverAutoloadTest.java | 1 - ...ullValueInResultSetForJdbcRestfulTest.java | 64 +++++++++++++++++++ .../com/taosdata/jdbc/utils/OSUtilsTest.java | 30 +++++++++ 11 files changed, 133 insertions(+), 47 deletions(-) create mode 100644 src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/OSUtils.java create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetForJdbcRestfulTest.java create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/OSUtilsTest.java diff --git a/cmake/install.inc b/cmake/install.inc index 01d3c8a4df..5823ef743e 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS) #INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS shell RUNTIME DESTINATION .) IF (TD_MVN_INSTALLED) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.24-dist.jar DESTINATION connector/jdbc) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.25-dist.jar DESTINATION connector/jdbc) ENDIF () ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") diff --git a/src/connector/jdbc/CMakeLists.txt b/src/connector/jdbc/CMakeLists.txt index 540dc8eb58..eb158b1f76 100644 --- a/src/connector/jdbc/CMakeLists.txt +++ b/src/connector/jdbc/CMakeLists.txt @@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED) ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME} POST_BUILD COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.24-dist.jar ${LIBRARY_OUTPUT_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.25-dist.jar ${LIBRARY_OUTPUT_PATH} COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMENT "build jdbc driver") ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME}) diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index 935b3f7e4a..eb8c92575c 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -5,7 +5,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.24 + 2.0.25 jar JDBCDriver diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index 5d7c89e2d2..1f75754b0c 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.taosdata.jdbc taos-jdbcdriver - 2.0.24 + 2.0.25 jar JDBCDriver https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc @@ -62,6 +62,14 @@ + + + src/main/resources + + **/*.md + + + org.apache.maven.plugins diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java index 999df06fc7..7df7252ae2 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java @@ -15,7 +15,7 @@ public abstract class AbstractParameterMetaData extends WrapperImpl implements P @Override public int getParameterCount() throws SQLException { - return parameters.length; + return parameters == null ? 0 : parameters.length; } @Override diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java index 7d3741917c..5e3ffffa4f 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java @@ -29,45 +29,25 @@ public class TSDBJNIConnector { private static volatile Boolean isInitialized = false; private TaosInfo taosInfo = TaosInfo.getInstance(); + // Connection pointer used in C + private long taos = TSDBConstants.JNI_NULL_POINTER; + // result set status in current connection + private boolean isResultsetClosed = true; + private int affectedRows = -1; static { System.loadLibrary("taos"); System.out.println("java.library.path:" + System.getProperty("java.library.path")); } - /** - * Connection pointer used in C - */ - private long taos = TSDBConstants.JNI_NULL_POINTER; - - /** - * Result set pointer for the current connection - */ -// private long taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER; - - /** - * result set status in current connection - */ - private boolean isResultsetClosed = true; - private int affectedRows = -1; - - /** - * Whether the connection is closed - */ public boolean isClosed() { return this.taos == TSDBConstants.JNI_NULL_POINTER; } - /** - * Returns the status of last result set in current connection - */ public boolean isResultsetClosed() { return this.isResultsetClosed; } - /** - * Initialize static variables in JNI to optimize performance - */ public static void init(String configDir, String locale, String charset, String timezone) throws SQLWarning { synchronized (isInitialized) { if (!isInitialized) { @@ -93,11 +73,6 @@ public class TSDBJNIConnector { public static native String getTsCharset(); - /** - * Get connection pointer - * - * @throws SQLException - */ public boolean connect(String host, int port, String dbName, String user, String password) throws SQLException { if (this.taos != TSDBConstants.JNI_NULL_POINTER) { // this.closeConnectionImp(this.taos); @@ -185,13 +160,6 @@ public class TSDBJNIConnector { private native String getErrMsgImp(long pSql); - /** - * Get resultset pointer - * Each connection should have a single open result set at a time - */ -// public long getResultSet() { -// return taosResultSetPointer; -// } private native long getResultSetImp(long connection, long pSql); public boolean isUpdateQuery(long pSql) { @@ -231,6 +199,7 @@ public class TSDBJNIConnector { // } // return resCode; // } + private native int freeResultSetImp(long connection, long result); /** @@ -323,8 +292,7 @@ public class TSDBJNIConnector { * Validate if a create table sql statement is correct without actually creating that table */ public boolean validateCreateTableSql(String sql) { - long connection = taos; - int res = validateCreateTableSqlImp(connection, sql.getBytes()); + int res = validateCreateTableSqlImp(taos, sql.getBytes()); return res != 0 ? false : true; } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java index 856f5257bf..5c2d4c45b0 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java @@ -84,9 +84,9 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: return new Timestamp(row.getDate(colIndex).getTime()); case TSDBConstants.TSDB_DATA_TYPE_BINARY: - return row.getString(colIndex).getBytes(); + return row.getString(colIndex) == null ? null : row.getString(colIndex).getBytes(); case TSDBConstants.TSDB_DATA_TYPE_NCHAR: - return row.getString(colIndex); + return row.getString(colIndex) == null ? null : row.getString(colIndex); default: return row.get(colIndex); } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/OSUtils.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/OSUtils.java new file mode 100644 index 0000000000..a67b4763f9 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/OSUtils.java @@ -0,0 +1,17 @@ +package com.taosdata.jdbc.utils; + +public class OSUtils { + private static final String OS = System.getProperty("os.name").toLowerCase(); + + public static boolean isWindows() { + return OS.indexOf("win") >= 0; + } + + public static boolean isMac() { + return OS.indexOf("mac") >= 0; + } + + public static boolean isLinux() { + return OS.indexOf("nux") >= 0; + } +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java index 580b2ac1b5..9826e6ed76 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java @@ -17,7 +17,6 @@ public class DriverAutoloadTest { @Test public void testRestful() throws SQLException { -// Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); final String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; Connection conn = DriverManager.getConnection(url, properties); Assert.assertNotNull(conn); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetForJdbcRestfulTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetForJdbcRestfulTest.java new file mode 100644 index 0000000000..f2ac94adc1 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetForJdbcRestfulTest.java @@ -0,0 +1,64 @@ +package com.taosdata.jdbc.cases; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.sql.*; + +public class NullValueInResultSetForJdbcRestfulTest { + + private static final String host = "127.0.0.1"; + Connection conn; + + @Test + public void test() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select * from weather"); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + Object value = rs.getObject(i); + System.out.print(meta.getColumnLabel(i) + ": " + value + "\t"); + } + System.out.println(); + } + + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Before + public void before() throws SQLException { + final String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; + conn = DriverManager.getConnection(url); + try (Statement stmt = conn.createStatement()) { + stmt.execute("drop database if exists test_null"); + stmt.execute("create database if not exists test_null"); + stmt.execute("use test_null"); + stmt.execute("create table weather(ts timestamp, f1 int, f2 bigint, f3 float, f4 double, f5 smallint, f6 tinyint, f7 bool, f8 binary(64), f9 nchar(64))"); + stmt.executeUpdate("insert into weather(ts, f1) values(now+1s, 1)"); + stmt.executeUpdate("insert into weather(ts, f2) values(now+2s, 2)"); + stmt.executeUpdate("insert into weather(ts, f3) values(now+3s, 3.0)"); + stmt.executeUpdate("insert into weather(ts, f4) values(now+4s, 4.0)"); + stmt.executeUpdate("insert into weather(ts, f5) values(now+5s, 5)"); + stmt.executeUpdate("insert into weather(ts, f6) values(now+6s, 6)"); + stmt.executeUpdate("insert into weather(ts, f7) values(now+7s, true)"); + stmt.executeUpdate("insert into weather(ts, f8) values(now+8s, 'hello')"); + stmt.executeUpdate("insert into weather(ts, f9) values(now+9s, '涛思数据')"); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @After + public void after() { + try { + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/OSUtilsTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/OSUtilsTest.java new file mode 100644 index 0000000000..fd6c83ad1c --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/OSUtilsTest.java @@ -0,0 +1,30 @@ +package com.taosdata.jdbc.utils; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class OSUtilsTest { + + private String OS; + + @Test + public void inWindows() { + Assert.assertEquals(OS.indexOf("win") >= 0, OSUtils.isWindows()); + } + + @Test + public void isMac() { + Assert.assertEquals(OS.indexOf("mac") >= 0, OSUtils.isMac()); + } + + @Test + public void isLinux() { + Assert.assertEquals(OS.indexOf("nux") >= 0, OSUtils.isLinux()); + } + + @Before + public void before() { + OS = System.getProperty("os.name").toLowerCase(); + } +} \ No newline at end of file From e39762deb1571554e603d26b6a404e8d90fa7ccb Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 1 Apr 2021 16:06:25 +0800 Subject: [PATCH 70/71] Hotfix/sangshuduo/td 3631 max stb count (#5645) * [TD-3631] : increase max super table count to 10000 * [TD-3631] : use smaller max stb count due to data structure limit. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 9f367b41f8..d76185ca43 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -85,7 +85,7 @@ enum TEST_MODE { #define MAX_NUM_DATATYPE 10 #define MAX_DB_COUNT 8 -#define MAX_SUPER_TABLE_COUNT 8 +#define MAX_SUPER_TABLE_COUNT 200 #define MAX_COLUMN_COUNT 1024 #define MAX_TAG_COUNT 128 @@ -4090,7 +4090,7 @@ static bool getInfoFromJsonFile(char* file) { } bool ret = false; - int maxLen = 64000; + int maxLen = 6400000; char *content = calloc(1, maxLen + 1); int len = fread(content, 1, maxLen, fp); if (len <= 0) { From ad1c6534a93602953c7baf53c548b5f1747e7938 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Thu, 1 Apr 2021 16:40:04 +0800 Subject: [PATCH 71/71] [TD-3645] : add description about dimension microsecond in timestamp. --- documentation20/cn/12.taos-sql/docs.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index dfda8997c2..58191e0bd8 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -29,17 +29,17 @@ taos> DESCRIBE meters; ## 支持的数据类型 -使用TDengine，最重要的是时间戳。创建并插入记录、查询历史记录的时候，均需要指定时间戳。时间戳有如下规则： +使用 TDengine，最重要的是时间戳。创建并插入记录、查询历史记录的时候，均需要指定时间戳。时间戳有如下规则： -- 时间格式为```YYYY-MM-DD HH:mm:ss.MS```, 默认时间分辨率为毫秒。比如：```2017-08-12 18:25:58.128``` -- 内部函数now是服务器的当前时间 -- 插入记录时，如果时间戳为now，插入数据时使用服务器当前时间 -- Epoch Time: 时间戳也可以是一个长整数，表示从1970-01-01 08:00:00.000开始的毫秒数 -- 时间可以加减，比如 now-2h，表明查询时刻向前推2个小时(最近2小时)。 数字后面的时间单位可以是 a(毫秒)、s(秒)、 m(分)、h(小时)、d(天)、w(周)。 比如select * from t1 where ts > now-2w and ts <= now-1w, 表示查询两周前整整一周的数据。 在指定降频操作(down sampling)的时间窗口(interval)时，时间单位还可以使用 n(自然月) 和 y(自然年)。 +- 时间格式为 ```YYYY-MM-DD HH:mm:ss.MS```，默认时间分辨率为毫秒。比如：```2017-08-12 18:25:58.128``` +- 内部函数 now 是客户端的当前时间 +- 插入记录时，如果时间戳为 now，插入数据时使用提交这条记录的客户端的当前时间 +- Epoch Time：时间戳也可以是一个长整数，表示从 1970-01-01 08:00:00.000 开始的毫秒数 +- 时间可以加减，比如 now-2h，表明查询时刻向前推 2 个小时（最近 2 小时）。数字后面的时间单位可以是 u(微秒)、a(毫秒)、s(秒)、m(分)、h(小时)、d(天)、w(周)。 比如 `select * from t1 where ts > now-2w and ts <= now-1w`，表示查询两周前整整一周的数据。在指定降频操作（down sampling）的时间窗口（interval）时，时间单位还可以使用 n(自然月) 和 y(自然年)。 -TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMicrosecond就可支持微秒。 +TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableMicrosecond 就可以支持微秒。 -在TDengine中，普通表的数据模型中可使用以下10种数据类型。 +在TDengine中，普通表的数据模型中可使用以下 10 种数据类型。 | | 类型 | Bytes | 说明 | | ---- | :-------: | ------ | ------------------------------------------------------------ |