diff --git a/tests/army/sys/checkErrorCode.py b/tests/army/sys/checkErrorCode.py deleted file mode 100644 index 71aa43b02d..0000000000 --- a/tests/army/sys/checkErrorCode.py +++ /dev/null @@ -1,181 +0,0 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- - -import sys -import time -import random -import taos - -import frame -import frame.eos -import frame.etime -import frame.etool -from frame.log import * -from frame.sql import * -from frame.cases import * -from frame.caseBase import * -from frame.srvCtl import * -from frame import * - - -class TDTestCase(TBase): - # parse line - def parseLine(self, line): - line = line.strip() - PRE_DEFINE = "#define TSDB_CODE_" - n = len(PRE_DEFINE) - if line[:n] != PRE_DEFINE: - return None - # TAOS_DEF_ERROR_CODE(0, 0x000B) - pos = line.find("TAOS_DEF_ERROR_CODE(0, 0x", n) - if pos == -1: - tdLog.info(f"not found \"TAOS_DEF_ERROR_CODE(0, \" line={line}") - return None - - code = line[pos:].strip() - pos = code.find(")") - if pos == -1: - tdLog.info(f"not found \")\", line={line}") - return None - code = code[:pos] - if len(code) != 4: - tdLog.info(f"code is len not 4 len:{len(code)} subcode={code}\")\", line={line}") - return None - - # return - return "0x8000" + code - - # ignore error - def ignoreCode(self, code): - ignoreCodes = {"0x00008, 0x000009"} - if code in ignoreCodes: - return True - else: - return False - - # read valid code - def readHeadCodes(self, hFile): - codes = [] - start = False - # read - with open(hFile) as file: - for line in file: - code = self.parseLine(line) - # invalid - if code == None: - continue - # ignore - if self.ignoreCode(code): - tdLog.info(f"ignore error {code}\n") - # valid - if code == 0: - start = True - if start: - codes.append(code) - # return - return codes - - # parse doc lines - def parseDocLine(self, line): - line = line.strip() - PRE_DEFINE = "| 0x8000" - n = len(PRE_DEFINE) - if line[:n] != PRE_DEFINE: - return None - line = line[2:] - cols = line.split("|") - # remove blank - cols = [col.strip() for col in cols] - - # return - return cols - - - # read valid code - def readDocCodes(self, docFile): - codes = [] - start = False - # read - with open(docFile) as file: - for line in file: - code = self.parseDocLine(line) - # invalid - if code == None: - continue - # valid - if start: - codes.append(code) - # return - return codes - - # check - def checkConsistency(self, docCodes, codes): - diff = False - # len - docLen = len(docCodes) - len = len(codes) - tdLog.info("head file codes = {len} doc file codes={docLen} \n") - - if docLen > len: - maxLen = docLen - else: - maxLen = len - - for i in range(maxLen): - if i < len and i < docLen: - if codes[i] == docCodes[i][0]: - tdLog.info(f" i={i} same head code: {codes[i]} doc code:{docCodes[i][0]}\n") - else: - tdLog.info(f" i={i} diff head code: {codes[i]} doc code:{docCodes[i][0]}\n") - diff = True - elif i < len: - tdLog.info(f" i={i} diff head code: {codes[i]} doc code: None\n") - diff = True - elif i < docLen: - tdLog.info(f" i={i} diff head code: None doc code: {docCodes[i][0]}\n") - diff = True - - # result - if diff: - tdLog.exit("check error code consistency failed.\n") - - - # run - def run(self): - tdLog.debug(f"start to excute {__file__}") - - # read head error code - hFile = "../../include/util/taoserror.h" - codes = self.readHeadCodes(hFile) - - # read zh codes - zhDoc = "../../docs/zh/14-reference/09-error-code.md" - zhCodes = self.readDocCodes(zhDoc, codes) - - # read en codes - enDoc = "../../docs/en/14-reference/09-error-code.md" - enCodes = self.readDocCodes(enDoc, codes) - - # check zh - tdLog.info(f"check zh docs ...\n") - self.checkConsistency(zhCodes, codes) - - # check en - tdLog.info(f"check en docs ...\n") - self.checkConsistency(enCodes, codes) - - tdLog.success(f"{__file__} successfully executed") - - -tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/tools/benchmark/basic/bugs.py b/tests/army/tools/benchmark/basic/bugs.py index d1b45d95ed..2af1e1d0f3 100644 --- a/tests/army/tools/benchmark/basic/bugs.py +++ b/tests/army/tools/benchmark/basic/bugs.py @@ -21,15 +21,6 @@ from frame.caseBase import * from frame import * -# reomve single and double quotation -def removeQuotation(origin): - value = "" - for c in origin: - if c != '\'' and c != '"': - value += c - - return value - class TDTestCase(TBase): def caseDescription(self): """ @@ -111,7 +102,7 @@ class TDTestCase(TBase): tdSql.query(sql) if cachemode != None: - value = removeQuotation(cachemode) + value = frame.eutil.removeQuota(cachemode) tdLog.info(f" deal both origin={cachemode} after={value}") tdSql.checkData(0, 1, value) diff --git a/tests/army/tools/benchmark/basic/exportCsv.py b/tests/army/tools/benchmark/basic/exportCsv.py index 625fac2641..b8b3828ea6 100644 --- a/tests/army/tools/benchmark/basic/exportCsv.py +++ b/tests/army/tools/benchmark/basic/exportCsv.py @@ -23,15 +23,6 @@ from frame.caseBase import * from frame import * -# reomve single and double quotation -def removeQuotation(origin): - value = "" - for c in origin: - if c != '\'' and c != '"': - value += c - - return value - class TDTestCase(TBase): def caseDescription(self): """ diff --git a/tests/army/tools/benchmark/basic/insertBasic.py b/tests/army/tools/benchmark/basic/insertBasic.py new file mode 100644 index 0000000000..f1d3b81732 --- /dev/null +++ b/tests/army/tools/benchmark/basic/insertBasic.py @@ -0,0 +1,133 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +import os +import json +import frame +import frame.etool +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * + + +class TDTestCase(TBase): + def caseDescription(self): + """ + taosBenchmark Insert->Basic test cases + """ + + def benchmarkQuery(self, benchmark, jsonFile, keys, options=""): + # exe insert + result = "query.log" + os.system(f"rm -f {result}") + cmd = f"{benchmark} {options} -f {jsonFile} >> {result}" + os.system(cmd) + tdLog.info(cmd) + with open(result) as file: + content = file.read() + for key in keys: + if content.find(key) == -1: + tdLog.exit(f"not found key: {key} in content={content}") + else: + tdLog.info(f"found key:{key} successful.") + + + def testBenchmarkJson(self, benchmark, jsonFile, options = "", checkTimeStep = False): + # exe insert + cmd = f"{benchmark} {options} -f {jsonFile}" + os.system(cmd) + + # + # check insert result + # + with open(jsonFile, "r") as file: + data = json.load(file) + + db = data["databases"][0]["dbinfo"]["name"] + stb = data["databases"][0]["super_tables"][0]["name"] + child_count = data["databases"][0]["super_tables"][0]["childtable_count"] + insert_rows = data["databases"][0]["super_tables"][0]["insert_rows"] + timestamp_step = data["databases"][0]["super_tables"][0]["timestamp_step"] + + # drop + try: + drop = data["databases"][0]["dbinfo"]["drop"] + except: + drop = "yes" + + # command is first + if options.find("-Q") != -1: + drop = "no" + + # only support + cmdVG = None + pos = options.find("=") + if pos != -1: + arr = options.split("=") + if arr[0] == "--vgroups": + cmdVG = arr[1] + + # vgropus + try: + if cmdVG != None: + # command special vgroups first priority + vgroups = cmdVG + else: + vgroups = data["databases"][0]["dbinfo"]["vgroups"] + except: + vgroups = None + + tdLog.info(f"get json info: db={db} stb={stb} child_count={child_count} insert_rows={insert_rows} cmdVG={cmdVG}\n") + + # all count insert_rows * child_table_count + sql = f"select * from {db}.{stb}" + tdSql.query(sql) + tdSql.checkRows(child_count * insert_rows) + + # timestamp step + if checkTimeStep: + sql = f"select * from (select diff(ts) as dif from {db}.{stb} partition by tbname) where dif != {timestamp_step};" + tdSql.query(sql) + tdSql.checkRows(0) + + if drop.lower() == "yes": + # check database optins + sql = f"select `vgroups` from information_schema.ins_databases where name='{db}';" + tdSql.query(sql) + if vgroups != None: + tdLog.info(f" vgroups real={tdSql.getData(0,0)} expect={vgroups}") + tdSql.checkData(0, 0, vgroups, True) + + + # bugs ts + def checkVGroups(self, benchmark): + # vgroups with command line set + self.testBenchmarkJson(benchmark, "./tools/benchmark/basic/json/insertBasic.json", "--vgroups=3", True) + # vgroups with json file + self.testBenchmarkJson(benchmark, "./tools/benchmark/basic/json/insertBasic.json", "", True) + + def run(self): + benchmark = etool.benchMarkFile() + + # vgroups + self.checkVGroups(benchmark) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/army/tools/benchmark/basic/insertBindVGroup.py b/tests/army/tools/benchmark/basic/insertBindVGroup.py new file mode 100644 index 0000000000..6086e20f5f --- /dev/null +++ b/tests/army/tools/benchmark/basic/insertBindVGroup.py @@ -0,0 +1,48 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +import os +import json +import frame +import frame.etool +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * + + +class TDTestCase(TBase): + def caseDescription(self): + """ + taosBenchmark insert->BindVGroup test cases + """ + + # bugs ts + def checkBasic(self): + # thread equal vgroups + self.insertBenchJson("./tools/benchmark/basic/json/insertBindVGroup.json", "", True) + # thread is limited + self.insertBenchJson("./tools/benchmark/basic/json/insertBindVGroup.json", "-T 2", True) + + def run(self): + # basic + self.checkBasic() + # advance + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/army/tools/benchmark/basic/insertPrecision.py b/tests/army/tools/benchmark/basic/insertPrecision.py new file mode 100644 index 0000000000..c0f0f25851 --- /dev/null +++ b/tests/army/tools/benchmark/basic/insertPrecision.py @@ -0,0 +1,89 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +import os +import json +import frame +import frame.etool +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * + + +class TDTestCase(TBase): + def caseDescription(self): + """ + taosBenchmark insert->Precision test cases + """ + + def testBenchmarkJson(self, benchmark, jsonFile, options = ""): + # exe insert + cmd = f"{benchmark} {options} -f {jsonFile}" + os.system(cmd) + + # + # check insert result + # + with open(jsonFile, "r") as file: + data = json.load(file) + + db = data["databases"][0]["dbinfo"]["name"] + precison = data["databases"][0]["dbinfo"]["precision"] + stb = data["databases"][0]["super_tables"][0]["name"] + child_count = data["databases"][0]["super_tables"][0]["childtable_count"] + insert_rows = data["databases"][0]["super_tables"][0]["insert_rows"] + timestamp_step = data["databases"][0]["super_tables"][0]["timestamp_step"] + start_timestamp = data["databases"][0]["super_tables"][0]["start_timestamp"] + + tdLog.info(f"get json info: db={db} precision={precison} stb={stb} child_count={child_count} insert_rows={insert_rows} " + f"start_timestamp={start_timestamp} timestamp_step={timestamp_step} \n") + + # all count insert_rows * child_table_count + sql = f"select * from {db}.{stb}" + tdSql.query(sql) + tdSql.checkRows(child_count * insert_rows) + + # timestamp step + sql = f"select * from (select diff(ts) as dif from {db}.{stb} partition by tbname) where dif != {timestamp_step};" + tdSql.query(sql) + tdSql.checkRows(0) + + # check last ts + lastTime = start_timestamp + timestamp_step * (insert_rows - 1) + sql = f"select last(ts) from {db}.{stb}" + tdSql.checkAgg(sql, lastTime) + + # bugs ts + def checkBasic(self, benchmark): + # MS + self.testBenchmarkJson(benchmark, "./tools/benchmark/basic/json/insertPrecisionMS.json", "") + # US + self.testBenchmarkJson(benchmark, "./tools/benchmark/basic/json/insertPrecisionUS.json", "") + # NS + self.testBenchmarkJson(benchmark, "./tools/benchmark/basic/json/insertPrecisionNS.json", "") + + def run(self): + benchmark = etool.benchMarkFile() + + # vgroups + self.checkBasic(benchmark) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/army/tools/benchmark/basic/json/insertBasic.json b/tests/army/tools/benchmark/basic/json/insertBasic.json new file mode 100644 index 0000000000..2fc59befe9 --- /dev/null +++ b/tests/army/tools/benchmark/basic/json/insertBasic.json @@ -0,0 +1,64 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "num_of_records_per_req": 3000, + "thread_count": 2, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "test", + "drop": "yes", + "precision": "ns", + "vgroups": 2 + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 2, + "insert_rows": 1000, + "childtable_prefix": "d", + "insert_mode": "taosc", + "insert_interval": 0, + "timestamp_step": 1000, + "start_timestamp":1700000000000000000, + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "max": 1, "min": 0 }, + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "smallint", "name": "si", "max": 100, "min": -50 }, + { "type": "int", "name": "ic", "max": 1000, "min": -1000 }, + { "type": "bigint", "name": "bi", "max": 100, "min": -1000 }, + { "type": "utinyint", "name": "uti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "usi", "max": 100, "min": 0 }, + { "type": "uint", "name": "ui", "max": 1000, "min": 0 }, + { "type": "ubigint", "name": "ubi", "max": 10000, "min": 0 }, + { "type": "binary", "name": "bin", "len": 4}, + { "type": "nchar", "name": "nch", "len": 8} + ], + "tags": [ + { "type": "bool", "name": "tbc"}, + { "type": "float", "name": "tfc", "max": 1, "min": 0 }, + { "type": "double", "name": "tdc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "tti", "max": 100, "min": -100 }, + { "type": "smallint", "name": "tsi", "max": 100, "min": -50 }, + { "type": "int", "name": "tic", "max": 1000, "min": -1000 }, + { "type": "bigint", "name": "tbi", "max": 100, "min": -1000 }, + { "type": "utinyint", "name": "tuti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 }, + { "type": "ubigint", "name": "tubi", "max": 10000, "min": 0 }, + { "type": "binary", "name": "tbin", "len": 4}, + { "type": "nchar", "name": "tnch", "len": 8} + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/army/tools/benchmark/basic/json/insertBindVGroup.json b/tests/army/tools/benchmark/basic/json/insertBindVGroup.json new file mode 100644 index 0000000000..d8f1a5fef7 --- /dev/null +++ b/tests/army/tools/benchmark/basic/json/insertBindVGroup.json @@ -0,0 +1,47 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "num_of_records_per_req": 6000, + "thread_count": 10, + "thread_bind_vgroup": "yes", + "create_table_thread_count": 1, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "binddb", + "drop": "yes", + "vgroups": 10 + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 20, + "insert_rows": 5000, + "childtable_prefix": "d", + "insert_mode": "stmt2", + "timestamp_step": 1000, + "start_timestamp":1600000000000, + "columns": [ + {"type": "FLOAT", "name": "current", "count": 1, "max": 12, "min": 8 }, + { "type": "INT", "name": "voltage", "max": 225, "min": 100 }, + { "type": "FLOAT", "name": "phase", "max": 1, "min": 0 } + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"type": "binary", "name": "location", "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/tools/taos-tools/case/insertBindVGroup.json b/tests/army/tools/benchmark/basic/json/insertPrecisionMS.json similarity index 50% rename from tools/taos-tools/case/insertBindVGroup.json rename to tests/army/tools/benchmark/basic/json/insertPrecisionMS.json index beaadfb58c..a07881511d 100644 --- a/tools/taos-tools/case/insertBindVGroup.json +++ b/tests/army/tools/benchmark/basic/json/insertPrecisionMS.json @@ -5,16 +5,15 @@ "port": 6030, "user": "root", "password": "taosdata", - "num_of_records_per_req": 200, - "thread_count": 20, - "thread_bind_vgroup": "yes", - "create_table_thread_count": 1, + "num_of_records_per_req": 3000, + "thread_count": 2, "confirm_parameter_prompt": "no", "databases": [ { "dbinfo": { - "name": "binddb", + "name": "test", "drop": "yes", + "precision": "ms", "vgroups": 2 }, "super_tables": [ @@ -22,37 +21,44 @@ "name": "meters", "child_table_exists": "no", "childtable_count": 4, - "insert_rows": 100, - "interlace_rows": 10, + "insert_rows": 1000, "childtable_prefix": "d", "insert_mode": "taosc", - "timestamp_step": 1000, - "start_timestamp":1500000000000, + "interlace_rows": 0, + "timestamp_step": 100, + "start_timestamp":1700000000111, "columns": [ { "type": "bool", "name": "bc"}, { "type": "float", "name": "fc", "max": 1, "min": 0 }, - { "type": "double", "name": "dc", "max": 1, "min": 0 }, - { "type": "tinyint", "name": "ti", "max": 100, "min": 0 }, - { "type": "smallint", "name": "si", "max": 100, "min": 0 }, - { "type": "int", "name": "ic", "max": 100, "min": 0 }, - { "type": "bigint", "name": "bi", "max": 100, "min": 0 }, + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "smallint", "name": "si", "max": 100, "min": -50 }, + { "type": "int", "name": "ic", "max": 1000, "min": -1000 }, + { "type": "bigint", "name": "bi", "max": 100, "min": -1000 }, { "type": "utinyint", "name": "uti", "max": 100, "min": 0 }, { "type": "usmallint", "name": "usi", "max": 100, "min": 0 }, - { "type": "uint", "name": "ui", "max": 100, "min": 0 }, - { "type": "ubigint", "name": "ubi", "max": 100, "min": 0 }, - { "type": "binary", "name": "bin", "len": 32}, - { "type": "nchar", "name": "nch", "len": 64} + { "type": "uint", "name": "ui", "max": 1000, "min": 0 }, + { "type": "ubigint", "name": "ubi", "max": 10000, "min": 0 }, + { "type": "binary", "name": "bin", "len": 4}, + { "type": "nchar", "name": "nch", "len": 8} ], "tags": [ - {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, - {"type": "binary", "name": "location", "len": 16, - "values": ["San Francisco", "Los Angles", "San Diego", - "San Jose", "Palo Alto", "Campbell", "Mountain View", - "Sunnyvale", "Santa Clara", "Cupertino"] - } + { "type": "bool", "name": "tbc"}, + { "type": "float", "name": "tfc", "max": 1, "min": 0 }, + { "type": "double", "name": "tdc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "tti", "max": 100, "min": -100 }, + { "type": "smallint", "name": "tsi", "max": 100, "min": -50 }, + { "type": "int", "name": "tic", "max": 1000, "min": -1000 }, + { "type": "bigint", "name": "tbi", "max": 100, "min": -1000 }, + { "type": "utinyint", "name": "tuti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 }, + { "type": "ubigint", "name": "tubi", "max": 10000, "min": 0 }, + { "type": "binary", "name": "tbin", "len": 4}, + { "type": "nchar", "name": "tnch", "len": 8} ] } ] } ] -} +} \ No newline at end of file diff --git a/tests/army/tools/benchmark/basic/json/insertPrecisionNS.json b/tests/army/tools/benchmark/basic/json/insertPrecisionNS.json new file mode 100644 index 0000000000..e18eef9508 --- /dev/null +++ b/tests/army/tools/benchmark/basic/json/insertPrecisionNS.json @@ -0,0 +1,63 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "num_of_records_per_req": 3000, + "thread_count": 2, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "test", + "drop": "yes", + "precision": "ns", + "vgroups": 2 + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 3, + "insert_rows": 1000, + "childtable_prefix": "d", + "insert_mode": "stmt2", + "timestamp_step": 10, + "start_timestamp":1700000000000000111, + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "max": 1, "min": 0 }, + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "smallint", "name": "si", "max": 100, "min": -50 }, + { "type": "int", "name": "ic", "max": 1000, "min": -1000 }, + { "type": "bigint", "name": "bi", "max": 100, "min": -1000 }, + { "type": "utinyint", "name": "uti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "usi", "max": 100, "min": 0 }, + { "type": "uint", "name": "ui", "max": 1000, "min": 0 }, + { "type": "ubigint", "name": "ubi", "max": 10000, "min": 0 }, + { "type": "binary", "name": "bin", "len": 4}, + { "type": "nchar", "name": "nch", "len": 8} + ], + "tags": [ + { "type": "bool", "name": "tbc"}, + { "type": "float", "name": "tfc", "max": 1, "min": 0 }, + { "type": "double", "name": "tdc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "tti", "max": 100, "min": -100 }, + { "type": "smallint", "name": "tsi", "max": 100, "min": -50 }, + { "type": "int", "name": "tic", "max": 1000, "min": -1000 }, + { "type": "bigint", "name": "tbi", "max": 100, "min": -1000 }, + { "type": "utinyint", "name": "tuti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 }, + { "type": "ubigint", "name": "tubi", "max": 10000, "min": 0 }, + { "type": "binary", "name": "tbin", "len": 4}, + { "type": "nchar", "name": "tnch", "len": 8} + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/army/tools/benchmark/basic/json/insertPrecisionUS.json b/tests/army/tools/benchmark/basic/json/insertPrecisionUS.json new file mode 100644 index 0000000000..a29c50a95e --- /dev/null +++ b/tests/army/tools/benchmark/basic/json/insertPrecisionUS.json @@ -0,0 +1,64 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "num_of_records_per_req": 3000, + "thread_count": 2, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "test", + "drop": "yes", + "precision": "us", + "vgroups": 2 + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 3, + "insert_rows": 5000, + "childtable_prefix": "d", + "insert_mode": "stmt", + "interlace_rows": 100, + "timestamp_step": 10, + "start_timestamp":1700000000000321, + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "max": 1, "min": 0 }, + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "smallint", "name": "si", "max": 100, "min": -50 }, + { "type": "int", "name": "ic", "max": 1000, "min": -1000 }, + { "type": "bigint", "name": "bi", "max": 100, "min": -1000 }, + { "type": "utinyint", "name": "uti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "usi", "max": 100, "min": 0 }, + { "type": "uint", "name": "ui", "max": 1000, "min": 0 }, + { "type": "ubigint", "name": "ubi", "max": 10000, "min": 0 }, + { "type": "binary", "name": "bin", "len": 4}, + { "type": "nchar", "name": "nch", "len": 8} + ], + "tags": [ + { "type": "bool", "name": "tbc"}, + { "type": "float", "name": "tfc", "max": 1, "min": 0 }, + { "type": "double", "name": "tdc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "tti", "max": 100, "min": -100 }, + { "type": "smallint", "name": "tsi", "max": 100, "min": -50 }, + { "type": "int", "name": "tic", "max": 1000, "min": -1000 }, + { "type": "bigint", "name": "tbi", "max": 100, "min": -1000 }, + { "type": "utinyint", "name": "tuti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 }, + { "type": "ubigint", "name": "tubi", "max": 10000, "min": 0 }, + { "type": "binary", "name": "tbin", "len": 4}, + { "type": "nchar", "name": "tnch", "len": 8} + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/army/tools/benchmark/basic/queryMain.py b/tests/army/tools/benchmark/basic/queryMain.py index 1d52163e31..bd6fa9ca31 100644 --- a/tests/army/tools/benchmark/basic/queryMain.py +++ b/tests/army/tools/benchmark/basic/queryMain.py @@ -28,19 +28,10 @@ from frame.caseBase import * from frame import * -# reomve single and double quotation -def removeQuotation(origin): - value = "" - for c in origin: - if c != '\'' and c != '"': - value += c - - return value - class TDTestCase(TBase): def caseDescription(self): """ - [TD-11510] taosBenchmark test cases + taosBenchmark query->Basic test cases """ def runSeconds(self, command, timeout = 180): diff --git a/tests/army/tools/taosdump/native/taosdumpCommandline.py b/tests/army/tools/taosdump/native/taosdumpCommandline.py new file mode 100644 index 0000000000..0e7c3dd29f --- /dev/null +++ b/tests/army/tools/taosdump/native/taosdumpCommandline.py @@ -0,0 +1,217 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import os +import json +import frame +import frame.etool +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * + + +class TDTestCase(TBase): + def caseDescription(self): + """ + test taosdump support commandline arguments + """ + + def exec(self, command): + tdLog.info(command) + return os.system(command) + + def clearPath(self, path): + os.system("rm -rf %s/*" % path) + + def findPrograme(self): + # taosdump + taosdump = etool.taosDumpFile() + if taosdump == "": + tdLog.exit("taosdump not found!") + else: + tdLog.info("taosdump found in %s" % taosdump) + + # taosBenchmark + benchmark = etool.benchMarkFile() + if benchmark == "": + tdLog.exit("benchmark not found!") + else: + tdLog.info("benchmark found in %s" % benchmark) + + # tmp dir + tmpdir = "./tmp" + if not os.path.exists(tmpdir): + os.makedirs(tmpdir) + else: + print("directory exists") + self.clearPath(tmpdir) + + return taosdump, benchmark,tmpdir + + def checkCorrectWithJson(self, jsonFile, newdb = None, checkInterval=False): + # + # check insert result + # + with open(jsonFile, "r") as file: + data = json.load(file) + + # db come from arguments + if newdb is None: + db = data["databases"][0]["dbinfo"]["name"] + else: + db = newdb + + stb = data["databases"][0]["super_tables"][0]["name"] + child_count = data["databases"][0]["super_tables"][0]["childtable_count"] + insert_rows = data["databases"][0]["super_tables"][0]["insert_rows"] + timestamp_step = data["databases"][0]["super_tables"][0]["timestamp_step"] + + tdLog.info(f"get json: db={db} stb={stb} child_count={child_count} insert_rows={insert_rows} \n") + + # all count insert_rows * child_table_count + sql = f"select * from {db}.{stb}" + tdSql.query(sql) + tdSql.checkRows(child_count * insert_rows) + + # timestamp step + if checkInterval: + sql = f"select * from (select diff(ts) as dif from {db}.{stb} partition by tbname) where dif != {timestamp_step};" + tdSql.query(sql) + tdSql.checkRows(0) + + def testBenchmarkJson(self, benchmark, jsonFile, options="", checkInterval=False): + # exe insert + cmd = f"{benchmark} {options} -f {jsonFile}" + self.exec(cmd) + self.checkCorrectWithJson(jsonFile) + + def insertData(self, benchmark, json, db): + # insert super table + self.testBenchmarkJson(benchmark, json) + + # normal table + sqls = [ + f"create table {db}.ntb(st timestamp, c1 int, c2 binary(32))", + f"insert into {db}.ntb values(now, 1, 'abc1')", + f"insert into {db}.ntb values(now, 2, 'abc2')", + f"insert into {db}.ntb values(now, 3, 'abc3')", + f"insert into {db}.ntb values(now, 4, 'abc4')", + f"insert into {db}.ntb values(now, 5, 'abc5')", + ] + for sql in sqls: + tdSql.execute(sql) + + def dumpOut(self, taosdump, db , outdir): + # dump out + self.exec(f"{taosdump} -D {db} -o {outdir}") + + def dumpIn(self, taosdump, db, newdb, indir): + # dump in + self.exec(f'{taosdump} -W "{db}={newdb}" -i {indir}') + + def checkSame(self, db, newdb, stb, aggfun): + # sum pk db + sql = f"select {aggfun} from {db}.{stb}" + tdSql.query(sql) + sum1 = tdSql.getData(0,0) + # sum pk newdb + sql = f"select {aggfun} from {newdb}.{stb}" + tdSql.query(sql) + sum2 = tdSql.getData(0,0) + + if sum1 == sum2: + tdLog.info(f"{aggfun} source db:{sum1} import db:{sum2} both equal.") + else: + tdLog.exit(f"{aggfun} source db:{sum1} import db:{sum2} not equal.") + + + def verifyResult(self, db, newdb, json): + # compare with insert json + self.checkCorrectWithJson(json, newdb) + + # compare sum(pk) + stb = "meters" + self.checkSame(db, newdb, stb, "sum(pk)") + self.checkSame(db, newdb, stb, "sum(usi)") + self.checkSame(db, newdb, stb, "sum(ic)") + + # check normal table + self.checkSame(db, newdb, "ntb", "sum(c1)") + + + # basic commandline + def basicCommandLine(self, taosdump, tmpdir): + # -h -P -u -p -o + self.exec(taosdump + f" -h 127.0.0.1 -P 6030 -uroot -ptaosdata -A -N -o {tmpdir}") + self.clearPath(tmpdir) + + # check except + def checkExcept(self, command): + try: + code = self.exec(command) + if code == 0: + tdLog.exit(f"Failed, not report error cmd:{command}") + else: + tdLog.info(f"Passed, report error code={code} is expect, cmd:{command}") + except: + tdLog.info(f"Passed, catch expect report error for command {command}") + + + # except commandline + def exceptCommandLine(self, taosdump, tmpdir): + # -o + self.checkExcept(taosdump + " -o= ") + self.checkExcept(taosdump + " -o") + self.checkExcept(taosdump + " -A -o=") + self.checkExcept(taosdump + " -A -o ") + self.checkExcept(taosdump + " -A -o ./noexistpath/") + self.checkExcept(taosdump + f" -d invalidAVRO -o {tmpdir}") + + # run + def run(self): + # database + db = "pridb" + newdb = "npridb" + + # find + taosdump, benchmark, tmpdir = self.findPrograme() + json = "./tools/taosdump/ws/json/primaryKey.json" + + # insert data with taosBenchmark + self.insertData(benchmark, json, db) + + # basic commandline + self.basicCommandLine(taosdump, tmpdir) + + # except commandline + self.exceptCommandLine(taosdump, tmpdir) + + # dump out + #self.dumpOut(taosdump, db, tmpdir) + + # dump in + #self.dumpIn(taosdump, db, newdb, tmpdir) + + # verify db + #self.verifyResult(db, newdb, json) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 0d14722aaf..50f5879a14 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -77,73 +77,88 @@ # army/tools # -# benchmark 64 cases -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/rest_insert_alltypes_json.py -R -,,n,army,python3 ./test.py -f tools/benchmark/basic/taosdemoTestQueryWithJson-mixed-query.py -R -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/stmt_sample_csv_json.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/taosdemoTestInsertWithJsonStmt-otherPara.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/sml_telnet_insert_alltypes-same-min-max.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/default_tmq_json.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/reuse-exist-stb.py -,,n,army,python3 ./test.py -f tools/benchmark/basic/sml_interlace.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/stmt2_insert.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/stmt_offset_json.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/json_tag.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/commandline-sml-rest.py -R -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/taosc_auto_create_table_json.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/insert-json-csv.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/taosc_insert-table-creating-interval.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/insertMix.py -,,n,army,python3 ./test.py -f tools/benchmark/basic/taosc_insert-mix.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/stream_function_test.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/telnet_tcp.py -R -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/stmt_sample_csv_json-subtable.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/from-to-continue.py -,,n,army,python3 ./test.py -f tools/benchmark/basic/sml_json_alltypes-interlace.py -,,n,army,python3 ./test.py -f tools/benchmark/basic/commandline-retry.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/tmq_case.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/limit_offset_json.py -,,n,army,python3 ./test.py -f tools/benchmark/basic/commandline-sml.py -,,n,army,python3 ./test.py -f tools/benchmark/basic/sml_insert_alltypes_json.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/taosc_insert_alltypes_json.py +# benchmark 66 cases ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/commandline.py -,,n,army,python3 ./test.py -f tools/benchmark/basic/sml_taosjson_insert_alltypes-same-min-max.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/taosc_insert_alltypes-same-min-max.py -,,n,army,python3 ./test.py -f tools/benchmark/basic/bugs.py -B -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/taosc_sample_csv_json-subtable.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/query_json-with-error-sqlfile.py -,,n,army,python3 ./test.py -f tools/benchmark/basic/taosc_insert-retry-json-global.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/from-to.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/exportCsv.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/taosdemoTestQueryWithJson.py -R ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/commandline-partial-col-numpy.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/query_json-with-sqlfile.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/query_json.py -B -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/sml_json_alltypes.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/invalid_commandline.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/sml_json_insert_alltypes-same-min-max.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/default_json.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/taosc_sample_csv_json.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/stmt_sample_csv_json_doesnt_use_ts.py -,,n,army,python3 ./test.py -f tools/benchmark/basic/taosadapter_json.py -B -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/demo.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/commandline-sml-rest.py -R +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/commandline-single-table.py ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/commandline-supplement-insert.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/commandline-vgroups.py + ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/custom_col_tag.py + +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/default_json.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/default_tmq_json.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/demo.py + +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/exportCsv.py + +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/from-to.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/from-to-continue.py + +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/json_tag.py + +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/insert-json-csv.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/insertBasic.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/insertBindVGroup.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/insertMix.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/insertPrecision.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/invalid_commandline.py + +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/query_json.py -B +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/query_json-with-error-sqlfile.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/query_json-with-sqlfile.py + +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/rest_insert_alltypes_json.py -R +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/reuse-exist-stb.py + +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/sml_auto_create_table_json.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/sml_json_alltypes.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/sml_json_insert_alltypes-same-min-max.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/sml_taosjson_alltypes.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/sml_telnet_alltypes.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/sml_telnet_insert_alltypes-same-min-max.py ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/stmt_auto_create_table_json.py ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/stmt_insert_alltypes_json.py ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/stmt_insert_alltypes-same-min-max.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/commandline-vgroups.py -,,n,army,python3 ./test.py -f tools/benchmark/basic/taosc_insert-retry-json-stb.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/sml_auto_create_table_json.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/sml_telnet_alltypes.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/stmt_offset_json.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/stmt_sample_csv_json_doesnt_use_ts.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/stmt_sample_csv_json-subtable.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/stmt_sample_csv_json.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/stmt2_insert.py ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/stream-test.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/sml_taosjson_alltypes.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/commandline-single-table.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/stream_function_test.py ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/stt.py + +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/taosc_auto_create_table_json.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/taosc_sample_csv_json-subtable.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/taosc_insert_alltypes-same-min-max.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/taosc_insert_alltypes_json.py ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/taosc_insert_alltypes_json-partial-col.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/taosc_insert-table-creating-interval.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/taosc_sample_csv_json.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/taosdemoTestInsertWithJsonStmt-otherPara.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/taosdemoTestQueryWithJson.py -R +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/telnet_tcp.py -R +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/tmq_case.py + ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/cloud/cloud-test.py ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/ws/websocket.py -R +,,n,army,python3 ./test.py -f tools/benchmark/basic/bugs.py -B +,,n,army,python3 ./test.py -f tools/benchmark/basic/commandline-retry.py +,,n,army,python3 ./test.py -f tools/benchmark/basic/commandline-sml.py +,,n,army,python3 ./test.py -f tools/benchmark/basic/sml_json_alltypes-interlace.py +,,n,army,python3 ./test.py -f tools/benchmark/basic/sml_insert_alltypes_json.py +,,n,army,python3 ./test.py -f tools/benchmark/basic/sml_interlace.py +,,n,army,python3 ./test.py -f tools/benchmark/basic/sml_taosjson_insert_alltypes-same-min-max.py +,,n,army,python3 ./test.py -f tools/benchmark/basic/taosadapter_json.py -B +,,n,army,python3 ./test.py -f tools/benchmark/basic/taosc_insert-mix.py +,,n,army,python3 ./test.py -f tools/benchmark/basic/taosc_insert-retry-json-global.py +,,n,army,python3 ./test.py -f tools/benchmark/basic/taosc_insert-retry-json-stb.py +,,n,army,python3 ./test.py -f tools/benchmark/basic/taosdemoTestQueryWithJson-mixed-query.py -R + + # taosdump 43 cases ,,y,army,./pytest.sh python3 ./test.py -f tools/taosdump/native/taosdumpCompa.py ,,y,army,./pytest.sh python3 ./test.py -f tools/taosdump/native/taosdumpTest.py @@ -191,6 +206,7 @@ ,,n,army,python3 ./test.py -f tools/taosdump/ws/taosdumpTestTypeBool.py -B ,,n,army,python3 ./test.py -f tools/taosdump/ws/taosdumpRetry.py -B ,,n,army,python3 ./test.py -f tools/taosdump/ws/taosdumpTestTypeTinyInt.py -B +,,y,army,./pytest.sh python3 ./test.py -f tools/taosdump/native/taosdumpCommandline.py # # system test diff --git a/tools/shell/test/shellTest.cpp b/tools/shell/test/shellTest.cpp index cf0ec503fe..7f0a7e38c8 100644 --- a/tools/shell/test/shellTest.cpp +++ b/tools/shell/test/shellTest.cpp @@ -18,7 +18,7 @@ #include "shellAuto.h" TEST(fieldOptionsArea, autoTabTest) { - printf("hellow world SHELL tab test\n"); + printf("hello world SHELL tab test\n"); // str false const char *s0[] = { diff --git a/tools/taos-tools/CMakeLists.txt b/tools/taos-tools/CMakeLists.txt index 9c50674c66..1c9a0057cf 100644 --- a/tools/taos-tools/CMakeLists.txt +++ b/tools/taos-tools/CMakeLists.txt @@ -118,3 +118,11 @@ ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows") SET(MAKE_INSTALL_SH "${PROJECT_SOURCE_DIR}/packaging/tools/make_install.bat") INSTALL(CODE "execute_process(COMMAND ${MAKE_INSTALL_SH} :needAdmin ${PROJECT_SOURCE_DIR} ${PROJECT_BINARY_DIR} Windows)") ENDIF () + +# add test +IF(TD_LINUX) + # unit test + IF(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) + ENDIF(${BUILD_TEST}) +ENDIF() \ No newline at end of file diff --git a/tools/taos-tools/deps/toolscJson/src/toolscJson.c b/tools/taos-tools/deps/toolscJson/src/toolscJson.c index 56aa53a7e3..b52538912c 100644 --- a/tools/taos-tools/deps/toolscJson/src/toolscJson.c +++ b/tools/taos-tools/deps/toolscJson/src/toolscJson.c @@ -222,6 +222,7 @@ static tools_cJSON_bool parse_number(tools_cJSON * const item, parse_buffer * co unsigned char number_c_string[64]; unsigned char decimal_point = get_decimal_point(); size_t i = 0; + int isFloat = 0; if ((input_buffer == NULL) || (input_buffer->content == NULL)) { @@ -245,14 +246,18 @@ static tools_cJSON_bool parse_number(tools_cJSON * const item, parse_buffer * co case '7': case '8': case '9': + number_c_string[i] = buffer_at_offset(input_buffer)[i]; + break; case '+': case '-': case 'e': case 'E': + isFloat = 1; number_c_string[i] = buffer_at_offset(input_buffer)[i]; break; case '.': + isFloat = 1; number_c_string[i] = decimal_point; break; @@ -282,7 +287,10 @@ loop_end: } else { - item->valueint = (int64_t)number; + if(isFloat) + item->valueint = (int64_t)number; + else + item->valueint = strtoll((const char*)number_c_string, (char**)&after_end, 10); } item->type = tools_cJSON_Number; diff --git a/tools/taos-tools/inc/bench.h b/tools/taos-tools/inc/bench.h index d47bafbaf0..078bc1ac20 100644 --- a/tools/taos-tools/inc/bench.h +++ b/tools/taos-tools/inc/bench.h @@ -773,9 +773,7 @@ typedef struct SArguments_S { int rest_server_ver_major; bool check_sql; int suit; // see define SUIT_ -#ifdef TD_VER_COMPATIBLE_3_0_0_0 int16_t inputted_vgroups; -#endif enum CONTINUE_IF_FAIL_MODE continueIfFail; bool mistMode; bool escape_character; diff --git a/tools/taos-tools/inc/toolsdef.h b/tools/taos-tools/inc/toolsdef.h index aaf918b897..ed17380251 100644 --- a/tools/taos-tools/inc/toolsdef.h +++ b/tools/taos-tools/inc/toolsdef.h @@ -16,9 +16,14 @@ #ifndef __TOOLSDEF_H_ #define __TOOLSDEF_H_ +#ifdef __cplusplus +extern "C" { +#endif + #include #include + #define TINY_BUFF_LEN 8 #define SMALL_BUFF_LEN 20 #define MIDDLE_BUFF_LEN 64 @@ -236,4 +241,8 @@ int setConsoleEcho(bool on); char *toolsFormatTimestamp(char *buf, int64_t val, int32_t precision); +#ifdef __cplusplus +} /* end extern "C" */ +#endif + #endif // __TOOLSDEF_H_ diff --git a/tools/taos-tools/src/benchInsert.c b/tools/taos-tools/src/benchInsert.c index 4c86d8a24d..885a6e529f 100644 --- a/tools/taos-tools/src/benchInsert.c +++ b/tools/taos-tools/src/benchInsert.c @@ -607,14 +607,10 @@ int32_t toolsGetDefaultVGroups() { infoPrint("check local machine CPU: %d Memory:%d MB \n", cores, (int32_t)(MemKB/1024)); if (MemKB <= 2*1024*1024) { // 2G return 1; - } else if (MemKB <= 4*1024*1024) { // 4G + } else if (MemKB <= 256*1024*1024) { // 256G return 2; - } else if (MemKB <= 8*1024*1024) { // 8G - return 3; - } else if (MemKB <= 16*1024*1024) { // 16G + } else if (MemKB <= 512*1024*1024) { // 512G return 4; - } else if (MemKB <= 32*1024*1024) { // 32G - return 5; } else { return cores / 2; } @@ -623,22 +619,13 @@ int32_t toolsGetDefaultVGroups() { int geneDbCreateCmd(SDataBase *database, char *command, int remainVnodes) { int dataLen = 0; int n; - if (-1 != g_arguments->inputted_vgroups) { - n = snprintf(command + dataLen, SHORT_1K_SQL_BUFF_LEN - dataLen, - g_arguments->escape_character - ? "CREATE DATABASE IF NOT EXISTS `%s` VGROUPS %d" - : "CREATE DATABASE IF NOT EXISTS %s VGROUPS %d", - database->dbName, - (-1 != g_arguments->inputted_vgroups)? - g_arguments->inputted_vgroups: - min(remainVnodes, toolsGetNumberOfCores())); - } else { - n = snprintf(command + dataLen, SHORT_1K_SQL_BUFF_LEN - dataLen, - g_arguments->escape_character - ? "CREATE DATABASE IF NOT EXISTS `%s`" - : "CREATE DATABASE IF NOT EXISTS %s", - database->dbName); - } + + // create database + n = snprintf(command + dataLen, SHORT_1K_SQL_BUFF_LEN - dataLen, + g_arguments->escape_character + ? "CREATE DATABASE IF NOT EXISTS `%s`" + : "CREATE DATABASE IF NOT EXISTS %s", + database->dbName); if (n < 0 || n >= SHORT_1K_SQL_BUFF_LEN - dataLen) { errorPrint("%s() LN%d snprintf overflow\n", @@ -648,9 +635,24 @@ int geneDbCreateCmd(SDataBase *database, char *command, int remainVnodes) { dataLen += n; } + int vgroups = g_arguments->inputted_vgroups; + + // append config items if (database->cfgs) { for (int i = 0; i < database->cfgs->size; i++) { SDbCfg* cfg = benchArrayGet(database->cfgs, i); + + // check vgroups + if (strcasecmp(cfg->name, "vgroups") == 0) { + if (vgroups > 0) { + // inputted vgroups by commandline + infoPrint("ignore config set vgroups %d\n", cfg->valueint); + } else { + vgroups = cfg->valueint; + } + continue; + } + if (cfg->valuestring) { n = snprintf(command + dataLen, TSDB_MAX_ALLOWED_SQL_LEN - dataLen, @@ -670,6 +672,17 @@ int geneDbCreateCmd(SDataBase *database, char *command, int remainVnodes) { } } + // benchmark default + if (vgroups < 1) { + vgroups = toolsGetDefaultVGroups(); + debugPrint("vgroup set with toolsGetDefaultVGroups(). vgroups=%d\n", vgroups); + } + + // not found vgroups + if (vgroups > 0) { + dataLen += snprintf(command + dataLen, TSDB_MAX_ALLOWED_SQL_LEN - dataLen, " VGROUPS %d", vgroups); + } + switch (database->precision) { case TSDB_TIME_PRECISION_MILLI: snprintf(command + dataLen, TSDB_MAX_ALLOWED_SQL_LEN - dataLen, @@ -4137,6 +4150,8 @@ int32_t runInsertLimitThread(SDataBase* database, SSuperTable* stbInfo, int32_t } } + tmfree(slot); + return 0; } #endif diff --git a/tools/taos-tools/src/toolsDir.c b/tools/taos-tools/src/toolsDir.c index 8b42b6454d..666836f801 100644 --- a/tools/taos-tools/src/toolsDir.c +++ b/tools/taos-tools/src/toolsDir.c @@ -93,36 +93,6 @@ typedef struct dirent TdDirEntry; #endif -int32_t toolsExpandDir(const char *dirname, char *outname, int32_t maxlen) { - wordexp_t full_path; - switch (wordexp(dirname, &full_path, 0)) { - case 0: - break; - case WRDE_NOSPACE: - wordfree(&full_path); - // printf("failed to expand path:%s since Out of memory\n", dirname); - return -1; - case WRDE_BADCHAR: - // printf("failed to expand path:%s since illegal occurrence of newline or one of |, &, ;, <, >, (, ), {, }\n", - // dirname); - return -1; - case WRDE_SYNTAX: - // printf("failed to expand path:%s since Shell syntax error, such as unbalanced parentheses or unmatched - // quotes\n", dirname); - return -1; - default: - // printf("failed to expand path:%s since %s\n", dirname, strerror(errno)); - return -1; - } - - if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) { - strncpy(outname, full_path.we_wordv[0], maxlen); - } - - wordfree(&full_path); - - return 0; -} TdDirPtr toolsOpenDir(const char *dirname) { if (dirname == NULL) { diff --git a/tools/taos-tools/test/CMakeLists.txt b/tools/taos-tools/test/CMakeLists.txt new file mode 100644 index 0000000000..1586dae65d --- /dev/null +++ b/tools/taos-tools/test/CMakeLists.txt @@ -0,0 +1,44 @@ + +MESSAGE(STATUS "build taos-tools unit test") + +IF(TD_LINUX) + + # GoogleTest requires at least C++11 + SET(CMAKE_CXX_STANDARD 11) + AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + + # benchmark + ADD_EXECUTABLE(benchmarkTest benchmarkTest.cpp) + TARGET_LINK_LIBRARIES( + benchmarkTest + PRIVATE gtest + ) + + target_include_directories( + benchmarkTest + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../inc" + ) + + add_test( + NAME benchmarkTest + COMMAND benchmarkTest + ) + + # taosdump + ADD_EXECUTABLE(taosdumpTest taosdumpTest.cpp ../src/toolsSys.c) + TARGET_LINK_LIBRARIES( + taosdumpTest + PRIVATE gtest + ) + + target_include_directories( + taosdumpTest + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../inc" + ) + + add_test( + NAME taosdumpTest + COMMAND taosdumpTest + ) + +ENDIF() diff --git a/tools/taos-tools/test/benchmarkTest.cpp b/tools/taos-tools/test/benchmarkTest.cpp new file mode 100644 index 0000000000..5ea296e4cb --- /dev/null +++ b/tools/taos-tools/test/benchmarkTest.cpp @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include + +TEST(jsonTest, taosBenchmarkTest) { + printf("hello world taosBenchmark unit test for C \n"); +} + +int main(int argc, char **argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/tools/taos-tools/test/taosdumpTest.cpp b/tools/taos-tools/test/taosdumpTest.cpp new file mode 100644 index 0000000000..2e327c15d6 --- /dev/null +++ b/tools/taos-tools/test/taosdumpTest.cpp @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include + +#include "toolsdef.h" + +TEST(taosdump, toolsSys) { + // errorPrintReqArg3 + errorPrintReqArg3((char *)"taosdump", (char *)"test parameters"); + printf("ut function errorPrintReqArg3 .................... [Passed]\n"); + + // setConsoleEcho + setConsoleEcho(true); + setConsoleEcho(false); + printf("ut function setConsoleEcho ....................... [Passed]\n"); +} + +int main(int argc, char **argv) { + printf("hello world taosdump unit test for C\n"); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file