From 6bfc24d4ae65f613107822c2e791b14c696a3457 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 7 Mar 2025 21:04:52 +0800 Subject: [PATCH] fix: stmt not support many stb with single = false stmt_init --- tests/army/frame/caseBase.py | 36 +++++ .../army/tools/benchmark/basic/insertBasic.py | 11 +- .../benchmark/basic/json/insertManyStb.json | 151 ++++++++++++++++++ tools/taos-tools/src/benchInsert.c | 9 +- 4 files changed, 199 insertions(+), 8 deletions(-) create mode 100644 tests/army/tools/benchmark/basic/json/insertManyStb.json diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 4427ddcea5..e527eae013 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -19,6 +19,7 @@ import random import copy import json +import frame.eos import frame.etool import frame.eutil from frame.log import * @@ -427,6 +428,41 @@ class TBase: return db, stb, child_count, insert_rows + # insert & check + def benchInsert(self, jsonFile, options = "", results = None): + # exe insert + benchmark = frame.etool.benchMarkFile() + cmd = f"{benchmark} {options} -f {jsonFile}" + rlist = frame.eos.runRetList(cmd, True, True, True) + if results != None: + for result in results: + self.checkListString(rlist, result) + + # open json + with open(jsonFile, "r") as file: + data = json.load(file) + + # read json + dbs = data["databases"] + for db in dbs: + dbName = db["dbinfo"]["name"] + stbs = db["super_tables"] + for stb in stbs: + child_count = stb["childtable_count"] + insert_rows = stb["insert_rows"] + timestamp_step = stb["timestamp_step"] + + # check result + + # count + sql = f"select count(*) from {db}.{stb}" + tdSql.checkAgg(sql, child_count * insert_rows) + # diff + sql = f"select * from (select diff(ts) as dif from {db}.{stb} partition by tbname) where dif != {timestamp_step};" + tdSql.query(sql) + tdSql.checkRows(0) + # show + tdLog.info(f"insert check passed. db:{db} stb:{stb} child_count:{child_count} insert_rows:{insert_rows}\n") # tmq def tmqBenchJson(self, jsonFile, options="", checkStep=False): diff --git a/tests/army/tools/benchmark/basic/insertBasic.py b/tests/army/tools/benchmark/basic/insertBasic.py index f1d3b81732..9cf7a515cf 100644 --- a/tests/army/tools/benchmark/basic/insertBasic.py +++ b/tests/army/tools/benchmark/basic/insertBasic.py @@ -13,6 +13,7 @@ import os import json import frame +import frame.eos import frame.etool from frame.log import * from frame.cases import * @@ -109,7 +110,6 @@ class TDTestCase(TBase): tdLog.info(f" vgroups real={tdSql.getData(0,0)} expect={vgroups}") tdSql.checkData(0, 0, vgroups, True) - # bugs ts def checkVGroups(self, benchmark): # vgroups with command line set @@ -117,12 +117,19 @@ class TDTestCase(TBase): # vgroups with json file self.testBenchmarkJson(benchmark, "./tools/benchmark/basic/json/insertBasic.json", "", True) + + def checkInsertManyStb(self): + # many stb + self.benchInsert("./tools/benchmark/basic/json/insertManyStb.json", "", True) + def run(self): - benchmark = etool.benchMarkFile() + benchmark = frame.etool.benchMarkFile() # vgroups self.checkVGroups(benchmark) + # check many stable + self.checkInsertManyStb() def stop(self): tdSql.close() diff --git a/tests/army/tools/benchmark/basic/json/insertManyStb.json b/tests/army/tools/benchmark/basic/json/insertManyStb.json new file mode 100644 index 0000000000..f00308da23 --- /dev/null +++ b/tests/army/tools/benchmark/basic/json/insertManyStb.json @@ -0,0 +1,151 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "num_of_records_per_req": 3000, + "thread_count": 2, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "test1", + "drop": "yes", + "precision": "us", + "vgroups": 1 + }, + "super_tables": [ + { + "name": "meters1", + "child_table_exists": "no", + "childtable_count": 2, + "insert_rows": 1000, + "childtable_prefix": "d", + "insert_mode": "stmt", + "timestamp_step": 15, + "start_timestamp":1700000000000000000, + "columns": [ + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "binary", "name": "bin", "len": 4} + ], + "tags": [ + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 } + ] + }, + { + "name": "meters2", + "child_table_exists": "no", + "childtable_count": 3, + "insert_rows": 100, + "childtable_prefix": "d", + "insert_mode": "stmt", + "timestamp_step": 20, + "interlace_rows": 1, + "start_timestamp":1700000000000000000, + "columns": [ + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "binary", "name": "bin", "len": 4} + ], + "tags": [ + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 } + ] + }, + { + "name": "meters3", + "child_table_exists": "no", + "childtable_count": 5, + "insert_rows": 100, + "childtable_prefix": "d", + "insert_mode": "stmt2", + "timestamp_step": 20, + "interlace_rows": 1, + "start_timestamp":1700000000000000000, + "columns": [ + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "binary", "name": "bin", "len": 4} + ], + "tags": [ + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 } + ] + }, + { + "name": "meters4", + "child_table_exists": "no", + "childtable_count": 2, + "insert_rows": 70, + "childtable_prefix": "d", + "insert_mode": "stmt2", + "timestamp_step": 50, + "interlace_rows": 0, + "start_timestamp":1700000000000000000, + "columns": [ + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "binary", "name": "bin", "len": 4} + ], + "tags": [ + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 } + ] + } + ] + }, + { + "dbinfo": { + "name": "test2", + "drop": "yes", + "precision": "ns", + "vgroups": 2 + }, + "super_tables": [ + { + "name": "meters1", + "child_table_exists": "no", + "childtable_count": 3, + "insert_rows": 120, + "childtable_prefix": "d", + "insert_mode": "taosc", + "timestamp_step": 15, + "start_timestamp":1700000000000000000, + "columns": [ + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "binary", "name": "bin", "len": 4} + ], + "tags": [ + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 } + ] + }, + { + "name": "meters2", + "child_table_exists": "no", + "childtable_count": 2, + "insert_rows": 200, + "childtable_prefix": "d", + "insert_mode": "taosc", + "timestamp_step": 2, + "interlace_rows": 4, + "start_timestamp":1700000000000000000, + "columns": [ + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "binary", "name": "bin", "len": 4} + ], + "tags": [ + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tools/taos-tools/src/benchInsert.c b/tools/taos-tools/src/benchInsert.c index 885a6e529f..894e4621a0 100644 --- a/tools/taos-tools/src/benchInsert.c +++ b/tools/taos-tools/src/benchInsert.c @@ -1928,7 +1928,7 @@ static void *syncWriteInterlace(void *sarg) { bindv = createBindV(nBatchTable, tagCnt, stbInfo->cols->size + 1); } - bool oldInitStmt = stbInfo->autoTblCreating || database->superTbls->size > 1; + bool oldInitStmt = stbInfo->autoTblCreating; // not auto create table call once if(stbInfo->iface == STMT_IFACE && !oldInitStmt) { debugPrint("call prepareStmt for stable:%s\n", stbInfo->stbName); @@ -2900,7 +2900,7 @@ void *syncWriteProgressive(void *sarg) { tagData = benchCalloc(TAG_BATCH_COUNT, stbInfo->lenOfTags, false); } - bool oldInitStmt = stbInfo->autoTblCreating || database->superTbls->size > 1; + bool oldInitStmt = stbInfo->autoTblCreating; // stmt. not auto table create call on stmt if (stbInfo->iface == STMT_IFACE && !oldInitStmt) { if (prepareStmt(pThreadInfo->conn->stmt, stbInfo, tagData, w)) { @@ -3909,11 +3909,8 @@ int32_t initInsertThread(SDataBase* database, SSuperTable* stbInfo, int32_t nthr if (NULL == pThreadInfo->conn) { goto END; } + // single always true for benchmark bool single = true; - if (database->superTbls->size > 1) { - single = false; - } - if (stbInfo->iface == STMT2_IFACE) { // stmt2 init if (pThreadInfo->conn->stmt2)