From b5ea56d80999d1d8b8dd6d530999a44923c3272d Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 4 Mar 2025 11:12:41 +0800 Subject: [PATCH] Test(insert):Add some test for auto create. --- source/common/src/msg/tmsg.c | 6 +- source/libs/parser/src/parInsertSql.c | 2 +- tests/army/insert/auto_create_bench.py | 114 +++++++++++++++ tests/army/insert/auto_create_insert.py | 179 ++++++++++++++++++++++++ 4 files changed, 299 insertions(+), 2 deletions(-) create mode 100644 tests/army/insert/auto_create_bench.py create mode 100644 tests/army/insert/auto_create_insert.py diff --git a/source/common/src/msg/tmsg.c b/source/common/src/msg/tmsg.c index b81b094882..1625d11943 100644 --- a/source/common/src/msg/tmsg.c +++ b/source/common/src/msg/tmsg.c @@ -11952,7 +11952,11 @@ int32_t tEncodeSubmitReq(SEncoder *pCoder, const SSubmitReq2 *pReq) { } } else{ for (uint64_t i = 0; i < taosArrayGetSize(pReq->aSubmitTbData); i++) { - TAOS_CHECK_EXIT(tEncodeSSubmitTbData(pCoder, taosArrayGet(pReq->aSubmitTbData, i))); + SSubmitTbData *pSubmitTbData = taosArrayGet(pReq->aSubmitTbData, i); + if ((pSubmitTbData->flags & SUBMIT_REQ_AUTO_CREATE_TABLE) && pSubmitTbData->pCreateTbReq == NULL) { + pSubmitTbData->flags = 0; + } + TAOS_CHECK_EXIT(tEncodeSSubmitTbData(pCoder, pSubmitTbData)); } } diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 0c60a787ce..74fb667733 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1019,7 +1019,7 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt code = tTagNew(pTagVals, 1, false, &pTag); } - if (TSDB_CODE_SUCCESS == code && !isParseBindParam) { + if (TSDB_CODE_SUCCESS == code && !isParseBindParam && !autoCreate) { code = buildCreateTbReq(pStmt, pTag, pTagName); pTag = NULL; } diff --git a/tests/army/insert/auto_create_bench.py b/tests/army/insert/auto_create_bench.py new file mode 100644 index 0000000000..0996aac87a --- /dev/null +++ b/tests/army/insert/auto_create_bench.py @@ -0,0 +1,114 @@ +import time +import taos + +conn = taos.connect() + +total_batches = 100 +tables_per_batch = 100 + +def prepare_database(): + cursor = conn.cursor() + cursor.execute("DROP DATABASE IF EXISTS test") + cursor.execute("CREATE DATABASE IF NOT EXISTS test") + cursor.execute("USE test") + cursor.execute("CREATE STABLE IF NOT EXISTS stb (ts TIMESTAMP, a INT, b FLOAT, c BINARY(10)) TAGS (e_id INT)") + cursor.close() + +def test_auto_create_tables(): + """测试场景1：自动建表插入""" + cursor = conn.cursor() + cursor.execute("USE test") + print("开始测试自动建表插入...") + + start_time = time.time() + for _ in range(100): + for batch in range(total_batches): + # 生成当前批次的子表ID范围 + start_id = batch * tables_per_batch + end_id = start_id + tables_per_batch + + # 构建批量插入SQL + sql_parts = [] + for i in range(start_id, end_id): + sql_part = f"t_{i} USING stb TAGS ({i}) VALUES ('2024-01-01 00:00:00', 1, 2.0, 'test')" + sql_parts.append(sql_part) + + # 执行批量插入 + full_sql = "INSERT INTO " + " ".join(sql_parts) + cursor.execute(full_sql) + + + elapsed = time.time() - start_time + print(f"自动建表插入耗时: {elapsed:.2f} 秒") + + cursor.close() + return elapsed + +def precreate_tables(): + """预处理：创建所有子表结构""" + cursor = conn.cursor() + cursor.execute("USE test") + + print("\n开始预创建子表...") + start_time = time.time() + + for batch in range(total_batches): + start_id = batch * tables_per_batch + end_id = start_id + tables_per_batch + + for i in range(start_id, end_id): + sql_part = f"CREATE TABLE t_{i} USING stb TAGS ({i})" + cursor.execute(sql_part) + + elapsed = time.time() - start_time + print(f"子表预创建耗时: {elapsed:.2f} 秒") + + cursor.close() + +def test_direct_insert(): + """测试场景2：直接插入已存在的子表""" + cursor = conn.cursor() + cursor.execute("USE test") + + print("\n开始测试直接插入...") + start_time = time.time() + for _ in range(100): + for batch in range(total_batches): + start_id = batch * tables_per_batch + end_id = start_id + tables_per_batch + + # 构建批量插入SQL + sql_parts = [] + for i in range(start_id, end_id): + sql_part = f"t_{i} VALUES ('2024-01-01 00:00:01', 1, 2.0, 'test')" + sql_parts.append(sql_part) + + # 执行批量插入 + full_sql = "INSERT INTO " + " ".join(sql_parts) + cursor.execute(full_sql) + + elapsed = time.time() - start_time + print(f"直接插入耗时: {elapsed:.2f} 秒") + + cursor.close() + return elapsed + +if __name__ == "__main__": + # 初始化数据库环境 + prepare_database() + # 预创建所有子表 + precreate_tables() + # 测试场景1：自动建表插入 + auto_create_time = test_auto_create_tables() + # 清理环境并重新初始化 + prepare_database() + # 预创建所有子表 + precreate_tables() + # 测试场景2：直接插入 + direct_insert_time = test_direct_insert() + + # 打印最终结果 + print("\n测试结果对比:") + print(f"自动建表插入耗时: {auto_create_time:.2f} 秒") + print(f"直接插入耗时: {direct_insert_time:.2f} 秒") + print(f"性能差异: {auto_create_time/direct_insert_time:.1f} 倍") \ No newline at end of file diff --git a/tests/army/insert/auto_create_insert.py b/tests/army/insert/auto_create_insert.py new file mode 100644 index 0000000000..4137c0018c --- /dev/null +++ b/tests/army/insert/auto_create_insert.py @@ -0,0 +1,179 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import random + +import taos +import frame +import frame.etool + + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * + + +class TDTestCase(TBase): + + def prepare_database(self): + tdLog.info(f"prepare database") + tdSql.execute("DROP DATABASE IF EXISTS test") + tdSql.execute("CREATE DATABASE IF NOT EXISTS test") + tdSql.execute("USE test") + tdSql.execute("CREATE STABLE IF NOT EXISTS stb (ts TIMESTAMP, a INT, b FLOAT, c BINARY(10)) TAGS (e_id INT)") + + + def insert_table_auto_create(self): + tdLog.info(f"insert table auto create") + tdSql.execute("USE test") + tdLog.info("start to test auto create insert...") + tdSql.execute("INSERT INTO t_0 USING stb TAGS (0) VALUES ('2024-01-01 00:00:00', 1, 2.0, 'test')") + tdSql.execute("INSERT INTO t_0 USING stb TAGS (0) VALUES ('2024-01-01 00:00:01', 1, 2.0, 'test')") + tdSql.query("select * from t_0") + tdSql.checkRows(2) + + def insert_table_pre_create(self): + tdLog.info(f"insert table pre create") + tdSql.execute("USE test") + tdLog.info("start to pre create table...") + tdSql.execute("CREATE TABLE t_1 USING stb TAGS (1)") + tdLog.info("start to test pre create insert...") + tdSql.execute("INSERT INTO t_1 USING stb TAGS (1) VALUES ('2024-01-01 00:00:00', 1, 2.0, 'test')") + tdSql.execute("INSERT INTO t_1 VALUES ('2024-01-01 00:00:01', 1, 2.0, 'test')") + tdSql.query("select * from t_1") + tdSql.checkRows(2) + + def insert_table_auto_insert_with_cache(self): + tdLog.info(f"insert table auto insert with cache") + tdSql.execute("USE test") + tdLog.info("start to test auto insert with cache...") + tdSql.execute("CREATE TABLE t_2 USING stb TAGS (2)") + tdLog.info("start to insert to init cache...") + tdSql.execute("INSERT INTO t_2 VALUES ('2024-01-01 00:00:00', 1, 2.0, 'test')") + tdSql.execute("INSERT INTO t_2 USING stb TAGS (2) VALUES ('2024-01-01 00:00:01', 1, 2.0, 'test')") + tdSql.query("select * from t_2") + tdSql.checkRows(2) + + def insert_table_auto_insert_with_multi_rows(self): + tdLog.info(f"insert table auto insert with multi rows") + tdSql.execute("USE test") + tdLog.info("start to test auto insert with multi rows...") + tdSql.execute("CREATE TABLE t_3 USING stb TAGS (3)") + tdLog.info("start to insert multi rows...") + tdSql.execute("INSERT INTO t_3 VALUES ('2024-01-01 00:00:00', 1, 2.0, 'test'), ('2024-01-01 00:00:01', 1, 2.0, 'test')") + tdSql.query("select * from t_3") + tdSql.checkRows(2) + + tdLog.info("start to insert multi rows with direct insert and auto create...") + tdSql.execute("INSERT INTO t_4 USING stb TAGS (4) VALUES ('2024-01-01 00:00:00', 1, 2.0, 'test'), t_3 VALUES ('2024-01-01 00:00:02', 1, 2.0, 'test')") + tdSql.query("select * from t_4") + tdSql.checkRows(1) + tdSql.query("select * from t_3") + tdSql.checkRows(3) + + tdLog.info("start to insert multi rows with auto create and direct insert...") + tdSql.execute("INSERT INTO t_3 VALUES ('2024-01-01 00:00:03', 1, 2.0, 'test'),t_4 USING stb TAGS (4) VALUES ('2024-01-01 00:00:01', 1, 2.0, 'test'),") + tdSql.query("select * from t_4") + tdSql.checkRows(2) + tdSql.query("select * from t_3") + tdSql.checkRows(4) + + tdLog.info("start to insert multi rows with auto create into same table...") + tdSql.execute("INSERT INTO t_10 USING stb TAGS (10) VALUES ('2024-01-01 00:00:04', 1, 2.0, 'test'),t_10 USING stb TAGS (10) VALUES ('2024-01-01 00:00:05', 1, 2.0, 'test'),") + tdSql.query("select * from t_10") + tdSql.checkRows(2) + + def check_some_err_case(self): + tdLog.info(f"check some err case") + tdSql.execute("USE test") + + tdLog.info("start to test err stb name...") + tdSql.error("INSERT INTO t_5 USING errrrxx TAGS (5) VALUES ('2024-01-01 00:00:00', 1, 2.0, 'test')", expectErrInfo="Table does not exist") + + tdLog.info("start to test err syntax name...") + tdSql.error("INSERT INTO t_5 USING stb TAG (5) VALUES ('2024-01-01 00:00:00', 1, 2.0, 'test')", expectErrInfo="syntax error") + + tdLog.info("start to test err syntax values...") + tdSql.error("INSERT INTO t_5 USING stb TAG (5) VALUS ('2024-01-01 00:00:00', 1, 2.0, 'test')", expectErrInfo="syntax error") + + tdLog.info("start to test err tag counts...") + tdSql.error("INSERT INTO t_5 USING stb TAG (5,1) VALUS ('2024-01-01 00:00:00', 1, 2.0, 'test')", expectErrInfo="syntax error") + + tdLog.info("start to test err tag counts...") + tdSql.error("INSERT INTO t_5 USING stb TAG ('dasds') VALUS ('2024-01-01 00:00:00', 1, 2.0, 'test')", expectErrInfo="syntax error") + + tdLog.info("start to test err values counts...") + tdSql.error("INSERT INTO t_5 USING stb TAGS (5) VALUES ('2024-01-01 00:00:00', 1, 1 ,2.0, 'test')", expectErrInfo="Illegal number of columns") + + tdLog.info("start to test err values...") + tdSql.error("INSERT INTO t_5 USING stb TAGS (5) VALUES ('2024-01-01 00:00:00', 'dasdsa', 1 ,2.0, 'test')", expectErrInfo="syntax error") + + def check_same_table_same_ts(self): + tdLog.info(f"check same table same ts") + tdSql.execute("USE test") + tdSql.execute("INSERT INTO t_6 USING stb TAGS (6) VALUES ('2024-01-01 00:00:00', 1, 2.0, 'test') t_6 USING stb TAGS (6) VALUES ('2024-01-01 00:00:00', 1, 2.0, 'test')") + tdSql.query("select * from t_6") + tdSql.checkRows(1) + + def check_tag_parse_error_with_cache(self): + tdLog.info(f"check tag parse error with cache") + tdSql.execute("USE test") + tdSql.execute("INSERT INTO t_7 USING stb TAGS (7) VALUES ('2024-01-01 00:00:00', 1, 2.0, 'test')") + tdSql.error("INSERT INTO t_7 USING stb TAGS ('ddd') VALUES ('2024-01-01 00:00:00', 1, 2.0, 'test')", expectErrInfo="syntax error") + tdSql.query("select * from t_7") + tdSql.checkRows(1) + + def check_duplicate_table_with_err_tag(self): + tdLog.info(f"check tag parse error with cache") + tdSql.execute("USE test") + tdSql.error("INSERT INTO t_8 USING stb TAGS (8) VALUES ('2024-01-01 00:00:00', 1, 2.0, 'test') t_8 USING stb TAGS (ddd) VALUES ('2024-01-01 00:00:00', 1, 2.0, 'test')", expectErrInfo="syntax error") + + # run + def run(self): + tdLog.debug(f"start to excute {__file__}") + + # prepare database + self.prepare_database() + + # insert table auto create + self.insert_table_auto_create() + + # insert table pre create + self.insert_table_pre_create() + + # insert table auto insert with cache + self.insert_table_auto_insert_with_cache() + + # insert table auto insert with multi rows + self.insert_table_auto_insert_with_multi_rows() + + # check some err case + self.check_some_err_case() + + # check same table same ts + self.check_same_table_same_ts() + + # check tag parse error with cache + self.check_tag_parse_error_with_cache() + + # check duplicate table with err tag + self.check_duplicate_table_with_err_tag() + + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file