From 125794c3a0265b59c959a9888980ad4b9cb5f130 Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 12 Mar 2024 01:54:12 +0000 Subject: [PATCH 01/21] fix/TD-29107 --- source/libs/audit/src/auditMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/audit/src/auditMain.c b/source/libs/audit/src/auditMain.c index 19dc771c56..96934888eb 100644 --- a/source/libs/audit/src/auditMain.c +++ b/source/libs/audit/src/auditMain.c @@ -27,7 +27,7 @@ #include "osMemory.h" SAudit tsAudit = {0}; -char* tsAuditUri = "/audit"; +char* tsAuditUri = "/audit_v2"; char* tsAuditBatchUri = "/audit-batch"; int32_t auditInit(const SAuditCfg *pCfg) { From b61bf20ebf86ca965e76be6a01c85234037999ac Mon Sep 17 00:00:00 2001 From: Chris Zhai Date: Wed, 3 Apr 2024 18:05:21 +0800 Subject: [PATCH 02/21] add test scripts for TS-4243 --- tests/parallel_test/cases.task | 3 + tests/pytest/util/csv.py | 62 ++ .../1-insert/composite_primary_key_create.py | 222 +++++ .../1-insert/composite_primary_key_delete.py | 244 ++++++ .../1-insert/composite_primary_key_insert.py | 776 ++++++++++++++++++ 5 files changed, 1307 insertions(+) create mode 100644 tests/pytest/util/csv.py create mode 100644 tests/system-test/1-insert/composite_primary_key_create.py create mode 100644 tests/system-test/1-insert/composite_primary_key_delete.py create mode 100644 tests/system-test/1-insert/composite_primary_key_insert.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 0cdac26a3a..101201a85e 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -308,6 +308,9 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/test_hot_refresh_configurations.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/subscribe_stream_privilege.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/composite_primary_key_create.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/composite_primary_key_insert.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/composite_primary_key_delete.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_double.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_replica.py -N 3 diff --git a/tests/pytest/util/csv.py b/tests/pytest/util/csv.py new file mode 100644 index 0000000000..6e1ee1732c --- /dev/null +++ b/tests/pytest/util/csv.py @@ -0,0 +1,62 @@ +import csv +import os + +class TDCsv: + def __init__(self): + self.__file_name = '' + self.__file_path = '' + + @property + def file_name(self): + return self.__file_name + + @file_name.setter + def file_name(self, value): + self.__file_name = value + + @property + def file_path(self): + return self.__file_path + + @file_path.setter + def file_path(self, value): + self.__file_path = value + + @property + def file(self): + if self.file_name and self.file_path: + return os.path.join(self.file_path, self.file_name) + return None + + + def read(self): + try: + with open(self.file, newline='') as csvfile: + reader = csv.reader(csvfile) + for row in reader: + print(row) + except Exception as errMsg: + raise Exception(errMsg) + + def __write_with_quotes(self, csv_writer, data): + for row in data: + row_with_quotes = [f'"{field}"' if isinstance(field, str) else field for field in row] + csv_writer.writerow(row_with_quotes) + + def write(self, data: dict): + try: + with open(self.file, 'w', newline='') as csvfile: + writer = csv.writer(csvfile) + writer.writerows(data) + # self.__write_with_quotes(writer, data) + except Exception as errMsg: + raise Exception(errMsg) + + def delete(self): + try: + if os.path.exists(self.file): + os.remove(self.file) + except Exception as errMsg: + raise Exception(errMsg) + + diff --git a/tests/system-test/1-insert/composite_primary_key_create.py b/tests/system-test/1-insert/composite_primary_key_create.py new file mode 100644 index 0000000000..503aa6df7d --- /dev/null +++ b/tests/system-test/1-insert/composite_primary_key_create.py @@ -0,0 +1,222 @@ +from enum import Enum + +from util.log import * +from util.sql import * +from util.cases import * + + +class IllegalDataType(Enum): + NULL = 'null' + BOOL = 'bool' + TINYINT = 'tinyint' + SMALLINT = 'smallint' + FLOAT = 'float' + DOUBLE = 'double' + TIMESTAMP = 'timestamp' + NCHAR = 'nchar(100)' + UTINYINT = 'tinyint unsigned' + USMALLINT = 'smallint unsigned' + JSON = 'json' + VARBINARY = 'varbinary(100)' + DECIMAL = 'decimal' + BLOB = 'blob' + MEDIUMBLOB = 'mediumblob' + GEOMETRY = 'geometry(512)' + EMPTY = '\'\'' + SPACE = '\' \'' + + +class LegalDataType(Enum): + INT = 'INT' + UINT = 'INT UNSIGNED' + BIGINT = 'BIGINT' + UBIGINT = 'BIGINT UNSIGNED' + VARCHAR = 'VARCHAR(100)' + BINARY = 'BINARY(100)' + +class LegalSpell(Enum): + CASE1 = 'primary key' + CASE2 = 'PRIMARY KEY' + CASE3 = 'Primary Key' + CASE4 = 'PriMary keY' + +class IllegalSpell(Enum): + CASE1 = 'primary' + CASE2 = 'key' + CASE3 = 'primay key' + CASE4 = 'primary ky' + CASE5 = 'primarykey' + CASE6 = 'key primary' + CASE6 = 'primay key primay key' + + +class TableType(Enum): + SUPERTABLE = 0 + CHILDTABLE = 1 + NORNALTABLE = 2 + +SHOW_LOG = True + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + self.database = "db_create_composite_primary_key" + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + def prepare_db(self): + tdSql.execute(f"drop database if exists {self.database}") + tdSql.execute(f"create database {self.database}") + tdSql.execute(f"use {self.database}") + + def check_pk_definition(self, table_name: str, d_type: LegalDataType, t_type: TableType): + tdSql.query(f"describe {table_name}", show=SHOW_LOG) + tdSql.checkData(1, 3, "PRIMARY KEY") + + if d_type == LegalDataType.BINARY: + d_type = LegalDataType.VARCHAR + + if t_type == TableType.SUPERTABLE: + expected_value = f'CREATE STABLE `{table_name}` (`ts` TIMESTAMP, `pk` {d_type.value} PRIMARY KEY, `c2` INT) TAGS (`engine` INT)' + elif t_type == TableType.NORNALTABLE: + expected_value = f'CREATE TABLE `{table_name}` (`ts` TIMESTAMP, `pk` {d_type.value} PRIMARY KEY, `c2` INT)' + + tdSql.query(f"show create table {table_name}", show=SHOW_LOG) + tdSql.checkData(0, 1, expected_value) + + def test_pk_datatype_legal(self, stable_name: str, ctable_name: str, ntable_name: str, dtype: LegalDataType): + # create super table and child table + tdSql.execute(f"drop table if exists {stable_name}", show=SHOW_LOG) + tdSql.execute(f"drop table if exists {ntable_name}", show=SHOW_LOG) + + tdSql.execute(f"create table {stable_name} (ts timestamp, pk {dtype.value} primary key, c2 int) tags (engine int)", show=SHOW_LOG) + self.check_pk_definition(stable_name, dtype, TableType.SUPERTABLE) + + tdSql.execute(f"create table {ctable_name} using {stable_name} tags (0)", show=SHOW_LOG) + tdSql.execute(f"create table {ctable_name}_1 using {stable_name} tags (1) {ctable_name}_2 using {stable_name} tags (2) {ctable_name}_3 using {stable_name} tags (3)", show=SHOW_LOG) + + # create normal table + tdSql.execute(f"create table {ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 int)", show=SHOW_LOG) + self.check_pk_definition(ntable_name, dtype, TableType.NORNALTABLE) + + def test_pk_datatype_illegal(self, stable_name: str, ntable_name: str, dtype: LegalDataType): + # create super table and child table + tdSql.execute(f"drop table if exists {stable_name}", show=SHOW_LOG) + tdSql.execute(f"drop table if exists {ntable_name}", show=SHOW_LOG) + + tdSql.error(f"create table {stable_name} (ts timestamp, pk {dtype.value} primary key, c2 int) tags (engine int)", show=SHOW_LOG) + + # create normal table + tdSql.error(f"create table {ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 int)", show=SHOW_LOG) + + def test_pk_spell_legal(self, stable_name: str, ntable_name: str, pk_spell: LegalSpell): + # create super table and child table + tdSql.execute(f"drop table if exists {stable_name}", show=SHOW_LOG) + tdSql.execute(f"drop table if exists {ntable_name}", show=SHOW_LOG) + + tdSql.execute(f"create table {stable_name} (ts timestamp, pk int {pk_spell.value}, c2 int) tags (engine int)", show=SHOW_LOG) + + # create normal table + tdSql.execute(f"create table {ntable_name} (ts timestamp, pk int {pk_spell.value}, c2 int)", show=SHOW_LOG) + + def test_pk_spell_illegal(self, stable_name: str, ntable_name: str, pk_spell: LegalSpell): + # create super table and child table + tdSql.execute(f"drop table if exists {stable_name}", show=SHOW_LOG) + tdSql.execute(f"drop table if exists {ntable_name}", show=SHOW_LOG) + + tdSql.error(f"create table {stable_name} (ts timestamp, pk int {pk_spell.value}, c2 int) tags (engine int)", show=SHOW_LOG) + + # create normal table + tdSql.error(f"create table {ntable_name} (ts timestamp, pk int {pk_spell.value}, c2 int)", show=SHOW_LOG) + + def test_update_pk(self, table_name: str, t_type: TableType): + # create super table and child table + tdSql.execute(f"drop table if exists {table_name}", show=SHOW_LOG) + + if t_type == TableType.SUPERTABLE: + tdSql.execute(f"create table {table_name} (ts timestamp, c2 int) tags (engine int)", show=SHOW_LOG) + elif t_type == TableType.NORNALTABLE: + # create normal table + tdSql.execute(f"create table {table_name} (ts timestamp, c2 int)", show=SHOW_LOG) + + tdSql.error(f"alter table {table_name} add column pk varchar(100) primary key", show=SHOW_LOG) + + tdSql.execute(f"drop table if exists {table_name}", show=SHOW_LOG) + if t_type == TableType.SUPERTABLE: + tdSql.execute(f"create table {table_name} (ts timestamp, pk varchar(200) primary key, c2 int) tags (engine int)", show=SHOW_LOG) + elif t_type == TableType.NORNALTABLE: + # create normal table + tdSql.execute(f"create table {table_name} (ts timestamp, pk varchar(200) primary key, c2 int)", show=SHOW_LOG) + + tdSql.error(f"alter table {table_name} add column new_pk varchar(20) primary key", show=SHOW_LOG) + for date_type in IllegalDataType.__members__.items(): + tdSql.error(f"alter table {table_name} modify column pk {date_type[1].value}", show=SHOW_LOG) + for date_type in LegalDataType.__members__.items(): + tdSql.error(f"alter table {table_name} modify column pk {date_type[1].value}", show=SHOW_LOG) + + tdSql.error(f"alter table {table_name} modify column pk varchar(300)", show=SHOW_LOG) + tdSql.error(f"alter table {table_name} rename column pk new_pk", show=SHOW_LOG) + tdSql.error(f"alter table {table_name} drop column pk", show=SHOW_LOG) + + def run(self): + tdSql.prepare(replica = self.replicaVar) + self.prepare_db() + + # 1.check legal data type + for date_type in LegalDataType.__members__.items(): + self.test_pk_datatype_legal('s_table', 'c_table', 'n_table', date_type[1]) + + # 2.check illegal data type + for date_type in IllegalDataType.__members__.items(): + self.test_pk_datatype_illegal('s_table', 'n_table', date_type[1]) + + # 3.check legal spell + for date_type in LegalSpell.__members__.items(): + self.test_pk_spell_legal('s_table', 'n_table', date_type[1]) + + # 4.check illegal spell + for date_type in IllegalSpell.__members__.items(): + self.test_pk_spell_illegal('s_table', 'n_table', date_type[1]) + + # 5.only define ts and pk columns + # create super table and child table + tdSql.execute(f"drop table if exists s_table", show=SHOW_LOG) + tdSql.execute(f"drop table if exists n_table", show=SHOW_LOG) + + tdSql.execute(f"create table s_table (ts timestamp, pk int primary key) tags (engine int)", show=SHOW_LOG) + tdSql.execute(f"create table c_table using s_table tags (1)", show=SHOW_LOG) + tdSql.execute(f"insert into c_table values(now, 1)", show=SHOW_LOG) + tdSql.query(f"select * from s_table", show=SHOW_LOG) + tdSql.checkRows(1) + + # create normal table + tdSql.execute(f"create table n_table (ts timestamp, pk int primary key)", show=SHOW_LOG) + tdSql.execute(f"insert into n_table values(now, 1)", show=SHOW_LOG) + tdSql.query(f"select * from n_table", show=SHOW_LOG) + tdSql.checkRows(1) + + # 6.mutiple pk & pk not defined as sencod column + tdSql.execute(f"drop table if exists s_table", show=SHOW_LOG) + tdSql.execute(f"drop table if exists n_table", show=SHOW_LOG) + + # create super table + tdSql.error(f"create table s_table (ts timestamp, c1 int, pk1 int primary key) tags (engine int)", show=SHOW_LOG) + tdSql.error(f"create table s_table (ts timestamp, pk1 int primary key, pk2 int primary key) tags (engine int)", show=SHOW_LOG) + tdSql.error(f"create table s_table (ts timestamp, pk1 int primary key, c2 int, pk2 int primary key) tags (engine int)", show=SHOW_LOG) + # create normal table + tdSql.error(f"create table n_table (ts timestamp, c1 int, pk1 int primary key)", show=SHOW_LOG) + tdSql.error(f"create table n_table (ts timestamp, pk1 int primary key, pk2 int primary key)", show=SHOW_LOG) + tdSql.error(f"create table n_table (ts timestamp, pk1 int primary key, c2 int, pk2 int primary key)", show=SHOW_LOG) + + # 7.add\update\delete pk column is not support + self.test_update_pk('s_table', TableType.SUPERTABLE) + self.test_update_pk('n_table', TableType.NORNALTABLE) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/1-insert/composite_primary_key_delete.py b/tests/system-test/1-insert/composite_primary_key_delete.py new file mode 100644 index 0000000000..31c7d7428e --- /dev/null +++ b/tests/system-test/1-insert/composite_primary_key_delete.py @@ -0,0 +1,244 @@ +from enum import Enum + +from util.log import * +from util.sql import * +from util.cases import * +from util.csv import * +import os +import taos +import json +from taos import SmlProtocol, SmlPrecision +from taos.error import SchemalessError + + +class LegalDataType(Enum): + INT = 'INT' + UINT = 'INT UNSIGNED' + BIGINT = 'BIGINT' + UBIGINT = 'BIGINT UNSIGNED' + VARCHAR = 'VARCHAR(100)' + BINARY = 'BINARY(100)' + + +class TableType(Enum): + SUPERTABLE = 0 + CHILDTABLE = 1 + NORNALTABLE = 2 + +SHOW_LOG = True +STAET_TS = '2023-10-01 00:00:00.000' + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + self.database = "db_insert_composite_primary_key" + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + self.testcasePath = os.path.split(__file__)[0] + self.testcasePath = self.testcasePath.replace('\\', '//') + self.testcaseFilename = os.path.split(__file__)[-1] + os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) + + self.stable_name = 's_table' + self.ctable_name = 'c_table' + self.ntable_name = 'n_table' + self.ts_list = {} + + + def prepare_db(self): + tdSql.execute(f"drop database if exists {self.database}") + tdSql.execute(f"create database {self.database} CACHEMODEL 'none'") + tdSql.execute(f"use {self.database}") + + def get_latest_ts(self, table_name: str): + tdSql.query(f'select last(ts) from {table_name}') + now_time = tdSql.queryResult[0][0].strftime("%Y-%m-%d %H:%M:%S.%f") + return f"'{now_time}'" + + def prepare_data(self, dtype: LegalDataType): + # drop super table and child table + tdSql.execute(f"drop table if exists {self.stable_name}") + tdSql.execute(f"drop table if exists {self.ntable_name}") + + # create super table & child table + tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c1 smallint, c2 varchar(10)) tags (engine int)", show=SHOW_LOG) + + table_name1 = f'{self.ctable_name}_1' + tdSql.execute(f"insert into {table_name1} using {self.stable_name} tags(1) values(now, 1, '7', '6')", show=SHOW_LOG) + child_ts_1 = self.get_latest_ts(table_name1) + tdSql.execute(f"insert into {table_name1} values({child_ts_1}, 2, '8', '6') ({child_ts_1}, 3, '9', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {table_name1} (ts, pk, c2) values({child_ts_1}, 4, '8') ({child_ts_1}, 5, '9')", show=SHOW_LOG) + + tdSql.execute(f"insert into {table_name1} values(now, 1, '7', '6')", show=SHOW_LOG) + child_ts_2 = self.get_latest_ts(table_name1) + tdSql.execute(f"insert into {table_name1} values({child_ts_2}, 2, '8', '6') ({child_ts_2}, 3, '9', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {table_name1} (ts, pk, c2) values({child_ts_2}, 4, '8') ({child_ts_2}, 5, '9')", show=SHOW_LOG) + + table_name2 = f'{self.ctable_name}_2' + tdSql.execute(f"insert into {table_name2} using {self.stable_name} tags(2) values(now, 1, '7', '6')", show=SHOW_LOG) + child_ts_3 = self.get_latest_ts(table_name2) + tdSql.execute(f"insert into {table_name2} values({child_ts_3}, 2, '8', '6') ({child_ts_3}, 3, '9', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {table_name2} (ts, pk, c2) values({child_ts_3}, 4, '8') ({child_ts_3}, 5, '9')", show=SHOW_LOG) + + tdSql.execute(f"insert into {table_name2} values(now, 1, '7', '6')", show=SHOW_LOG) + child_ts_4 = self.get_latest_ts(table_name2) + tdSql.execute(f"insert into {table_name2} values({child_ts_4}, 2, '8', '6') ({child_ts_4}, 3, '9', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {table_name2} (ts, pk, c2) values({child_ts_4}, 4, '8') ({child_ts_4}, 5, '9')", show=SHOW_LOG) + + table_name3 = f'{self.ctable_name}_3' + tdSql.execute(f"insert into {table_name3} using {self.stable_name} tags(3) values(now, 1, '7', '6')", show=SHOW_LOG) + child_ts_5 = self.get_latest_ts(table_name3) + tdSql.execute(f"insert into {table_name3} values({child_ts_5}, 2, '8', '6') ({child_ts_5}, 3, '9', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {table_name3} (ts, pk, c2) values({child_ts_5}, 4, '8') ({child_ts_5}, 5, '9')", show=SHOW_LOG) + + tdSql.execute(f"insert into {table_name3} values(now, 1, '7', '6')", show=SHOW_LOG) + child_ts_6 = self.get_latest_ts(table_name3) + tdSql.execute(f"insert into {table_name3} values({child_ts_6}, 2, '8', '6') ({child_ts_6}, 3, '9', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {table_name3} (ts, pk, c2) values({child_ts_6}, 4, '8') ({child_ts_6}, 5, '9')", show=SHOW_LOG) + + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(30) + + # create normal table + tdSql.execute(f"create table {self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c1 smallint, c2 varchar(10))", show=SHOW_LOG) + + tdSql.execute(f"insert into {self.ntable_name} values(now, 1, '7', '6')", show=SHOW_LOG) + normal_ts_1 = self.get_latest_ts(self.ntable_name) + tdSql.execute(f"insert into {self.ntable_name} values({normal_ts_1}, 2, '8', '6') ({normal_ts_1}, 3, '9', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {self.ntable_name} (ts, pk, c2) values({normal_ts_1}, 4, '8') ({normal_ts_1}, 5, '9')", show=SHOW_LOG) + + tdSql.execute(f"insert into {self.ntable_name} values(now, 1, '7', '6')", show=SHOW_LOG) + normal_ts_2 = self.get_latest_ts(self.ntable_name) + tdSql.execute(f"insert into {self.ntable_name} values({normal_ts_2}, 2, '8', '6') ({normal_ts_2}, 3, '9', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {self.ntable_name} (ts, pk, c2) values({normal_ts_2}, 4, '8') ({normal_ts_2}, 5, '9')", show=SHOW_LOG) + + tdSql.execute(f"insert into {self.ntable_name} values(now, 1, '7', '6')", show=SHOW_LOG) + normal_ts_3 = self.get_latest_ts(self.ntable_name) + tdSql.execute(f"insert into {self.ntable_name} values({normal_ts_3}, 2, '8', '6') ({normal_ts_3}, 3, '9', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {self.ntable_name} (ts, pk, c2) values({normal_ts_3}, 4, '8') ({normal_ts_3}, 5, '9')", show=SHOW_LOG) + + tdSql.execute(f"insert into {self.ntable_name} values(now, 1, '7', '6')", show=SHOW_LOG) + normal_ts_4 = self.get_latest_ts(self.ntable_name) + tdSql.execute(f"insert into {self.ntable_name} values({normal_ts_4}, 2, '8', '6') ({normal_ts_4}, 3, '9', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {self.ntable_name} (ts, pk, c2) values({normal_ts_4}, 4, '8') ({normal_ts_4}, 5, '9')", show=SHOW_LOG) + + tdSql.execute(f"insert into {self.ntable_name} values(now, 1, '7', '6')", show=SHOW_LOG) + normal_ts_5 = self.get_latest_ts(self.ntable_name) + tdSql.execute(f"insert into {self.ntable_name} values({normal_ts_5}, 2, '8', '6') ({normal_ts_5}, 3, '9', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {self.ntable_name} (ts, pk, c2) values({normal_ts_5}, 4, '8') ({normal_ts_5}, 5, '9')", show=SHOW_LOG) + + tdSql.execute(f"insert into {self.ntable_name} values(now, 1, '7', '6')", show=SHOW_LOG) + normal_ts_6 = self.get_latest_ts(self.ntable_name) + tdSql.execute(f"insert into {self.ntable_name} values({normal_ts_6}, 2, '8', '6') ({normal_ts_6}, 3, '9', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {self.ntable_name} (ts, pk, c2) values({normal_ts_6}, 4, '8') ({normal_ts_6}, 5, '9')", show=SHOW_LOG) + + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(30) + + self.ts_list['child_ts_1'] = child_ts_1 + self.ts_list['child_ts_2'] = child_ts_2 + self.ts_list['child_ts_3'] = child_ts_3 + self.ts_list['child_ts_4'] = child_ts_4 + self.ts_list['child_ts_5'] = child_ts_5 + self.ts_list['child_ts_6'] = child_ts_6 + self.ts_list['normal_ts_1'] = normal_ts_1 + self.ts_list['normal_ts_2'] = normal_ts_2 + self.ts_list['normal_ts_3'] = normal_ts_3 + self.ts_list['normal_ts_4'] = normal_ts_4 + self.ts_list['normal_ts_5'] = normal_ts_5 + self.ts_list['normal_ts_6'] = normal_ts_6 + + + + def test_delete_data(self): + # delete with ts + tdSql.execute(f"delete from {self.stable_name} where ts={self.ts_list['child_ts_1']} ", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(25) + tdSql.execute(f"delete from {self.ctable_name}_1 where ts={self.ts_list['child_ts_2']} ", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(20) + tdSql.execute(f"delete from {self.ntable_name} where ts={self.ts_list['normal_ts_1']} ", show=SHOW_LOG) + tdSql.query(f'select * from {self.ntable_name}') + tdSql.checkRows(25) + + # delete with ts range + tdSql.execute(f"delete from {self.stable_name} where ts>{self.ts_list['child_ts_2']} and ts<{self.ts_list['child_ts_4']}", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(15) + tdSql.execute(f"delete from {self.ctable_name}_2 where ts>{self.ts_list['child_ts_2']} and ts<{self.ts_list['child_ts_5']}", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(10) + tdSql.execute(f"delete from {self.ntable_name} where ts>{self.ts_list['normal_ts_2']} and ts<{self.ts_list['normal_ts_5']}", show=SHOW_LOG) + tdSql.query(f'select * from {self.ntable_name}') + tdSql.checkRows(15) + + tdSql.execute(f"delete from {self.stable_name}", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + + table_name4 = f'{self.ctable_name}_4' + tdSql.execute(f"insert into {table_name4} using {self.stable_name} tags(3) values(now, 1, '7', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {table_name4} values(now+1s, 2, '8', '6') (now+2s, 3, '9', '6')", show=SHOW_LOG) + + tdSql.execute(f"delete from {table_name4}", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + tdSql.execute(f"delete from {self.ntable_name}", show=SHOW_LOG) + tdSql.query(f'select * from {self.ntable_name}') + tdSql.checkRows(0) + + tdSql.execute(f"delete from {self.stable_name}", show=SHOW_LOG) + tdSql.execute(f"delete from {table_name4}", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + tdSql.execute(f"delete from {self.ntable_name}", show=SHOW_LOG) + tdSql.query(f'select * from {self.ntable_name}') + tdSql.checkRows(0) + + def test_delete_data_illegal(self, dtype: LegalDataType): + if dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY: + pk_condition_value = '\'1\'' + else: + pk_condition_value = '1' + # delete with ts & pk + tdSql.error(f"delete from {self.stable_name} where ts={self.ts_list['child_ts_1']} and pk={pk_condition_value}", show=SHOW_LOG) + tdSql.error(f"delete from {self.ctable_name}_2 where ts={self.ts_list['child_ts_1']} and pk={pk_condition_value}", show=SHOW_LOG) + tdSql.error(f"delete from {self.ntable_name} where ts={self.ts_list['normal_ts_1']} and pk={pk_condition_value}", show=SHOW_LOG) + + # delete with ts range & pk + tdSql.error(f"delete from {self.stable_name} where ts>{self.ts_list['child_ts_1']} and ts<{self.ts_list['child_ts_2']} and pk={pk_condition_value}", show=SHOW_LOG) + tdSql.error(f"delete from {self.ctable_name}_2 where ts>{self.ts_list['child_ts_1']} and ts<{self.ts_list['child_ts_2']} and pk={pk_condition_value}", show=SHOW_LOG) + tdSql.error(f"delete from {self.ntable_name} where ts>{self.ts_list['normal_ts_1']} and ts<{self.ts_list['normal_ts_2']} and pk={pk_condition_value}", show=SHOW_LOG) + + # delete with pk + tdSql.error(f"delete from {self.stable_name} where pk={pk_condition_value}", show=SHOW_LOG) + tdSql.error(f"delete from {self.ctable_name}_2 where pk={pk_condition_value}", show=SHOW_LOG) + tdSql.error(f"delete from {self.ntable_name} where pk={pk_condition_value}", show=SHOW_LOG) + + def _compare_table_data(self, result1, result2, row = 0, col = 0): + for i in range(row): + for j in range(col): + if result1[i][j] != result2[i][j]: + tdSql.checkEqual(False, True) + + def run(self): + tdSql.prepare(replica = self.replicaVar) + self.prepare_db() + + for date_type in LegalDataType.__members__.items(): + self.prepare_data(date_type[1]) + self.test_delete_data_illegal(date_type[1]) + self.test_delete_data() + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/1-insert/composite_primary_key_insert.py b/tests/system-test/1-insert/composite_primary_key_insert.py new file mode 100644 index 0000000000..a867ac2a4a --- /dev/null +++ b/tests/system-test/1-insert/composite_primary_key_insert.py @@ -0,0 +1,776 @@ +from enum import Enum + +from util.log import * +from util.sql import * +from util.cases import * +from util.csv import * +import os +import taos +import json +from taos import SmlProtocol, SmlPrecision +from taos.error import SchemalessError + + +class IllegalData(Enum): + NULL = 'null' + # EMPTY = '\'\'' + NONE = 'none' + # TRUE = 'true' + # FALSE = 'false' + +# class IllegalDataVar(Enum): +# NULL = 'null' +# EMPTY = '\'\'' +# NONE = 'none' +# TRUE = 'true' +# FALSE = 'false' + + +class LegalDataType(Enum): + INT = 'INT' + UINT = 'INT UNSIGNED' + BIGINT = 'BIGINT' + UBIGINT = 'BIGINT UNSIGNED' + VARCHAR = 'VARCHAR(100)' + BINARY = 'BINARY(100)' + + +class TableType(Enum): + SUPERTABLE = 0 + CHILDTABLE = 1 + NORNALTABLE = 2 + +class HasPK(Enum): + NO = 0 + YES = 1 + +SHOW_LOG = True +STAET_TS = '2023-10-01 00:00:00.000' + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + self.database = "db_insert_composite_primary_key" + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + self.testcasePath = os.path.split(__file__)[0] + self.testcasePath = self.testcasePath.replace('\\', '//') + self.testcaseFilename = os.path.split(__file__)[-1] + os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) + # tdSql.execute(f"insert into db4096.ctb00 file '{self.testcasePath}//tableColumn4096csvLength64k.csv'") + + self.tdCsv = TDCsv() + self.tdCsv.file_path = self.testcasePath + self.tdCsv.file_name = 'file1.csv' + + self.stable_name = 's_table' + self.ctable_name = 'c_table' + self.ntable_name = 'n_table' + + + def prepare_db(self): + tdSql.execute(f"drop database if exists {self.database}") + tdSql.execute(f"create database {self.database} CACHEMODEL 'none'") + tdSql.execute(f"use {self.database}") + + def get_latest_ts(self, table_name: str): + tdSql.query(f'select last(ts) from {table_name}') + now_time = tdSql.queryResult[0][0].strftime("%Y-%m-%d %H:%M:%S.%f") + return f"'{now_time}'" + + def test_insert_data(self, dtype: LegalDataType, hasPk: HasPK): + # drop super table and child table + tdSql.execute(f"drop table if exists {self.stable_name}") + tdSql.execute(f"drop table if exists {self.ntable_name}_1") + tdSql.execute(f"drop table if exists {self.ntable_name}_2") + if hasPk: + tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}, c3 {dtype.value}) tags (engine int)", show=SHOW_LOG) + else: + tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value}, c2 {dtype.value}, c3 {dtype.value}) tags (engine int)", show=SHOW_LOG) + + # 1.insert into value through supper table + table_name = f'{self.ctable_name}_1' + tdSql.execute(f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) values('{table_name}', 1, now, 1, '1')", show=SHOW_LOG) + current_ts1 = self.get_latest_ts(self.stable_name) + tdSql.execute(f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) values('{table_name}', 1, {current_ts1}, 2, '2')", show=SHOW_LOG) + tdSql.execute(f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) values('{table_name}', 1, {current_ts1}, 3, '3')", show=SHOW_LOG) + + tdSql.execute(f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) values('{table_name}', 1, now + 1s, 1, '4')", show=SHOW_LOG) + current_ts2 = self.get_latest_ts(self.stable_name) + tdSql.execute(f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) values('{table_name}', 1, {current_ts2}, 2, '5')", show=SHOW_LOG) + tdSql.execute(f"insert into {self.stable_name} (tbname, engine, ts, pk) values('{table_name}', 1, now + 2s, 2)", show=SHOW_LOG) + + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(6) + tdSql.query(f"select * from {self.stable_name} where engine = 1 and ts ={current_ts1}", show=SHOW_LOG) + tdSql.checkRows(3) + tdSql.query(f"select * from {self.stable_name} where engine = 1 and ts ={current_ts2}", show=SHOW_LOG) + tdSql.checkRows(2) + tdSql.query(f"select * from {self.stable_name} where engine = 1 and ts ={current_ts2} and pk = 2", show=SHOW_LOG) + tdSql.checkRows(1) + + # 2.insert into value through child table + table_name = f'{self.ctable_name}_2' + tdSql.execute(f"insert into {table_name} using {self.stable_name} tags(2) values(now, 1, '7', '6')", show=SHOW_LOG) + current_ts3 = self.get_latest_ts(table_name) + tdSql.execute(f"insert into {table_name} values({current_ts3}, 2, '8', '6') ({current_ts3}, 3, '9', '6')", show=SHOW_LOG) + + tdSql.execute(f"insert into {table_name} using {self.stable_name} tags(2) values(now + 2s, 1, '10', '6')", show=SHOW_LOG) + current_ts4 = self.get_latest_ts(table_name) + tdSql.execute(f"insert into {table_name} using {self.stable_name} tags(2) (ts, pk, c2)values({current_ts4}, 2, '11')", show=SHOW_LOG) + tdSql.execute(f"insert into {table_name} using {self.stable_name} tags(2) (ts, pk) values(now + 4s, 2)", show=SHOW_LOG) + + tdSql.query(f'select * from {table_name}') + tdSql.checkRows(6) + tdSql.query(f"select * from {table_name} where engine = 2 and ts ={current_ts3}", show=SHOW_LOG) + tdSql.checkRows(3) + tdSql.query(f"select * from {table_name} where engine = 2 and ts ={current_ts4}", show=SHOW_LOG) + tdSql.checkRows(2) + tdSql.query(f"select * from {table_name} where engine = 2 and ts ={current_ts4} and pk = 2", show=SHOW_LOG) + tdSql.checkRows(1) + + # 3.insert value into child table from csv file + data = [ + ['ts','pk','c2'], + ['\'2024-03-29 16:55:42.572\'','1','1','100'], + ['\'2024-03-29 16:55:42.572\'','2','2','100'], + ['\'2024-03-29 16:55:42.572\'','3','3','100'], + ['\'2024-03-29 16:55:43.586\'','1','4','100'], + ['\'2024-03-29 16:55:43.586\'','2','5','100'], + ['\'2024-03-29 16:55:44.595\'','2','6','100'] + ] + + self.tdCsv.delete() + self.tdCsv.write(data) + + table_name = f'{self.ctable_name}_3' + tdSql.execute(f"create table {table_name} using {self.stable_name} tags(3)", show=SHOW_LOG) + tdSql.execute(f"insert into {table_name} file '{self.tdCsv.file}'", show=SHOW_LOG) + + tdSql.query(f'select * from {table_name}') + tdSql.checkRows(6) + tdSql.query(f"select * from {table_name} where engine=3 and ts='2024-03-29 16:55:42.572'", show=SHOW_LOG) + tdSql.checkRows(3) + tdSql.query(f"select * from {table_name} where engine=3 and ts='2024-03-29 16:55:43.586'", show=SHOW_LOG) + tdSql.checkRows(2) + tdSql.query(f"select * from {table_name} where engine=3 and ts='2024-03-29 16:55:43.586' and pk=2", show=SHOW_LOG) + tdSql.checkRows(1) + + # 4.insert value into child table from csv file, create table automatically + data = [ + ['ts','pk','c2'], + ['\'2024-03-28 16:55:42.572\'','1','1','100'], + ['\'2024-03-28 16:55:42.572\'','2','2','100'], + ['\'2024-03-28 16:55:42.572\'','3','3','100'], + ['\'2024-03-28 16:55:43.586\'','1','4','100'], + ['\'2024-03-28 16:55:43.586\'','2','5','100'], + ['\'2024-03-28 16:55:44.595\'','2','6','100'] + ] + + self.tdCsv.delete() + self.tdCsv.write(data) + + table_name = f'{self.ctable_name}_4' + tdSql.execute(f"insert into {table_name} using {self.stable_name} tags(4) file '{self.tdCsv.file}'", show=SHOW_LOG) + + tdSql.query(f'select * from {table_name}') + tdSql.checkRows(6) + tdSql.query(f"select * from {self.stable_name} where engine=4 and ts='2024-03-28 16:55:42.572'", show=SHOW_LOG) + tdSql.checkRows(3) + tdSql.query(f"select * from {self.stable_name} where engine=4 and ts='2024-03-28 16:55:43.586'", show=SHOW_LOG) + tdSql.checkRows(2) + tdSql.query(f"select * from {self.stable_name} where engine=4 and ts='2024-03-28 16:55:43.586' and pk=2", show=SHOW_LOG) + tdSql.checkRows(1) + + # 5.insert value into normal table from csv file + table_name = f'{self.ntable_name}_1' + if hasPk: + tdSql.execute(f"create table {table_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}, c3 {dtype.value})", show=SHOW_LOG) + else: + tdSql.execute(f"create table {table_name} (ts timestamp, pk {dtype.value}, c2 {dtype.value}, c3 {dtype.value})", show=SHOW_LOG) + + tdSql.execute(f"insert into {table_name} file '{self.tdCsv.file}'", show=SHOW_LOG) + + tdSql.query(f'select * from {table_name}') + tdSql.checkRows(6) + tdSql.query(f"select * from {table_name} where ts='2024-03-28 16:55:42.572'", show=SHOW_LOG) + tdSql.checkRows(3) + tdSql.query(f"select * from {table_name} where ts='2024-03-28 16:55:43.586'", show=SHOW_LOG) + tdSql.checkRows(2) + tdSql.query(f"select * from {table_name} where ts='2024-03-28 16:55:43.586' and pk=2", show=SHOW_LOG) + tdSql.checkRows(1) + + # 6.insert value into normal table + table_name = f'{self.ntable_name}_2' + if hasPk: + tdSql.execute(f"create table {table_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}, c3 {dtype.value})", show=SHOW_LOG) + else: + tdSql.execute(f"create table {table_name} (ts timestamp, pk {dtype.value}, c2 {dtype.value}, c3 {dtype.value})", show=SHOW_LOG) + + tdSql.execute(f"insert into {table_name} values(now, 1, '1', '234')", show=SHOW_LOG) + current_ts1 = self.get_latest_ts(table_name) + tdSql.execute(f"insert into {table_name} values({current_ts1}, 2, '2', '234') ({current_ts1}, 3, '3', '234')", show=SHOW_LOG) + + tdSql.execute(f"insert into {table_name} values(now + 1s, 1, '4', '234')", show=SHOW_LOG) + current_ts2 = self.get_latest_ts(table_name) + tdSql.execute(f"insert into {table_name} (ts, pk, c2) values({current_ts2}, 2, '5')", show=SHOW_LOG) + tdSql.execute(f"insert into {table_name} (ts, pk) values(now + 2s, 2)", show=SHOW_LOG) + + tdSql.query(f'select * from {table_name}') + tdSql.checkRows(6) + tdSql.query(f"select * from {table_name} where ts ={current_ts1}", show=SHOW_LOG) + tdSql.checkRows(3) + tdSql.query(f"select * from {table_name} where ts ={current_ts2}", show=SHOW_LOG) + tdSql.checkRows(2) + tdSql.query(f"select * from {table_name} where ts ={current_ts2} and pk = 2", show=SHOW_LOG) + tdSql.checkRows(1) + + def test_insert_data_illegal(self, dtype: LegalDataType, illegal_data: IllegalData): + # drop tables + tdSql.execute(f"drop table if exists {self.stable_name}") + tdSql.execute(f"drop table if exists {self.ntable_name}_1") + tdSql.execute(f"drop table if exists {self.ntable_name}_2") + + tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}) tags (engine int)", show=SHOW_LOG) + + # 1.insert into value through supper table + table_name = f'{self.ctable_name}_1' + tdSql.error(f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) values('{table_name}', 1, now, {illegal_data.value}, '1')", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + + # 2.insert into value through child table + table_name = f'{self.ctable_name}_2' + tdSql.error(f"insert into {table_name} using {self.stable_name} tags(2) values(now, {illegal_data.value}, '7')", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + + # 4.insert value into child table from csv file + data = [ + ['ts','pk','c2'], + ['\'2024-03-29 16:55:42.572\'','1','1'], + ['\'2024-03-29 16:55:42.572\'',f'{illegal_data.value}','2'], + ['\'2024-03-29 16:55:42.572\'','3','3'], + ['\'2024-03-29 16:55:43.586\'','1','4'], + ['\'2024-03-29 16:55:43.586\'','2','5'], + ['\'2024-03-29 16:55:44.595\'','2','6'] + ] + + self.tdCsv.delete() + self.tdCsv.write(data) + + table_name = f'{self.ctable_name}_3' + tdSql.execute(f"create table {table_name} using {self.stable_name} tags(3)", show=SHOW_LOG) + tdSql.error(f"insert into {table_name} file '{self.tdCsv.file}'", show=SHOW_LOG) + tdSql.query(f'select * from {table_name}') + tdSql.checkRows(0) + + # 5.insert value into child table from csv file, create table automatically + table_name = f'{self.ctable_name}_4' + tdSql.error(f"insert into {table_name} using {self.stable_name} tags(4) file '{self.tdCsv.file}'", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + + # 6.insert value into normal table from csv file + table_name = f'{self.ntable_name}_1' + tdSql.execute(f"create table {table_name} using {self.stable_name} tags(3)", show=SHOW_LOG) + tdSql.error(f"insert into {table_name} file '{self.tdCsv.file}'", show=SHOW_LOG) + tdSql.query(f'select * from {table_name}') + tdSql.checkRows(0) + + # 7.insert value into normal table + table_name = f'{self.ntable_name}_2' + tdSql.execute(f"create table {table_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value})", show=SHOW_LOG) + tdSql.error(f"insert into {table_name} values(now, {illegal_data.value}, '1')", show=SHOW_LOG) + tdSql.query(f'select * from {table_name}') + tdSql.checkRows(0) + + def test_insert_select(self, dtype: LegalDataType): + # # 1.pk table to non-pk table, throw error + tdSql.execute(f"drop table if exists source_{self.stable_name}") + tdSql.execute(f"drop table if exists source_{self.ctable_name}") + tdSql.execute(f"drop table if exists source_{self.ntable_name}") + tdSql.execute(f"drop table if exists dest_{self.ntable_name}") + + tdSql.execute(f"create table source_{self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}) tags (engine int)", show=SHOW_LOG) + tdSql.execute(f"create table source_{self.ctable_name} using source_{self.stable_name} tags(3)", show=SHOW_LOG) + tdSql.execute(f"create table source_{self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value})", show=SHOW_LOG) + tdSql.execute(f"create table dest_{self.ntable_name} (ts timestamp, pk {dtype.value}, c2 {dtype.value})", show=SHOW_LOG) + + tdSql.error(f"insert into dest_{self.ntable_name} select * from source_{self.stable_name})", show=SHOW_LOG) + tdSql.error(f"insert into dest_{self.ntable_name} select * from source_{self.ctable_name})", show=SHOW_LOG) + tdSql.error(f"insert into dest_{self.ntable_name} select * from source_{self.ntable_name})", show=SHOW_LOG) + + # 2.non-pk table to pk table + tdSql.execute(f"drop table if exists source_{self.stable_name}") + tdSql.execute(f"drop table if exists source_{self.ctable_name}") + tdSql.execute(f"drop table if exists source_{self.ntable_name}") + tdSql.execute(f"drop table if exists dest_{self.ntable_name}") + + # create dest super & child table + tdSql.execute(f"create table dest_{self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}) tags (engine int)", show=SHOW_LOG) + tdSql.execute(f"create table dest_{self.ctable_name} using dest_{self.stable_name} tags(3)", show=SHOW_LOG) + # tdSql.execute(f"insert into source_{self.ctable_name} values(now, 1, 1) (now+1s, 2, 2) (now+2s, 3, 3)", show=SHOW_LOG) + + # create normal dest table + tdSql.execute(f"create table dest_{self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value})", show=SHOW_LOG) + + # create source table & insert data + tdSql.execute(f"create table source_{self.ntable_name} (ts timestamp, pk {dtype.value}, c2 {dtype.value})", show=SHOW_LOG) + tdSql.execute(f"insert into source_{self.ntable_name} values(now, 1, 1) (now+1s, 2, 2) (now+2s, 3, 3)", show=SHOW_LOG) + tdSql.query(f'select * from source_{self.ntable_name}') + source_data = tdSql.queryResult + + tdSql.execute(f"insert into dest_{self.ctable_name} select ts, pk, c2 from source_{self.ntable_name}", show=SHOW_LOG) + tdSql.query(f'select * from dest_{self.ctable_name}') + dest_data = tdSql.queryResult + self._compare_table_data(source_data, dest_data, 3, 3) + tdSql.execute(f"delete from dest_{self.ctable_name}", show=SHOW_LOG) + + tdSql.execute(f"insert into dest_{self.ntable_name} select * from source_{self.ntable_name}", show=SHOW_LOG) + tdSql.query(f'select * from dest_{self.ntable_name}') + dest_data = tdSql.queryResult + self._compare_table_data(source_data, dest_data, 3, 3) + tdSql.execute(f"delete from dest_{self.ntable_name}", show=SHOW_LOG) + + # TD-29363 + tdSql.execute(f"create table source_null (ts timestamp, pk {dtype.value}, c2 {dtype.value})", show=SHOW_LOG) + tdSql.execute(f"insert into source_null values(now, null, 1) (now+1s, 2, 2) (now+2s, 3, 3)", show=SHOW_LOG) + tdSql.error(f"insert into dest_{self.ntable_name} values(now, null, 1) (now+1s, 2, 2) (now+2s, 3, 3)", show=SHOW_LOG) + tdSql.error(f"insert into dest_{self.ctable_name} select ts, pk, c2 from source_null", show=SHOW_LOG) + tdSql.error(f"insert into dest_{self.ntable_name} select * from source_null", show=SHOW_LOG) + + # 3.pk table to pk table + tdSql.execute(f"drop table if exists source_{self.stable_name}") + tdSql.execute(f"drop table if exists source_{self.ctable_name}") + tdSql.execute(f"drop table if exists source_{self.ntable_name}") + tdSql.execute(f"drop table if exists dest_{self.stable_name}") + tdSql.execute(f"drop table if exists dest_{self.ntable_name}") + + # create dest super & child table + tdSql.execute(f"create table dest_{self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}, c3 {dtype.value}) tags (engine int)", show=SHOW_LOG) + tdSql.execute(f"create table dest_{self.ctable_name} using dest_{self.stable_name} tags(3)", show=SHOW_LOG) + tdSql.execute(f"insert into dest_{self.ctable_name} values('2024-04-01 17:38:08.764', 1, 1, 100) ('2024-04-01 17:38:08.764', 2, 2, 200) ('2024-04-01 17:38:08.764', 3, 3, 300)", show=SHOW_LOG) + tdSql.execute(f"insert into dest_{self.ctable_name} values('2024-04-02 17:38:08.764', 1, 4, 400) ('2024-04-02 17:38:08.764', 2, 1, 500) ('2024-04-02 17:38:08.764', 3, 1, 600)", show=SHOW_LOG) + + # create normal dest table + tdSql.execute(f"create table dest_{self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}, c3 {dtype.value})", show=SHOW_LOG) + tdSql.execute(f"insert into dest_{self.ntable_name} values('2024-04-01 17:38:08.764', 1, 1, 100) ('2024-04-01 17:38:08.764', 2, 2, 200) ('2024-04-01 17:38:08.764', 3, 3, 300)", show=SHOW_LOG) + tdSql.execute(f"insert into dest_{self.ntable_name} values('2024-04-02 17:38:08.764', 1, 4, 400) ('2024-04-02 17:38:08.764', 2, 1, 500) ('2024-04-02 17:38:08.764', 3, 1, 600)", show=SHOW_LOG) + + # create source table & insert data + tdSql.execute(f"create table source_{self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}, c3 {dtype.value})", show=SHOW_LOG) + tdSql.execute(f"insert into source_{self.ntable_name} values('2024-04-02 17:38:08.764', 1, 4, 800) ('2024-04-02 17:38:08.764', 2, 5, 400) ('2024-04-02 17:38:08.764', 3, 6, 600)", show=SHOW_LOG) + tdSql.execute(f"insert into source_{self.ntable_name} values('2024-04-04 17:38:08.764', 1, 7, 365)", show=SHOW_LOG) + + # insert into select + tdSql.execute(f"insert into dest_{self.ctable_name} (ts, pk, c2) select ts, pk, c2 from source_{self.ntable_name}", show=SHOW_LOG) + tdSql.query(f'select * from dest_{self.ctable_name}') + tdSql.checkRows(7) + if dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY: + tdSql.query(f"select * from dest_{self.ctable_name} where c2='5' and c2='6'", show=SHOW_LOG) + else: + tdSql.query(f'select * from dest_{self.ctable_name} where c2=5 and c2=6', show=SHOW_LOG) + tdSql.checkRows(2) + tdSql.execute(f"delete from dest_{self.ctable_name}", show=SHOW_LOG) + + tdSql.execute(f"insert into dest_{self.ntable_name} (ts, pk, c2) select * from source_{self.ntable_name}", show=SHOW_LOG) + tdSql.query(f'select * from dest_{self.ntable_name}') + tdSql.checkRows(7) + if dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY: + tdSql.query(f"select * from dest_{self.ctable_name} where c2='5' and c2='6'") + else: + tdSql.query(f'select * from dest_{self.ctable_name} where c2=5 and c2=6') + tdSql.checkRows(2) + tdSql.execute(f"delete from dest_{self.ntable_name}", show=SHOW_LOG) + + def test_schemaless_error(self): + # 5.1.insert into values via influxDB + lines = ["meters,location=California.LosAngeles,groupid=2 current=11i32,voltage=221,phase=0.28 1648432611249000", + "meters,location=California.LosAngeles,groupid=2 current=13i32,voltage=223,phase=0.29 1648432611249000", + "meters,location=California.LosAngeles,groupid=3 current=10i32,voltage=223,phase=0.29 1648432611249300", + "meters,location=California.LosAngeles,groupid=3 current=11i32,voltage=221,phase=0.35 1648432611249300", + ] + try: + conn = taos.connect() + conn.execute("drop database if exists influxDB") + conn.execute("CREATE DATABASE influxDB precision 'us'") + conn.execute("USE influxDB") + conn.execute("CREATE STABLE `meters` (`_ts` TIMESTAMP, `current` int primary key, `voltage` DOUBLE, `phase` DOUBLE) TAGS (`location` NCHAR(32), `groupid` NCHAR(2))") + conn.execute("create table t_be97833a0e1f523fcdaeb6291d6fdf27 using meters tags('California.LosAngeles', 2)") + conn.execute("create table t_10b65f71ff8970369c8c18de0d6be028 using meters tags('California.LosAngeles', 3)") + conn.schemaless_insert(lines, SmlProtocol.LINE_PROTOCOL, SmlPrecision.MICRO_SECONDS) + except SchemalessError: + tdSql.checkEqual(False, True) + + # 5.2.insert into values via OpenTSDB + lines = ["meters.current 1648432611249 10i32 location=California.SanFrancisco groupid=2", + "meters.current 1648432611250 12i32 location=California.SanFrancisco groupid=2", + "meters.current 1648432611249 10i32 location=California.LosAngeles groupid=3", + "meters.current 1648432611250 11i32 location=California.LosAngeles groupid=3", + "meters.voltage 1648432611249 219i32 location=California.SanFrancisco groupid=2", + "meters.voltage 1648432611250 218i32 location=California.SanFrancisco groupid=2", + "meters.voltage 1648432611249 221i32 location=California.LosAngeles groupid=3", + "meters.voltage 1648432611250 217i32 location=California.LosAngeles groupid=3", + ] + try: + conn = taos.connect() + conn.execute("drop database if exists OpenTSDB") + conn.execute("CREATE DATABASE OpenTSDB precision 'us'") + conn.execute("USE OpenTSDB") + conn.execute("CREATE STABLE `meters_current` (`_ts` TIMESTAMP, `_value` INT primary key) TAGS (`location` NCHAR(32), `groupid` NCHAR(2))") + conn.execute("CREATE TABLE `t_c66ea0b2497be26ca9d328b59c39dd61` USING `meters_current` (`location`, `groupid`) TAGS ('California.LosAngeles', '3')") + conn.execute("CREATE TABLE `t_e71c6cf63cfcabb0e261886adea02274` USING `meters_current` (`location`, `groupid`) TAGS ('California.SanFrancisco', '2')") + conn.schemaless_insert(lines, SmlProtocol.TELNET_PROTOCOL, SmlPrecision.NOT_CONFIGURED) + except SchemalessError: + tdSql.checkEqual(False, True) + + # 5.3.insert into values via OpenTSDB Json + lines = [{"metric": "meters.current", "timestamp": 1648432611249, "value": "a32", "tags": {"location": "California.SanFrancisco", "groupid": 2}}] + try: + conn = taos.connect() + conn.execute("drop database if exists OpenTSDBJson") + conn.execute("CREATE DATABASE OpenTSDBJson") + conn.execute("USE OpenTSDBJson") + conn.execute("CREATE STABLE `meters_current` (`_ts` TIMESTAMP, `_value` varchar(10) primary key) TAGS (`location` VARCHAR(32), `groupid` DOUBLE)") + conn.execute("CREATE TABLE `t_71d176bfc4c952b64d30d719004807a0` USING `meters_current` (`location`, `groupid`) TAGS ('California.SanFrancisco', 2.000000e+00)") + # global lines + lines = json.dumps(lines) + # note: the first parameter must be a list with only one element. + conn.schemaless_insert([lines], SmlProtocol.JSON_PROTOCOL, SmlPrecision.NOT_CONFIGURED) + except SchemalessError as errMsg: + tdSql.checkEqual(False, True) + + def test_insert_values_special(self, dtype: LegalDataType): + # 4.insert into values without ts column + # drop tables + tdSql.execute(f"drop table if exists {self.stable_name}") + tdSql.execute(f"drop table if exists {self.ntable_name}") + + tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 binary(10)) tags (engine int)", show=SHOW_LOG) + + # # 4.1.insert into value through supper table + tdSql.error(f"insert into {self.stable_name} (tbname, engine, pk, c2) values('{self.ctable_name}', 1, '1', '1')", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + + # # 4.2.insert into value through child table + tdSql.error(f"insert into {self.ctable_name} using {self.stable_name} tags(2) (pk, c2) values('1', '7')", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + + # # 4.3.insert value into normal table + tdSql.execute(f"create table {self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 varchar(20))", show=SHOW_LOG) + tdSql.error(f"insert into {self.ntable_name} (pk, c2) values('1', '1')", show=SHOW_LOG) + tdSql.query(f'select * from {self.ntable_name}') + tdSql.checkRows(0) + + # 5.insert into values without pk column + # drop tables + tdSql.execute(f"drop table if exists {self.stable_name}") + tdSql.execute(f"drop table if exists {self.ntable_name}") + + tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 binary(10)) tags (engine int)", show=SHOW_LOG) + + # # 5.1.insert into value through supper table + tdSql.error(f"insert into {self.stable_name} (tbname, engine, ts, c2) values('{self.ctable_name}', 1, now, '1')", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + + # # 5.2.insert into value through child table + tdSql.error(f"insert into {self.ctable_name} using {self.stable_name} tags(2) (ts, c2) values(now, '7')", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + + # # 5.3.insert value into normal table + tdSql.execute(f"create table {self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 varchar(20))", show=SHOW_LOG) + tdSql.error(f"insert into {self.ntable_name} (ts, c2) values(now, '1')", show=SHOW_LOG) + tdSql.query(f'select * from {self.ntable_name}') + tdSql.checkRows(0) + + def test_insert_into_mutiple_tables(self, dtype: LegalDataType): + # drop super table and child table + tdSql.execute(f"drop table if exists {self.stable_name}") + tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}) tags (engine int)", show=SHOW_LOG) + + # 1.insert into value by supper table syntax + error_sql = f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) " \ + f"values('{self.ctable_name}_1', 1, '2021-07-13 14:06:34.630', 1, 100) " \ + f"('{self.ctable_name}_1', 1, '2021-07-13 14:06:34.630', 2, 200) " \ + f"('{self.ctable_name}_2', 2, '2021-07-14 14:06:34.630', 1, 300) " \ + f"('{self.ctable_name}_2', 2, '2021-07-14 14:06:34.630', 2, 400) " \ + f"('{self.ctable_name}_3', 3, '2021-07-15 14:06:34.630', null, 500)" + + tdSql.error(error_sql, show=SHOW_LOG) + + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + + sql = f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) " \ + f"values('{self.ctable_name}_1', 1, '2021-07-13 14:06:34.630', 1, 100) " \ + f"('{self.ctable_name}_1', 1, '2021-07-13 14:06:34.630', 2, 200) " \ + f"('{self.ctable_name}_2', 2, '2021-07-14 14:06:34.630', 1, 300) " \ + f"('{self.ctable_name}_2', 2, '2021-07-14 14:06:34.630', 2, 400) " \ + f"('{self.ctable_name}_3', 3, '2021-07-15 14:06:34.630', 1, 500)" + + tdSql.execute(sql, show=SHOW_LOG) + + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(5) + tdSql.query(f"select * from {self.stable_name} where ts='2021-07-13 14:06:34.630'", show=SHOW_LOG) + tdSql.checkRows(2) + tdSql.query(f"select * from {self.stable_name} where ts='2021-07-13 14:06:34.630' and pk=2", show=SHOW_LOG) + tdSql.checkRows(1) + + # 2.insert into value and create table automatically + tdSql.execute(f"drop table if exists {self.stable_name}") + tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}) tags (engine int)", show=SHOW_LOG) + + error_sql = f"insert into {self.ctable_name}_1 using {self.stable_name} tags(1) values('2021-07-13 14:06:34.630', 1, 100) ('2021-07-13 14:06:34.630', 2, 200) " \ + f"{self.ctable_name}_2 using {self.stable_name} (engine) tags(2) values('2021-07-14 14:06:34.630', 1, 300) ('2021-07-14 14:06:34.630', 2, 400) " \ + f"{self.ctable_name}_3 using {self.stable_name} (engine) tags(3) values('2021-07-15 14:06:34.630', null, 500)" + + tdSql.error(error_sql, show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + + sql = f"insert into {self.ctable_name}_1 using {self.stable_name} tags(1) values('2021-07-13 14:06:34.630', 1, 100) ('2021-07-13 14:06:34.630', 2, 200) " \ + f"{self.ctable_name}_2 using {self.stable_name} (engine) tags(2) values('2021-07-14 14:06:34.630', 1, 300) ('2021-07-14 14:06:34.630', 2, 400) " \ + f"{self.ctable_name}_3 using {self.stable_name} (engine) tags(3) values('2021-07-15 14:06:34.630', 1, 500)" + + tdSql.execute(sql, show=SHOW_LOG) + + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(5) + tdSql.query(f"select * from {self.stable_name} where ts='2021-07-13 14:06:34.630'", show=SHOW_LOG) + tdSql.checkRows(2) + tdSql.query(f"select * from {self.stable_name} where ts='2021-07-13 14:06:34.630' and pk=2", show=SHOW_LOG) + tdSql.checkRows(1) + + # 3.insert value into child table from csv file, create table automatically + tdSql.execute(f"drop table if exists {self.stable_name}") + tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}) tags (engine int)", show=SHOW_LOG) + + error_data = [ + ['ts','pk','c2'], + ['\'2024-03-29 16:55:42.572\'','1','1'], + ['\'2024-03-29 16:55:42.572\'','2','2'], + ['\'2024-03-29 16:55:42.572\'','null','3'], + ['\'2024-03-29 16:55:43.586\'','1','4'], + ['\'2024-03-29 16:55:43.586\'','2','5'], + ['\'2024-03-29 16:55:44.595\'','2','6'] + ] + + self.tdCsv.delete() + self.tdCsv.write(error_data) + + sql = f"insert into {self.ctable_name}_1 using {self.stable_name} tags(1) FILE '{self.tdCsv.file}' " \ + f"{self.ctable_name}_2 using {self.stable_name} (engine) tags(2) FILE '{self.tdCsv.file}' " \ + f"{self.ctable_name}_3 using {self.stable_name} (engine) tags(3) FILE '{self.tdCsv.file}'" + + tdSql.error(sql, show=SHOW_LOG) + + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + + data = [ + ['ts','pk','c2'], + ['\'2024-03-29 16:55:42.572\'','1','1'], + ['\'2024-03-29 16:55:42.572\'','2','2'], + ['\'2024-03-29 16:55:42.572\'','2','3'], + ['\'2024-03-29 16:55:43.586\'','1','4'], + ['\'2024-03-29 16:55:43.586\'','2','5'], + ['\'2024-03-29 16:55:44.595\'','2','6'] + ] + + self.tdCsv.delete() + self.tdCsv.write(data) + + sql = f"insert into {self.ctable_name}_1 using {self.stable_name} tags(1) FILE '{self.tdCsv.file}'" \ + f"{self.ctable_name}_2 using {self.stable_name} (engine) tags(2) FILE '{self.tdCsv.file}'" \ + f"{self.ctable_name}_3 using {self.stable_name} (engine) tags(3) FILE '{self.tdCsv.file}'" + + tdSql.execute(sql, show=SHOW_LOG) + + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(15) + tdSql.query(f"select * from {self.stable_name} where ts='2024-03-29 16:55:42.572'", show=SHOW_LOG) + tdSql.checkRows(6) + tdSql.query(f"select * from {self.stable_name} where ts='2024-03-29 16:55:42.572' and pk=2", show=SHOW_LOG) + tdSql.checkRows(3) + + # 6.insert value into normal table + tdSql.execute(f"drop table if exists {self.ntable_name}_1") + tdSql.execute(f"drop table if exists {self.ntable_name}_2") + tdSql.execute(f"create table {self.ntable_name}_1 (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value})", show=SHOW_LOG) + tdSql.execute(f"create table {self.ntable_name}_2 (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value})", show=SHOW_LOG) + + sql = f"insert into {self.ntable_name}_1 values('2021-07-13 14:06:34.630', 1, 100) ('2021-07-13 14:06:34.630', 2, 200) " \ + f"{self.ntable_name}_2 values('2021-07-14 14:06:34.630', 1, 300) ('2021-07-14 14:06:34.630', 2, 400) " + + tdSql.execute(sql, show=SHOW_LOG) + + tdSql.query(f'select * from {self.ntable_name}_1') + tdSql.checkRows(2) + tdSql.query(f"select * from {self.ntable_name}_2 where ts='2021-07-14 14:06:34.630' and pk=2", show=SHOW_LOG) + tdSql.checkRows(1) + + # 7. insert value into child and normal table + tdSql.execute(f"drop table if exists {self.stable_name}") + tdSql.execute(f"drop table if exists {self.ctable_name}") + tdSql.execute(f"drop table if exists {self.ntable_name}") + + tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}) tags (engine int)", show=SHOW_LOG) + tdSql.execute(f"create table {self.ctable_name} using {self.stable_name} tags (1)", show=SHOW_LOG) + tdSql.execute(f"create table {self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value})", show=SHOW_LOG) + + error_sql = f"insert into {self.ctable_name} values('2021-07-13 14:06:34.630', null, 100) ('2021-07-13 14:06:34.630', 2, 200) " \ + f"{self.ntable_name} values('2021-07-14 14:06:34.630', 1, 300) ('2021-07-14 14:06:34.630', 2, 400) " + + sql = f"insert into {self.ctable_name} values('2021-07-13 14:06:34.630', 1, 100) ('2021-07-13 14:06:34.630', 2, 200) " \ + f"{self.ntable_name} values('2021-07-14 14:06:34.630', 1, 300) ('2021-07-14 14:06:34.630', 2, 400) " + + tdSql.error(error_sql, show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + tdSql.query(f'select * from {self.ntable_name}') + tdSql.checkRows(0) + + tdSql.execute(sql, show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(2) + tdSql.query(f'select * from {self.ntable_name}') + tdSql.checkRows(2) + + def test_stmt(self, dtype: LegalDataType): + tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}, c3 float) tags (engine int)", show=SHOW_LOG) + + sql = f"INSERT INTO ? USING {self.stable_name} TAGS(?) VALUES (?,?,?,?)" + + conn = taos.connect() + conn.select_db(self.database) + stmt = conn.statement(sql) + + tbname = f"d1001" + + tags = taos.new_bind_params(1) + tags[0].int([2]) + + stmt.set_tbname_tags(tbname, tags) + + params = taos.new_bind_params(4) + params[0].timestamp((1626861392589, 1626861392589, 1626861392592)) + if dtype == LegalDataType.INT : + params[1].int((10, 12, 12)) + params[2].int([194, 200, 201]) + elif dtype == LegalDataType.UINT: + params[1].int_unsigned((10, 12, 12)) + params[2].int_unsigned([194, 200, 201]) + elif dtype == LegalDataType.BIGINT: + params[1].bigint((10, 12, 12)) + params[2].bigint([194, 200, 201]) + elif dtype == LegalDataType.UBIGINT: + params[1].bigint_unsigned((10, 12, 12)) + params[2].bigint_unsigned([194, 200, 201]) + elif dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BIGINT: + params[1].binary(("s10", "s12", "s12")) + params[2].binary(["s194", "s200", "s201"]) + params[3].float([0.31, 0.33, 0.31]) + + stmt.bind_param_batch(params) + + params = taos.new_bind_params(4) + params[0].timestamp((1626861392589)) + if dtype == LegalDataType.INT : + params[1].int((11)) + params[2].int([100]) + elif dtype == LegalDataType.UINT: + params[1].int_unsigned((10)) + params[2].int_unsigned([100]) + elif dtype == LegalDataType.BIGINT: + params[1].bigint((10)) + params[2].bigint([100]) + elif dtype == LegalDataType.UBIGINT: + params[1].bigint_unsigned((10)) + params[2].bigint_unsigned([100]) + elif dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BIGINT: + params[1].binary(("s10")) + params[2].binary(["s100"]) + params[3].float([0.31]) + + stmt.bind_param(params) + + stmt.execute() + + stmt.close() + + def test_implicit_conversion(self, dtype: LegalDataType): + + tdSql.execute(f"drop table if exists dest_table", show=SHOW_LOG) + tdSql.execute(f"create table dest_table (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value})", show=SHOW_LOG) + for type in LegalDataType: + if type == dtype: + continue + tdSql.execute(f"drop table if exists source_table", show=SHOW_LOG) + tdSql.execute(f"create table source_table (ts timestamp, pk {dtype.value} primary key, c2 int)", show=SHOW_LOG) + tdSql.execute(f"insert into source_table values(now, 100, 1000)", show=SHOW_LOG) + tdSql.execute(f"insert into dest_table select * from source_table", show=SHOW_LOG) + + tdSql.query(f'select * from dest_table where c2=1000') + tdSql.checkRows(1) + tdSql.execute(f"delete from dest_table", show=SHOW_LOG) + + def _compare_table_data(self, result1, result2, row = 0, col = 0): + for i in range(row): + for j in range(col): + if result1[i][j] != result2[i][j]: + tdSql.checkEqual(False, True) + + def run(self): + tdSql.prepare(replica = self.replicaVar) + self.prepare_db() + + # for date_type in LegalDataType.__members__.items(): + # # 1.insert into value with pk - pass + # self.test_insert_data(date_type[1], HasPK.YES) + + # # 2.insert into value without pk - pass + # self.test_insert_data(date_type[1], HasPK.NO) + + # # 3.insert into illegal data - pass + # for illegal_data in IllegalData.__members__.items(): + # self.test_insert_data_illegal(date_type[1], illegal_data[1]) + + # # 4. insert into select bug!!! + # self.test_insert_select(date_type[1]) + + # # 5. insert into values special cases - pass + # self.test_insert_values_special(date_type[1]) + + # 6. insert into value to mutiple tables - pass + # self.test_implicit_conversion(date_type[1]) + + # # 7. insert into value to mutiple tables - pass + # self.test_insert_into_mutiple_tables(date_type[1]) + + # 8. stmt wait for test!!!! + # self.test_stmt(date_type[1]) + + # 9. insert data by schemaless model is not allowed - bug!!! + self.test_schemaless_error() + + + + + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From ea81f194c0b721d6747e337c77d7324e8895c1af Mon Sep 17 00:00:00 2001 From: Chris Zhai Date: Wed, 10 Apr 2024 16:21:30 +0800 Subject: [PATCH 03/21] update composite_primary_key_insert.py --- .../1-insert/composite_primary_key_insert.py | 65 ++++++++++--------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/tests/system-test/1-insert/composite_primary_key_insert.py b/tests/system-test/1-insert/composite_primary_key_insert.py index a867ac2a4a..d818c5ac7e 100644 --- a/tests/system-test/1-insert/composite_primary_key_insert.py +++ b/tests/system-test/1-insert/composite_primary_key_insert.py @@ -119,7 +119,7 @@ class TDTestCase: tdSql.execute(f"insert into {table_name} using {self.stable_name} tags(2) values(now + 2s, 1, '10', '6')", show=SHOW_LOG) current_ts4 = self.get_latest_ts(table_name) - tdSql.execute(f"insert into {table_name} using {self.stable_name} tags(2) (ts, pk, c2)values({current_ts4}, 2, '11')", show=SHOW_LOG) + tdSql.execute(f"insert into {table_name} using {self.stable_name} tags(2) (ts, pk, c2) values({current_ts4}, 2, '11')", show=SHOW_LOG) tdSql.execute(f"insert into {table_name} using {self.stable_name} tags(2) (ts, pk) values(now + 4s, 2)", show=SHOW_LOG) tdSql.query(f'select * from {table_name}') @@ -292,6 +292,7 @@ class TDTestCase: tdSql.execute(f"drop table if exists source_{self.stable_name}") tdSql.execute(f"drop table if exists source_{self.ctable_name}") tdSql.execute(f"drop table if exists source_{self.ntable_name}") + tdSql.execute(f"drop table if exists dest_{self.stable_name}") tdSql.execute(f"drop table if exists dest_{self.ntable_name}") tdSql.execute(f"create table source_{self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}) tags (engine int)", show=SHOW_LOG) @@ -307,6 +308,7 @@ class TDTestCase: tdSql.execute(f"drop table if exists source_{self.stable_name}") tdSql.execute(f"drop table if exists source_{self.ctable_name}") tdSql.execute(f"drop table if exists source_{self.ntable_name}") + tdSql.execute(f"drop table if exists dest_{self.stable_name}") tdSql.execute(f"drop table if exists dest_{self.ntable_name}") # create dest super & child table @@ -336,6 +338,8 @@ class TDTestCase: tdSql.execute(f"delete from dest_{self.ntable_name}", show=SHOW_LOG) # TD-29363 + tdSql.execute(f"drop table if exists source_null") + tdSql.execute(f"create table source_null (ts timestamp, pk {dtype.value}, c2 {dtype.value})", show=SHOW_LOG) tdSql.execute(f"insert into source_null values(now, null, 1) (now+1s, 2, 2) (now+2s, 3, 3)", show=SHOW_LOG) tdSql.error(f"insert into dest_{self.ntable_name} values(now, null, 1) (now+1s, 2, 2) (now+2s, 3, 3)", show=SHOW_LOG) @@ -370,19 +374,19 @@ class TDTestCase: tdSql.query(f'select * from dest_{self.ctable_name}') tdSql.checkRows(7) if dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY: - tdSql.query(f"select * from dest_{self.ctable_name} where c2='5' and c2='6'", show=SHOW_LOG) + tdSql.query(f"select * from dest_{self.ctable_name} where c2='5' or c2='6'", show=SHOW_LOG) else: - tdSql.query(f'select * from dest_{self.ctable_name} where c2=5 and c2=6', show=SHOW_LOG) + tdSql.query(f'select * from dest_{self.ctable_name} where c2=5 or c2=6', show=SHOW_LOG) tdSql.checkRows(2) tdSql.execute(f"delete from dest_{self.ctable_name}", show=SHOW_LOG) - tdSql.execute(f"insert into dest_{self.ntable_name} (ts, pk, c2) select * from source_{self.ntable_name}", show=SHOW_LOG) + tdSql.execute(f"insert into dest_{self.ntable_name} (ts, pk, c2) select ts, pk, c2 from source_{self.ntable_name}", show=SHOW_LOG) tdSql.query(f'select * from dest_{self.ntable_name}') tdSql.checkRows(7) if dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY: - tdSql.query(f"select * from dest_{self.ctable_name} where c2='5' and c2='6'") + tdSql.query(f"select * from dest_{self.ntable_name} where c2='5' or c2='6'") else: - tdSql.query(f'select * from dest_{self.ctable_name} where c2=5 and c2=6') + tdSql.query(f'select * from dest_{self.ntable_name} where c2=5 or c2=6') tdSql.checkRows(2) tdSql.execute(f"delete from dest_{self.ntable_name}", show=SHOW_LOG) @@ -402,8 +406,9 @@ class TDTestCase: conn.execute("create table t_be97833a0e1f523fcdaeb6291d6fdf27 using meters tags('California.LosAngeles', 2)") conn.execute("create table t_10b65f71ff8970369c8c18de0d6be028 using meters tags('California.LosAngeles', 3)") conn.schemaless_insert(lines, SmlProtocol.LINE_PROTOCOL, SmlPrecision.MICRO_SECONDS) - except SchemalessError: tdSql.checkEqual(False, True) + except SchemalessError as errMsg: + tdSql.checkEqual(errMsg.msg == 'Can not insert data into table with primary key', True) # 5.2.insert into values via OpenTSDB lines = ["meters.current 1648432611249 10i32 location=California.SanFrancisco groupid=2", @@ -424,8 +429,9 @@ class TDTestCase: conn.execute("CREATE TABLE `t_c66ea0b2497be26ca9d328b59c39dd61` USING `meters_current` (`location`, `groupid`) TAGS ('California.LosAngeles', '3')") conn.execute("CREATE TABLE `t_e71c6cf63cfcabb0e261886adea02274` USING `meters_current` (`location`, `groupid`) TAGS ('California.SanFrancisco', '2')") conn.schemaless_insert(lines, SmlProtocol.TELNET_PROTOCOL, SmlPrecision.NOT_CONFIGURED) - except SchemalessError: tdSql.checkEqual(False, True) + except SchemalessError as errMsg: + tdSql.checkEqual(errMsg.msg == 'Can not insert data into table with primary key', True) # 5.3.insert into values via OpenTSDB Json lines = [{"metric": "meters.current", "timestamp": 1648432611249, "value": "a32", "tags": {"location": "California.SanFrancisco", "groupid": 2}}] @@ -440,8 +446,9 @@ class TDTestCase: lines = json.dumps(lines) # note: the first parameter must be a list with only one element. conn.schemaless_insert([lines], SmlProtocol.JSON_PROTOCOL, SmlPrecision.NOT_CONFIGURED) - except SchemalessError as errMsg: tdSql.checkEqual(False, True) + except SchemalessError as errMsg: + tdSql.checkEqual(errMsg.msg == 'Can not insert data into table with primary key', True) def test_insert_values_special(self, dtype: LegalDataType): # 4.insert into values without ts column @@ -732,40 +739,34 @@ class TDTestCase: tdSql.prepare(replica = self.replicaVar) self.prepare_db() - # for date_type in LegalDataType.__members__.items(): - # # 1.insert into value with pk - pass - # self.test_insert_data(date_type[1], HasPK.YES) + for date_type in LegalDataType.__members__.items(): + # 1.insert into value with pk - pass + self.test_insert_data(date_type[1], HasPK.YES) - # # 2.insert into value without pk - pass - # self.test_insert_data(date_type[1], HasPK.NO) + # 2.insert into value without pk - pass + self.test_insert_data(date_type[1], HasPK.NO) - # # 3.insert into illegal data - pass - # for illegal_data in IllegalData.__members__.items(): - # self.test_insert_data_illegal(date_type[1], illegal_data[1]) + # 3.insert into illegal data - pass + for illegal_data in IllegalData.__members__.items(): + self.test_insert_data_illegal(date_type[1], illegal_data[1]) - # # 4. insert into select bug!!! - # self.test_insert_select(date_type[1]) + # 4. insert into select bug!!! + self.test_insert_select(date_type[1]) - # # 5. insert into values special cases - pass - # self.test_insert_values_special(date_type[1]) + # 5. insert into values special cases - pass + self.test_insert_values_special(date_type[1]) # 6. insert into value to mutiple tables - pass - # self.test_implicit_conversion(date_type[1]) + self.test_implicit_conversion(date_type[1]) - # # 7. insert into value to mutiple tables - pass - # self.test_insert_into_mutiple_tables(date_type[1]) + # 7. insert into value to mutiple tables - pass + self.test_insert_into_mutiple_tables(date_type[1]) # 8. stmt wait for test!!!! - # self.test_stmt(date_type[1]) + self.test_stmt(date_type[1]) - # 9. insert data by schemaless model is not allowed - bug!!! + # 9. insert data by schemaless model is not allowed - pass self.test_schemaless_error() - - - - - - def stop(self): tdSql.close() From e0b60125b564dd63183d1590799f0213c4d225dd Mon Sep 17 00:00:00 2001 From: Chris Zhai Date: Wed, 10 Apr 2024 17:05:16 +0800 Subject: [PATCH 04/21] chore: add test case for composite primary key --- .../1-insert/composite_primary_key_insert.py | 104 +++++++++--------- 1 file changed, 53 insertions(+), 51 deletions(-) diff --git a/tests/system-test/1-insert/composite_primary_key_insert.py b/tests/system-test/1-insert/composite_primary_key_insert.py index d818c5ac7e..a0cecd2b17 100644 --- a/tests/system-test/1-insert/composite_primary_key_insert.py +++ b/tests/system-test/1-insert/composite_primary_key_insert.py @@ -667,44 +667,46 @@ class TDTestCase: stmt.set_tbname_tags(tbname, tags) - params = taos.new_bind_params(4) - params[0].timestamp((1626861392589, 1626861392589, 1626861392592)) - if dtype == LegalDataType.INT : - params[1].int((10, 12, 12)) - params[2].int([194, 200, 201]) - elif dtype == LegalDataType.UINT: - params[1].int_unsigned((10, 12, 12)) - params[2].int_unsigned([194, 200, 201]) - elif dtype == LegalDataType.BIGINT: - params[1].bigint((10, 12, 12)) - params[2].bigint([194, 200, 201]) - elif dtype == LegalDataType.UBIGINT: - params[1].bigint_unsigned((10, 12, 12)) - params[2].bigint_unsigned([194, 200, 201]) - elif dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BIGINT: - params[1].binary(("s10", "s12", "s12")) - params[2].binary(["s194", "s200", "s201"]) - params[3].float([0.31, 0.33, 0.31]) + # params = taos.new_bind_params(4) + # params[0].timestamp((1626861392589, 1626861392589, 1626861392592)) + # if dtype == LegalDataType.INT : + # params[1].int((10, 12, 12)) + # params[2].int([194, 200, 201]) + # elif dtype == LegalDataType.UINT: + # params[1].int_unsigned((10, 12, 12)) + # params[2].int_unsigned([194, 200, 201]) + # elif dtype == LegalDataType.BIGINT: + # params[1].bigint((10, 12, 12)) + # params[2].bigint([194, 200, 201]) + # elif dtype == LegalDataType.UBIGINT: + # params[1].bigint_unsigned((10, 12, 12)) + # params[2].bigint_unsigned([194, 200, 201]) + # elif dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BIGINT: + # params[1].binary(("s10", "s12", "s12")) + # params[2].binary(["s194", "s200", "s201"]) + # params[3].float([0.31, 0.33, 0.31]) - stmt.bind_param_batch(params) + # stmt.bind_param_batch(params) params = taos.new_bind_params(4) params[0].timestamp((1626861392589)) - if dtype == LegalDataType.INT : - params[1].int((11)) - params[2].int([100]) - elif dtype == LegalDataType.UINT: - params[1].int_unsigned((10)) - params[2].int_unsigned([100]) - elif dtype == LegalDataType.BIGINT: - params[1].bigint((10)) - params[2].bigint([100]) - elif dtype == LegalDataType.UBIGINT: - params[1].bigint_unsigned((10)) - params[2].bigint_unsigned([100]) - elif dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BIGINT: - params[1].binary(("s10")) - params[2].binary(["s100"]) + params[1].int((11)) + params[2].int([100]) + # if dtype == LegalDataType.INT : + # params[1].int((11)) + # params[2].int([100]) + # elif dtype == LegalDataType.UINT: + # params[1].int_unsigned((10)) + # params[2].int_unsigned([100]) + # elif dtype == LegalDataType.BIGINT: + # params[1].bigint((10)) + # params[2].bigint([100]) + # elif dtype == LegalDataType.UBIGINT: + # params[1].bigint_unsigned((10)) + # params[2].bigint_unsigned([100]) + # elif dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BIGINT: + # params[1].binary(("s10")) + # params[2].binary(["s100"]) params[3].float([0.31]) stmt.bind_param(params) @@ -740,33 +742,33 @@ class TDTestCase: self.prepare_db() for date_type in LegalDataType.__members__.items(): - # 1.insert into value with pk - pass - self.test_insert_data(date_type[1], HasPK.YES) + # # 1.insert into value with pk - pass + # self.test_insert_data(date_type[1], HasPK.YES) - # 2.insert into value without pk - pass - self.test_insert_data(date_type[1], HasPK.NO) + # # 2.insert into value without pk - pass + # self.test_insert_data(date_type[1], HasPK.NO) - # 3.insert into illegal data - pass - for illegal_data in IllegalData.__members__.items(): - self.test_insert_data_illegal(date_type[1], illegal_data[1]) + # # 3.insert into illegal data - pass + # for illegal_data in IllegalData.__members__.items(): + # self.test_insert_data_illegal(date_type[1], illegal_data[1]) - # 4. insert into select bug!!! - self.test_insert_select(date_type[1]) + # # 4. insert into select bug!!! + # self.test_insert_select(date_type[1]) - # 5. insert into values special cases - pass - self.test_insert_values_special(date_type[1]) + # # 5. insert into values special cases - pass + # self.test_insert_values_special(date_type[1]) - # 6. insert into value to mutiple tables - pass - self.test_implicit_conversion(date_type[1]) + # # 6. insert into value to mutiple tables - pass + # self.test_implicit_conversion(date_type[1]) - # 7. insert into value to mutiple tables - pass - self.test_insert_into_mutiple_tables(date_type[1]) + # # 7. insert into value to mutiple tables - pass + # self.test_insert_into_mutiple_tables(date_type[1]) # 8. stmt wait for test!!!! self.test_stmt(date_type[1]) - # 9. insert data by schemaless model is not allowed - pass - self.test_schemaless_error() + # # 9. insert data by schemaless model is not allowed - pass + # self.test_schemaless_error() def stop(self): tdSql.close() From 42bf46139af4a0cd3cc1f858129a7d6f37b2bef9 Mon Sep 17 00:00:00 2001 From: Chris Zhai Date: Thu, 11 Apr 2024 11:17:00 +0800 Subject: [PATCH 05/21] update stmt test cases --- .../1-insert/composite_primary_key_insert.py | 171 ++++++++++++++---- 1 file changed, 134 insertions(+), 37 deletions(-) diff --git a/tests/system-test/1-insert/composite_primary_key_insert.py b/tests/system-test/1-insert/composite_primary_key_insert.py index a0cecd2b17..b9a92194e7 100644 --- a/tests/system-test/1-insert/composite_primary_key_insert.py +++ b/tests/system-test/1-insert/composite_primary_key_insert.py @@ -652,6 +652,7 @@ class TDTestCase: tdSql.checkRows(2) def test_stmt(self, dtype: LegalDataType): + tdSql.execute(f"drop table if exists {self.stable_name}", show=SHOW_LOG) tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}, c3 float) tags (engine int)", show=SHOW_LOG) sql = f"INSERT INTO ? USING {self.stable_name} TAGS(?) VALUES (?,?,?,?)" @@ -667,52 +668,146 @@ class TDTestCase: stmt.set_tbname_tags(tbname, tags) - # params = taos.new_bind_params(4) - # params[0].timestamp((1626861392589, 1626861392589, 1626861392592)) - # if dtype == LegalDataType.INT : - # params[1].int((10, 12, 12)) - # params[2].int([194, 200, 201]) - # elif dtype == LegalDataType.UINT: - # params[1].int_unsigned((10, 12, 12)) - # params[2].int_unsigned([194, 200, 201]) - # elif dtype == LegalDataType.BIGINT: - # params[1].bigint((10, 12, 12)) - # params[2].bigint([194, 200, 201]) - # elif dtype == LegalDataType.UBIGINT: - # params[1].bigint_unsigned((10, 12, 12)) - # params[2].bigint_unsigned([194, 200, 201]) - # elif dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BIGINT: - # params[1].binary(("s10", "s12", "s12")) - # params[2].binary(["s194", "s200", "s201"]) - # params[3].float([0.31, 0.33, 0.31]) + params = taos.new_bind_params(4) + params[0].timestamp((1626861392589, 1626861392589, 1626861392592)) + if dtype == LegalDataType.INT : + params[1].int((10, 12, 12)) + params[2].int([194, 200, 201]) + elif dtype == LegalDataType.UINT: + params[1].int_unsigned((10, 12, 12)) + params[2].int_unsigned([194, 200, 201]) + elif dtype == LegalDataType.BIGINT: + params[1].bigint((10, 12, 12)) + params[2].bigint([194, 200, 201]) + elif dtype == LegalDataType.UBIGINT: + params[1].bigint_unsigned((10, 12, 12)) + params[2].bigint_unsigned([194, 200, 201]) + elif dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY: + params[1].binary(("s10", "s12", "s12")) + params[2].binary(["s194", "s200", "s201"]) + params[3].float([0.31, 0.33, 0.31]) - # stmt.bind_param_batch(params) + stmt.bind_param_batch(params) + + stmt.execute() + + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(3) params = taos.new_bind_params(4) params[0].timestamp((1626861392589)) - params[1].int((11)) - params[2].int([100]) - # if dtype == LegalDataType.INT : - # params[1].int((11)) - # params[2].int([100]) - # elif dtype == LegalDataType.UINT: - # params[1].int_unsigned((10)) - # params[2].int_unsigned([100]) - # elif dtype == LegalDataType.BIGINT: - # params[1].bigint((10)) - # params[2].bigint([100]) - # elif dtype == LegalDataType.UBIGINT: - # params[1].bigint_unsigned((10)) - # params[2].bigint_unsigned([100]) - # elif dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BIGINT: - # params[1].binary(("s10")) - # params[2].binary(["s100"]) + if dtype == LegalDataType.INT : + params[1].int((11)) + params[2].int([199]) + elif dtype == LegalDataType.UINT: + params[1].int_unsigned((11)) + params[2].int_unsigned([199]) + elif dtype == LegalDataType.BIGINT: + params[1].bigint((11)) + params[2].bigint([199]) + elif dtype == LegalDataType.UBIGINT: + params[1].bigint_unsigned((11)) + params[2].bigint_unsigned([199]) + elif dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY: + params[1].binary(("s11")) + params[2].binary(["s199"]) params[3].float([0.31]) stmt.bind_param(params) stmt.execute() + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(4) + + params = taos.new_bind_params(4) + params[0].timestamp((1626861392589)) + if dtype == LegalDataType.INT : + params[1].int((11)) + params[2].int([1000]) + elif dtype == LegalDataType.UINT: + params[1].int_unsigned((11)) + params[2].int_unsigned([1000]) + elif dtype == LegalDataType.BIGINT: + params[1].bigint((11)) + params[2].bigint([1000]) + elif dtype == LegalDataType.UBIGINT: + params[1].bigint_unsigned((11)) + params[2].bigint_unsigned([1000]) + elif dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY: + params[1].binary(("s11")) + params[2].binary(["1000"]) + params[3].float([0.31]) + + stmt.bind_param(params) + + stmt.execute() + + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(4) + + params = taos.new_bind_params(4) + params[0].timestamp((1626861392589)) + if dtype == LegalDataType.INT : + params[1].int(None) + params[2].int([199]) + elif dtype == LegalDataType.UINT: + params[1].int_unsigned(None) + params[2].int_unsigned([199]) + elif dtype == LegalDataType.BIGINT: + params[1].bigint(None) + params[2].bigint([199]) + elif dtype == LegalDataType.UBIGINT: + params[1].bigint_unsigned(None) + params[2].bigint_unsigned([199]) + elif dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY: + params[1].binary(None) + params[2].binary(["s199"]) + params[3].float([0.31]) + + try: + stmt.bind_param(params) + tdSql.checkEqual(False, True) + except Exception as errMsg: + pass + + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(4) + + params = taos.new_bind_params(4) + params[0].timestamp((1626861392589, 1626861392589, )) + if dtype == LegalDataType.INT : + params[1].int((None, 18,)) + params[2].int([194, 200]) + elif dtype == LegalDataType.UINT: + params[1].int_unsigned((None, 18)) + params[2].int_unsigned([194, 200]) + elif dtype == LegalDataType.BIGINT: + params[1].bigint((None, 18)) + params[2].bigint([194, 200]) + elif dtype == LegalDataType.UBIGINT: + params[1].bigint_unsigned((None, 18)) + params[2].bigint_unsigned([194, 200]) + elif dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY: + params[1].binary((None, "s18")) + params[2].binary(["s194", "s200"]) + params[3].float([0.31, 0.33, 0.31]) + + try: + stmt.bind_param(params) + tdSql.checkEqual(False, True) + except Exception as errMsg: + pass + + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(4) + + if dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY: + tdSql.query(f'select * from {self.stable_name} where pk="s11"') + tdSql.checkEqual(tdSql.queryResult[0][2] == '1000', True) + else: + tdSql.query(f'select * from {self.stable_name} where pk=11') + tdSql.checkEqual(tdSql.queryResult[0][2] == 1000, True) stmt.close() def test_implicit_conversion(self, dtype: LegalDataType): @@ -752,7 +847,7 @@ class TDTestCase: # for illegal_data in IllegalData.__members__.items(): # self.test_insert_data_illegal(date_type[1], illegal_data[1]) - # # 4. insert into select bug!!! + # # 4. insert into select - pass # self.test_insert_select(date_type[1]) # # 5. insert into values special cases - pass @@ -769,6 +864,8 @@ class TDTestCase: # # 9. insert data by schemaless model is not allowed - pass # self.test_schemaless_error() + # while(True): + # self.test_stmt(LegalDataType.VARCHAR) def stop(self): tdSql.close() From 49639714f3854b72dbd0371fccfd48a23a77425e Mon Sep 17 00:00:00 2001 From: Chris Zhai Date: Fri, 12 Apr 2024 09:47:41 +0800 Subject: [PATCH 06/21] update test cases for composite_primary_key_insert --- .../1-insert/composite_primary_key_insert.py | 97 ++++++++++++++++--- 1 file changed, 81 insertions(+), 16 deletions(-) diff --git a/tests/system-test/1-insert/composite_primary_key_insert.py b/tests/system-test/1-insert/composite_primary_key_insert.py index b9a92194e7..7261a75717 100644 --- a/tests/system-test/1-insert/composite_primary_key_insert.py +++ b/tests/system-test/1-insert/composite_primary_key_insert.py @@ -110,6 +110,7 @@ class TDTestCase: tdSql.checkRows(2) tdSql.query(f"select * from {self.stable_name} where engine = 1 and ts ={current_ts2} and pk = 2", show=SHOW_LOG) tdSql.checkRows(1) + self._check_select(self.stable_name) # 2.insert into value through child table table_name = f'{self.ctable_name}_2' @@ -130,6 +131,7 @@ class TDTestCase: tdSql.checkRows(2) tdSql.query(f"select * from {table_name} where engine = 2 and ts ={current_ts4} and pk = 2", show=SHOW_LOG) tdSql.checkRows(1) + self._check_select(table_name) # 3.insert value into child table from csv file data = [ @@ -157,6 +159,7 @@ class TDTestCase: tdSql.checkRows(2) tdSql.query(f"select * from {table_name} where engine=3 and ts='2024-03-29 16:55:43.586' and pk=2", show=SHOW_LOG) tdSql.checkRows(1) + self._check_select(table_name) # 4.insert value into child table from csv file, create table automatically data = [ @@ -173,6 +176,7 @@ class TDTestCase: self.tdCsv.write(data) table_name = f'{self.ctable_name}_4' + self._check_select(self.stable_name) tdSql.execute(f"insert into {table_name} using {self.stable_name} tags(4) file '{self.tdCsv.file}'", show=SHOW_LOG) tdSql.query(f'select * from {table_name}') @@ -183,6 +187,7 @@ class TDTestCase: tdSql.checkRows(2) tdSql.query(f"select * from {self.stable_name} where engine=4 and ts='2024-03-28 16:55:43.586' and pk=2", show=SHOW_LOG) tdSql.checkRows(1) + self._check_select(self.stable_name) # 5.insert value into normal table from csv file table_name = f'{self.ntable_name}_1' @@ -201,6 +206,7 @@ class TDTestCase: tdSql.checkRows(2) tdSql.query(f"select * from {table_name} where ts='2024-03-28 16:55:43.586' and pk=2", show=SHOW_LOG) tdSql.checkRows(1) + self._check_select(table_name) # 6.insert value into normal table table_name = f'{self.ntable_name}_2' @@ -226,6 +232,7 @@ class TDTestCase: tdSql.checkRows(2) tdSql.query(f"select * from {table_name} where ts ={current_ts2} and pk = 2", show=SHOW_LOG) tdSql.checkRows(1) + self._check_select(table_name) def test_insert_data_illegal(self, dtype: LegalDataType, illegal_data: IllegalData): # drop tables @@ -240,12 +247,14 @@ class TDTestCase: tdSql.error(f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) values('{table_name}', 1, now, {illegal_data.value}, '1')", show=SHOW_LOG) tdSql.query(f'select * from {self.stable_name}') tdSql.checkRows(0) + self._check_select(self.stable_name) # 2.insert into value through child table table_name = f'{self.ctable_name}_2' tdSql.error(f"insert into {table_name} using {self.stable_name} tags(2) values(now, {illegal_data.value}, '7')", show=SHOW_LOG) tdSql.query(f'select * from {self.stable_name}') tdSql.checkRows(0) + self._check_select(self.stable_name) # 4.insert value into child table from csv file data = [ @@ -266,12 +275,14 @@ class TDTestCase: tdSql.error(f"insert into {table_name} file '{self.tdCsv.file}'", show=SHOW_LOG) tdSql.query(f'select * from {table_name}') tdSql.checkRows(0) + self._check_select(table_name) # 5.insert value into child table from csv file, create table automatically table_name = f'{self.ctable_name}_4' tdSql.error(f"insert into {table_name} using {self.stable_name} tags(4) file '{self.tdCsv.file}'", show=SHOW_LOG) tdSql.query(f'select * from {self.stable_name}') tdSql.checkRows(0) + self._check_select(table_name) # 6.insert value into normal table from csv file table_name = f'{self.ntable_name}_1' @@ -279,6 +290,7 @@ class TDTestCase: tdSql.error(f"insert into {table_name} file '{self.tdCsv.file}'", show=SHOW_LOG) tdSql.query(f'select * from {table_name}') tdSql.checkRows(0) + self._check_select(table_name) # 7.insert value into normal table table_name = f'{self.ntable_name}_2' @@ -286,6 +298,7 @@ class TDTestCase: tdSql.error(f"insert into {table_name} values(now, {illegal_data.value}, '1')", show=SHOW_LOG) tdSql.query(f'select * from {table_name}') tdSql.checkRows(0) + self._check_select(table_name) def test_insert_select(self, dtype: LegalDataType): # # 1.pk table to non-pk table, throw error @@ -329,13 +342,17 @@ class TDTestCase: tdSql.query(f'select * from dest_{self.ctable_name}') dest_data = tdSql.queryResult self._compare_table_data(source_data, dest_data, 3, 3) + self._check_select(f'dest_{self.ctable_name}') tdSql.execute(f"delete from dest_{self.ctable_name}", show=SHOW_LOG) + self._check_select(f'dest_{self.ctable_name}') tdSql.execute(f"insert into dest_{self.ntable_name} select * from source_{self.ntable_name}", show=SHOW_LOG) tdSql.query(f'select * from dest_{self.ntable_name}') dest_data = tdSql.queryResult self._compare_table_data(source_data, dest_data, 3, 3) + self._check_select(f'dest_{self.ntable_name}') tdSql.execute(f"delete from dest_{self.ntable_name}", show=SHOW_LOG) + self._check_select(f'dest_{self.ntable_name}') # TD-29363 tdSql.execute(f"drop table if exists source_null") @@ -378,7 +395,9 @@ class TDTestCase: else: tdSql.query(f'select * from dest_{self.ctable_name} where c2=5 or c2=6', show=SHOW_LOG) tdSql.checkRows(2) + self._check_select(f'dest_{self.ctable_name}') tdSql.execute(f"delete from dest_{self.ctable_name}", show=SHOW_LOG) + self._check_select(f'dest_{self.ctable_name}') tdSql.execute(f"insert into dest_{self.ntable_name} (ts, pk, c2) select ts, pk, c2 from source_{self.ntable_name}", show=SHOW_LOG) tdSql.query(f'select * from dest_{self.ntable_name}') @@ -388,7 +407,9 @@ class TDTestCase: else: tdSql.query(f'select * from dest_{self.ntable_name} where c2=5 or c2=6') tdSql.checkRows(2) + self._check_select(f'dest_{self.ntable_name}') tdSql.execute(f"delete from dest_{self.ntable_name}", show=SHOW_LOG) + self._check_select(f'dest_{self.ntable_name}') def test_schemaless_error(self): # 5.1.insert into values via influxDB @@ -462,17 +483,20 @@ class TDTestCase: tdSql.error(f"insert into {self.stable_name} (tbname, engine, pk, c2) values('{self.ctable_name}', 1, '1', '1')", show=SHOW_LOG) tdSql.query(f'select * from {self.stable_name}') tdSql.checkRows(0) + self._check_select(self.stable_name) # # 4.2.insert into value through child table tdSql.error(f"insert into {self.ctable_name} using {self.stable_name} tags(2) (pk, c2) values('1', '7')", show=SHOW_LOG) tdSql.query(f'select * from {self.stable_name}') tdSql.checkRows(0) + self._check_select(self.stable_name) # # 4.3.insert value into normal table tdSql.execute(f"create table {self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 varchar(20))", show=SHOW_LOG) tdSql.error(f"insert into {self.ntable_name} (pk, c2) values('1', '1')", show=SHOW_LOG) tdSql.query(f'select * from {self.ntable_name}') tdSql.checkRows(0) + self._check_select(self.ntable_name) # 5.insert into values without pk column # drop tables @@ -485,17 +509,20 @@ class TDTestCase: tdSql.error(f"insert into {self.stable_name} (tbname, engine, ts, c2) values('{self.ctable_name}', 1, now, '1')", show=SHOW_LOG) tdSql.query(f'select * from {self.stable_name}') tdSql.checkRows(0) + self._check_select(self.stable_name) # # 5.2.insert into value through child table tdSql.error(f"insert into {self.ctable_name} using {self.stable_name} tags(2) (ts, c2) values(now, '7')", show=SHOW_LOG) tdSql.query(f'select * from {self.stable_name}') tdSql.checkRows(0) + self._check_select(self.stable_name) # # 5.3.insert value into normal table tdSql.execute(f"create table {self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 varchar(20))", show=SHOW_LOG) tdSql.error(f"insert into {self.ntable_name} (ts, c2) values(now, '1')", show=SHOW_LOG) tdSql.query(f'select * from {self.ntable_name}') tdSql.checkRows(0) + self._check_select(self.ntable_name) def test_insert_into_mutiple_tables(self, dtype: LegalDataType): # drop super table and child table @@ -514,7 +541,8 @@ class TDTestCase: tdSql.query(f'select * from {self.stable_name}') tdSql.checkRows(0) - + self._check_select(self.stable_name) + sql = f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) " \ f"values('{self.ctable_name}_1', 1, '2021-07-13 14:06:34.630', 1, 100) " \ f"('{self.ctable_name}_1', 1, '2021-07-13 14:06:34.630', 2, 200) " \ @@ -530,6 +558,7 @@ class TDTestCase: tdSql.checkRows(2) tdSql.query(f"select * from {self.stable_name} where ts='2021-07-13 14:06:34.630' and pk=2", show=SHOW_LOG) tdSql.checkRows(1) + self._check_select(self.stable_name) # 2.insert into value and create table automatically tdSql.execute(f"drop table if exists {self.stable_name}") @@ -542,6 +571,7 @@ class TDTestCase: tdSql.error(error_sql, show=SHOW_LOG) tdSql.query(f'select * from {self.stable_name}') tdSql.checkRows(0) + self._check_select(self.stable_name) sql = f"insert into {self.ctable_name}_1 using {self.stable_name} tags(1) values('2021-07-13 14:06:34.630', 1, 100) ('2021-07-13 14:06:34.630', 2, 200) " \ f"{self.ctable_name}_2 using {self.stable_name} (engine) tags(2) values('2021-07-14 14:06:34.630', 1, 300) ('2021-07-14 14:06:34.630', 2, 400) " \ @@ -555,6 +585,7 @@ class TDTestCase: tdSql.checkRows(2) tdSql.query(f"select * from {self.stable_name} where ts='2021-07-13 14:06:34.630' and pk=2", show=SHOW_LOG) tdSql.checkRows(1) + self._check_select(self.stable_name) # 3.insert value into child table from csv file, create table automatically tdSql.execute(f"drop table if exists {self.stable_name}") @@ -581,6 +612,7 @@ class TDTestCase: tdSql.query(f'select * from {self.stable_name}') tdSql.checkRows(0) + self._check_select(self.stable_name) data = [ ['ts','pk','c2'], @@ -607,6 +639,7 @@ class TDTestCase: tdSql.checkRows(6) tdSql.query(f"select * from {self.stable_name} where ts='2024-03-29 16:55:42.572' and pk=2", show=SHOW_LOG) tdSql.checkRows(3) + self._check_select(self.stable_name) # 6.insert value into normal table tdSql.execute(f"drop table if exists {self.ntable_name}_1") @@ -623,6 +656,7 @@ class TDTestCase: tdSql.checkRows(2) tdSql.query(f"select * from {self.ntable_name}_2 where ts='2021-07-14 14:06:34.630' and pk=2", show=SHOW_LOG) tdSql.checkRows(1) + self._check_select(self.ntable_name) # 7. insert value into child and normal table tdSql.execute(f"drop table if exists {self.stable_name}") @@ -650,6 +684,8 @@ class TDTestCase: tdSql.checkRows(2) tdSql.query(f'select * from {self.ntable_name}') tdSql.checkRows(2) + self._check_select(self.stable_name) + self._check_select(self.ntable_name) def test_stmt(self, dtype: LegalDataType): tdSql.execute(f"drop table if exists {self.stable_name}", show=SHOW_LOG) @@ -693,6 +729,7 @@ class TDTestCase: tdSql.query(f'select * from {self.stable_name}') tdSql.checkRows(3) + self._check_select(self.stable_name) params = taos.new_bind_params(4) params[0].timestamp((1626861392589)) @@ -719,6 +756,7 @@ class TDTestCase: tdSql.query(f'select * from {self.stable_name}') tdSql.checkRows(4) + self._check_select(self.stable_name) params = taos.new_bind_params(4) params[0].timestamp((1626861392589)) @@ -745,6 +783,7 @@ class TDTestCase: tdSql.query(f'select * from {self.stable_name}') tdSql.checkRows(4) + self._check_select(self.stable_name) params = taos.new_bind_params(4) params[0].timestamp((1626861392589)) @@ -773,6 +812,7 @@ class TDTestCase: tdSql.query(f'select * from {self.stable_name}') tdSql.checkRows(4) + self._check_select(self.stable_name) params = taos.new_bind_params(4) params[0].timestamp((1626861392589, 1626861392589, )) @@ -801,6 +841,7 @@ class TDTestCase: tdSql.query(f'select * from {self.stable_name}') tdSql.checkRows(4) + self._check_select(self.stable_name) if dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY: tdSql.query(f'select * from {self.stable_name} where pk="s11"') @@ -825,6 +866,7 @@ class TDTestCase: tdSql.query(f'select * from dest_table where c2=1000') tdSql.checkRows(1) tdSql.execute(f"delete from dest_table", show=SHOW_LOG) + self._check_select('dest_table') def _compare_table_data(self, result1, result2, row = 0, col = 0): for i in range(row): @@ -832,38 +874,61 @@ class TDTestCase: if result1[i][j] != result2[i][j]: tdSql.checkEqual(False, True) + def _check_select(self, table_nam: str): + tdSql.query(f'select count(*) from {table_nam} ') + tdSql.query(f'select * from {table_nam}') + tdSql.query(f'select last_row(*) from {table_nam}') + tdSql.query(f'select first(*) from {table_nam}') + tdSql.query(f'select last(*) from {table_nam}') + tdSql.query(f'select * from {table_nam} order by ts asc') + tdSql.query(f'select * from {table_nam} order by ts desc') + tdSql.query(f'select * from {table_nam} order by pk asc') + tdSql.query(f'select * from {table_nam} order by pk desc') + tdSql.query(f'select * from {table_nam} order by ts asc, pk desc') + tdSql.query(f'select * from {table_nam} order by ts desc, pk asc') + def run(self): tdSql.prepare(replica = self.replicaVar) self.prepare_db() for date_type in LegalDataType.__members__.items(): - # # 1.insert into value with pk - pass - # self.test_insert_data(date_type[1], HasPK.YES) + tdLog.info(f'') + # 1.insert into value with pk - pass + tdLog.info('[1.insert into value with pk]') + self.test_insert_data(date_type[1], HasPK.YES) - # # 2.insert into value without pk - pass - # self.test_insert_data(date_type[1], HasPK.NO) + # 2.insert into value without pk - pass + tdLog.info('[2.insert into value without pk]') + self.test_insert_data(date_type[1], HasPK.NO) - # # 3.insert into illegal data - pass + # 3.insert into illegal data - pass + tdLog.info('[3.insert into illegal data]') # for illegal_data in IllegalData.__members__.items(): # self.test_insert_data_illegal(date_type[1], illegal_data[1]) - # # 4. insert into select - pass - # self.test_insert_select(date_type[1]) + # 4. insert into select - pass + tdLog.info('[4. insert into select]') + self.test_insert_select(date_type[1]) - # # 5. insert into values special cases - pass - # self.test_insert_values_special(date_type[1]) + # 5. insert into values special cases - pass + tdLog.info('[5. insert into values special cases]') + self.test_insert_values_special(date_type[1]) - # # 6. insert into value to mutiple tables - pass - # self.test_implicit_conversion(date_type[1]) + # 6. test implicit conversion - pass + tdLog.info('[6. test implicit conversion]') + self.test_implicit_conversion(date_type[1]) - # # 7. insert into value to mutiple tables - pass - # self.test_insert_into_mutiple_tables(date_type[1]) + # 7. insert into value to mutiple tables - pass + tdLog.info('[7. insert into value to mutiple tables]') + self.test_insert_into_mutiple_tables(date_type[1]) # 8. stmt wait for test!!!! + tdLog.info('[8. stmt wait for test]') self.test_stmt(date_type[1]) - # # 9. insert data by schemaless model is not allowed - pass - # self.test_schemaless_error() + # 9. insert data by schemaless model is not allowed - pass + tdLog.info('[9. insert data by schemaless model is not allowed]') + self.test_schemaless_error() # while(True): # self.test_stmt(LegalDataType.VARCHAR) From 0241be8b083e9e148e0939fa41deab956403d6f6 Mon Sep 17 00:00:00 2001 From: Chris Zhai Date: Fri, 12 Apr 2024 10:57:46 +0800 Subject: [PATCH 07/21] add flush test to composite primary key insert --- .../1-insert/composite_primary_key_insert.py | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/tests/system-test/1-insert/composite_primary_key_insert.py b/tests/system-test/1-insert/composite_primary_key_insert.py index 7261a75717..84fa24247a 100644 --- a/tests/system-test/1-insert/composite_primary_key_insert.py +++ b/tests/system-test/1-insert/composite_primary_key_insert.py @@ -102,6 +102,7 @@ class TDTestCase: tdSql.execute(f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) values('{table_name}', 1, {current_ts2}, 2, '5')", show=SHOW_LOG) tdSql.execute(f"insert into {self.stable_name} (tbname, engine, ts, pk) values('{table_name}', 1, now + 2s, 2)", show=SHOW_LOG) + tdSql.execute(f"flush database {table_name}", show=SHOW_LOG) tdSql.query(f'select * from {self.stable_name}') tdSql.checkRows(6) tdSql.query(f"select * from {self.stable_name} where engine = 1 and ts ={current_ts1}", show=SHOW_LOG) @@ -151,6 +152,7 @@ class TDTestCase: tdSql.execute(f"create table {table_name} using {self.stable_name} tags(3)", show=SHOW_LOG) tdSql.execute(f"insert into {table_name} file '{self.tdCsv.file}'", show=SHOW_LOG) + tdSql.execute(f"flush database {table_name}", show=SHOW_LOG) tdSql.query(f'select * from {table_name}') tdSql.checkRows(6) tdSql.query(f"select * from {table_name} where engine=3 and ts='2024-03-29 16:55:42.572'", show=SHOW_LOG) @@ -224,6 +226,7 @@ class TDTestCase: tdSql.execute(f"insert into {table_name} (ts, pk, c2) values({current_ts2}, 2, '5')", show=SHOW_LOG) tdSql.execute(f"insert into {table_name} (ts, pk) values(now + 2s, 2)", show=SHOW_LOG) + tdSql.execute(f"flush database {table_name}", show=SHOW_LOG) tdSql.query(f'select * from {table_name}') tdSql.checkRows(6) tdSql.query(f"select * from {table_name} where ts ={current_ts1}", show=SHOW_LOG) @@ -288,6 +291,7 @@ class TDTestCase: table_name = f'{self.ntable_name}_1' tdSql.execute(f"create table {table_name} using {self.stable_name} tags(3)", show=SHOW_LOG) tdSql.error(f"insert into {table_name} file '{self.tdCsv.file}'", show=SHOW_LOG) + tdSql.execute(f"flush database {table_name}", show=SHOW_LOG) tdSql.query(f'select * from {table_name}') tdSql.checkRows(0) self._check_select(table_name) @@ -339,6 +343,7 @@ class TDTestCase: source_data = tdSql.queryResult tdSql.execute(f"insert into dest_{self.ctable_name} select ts, pk, c2 from source_{self.ntable_name}", show=SHOW_LOG) + tdSql.execute(f"flush database {table_name}", show=SHOW_LOG) tdSql.query(f'select * from dest_{self.ctable_name}') dest_data = tdSql.queryResult self._compare_table_data(source_data, dest_data, 3, 3) @@ -471,6 +476,8 @@ class TDTestCase: except SchemalessError as errMsg: tdSql.checkEqual(errMsg.msg == 'Can not insert data into table with primary key', True) + tdSql.execute(f"flush database {table_name}", show=SHOW_LOG) + def test_insert_values_special(self, dtype: LegalDataType): # 4.insert into values without ts column # drop tables @@ -520,6 +527,7 @@ class TDTestCase: # # 5.3.insert value into normal table tdSql.execute(f"create table {self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 varchar(20))", show=SHOW_LOG) tdSql.error(f"insert into {self.ntable_name} (ts, c2) values(now, '1')", show=SHOW_LOG) + tdSql.execute(f"flush database {table_name}", show=SHOW_LOG) tdSql.query(f'select * from {self.ntable_name}') tdSql.checkRows(0) self._check_select(self.ntable_name) @@ -569,6 +577,7 @@ class TDTestCase: f"{self.ctable_name}_3 using {self.stable_name} (engine) tags(3) values('2021-07-15 14:06:34.630', null, 500)" tdSql.error(error_sql, show=SHOW_LOG) + tdSql.execute(f"flush database {table_name}", show=SHOW_LOG) tdSql.query(f'select * from {self.stable_name}') tdSql.checkRows(0) self._check_select(self.stable_name) @@ -674,6 +683,7 @@ class TDTestCase: f"{self.ntable_name} values('2021-07-14 14:06:34.630', 1, 300) ('2021-07-14 14:06:34.630', 2, 400) " tdSql.error(error_sql, show=SHOW_LOG) + tdSql.execute(f"flush database {table_name}", show=SHOW_LOG) tdSql.query(f'select * from {self.stable_name}') tdSql.checkRows(0) tdSql.query(f'select * from {self.ntable_name}') @@ -727,6 +737,7 @@ class TDTestCase: stmt.execute() + tdSql.execute(f"flush database {table_name}", show=SHOW_LOG) tdSql.query(f'select * from {self.stable_name}') tdSql.checkRows(3) self._check_select(self.stable_name) @@ -863,6 +874,7 @@ class TDTestCase: tdSql.execute(f"insert into source_table values(now, 100, 1000)", show=SHOW_LOG) tdSql.execute(f"insert into dest_table select * from source_table", show=SHOW_LOG) + tdSql.execute(f"flush database {table_name}", show=SHOW_LOG) tdSql.query(f'select * from dest_table where c2=1000') tdSql.checkRows(1) tdSql.execute(f"delete from dest_table", show=SHOW_LOG) @@ -892,19 +904,19 @@ class TDTestCase: self.prepare_db() for date_type in LegalDataType.__members__.items(): - tdLog.info(f'') - # 1.insert into value with pk - pass - tdLog.info('[1.insert into value with pk]') - self.test_insert_data(date_type[1], HasPK.YES) + # tdLog.info(f'') + # # 1.insert into value with pk - pass + # tdLog.info('[1.insert into value with pk]') + # self.test_insert_data(date_type[1], HasPK.YES) - # 2.insert into value without pk - pass - tdLog.info('[2.insert into value without pk]') - self.test_insert_data(date_type[1], HasPK.NO) + # # 2.insert into value without pk - pass + # tdLog.info('[2.insert into value without pk]') + # self.test_insert_data(date_type[1], HasPK.NO) # 3.insert into illegal data - pass tdLog.info('[3.insert into illegal data]') - # for illegal_data in IllegalData.__members__.items(): - # self.test_insert_data_illegal(date_type[1], illegal_data[1]) + for illegal_data in IllegalData.__members__.items(): + self.test_insert_data_illegal(date_type[1], illegal_data[1]) # 4. insert into select - pass tdLog.info('[4. insert into select]') From a7fb34991688909b1e0297319bbafdcfba463a80 Mon Sep 17 00:00:00 2001 From: Chris Zhai Date: Fri, 12 Apr 2024 16:42:48 +0800 Subject: [PATCH 08/21] add flush before select --- .../1-insert/composite_primary_key_insert.py | 43 +++++++++++-------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/tests/system-test/1-insert/composite_primary_key_insert.py b/tests/system-test/1-insert/composite_primary_key_insert.py index 84fa24247a..7f73000b05 100644 --- a/tests/system-test/1-insert/composite_primary_key_insert.py +++ b/tests/system-test/1-insert/composite_primary_key_insert.py @@ -72,7 +72,7 @@ class TDTestCase: def prepare_db(self): tdSql.execute(f"drop database if exists {self.database}") - tdSql.execute(f"create database {self.database} CACHEMODEL 'none'") + tdSql.execute(f"create database {self.database}") tdSql.execute(f"use {self.database}") def get_latest_ts(self, table_name: str): @@ -102,7 +102,7 @@ class TDTestCase: tdSql.execute(f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) values('{table_name}', 1, {current_ts2}, 2, '5')", show=SHOW_LOG) tdSql.execute(f"insert into {self.stable_name} (tbname, engine, ts, pk) values('{table_name}', 1, now + 2s, 2)", show=SHOW_LOG) - tdSql.execute(f"flush database {table_name}", show=SHOW_LOG) + tdSql.execute(f"flush database {self.database}", show=SHOW_LOG) tdSql.query(f'select * from {self.stable_name}') tdSql.checkRows(6) tdSql.query(f"select * from {self.stable_name} where engine = 1 and ts ={current_ts1}", show=SHOW_LOG) @@ -152,7 +152,7 @@ class TDTestCase: tdSql.execute(f"create table {table_name} using {self.stable_name} tags(3)", show=SHOW_LOG) tdSql.execute(f"insert into {table_name} file '{self.tdCsv.file}'", show=SHOW_LOG) - tdSql.execute(f"flush database {table_name}", show=SHOW_LOG) + tdSql.execute(f"flush database {self.database}", show=SHOW_LOG) tdSql.query(f'select * from {table_name}') tdSql.checkRows(6) tdSql.query(f"select * from {table_name} where engine=3 and ts='2024-03-29 16:55:42.572'", show=SHOW_LOG) @@ -226,7 +226,7 @@ class TDTestCase: tdSql.execute(f"insert into {table_name} (ts, pk, c2) values({current_ts2}, 2, '5')", show=SHOW_LOG) tdSql.execute(f"insert into {table_name} (ts, pk) values(now + 2s, 2)", show=SHOW_LOG) - tdSql.execute(f"flush database {table_name}", show=SHOW_LOG) + tdSql.execute(f"flush database {self.database}", show=SHOW_LOG) tdSql.query(f'select * from {table_name}') tdSql.checkRows(6) tdSql.query(f"select * from {table_name} where ts ={current_ts1}", show=SHOW_LOG) @@ -291,7 +291,7 @@ class TDTestCase: table_name = f'{self.ntable_name}_1' tdSql.execute(f"create table {table_name} using {self.stable_name} tags(3)", show=SHOW_LOG) tdSql.error(f"insert into {table_name} file '{self.tdCsv.file}'", show=SHOW_LOG) - tdSql.execute(f"flush database {table_name}", show=SHOW_LOG) + tdSql.execute(f"flush database {self.database}", show=SHOW_LOG) tdSql.query(f'select * from {table_name}') tdSql.checkRows(0) self._check_select(table_name) @@ -343,7 +343,7 @@ class TDTestCase: source_data = tdSql.queryResult tdSql.execute(f"insert into dest_{self.ctable_name} select ts, pk, c2 from source_{self.ntable_name}", show=SHOW_LOG) - tdSql.execute(f"flush database {table_name}", show=SHOW_LOG) + tdSql.execute(f"flush database {self.database}", show=SHOW_LOG) tdSql.query(f'select * from dest_{self.ctable_name}') dest_data = tdSql.queryResult self._compare_table_data(source_data, dest_data, 3, 3) @@ -476,7 +476,7 @@ class TDTestCase: except SchemalessError as errMsg: tdSql.checkEqual(errMsg.msg == 'Can not insert data into table with primary key', True) - tdSql.execute(f"flush database {table_name}", show=SHOW_LOG) + tdSql.execute(f"flush database {self.database}", show=SHOW_LOG) def test_insert_values_special(self, dtype: LegalDataType): # 4.insert into values without ts column @@ -527,7 +527,7 @@ class TDTestCase: # # 5.3.insert value into normal table tdSql.execute(f"create table {self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 varchar(20))", show=SHOW_LOG) tdSql.error(f"insert into {self.ntable_name} (ts, c2) values(now, '1')", show=SHOW_LOG) - tdSql.execute(f"flush database {table_name}", show=SHOW_LOG) + tdSql.execute(f"flush database {self.database}", show=SHOW_LOG) tdSql.query(f'select * from {self.ntable_name}') tdSql.checkRows(0) self._check_select(self.ntable_name) @@ -577,7 +577,7 @@ class TDTestCase: f"{self.ctable_name}_3 using {self.stable_name} (engine) tags(3) values('2021-07-15 14:06:34.630', null, 500)" tdSql.error(error_sql, show=SHOW_LOG) - tdSql.execute(f"flush database {table_name}", show=SHOW_LOG) + tdSql.execute(f"flush database {self.database}", show=SHOW_LOG) tdSql.query(f'select * from {self.stable_name}') tdSql.checkRows(0) self._check_select(self.stable_name) @@ -683,7 +683,7 @@ class TDTestCase: f"{self.ntable_name} values('2021-07-14 14:06:34.630', 1, 300) ('2021-07-14 14:06:34.630', 2, 400) " tdSql.error(error_sql, show=SHOW_LOG) - tdSql.execute(f"flush database {table_name}", show=SHOW_LOG) + tdSql.execute(f"flush database {self.database}", show=SHOW_LOG) tdSql.query(f'select * from {self.stable_name}') tdSql.checkRows(0) tdSql.query(f'select * from {self.ntable_name}') @@ -737,7 +737,7 @@ class TDTestCase: stmt.execute() - tdSql.execute(f"flush database {table_name}", show=SHOW_LOG) + tdSql.execute(f"flush database {self.database}", show=SHOW_LOG) tdSql.query(f'select * from {self.stable_name}') tdSql.checkRows(3) self._check_select(self.stable_name) @@ -874,7 +874,7 @@ class TDTestCase: tdSql.execute(f"insert into source_table values(now, 100, 1000)", show=SHOW_LOG) tdSql.execute(f"insert into dest_table select * from source_table", show=SHOW_LOG) - tdSql.execute(f"flush database {table_name}", show=SHOW_LOG) + tdSql.execute(f"flush database {self.database}", show=SHOW_LOG) tdSql.query(f'select * from dest_table where c2=1000') tdSql.checkRows(1) tdSql.execute(f"delete from dest_table", show=SHOW_LOG) @@ -890,8 +890,13 @@ class TDTestCase: tdSql.query(f'select count(*) from {table_nam} ') tdSql.query(f'select * from {table_nam}') tdSql.query(f'select last_row(*) from {table_nam}') + tdSql.query(f'select last_row(*) from {table_nam} group by tbname') tdSql.query(f'select first(*) from {table_nam}') + tdSql.query(f'select first(*) from {table_nam} group by tbname') tdSql.query(f'select last(*) from {table_nam}') + tdSql.query(f'select last(*) from {table_nam} group by tbname') + tdSql.query(f'select ts, last(pk) from {table_nam} order by ts') + tdSql.query(f'select * from {table_nam} order by ts asc') tdSql.query(f'select * from {table_nam} order by ts desc') tdSql.query(f'select * from {table_nam} order by pk asc') @@ -904,14 +909,14 @@ class TDTestCase: self.prepare_db() for date_type in LegalDataType.__members__.items(): - # tdLog.info(f'') - # # 1.insert into value with pk - pass - # tdLog.info('[1.insert into value with pk]') - # self.test_insert_data(date_type[1], HasPK.YES) + tdLog.info(f'') + # 1.insert into value with pk - pass + tdLog.info('[1.insert into value with pk]') + self.test_insert_data(date_type[1], HasPK.YES) - # # 2.insert into value without pk - pass - # tdLog.info('[2.insert into value without pk]') - # self.test_insert_data(date_type[1], HasPK.NO) + # 2.insert into value without pk - pass + tdLog.info('[2.insert into value without pk]') + self.test_insert_data(date_type[1], HasPK.NO) # 3.insert into illegal data - pass tdLog.info('[3.insert into illegal data]') From 88d227a97d5edc878363fa9e1ff131eae55e5e83 Mon Sep 17 00:00:00 2001 From: Chris Zhai Date: Sun, 14 Apr 2024 12:19:04 +0800 Subject: [PATCH 09/21] update composite primary ke case --- tests/system-test/1-insert/composite_primary_key_insert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/composite_primary_key_insert.py b/tests/system-test/1-insert/composite_primary_key_insert.py index 7f73000b05..1a22eb781c 100644 --- a/tests/system-test/1-insert/composite_primary_key_insert.py +++ b/tests/system-test/1-insert/composite_primary_key_insert.py @@ -285,7 +285,7 @@ class TDTestCase: tdSql.error(f"insert into {table_name} using {self.stable_name} tags(4) file '{self.tdCsv.file}'", show=SHOW_LOG) tdSql.query(f'select * from {self.stable_name}') tdSql.checkRows(0) - self._check_select(table_name) + self._check_select(self.stable_name) # 6.insert value into normal table from csv file table_name = f'{self.ntable_name}_1' From 529ca1d1c12ac9cdff25bdb78a1623ca36e2fc95 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 18 Apr 2024 13:56:43 +0800 Subject: [PATCH 10/21] opti:add local thread error msg array --- include/libs/parser/parser.h | 3 +- include/util/taoserror.h | 7 +++++ source/client/src/clientRawBlockWrite.c | 38 +++++++++++++++++++++---- source/client/src/clientTmq.c | 16 +++++------ source/libs/parser/src/parInsertUtil.c | 15 +++++----- source/util/src/terror.c | 11 ++++++- utils/test/c/tmq_taosx_ci.c | 12 ++++++++ 7 files changed, 79 insertions(+), 23 deletions(-) diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 7ae393f5a6..37d5f3f0b6 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -152,7 +152,8 @@ int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsS STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, int32_t msgBufLen); int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash); -int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCreateTbReq** pCreateTb, TAOS_FIELD *fields, int numFields, bool needChangeLength); +int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCreateTbReq** pCreateTb, TAOS_FIELD *fields, + int numFields, bool needChangeLength, char* errstr, int32_t errstrLen); int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray); SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index c13c8586f5..fe6f3f53e9 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -33,8 +33,15 @@ extern "C" { const char* tstrerror(int32_t err); const char* terrstr(); +#define ERR_MSG_LEN 256 + +char* taosGetErrMsg(); int32_t* taosGetErrno(); #define terrno (*taosGetErrno()) +#define terrMsg (taosGetErrMsg()) + +#define SET_ERROR_MSG(MSG, ...) \ + snprintf(terrMsg, ERR_MSG_LEN, MSG, ##__VA_ARGS__) #define TSDB_CODE_SUCCESS 0 #define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 84a2bf3e03..c6e864f0c7 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1486,7 +1486,7 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pDat pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)); - code = rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false); + code = rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false, NULL, 0); if (code != TSDB_CODE_SUCCESS) { goto end; } @@ -1570,7 +1570,7 @@ int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const cha pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)); - code = rawBlockBindData(pQuery, pTableMeta, pData, NULL, NULL, 0, false); + code = rawBlockBindData(pQuery, pTableMeta, pData, NULL, NULL, 0, false, NULL, 0); if (code != TSDB_CODE_SUCCESS) { goto end; } @@ -1608,6 +1608,7 @@ static void* getRawDataFromRes(void* pRetrieve) { static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { if (taos == NULL || data == NULL) { terrno = TSDB_CODE_INVALID_PARA; + SET_ERROR_MSG("taos:%p or data:%p is NULL", taos, data); return terrno; } int32_t code = TSDB_CODE_SUCCESS; @@ -1620,6 +1621,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { terrno = TSDB_CODE_SUCCESS; SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); if (!pRequest) { + SET_ERROR_MSG("pRequest is NULL"); return terrno; } @@ -1631,6 +1633,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { tDecoderInit(&decoder, data, dataLen); code = tDecodeMqDataRsp(&decoder, &rspObj.rsp, *(int8_t*)data); if (code != 0) { + SET_ERROR_MSG("decode mq data rsp failed"); code = TSDB_CODE_INVALID_MSG; goto end; } @@ -1643,6 +1646,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { struct SCatalog* pCatalog = NULL; code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { + SET_ERROR_MSG("cata log get handle failed"); goto end; } @@ -1655,6 +1659,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { pQuery = smlInitHandle(); if (pQuery == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; + SET_ERROR_MSG("init sml handle failed"); goto end; } pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); @@ -1666,6 +1671,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.blockTbName, rspObj.resIter); if (!tbName) { + SET_ERROR_MSG("block tbname is null"); code = TSDB_CODE_TMQ_INVALID_MSG; goto end; } @@ -1681,12 +1687,14 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { // continue; // } if (code != TSDB_CODE_SUCCESS) { + SET_ERROR_MSG("cata log get table:%s meta failed", tbName); goto end; } SVgroupInfo vg; code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg); if (code != TSDB_CODE_SUCCESS) { + SET_ERROR_MSG("cata log get table:%s vgroup failed", tbName); goto end; } @@ -1698,6 +1706,8 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.blockSchema, rspObj.resIter); TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD)); if (fields == NULL) { + SET_ERROR_MSG("calloc fields failed"); + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } for (int i = 0; i < pSW->nCols; i++) { @@ -1706,16 +1716,18 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name)); } void* rawData = getRawDataFromRes(pRetrieve); - code = rawBlockBindData(pQuery, pTableMeta, rawData, NULL, fields, pSW->nCols, true); + char err[ERR_MSG_LEN] = {0}; + code = rawBlockBindData(pQuery, pTableMeta, rawData, NULL, fields, pSW->nCols, true, err, ERR_MSG_LEN); taosMemoryFree(fields); if (code != TSDB_CODE_SUCCESS) { + SET_ERROR_MSG("table:%s, err:%s", tbName, err); goto end; } - taosMemoryFreeClear(pTableMeta); } code = smlBuildOutput(pQuery, pVgHash); if (code != TSDB_CODE_SUCCESS) { + SET_ERROR_MSG("sml build output failed"); goto end; } @@ -1737,6 +1749,7 @@ end: static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) { if (taos == NULL || data == NULL) { terrno = TSDB_CODE_INVALID_PARA; + SET_ERROR_MSG("taos:%p or data:%p is NULL", taos, data); return terrno; } int32_t code = TSDB_CODE_SUCCESS; @@ -1750,6 +1763,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) terrno = TSDB_CODE_SUCCESS; SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); if (!pRequest) { + SET_ERROR_MSG("pRequest is NULL"); return terrno; } uDebug(LOG_ID_TAG " write raw metadata, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen); @@ -1760,6 +1774,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) tDecoderInit(&decoder, data, dataLen); code = tDecodeSTaosxRsp(&decoder, &rspObj.rsp, *(int8_t*)data); if (code != 0) { + SET_ERROR_MSG("decode mq taosx data rsp failed"); code = TSDB_CODE_INVALID_MSG; goto end; } @@ -1772,6 +1787,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) struct SCatalog* pCatalog = NULL; code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { + SET_ERROR_MSG("cata log get handle failed"); goto end; } @@ -1783,6 +1799,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) pQuery = smlInitHandle(); if (pQuery == NULL) { + SET_ERROR_MSG("init sml handle failed"); code = TSDB_CODE_OUT_OF_MEMORY; goto end; } @@ -1797,6 +1814,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.blockTbName, rspObj.resIter); if (!tbName) { + SET_ERROR_MSG("block tbname is null"); code = TSDB_CODE_TMQ_INVALID_MSG; goto end; } @@ -1818,6 +1836,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) tDecoderClear(&decoderTmp); tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE); code = TSDB_CODE_TMQ_INVALID_MSG; + SET_ERROR_MSG("decode create table:%s req failed", tbName); goto end; } @@ -1825,6 +1844,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) code = TSDB_CODE_TSC_INVALID_VALUE; tDecoderClear(&decoderTmp); tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE); + SET_ERROR_MSG("create table req type is not child table: %s, type: %d", tbName, pCreateReq.type); goto end; } if (strcmp(tbName, pCreateReq.name) == 0) { @@ -1841,6 +1861,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) SVgroupInfo vg; code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg); if (code != TSDB_CODE_SUCCESS) { + SET_ERROR_MSG("cata log get table:%s vgroup failed", tbName); goto end; } @@ -1854,6 +1875,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) // continue; // } if (code != TSDB_CODE_SUCCESS) { + SET_ERROR_MSG("cata log get table:%s meta failed", tbName); goto end; } @@ -1870,6 +1892,8 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.blockSchema, rspObj.resIter); TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD)); if (fields == NULL) { + SET_ERROR_MSG("calloc fields failed"); + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } for (int i = 0; i < pSW->nCols; i++) { @@ -1878,12 +1902,13 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name)); } void* rawData = getRawDataFromRes(pRetrieve); - code = rawBlockBindData(pQuery, pTableMeta, rawData, &pCreateReqDst, fields, pSW->nCols, true); + char err[ERR_MSG_LEN] = {0}; + code = rawBlockBindData(pQuery, pTableMeta, rawData, &pCreateReqDst, fields, pSW->nCols, true, err, ERR_MSG_LEN); taosMemoryFree(fields); if (code != TSDB_CODE_SUCCESS) { + SET_ERROR_MSG("table:%s, err:%s", tbName, err); goto end; } - taosMemoryFreeClear(pTableMeta); } code = smlBuildOutput(pQuery, pVgHash); @@ -2010,6 +2035,7 @@ void tmq_free_raw(tmq_raw_data raw) { if (raw.raw_type == RES_TYPE__TMQ || raw.raw_type == RES_TYPE__TMQ_METADATA) { taosMemoryFree(raw.raw); } + memset(terrMsg, 0, ERR_MSG_LEN); } int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) { diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 4cac1febe5..df5f8e2e00 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1056,17 +1056,17 @@ static void tmqMgmtInit(void) { } } -#define SET_ERROR_MSG(MSG) \ +#define SET_ERROR_MSG_TMQ(MSG) \ if (errstr != NULL) snprintf(errstr, errstrLen, MSG); tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { if (conf == NULL) { - SET_ERROR_MSG("configure is null") + SET_ERROR_MSG_TMQ("configure is null") return NULL; } taosThreadOnce(&tmqInit, tmqMgmtInit); if (tmqInitRes != 0) { terrno = tmqInitRes; - SET_ERROR_MSG("tmq timer init error") + SET_ERROR_MSG_TMQ("tmq timer init error") return NULL; } @@ -1074,7 +1074,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { if (pTmq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tscError("failed to create consumer, groupId:%s, code:%s", conf->groupId, terrstr()); - SET_ERROR_MSG("malloc tmq failed") + SET_ERROR_MSG_TMQ("malloc tmq failed") return NULL; } @@ -1090,7 +1090,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { conf->groupId[0] == 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); - SET_ERROR_MSG("malloc tmq element failed or group is empty") + SET_ERROR_MSG_TMQ("malloc tmq element failed or group is empty") goto _failed; } @@ -1123,7 +1123,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { if (tsem_init(&pTmq->rspSem, 0, 0) != 0) { tscError("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(), pTmq->groupId); - SET_ERROR_MSG("init t_sem failed") + SET_ERROR_MSG_TMQ("init t_sem failed") goto _failed; } @@ -1132,13 +1132,13 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { if (pTmq->pTscObj == NULL) { tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); tsem_destroy(&pTmq->rspSem); - SET_ERROR_MSG("init tscObj failed") + SET_ERROR_MSG_TMQ("init tscObj failed") goto _failed; } pTmq->refId = taosAddRef(tmqMgmt.rsetId, pTmq); if (pTmq->refId < 0) { - SET_ERROR_MSG("add tscObj ref failed") + SET_ERROR_MSG_TMQ("add tscObj ref failed") goto _failed; } diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 7e99867b2a..6cd10f8a1f 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -641,7 +641,7 @@ static bool findFileds(SSchema* pSchema, TAOS_FIELD* fields, int numFields) { } int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq** pCreateTb, TAOS_FIELD* tFields, - int numFields, bool needChangeLength) { + int numFields, bool needChangeLength, char* errstr, int32_t errstrLen) { void* tmp = taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid)); STableDataCxt* pTableCxt = NULL; @@ -662,8 +662,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate } char* p = (char*)data; - // | version | total length | total rows | blankFill | total columns | flag seg| block group id | column schema | each - // column length | + // | version | total length | total rows | blankFill | total columns | flag seg| block group id | column schema | each column length | int32_t version = *(int32_t*)data; p += sizeof(int32_t); p += sizeof(int32_t); @@ -689,12 +688,12 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate SBoundColInfo* boundInfo = &pTableCxt->boundColsInfo; if (tFields != NULL && numFields != numOfCols) { - uError("numFields:%d != numOfCols:%d", numFields, numOfCols); + if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d != raw numOfCols:%d", numFields, numOfCols); ret = TSDB_CODE_INVALID_PARA; goto end; } if (tFields != NULL && numFields > boundInfo->numOfBound) { - uError("numFields:%d > boundInfo->numOfBound:%d", numFields, boundInfo->numOfBound); + if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d > boundInfo->numOfBound:%d", numFields, boundInfo->numOfBound); ret = TSDB_CODE_INVALID_PARA; goto end; } @@ -703,7 +702,8 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate SSchema* pColSchema = &pSchema[j]; SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, j); if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { - uError("type or bytes not equal"); + if (errstr != NULL) snprintf(errstr, errstrLen, "type or bytes not equal, id:%d, type:%d, raw type:%d. bytes:%d, raw bytes:%d", + pColSchema->colId, pColSchema->type, *fields, pColSchema->bytes, *(int32_t*)(fields + sizeof(int8_t))); ret = TSDB_CODE_INVALID_PARA; goto end; } @@ -732,7 +732,8 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate SSchema* pColSchema = &pSchema[j]; if (strcmp(pColSchema->name, tFields[i].name) == 0) { if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { - uError("type or bytes not equal"); + if (errstr != NULL) snprintf(errstr, errstrLen, "type or bytes not equal, id:%d, type:%d, raw type:%d. bytes:%d, raw bytes:%d", + pColSchema->colId, pColSchema->type, *fields, pColSchema->bytes, *(int32_t*)(fields + sizeof(int8_t))); ret = TSDB_CODE_INVALID_PARA; goto end; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 5947652096..c373d29236 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -27,8 +27,11 @@ typedef struct { } STaosError; static threadlocal int32_t tsErrno; +static threadlocal char tsErrMsgDetail[ERR_MSG_LEN] = {0}; +static threadlocal char tsErrMsgReturn[ERR_MSG_LEN] = {0}; int32_t* taosGetErrno() { return &tsErrno; } +char* taosGetErrMsg() { return tsErrMsgDetail; } #ifdef TAOS_ERROR_C #define TAOS_DEFINE_ERROR(name, msg) {.val = (name), .str = (msg)}, @@ -793,7 +796,13 @@ const char* tstrerror(int32_t err) { } else if (err < val) { e = mid; } else if (err == val) { - return errors[mid].str; + if(strlen(tsErrMsgDetail) == 0){ + return errors[mid].str; + } else{ + memset(tsErrMsgReturn, 0, ERR_MSG_LEN); + snprintf(tsErrMsgReturn, ERR_MSG_LEN, "%s,detail:%s", errors[mid].str, tsErrMsgDetail); + return (const char*)tsErrMsgReturn; + } } else { break; } diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 6d99694df0..d5ea2b1ce7 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -1000,6 +1000,17 @@ void testConsumeExcluded(int topic_type){ } taos_free_result(pRes); } + +void testDetailError(){ + tmq_raw_data raw = {0}; + raw.raw_type = 2; + int32_t code = tmq_write_raw((TAOS *)1, raw); + ASSERT(code); + const char *err = taos_errstr(NULL); + code = strcmp("Invalid parameters,detail:taos:0x1 or data:0x0 is NULL", err); + ASSERT(code == 0); +} + int main(int argc, char* argv[]) { for (int32_t i = 1; i < argc; i++) { if (strcmp(argv[i], "-c") == 0) { @@ -1036,5 +1047,6 @@ int main(int argc, char* argv[]) { testConsumeExcluded(1); testConsumeExcluded(2); + testDetailError(); taosCloseFile(&g_fp); } From fb114b99a00e23138e8cd1a8e9df9c2800a0e31c Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Fri, 19 Apr 2024 04:58:44 +0000 Subject: [PATCH 11/21] fix compile err --- contrib/CMakeLists.txt | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index bb18516ba4..ccd60df19a 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -109,6 +109,13 @@ cat("${TD_SUPPORT_DIR}/zlib_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) # cJson cat("${TD_SUPPORT_DIR}/cjson_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +# xz +#cat("${TD_SUPPORT_DIR}/xz_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + +#lzma2 +cat("${TD_SUPPORT_DIR}/lzma_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + + if (${BUILD_CONTRIB}) if(${BUILD_WITH_ROCKSDB}) cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -127,6 +134,8 @@ else() endif() endif() +#cat("${TD_SUPPORT_DIR}/zstd_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + #libuv if(${BUILD_WITH_UV}) cat("${TD_SUPPORT_DIR}/libuv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -254,6 +263,13 @@ target_include_directories( ) unset(CMAKE_PROJECT_INCLUDE_BEFORE) +# add_subdirectory(xz EXCLUDE_FROM_ALL) +# target_include_directories( +# xz +# PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/xz +# PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/xz +# ) + # leveldb if(${BUILD_WITH_LEVELDB}) option(LEVELDB_BUILD_TESTS "" OFF) @@ -388,7 +404,6 @@ endif() if(${BUILD_WITH_S3}) INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.2/include) MESSAGE("build with s3: ${BUILD_WITH_S3}") - # cos elseif(${BUILD_WITH_COS}) if(${TD_LINUX}) @@ -427,6 +442,7 @@ if(${BUILD_PTHREAD}) target_link_libraries(pthread INTERFACE libpthreadVC3) endif() + # jemalloc if(${JEMALLOC_ENABLED}) include(ExternalProject) From 35453772eb8aeb8bac94064b97fbb786d610b1e6 Mon Sep 17 00:00:00 2001 From: Chris Zhai Date: Fri, 19 Apr 2024 13:33:38 +0800 Subject: [PATCH 12/21] update composite_primary_key_create --- tests/system-test/1-insert/composite_primary_key_create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/composite_primary_key_create.py b/tests/system-test/1-insert/composite_primary_key_create.py index 503aa6df7d..cafedc16ac 100644 --- a/tests/system-test/1-insert/composite_primary_key_create.py +++ b/tests/system-test/1-insert/composite_primary_key_create.py @@ -47,7 +47,7 @@ class IllegalSpell(Enum): CASE4 = 'primary ky' CASE5 = 'primarykey' CASE6 = 'key primary' - CASE6 = 'primay key primay key' + CASE7 = 'primay key primay key' class TableType(Enum): From 42467c0e00dd8abd036beca0ae30e9bcbd775820 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 Apr 2024 13:48:17 +0800 Subject: [PATCH 13/21] opti:raw data from tmq --- include/common/tmsg.h | 37 +++--- source/client/inc/clientInt.h | 18 ++- source/client/src/clientMain.c | 6 +- source/client/src/clientRawBlockWrite.c | 138 +++++++++++++------- source/client/src/clientTmq.c | 149 ++++++++++------------ source/common/src/tmsg.c | 52 ++++---- source/dnode/mnode/impl/src/mndConsumer.c | 1 - source/dnode/vnode/src/inc/tq.h | 8 +- source/dnode/vnode/src/tq/tq.c | 30 ++--- source/dnode/vnode/src/tq/tqScan.c | 50 ++++---- source/dnode/vnode/src/tq/tqUtil.c | 52 ++++---- 11 files changed, 279 insertions(+), 262 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index ff6e4de128..3d46342d6a 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3807,6 +3807,7 @@ int32_t tDecodeMqMetaRsp(SDecoder* pDecoder, SMqMetaRsp* pRsp); void tDeleteMqMetaRsp(SMqMetaRsp *pRsp); #define MQ_DATA_RSP_VERSION 100 + typedef struct { SMqRspHead head; STqOffsetVal reqOffset; @@ -3818,33 +3819,27 @@ typedef struct { SArray* blockData; SArray* blockTbName; SArray* blockSchema; - int64_t sleepTime; +} SMqDataRspCommon; + +typedef struct { + SMqDataRspCommon common; + int64_t sleepTime; } SMqDataRsp; -int32_t tEncodeMqDataRsp(SEncoder* pEncoder, const SMqDataRsp* pRsp); -int32_t tDecodeMqDataRsp(SDecoder* pDecoder, SMqDataRsp* pRsp, int8_t dataVersion); -void tDeleteMqDataRsp(SMqDataRsp* pRsp); +int32_t tEncodeMqDataRsp(SEncoder* pEncoder, const void* pRsp); +int32_t tDecodeMqDataRsp(SDecoder* pDecoder, void* pRsp); +void tDeleteMqDataRsp(void* pRsp); typedef struct { - SMqRspHead head; - STqOffsetVal reqOffset; - STqOffsetVal rspOffset; - int32_t blockNum; - int8_t withTbName; - int8_t withSchema; - SArray* blockDataLen; - SArray* blockData; - SArray* blockTbName; - SArray* blockSchema; - // the following attributes are extended from SMqDataRsp - int32_t createTableNum; - SArray* createTableLen; - SArray* createTableReq; + SMqDataRspCommon common; + int32_t createTableNum; + SArray* createTableLen; + SArray* createTableReq; } STaosxRsp; -int32_t tEncodeSTaosxRsp(SEncoder* pEncoder, const STaosxRsp* pRsp); -int32_t tDecodeSTaosxRsp(SDecoder* pDecoder, STaosxRsp* pRsp, int8_t dateVersion); -void tDeleteSTaosxRsp(STaosxRsp* pRsp); +int32_t tEncodeSTaosxRsp(SEncoder* pEncoder, const void* pRsp); +int32_t tDecodeSTaosxRsp(SDecoder* pDecoder, void* pRsp); +void tDeleteSTaosxRsp(void* pRsp); typedef struct { SMqRspHead head; diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 6c3603b4e0..9507472df0 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -221,7 +221,11 @@ typedef struct { SSchemaWrapper schema; int32_t resIter; SReqResultInfo resInfo; - SMqDataRsp rsp; +} SMqRspObjCommon; + +typedef struct { + SMqRspObjCommon common; + SMqDataRsp rsp; } SMqRspObj; typedef struct { @@ -233,14 +237,8 @@ typedef struct { } SMqMetaRspObj; typedef struct { - int8_t resType; - char topic[TSDB_TOPIC_FNAME_LEN]; - char db[TSDB_DB_FNAME_LEN]; - int32_t vgId; - SSchemaWrapper schema; - int32_t resIter; - SReqResultInfo resInfo; - STaosxRsp rsp; + SMqRspObjCommon common; + STaosxRsp rsp; } SMqTaosxRspObj; typedef struct SReqRelInfo { @@ -320,7 +318,7 @@ int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols); static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) { SMqRspObj* msg = (SMqRspObj*)res; - return (SReqResultInfo*)&msg->resInfo; + return (SReqResultInfo*)&msg->common.resInfo; } SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 9464baea52..2ec98b3d67 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -343,12 +343,12 @@ void taos_free_result(TAOS_RES *res) { } else if (TD_RES_TMQ_METADATA(res)) { SMqTaosxRspObj *pRsp = (SMqTaosxRspObj *)res; tDeleteSTaosxRsp(&pRsp->rsp); - doFreeReqResultInfo(&pRsp->resInfo); + doFreeReqResultInfo(&pRsp->common.resInfo); taosMemoryFree(pRsp); } else if (TD_RES_TMQ(res)) { SMqRspObj *pRsp = (SMqRspObj *)res; tDeleteMqDataRsp(&pRsp->rsp); - doFreeReqResultInfo(&pRsp->resInfo); + doFreeReqResultInfo(&pRsp->common.resInfo); taosMemoryFree(pRsp); } else if (TD_RES_TMQ_META(res)) { SMqMetaRspObj *pRspObj = (SMqMetaRspObj *)res; @@ -417,7 +417,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { } else if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { SMqRspObj *msg = ((SMqRspObj *)res); SReqResultInfo *pResultInfo; - if (msg->resIter == -1) { + if (msg->common.resIter == -1) { pResultInfo = tmqGetNextResInfo(res, true); } else { pResultInfo = tmqGetCurResInfo(res); diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index d25a5332d7..ae60270d8b 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1623,11 +1623,16 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { uDebug(LOG_ID_TAG " write raw data, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen); pRequest->syncQuery = true; - rspObj.resIter = -1; - rspObj.resType = RES_TYPE__TMQ; + rspObj.common.resIter = -1; + rspObj.common.resType = RES_TYPE__TMQ; + int8_t dataVersion = *(int8_t*)data; + if (dataVersion >= MQ_DATA_RSP_VERSION){ + data += sizeof(int8_t) + sizeof(int32_t); + dataLen -= sizeof(int8_t) + sizeof(int32_t); + } tDecoderInit(&decoder, data, dataLen); - code = tDecodeMqDataRsp(&decoder, &rspObj.rsp, *(int8_t*)data); + code = tDecodeMqDataRsp(&decoder, &rspObj.rsp); if (code != 0) { code = TSDB_CODE_INVALID_MSG; goto end; @@ -1656,13 +1661,13 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { goto end; } pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - while (++rspObj.resIter < rspObj.rsp.blockNum) { - void* pRetrieve = taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter); - if (!rspObj.rsp.withSchema) { + while (++rspObj.common.resIter < rspObj.rsp.common.blockNum) { + void* pRetrieve = taosArrayGetP(rspObj.rsp.common.blockData, rspObj.common.resIter); + if (!rspObj.rsp.common.withSchema) { goto end; } - const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.blockTbName, rspObj.resIter); + const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.common.blockTbName, rspObj.common.resIter); if (!tbName) { code = TSDB_CODE_TMQ_INVALID_MSG; goto end; @@ -1693,7 +1698,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); } - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.blockSchema, rspObj.resIter); + SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.common.blockSchema, rspObj.common.resIter); TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD)); if (fields == NULL) { goto end; @@ -1752,11 +1757,17 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) } uDebug(LOG_ID_TAG " write raw metadata, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen); pRequest->syncQuery = true; - rspObj.resIter = -1; - rspObj.resType = RES_TYPE__TMQ_METADATA; + rspObj.common.resIter = -1; + rspObj.common.resType = RES_TYPE__TMQ_METADATA; + + int8_t dataVersion = *(int8_t*)data; + if (dataVersion >= MQ_DATA_RSP_VERSION){ + data += sizeof(int8_t) + sizeof(int32_t); + dataLen -= sizeof(int8_t) + sizeof(int32_t); + } tDecoderInit(&decoder, data, dataLen); - code = tDecodeSTaosxRsp(&decoder, &rspObj.rsp, *(int8_t*)data); + code = tDecodeSTaosxRsp(&decoder, &rspObj.rsp); if (code != 0) { code = TSDB_CODE_INVALID_MSG; goto end; @@ -1786,14 +1797,14 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) } pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.blockNum); - while (++rspObj.resIter < rspObj.rsp.blockNum) { - void* pRetrieve = taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter); - if (!rspObj.rsp.withSchema) { + uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.common.blockNum); + while (++rspObj.common.resIter < rspObj.rsp.common.blockNum) { + void* pRetrieve = taosArrayGetP(rspObj.rsp.common.blockData, rspObj.common.resIter); + if (!rspObj.rsp.common.withSchema) { goto end; } - const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.blockTbName, rspObj.resIter); + const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.common.blockTbName, rspObj.common.resIter); if (!tbName) { code = TSDB_CODE_TMQ_INVALID_MSG; goto end; @@ -1865,7 +1876,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); } - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.blockSchema, rspObj.resIter); + SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.common.blockSchema, rspObj.common.resIter); TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD)); if (fields == NULL) { goto end; @@ -1944,6 +1955,64 @@ char* tmq_get_json_meta(TAOS_RES* res) { void tmq_free_json_meta(char* jsonMeta) { taosMemoryFreeClear(jsonMeta); } +static int32_t getOffSetLen(const void *rsp){ + const SMqDataRspCommon *pRsp = rsp; + SEncoder coder = {0}; + tEncoderInit(&coder, NULL, 0); + if (tEncodeSTqOffsetVal(&coder, &pRsp->reqOffset) < 0) return -1; + if (tEncodeSTqOffsetVal(&coder, &pRsp->rspOffset) < 0) return -1; + int32_t pos = coder.pos; + tEncoderClear(&coder); + return pos; +} + +typedef int32_t __encode_func__(SEncoder *pEncoder, const void *pRsp); + +static int32_t encodeMqDataRsp(__encode_func__* encodeFunc, void* rspObj, tmq_raw_data* raw){ + int32_t len = 0; + int32_t code = 0; + SEncoder encoder = {0}; + void* buf = NULL; + tEncodeSize(encodeFunc, rspObj, len, code); + if (code < 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto FAILED; + } + len += sizeof(int8_t) + sizeof(int32_t); + buf = taosMemoryCalloc(1, len); + if(buf == NULL){ + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto FAILED; + } + tEncoderInit(&encoder, buf, len); + if (tEncodeI8(&encoder, MQ_DATA_RSP_VERSION) < 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto FAILED; + } + int32_t offsetLen = getOffSetLen(rspObj); + if(offsetLen <= 0){ + terrno = TSDB_CODE_INVALID_MSG; + goto FAILED; + } + if (tEncodeI32(&encoder, offsetLen) < 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto FAILED; + } + if(encodeFunc(&encoder, rspObj) < 0){ + terrno = TSDB_CODE_INVALID_MSG; + goto FAILED; + } + tEncoderClear(&encoder); + + raw->raw = buf; + raw->raw_len = len; + return 0; +FAILED: + tEncoderClear(&encoder); + taosMemoryFree(buf); + return terrno; +} + int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { if (!raw || !res) { terrno = TSDB_CODE_INVALID_PARA; @@ -1957,42 +2026,19 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { uDebug("tmq get raw type meta:%p", raw); } else if (TD_RES_TMQ(res)) { SMqRspObj* rspObj = ((SMqRspObj*)res); - - int32_t len = 0; - int32_t code = 0; - tEncodeSize(tEncodeMqDataRsp, &rspObj->rsp, len, code); - if (code < 0) { - return -1; + if(encodeMqDataRsp(tEncodeMqDataRsp, &rspObj->rsp, raw) != 0){ + uError("tmq get raw type error:%d", terrno); + return terrno; } - - void* buf = taosMemoryCalloc(1, len); - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, len); - tEncodeMqDataRsp(&encoder, &rspObj->rsp); - tEncoderClear(&encoder); - - raw->raw = buf; - raw->raw_len = len; raw->raw_type = RES_TYPE__TMQ; uDebug("tmq get raw type data:%p", raw); } else if (TD_RES_TMQ_METADATA(res)) { SMqTaosxRspObj* rspObj = ((SMqTaosxRspObj*)res); - int32_t len = 0; - int32_t code = 0; - tEncodeSize(tEncodeSTaosxRsp, &rspObj->rsp, len, code); - if (code < 0) { - return -1; + if(encodeMqDataRsp(tEncodeSTaosxRsp, &rspObj->rsp, raw) != 0){ + uError("tmq get raw type error:%d", terrno); + return terrno; } - - void* buf = taosMemoryCalloc(1, len); - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, len); - tEncodeSTaosxRsp(&encoder, &rspObj->rsp); - tEncoderClear(&encoder); - - raw->raw = buf; - raw->raw_len = len; raw->raw_type = RES_TYPE__TMQ_METADATA; uDebug("tmq get raw type metadata:%p", raw); } else { diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 4cac1febe5..d4b7f59d5c 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -622,9 +622,9 @@ static void asyncCommitFromResult(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_c if (TD_RES_TMQ(pRes)) { SMqRspObj* pRspObj = (SMqRspObj*)pRes; - pTopicName = pRspObj->topic; - vgId = pRspObj->vgId; - offsetVal = pRspObj->rsp.rspOffset; + pTopicName = pRspObj->common.topic; + vgId = pRspObj->common.vgId; + offsetVal = pRspObj->rsp.common.rspOffset; } else if (TD_RES_TMQ_META(pRes)) { SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)pRes; pTopicName = pMetaRspObj->topic; @@ -632,9 +632,9 @@ static void asyncCommitFromResult(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_c offsetVal = pMetaRspObj->metaRsp.rspOffset; } else if (TD_RES_TMQ_METADATA(pRes)) { SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)pRes; - pTopicName = pRspObj->topic; - vgId = pRspObj->vgId; - offsetVal = pRspObj->rsp.rspOffset; + pTopicName = pRspObj->common.topic; + vgId = pRspObj->common.vgId; + offsetVal = pRspObj->rsp.common.rspOffset; } else { code = TSDB_CODE_TMQ_INVALID_MSG; goto end; @@ -1377,7 +1377,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { if (rspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { SDecoder decoder; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); - if(tDecodeMqDataRsp(&decoder, &pRspWrapper->dataRsp, *(int8_t*)POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead))) < 0){ + if(tDecodeMqDataRsp(&decoder, &pRspWrapper->dataRsp) < 0){ tDecoderClear(&decoder); taosReleaseRef(tmqMgmt.rsetId, refId); code = TSDB_CODE_OUT_OF_MEMORY; @@ -1387,9 +1387,9 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead)); char buf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf, TSDB_OFFSET_LEN, &pRspWrapper->dataRsp.rspOffset); + tFormatOffset(buf, TSDB_OFFSET_LEN, &pRspWrapper->dataRsp.common.rspOffset); tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req ver:%" PRId64 ", rsp:%s type %d, reqId:0x%" PRIx64, - tmq->consumerId, vgId, pRspWrapper->dataRsp.reqOffset.version, buf, rspType, requestId); + tmq->consumerId, vgId, pRspWrapper->dataRsp.common.reqOffset.version, buf, rspType, requestId); } else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) { SDecoder decoder; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); @@ -1404,7 +1404,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { } else if (rspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { SDecoder decoder; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); - if(tDecodeSTaosxRsp(&decoder, &pRspWrapper->taosxRsp, *(int8_t*)POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead))) < 0){ + if(tDecodeSTaosxRsp(&decoder, &pRspWrapper->taosxRsp) < 0){ tDecoderClear(&decoder); taosReleaseRef(tmqMgmt.rsetId, refId); code = TSDB_CODE_OUT_OF_MEMORY; @@ -1598,6 +1598,9 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { SMqMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqMetaRspObj)); + if(pRspObj == NULL) { + return NULL; + } pRspObj->resType = RES_TYPE__TMQ_META; tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); @@ -1611,8 +1614,7 @@ SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { void changeByteEndian(char* pData){ char* p = pData; - // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | - // version: + // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | version: int32_t blockVersion = *(int32_t*)p; ASSERT(blockVersion == BLOCK_VERSION_1); *(int32_t*)p = BLOCK_VERSION_2; @@ -1649,7 +1651,7 @@ static void tmqGetRawDataRowsPrecisionFromRes(void *pRetrieve, void** rawData, i } } -static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqRspObj* pRspObj) { +static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqRspObjCommon* pRspObj, SMqDataRspCommon* pDataRsp) { (*numOfRows) = 0; tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); @@ -1660,14 +1662,14 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg pRspObj->resInfo.totalRows = 0; pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; - bool needTransformSchema = !pRspObj->rsp.withSchema; - if (!pRspObj->rsp.withSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable - pRspObj->rsp.withSchema = true; - pRspObj->rsp.blockSchema = taosArrayInit(pRspObj->rsp.blockNum, sizeof(void*)); + bool needTransformSchema = !pDataRsp->withSchema; + if (!pDataRsp->withSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable + pDataRsp->withSchema = true; + pDataRsp->blockSchema = taosArrayInit(pDataRsp->blockNum, sizeof(void*)); } // extract the rows in this data packet - for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) { - void* pRetrieve = taosArrayGetP(pRspObj->rsp.blockData, i); + for (int32_t i = 0; i < pDataRsp->blockNum; ++i) { + void* pRetrieve = taosArrayGetP(pDataRsp->blockData, i); void* rawData = NULL; int64_t rows = 0; // deal with compatibility @@ -1679,7 +1681,7 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg if (needTransformSchema) { //withSchema is false if subscribe subquery, true if subscribe db or stable SSchemaWrapper *schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema); if(schema){ - taosArrayPush(pRspObj->rsp.blockSchema, &schema); + taosArrayPush(pDataRsp->blockSchema, &schema); } } } @@ -1687,18 +1689,24 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) { SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); - pRspObj->resType = RES_TYPE__TMQ; + if(pRspObj == NULL){ + return NULL; + } + pRspObj->common.resType = RES_TYPE__TMQ; memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp)); - tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, pRspObj); + tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, &pRspObj->common, &pRspObj->rsp.common); return pRspObj; } SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) { SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj)); - pRspObj->resType = RES_TYPE__TMQ_METADATA; + if(pRspObj == NULL){ + return NULL; + } + pRspObj->common.resType = RES_TYPE__TMQ_METADATA; memcpy(&pRspObj->rsp, &pWrapper->taosxRsp, sizeof(STaosxRsp)); - tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, (SMqRspObj*)pRspObj); + tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, &pRspObj->common, &pRspObj->rsp.common); return pRspObj; } @@ -1891,7 +1899,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; int32_t consumerEpoch = atomic_load_32(&tmq->epoch); - SMqDataRsp* pDataRsp = &pollRspWrapper->dataRsp; + SMqDataRspCommon* pDataRsp = (SMqDataRspCommon*)&pollRspWrapper->dataRsp; if (pDataRsp->head.epoch == consumerEpoch) { taosWLockLatch(&tmq->lock); @@ -1994,8 +2002,9 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; int32_t consumerEpoch = atomic_load_32(&tmq->epoch); + SMqDataRspCommon* pDataRsp = (SMqDataRspCommon*)&pollRspWrapper->taosxRsp; - if (pollRspWrapper->taosxRsp.head.epoch == consumerEpoch) { + if (pDataRsp->head.epoch == consumerEpoch) { taosWLockLatch(&tmq->lock); SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); pollRspWrapper->vgHandle = pVg; @@ -2009,11 +2018,11 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { return NULL; } - updateVgInfo(pVg, &pollRspWrapper->taosxRsp.reqOffset, &pollRspWrapper->taosxRsp.rspOffset, - pollRspWrapper->taosxRsp.head.walsver, pollRspWrapper->taosxRsp.head.walever, tmq->consumerId, - pollRspWrapper->taosxRsp.blockNum != 0); + updateVgInfo(pVg, &pDataRsp->reqOffset, &pDataRsp->rspOffset, + pDataRsp->head.walsver, pDataRsp->head.walever, tmq->consumerId, + pDataRsp->blockNum != 0); - if (pollRspWrapper->taosxRsp.blockNum == 0) { + if (pDataRsp->blockNum == 0) { tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ", reqId:0x%" PRIx64, tmq->consumerId, pVg->vgId, pVg->numOfRows, pollRspWrapper->reqId); pVg->emptyBlockReceiveTs = taosGetTimestampMs(); @@ -2034,7 +2043,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.endOffset); tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 ", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64, - tmq->consumerId, pVg->vgId, buf, pollRspWrapper->dataRsp.blockNum, numOfRows, pVg->numOfRows, + tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId); taosFreeQitem(pRspWrapper); @@ -2042,7 +2051,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { return pRsp; } else { tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", - tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->taosxRsp.head.epoch, consumerEpoch); + tmq->consumerId, pollRspWrapper->vgId, pDataRsp->head.epoch, consumerEpoch); setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); tmqFreeRspWrapper(pRspWrapper); taosFreeQitem(pRspWrapper); @@ -2205,15 +2214,11 @@ const char* tmq_get_topic_name(TAOS_RES* res) { if (res == NULL) { return NULL; } - if (TD_RES_TMQ(res)) { - SMqRspObj* pRspObj = (SMqRspObj*)res; - return strchr(pRspObj->topic, '.') + 1; + if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { + return strchr(((SMqRspObjCommon*)res)->topic, '.') + 1; } else if (TD_RES_TMQ_META(res)) { SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; return strchr(pMetaRspObj->topic, '.') + 1; - } else if (TD_RES_TMQ_METADATA(res)) { - SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)res; - return strchr(pRspObj->topic, '.') + 1; } else { return NULL; } @@ -2224,15 +2229,11 @@ const char* tmq_get_db_name(TAOS_RES* res) { return NULL; } - if (TD_RES_TMQ(res)) { - SMqRspObj* pRspObj = (SMqRspObj*)res; - return strchr(pRspObj->db, '.') + 1; + if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { + return strchr(((SMqRspObjCommon*)res)->db, '.') + 1; } else if (TD_RES_TMQ_META(res)) { SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; return strchr(pMetaRspObj->db, '.') + 1; - } else if (TD_RES_TMQ_METADATA(res)) { - SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)res; - return strchr(pRspObj->db, '.') + 1; } else { return NULL; } @@ -2242,15 +2243,11 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) { if (res == NULL) { return -1; } - if (TD_RES_TMQ(res)) { - SMqRspObj* pRspObj = (SMqRspObj*)res; - return pRspObj->vgId; + if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { + return ((SMqRspObjCommon*)res)->vgId; } else if (TD_RES_TMQ_META(res)) { SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; return pMetaRspObj->vgId; - } else if (TD_RES_TMQ_METADATA(res)) { - SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)res; - return pRspObj->vgId; } else { return -1; } @@ -2260,24 +2257,19 @@ int64_t tmq_get_vgroup_offset(TAOS_RES* res) { if (res == NULL) { return TSDB_CODE_INVALID_PARA; } - if (TD_RES_TMQ(res)) { - SMqRspObj* pRspObj = (SMqRspObj*)res; - STqOffsetVal* pOffset = &pRspObj->rsp.reqOffset; - if (pOffset->type == TMQ_OFFSET__LOG) { - return pRspObj->rsp.reqOffset.version; + if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { + SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon)); + STqOffsetVal* pOffset = &common->reqOffset; + if (common->reqOffset.type == TMQ_OFFSET__LOG) { + return common->reqOffset.version; } else { - tscError("invalid offset type:%d", pOffset->type); + tscError("invalid offset type:%d", common->reqOffset.type); } } else if (TD_RES_TMQ_META(res)) { SMqMetaRspObj* pRspObj = (SMqMetaRspObj*)res; if (pRspObj->metaRsp.rspOffset.type == TMQ_OFFSET__LOG) { return pRspObj->metaRsp.rspOffset.version; } - } else if (TD_RES_TMQ_METADATA(res)) { - SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)res; - if (pRspObj->rsp.reqOffset.type == TMQ_OFFSET__LOG) { - return pRspObj->rsp.reqOffset.version; - } } else { tscError("invalid tmq type:%d", *(int8_t*)res); } @@ -2290,20 +2282,15 @@ const char* tmq_get_table_name(TAOS_RES* res) { if (res == NULL) { return NULL; } - if (TD_RES_TMQ(res)) { - SMqRspObj* pRspObj = (SMqRspObj*)res; - if (!pRspObj->rsp.withTbName || pRspObj->rsp.blockTbName == NULL || pRspObj->resIter < 0 || - pRspObj->resIter >= pRspObj->rsp.blockNum) { + if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { + SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon)); + + SMqRspObjCommon* pRspObj = (SMqRspObjCommon*)res; + if (!common->withTbName || common->blockTbName == NULL || pRspObj->resIter < 0 || + pRspObj->resIter >= common->blockNum) { return NULL; } - return (const char*)taosArrayGetP(pRspObj->rsp.blockTbName, pRspObj->resIter); - } else if (TD_RES_TMQ_METADATA(res)) { - SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)res; - if (!pRspObj->rsp.withTbName || pRspObj->rsp.blockTbName == NULL || pRspObj->resIter < 0 || - pRspObj->resIter >= pRspObj->rsp.blockNum) { - return NULL; - } - return (const char*)taosArrayGetP(pRspObj->rsp.blockTbName, pRspObj->resIter); + return (const char*)taosArrayGetP(common->blockTbName, pRspObj->resIter); } return NULL; } @@ -2640,17 +2627,17 @@ void commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, cons } SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) { - SMqRspObj* pRspObj = (SMqRspObj*)res; + SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon)); + SMqRspObjCommon* pRspObj = (SMqRspObjCommon*)res; pRspObj->resIter++; - - if (pRspObj->resIter < pRspObj->rsp.blockNum) { - if (pRspObj->rsp.withSchema) { + if (pRspObj->resIter < common->blockNum) { + if (common->withSchema) { doFreeReqResultInfo(&pRspObj->resInfo); - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRspObj->rsp.blockSchema, pRspObj->resIter); + SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(common->blockSchema, pRspObj->resIter); setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols); } - void* pRetrieve = taosArrayGetP(pRspObj->rsp.blockData, pRspObj->resIter); + void* pRetrieve = taosArrayGetP(common->blockData, pRspObj->resIter); void* rawData = NULL; int64_t rows = 0; int32_t precision = 0; @@ -2685,14 +2672,14 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { SMqDataRsp rsp; SDecoder decoder; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); - tDecodeMqDataRsp(&decoder, &rsp, *(int8_t*)POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead))); + tDecodeMqDataRsp(&decoder, &rsp); tDecoderClear(&decoder); SMqRspHead* pHead = pMsg->pData; tmq_topic_assignment assignment = {.begin = pHead->walsver, .end = pHead->walever + 1, - .currentOffset = rsp.rspOffset.version, + .currentOffset = rsp.common.rspOffset.version, .vgId = pParam->vgId}; taosThreadMutexLock(&pCommon->mutex); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index a38b745db8..ae230d21aa 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -8897,7 +8897,7 @@ void tDeleteMqMetaRsp(SMqMetaRsp *pRsp) { taosMemoryFree(pRsp->metaRsp); } -int32_t tEncodeMqDataRspCommon(SEncoder *pEncoder, const SMqDataRsp *pRsp) { +int32_t tEncodeMqDataRspCommon(SEncoder *pEncoder, const SMqDataRspCommon *pRsp) { if (tEncodeSTqOffsetVal(pEncoder, &pRsp->reqOffset) < 0) return -1; if (tEncodeSTqOffsetVal(pEncoder, &pRsp->rspOffset) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->blockNum) < 0) return -1; @@ -8922,14 +8922,13 @@ int32_t tEncodeMqDataRspCommon(SEncoder *pEncoder, const SMqDataRsp *pRsp) { return 0; } -int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { - if (tEncodeI8(pEncoder, MQ_DATA_RSP_VERSION) < 0) return -1; +int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const void *pRsp) { if (tEncodeMqDataRspCommon(pEncoder, pRsp) < 0) return -1; - if (tEncodeI64(pEncoder, pRsp->sleepTime) < 0) return -1; + if (tEncodeI64(pEncoder, ((SMqDataRsp*)pRsp)->sleepTime) < 0) return -1; return 0; } -int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRsp *pRsp) { +int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRspCommon *pRsp) { if (tDecodeSTqOffsetVal(pDecoder, &pRsp->reqOffset) < 0) return -1; if (tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->blockNum) < 0) return -1; @@ -8975,19 +8974,17 @@ int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRsp *pRsp) { return 0; } -int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp, int8_t dataVersion) { - if (dataVersion >= MQ_DATA_RSP_VERSION){ - if (tDecodeI8(pDecoder, &dataVersion) < 0) return -1; - } +int32_t tDecodeMqDataRsp(SDecoder *pDecoder, void *pRsp) { if (tDecodeMqDataRspCommon(pDecoder, pRsp) < 0) return -1; if (!tDecodeIsEnd(pDecoder)) { - if (tDecodeI64(pDecoder, &pRsp->sleepTime) < 0) return -1; + if (tDecodeI64(pDecoder, &((SMqDataRsp*)pRsp)->sleepTime) < 0) return -1; } return 0; } -void tDeleteMqDataRsp(SMqDataRsp *pRsp) { +static void tDeleteMqDataRspCommon(void *rsp) { + SMqDataRspCommon *pRsp = rsp; pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen); taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree); pRsp->blockData = NULL; @@ -8999,10 +8996,14 @@ void tDeleteMqDataRsp(SMqDataRsp *pRsp) { tOffsetDestroy(&pRsp->rspOffset); } -int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) { - if (tEncodeI8(pEncoder, MQ_DATA_RSP_VERSION) < 0) return -1; - if (tEncodeMqDataRspCommon(pEncoder, (const SMqDataRsp *)pRsp) < 0) return -1; +void tDeleteMqDataRsp(void *rsp) { + tDeleteMqDataRspCommon(rsp); +} +int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const void *rsp) { + if (tEncodeMqDataRspCommon(pEncoder, rsp) < 0) return -1; + + const STaosxRsp *pRsp = (const STaosxRsp *)rsp; if (tEncodeI32(pEncoder, pRsp->createTableNum) < 0) return -1; if (pRsp->createTableNum) { for (int32_t i = 0; i < pRsp->createTableNum; i++) { @@ -9014,19 +9015,17 @@ int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) { return 0; } -int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp, int8_t dataVersion) { - if (dataVersion >= MQ_DATA_RSP_VERSION){ - if (tDecodeI8(pDecoder, &dataVersion) < 0) return -1; - } - if (tDecodeMqDataRspCommon(pDecoder, (SMqDataRsp*)pRsp) < 0) return -1; +int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, void *rsp) { + if (tDecodeMqDataRspCommon(pDecoder, rsp) < 0) return -1; + STaosxRsp *pRsp = (STaosxRsp *)rsp; if (tDecodeI32(pDecoder, &pRsp->createTableNum) < 0) return -1; if (pRsp->createTableNum) { pRsp->createTableLen = taosArrayInit(pRsp->createTableNum, sizeof(int32_t)); pRsp->createTableReq = taosArrayInit(pRsp->createTableNum, sizeof(void *)); for (int32_t i = 0; i < pRsp->createTableNum; i++) { void * pCreate = NULL; - uint64_t len; + uint64_t len = 0; if (tDecodeBinaryAlloc(pDecoder, &pCreate, &len) < 0) return -1; int32_t l = (int32_t)len; taosArrayPush(pRsp->createTableLen, &l); @@ -9037,20 +9036,13 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp, int8_t dataVersion return 0; } -void tDeleteSTaosxRsp(STaosxRsp *pRsp) { - pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen); - taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree); - pRsp->blockData = NULL; - taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSchemaWrapper); - pRsp->blockSchema = NULL; - taosArrayDestroyP(pRsp->blockTbName, (FDelete)taosMemoryFree); - pRsp->blockTbName = NULL; +void tDeleteSTaosxRsp(void *rsp) { + tDeleteMqDataRspCommon(rsp); + STaosxRsp *pRsp = (STaosxRsp *)rsp; pRsp->createTableLen = taosArrayDestroy(pRsp->createTableLen); taosArrayDestroyP(pRsp->createTableReq, (FDelete)taosMemoryFree); pRsp->createTableReq = NULL; - tOffsetDestroy(&pRsp->reqOffset); - tOffsetDestroy(&pRsp->rspOffset); } int32_t tEncodeSSingleDeleteReq(SEncoder *pEncoder, const SSingleDeleteReq *pReq) { diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index ed9333f480..c7ae941389 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -648,7 +648,6 @@ static SMqConsumerObj* buildSubConsumer(SMnode *pMnode, SCMSubscribeReq *subscri _over: mndReleaseConsumer(pMnode, pExistedConsumer); tDeleteSMqConsumerObj(pConsumerNew); - taosArrayDestroyP(subscribe->topicNames, (FDelete)taosMemoryFree); return NULL; } diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 2a076cfc61..bd8b73ed33 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -119,8 +119,8 @@ int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t // tqExec int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded); -int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision); -int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, +int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, void* pRsp, int32_t numOfCols, int8_t precision); +int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const void* pRsp, int32_t type, int32_t vgId); int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId); @@ -154,9 +154,9 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname); // tq util int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type); int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg); -int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, +int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const void* pRsp, int32_t epoch, int64_t consumerId, int32_t type, int64_t sver, int64_t ever); -int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset); +int32_t tqInitDataRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset); void tqUpdateNodeStage(STQ* pTq, bool isLeader); int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema* pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock, SSubmitTbData* pTableData, const char* id); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index b2b65b54cb..43308ce1f5 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -153,10 +153,10 @@ int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) { } SMqDataRsp dataRsp = {0}; - tqInitDataRsp(&dataRsp, req.reqOffset); - dataRsp.blockNum = 0; + tqInitDataRsp(&dataRsp.common, req.reqOffset); + dataRsp.common.blockNum = 0; char buf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.reqOffset); + tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.common.reqOffset); tqInfo("tqPushEmptyDataRsp to consumer:0x%" PRIx64 " vgId:%d, offset:%s, reqId:0x%" PRIx64, req.consumerId, vgId, buf, req.reqId); @@ -165,7 +165,7 @@ int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) { return 0; } -int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, +int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const void* pRsp, int32_t type, int32_t vgId) { int64_t sver = 0, ever = 0; walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); @@ -174,11 +174,11 @@ int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* char buf1[TSDB_OFFSET_LEN] = {0}; char buf2[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf1, TSDB_OFFSET_LEN, &pRsp->reqOffset); - tFormatOffset(buf2, TSDB_OFFSET_LEN, &pRsp->rspOffset); + tFormatOffset(buf1, TSDB_OFFSET_LEN, &((SMqDataRspCommon*)pRsp)->reqOffset); + tFormatOffset(buf2, TSDB_OFFSET_LEN, &((SMqDataRspCommon*)pRsp)->rspOffset); tqDebug("tmq poll vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s, reqId:0x%" PRIx64, - vgId, pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2, pReq->reqId); + vgId, pReq->consumerId, pReq->epoch, ((SMqDataRspCommon*)pRsp)->blockNum, buf1, buf2, pReq->reqId); return 0; } @@ -499,7 +499,7 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { taosRUnLockLatch(&pTq->lock); SMqDataRsp dataRsp = {0}; - tqInitDataRsp(&dataRsp, req.reqOffset); + tqInitDataRsp(&dataRsp.common, req.reqOffset); if (req.useSnapshot == true) { tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s snapshot not support wal info", consumerId, vgId, req.subKey); @@ -508,10 +508,10 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { return -1; } - dataRsp.rspOffset.type = TMQ_OFFSET__LOG; + dataRsp.common.rspOffset.type = TMQ_OFFSET__LOG; if (reqOffset.type == TMQ_OFFSET__LOG) { - dataRsp.rspOffset.version = reqOffset.version; + dataRsp.common.rspOffset.version = reqOffset.version; } else if (reqOffset.type < 0) { STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, req.subKey); if (pOffset != NULL) { @@ -522,17 +522,17 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { return -1; } - dataRsp.rspOffset.version = pOffset->val.version; + dataRsp.common.rspOffset.version = pOffset->val.version; tqInfo("consumer:0x%" PRIx64 " vgId:%d subkey:%s get assignment from store:%" PRId64, consumerId, vgId, - req.subKey, dataRsp.rspOffset.version); + req.subKey, dataRsp.common.rspOffset.version); } else { if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEST) { - dataRsp.rspOffset.version = sver; // not consume yet, set the earliest position + dataRsp.common.rspOffset.version = sver; // not consume yet, set the earliest position } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { - dataRsp.rspOffset.version = ever; + dataRsp.common.rspOffset.version = ever; } tqInfo("consumer:0x%" PRIx64 " vgId:%d subkey:%s get assignment from init:%" PRId64, consumerId, vgId, req.subKey, - dataRsp.rspOffset.version); + dataRsp.common.rspOffset.version); } } else { tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s invalid offset type:%d", consumerId, vgId, req.subKey, diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 017d5247d8..08f1689f2f 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -15,7 +15,7 @@ #include "tq.h" -int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision) { +int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, void* pRsp, int32_t numOfCols, int8_t precision) { int32_t dataStrLen = sizeof(SRetrieveTableRspForTmq) + blockGetEncodeSize(pBlock); void* buf = taosMemoryCalloc(1, dataStrLen); if (buf == NULL) { @@ -30,22 +30,22 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); actualLen += sizeof(SRetrieveTableRspForTmq); - taosArrayPush(pRsp->blockDataLen, &actualLen); - taosArrayPush(pRsp->blockData, &buf); + taosArrayPush(((SMqDataRspCommon*)pRsp)->blockDataLen, &actualLen); + taosArrayPush(((SMqDataRspCommon*)pRsp)->blockData, &buf); return TSDB_CODE_SUCCESS; } -static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, STaosxRsp* pRsp) { +static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, void* pRsp) { SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pTqReader->pSchemaWrapper); if (pSW == NULL) { return -1; } - taosArrayPush(pRsp->blockSchema, &pSW); + taosArrayPush(((SMqDataRspCommon*)pRsp)->blockSchema, &pSW); return 0; } -static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, int32_t n) { +static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, void* pRsp, int32_t n) { SMetaReader mr = {0}; metaReaderDoInit(&mr, pTq->pVnode->pMeta, META_READER_LOCK); @@ -57,7 +57,7 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, in for (int32_t i = 0; i < n; i++) { char* tbName = taosStrdup(mr.me.name); - taosArrayPush(pRsp->blockTbName, &tbName); + taosArrayPush(((SMqDataRspCommon*)pRsp)->blockTbName, &tbName); } metaReaderClear(&mr); return 0; @@ -125,7 +125,7 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* return code; } - pRsp->blockNum++; + pRsp->common.blockNum++; if (pDataBlock == NULL) { blockDataDestroy(pHandle->block); pHandle->block = NULL; @@ -149,7 +149,7 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* return code; } - pRsp->blockNum++; + pRsp->common.blockNum++; totalRows += pDataBlock->info.rows; if (totalRows >= tmqRowSize) { break; @@ -158,8 +158,8 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* } tqDebug("consumer:0x%" PRIx64 " vgId:%d tmq task executed finished, total blocks:%d, totalRows:%d", - pHandle->consumerId, vgId, pRsp->blockNum, totalRows); - qStreamExtractOffset(task, &pRsp->rspOffset); + pHandle->consumerId, vgId, pRsp->common.blockNum, totalRows); + qStreamExtractOffset(task, &pRsp->common.rspOffset); return 0; } @@ -186,7 +186,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta tqDebug("tmqsnap task execute end, get %p", pDataBlock); if (pDataBlock != NULL && pDataBlock->info.rows > 0) { - if (pRsp->withTbName) { + if (pRsp->common.withTbName) { if (pOffset->type == TMQ_OFFSET__LOG) { int64_t uid = pExec->pTqReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, pRsp, 1) < 0) { @@ -194,21 +194,21 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta } } else { char* tbName = taosStrdup(qExtractTbnameFromTask(task)); - taosArrayPush(pRsp->blockTbName, &tbName); + taosArrayPush(pRsp->common.blockTbName, &tbName); } } - if (pRsp->withSchema) { + if (pRsp->common.withSchema) { if (pOffset->type == TMQ_OFFSET__LOG) { tqAddBlockSchemaToRsp(pExec, pRsp); } else { SSchemaWrapper* pSW = tCloneSSchemaWrapper(qExtractSchemaFromTask(task)); - taosArrayPush(pRsp->blockSchema, &pSW); + taosArrayPush(pRsp->common.blockSchema, &pSW); } } tqAddBlockDataToRsp(pDataBlock, (SMqDataRsp*)pRsp, taosArrayGetSize(pDataBlock->pDataBlock), pTq->pVnode->config.tsdbCfg.precision); - pRsp->blockNum++; + pRsp->common.blockNum++; if (pOffset->type == TMQ_OFFSET__LOG) { continue; } else { @@ -234,13 +234,13 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta } tqDebug("tmqsnap vgId: %d, tsdb consume over, switch to wal, ver %" PRId64, TD_VID(pTq->pVnode), pHandle->snapshotVer + 1); - qStreamExtractOffset(task, &pRsp->rspOffset); + qStreamExtractOffset(task, &pRsp->common.rspOffset); break; } - if (pRsp->blockNum > 0) { + if (pRsp->common.blockNum > 0) { tqDebug("tmqsnap task exec exited, get data"); - qStreamExtractOffset(task, &pRsp->rspOffset); + qStreamExtractOffset(task, &pRsp->common.rspOffset); break; } } @@ -268,7 +268,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR if ((pSubmitTbDataRet->flags & sourceExcluded) != 0) { goto loop_table; } - if (pRsp->withTbName) { + if (pRsp->common.withTbName) { int64_t uid = pExec->pTqReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) { goto loop_table; @@ -312,8 +312,8 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR *totalRows += pBlock->info.rows; blockDataFreeRes(pBlock); SSchemaWrapper* pSW = taosArrayGetP(pSchemas, i); - taosArrayPush(pRsp->blockSchema, &pSW); - pRsp->blockNum++; + taosArrayPush(pRsp->common.blockSchema, &pSW); + pRsp->common.blockNum++; } continue; loop_table: @@ -336,7 +336,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR if ((pSubmitTbDataRet->flags & sourceExcluded) != 0) { goto loop_db; } - if (pRsp->withTbName) { + if (pRsp->common.withTbName) { int64_t uid = pExec->pTqReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) { goto loop_db; @@ -380,8 +380,8 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR *totalRows += pBlock->info.rows; blockDataFreeRes(pBlock); SSchemaWrapper* pSW = taosArrayGetP(pSchemas, i); - taosArrayPush(pRsp->blockSchema, &pSW); - pRsp->blockNum++; + taosArrayPush(pRsp->common.blockSchema, &pSW); + pRsp->common.blockNum++; } continue; loop_db: diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 5e5c77265b..a5bb01f6c0 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -18,7 +18,7 @@ static int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp, int32_t vgId); -int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) { +int32_t tqInitDataRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset) { tOffsetCopy(&pRsp->reqOffset, &pOffset); tOffsetCopy(&pRsp->rspOffset, &pOffset); @@ -39,7 +39,7 @@ void tqUpdateNodeStage(STQ* pTq, bool isLeader) { streamMetaUpdateStageRole(pTq->pStreamMeta, state.term, isLeader); } -static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, STqOffsetVal pOffset) { +static int32_t tqInitTaosxRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset) { tOffsetCopy(&pRsp->reqOffset, &pOffset); tOffsetCopy(&pRsp->rspOffset, &pOffset); @@ -110,9 +110,9 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand SMqDataRsp dataRsp = {0}; tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer + 1); - tqInitDataRsp(&dataRsp, *pOffsetVal); + tqInitDataRsp(&dataRsp.common, *pOffsetVal); tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId, - pHandle->subKey, vgId, dataRsp.rspOffset.version); + pHandle->subKey, vgId, dataRsp.common.rspOffset.version); int32_t code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); tDeleteMqDataRsp(&dataRsp); @@ -137,7 +137,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, terrno = 0; SMqDataRsp dataRsp = {0}; - tqInitDataRsp(&dataRsp, *pOffset); + tqInitDataRsp(&dataRsp.common, *pOffset); qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); int code = tqScanData(pTq, pHandle, &dataRsp, pOffset, pRequest); @@ -146,11 +146,11 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, } // till now, all data has been transferred to consumer, new data needs to push client once arrived. - if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST && dataRsp.blockNum == 0) { + if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST && dataRsp.common.blockNum == 0) { // lock taosWLockLatch(&pTq->lock); int64_t ver = walGetCommittedVer(pTq->pVnode->pWal); - if (dataRsp.rspOffset.version > ver) { // check if there are data again to avoid lost data + if (dataRsp.common.rspOffset.version > ver) { // check if there are data again to avoid lost data code = tqRegisterPushHandle(pTq, pHandle, pMsg); taosWUnLockLatch(&pTq->lock); goto end; @@ -158,15 +158,15 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, taosWUnLockLatch(&pTq->lock); } - tOffsetCopy(&dataRsp.reqOffset, pOffset); // reqOffset represents the current date offset, may be changed if wal not exists - code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); + tOffsetCopy(&dataRsp.common.reqOffset, pOffset); // reqOffset represents the current date offset, may be changed if wal not exists + code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); end : { char buf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.rspOffset); + tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.common.rspOffset); tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, reqId:0x%" PRIx64 " code:%d", - consumerId, pHandle->subKey, vgId, dataRsp.blockNum, buf, pRequest->reqId, code); + consumerId, pHandle->subKey, vgId, dataRsp.common.blockNum, buf, pRequest->reqId, code); tDeleteMqDataRsp(&dataRsp); return code; } @@ -194,7 +194,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, int32_t vgId = TD_VID(pTq->pVnode); SMqMetaRsp metaRsp = {0}; STaosxRsp taosxRsp = {0}; - tqInitTaosxRsp(&taosxRsp, *offset); + tqInitTaosxRsp(&taosxRsp.common, *offset); if (offset->type != TMQ_OFFSET__LOG) { if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) { @@ -214,13 +214,13 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, tqDebug("taosx poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send data blockNum:%d, offset type:%d,uid:%" PRId64 ",ts:%" PRId64, - pRequest->consumerId, pHandle->subKey, vgId, taosxRsp.blockNum, taosxRsp.rspOffset.type, - taosxRsp.rspOffset.uid, taosxRsp.rspOffset.ts); - if (taosxRsp.blockNum > 0) { - code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); + pRequest->consumerId, pHandle->subKey, vgId, taosxRsp.common.blockNum, taosxRsp.common.rspOffset.type, + taosxRsp.common.rspOffset.uid, taosxRsp.common.rspOffset.ts); + if (taosxRsp.common.blockNum > 0) { + code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); goto end; } else { - tOffsetCopy(offset, &taosxRsp.rspOffset); + tOffsetCopy(offset, &taosxRsp.common.rspOffset); } } @@ -235,9 +235,9 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, ASSERT(savedEpoch <= pRequest->epoch); if (tqFetchLog(pTq, pHandle, &fetchVer, pRequest->reqId) < 0) { - tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); + tqOffsetResetToLog(&taosxRsp.common.rspOffset, fetchVer); code = tqSendDataRsp( - pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, + pHandle, pMsg, pRequest, &taosxRsp, taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); goto end; } @@ -249,9 +249,9 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, // process meta if (pHead->msgType != TDMT_VND_SUBMIT) { if (totalRows > 0) { - tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); + tqOffsetResetToLog(&taosxRsp.common.rspOffset, fetchVer); code = tqSendDataRsp( - pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, + pHandle, pMsg, pRequest, &taosxRsp, taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); goto end; } @@ -292,9 +292,9 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, } if (totalRows >= tmqRowSize || (taosGetTimestampMs() - st > 1000)) { - tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer + 1); + tqOffsetResetToLog(&taosxRsp.common.rspOffset, fetchVer + 1); code = tqSendDataRsp( - pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, + pHandle, pMsg, pRequest, &taosxRsp, taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); goto end; } else { @@ -386,7 +386,7 @@ int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPoll return 0; } -int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, +int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const void* pRsp, int32_t epoch, int64_t consumerId, int32_t type, int64_t sver, int64_t ever) { int32_t len = 0; int32_t code = 0; @@ -394,7 +394,7 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* if (type == TMQ_MSG_TYPE__POLL_DATA_RSP || type == TMQ_MSG_TYPE__WALINFO_RSP) { tEncodeSize(tEncodeMqDataRsp, pRsp, len, code); } else if (type == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { - tEncodeSize(tEncodeSTaosxRsp, (STaosxRsp*)pRsp, len, code); + tEncodeSize(tEncodeSTaosxRsp, pRsp, len, code); } if (code < 0) { @@ -418,7 +418,7 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* if (type == TMQ_MSG_TYPE__POLL_DATA_RSP || type == TMQ_MSG_TYPE__WALINFO_RSP) { tEncodeMqDataRsp(&encoder, pRsp); } else if (type == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { - tEncodeSTaosxRsp(&encoder, (STaosxRsp*)pRsp); + tEncodeSTaosxRsp(&encoder, pRsp); } tEncoderClear(&encoder); From fe7bfc88c0276280990cbd43dc604591962e91db Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 Apr 2024 14:14:09 +0800 Subject: [PATCH 14/21] fix:ci error --- utils/test/c/tmq_taosx_ci.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index d5ea2b1ce7..d4300adaf5 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -1007,8 +1007,8 @@ void testDetailError(){ int32_t code = tmq_write_raw((TAOS *)1, raw); ASSERT(code); const char *err = taos_errstr(NULL); - code = strcmp("Invalid parameters,detail:taos:0x1 or data:0x0 is NULL", err); - ASSERT(code == 0); + char* tmp = strstr(err, "Invalid parameters,detail:taos:0x1 or data"); + ASSERT(tmp != NULL); } int main(int argc, char* argv[]) { From 32f6d525e61861329b8a918156be73f2df8f5bd1 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 Apr 2024 15:27:56 +0800 Subject: [PATCH 15/21] fix:compile error in windows & wait for heatbeat in case --- source/client/src/clientRawBlockWrite.c | 4 ++-- tests/system-test/7-tmq/tmq_primary_key.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index ec748d0270..4188d84573 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1630,7 +1630,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { int8_t dataVersion = *(int8_t*)data; if (dataVersion >= MQ_DATA_RSP_VERSION){ - data += sizeof(int8_t) + sizeof(int32_t); + data = POINTER_SHIFT(data, sizeof(int8_t) + sizeof(int32_t)); dataLen -= sizeof(int8_t) + sizeof(int32_t); } tDecoderInit(&decoder, data, dataLen); @@ -1764,7 +1764,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) int8_t dataVersion = *(int8_t*)data; if (dataVersion >= MQ_DATA_RSP_VERSION){ - data += sizeof(int8_t) + sizeof(int32_t); + data = POINTER_SHIFT(data, sizeof(int8_t) + sizeof(int32_t)); dataLen -= sizeof(int8_t) + sizeof(int32_t); } diff --git a/tests/system-test/7-tmq/tmq_primary_key.py b/tests/system-test/7-tmq/tmq_primary_key.py index 44de5466a8..8f62dc8783 100644 --- a/tests/system-test/7-tmq/tmq_primary_key.py +++ b/tests/system-test/7-tmq/tmq_primary_key.py @@ -81,6 +81,8 @@ class TDTestCase: finally: consumer.close() + time.sleep(4) # wait for heart beat + tdSql.query(f'show subscriptions;') sub = tdSql.getData(0, 4); print(sub) @@ -190,6 +192,7 @@ class TDTestCase: finally: consumer.close() + time.sleep(4) # wait for heart beat tdSql.query(f'show subscriptions;') sub = tdSql.getData(0, 4); print(sub) @@ -298,6 +301,7 @@ class TDTestCase: finally: consumer.close() + time.sleep(4) # wait for heart beat tdSql.query(f'show subscriptions;') sub = tdSql.getData(0, 4); print(sub) @@ -406,6 +410,7 @@ class TDTestCase: finally: consumer.close() + time.sleep(4) # wait for heart beat tdSql.query(f'show subscriptions;') sub = tdSql.getData(0, 4); print(sub) From 3baa47f266419954ea66eee92e9f9e9d3c438a7b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 19 Apr 2024 16:21:05 +0800 Subject: [PATCH 16/21] fix(stream): record the task check downstream failed info. --- source/libs/stream/src/streamMeta.c | 9 +++- source/libs/stream/src/streamTask.c | 64 ++++++++++++++++------------- 2 files changed, 44 insertions(+), 29 deletions(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 4aecb6cc9d..888d6df171 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1288,7 +1288,7 @@ void streamMetaNotifyClose(SStreamMeta* pMeta) { } SStreamTask* pTask = *(SStreamTask**)pIter; - stDebug("vgId:%d s-task:%s set closing flag", vgId, pTask->id.idStr); + stDebug("vgId:%d s-task:%s set task closing flag", vgId, pTask->id.idStr); streamTaskStop(pTask); } @@ -1649,6 +1649,13 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int3 return 0; } + void* p = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); + if (p == NULL) { // task does not exists in current vnode, not record the complete info + qError("vgId:%d s-task:0x%x not exists discard the check downstream info", pMeta->vgId, taskId); + streamMetaWUnLock(pMeta); + return 0; + } + SHashObj* pDst = ready ? pStartInfo->pReadyTaskSet : pStartInfo->pFailedTaskSet; STaskInitTs initTs = {.start = startTs, .end = endTs, .success = ready}; diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 70c2619f6f..8ad24b8edd 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -1026,7 +1026,7 @@ int32_t streamTaskStartCheckDownstream(STaskCheckInfo* pInfo, const char* id) { } taosThreadMutexUnlock(&pInfo->checkInfoLock); - stDebug("s-task:%s set the in check procedure flag", id); + stDebug("s-task:%s set the in-check-procedure flag", id); return 0; } @@ -1038,7 +1038,7 @@ int32_t streamTaskCompleteCheck(STaskCheckInfo* pInfo, const char* id) { } int64_t el = taosGetTimestampMs() - pInfo->startTs; - stDebug("s-task:%s check downstream completed, elapsed time:%" PRId64 " ms", id, el); + stDebug("s-task:%s clear the in-check-procedure flag, elapsed time:%" PRId64 " ms", id, el); pInfo->startTs = 0; pInfo->inCheckProcess = 0; @@ -1098,27 +1098,36 @@ static void rspMonitorFn(void* param, void* tmrId) { int64_t now = taosGetTimestampMs(); int64_t el = now - pInfo->startTs; ETaskStatus state = pStat->state; - int32_t numOfReady = 0; int32_t numOfFault = 0; + const char* id = pTask->id.idStr; - stDebug("s-task:%s start to do check downstream rsp check", pTask->id.idStr); + stDebug("s-task:%s start to do check downstream rsp check", id); - if (state == TASK_STATUS__STOP || state == TASK_STATUS__DROPPING || state == TASK_STATUS__READY) { + if (state == TASK_STATUS__STOP) { int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s status:%s vgId:%d quit from monitor rsp tmr, ref:%d", pTask->id.idStr, pStat->name, vgId, ref); - streamTaskCompleteCheck(pInfo, pTask->id.idStr); + stDebug("s-task:%s status:%s vgId:%d quit from monitor check-rsp tmr, ref:%d", id, pStat->name, vgId, ref); + streamTaskCompleteCheck(pInfo, id); + + streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, pInfo->startTs, now, false); + return; + } + + if (state == TASK_STATUS__DROPPING || state == TASK_STATUS__READY) { + int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); + stDebug("s-task:%s status:%s vgId:%d quit from monitor check-rsp tmr, ref:%d", id, pStat->name, vgId, ref); + streamTaskCompleteCheck(pInfo, id); return; } taosThreadMutexLock(&pInfo->checkInfoLock); if (pInfo->notReadyTasks == 0) { int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s status:%s vgId:%d all downstream ready, quit from monitor rsp tmr, ref:%d", pTask->id.idStr, - pStat->name, vgId, ref); + stDebug("s-task:%s status:%s vgId:%d all downstream ready, quit from monitor rsp tmr, ref:%d", id, pStat->name, + vgId, ref); taosThreadMutexUnlock(&pInfo->checkInfoLock); - streamTaskCompleteCheck(pInfo, pTask->id.idStr); + streamTaskCompleteCheck(pInfo, id); return; } @@ -1131,13 +1140,12 @@ static void rspMonitorFn(void* param, void* tmrId) { if (p->status == TASK_DOWNSTREAM_READY) { numOfReady += 1; } else if (p->status == TASK_UPSTREAM_NEW_STAGE || p->status == TASK_DOWNSTREAM_NOT_LEADER) { - stDebug("s-task:%s recv status from downstream, task:0x%x, quit from check downstream tasks", pTask->id.idStr, - p->taskId); + stDebug("s-task:%s recv status from downstream, task:0x%x, quit from check downstream tasks", id, p->taskId); numOfFault += 1; - } else { // TASK_DOWNSTREAM_NOT_READY - if (p->rspTs == 0) { // not response yet + } else { // TASK_DOWNSTREAM_NOT_READY + if (p->rspTs == 0) { // not response yet ASSERT(p->status == -1); - if (el >= CHECK_NOT_RSP_DURATION) { // not receive info for 10 sec. + if (el >= CHECK_NOT_RSP_DURATION) { // not receive info for 10 sec. taosArrayPush(pTimeoutList, &p->taskId); } else { // el < CHECK_NOT_RSP_DURATION // do nothing and continue waiting for their rsps @@ -1148,25 +1156,26 @@ static void rspMonitorFn(void* param, void* tmrId) { } } } else { // unexpected status - stError("s-task:%s unexpected task status:%s during waiting for check rsp", pTask->id.idStr, pStat->name); + stError("s-task:%s unexpected task status:%s during waiting for check rsp", id, pStat->name); } int32_t numOfNotReady = (int32_t)taosArrayGetSize(pNotReadyList); int32_t numOfTimeout = (int32_t)taosArrayGetSize(pTimeoutList); // fault tasks detected, not try anymore - if (((numOfReady + numOfFault + numOfNotReady + numOfTimeout) == taosArrayGetSize(pInfo->pList)) && (numOfFault > 0)) { + if (((numOfReady + numOfFault + numOfNotReady + numOfTimeout) == taosArrayGetSize(pInfo->pList)) && + (numOfFault > 0)) { int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); stDebug( "s-task:%s status:%s vgId:%d all rsp. quit from monitor rsp tmr, since vnode-transfer/leader-change/restart " - "detected, ref:%d", - pTask->id.idStr, pStat->name, vgId, ref); + "detected, notReady:%d, fault:%d, timeout:%d, ready:%d ref:%d", + id, pStat->name, vgId, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref); taosThreadMutexUnlock(&pInfo->checkInfoLock); taosArrayDestroy(pNotReadyList); taosArrayDestroy(pTimeoutList); - streamTaskCompleteCheck(pInfo, pTask->id.idStr); + streamTaskCompleteCheck(pInfo, id); return; } @@ -1176,14 +1185,14 @@ static void rspMonitorFn(void* param, void* tmrId) { stDebug( "s-task:%s status:%s vgId:%d stopped by other threads to check downstream process, notReady:%d, fault:%d, " "timeout:%d, ready:%d ref:%d", - pTask->id.idStr, pStat->name, vgId, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref); + id, pStat->name, vgId, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref); taosThreadMutexUnlock(&pInfo->checkInfoLock); // add the not-ready tasks into the final task status result buf, along with related fill-history task if exists. streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, pInfo->startTs, now, false); if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - SHistoryTaskInfo* pHTaskInfo = &pTask->hTaskInfo; - streamMetaAddTaskLaunchResult(pTask->pMeta, pHTaskInfo->id.streamId, pHTaskInfo->id.taskId, pInfo->startTs, now, false); + STaskId* pHId = &pTask->hTaskInfo.id; + streamMetaAddTaskLaunchResult(pTask->pMeta, pHId->streamId, pHId->taskId, pInfo->startTs, now, false); } return; } @@ -1205,7 +1214,7 @@ static void rspMonitorFn(void* param, void* tmrId) { } } - stDebug("s-task:%s %d downstream task(s) not ready, send check msg again", pTask->id.idStr, numOfNotReady); + stDebug("s-task:%s %d downstream task(s) not ready, send check msg again", id, numOfNotReady); } if (numOfTimeout > 0) { @@ -1225,15 +1234,14 @@ static void rspMonitorFn(void* param, void* tmrId) { } } - stDebug("s-task:%s %d downstream tasks timeout, send check msg again, start ts:%" PRId64, pTask->id.idStr, - numOfTimeout, now); + stDebug("s-task:%s %d downstream tasks timeout, send check msg again, start ts:%" PRId64, id, numOfTimeout, now); } taosThreadMutexUnlock(&pInfo->checkInfoLock); taosTmrReset(rspMonitorFn, CHECK_RSP_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr); - stDebug("s-task:%s continue checking rsp in 200ms, notReady:%d, fault:%d, timeout:%d, ready:%d", pTask->id.idStr, - numOfNotReady, numOfFault, numOfTimeout, numOfReady); + stDebug("s-task:%s continue checking rsp in 200ms, notReady:%d, fault:%d, timeout:%d, ready:%d", id, numOfNotReady, + numOfFault, numOfTimeout, numOfReady); taosArrayDestroy(pNotReadyList); taosArrayDestroy(pTimeoutList); From 16065e2a2d3f72d08fe2c9868b136027fb0b224b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 Apr 2024 16:36:11 +0800 Subject: [PATCH 17/21] fix:move err msg to tmq interface --- include/util/taoserror.h | 1 + source/client/src/clientTmq.c | 7 ++++++- source/util/src/terror.c | 9 ++------- utils/test/c/tmq_taosx_ci.c | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index fe6f3f53e9..48aeb4a073 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -35,6 +35,7 @@ const char* terrstr(); #define ERR_MSG_LEN 256 +char* taosGetErrMsgReturn(); char* taosGetErrMsg(); int32_t* taosGetErrno(); #define terrno (*taosGetErrno()) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index df5f8e2e00..5015399ab9 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -2182,7 +2182,12 @@ const char* tmq_err2str(int32_t err) { } else if (err == -1) { return "fail"; } else { - return tstrerror(err); + if(*(taosGetErrMsg()) == 0){ + return tstrerror(err); + } else{ + snprintf(taosGetErrMsgReturn(), ERR_MSG_LEN, "%s,detail:%s", tstrerror(err), taosGetErrMsg()); + return (const char*)taosGetErrMsgReturn(); + } } } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index c373d29236..94f660bc29 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -32,6 +32,7 @@ static threadlocal char tsErrMsgReturn[ERR_MSG_LEN] = {0}; int32_t* taosGetErrno() { return &tsErrno; } char* taosGetErrMsg() { return tsErrMsgDetail; } +char* taosGetErrMsgReturn() { return tsErrMsgReturn; } #ifdef TAOS_ERROR_C #define TAOS_DEFINE_ERROR(name, msg) {.val = (name), .str = (msg)}, @@ -796,13 +797,7 @@ const char* tstrerror(int32_t err) { } else if (err < val) { e = mid; } else if (err == val) { - if(strlen(tsErrMsgDetail) == 0){ - return errors[mid].str; - } else{ - memset(tsErrMsgReturn, 0, ERR_MSG_LEN); - snprintf(tsErrMsgReturn, ERR_MSG_LEN, "%s,detail:%s", errors[mid].str, tsErrMsgDetail); - return (const char*)tsErrMsgReturn; - } + return errors[mid].str; } else { break; } diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index d4300adaf5..8e9a67eb41 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -1006,7 +1006,7 @@ void testDetailError(){ raw.raw_type = 2; int32_t code = tmq_write_raw((TAOS *)1, raw); ASSERT(code); - const char *err = taos_errstr(NULL); + const char *err = tmq_err2str(code); char* tmp = strstr(err, "Invalid parameters,detail:taos:0x1 or data"); ASSERT(tmp != NULL); } From 5da1fffa996fbf73d8ca4b53989330d8078a45f8 Mon Sep 17 00:00:00 2001 From: Chris Zhai Date: Fri, 19 Apr 2024 16:41:53 +0800 Subject: [PATCH 18/21] update cases --- .../system-test/1-insert/composite_primary_key_create.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/system-test/1-insert/composite_primary_key_create.py b/tests/system-test/1-insert/composite_primary_key_create.py index cafedc16ac..cde960739c 100644 --- a/tests/system-test/1-insert/composite_primary_key_create.py +++ b/tests/system-test/1-insert/composite_primary_key_create.py @@ -78,12 +78,14 @@ class TDTestCase: d_type = LegalDataType.VARCHAR if t_type == TableType.SUPERTABLE: - expected_value = f'CREATE STABLE `{table_name}` (`ts` TIMESTAMP, `pk` {d_type.value} PRIMARY KEY, `c2` INT) TAGS (`engine` INT)' + expected_value = f"CREATE STABLE `{table_name}` (`ts` TIMESTAMP ENCODE 'delta-i' COMPRESS 'lz4' LEVEL 'medium', `pk` {d_type.value} ENCODE 'simple8b' COMPRESS 'lz4' LEVEL 'medium' PRIMARY KEY, `c2` INT ENCODE 'simple8b' COMPRESS 'lz4' LEVEL 'medium') TAGS (`engine` INT)" elif t_type == TableType.NORNALTABLE: - expected_value = f'CREATE TABLE `{table_name}` (`ts` TIMESTAMP, `pk` {d_type.value} PRIMARY KEY, `c2` INT)' + expected_value = f"CREATE TABLE `{table_name}` (`ts` TIMESTAMP ENCODE 'delta-i' COMPRESS 'lz4' LEVEL 'medium', `pk` {d_type.value} ENCODE 'simple8b' COMPRESS 'lz4' LEVEL 'medium' PRIMARY KEY, `c2` INT ENCODE 'simple8b' COMPRESS 'lz4' LEVEL 'medium')" tdSql.query(f"show create table {table_name}", show=SHOW_LOG) - tdSql.checkData(0, 1, expected_value) + result = tdSql.queryResult + + tdSql.checkEqual("PRIMARY KEY" in result[0][1], True) def test_pk_datatype_legal(self, stable_name: str, ctable_name: str, ntable_name: str, dtype: LegalDataType): # create super table and child table From b903872a4eb55d547046d11b6f6f3f820992630d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 19 Apr 2024 19:02:34 +0800 Subject: [PATCH 19/21] fix(stream): correct count down the ready tasks. --- source/libs/stream/src/streamTask.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 8ad24b8edd..7dc93ceccf 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -993,16 +993,17 @@ int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t SDownstreamStatusInfo* p = taosArrayGet(pInfo->pList, i); if (p->taskId == taskId) { ASSERT(reqId == p->reqId); - p->status = status; - p->rspTs = rspTs; // count down one, since it is ready now - if (p->status == TASK_DOWNSTREAM_READY) { + if ((p->status != TASK_DOWNSTREAM_READY) && (status == TASK_DOWNSTREAM_READY)) { *pNotReady = atomic_sub_fetch_32(&pInfo->notReadyTasks, 1); } else { *pNotReady = pInfo->notReadyTasks; } + p->status = status; + p->rspTs = rspTs; + taosThreadMutexUnlock(&pInfo->checkInfoLock); return TSDB_CODE_SUCCESS; } From ea527b19b852bd053dceb2e1492969854f04b5fc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 19 Apr 2024 19:12:23 +0800 Subject: [PATCH 20/21] fix(stream): set correct start version if no checkpoint exist. --- include/dnode/vnode/tqCommon.h | 1 + source/dnode/snode/src/snode.c | 12 +----------- source/dnode/vnode/src/tq/tq.c | 11 +---------- source/dnode/vnode/src/tqCommon/tqCommon.c | 16 ++++++++++++++++ 4 files changed, 19 insertions(+), 21 deletions(-) diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h index 93e0064192..cb616f7afc 100644 --- a/include/dnode/vnode/tqCommon.h +++ b/include/dnode/vnode/tqCommon.h @@ -41,5 +41,6 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg); int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* pMsg, bool fromVnode); int32_t tqExpandStreamTask(SStreamTask* pTask, SStreamMeta* pMeta, void* pVnode); +void tqSetRestoreVersionInfo(SStreamTask* pTask); #endif // TDENGINE_TQ_COMMON_H diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index f17716eda0..b717504e1e 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -58,17 +58,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer streamSetupScheduleTrigger(pTask); SCheckpointInfo *pChkInfo = &pTask->chkInfo; - - // checkpoint ver is the kept version, handled data should be the next version. - if (pChkInfo->checkpointId != 0) { - pChkInfo->nextProcessVer = pChkInfo->checkpointVer + 1; - pChkInfo->processedVer = pChkInfo->checkpointVer; - pTask->execInfo.startCheckpointVer = pChkInfo->nextProcessVer; - pTask->execInfo.startCheckpointId = pChkInfo->checkpointId; - - sndInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " nextProcessVer:%" PRId64, - pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); - } + tqSetRestoreVersionInfo(pTask); char* p = streamTaskGetStatus(pTask)->name; if (pTask->info.fillHistory) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 8edc0fed4d..21ca3290a0 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -760,16 +760,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) { streamSetupScheduleTrigger(pTask); SCheckpointInfo* pChkInfo = &pTask->chkInfo; - - // checkpoint ver is the kept version, handled data should be the next version. - if (pChkInfo->checkpointId != 0) { - pChkInfo->nextProcessVer = pChkInfo->checkpointVer + 1; - pChkInfo->processedVer = pChkInfo->checkpointVer; - pTask->execInfo.startCheckpointVer = pChkInfo->nextProcessVer; - pTask->execInfo.startCheckpointId = pChkInfo->checkpointId; - tqInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " currentVer:%" PRId64, pTask->id.idStr, - pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); - } + tqSetRestoreVersionInfo(pTask); char* p = streamTaskGetStatus(pTask)->name; const char* pNext = streamTaskGetStatusStr(pTask->status.taskStatus); diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index d2c7924cf5..4ce8579ea0 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -86,6 +86,22 @@ int32_t tqExpandStreamTask(SStreamTask* pTask, SStreamMeta* pMeta, void* pVnode) return TSDB_CODE_SUCCESS; } +void tqSetRestoreVersionInfo(SStreamTask* pTask) { + SCheckpointInfo* pChkInfo = &pTask->chkInfo; + + // checkpoint ver is the kept version, handled data should be the next version. + if (pChkInfo->checkpointId != 0) { + pChkInfo->nextProcessVer = pChkInfo->checkpointVer + 1; + pChkInfo->processedVer = pChkInfo->checkpointVer; + pTask->execInfo.startCheckpointId = pChkInfo->checkpointId; + + tqInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " currentVer:%" PRId64, pTask->id.idStr, + pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); + } + + pTask->execInfo.startCheckpointVer = pChkInfo->nextProcessVer; +} + int32_t tqStreamTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, bool restart) { int32_t vgId = pMeta->vgId; int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); From b75e77cb1570248c7644b6b88c3f68ba8ef3bb64 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 19 Apr 2024 19:13:07 +0800 Subject: [PATCH 21/21] refactor: do some internal refactor. --- source/libs/executor/src/cachescanoperator.c | 13 ++++++++----- source/libs/planner/src/planPhysiCreater.c | 3 ++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 23e873d335..115a61f647 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -108,6 +108,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe goto _error; } + // todd: the pk information should comes from the physical plan for(int32_t i = 0; i < taosArrayGetSize(pInfo->matchInfo.pList); ++i) { SColMatchItem* pItem = taosArrayGet(pInfo->matchInfo.pList, i); if (pItem->isPk) { @@ -223,8 +224,8 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { blockDataCleanup(pInfo->pBufferedRes); taosArrayClear(pInfo->pUidList); - int32_t code = pInfo->readHandle.api.cacheFn.retrieveRows(pInfo->pLastrowReader, pInfo->pBufferedRes, pInfo->pSlotIds, - pInfo->pDstSlotIds, pInfo->pUidList); + int32_t code = pInfo->readHandle.api.cacheFn.retrieveRows(pInfo->pLastrowReader, pInfo->pBufferedRes, + pInfo->pSlotIds, pInfo->pDstSlotIds, pInfo->pUidList); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -293,9 +294,11 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } if (NULL == pInfo->pLastrowReader) { - code = pInfo->readHandle.api.cacheFn.openReader(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num, - taosArrayGetSize(pInfo->matchInfo.pList), pInfo->pCidList, pInfo->pSlotIds, suid, &pInfo->pLastrowReader, - pTaskInfo->id.str, pInfo->pFuncTypeList, &pInfo->pkCol, pInfo->numOfPks); + code = pInfo->readHandle.api.cacheFn.openReader( + pInfo->readHandle.vnode, pInfo->retrieveType, pList, num, taosArrayGetSize(pInfo->matchInfo.pList), + pInfo->pCidList, pInfo->pSlotIds, suid, &pInfo->pLastrowReader, pTaskInfo->id.str, pInfo->pFuncTypeList, + &pInfo->pkCol, pInfo->numOfPks); + if (code != TSDB_CODE_SUCCESS) { pInfo->currentGroupIndex += 1; taosArrayClear(pInfo->pUidList); diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index e7e0be6d57..a824979801 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -570,13 +570,14 @@ static int32_t createLastRowScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSu if (pScanLogicNode->pVgroupList) { vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); } - int32_t code = createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); + int32_t code = createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); if (TSDB_CODE_SUCCESS == code && pScanLogicNode->pFuncTypes != NULL) { pScan->pFuncTypes = taosArrayInit(taosArrayGetSize(pScanLogicNode->pFuncTypes), sizeof(int32_t)); if (NULL == pScan->pFuncTypes) { return TSDB_CODE_OUT_OF_MEMORY; } + SNode* pTargetNode = NULL; int funcTypeIndex = 0; FOREACH(pTargetNode, ((SScanPhysiNode*)pScan)->pScanCols) {