From edba0063aab89f0d3b57738424c4d90b61c7c1e6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 27 Nov 2024 18:26:35 +0800 Subject: [PATCH 1/3] fix:[TD-33048] add ts to cols if dataFormat is true in schemaless to avoid schemal is old --- source/client/src/clientSml.c | 2 +- utils/test/c/sml_test.c | 81 +++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 8602421ed0..112fc8f5bc 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -233,7 +233,7 @@ int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSml goto END; } SML_CHECK_CODE(smlBuildSTableMeta(info->dataFormat, sMeta)); - for (int i = 1; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) { + for (int i = 0; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) { SSchema *col = pTableMeta->schema + i; SSmlKv kv = {.key = col->name, .keyLen = strlen(col->name), .type = col->type}; if (col->type == TSDB_DATA_TYPE_NCHAR) { diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index a8d4fafb03..bf04352232 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -2253,6 +2253,83 @@ int sml_ts5528_test(){ return 0; } +int sml_td33048_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "drop database if exists td33048"); + taos_free_result(pRes); + + pRes = taos_query(taos, "create database if not exists td33048"); + taos_free_result(pRes); + + // check column name duplication + const char *sql[] = { + "alarm_record,tag=alarm_record uid=\"3+8001+c939604c\",deviceId=\"3\",alarmId=\"8001\",alarmStatus=\"false\",lotNo=\"2411A0302\",subMode=\"11\",occurTime=\"2024-11-25 09:31:52.702\" 1732527117484", + }; + pRes = taos_query(taos, "use td33048"); + taos_free_result(pRes); + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + int code = taos_errno(pRes); + printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); + ASSERT(code == 0); + taos_free_result(pRes); + + // check tag name duplication + const char *sql1[] = { + "alarm_record,tag=alarm_record uid=\"2+100012+303fe9b5\",deviceId=\"2\",alarmId=\"100012\",alarmStatus=\"false\",lotNo=\"2411A0202\",subMode=\"11\",occurTime=\"2024-11-25 09:31:55.591\" 1732527119493", + }; + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + code = taos_errno(pRes); + printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); + ASSERT(code == 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from alarm_record"); + code = taos_errno(pRes); + printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); + ASSERT(code == 0); + taos_free_result(pRes); + + taos_close(taos); + + return code; +} + +int sml_td17324_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "drop database if exists gcbacaefqk"); + taos_free_result(pRes); + + pRes = taos_query(taos, "create database if not exists gcbacaefqk PRECISION 'ns'"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use gcbacaefqk"); + taos_free_result(pRes); + + pRes = taos_query(taos, "create stable gcbacaefqk.test_stb(_ts timestamp, f int) tags(t1 bigint)"); + taos_free_result(pRes); + // check column name duplication + const char *sql[] = { + "st123456,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1732700000364000000", + "st123456,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64 1732700000361000000", + "test_stb,t2=5f64,t3=L\"ste\" c1=true,c2=4i64,c3=\"iam\" 1732700000364316532" + }; + + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + int code = taos_errno(pRes); + printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); + ASSERT(code == 0); + taos_free_result(pRes); + + taos_close(taos); + + return code; +} + int main(int argc, char *argv[]) { if (argc == 2) { taos_options(TSDB_OPTION_CONFIGDIR, argv[1]); @@ -2262,6 +2339,10 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = sml_ts5528_test(); ASSERT(!ret); + ret = sml_td33048_Test(); + ASSERT(!ret); + ret = sml_td17324_Test(); + ASSERT(!ret); ret = sml_td29691_Test(); ASSERT(ret); ret = sml_td29373_Test(); From ce475566377131110b55c0441baa885dbc5c3add Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 27 Nov 2024 19:55:47 +0800 Subject: [PATCH 2/3] fix:[TD-33048] add ts to cols if dataFormat is true in schemaless to avoid schemal is old --- source/client/src/clientSml.c | 23 ++++++++++++++--------- tests/system-test/2-query/sml_TS-3724.py | 8 ++++---- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 112fc8f5bc..911e3664f5 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -772,22 +772,27 @@ END: RETURN } -static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) { +static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); SML_CHECK_NULL(hashTmp); - int32_t i = 0; - for (; i < length; i++) { - SML_CHECK_CODE(taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES)); + for (int32_t i = 0; i < length; i++) { + SML_CHECK_CODE(taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &schema[i], sizeof(SSchema))); } - i = isTag ? 0 : 1; - for (; i < taosArrayGetSize(cols); i++) { + for (int32_t i = 0; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); SML_CHECK_NULL(kv); - if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { + SSchema *sTmp = taosHashGet(hashTmp, kv->key, kv->keyLen); + if (sTmp == NULL) { SML_CHECK_CODE(TSDB_CODE_SML_INVALID_DATA); } + if ((IS_VAR_DATA_TYPE(kv->type) && kv->length + VARSTR_HEADER_SIZE > sTmp->bytes) || + (!IS_VAR_DATA_TYPE(kv->type) && kv->length != sTmp->bytes)){ + uError("column %s (type %s) bytes invalid. db bytes:%d, kv bytes:%zu", sTmp->name, + tDataTypes[sTmp->type].name, sTmp->bytes, kv->length); + SML_CHECK_CODE(TSDB_CODE_INTERNAL_ERROR); + } } END: @@ -1132,8 +1137,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { } if (needCheckMeta) { - SML_CHECK_CODE(smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags, sTableData->tags, true)); - SML_CHECK_CODE(smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols, false)); + SML_CHECK_CODE(smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags, sTableData->tags)); + SML_CHECK_CODE(smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols)); } taosMemoryFreeClear(sTableData->tableMeta); diff --git a/tests/system-test/2-query/sml_TS-3724.py b/tests/system-test/2-query/sml_TS-3724.py index b537ad9b9a..9511ee8aa1 100644 --- a/tests/system-test/2-query/sml_TS-3724.py +++ b/tests/system-test/2-query/sml_TS-3724.py @@ -14,7 +14,7 @@ sys.path.append("./7-tmq") from tmqCommon import * class TDTestCase: - updatecfgDict = {'clientCfg': {'smlChildTableName': 'dataModelName', 'fqdn': 'localhost', 'smlTsDefaultName': "times"}, 'fqdn': 'localhost'} + updatecfgDict = {'clientCfg': {'smlChildTableName': 'dataModelName', 'fqdn': 'localhost'}, 'fqdn': 'localhost'} print("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): @@ -58,7 +58,7 @@ class TDTestCase: tdSql.query(f"select distinct tbname from {dbname}.readings") tdSql.checkRows(4) - tdSql.query(f"select * from {dbname}.t_0799064f5487946e5d22164a822acfc8 order by times") + tdSql.query(f"select * from {dbname}.t_0799064f5487946e5d22164a822acfc8 order by _ts") tdSql.checkRows(2) tdSql.checkData(0, 3, "kk") tdSql.checkData(1, 3, "") @@ -67,7 +67,7 @@ class TDTestCase: tdSql.query(f"select distinct tbname from {dbname}.`sys_if_bytes_out`") tdSql.checkRows(2) - tdSql.query(f"select * from {dbname}.t_f67972b49aa8adf8bca5d0d54f0d850d order by times") + tdSql.query(f"select * from {dbname}.t_f67972b49aa8adf8bca5d0d54f0d850d order by _ts") tdSql.checkRows(2) tdSql.checkData(0, 1, 1.300000000) tdSql.checkData(1, 1, 13.000000000) @@ -80,7 +80,7 @@ class TDTestCase: tdSql.query(f"select distinct tbname from {dbname}.`sys_cpu_nice`") tdSql.checkRows(3) - tdSql.query(f"select * from {dbname}.`sys_cpu_nice` order by times") + tdSql.query(f"select * from {dbname}.`sys_cpu_nice` order by _ts") tdSql.checkRows(4) tdSql.checkData(0, 1, 13.000000000) tdSql.checkData(0, 2, "web01") From b036345382183f6227bbea334c13a3391ac7b650 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 27 Nov 2024 22:11:49 +0800 Subject: [PATCH 3/3] fix:[TD-33048] add ts to cols if dataFormat is true in schemaless to avoid schemal is old --- source/client/src/clientSml.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 911e3664f5..e1316af9a5 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -787,8 +787,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols) { if (sTmp == NULL) { SML_CHECK_CODE(TSDB_CODE_SML_INVALID_DATA); } - if ((IS_VAR_DATA_TYPE(kv->type) && kv->length + VARSTR_HEADER_SIZE > sTmp->bytes) || - (!IS_VAR_DATA_TYPE(kv->type) && kv->length != sTmp->bytes)){ + if ((IS_VAR_DATA_TYPE(kv->type) && kv->length + VARSTR_HEADER_SIZE > sTmp->bytes)){ uError("column %s (type %s) bytes invalid. db bytes:%d, kv bytes:%zu", sTmp->name, tDataTypes[sTmp->type].name, sTmp->bytes, kv->length); SML_CHECK_CODE(TSDB_CODE_INTERNAL_ERROR);