From 4b2918440829e2b085bb48a6ae953fd73d5dee96 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 25 Apr 2024 11:06:20 +0800 Subject: [PATCH 1/3] fix:[TD-29793] check if column and tag name are dumplicate in schemaless --- source/client/src/clientSml.c | 42 ++++++++++----- utils/test/c/sml_test.c | 99 ++++++++++++++++++++++++++++++++++- 2 files changed, 127 insertions(+), 14 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 0802ec672a..1681fd1551 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1290,17 +1290,24 @@ end: return code; } -static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) { +static int32_t smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, SHashObj *checkDuplicate) { + terrno = 0; for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); if (ret == 0) { taosArrayPush(metaArray, kv); + if(taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { + return TSDB_CODE_PAR_DUPLICATED_COLUMN; + } + }else if(terrno == TSDB_CODE_DUP_KEY){ + return TSDB_CODE_PAR_DUPLICATED_COLUMN; } } + return TSDB_CODE_SUCCESS; } -static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg) { +static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg, SHashObj* checkDuplicate) { for (int i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); @@ -1332,6 +1339,11 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); if (ret == 0) { taosArrayPush(metaArray, kv); + if(taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { + return TSDB_CODE_PAR_DUPLICATED_COLUMN; + } + }else{ + return ret; } } } @@ -1456,7 +1468,7 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); if (terrno == TSDB_CODE_DUP_KEY) { taosHashCleanup(kvHash); - return terrno; + return TSDB_CODE_PAR_DUPLICATED_COLUMN; } } @@ -1512,12 +1524,12 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { if (tableMeta) { // update meta uDebug("SML:0x%" PRIx64 " smlParseLineBottom update meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); - ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, elements->colArray, false, &info->msgBuf); + ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, elements->colArray, false, &info->msgBuf, (*tableMeta)->tagHash); if (ret == TSDB_CODE_SUCCESS) { - ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, tinfo->tags, true, &info->msgBuf); + ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, tinfo->tags, true, &info->msgBuf, (*tableMeta)->colHash); } if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); + uError("SML:0x%" PRIx64 " smlUpdateMeta failed, ret:%d", info->id, ret); return ret; } } else { @@ -1527,13 +1539,19 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { if (meta == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); - terrno = 0; - smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags); - if (terrno == TSDB_CODE_DUP_KEY) { - return terrno; + ret = taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); + if (ret != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " put measuer to hash failed", info->id); + return ret; + } + ret = smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags, NULL); + if (ret == TSDB_CODE_SUCCESS) { + ret = smlInsertMeta(meta->colHash, meta->cols, elements->colArray, meta->tagHash); + } + if (ret != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " insert meta failed:%s", info->id, tstrerror(ret)); + return ret; } - smlInsertMeta(meta->colHash, meta->cols, elements->colArray); } } uDebug("SML:0x%" PRIx64 " smlParseLineBottom end, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 9e04cfb75b..e51e4979f3 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -1789,6 +1789,88 @@ int sml_td24559_Test() { return code; } +int sml_td29691_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "drop database if exists td29691"); + taos_free_result(pRes); + + pRes = taos_query(taos, "create database if not exists td29691"); + taos_free_result(pRes); + + // check column name duplication + const char *sql[] = { + "vbin,t1=1 f1=283i32,f2=3,f2=b\"hello\" 1632299372000", + }; + pRes = taos_query(taos, "use td29691"); + taos_free_result(pRes); + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + int code = taos_errno(pRes); + printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); + ASSERT(code == TSDB_CODE_PAR_DUPLICATED_COLUMN); + taos_free_result(pRes); + + // check tag name duplication + const char *sql1[] = { + "vbin,t1=1,t1=2 f2=b\"hello\" 1632299372000", + }; + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + code = taos_errno(pRes); + printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); + ASSERT(code == TSDB_CODE_PAR_DUPLICATED_COLUMN); + taos_free_result(pRes); + + + // check column tag name duplication + const char *sql2[] = { + "vbin,t1=1,t2=2 t2=L\"ewe\",f2=b\"hello\" 1632299372000", + }; + pRes = taos_schemaless_insert(taos, (char **)sql2, sizeof(sql2) / sizeof(sql2[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + code = taos_errno(pRes); + printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); + ASSERT(code == TSDB_CODE_PAR_DUPLICATED_COLUMN); + taos_free_result(pRes); + + // insert data + const char *sql3[] = { + "vbin,t1=1,t2=2 f1=1,f2=b\"hello\" 1632299372000", + }; + pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + code = taos_errno(pRes); + printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); + ASSERT(code == 0); + taos_free_result(pRes); + + //check column tag name duplication when update + const char *sql4[] = { + "vbin,t1=1,t2=2,f1=ewe f1=1,f2=b\"hello\" 1632299372001", + }; + pRes = taos_schemaless_insert(taos, (char **)sql4, sizeof(sql4) / sizeof(sql4[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + code = taos_errno(pRes); + printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); + ASSERT(code == TSDB_CODE_PAR_DUPLICATED_COLUMN); + taos_free_result(pRes); + + //check column tag name duplication when update + const char *sql5[] = { + "vbin,t1=1,t2=2 f1=1,f2=b\"hello\",t1=3 1632299372002", + }; + pRes = taos_schemaless_insert(taos, (char **)sql5, sizeof(sql5) / sizeof(sql5[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + code = taos_errno(pRes); + printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); + ASSERT(code == TSDB_CODE_PAR_DUPLICATED_COLUMN); + taos_free_result(pRes); + + return code; +} + + int sml_td18789_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -1799,8 +1881,7 @@ int sml_td18789_Test() { taos_free_result(pRes); const char *sql[] = { - "vbin,t1=1 f1=283i32,f2=b\"hello\" 1632299372000", - "vbin,t1=1 f2=B\"\\x98f46e\",f1=106i32 1632299373000", + "vbin,t1=1 f1=283i32,f2=3,f2=b\"hello\" 1632299372000", }; pRes = taos_query(taos, "use td18789"); @@ -1809,10 +1890,22 @@ int sml_td18789_Test() { pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); + printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); + taos_free_result(pRes); + + const char *sql1[] = { + "vbin,t1=1,f1=2 f2=b\"hello\" 1632299372000", + }; + + + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + int code = taos_errno(pRes); printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); taos_free_result(pRes); + TAOS_ROW row = NULL; pRes = taos_query(taos, "select *,tbname from vbin order by _ts"); int rowIndex = 0; @@ -1952,6 +2045,8 @@ int main(int argc, char *argv[]) { } int ret = 0; + ret = sml_td29691_Test(); + ASSERT(ret); ret = sml_td29373_Test(); ASSERT(ret); ret = sml_td24559_Test(); From aed5e50eeea541ab656f728ded186f8dace0ece4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 25 Apr 2024 11:25:20 +0800 Subject: [PATCH 2/3] fix:[TD-29793] check if column and tag name are dumplicate in schemaless --- utils/test/c/sml_test.c | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index e51e4979f3..e656665b3a 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -1881,7 +1881,8 @@ int sml_td18789_Test() { taos_free_result(pRes); const char *sql[] = { - "vbin,t1=1 f1=283i32,f2=3,f2=b\"hello\" 1632299372000", + "vbin,t1=1 f1=283i32,f2=b\"hello\" 1632299372000", + "vbin,t1=1 f2=B\"\\x98f46e\",f1=106i32 1632299373000", }; pRes = taos_query(taos, "use td18789"); @@ -1889,18 +1890,6 @@ int sml_td18789_Test() { pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); - - printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); - taos_free_result(pRes); - - const char *sql1[] = { - "vbin,t1=1,f1=2 f2=b\"hello\" 1632299372000", - }; - - - pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, - TSDB_SML_TIMESTAMP_MILLI_SECONDS); - int code = taos_errno(pRes); printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); taos_free_result(pRes); From c5f45c5114d65b286621c4e7ea51d84cb78e7205 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 25 Apr 2024 14:28:28 +0800 Subject: [PATCH 3/3] fix:memory leak --- utils/test/c/sml_test.c | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index e656665b3a..6c6144244f 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -1866,6 +1866,7 @@ int sml_td29691_Test() { printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); ASSERT(code == TSDB_CODE_PAR_DUPLICATED_COLUMN); taos_free_result(pRes); + taos_close(taos); return code; }