From a00dfb52e10760423ae2f0b87fd9f57feb2f3d25 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 15 Oct 2024 14:00:15 +0800 Subject: [PATCH 1/4] fix:[TS-5528] insert error in sml --- source/client/inc/clientSml.h | 1 + source/client/src/clientSml.c | 8 ++++++-- source/client/src/clientSmlLine.c | 4 ++++ tests/system-test/2-query/sml.py | 4 ++++ utils/test/c/sml_test.c | 32 +++++++++++++++++++++++++++++++ 5 files changed, 47 insertions(+), 2 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 209c376f30..a6aca2fddf 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -199,6 +199,7 @@ typedef struct { SArray *preLineTagKV; SArray *maxTagKVs; SArray *maxColKVs; + SArray *escapedStringList; SSmlLineInfo preLine; STableMeta *currSTableMeta; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index d5cca55701..80f583bbee 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -479,6 +479,7 @@ int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) } clearColValArraySml(info->currTableDataCtx->pValues); + taosArrayClearP(info->escapedStringList, taosMemoryFree); if (unlikely(ret != TSDB_CODE_SUCCESS)) { smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); return ret; @@ -1608,6 +1609,7 @@ void smlDestroyInfo(SSmlHandle *info) { taosArrayDestroy(info->valueJsonArray); taosArrayDestroyEx(info->preLineTagKV, freeSSmlKv); + taosArrayDestroyP(info->escapedStringList, taosMemoryFree); if (!info->dataFormat) { for (int i = 0; i < info->lineNum; i++) { @@ -1667,8 +1669,9 @@ int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { info->tagJsonArray = taosArrayInit(8, POINTER_BYTES); info->valueJsonArray = taosArrayInit(8, POINTER_BYTES); info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv)); - - if (info->tagJsonArray == NULL || info->valueJsonArray == NULL || info->preLineTagKV == NULL) { + info->escapedStringList = taosArrayInit(8, POINTER_BYTES); + if (info->tagJsonArray == NULL || info->valueJsonArray == NULL || + info->preLineTagKV == NULL || info->escapedStringList == NULL) { uError("SML:0x%" PRIx64 " failed to allocate memory", info->id); code = terrno; goto FAILED; @@ -1949,6 +1952,7 @@ int32_t smlClearForRerun(SSmlHandle *info) { } } + taosArrayClearP(info->escapedStringList, taosMemoryFree); (void)memset(&info->preLine, 0, sizeof(SSmlLineInfo)); info->currSTableMeta = NULL; info->currTableDataCtx = NULL; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index e620ca9b0c..fabda47ae1 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -451,6 +451,10 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if (info->dataFormat) { bool isAligned = isSmlColAligned(info, cnt, &kv); + if (kv.type == TSDB_DATA_TYPE_BINARY && valueEscaped) { + taosArrayPush(info->escapedStringList, &kv.value); + kv.value = NULL; + } freeSSmlKv(&kv); if(!isAligned){ return TSDB_CODE_SUCCESS; diff --git a/tests/system-test/2-query/sml.py b/tests/system-test/2-query/sml.py index e28f3b1edd..4a8760cd67 100644 --- a/tests/system-test/2-query/sml.py +++ b/tests/system-test/2-query/sml.py @@ -105,6 +105,10 @@ class TDTestCase: tdSql.query(f"select * from ts3724.`stb2.`") tdSql.checkRows(1) + tdSql.query(f"select * from ts5528.device_log_yuelan_cs1") + tdSql.checkRows(2) + tdSql.checkData(0, 1, '{"deviceId":"星宇公司-861701069493741","headers":{"_uid":"4e3599eacd62834995c77b38ad95f88d","creatorId":"1199596756811550720","deviceNmae":"861701069493741","productId":"yuelan","productName":"悦蓝cat1穿戴设备"},"messageType":"REPORT_PROPERTY","properties":{"lat":35.265527067449185,"lng":118.49713144245987,"location":"118.49713144245987,35.265527067449185"},"timestamp":1728719963230}') + tdSql.checkData(1, 1, '{"deviceId":"星宇公司-861701069065507","headers":{"_uid":"9045d6b78b4ffaf1e2d244e912ebbff8","creatorId":"1199596756811550720","deviceNmae":"861701069065507","productId":"yuelan","productName":"悦蓝cat1穿戴设备"},"messageType":"REPORT_PROPERTY","properties":{"lat":36.788241914043425,"lng":119.15042325460891,"location":"119.15042325460891,36.788241914043425"},"timestamp":1728719964105}') # tdSql.query(f"select * from td24559.stb order by _ts") # tdSql.checkRows(4) # tdSql.checkData(0, 2, "POINT (4.343000 89.342000)") diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index a3830d2518..4f4f09bd85 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -2098,12 +2098,44 @@ int sml_td29373_Test() { return code; } +int sml_ts5528_test(){ + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "drop database if exists ts5528"); + taos_free_result(pRes); + + pRes = taos_query(taos, "create database if not exists ts5528"); + taos_free_result(pRes); + + // check column name duplication + char *sql[] = { + "device_log_yuelan_cs1,deviceId=861701069493741 content=\"{\\\"deviceId\\\":\\\"星宇公司-861701069493741\\\",\\\"headers\\\":{\\\"_uid\\\":\\\"4e3599eacd62834995c77b38ad95f88d\\\",\\\"creatorId\\\":\\\"1199596756811550720\\\",\\\"deviceNmae\\\":\\\"861701069493741\\\",\\\"productId\\\":\\\"yuelan\\\",\\\"productName\\\":\\\"悦蓝cat1穿戴设备\\\"},\\\"messageType\\\":\\\"REPORT_PROPERTY\\\",\\\"properties\\\":{\\\"lat\\\":35.265527067449185,\\\"lng\\\":118.49713144245987,\\\"location\\\":\\\"118.49713144245987,35.265527067449185\\\"},\\\"timestamp\\\":1728719963230}\",createTime=1728719963230i64,id=\"4e3599eacd62834995c77b38ad95f88d\",messageId=\"\",timestamp=1728719963230i64,type=\"reportProperty\" 1728719963230", + "device_log_yuelan_cs1,deviceId=861701069065507 content=\"{\\\"deviceId\\\":\\\"星宇公司-861701069065507\\\",\\\"headers\\\":{\\\"_uid\\\":\\\"9045d6b78b4ffaf1e2d244e912ebbff8\\\",\\\"creatorId\\\":\\\"1199596756811550720\\\",\\\"deviceNmae\\\":\\\"861701069065507\\\",\\\"productId\\\":\\\"yuelan\\\",\\\"productName\\\":\\\"悦蓝cat1穿戴设备\\\"},\\\"messageType\\\":\\\"REPORT_PROPERTY\\\",\\\"properties\\\":{\\\"lat\\\":36.788241914043425,\\\"lng\\\":119.15042325460891,\\\"location\\\":\\\"119.15042325460891,36.788241914043425\\\"},\\\"timestamp\\\":1728719964105}\",createTime=1728719964105i64,id=\"9045d6b78b4ffaf1e2d244e912ebbff8\",messageId=\"\",timestamp=1728719964105i64,type=\"reportProperty\" 1728719964105", + }; + pRes = taos_query(taos, "use ts5528"); + taos_free_result(pRes); + + for( int i = 0; i < 2; i++){ + int32_t totalRows = 0; + pRes = taos_schemaless_insert_raw(taos, sql[i], strlen(sql[i]), &totalRows, TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + taos_free_result(pRes); + int code = taos_errno(pRes); + if (code != 0) { + return code; + } + } + printf("%s result success\n", __FUNCTION__); + return 0; +} int main(int argc, char *argv[]) { if (argc == 2) { taos_options(TSDB_OPTION_CONFIGDIR, argv[1]); } int ret = 0; + ret = sml_ts5528_test(); + ASSERT(!ret); ret = sml_td29691_Test(); ASSERT(ret); ret = sml_td29373_Test(); From 986f92d31766482b6aab5784d94980e71760236f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 15 Oct 2024 14:30:36 +0800 Subject: [PATCH 2/4] fix:[TS-5528] insert error in sml --- utils/test/c/sml_test.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 4f4f09bd85..1d8d82ccb9 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -2119,12 +2119,14 @@ int sml_ts5528_test(){ int32_t totalRows = 0; pRes = taos_schemaless_insert_raw(taos, sql[i], strlen(sql[i]), &totalRows, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); - taos_free_result(pRes); int code = taos_errno(pRes); + taos_free_result(pRes); if (code != 0) { + taos_close(taos); return code; } } + taos_close(taos); printf("%s result success\n", __FUNCTION__); return 0; } From 490466747184737a44688eec38db16fd0b004020 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 15 Oct 2024 15:19:37 +0800 Subject: [PATCH 3/4] fix:[TS-5528] insert error in sml --- source/client/src/clientSml.c | 5 ++--- source/client/src/clientSmlLine.c | 7 ++++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 80f583bbee..46aaf29b6b 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -262,7 +262,7 @@ int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSml return TSDB_CODE_SUCCESS; } -bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { +void isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { // cnt begin 0, add ts so + 2 if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) { goto END; @@ -288,12 +288,11 @@ bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { maxKV->length = kv->length; info->needModifySchema = true; } - return true; + return; END: info->dataFormat = false; info->reRun = true; - return false; } bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index fabda47ae1..c1f3431698 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -452,7 +452,10 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if (info->dataFormat) { bool isAligned = isSmlColAligned(info, cnt, &kv); if (kv.type == TSDB_DATA_TYPE_BINARY && valueEscaped) { - taosArrayPush(info->escapedStringList, &kv.value); + if (taosArrayPush(info->escapedStringList, &kv.value) == NULL){ + freeSSmlKv(&kv); + return terrno; + } kv.value = NULL; } freeSSmlKv(&kv); @@ -463,10 +466,12 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if (currElement->colArray == NULL) { currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1); if (currElement->colArray == NULL) { + freeSSmlKv(&kv); return terrno; } } if (taosArrayPush(currElement->colArray, &kv) == NULL){ // reserve for timestamp + freeSSmlKv(&kv); return terrno; } } From c9f835e60d742e5608b902f11e0e910c4ccd0dd2 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 15 Oct 2024 15:25:28 +0800 Subject: [PATCH 4/4] fix:[TS-5528] insert error in sml --- source/client/src/clientSml.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 46aaf29b6b..80f583bbee 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -262,7 +262,7 @@ int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSml return TSDB_CODE_SUCCESS; } -void isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { +bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { // cnt begin 0, add ts so + 2 if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) { goto END; @@ -288,11 +288,12 @@ void isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { maxKV->length = kv->length; info->needModifySchema = true; } - return; + return true; END: info->dataFormat = false; info->reRun = true; + return false; } bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv) {