Merge pull request #28368 from taosdata/fix/TS-5528-3.0

fix:[TS-5528] insert error in sml

source/client/inc/clientSml.h

@@ -199,6 +199,7 @@ typedef struct {
   SArray      *preLineTagKV;
   SArray      *maxTagKVs;
   SArray      *maxColKVs;
+  SArray      *escapedStringList;
 
   SSmlLineInfo preLine;
   STableMeta  *currSTableMeta;

source/client/src/clientSml.c

@@ -479,6 +479,7 @@ int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs)
     }
 
     clearColValArraySml(info->currTableDataCtx->pValues);
+    taosArrayClearP(info->escapedStringList, taosMemoryFree);
     if (unlikely(ret != TSDB_CODE_SUCCESS)) {
       smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
       return ret;
@@ -1608,6 +1609,7 @@ void smlDestroyInfo(SSmlHandle *info) {
   taosArrayDestroy(info->valueJsonArray);
 
   taosArrayDestroyEx(info->preLineTagKV, freeSSmlKv);
+  taosArrayDestroyP(info->escapedStringList, taosMemoryFree);
 
   if (!info->dataFormat) {
     for (int i = 0; i < info->lineNum; i++) {
@@ -1667,8 +1669,9 @@ int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) {
   info->tagJsonArray = taosArrayInit(8, POINTER_BYTES);
   info->valueJsonArray = taosArrayInit(8, POINTER_BYTES);
   info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv));
-
+  info->escapedStringList = taosArrayInit(8, POINTER_BYTES);
-  if (info->tagJsonArray == NULL || info->valueJsonArray == NULL || info->preLineTagKV == NULL) {
+  if (info->tagJsonArray == NULL || info->valueJsonArray == NULL ||
+      info->preLineTagKV == NULL || info->escapedStringList == NULL) {
     uError("SML:0x%" PRIx64 " failed to allocate memory", info->id);
     code = terrno;
     goto FAILED;
@@ -1949,6 +1952,7 @@ int32_t smlClearForRerun(SSmlHandle *info) {
     }
   }
 
+  taosArrayClearP(info->escapedStringList, taosMemoryFree);
   (void)memset(&info->preLine, 0, sizeof(SSmlLineInfo));
   info->currSTableMeta = NULL;
   info->currTableDataCtx = NULL;

source/client/src/clientSmlLine.c

@@ -451,6 +451,13 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
 
     if (info->dataFormat) {
       bool isAligned = isSmlColAligned(info, cnt, &kv);
+      if (kv.type == TSDB_DATA_TYPE_BINARY && valueEscaped) {
+        if (taosArrayPush(info->escapedStringList, &kv.value) == NULL){
+          freeSSmlKv(&kv);
+          return terrno;
+        }
+        kv.value = NULL;
+      }
       freeSSmlKv(&kv);
       if(!isAligned){
         return TSDB_CODE_SUCCESS;
@@ -459,10 +466,12 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
       if (currElement->colArray == NULL) {
         currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1);
         if (currElement->colArray == NULL) {
+          freeSSmlKv(&kv);
           return terrno;
         }
       }
       if (taosArrayPush(currElement->colArray, &kv) == NULL){  // reserve for timestamp
+        freeSSmlKv(&kv);
         return terrno;
       }
     }

tests/system-test/2-query/sml.py

@@ -105,6 +105,10 @@ class TDTestCase:
         tdSql.query(f"select * from ts3724.`stb2.`")
         tdSql.checkRows(1)
 
+        tdSql.query(f"select * from ts5528.device_log_yuelan_cs1")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, '{"deviceId":"星宇公司-861701069493741","headers":{"_uid":"4e3599eacd62834995c77b38ad95f88d","creatorId":"1199596756811550720","deviceNmae":"861701069493741","productId":"yuelan","productName":"悦蓝cat1穿戴设备"},"messageType":"REPORT_PROPERTY","properties":{"lat":35.265527067449185,"lng":118.49713144245987,"location":"118.49713144245987,35.265527067449185"},"timestamp":1728719963230}')
+        tdSql.checkData(1, 1, '{"deviceId":"星宇公司-861701069065507","headers":{"_uid":"9045d6b78b4ffaf1e2d244e912ebbff8","creatorId":"1199596756811550720","deviceNmae":"861701069065507","productId":"yuelan","productName":"悦蓝cat1穿戴设备"},"messageType":"REPORT_PROPERTY","properties":{"lat":36.788241914043425,"lng":119.15042325460891,"location":"119.15042325460891,36.788241914043425"},"timestamp":1728719964105}')
         # tdSql.query(f"select * from td24559.stb order by _ts")
         # tdSql.checkRows(4)
         # tdSql.checkData(0, 2, "POINT (4.343000 89.342000)")

utils/test/c/sml_test.c

@@ -2098,12 +2098,46 @@ int sml_td29373_Test() {
   return code;
 }
 
+int sml_ts5528_test(){
+  TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
+
+  TAOS_RES *pRes = taos_query(taos, "drop database if exists ts5528");
+  taos_free_result(pRes);
+
+  pRes = taos_query(taos, "create database if not exists ts5528");
+  taos_free_result(pRes);
+
+  // check column name duplication
+  char *sql[] = {
+      "device_log_yuelan_cs1,deviceId=861701069493741 content=\"{\\\"deviceId\\\":\\\"星宇公司-861701069493741\\\",\\\"headers\\\":{\\\"_uid\\\":\\\"4e3599eacd62834995c77b38ad95f88d\\\",\\\"creatorId\\\":\\\"1199596756811550720\\\",\\\"deviceNmae\\\":\\\"861701069493741\\\",\\\"productId\\\":\\\"yuelan\\\",\\\"productName\\\":\\\"悦蓝cat1穿戴设备\\\"},\\\"messageType\\\":\\\"REPORT_PROPERTY\\\",\\\"properties\\\":{\\\"lat\\\":35.265527067449185,\\\"lng\\\":118.49713144245987,\\\"location\\\":\\\"118.49713144245987,35.265527067449185\\\"},\\\"timestamp\\\":1728719963230}\",createTime=1728719963230i64,id=\"4e3599eacd62834995c77b38ad95f88d\",messageId=\"\",timestamp=1728719963230i64,type=\"reportProperty\" 1728719963230",
+      "device_log_yuelan_cs1,deviceId=861701069065507 content=\"{\\\"deviceId\\\":\\\"星宇公司-861701069065507\\\",\\\"headers\\\":{\\\"_uid\\\":\\\"9045d6b78b4ffaf1e2d244e912ebbff8\\\",\\\"creatorId\\\":\\\"1199596756811550720\\\",\\\"deviceNmae\\\":\\\"861701069065507\\\",\\\"productId\\\":\\\"yuelan\\\",\\\"productName\\\":\\\"悦蓝cat1穿戴设备\\\"},\\\"messageType\\\":\\\"REPORT_PROPERTY\\\",\\\"properties\\\":{\\\"lat\\\":36.788241914043425,\\\"lng\\\":119.15042325460891,\\\"location\\\":\\\"119.15042325460891,36.788241914043425\\\"},\\\"timestamp\\\":1728719964105}\",createTime=1728719964105i64,id=\"9045d6b78b4ffaf1e2d244e912ebbff8\",messageId=\"\",timestamp=1728719964105i64,type=\"reportProperty\" 1728719964105",
+  };
+  pRes = taos_query(taos, "use ts5528");
+  taos_free_result(pRes);
+
+  for( int  i = 0; i < 2; i++){
+    int32_t totalRows = 0;
+    pRes = taos_schemaless_insert_raw(taos, sql[i], strlen(sql[i]), &totalRows, TSDB_SML_LINE_PROTOCOL,
+                                      TSDB_SML_TIMESTAMP_MILLI_SECONDS);
+    int code = taos_errno(pRes);
+    taos_free_result(pRes);
+    if (code != 0) {
+      taos_close(taos);
+      return code;
+    }
+  }
+  taos_close(taos);
+  printf("%s result success\n", __FUNCTION__);
+  return 0;
+}
 int main(int argc, char *argv[]) {
   if (argc == 2) {
     taos_options(TSDB_OPTION_CONFIGDIR, argv[1]);
   }
 
   int ret = 0;
+  ret = sml_ts5528_test();
+  ASSERT(!ret);
   ret = sml_td29691_Test();
   ASSERT(ret);
   ret = sml_td29373_Test();