feat:add new logic for new tag format

source/libs/parser/src/parInsert.c

@@ -958,6 +958,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint
   SArray *pTagVals = taosArrayInit(pCxt->tags.numOfBound, sizeof(STagVal));
   SToken   sToken;
   bool     isParseBindParam = false;
+  bool isJson = false;
   for (int i = 0; i < pCxt->tags.numOfBound; ++i) {
     NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken);
 
@@ -994,6 +995,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint
       if(code != TSDB_CODE_SUCCESS){
         goto end;
       }
+      isJson = true;
     }else{
       STagVal val = {0};
       code = parseTagToken(&pCxt->pSql, &sToken, pTagSchema, precision, tmpTokenBuf, &val, &pCxt->msg);
@@ -1014,7 +1016,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint
   }
 
   STag* pTag = NULL;
-  code = tTagNew(pTagVals, 1, false, &pTag);
+  code = tTagNew(pTagVals, 1, isJson, &pTag);
   if (code != TSDB_CODE_SUCCESS) {
     goto end;
   }
@@ -1404,7 +1406,6 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) {
       .pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK),
       .pTableNameHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK),
       .totalNum = 0,
-      .pTagVals = NULL,
       .pOutput = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT),
       .pStmtCb = pContext->pStmtCb};
 
@@ -1526,6 +1527,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN
   int32_t code = TSDB_CODE_SUCCESS;
   SSchema* pSchema = pDataBlock->pTableMeta->schema;
 
+  bool isJson = false;
   for (int c = 0; c < tags->numOfBound; ++c) {
     if (bind[c].is_null && bind[c].is_null[0]) {
       continue;
@@ -1537,42 +1539,56 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN
     if (IS_VAR_DATA_TYPE(pTagSchema->type)) {
       colLen = bind[c].length[0];
     }
-
+    if (pTagSchema->type == TSDB_DATA_TYPE_JSON) {
-    STagVal val = {0};
+      if (colLen > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
-    if(pTagSchema->type == TSDB_DATA_TYPE_BINARY){
+        code = buildSyntaxErrMsg(&pBuf, "json string too long than 4095", bind[c].buffer);
-      val.pData = (uint8_t*)bind[c].buffer;
-      val.nData = colLen;
-    }else if(pTagSchema->type == TSDB_DATA_TYPE_NCHAR){
-      int32_t output = 0;
-      void *p = taosMemoryCalloc(1, colLen * TSDB_NCHAR_SIZE);
-      if(p == NULL){
-        code = TSDB_CODE_OUT_OF_MEMORY;
         goto end;
       }
-      if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) {
+
-        if (errno == E2BIG) {
+      isJson = true;
-          taosMemoryFree(p);
+      char *tmp = taosMemoryCalloc(1, colLen + 1);
-          code = generateSyntaxErrMsg(&pBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
+      memcpy(tmp, bind[c].buffer, colLen);
+      code = parseJsontoTagData(tmp, pTagArray, &pBuf);
+      taosMemoryFree(tmp);
+      if(code != TSDB_CODE_SUCCESS){
+        goto end;
+      }
+    }else{
+      STagVal val = {0};
+      if(pTagSchema->type == TSDB_DATA_TYPE_BINARY){
+        val.pData = (uint8_t*)bind[c].buffer;
+        val.nData = colLen;
+      }else if(pTagSchema->type == TSDB_DATA_TYPE_NCHAR){
+        int32_t output = 0;
+        void *p = taosMemoryCalloc(1, colLen * TSDB_NCHAR_SIZE);
+        if(p == NULL){
+          code = TSDB_CODE_OUT_OF_MEMORY;
           goto end;
         }
-        char buf[512] = {0};
+        if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) {
-        snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno));
+          if (errno == E2BIG) {
-        taosMemoryFree(p);
+            taosMemoryFree(p);
-        code = buildSyntaxErrMsg(&pBuf, buf, bind[c].buffer);
+            code = generateSyntaxErrMsg(&pBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
-        goto end;
+            goto end;
+          }
+          char buf[512] = {0};
+          snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno));
+          taosMemoryFree(p);
+          code = buildSyntaxErrMsg(&pBuf, buf, bind[c].buffer);
+          goto end;
+        }
+        val.pData = p;
+        val.nData = output;
+      }else{
+        memcpy(&val.i64, bind[c].buffer, colLen);
       }
-      val.pData = p;
+      taosArrayPush(pTagArray, &val);
-      val.nData = output;
-    }else{
-      memcpy(&val.i64, bind[c].buffer, colLen);
     }
-    taosArrayPush(pTagArray, &val);
   }
 
   STag* pTag = NULL;
 
-  // TODO: stmt support json
+  if (0 != tTagNew(pTagArray, 1, isJson, &pTag)) {
-  if (0 != tTagNew(pTagArray, 1, false, &pTag)) {
     code = buildInvalidOperationMsg(&pBuf, "out of memory");
     goto end;
   }

source/libs/parser/src/parTranslater.c

@@ -4235,7 +4235,7 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla
     }
   }
 
-  code = tTagNew(pTagArray, 1, false, ppTag);
+  code = tTagNew(pTagArray, 1, isJson, ppTag);
 
 end:
   if(isJson){
@@ -4302,7 +4302,7 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau
     }
     ++index;
   }
-  code = tTagNew(pTagArray, 1, false, ppTag);
+  code = tTagNew(pTagArray, 1, isJson, ppTag);
 
 end:
   if(isJson){
@@ -4584,7 +4584,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS
       if (TSDB_CODE_SUCCESS != code) {
         break;
       }
-      code = tTagNew(pTagVals, 1, false, &pTag);
+      code = tTagNew(pTagVals, 1, true, &pTag);
     }while(0);
     for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) {
       STagVal *p = (STagVal *)taosArrayGet(pTagVals, i);

source/libs/scalar/src/sclvector.c

@@ -951,7 +951,7 @@ void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO
     char *data = tTagValToData(value, true);
     colDataAppend(pOutputCol, i, data, data == NULL);
     if(value && IS_VAR_DATA_TYPE(value->type) && data){
-      taosMemoryFree(data)
+      taosMemoryFree(data);
     }
   }
 }

source/libs/scalar/test/scalar/scalarTests.cpp

@@ -1105,24 +1105,16 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do
   nodesDestroyNode(opNode);
 }
 
-#if 0
 TEST(columnTest, json_column_arith_op) {
   scltInitLogFile();
   char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44}";
 
   char rightv[256] = {0};
   memcpy(rightv, rightvTmp, strlen(rightvTmp));
-  SKVRowBuilder kvRowBuilder;
+  SArray *tags = taosArrayInit(1, sizeof(STagVal));
-  tdInitKVRowBuilder(&kvRowBuilder);
+  parseJsontoTagData(rightv, tags, NULL);
-  parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0);
+  STag* row = NULL;
-  SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder);
+  tTagNew(tags, 1, true, &row);
-  char *tmp = (char *)taosMemoryRealloc(row, kvRowLen(row)+1);
-  if(tmp == NULL){
-    ASSERT_TRUE(0);
-  }
-  memmove(tmp+1, tmp, kvRowLen(tmp));
-  *tmp = TSDB_DATA_TYPE_JSON;
-  row = tmp;
 
   const int32_t len = 8;
   EOperatorType op[len] = {OP_TYPE_ADD, OP_TYPE_SUB, OP_TYPE_MULTI, OP_TYPE_DIV,
@@ -1176,10 +1168,10 @@ TEST(columnTest, json_column_arith_op) {
     makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i]);
   }
 
-  tdDestroyKVRowBuilder(&kvRowBuilder);
+  taosArrayDestroy(tags);
   taosMemoryFree(row);
 }
-#endif
+
 void *prepareNchar(char* rightData){
   int32_t len = 0;
   int32_t inputLen = strlen(rightData);
@@ -1189,24 +1181,17 @@ void *prepareNchar(char* rightData){
   varDataSetLen(t, len);
   return t;
 }
-#if 0
+
 TEST(columnTest, json_column_logic_op) {
   scltInitLogFile();
   char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44,\"k6\":\"6.6hello\"}";
 
   char rightv[256] = {0};
   memcpy(rightv, rightvTmp, strlen(rightvTmp));
-  SKVRowBuilder kvRowBuilder;
+  SArray *tags = taosArrayInit(1, sizeof(STagVal));
-  tdInitKVRowBuilder(&kvRowBuilder);
+  parseJsontoTagData(rightv, tags, NULL);
-  parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0);
+  STag* row = NULL;
-  SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder);
+  tTagNew(tags, 1, true, &row);
-  char *tmp = (char *)taosMemoryRealloc(row, kvRowLen(row)+1);
-  if(tmp == NULL){
-    ASSERT_TRUE(0);
-  }
-  memmove(tmp+1, tmp, kvRowLen(tmp));
-  *tmp = TSDB_DATA_TYPE_JSON;
-  row = tmp;
 
   const int32_t len = 9;
   const int32_t len1 = 4;
@@ -1306,10 +1291,9 @@ TEST(columnTest, json_column_logic_op) {
     taosMemoryFree(rightData);
   }
 
-  tdDestroyKVRowBuilder(&kvRowBuilder);
+  taosArrayDestroy(tags);
   taosMemoryFree(row);
 }
-#endif
 
 TEST(columnTest, smallint_value_add_int_column) {
   scltInitLogFile();