Merge pull request #18833 from taosdata/refact/submit_req_marks

opti:performance of schemaless

include/libs/parser/parser.h

@@ -58,7 +58,6 @@ typedef struct SParseContext {
   bool             isSuperUser;
   bool             enableSysInfo;
   bool             async;
-  int8_t           schemalessType;
   const char*      svrVer;
   bool             nodeOffline;
   SArray*          pTableMetaPos;    // sql table pos => catalog data pos
@@ -108,7 +107,11 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char*
 void    qDestroyBoundColInfo(void* pInfo);
 
 SQuery* smlInitHandle();
-int32_t smlBindData(SQuery* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta,
+int32_t smlBuildRow(STableDataCxt* pTableCxt);
+int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void *kv, int32_t index);
+STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta);
+
+int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, STableMeta* pTableMeta,
                     char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, int16_t msgBufLen);
 int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash);
 

source/client/inc/clientInt.h

@@ -149,7 +149,6 @@ typedef struct STscObj {
   int32_t       numOfReqs;  // number of sqlObj bound to this connection
   SAppInstInfo* pAppInfo;
   SHashObj*     pRequests;
-  int8_t        schemalessType;  // todo remove it, this attribute should be move to request
 } STscObj;
 
 typedef struct SResultColumn {

source/client/inc/clientLog.h

@@ -30,7 +30,7 @@ extern "C" {
 #define tscDebug(...)  do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0)
 #define tscTrace(...)  do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", DEBUG_TRACE, cDebugFlag, __VA_ARGS__); }} while(0)
 #define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0)
-//#define tscPerf(...)   do { if (cDebugFlag & DEBUG_INFO)  { taosPrintLog("TSC ", DEBUG_INFO, cDebugFlag, __VA_ARGS__); }} while(0)
+#define tscPerf(...)   do { if (cDebugFlag & DEBUG_INFO)  { taosPrintLog("TSC ", 0, cDebugFlag, __VA_ARGS__); }} while(0)
 // clang-format on
 
 #ifdef __cplusplus

source/client/src/clientEnv.c

@@ -76,13 +76,20 @@ static void deregisterRequest(SRequestObj *pRequest) {
            "current:%d, app current:%d",
            pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000.0, num, currentInst);
 
+  tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64
+              "us, exec:%" PRId64 "us, stmtType:%d",
+          duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart,
+          pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd -
+                                                               pRequest->metric.ctgEnd, pRequest->metric.execEnd - pRequest->metric.semanticEnd,
+          pRequest->stmtType);
+
   if (QUERY_NODE_VNODE_MODIF_STMT == pRequest->stmtType) {
-    //    tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64
+//        tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64
-    //            "us, exec:%" PRId64 "us",
+//                "us, exec:%" PRId64 "us",
-    //            duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart,
+//                duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart,
-    //            pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd -
+//                pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd -
-    //            pRequest->metric.ctgEnd, pRequest->metric.execEnd - pRequest->metric.semanticEnd);
+//                pRequest->metric.ctgEnd, pRequest->metric.execEnd - pRequest->metric.semanticEnd);
-    atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration);
+//    atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration);
   } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) {
     //    tscPerf("select duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64
     //            "us, planner:%" PRId64 "us, exec:%" PRId64 "us, reqId:0x%" PRIx64,
@@ -265,7 +272,6 @@ void *createTscObj(const char *user, const char *auth, const char *db, int32_t c
 
   taosThreadMutexInit(&pObj->mutex, NULL);
   pObj->id = taosAddRef(clientConnRefPool, pObj);
-  pObj->schemalessType = 1;
 
   atomic_add_fetch_64(&pObj->pAppInfo->numOfConns, 1);
 

source/client/src/clientImpl.c

@@ -232,7 +232,6 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC
                        .pTransporter = pTscObj->pAppInfo->pTransporter,
                        .pStmtCb = pStmtCb,
                        .pUser = pTscObj->user,
-                       .schemalessType = pTscObj->schemalessType,
                        .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)),
                        .enableSysInfo = pTscObj->sysInfo,
                        .svrVer = pTscObj->sVer,

source/client/src/clientMain.c

@@ -863,7 +863,6 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) {
                            .pTransporter = pTscObj->pAppInfo->pTransporter,
                            .pStmtCb = NULL,
                            .pUser = pTscObj->user,
-                           .schemalessType = pTscObj->schemalessType,
                            .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)),
                            .enableSysInfo = pTscObj->sysInfo,
                            .async = true,

source/common/src/tdatablock.c

@@ -2381,24 +2381,15 @@ char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) {
     return NULL;
   }
 
-  SSmlKv* pTag = taosMemoryCalloc(1, sizeof(SSmlKv));
-  if (pTag == NULL) {
-    taosArrayDestroy(tags);
-    return NULL;
-  }
-
   void* cname = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1);
   if (cname == NULL) {
     taosArrayDestroy(tags);
-    taosMemoryFree(pTag);
     return NULL;
   }
 
-  pTag->key = "group_id";
+  SSmlKv pTag = {.key = "group_id", .keyLen = sizeof("group_id") - 1,
-  pTag->keyLen = strlen(pTag->key);
+                 .type = TSDB_DATA_TYPE_UBIGINT, .u = groupId,
-  pTag->type = TSDB_DATA_TYPE_UBIGINT;
+                 .length = sizeof(uint64_t)};
-  pTag->u = groupId;
-  pTag->length = sizeof(uint64_t);
   taosArrayPush(tags, &pTag);
 
   RandTableName rname = {
@@ -2410,7 +2401,6 @@ char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) {
 
   buildChildTableName(&rname);
 
-  taosMemoryFree(pTag);
   taosArrayDestroy(tags);
 
   ASSERT(rname.ctbShortName && rname.ctbShortName[0]);

source/common/src/tname.c

@@ -298,8 +298,8 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) {
 }
 
 static int compareKv(const void* p1, const void* p2) {
-  SSmlKv* kv1 = *(SSmlKv**)p1;
+  SSmlKv* kv1 = (SSmlKv*)p1;
-  SSmlKv* kv2 = *(SSmlKv**)p2;
+  SSmlKv* kv2 = (SSmlKv*)p2;
   int32_t kvLen1 = kv1->keyLen;
   int32_t kvLen2 = kv2->keyLen;
   int32_t res = strncasecmp(kv1->key, kv2->key, TMIN(kvLen1, kvLen2));
@@ -320,7 +320,7 @@ void buildChildTableName(RandTableName* rName) {
   taosArraySort(rName->tags, compareKv);
   for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) {
     taosStringBuilderAppendChar(&sb, ',');
-    SSmlKv* tagKv = taosArrayGetP(rName->tags, j);
+    SSmlKv* tagKv = taosArrayGet(rName->tags, j);
     taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen);
     taosStringBuilderAppendChar(&sb, '=');
     if (IS_VAR_DATA_TYPE(tagKv->type)) {

source/libs/parser/src/parInsertSml.c

@@ -56,7 +56,7 @@ static int32_t smlBoundColumnData(SArray* cols, SBoundColInfo* pBoundInfo, SSche
   int32_t code = TSDB_CODE_SUCCESS;
 
   for (int i = 0; i < taosArrayGetSize(cols); ++i) {
-    SSmlKv*  kv = taosArrayGetP(cols, i);
+    SSmlKv*  kv = taosArrayGet(cols, i);
     SToken   sToken = {.n = kv->keyLen, .z = (char*)kv->key};
     col_id_t t = lastColIdx + 1;
     col_id_t index = ((t == 0 && !isTag) ? 0 : insFindCol(&sToken, t, pBoundInfo->numOfCols, pSchema));
@@ -111,7 +111,7 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem
   int32_t code = TSDB_CODE_SUCCESS;
   for (int i = 0; i < tags->numOfBound; ++i) {
     SSchema* pTagSchema = &pSchema[tags->pColIndex[i]];
-    SSmlKv*  kv = taosArrayGetP(cols, i);
+    SSmlKv*  kv = taosArrayGet(cols, i);
 
     taosArrayPush(*tagName, pTagSchema->name);
     STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type};
@@ -158,9 +158,68 @@ end:
   return code;
 }
 
-int32_t smlBindData(SQuery* query, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta,
+STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta){
-                    char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf,
+  STableDataCxt* pTableCxt = NULL;
-                    int16_t msgBufLen) {
+  SVCreateTbReq *pCreateTbReq = NULL;
+  int ret = insGetTableDataCxt(((SVnodeModifOpStmt *)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid),
+                           pTableMeta, &pCreateTbReq, &pTableCxt, false);
+  if (ret != TSDB_CODE_SUCCESS) {
+    return NULL;
+  }
+
+  ret = initTableColSubmitData(pTableCxt);
+  if (ret != TSDB_CODE_SUCCESS) {
+    return NULL;
+  }
+  return pTableCxt;
+}
+
+int32_t smlBuildRow(STableDataCxt* pTableCxt){
+  SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1);
+  int ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow);
+  if (TSDB_CODE_SUCCESS != ret) {
+    return ret;
+  }
+  insCheckTableDataOrder(pTableCxt, TD_ROW_KEY(*pRow));
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void *data, int32_t index){
+  int ret = TSDB_CODE_SUCCESS;
+  SSchema* pColSchema = schema + index;
+  SColVal* pVal = taosArrayGet(pTableCxt->pValues, index);
+  SSmlKv* kv = (SSmlKv*)data;
+  if (kv->type == TSDB_DATA_TYPE_NCHAR){
+    int32_t len = 0;
+    char*   pUcs4 = taosMemoryCalloc(1, pColSchema->bytes - VARSTR_HEADER_SIZE);
+    if (NULL == pUcs4) {
+      ret = TSDB_CODE_OUT_OF_MEMORY;
+      goto end;
+    }
+    if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, pColSchema->bytes - VARSTR_HEADER_SIZE, &len)) {
+      if (errno == E2BIG) {
+        ret = TSDB_CODE_PAR_VALUE_TOO_LONG;
+        goto end;
+      }
+      ret = TSDB_CODE_TSC_INVALID_VALUE;
+      goto end;
+    }
+    pVal->value.pData = pUcs4;
+    pVal->value.nData = len;
+  } else if(kv->type == TSDB_DATA_TYPE_BINARY) {
+    pVal->value.nData = kv->length;
+    pVal->value.pData = (uint8_t *)kv->value;
+  } else {
+    memcpy(&pVal->value.val, &(kv->value), kv->length);
+  }
+  pVal->flag = CV_FLAG_VALUE;
+
+end:
+  return ret;
+}
+
+int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, STableMeta* pTableMeta,
+                    char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, int16_t msgBufLen) {
   SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen};
 
   SSchema*       pTagsSchema = getTableTagSchema(pTableMeta);
@@ -193,6 +252,20 @@ int32_t smlBindData(SQuery* query, SArray* tags, SArray* colsSchema, SArray* col
   pCreateTblReq->ctb.stbName = taosMemoryCalloc(1, sTableNameLen + 1);
   memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen);
 
+  if(dataFormat){
+    STableDataCxt** pTableCxt = (STableDataCxt**)taosHashGet(((SVnodeModifOpStmt *)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid));
+    if (NULL == pTableCxt) {
+      ret = buildInvalidOperationMsg(&pBuf, "dataformat true. get tableDataCtx error");
+      goto end;
+    }
+    (*pTableCxt)->pData->flags |= SUBMIT_REQ_AUTO_CREATE_TABLE;
+    (*pTableCxt)->pData->pCreateTbReq = pCreateTblReq;
+    (*pTableCxt)->pMeta->uid = pTableMeta->uid;
+    (*pTableCxt)->pMeta->vgId = pTableMeta->vgId;
+    pCreateTblReq = NULL;
+    goto end;
+  }
+
   STableDataCxt* pTableCxt = NULL;
   ret = insGetTableDataCxt(((SVnodeModifOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid,
                            sizeof(pTableMeta->uid), pTableMeta, &pCreateTblReq, &pTableCxt, false);
@@ -227,15 +300,12 @@ int32_t smlBindData(SQuery* query, SArray* tags, SArray* colsSchema, SArray* col
     for (int c = 0; c < pTableCxt->boundColsInfo.numOfBound; ++c) {
       SSchema* pColSchema = &pSchema[pTableCxt->boundColsInfo.pColIndex[c]];
       SColVal* pVal = taosArrayGet(pTableCxt->pValues, pTableCxt->boundColsInfo.pColIndex[c]);
-      SSmlKv*  kv = NULL;
+      void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name));
-      if (!format) {
+      if (p == NULL) {
-        void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name));
-        if (p) kv = *p;
-      }
-
-      if (kv == NULL) {
         continue;
       }
+      SSmlKv *kv = *(SSmlKv **)p;
+
       if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) {
         kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision);
       }
@@ -297,7 +367,7 @@ SQuery* smlInitHandle() {
     qDestroyQuery(pQuery);
     return NULL;
   }
-  stmt->pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
+  stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
   stmt->freeHashFunc = insDestroyTableDataCxtHashMap;
   stmt->freeArrayFunc = insDestroyVgroupDataCxtList;
 

utils/test/c/sml_test.c

@@ -77,11 +77,20 @@ int smlProcess_telnet_Test() {
   pRes = taos_query(taos, "use sml_db");
   taos_free_result(pRes);
 
-  const char *sql[] = {"sys.if.bytes.out  1479496100 1.3E0 host=web01 interface=eth0",
+  char *sql[4] = {0};
+  sql[0] = taosMemoryCalloc(1, 128);
+  sql[1] = taosMemoryCalloc(1, 128);
+  sql[2] = taosMemoryCalloc(1, 128);
+  sql[3] = taosMemoryCalloc(1, 128);
+  const char *sql1[] = {"sys.if.bytes.out  1479496100 1.3E0 host=web01 interface=eth0",
                        "sys.if.bytes.out  1479496101 1.3E1 interface=eth0    host=web01   ",
                        "sys.if.bytes.out  1479496102 1.3E3 network=tcp",
                        " sys.procs.running   1479496100 42 host=web01   "};
 
+  for(int i = 0; i < 4; i++){
+    strncpy(sql[i], sql1[i], 128);
+  }
+
   pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL,
                                 TSDB_SML_TIMESTAMP_NANO_SECONDS);
   printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes));
@@ -1017,7 +1026,7 @@ int smlProcess_18784_Test() {
   printf("%s result:%s, rows:%d\n", __FUNCTION__, taos_errstr(pRes), taos_affected_rows(pRes));
   int code = taos_errno(pRes);
   ASSERT(!code);
-  ASSERT(taos_affected_rows(pRes) == 2);
+  ASSERT(taos_affected_rows(pRes) == 1);
   taos_free_result(pRes);
 
   pRes = taos_query(taos, "select * from disk");
@@ -1091,9 +1100,10 @@ int sml_ts2164_Test() {
   taos_free_result(pRes);
 
   const char *sql[] = {
+//      "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27",
+      "meters,location=la,groupid=ca current=11.8,voltage=221",
       "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27",
-      "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27",
+//      "meters,location=la,groupid=cb current=11.8,voltage=221,phase=0.27",
-      "meters,location=la,groupid=cb current=11.8,voltage=221,phase=0.27",
   };
 
   pRes = taos_query(taos, "use line_test");
@@ -1118,6 +1128,9 @@ int sml_ttl_Test() {
   const char *sql[] = {
       "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=\"2022-02-0210:22:22\" 1626006833739000000",
   };
+  const char *sql1[] = {
+      "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=\"2022-02-0210:22:22\" 1626006833339000000",
+  };
 
   pRes = taos_query(taos, "use sml_db");
   taos_free_result(pRes);
@@ -1127,6 +1140,11 @@ int sml_ttl_Test() {
   printf("%s result1:%s\n", __FUNCTION__, taos_errstr(pRes));
   taos_free_result(pRes);
 
+  pRes = taos_schemaless_insert_ttl(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, 20);
+
+  printf("%s result1:%s\n", __FUNCTION__, taos_errstr(pRes));
+  taos_free_result(pRes);
+
   pRes = taos_query(taos, "select `ttl` from information_schema.ins_tables where table_name='t_be97833a0e1f523fcdaeb6291d6fdf27'");
   printf("%s result2:%s\n", __FUNCTION__, taos_errstr(pRes));
   TAOS_ROW row = taos_fetch_row(pRes);