From f84fe05a671bcde33fdcb07722752d4516a97895 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Tue, 24 Sep 2024 18:54:52 +0800
Subject: [PATCH 01/28] enh:[TS-5441] cost too long in tmq write meta data

---
 source/client/src/clientRawBlockWrite.c | 96 ++++++++++++++-----------
 1 file changed, 55 insertions(+), 41 deletions(-)

diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 8a888a2a47..efd2eec678 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -26,7 +26,7 @@
 #define RAW_NULL_CHECK(c)             \
   do {                                \
     if (c == NULL) {                  \
-      code = TSDB_CODE_OUT_OF_MEMORY; \
+      code = terrno; \
       goto end;                       \
     }                                 \
   } while (0)
@@ -1780,6 +1780,42 @@ end:
   return code;
 }
 
+static int32_t buildCreateTbMap(STaosxRsp* rsp, SHashObj* pHashObj) {
+  // find schema data info
+  int32_t             code = 0;
+  SVCreateTbReq pCreateReq = {0};
+  SDecoder      decoderTmp = {0};
+
+  for (int j = 0; j < rsp->createTableNum; j++) {
+    void** dataTmp = taosArrayGet(rsp->createTableReq, j);
+    RAW_NULL_CHECK(dataTmp);
+    int32_t* lenTmp = taosArrayGet(rsp->createTableLen, j);
+    RAW_NULL_CHECK(dataTmp);
+
+    tDecoderInit(&decoderTmp, *dataTmp, *lenTmp);
+    RAW_RETURN_CHECK (tDecodeSVCreateTbReq(&decoderTmp, &pCreateReq));
+
+    if (pCreateReq.type != TSDB_CHILD_TABLE) {
+      code = TSDB_CODE_INVALID_MSG;
+      goto end;
+    }
+    if (taosHashGet(pHashObj, pCreateReq.name, strlen(pCreateReq.name)) == NULL){
+      RAW_RETURN_CHECK(taosHashPut(pHashObj, pCreateReq.name, strlen(pCreateReq.name), &pCreateReq, sizeof(SVCreateTbReq)));
+    } else{
+      tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE);
+      pCreateReq = (SVCreateTbReq){0};
+    }
+
+    tDecoderClear(&decoderTmp);
+  }
+  return 0;
+
+end:
+  tDecoderClear(&decoderTmp);
+  tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE);
+  return code;
+}
+
 static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) {
   if (taos == NULL || data == NULL) {
     SET_ERROR_MSG("taos:%p or data:%p is NULL", taos, data);
@@ -1787,11 +1823,11 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
   }
   int32_t        code = TSDB_CODE_SUCCESS;
   SHashObj*      pVgHash = NULL;
+  SHashObj*      pCreateTbHash = NULL;
   SQuery*        pQuery = NULL;
   SMqTaosxRspObj rspObj = {0};
   SDecoder       decoder = {0};
   STableMeta*    pTableMeta = NULL;
-  SVCreateTbReq* pCreateReqDst = NULL;
 
   SRequestObj* pRequest = NULL;
   RAW_RETURN_CHECK(createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0, &pRequest));
@@ -1832,6 +1868,11 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
   RAW_RETURN_CHECK(smlInitHandle(&pQuery));
   pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
   RAW_NULL_CHECK(pVgHash);
+  pCreateTbHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+  RAW_NULL_CHECK(pCreateTbHash);
+  taosHashSetFreeFp(pCreateTbHash, (_hash_free_fn_t)tdDestroySVCreateTbReq);
+
+  RAW_RETURN_CHECK(buildCreateTbMap(&rspObj.rsp, pCreateTbHash));
 
   uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.common.blockNum);
   while (++rspObj.common.resIter < rspObj.rsp.common.blockNum) {
@@ -1854,40 +1895,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
     (void)strcpy(pName.tname, tbName);
 
     // find schema data info
-    for (int j = 0; j < rspObj.rsp.createTableNum; j++) {
-      void** dataTmp = taosArrayGet(rspObj.rsp.createTableReq, j);
-      RAW_NULL_CHECK(dataTmp);
-      int32_t* lenTmp = taosArrayGet(rspObj.rsp.createTableLen, j);
-      RAW_NULL_CHECK(dataTmp);
-
-      SDecoder      decoderTmp = {0};
-      SVCreateTbReq pCreateReq = {0};
-      tDecoderInit(&decoderTmp, *dataTmp, *lenTmp);
-      if (tDecodeSVCreateTbReq(&decoderTmp, &pCreateReq) < 0) {
-        tDecoderClear(&decoderTmp);
-        tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE);
-        code = TSDB_CODE_TMQ_INVALID_MSG;
-        SET_ERROR_MSG("decode create table:%s req failed", tbName);
-        goto end;
-      }
-
-      if (pCreateReq.type != TSDB_CHILD_TABLE) {
-        code = TSDB_CODE_TSC_INVALID_VALUE;
-        tDecoderClear(&decoderTmp);
-        tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE);
-        SET_ERROR_MSG("create table req type is not child table: %s, type: %d", tbName, pCreateReq.type);
-        goto end;
-      }
-      if (strcmp(tbName, pCreateReq.name) == 0) {
-        RAW_RETURN_CHECK(cloneSVreateTbReq(&pCreateReq, &pCreateReqDst));
-        tDecoderClear(&decoderTmp);
-        tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE);
-        break;
-      }
-      tDecoderClear(&decoderTmp);
-      tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE);
-    }
-
+    SVCreateTbReq* pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, tbName, strlen(tbName));
     SVgroupInfo vg = {0};
     RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg));
     if (pCreateReqDst) {  // change stable name to get meta
@@ -1920,7 +1928,15 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
     }
     void* rawData = getRawDataFromRes(pRetrieve);
     char  err[ERR_MSG_LEN] = {0};
-    code = rawBlockBindData(pQuery, pTableMeta, rawData, &pCreateReqDst, fields, pSW->nCols, true, err, ERR_MSG_LEN);
+    SVCreateTbReq* pCreateReqTmp = NULL;
+    if (pCreateReqDst){
+      RAW_RETURN_CHECK(cloneSVreateTbReq(pCreateReqDst, &pCreateReqTmp));
+    }
+    code = rawBlockBindData(pQuery, pTableMeta, rawData, &pCreateReqTmp, fields, pSW->nCols, true, err, ERR_MSG_LEN);
+    if (pCreateReqTmp != NULL) {
+      tdDestroySVCreateTbReq(pCreateReqTmp);
+      taosMemoryFree(pCreateReqTmp);
+    }
     taosMemoryFree(fields);
     taosMemoryFreeClear(pTableMeta);
     if (code != TSDB_CODE_SUCCESS) {
@@ -1936,16 +1952,14 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
 
 end:
   uDebug(LOG_ID_TAG " write raw metadata return, msg:%s", LOG_ID_VALUE, tstrerror(code));
+  taosHashCleanup(pCreateTbHash);
   tDeleteSTaosxRsp(&rspObj.rsp);
   tDecoderClear(&decoder);
   qDestroyQuery(pQuery);
   destroyRequest(pRequest);
   taosHashCleanup(pVgHash);
   taosMemoryFreeClear(pTableMeta);
-  if (pCreateReqDst) {
-    tdDestroySVCreateTbReq(pCreateReqDst);
-    taosMemoryFree(pCreateReqDst);
-  }
+
   return code;
 }
 

From 33cab144faa092a93f77d30dbe2ecfeba65fb34d Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 25 Sep 2024 11:13:03 +0800
Subject: [PATCH 02/28] enh:[TS-5441] cost too long in tmq write meta data

---
 source/client/src/clientRawBlockWrite.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index efd2eec678..a97440eed7 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -1870,8 +1870,6 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
   RAW_NULL_CHECK(pVgHash);
   pCreateTbHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
   RAW_NULL_CHECK(pCreateTbHash);
-  taosHashSetFreeFp(pCreateTbHash, (_hash_free_fn_t)tdDestroySVCreateTbReq);
-
   RAW_RETURN_CHECK(buildCreateTbMap(&rspObj.rsp, pCreateTbHash));
 
   uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.common.blockNum);
@@ -1952,7 +1950,13 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
 
 end:
   uDebug(LOG_ID_TAG " write raw metadata return, msg:%s", LOG_ID_VALUE, tstrerror(code));
+  void* pIter = taosHashIterate(pCreateTbHash, NULL);
+  while (pIter) {
+    tDestroySVCreateTbReq(pIter, TSDB_MSG_FLG_DECODE);
+    pIter = taosHashIterate(pCreateTbHash, pIter);
+  }
   taosHashCleanup(pCreateTbHash);
+
   tDeleteSTaosxRsp(&rspObj.rsp);
   tDecoderClear(&decoder);
   qDestroyQuery(pQuery);

From 5399bffe993bc462938e44100bef90d5bf321a8d Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 25 Sep 2024 18:47:02 +0800
Subject: [PATCH 03/28] enh:[TS-5441] cost too long in tmq write meta data

---
 source/client/src/clientRawBlockWrite.c | 188 ++++++++++++++----------
 1 file changed, 112 insertions(+), 76 deletions(-)

diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index a97440eed7..856f4f59c1 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -165,7 +165,7 @@ static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sche
   }
   RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tags", tags));
 
-end:
+  end:
   *pJson = json;
 }
 
@@ -199,7 +199,7 @@ static int32_t setCompressOption(cJSON* json, uint32_t para) {
     return code;
   }
 
-end:
+  end:
   return code;
 }
 static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON** pJson) {
@@ -340,7 +340,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON**
       break;
   }
 
-end:
+  end:
   tFreeSMAltertbReq(&req);
   *pJson = json;
 }
@@ -360,7 +360,7 @@ static void processCreateStb(SMqMetaRsp* metaRsp, cJSON** pJson) {
   }
   buildCreateTableJson(&req.schemaRow, &req.schemaTag, req.name, req.suid, TSDB_SUPER_TABLE, &req.colCmpr, pJson);
 
-end:
+  end:
   uDebug("create stable return, sql json:%s", cJSON_PrintUnformatted(*pJson));
   tDecoderClear(&coder);
 }
@@ -380,7 +380,7 @@ static void processAlterStb(SMqMetaRsp* metaRsp, cJSON** pJson) {
   }
   buildAlterSTableJson(req.alterOriData, req.alterOriDataLen, pJson);
 
-end:
+  end:
   uDebug("alter stable return, sql json:%s", cJSON_PrintUnformatted(*pJson));
   tDecoderClear(&coder);
 }
@@ -485,7 +485,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
     RAW_FALSE_CHECK(cJSON_AddItemToArray(tags, tag));
   }
 
-end:
+  end:
   RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tags", tags));
   taosArrayDestroy(pTagVals);
 }
@@ -514,7 +514,7 @@ static void buildCreateCTableJson(SVCreateTbReq* pCreateReq, int32_t nReqs, cJSO
   }
   RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "createList", createList));
 
-end:
+  end:
   *pJson = json;
 }
 
@@ -542,7 +542,7 @@ static void processCreateTable(SMqMetaRsp* metaRsp, cJSON** pJson) {
     }
   }
 
-end:
+  end:
   uDebug("create table return, sql json:%s", cJSON_PrintUnformatted(*pJson));
   tDeleteSVCreateTbBatchReq(&req);
   tDecoderClear(&decoder);
@@ -585,7 +585,7 @@ static void processAutoCreateTable(STaosxRsp* rsp, char** string) {
   *string = cJSON_PrintUnformatted(pJson);
   cJSON_Delete(pJson);
 
-end:
+  end:
   uDebug("auto created table return, sql json:%s", *string);
   for (int i = 0; decoder && pCreateReq && i < rsp->createTableNum; i++) {
     tDecoderClear(&decoder[i]);
@@ -771,7 +771,7 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) {
       break;
   }
 
-end:
+  end:
   uDebug("alter table return, sql json:%s", cJSON_PrintUnformatted(json));
   tDecoderClear(&decoder);
   *pJson = json;
@@ -806,7 +806,7 @@ static void processDropSTable(SMqMetaRsp* metaRsp, cJSON** pJson) {
   RAW_NULL_CHECK(tableName);
   RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableName", tableName));
 
-end:
+  end:
   uDebug("processDropSTable return, sql json:%s", cJSON_PrintUnformatted(json));
   tDecoderClear(&decoder);
   *pJson = json;
@@ -842,7 +842,7 @@ static void processDeleteTable(SMqMetaRsp* metaRsp, cJSON** pJson) {
   RAW_NULL_CHECK(sqlJson);
   RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "sql", sqlJson));
 
-end:
+  end:
   uDebug("processDeleteTable return, sql json:%s", cJSON_PrintUnformatted(json));
   tDecoderClear(&coder);
   *pJson = json;
@@ -879,7 +879,7 @@ static void processDropTable(SMqMetaRsp* metaRsp, cJSON** pJson) {
   }
   RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableNameList", tableNameList));
 
-end:
+  end:
   uDebug("processDropTable return, json sql:%s", cJSON_PrintUnformatted(json));
   tDecoderClear(&decoder);
   *pJson = json;
@@ -990,7 +990,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
 
   code = pRequest->code;
 
-end:
+  end:
   uDebug(LOG_ID_TAG " create stable return, msg:%s", LOG_ID_VALUE, tstrerror(code));
   destroyRequest(pRequest);
   tFreeSMCreateStbReq(&pReq);
@@ -1027,9 +1027,9 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
   SCatalog* pCatalog = NULL;
   RAW_RETURN_CHECK(catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog));
   SRequestConnInfo conn = {.pTrans = pRequest->pTscObj->pAppInfo->pTransporter,
-                           .requestId = pRequest->requestId,
-                           .requestObjRefId = pRequest->self,
-                           .mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)};
+      .requestId = pRequest->requestId,
+      .requestObjRefId = pRequest->self,
+      .mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)};
   SName            pName = {0};
   toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, &pName);
   STableMeta* pTableMeta = NULL;
@@ -1092,7 +1092,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
 
   code = pRequest->code;
 
-end:
+  end:
   uDebug(LOG_ID_TAG " drop stable return, msg:%s", LOG_ID_VALUE, tstrerror(code));
   destroyRequest(pRequest);
   tDecoderClear(&coder);
@@ -1148,9 +1148,9 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
   taosHashSetFreeFp(pVgroupHashmap, destroyCreateTbReqBatch);
 
   SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter,
-                           .requestId = pRequest->requestId,
-                           .requestObjRefId = pRequest->self,
-                           .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)};
+      .requestId = pRequest->requestId,
+      .requestObjRefId = pRequest->self,
+      .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)};
 
   pRequest->tableList = taosArrayInit(req.nReqs, sizeof(SName));
   RAW_NULL_CHECK(pRequest->tableList);
@@ -1243,7 +1243,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
 
   code = pRequest->code;
 
-end:
+  end:
   uDebug(LOG_ID_TAG " create table return, msg:%s", LOG_ID_VALUE, tstrerror(code));
   tDeleteSVCreateTbBatchReq(&req);
 
@@ -1304,9 +1304,9 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
   taosHashSetFreeFp(pVgroupHashmap, destroyDropTbReqBatch);
 
   SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter,
-                           .requestId = pRequest->requestId,
-                           .requestObjRefId = pRequest->self,
-                           .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)};
+      .requestId = pRequest->requestId,
+      .requestObjRefId = pRequest->self,
+      .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)};
   pRequest->tableList = taosArrayInit(req.nReqs, sizeof(SName));
   RAW_NULL_CHECK(pRequest->tableList);
   // loop to create table
@@ -1371,7 +1371,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
   }
   code = pRequest->code;
 
-end:
+  end:
   uDebug(LOG_ID_TAG " drop table return, msg:%s", LOG_ID_VALUE, tstrerror(code));
   taosHashCleanup(pVgroupHashmap);
   destroyRequest(pRequest);
@@ -1412,7 +1412,7 @@ static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) {
   }
   taos_free_result(res);
 
-end:
+  end:
   uDebug("connId:0x%" PRIx64 " delete data sql:%s, code:%s", *(int64_t*)taos, sql, tstrerror(code));
   tDecoderClear(&coder);
   return code;
@@ -1455,9 +1455,9 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
   SCatalog* pCatalog = NULL;
   RAW_RETURN_CHECK(catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog));
   SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter,
-                           .requestId = pRequest->requestId,
-                           .requestObjRefId = pRequest->self,
-                           .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)};
+      .requestId = pRequest->requestId,
+      .requestObjRefId = pRequest->self,
+      .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)};
 
   SVgroupInfo pInfo = {0};
   SName       pName = {0};
@@ -1525,7 +1525,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
       code = handleAlterTbExecRes(pRes->res, pCatalog);
     }
   }
-end:
+  end:
   uDebug(LOG_ID_TAG " alter table return, meta:%p, len:%d, msg:%s", LOG_ID_VALUE, meta, metaLen, tstrerror(code));
   taosArrayDestroy(pArray);
   if (pVgData) taosMemoryFreeClear(pVgData->pData);
@@ -1590,7 +1590,7 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pDat
   (void)launchQueryImpl(pRequest, pQuery, true, NULL);
   code = pRequest->code;
 
-end:
+  end:
   uDebug(LOG_ID_TAG " write raw block with field return, msg:%s", LOG_ID_VALUE, tstrerror(code));
   taosMemoryFreeClear(pTableMeta);
   qDestroyQuery(pQuery);
@@ -1650,7 +1650,7 @@ int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const cha
   (void)launchQueryImpl(pRequest, pQuery, true, NULL);
   code = pRequest->code;
 
-end:
+  end:
   uDebug(LOG_ID_TAG " write raw block return, msg:%s", LOG_ID_VALUE, tstrerror(code));
   taosMemoryFreeClear(pTableMeta);
   qDestroyQuery(pQuery);
@@ -1769,7 +1769,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
   (void)launchQueryImpl(pRequest, pQuery, true, NULL);
   code = pRequest->code;
 
-end:
+  end:
   uDebug(LOG_ID_TAG " write raw data return, msg:%s", LOG_ID_VALUE, tstrerror(code));
   tDeleteMqDataRsp(&rspObj.rsp);
   tDecoderClear(&decoder);
@@ -1790,23 +1790,28 @@ static int32_t buildCreateTbMap(STaosxRsp* rsp, SHashObj* pHashObj) {
     void** dataTmp = taosArrayGet(rsp->createTableReq, j);
     RAW_NULL_CHECK(dataTmp);
     int32_t* lenTmp = taosArrayGet(rsp->createTableLen, j);
-    RAW_NULL_CHECK(dataTmp);
+    RAW_NULL_CHECK(lenTmp);
 
     tDecoderInit(&decoderTmp, *dataTmp, *lenTmp);
-    RAW_RETURN_CHECK (tDecodeSVCreateTbReq(&decoderTmp, &pCreateReq));
+    RAW_RETURN_CHECK(tDecodeSVCreateTbReq(&decoderTmp, &pCreateReq));
 
     if (pCreateReq.type != TSDB_CHILD_TABLE) {
       code = TSDB_CODE_INVALID_MSG;
       goto end;
     }
-    if (taosHashGet(pHashObj, pCreateReq.name, strlen(pCreateReq.name)) == NULL){
-      RAW_RETURN_CHECK(taosHashPut(pHashObj, pCreateReq.name, strlen(pCreateReq.name), &pCreateReq, sizeof(SVCreateTbReq)));
-    } else{
-      tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE);
-      pCreateReq = (SVCreateTbReq){0};
+    SVCreateTbReq** pCreateReqDst = taosHashGet(pHashObj, pCreateReq.name, strlen(pCreateReq.name));
+    if (pCreateReqDst == NULL){
+      RAW_RETURN_CHECK(cloneSVreateTbReq(&pCreateReq, pCreateReqDst));
+      code = taosHashPut(pHashObj, pCreateReq.name, strlen(pCreateReq.name), &pCreateReqDst, POINTER_BYTES);
+      if (code != 0){
+        tdDestroySVCreateTbReq(*pCreateReqDst);
+        goto end;
+      }
     }
 
     tDecoderClear(&decoderTmp);
+    tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE);
+    pCreateReq = (SVCreateTbReq){0};
   }
   return 0;
 
@@ -1816,19 +1821,20 @@ end:
   return code;
 }
 
+static threadlocal SHashObj*     pVgHash = NULL;
+static threadlocal SHashObj*     pCreateTbHash = NULL;
+static threadlocal SHashObj*     pNameHash = NULL;
+static threadlocal SHashObj*     pMetaHash = NULL;
+
 static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) {
   if (taos == NULL || data == NULL) {
     SET_ERROR_MSG("taos:%p or data:%p is NULL", taos, data);
     return TSDB_CODE_INVALID_PARA;
   }
   int32_t        code = TSDB_CODE_SUCCESS;
-  SHashObj*      pVgHash = NULL;
-  SHashObj*      pCreateTbHash = NULL;
   SQuery*        pQuery = NULL;
   SMqTaosxRspObj rspObj = {0};
   SDecoder       decoder = {0};
-  STableMeta*    pTableMeta = NULL;
-
   SRequestObj* pRequest = NULL;
   RAW_RETURN_CHECK(createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0, &pRequest));
 
@@ -1856,6 +1862,23 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
     goto end;
   }
 
+  if (pVgHash == NULL){
+    pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
+    RAW_NULL_CHECK(pVgHash);
+  }
+  if (pCreateTbHash == NULL){
+    pCreateTbHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
+    RAW_NULL_CHECK(pCreateTbHash);
+  }
+  if (pNameHash == NULL){
+    pNameHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
+    RAW_NULL_CHECK(pNameHash);
+  }
+  if (pMetaHash == NULL){
+    pMetaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
+    RAW_NULL_CHECK(pMetaHash);
+    taosHashSetFreeFp(pMetaHash, taosMemoryFree);
+  }
   struct SCatalog* pCatalog = NULL;
   RAW_RETURN_CHECK(catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog));
 
@@ -1866,11 +1889,6 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
   conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
 
   RAW_RETURN_CHECK(smlInitHandle(&pQuery));
-  pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
-  RAW_NULL_CHECK(pVgHash);
-  pCreateTbHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
-  RAW_NULL_CHECK(pCreateTbHash);
-  RAW_RETURN_CHECK(buildCreateTbMap(&rspObj.rsp, pCreateTbHash));
 
   uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.common.blockNum);
   while (++rspObj.common.resIter < rspObj.rsp.common.blockNum) {
@@ -1894,21 +1912,37 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
 
     // find schema data info
     SVCreateTbReq* pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, tbName, strlen(tbName));
-    SVgroupInfo vg = {0};
-    RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg));
-    if (pCreateReqDst) {  // change stable name to get meta
-      (void)strcpy(pName.tname, pCreateReqDst->ctb.stbName);
+    if (pCreateReqDst == NULL) {
+      RAW_RETURN_CHECK(buildCreateTbMap(&rspObj.rsp, pCreateTbHash));
+      pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, tbName, strlen(tbName));
     }
-    RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta));
 
-    if (pCreateReqDst) {
-      pTableMeta->vgId = vg.vgId;
-      pTableMeta->uid = pCreateReqDst->uid;
-      pCreateReqDst->ctb.suid = pTableMeta->suid;
+    int32_t vgId = 0;
+    SVgroupInfo* vg = (SVgroupInfo*)taosHashGet(pNameHash, tbName, strlen(tbName));
+    if (vg == NULL) {
+      SVgroupInfo vgTmp = {0};
+      RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vgTmp));
+      RAW_RETURN_CHECK(taosHashPut(pNameHash, tbName, strlen(tbName), &vgTmp, sizeof(SVgroupInfo)));
+      code = taosHashPut(pVgHash, &vgTmp.vgId, sizeof(vgTmp.vgId), &vgTmp, sizeof(SVgroupInfo));
+      code = (code == TSDB_CODE_DUP_KEY) ? 0 : code;
+      RAW_RETURN_CHECK(code);
+      vgId = vgTmp.vgId;
+    } else {
+      vgId = vg->vgId;
     }
-    void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId));
-    if (hData == NULL) {
-      RAW_RETURN_CHECK(taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)));
+
+    STableMeta** pTableMeta = (STableMeta**)taosHashGet(pMetaHash, tbName, strlen(tbName));
+    if (pTableMeta == NULL) {
+      if (pCreateReqDst) {  // change stable name to get meta
+        (void)strcpy(pName.tname, pCreateReqDst->ctb.stbName);
+      }
+      RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, &conn, &pName, pTableMeta));
+      RAW_RETURN_CHECK(taosHashPut(pMetaHash, tbName, strlen(tbName), pTableMeta, POINTER_BYTES));
+      if (pCreateReqDst) {
+        (*pTableMeta)->vgId = vgId;
+        (*pTableMeta)->uid = pCreateReqDst->uid;
+        pCreateReqDst->ctb.suid = (*pTableMeta)->suid;
+      }
     }
 
     SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.common.blockSchema, rspObj.common.resIter);
@@ -1930,13 +1964,12 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
     if (pCreateReqDst){
       RAW_RETURN_CHECK(cloneSVreateTbReq(pCreateReqDst, &pCreateReqTmp));
     }
-    code = rawBlockBindData(pQuery, pTableMeta, rawData, &pCreateReqTmp, fields, pSW->nCols, true, err, ERR_MSG_LEN);
+    code = rawBlockBindData(pQuery, *pTableMeta, rawData, &pCreateReqTmp, fields, pSW->nCols, true, err, ERR_MSG_LEN);
     if (pCreateReqTmp != NULL) {
       tdDestroySVCreateTbReq(pCreateReqTmp);
       taosMemoryFree(pCreateReqTmp);
     }
     taosMemoryFree(fields);
-    taosMemoryFreeClear(pTableMeta);
     if (code != TSDB_CODE_SUCCESS) {
       SET_ERROR_MSG("table:%s, err:%s", tbName, err);
       goto end;
@@ -1948,23 +1981,26 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
   (void)launchQueryImpl(pRequest, pQuery, true, NULL);
   code = pRequest->code;
 
-end:
+  end:
   uDebug(LOG_ID_TAG " write raw metadata return, msg:%s", LOG_ID_VALUE, tstrerror(code));
+  tDeleteSTaosxRsp(&rspObj.rsp);
+  tDecoderClear(&decoder);
+  qDestroyQuery(pQuery);
+  destroyRequest(pRequest);
+
+  return code;
+}
+
+void tmqClean() {
+  taosHashCleanup(pMetaHash);
+  taosHashCleanup(pNameHash);
   void* pIter = taosHashIterate(pCreateTbHash, NULL);
   while (pIter) {
     tDestroySVCreateTbReq(pIter, TSDB_MSG_FLG_DECODE);
     pIter = taosHashIterate(pCreateTbHash, pIter);
   }
   taosHashCleanup(pCreateTbHash);
-
-  tDeleteSTaosxRsp(&rspObj.rsp);
-  tDecoderClear(&decoder);
-  qDestroyQuery(pQuery);
-  destroyRequest(pRequest);
   taosHashCleanup(pVgHash);
-  taosMemoryFreeClear(pTableMeta);
-
-  return code;
 }
 
 static void processSimpleMeta(SMqMetaRsp* pMetaRsp, cJSON** meta) {
@@ -2027,7 +2063,7 @@ static void processBatchMetaToJson(SMqBatchMetaRsp* pMsgRsp, char** string) {
   *string = fullStr;
   return;
 
-end:
+  end:
   cJSON_Delete(pJson);
   tDeleteMqBatchMetaRsp(&rsp);
 }
@@ -2113,7 +2149,7 @@ static int32_t encodeMqDataRsp(__encode_func__* encodeFunc, void* rspObj, tmq_ra
   raw->raw = buf;
   raw->raw_len = len;
   return code;
-FAILED:
+  FAILED:
   tEncoderClear(&encoder);
   taosMemoryFree(buf);
   return code;
@@ -2236,7 +2272,7 @@ static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen
     }
   }
 
-end:
+  end:
   tDeleteMqBatchMetaRsp(&rsp);
   return code;
 }

From 67fe6091298de92b1bb0876d09e985e2f74cbcbf Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Tue, 8 Oct 2024 18:24:16 +0800
Subject: [PATCH 04/28] enh:[TS-5441] cost too long in tmq write meta data by
 cache meta and vg info

---
 include/libs/parser/parser.h            |  4 +--
 source/client/src/clientRawBlockWrite.c | 43 ++++++++++++-------------
 source/libs/parser/src/parInsertUtil.c  | 21 +++++++++---
 utils/test/c/tmq_taosx_ci.c             |  1 +
 4 files changed, 39 insertions(+), 30 deletions(-)

diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h
index c98b188d49..832e4f8863 100644
--- a/include/libs/parser/parser.h
+++ b/include/libs/parser/parser.h
@@ -176,8 +176,8 @@ int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsS
                     STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl,
                     char* msgBuf, int32_t msgBufLen);
 int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash);
-int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, TAOS_FIELD* fields,
-                     int numFields, bool needChangeLength, char* errstr, int32_t errstrLen);
+int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, void* fields,
+                     int numFields, bool needChangeLength, char* errstr, int32_t errstrLen, bool raw);
 
 int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray);
 int32_t serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap, SArray** pOut);
diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 17ebd9b558..064c3bdb2e 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -1618,7 +1618,7 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pDat
   RAW_NULL_CHECK(pVgHash);
   RAW_RETURN_CHECK(
       taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)));
-  RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false, NULL, 0));
+  RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false, NULL, 0, false));
   RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
 
   launchQueryImpl(pRequest, pQuery, true, NULL);
@@ -1678,7 +1678,7 @@ int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const cha
   RAW_NULL_CHECK(pVgHash);
   RAW_RETURN_CHECK(
       taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)));
-  RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, NULL, 0, false, NULL, 0));
+  RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, NULL, 0, false, NULL, 0, false));
   RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
 
   launchQueryImpl(pRequest, pQuery, true, NULL);
@@ -1960,35 +1960,32 @@ static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t da
       vgId = vg->vgId;
     }
 
-    STableMeta** pTableMeta = (STableMeta**)taosHashGet(pMetaHash, tbName, strlen(tbName));
-    if (pTableMeta == NULL) {
+    STableMeta* pTableMeta = NULL;
+    STableMeta** pTableMetaTmp = (STableMeta**)taosHashGet(pMetaHash, tbName, strlen(tbName));
+    if (pTableMetaTmp == NULL) {
       if (pCreateReqDst) {  // change stable name to get meta
         (void)strcpy(pName.tname, pCreateReqDst->ctb.stbName);
       }
-      RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, &conn, &pName, pTableMeta));
-      RAW_RETURN_CHECK(taosHashPut(pMetaHash, tbName, strlen(tbName), pTableMeta, POINTER_BYTES));
-      if (pCreateReqDst) {
-        (*pTableMeta)->vgId = vgId;
-        (*pTableMeta)->uid = pCreateReqDst->uid;
-        pCreateReqDst->ctb.suid = (*pTableMeta)->suid;
+      RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta));
+      code = taosHashPut(pMetaHash, tbName, strlen(tbName), &pTableMeta, POINTER_BYTES);
+      if (code != 0){
+        taosMemoryFree(pTableMeta);
+        goto end;
       }
+      if (pCreateReqDst) {
+        pTableMeta->vgId = vgId;
+        pTableMeta->uid = pCreateReqDst->uid;
+        pCreateReqDst->ctb.suid = pTableMeta->suid;
+      }
+    }else{
+      pTableMeta = *pTableMetaTmp;
     }
 
-//    SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter);
-//    RAW_NULL_CHECK(pSW);
-//    TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD));
-//    RAW_NULL_CHECK(fields);
-//
-//    for (int i = 0; i < pSW->nCols; i++) {
-//      fields[i].type = pSW->pSchema[i].type;
-//      fields[i].bytes = pSW->pSchema[i].bytes;
-//      tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name));
-//    }
+    SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter);
+    RAW_NULL_CHECK(pSW);
     void* rawData = getRawDataFromRes(pRetrieve);
     char  err[ERR_MSG_LEN] = {0};
-    code = rawBlockBindData(pQuery, *pTableMeta, rawData, pCreateReqDst, NULL, 0, true, err, ERR_MSG_LEN);
-//    code = rawBlockBindData(pQuery, *pTableMeta, rawData, pCreateReqDst, fields, pSW->nCols, true, err, ERR_MSG_LEN);
-//    taosMemoryFree(fields);
+    code = rawBlockBindData(pQuery, pTableMeta, rawData, pCreateReqDst, pSW, pSW->nCols, true, err, ERR_MSG_LEN, true);
     if (code != TSDB_CODE_SUCCESS) {
       SET_ERROR_MSG("table:%s, err:%s", tbName, err);
       goto end;
diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c
index 838f797394..f29ed79412 100644
--- a/source/libs/parser/src/parInsertUtil.c
+++ b/source/libs/parser/src/parInsertUtil.c
@@ -886,8 +886,8 @@ static bool findFileds(SSchema* pSchema, TAOS_FIELD* fields, int numFields) {
   return false;
 }
 
-int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, TAOS_FIELD* tFields,
-                     int numFields, bool needChangeLength, char* errstr, int32_t errstrLen) {
+int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, void* tFields,
+                     int numFields, bool needChangeLength, char* errstr, int32_t errstrLen, bool raw) {
   int ret = 0;
   if(data == NULL) {
     uError("rawBlockBindData, data is NULL");
@@ -964,11 +964,16 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
     goto end;
   }
   if (tFields != NULL && numFields > boundInfo->numOfBound) {
-    if (errstr != NULL)
-      snprintf(errstr, errstrLen, "numFields:%d bigger than num of bound cols:%d", numFields, boundInfo->numOfBound);
+    if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d bigger than num of bound cols:%d", numFields, boundInfo->numOfBound);
     ret = TSDB_CODE_INVALID_PARA;
     goto end;
   }
+  if (tFields == NULL && numOfCols != boundInfo->numOfBound) {
+    if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d not equal to num of bound cols:%d", numOfCols, boundInfo->numOfBound);
+    ret = TSDB_CODE_INVALID_PARA;
+    goto end;
+  }
+
   if (tFields == NULL) {
     for (int j = 0; j < boundInfo->numOfBound; j++) {
       SSchema*  pColSchema = &pSchema[j];
@@ -1006,7 +1011,13 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
     for (int i = 0; i < numFields; i++) {
       for (int j = 0; j < boundInfo->numOfBound; j++) {
         SSchema* pColSchema = &pSchema[j];
-        if (strcmp(pColSchema->name, tFields[i].name) == 0) {
+        char* fieldName = NULL;
+        if (raw) {
+          fieldName = ((SSchemaWrapper*)tFields)->pSchema[i].name;
+        } else {
+          fieldName = ((TAOS_FIELD*)tFields)[i].name;
+        }
+        if (strcmp(pColSchema->name, fieldName) == 0) {
           if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) {
             if (errstr != NULL)
               snprintf(errstr, errstrLen,
diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c
index 3a79a3763c..d49c7e4ad4 100644
--- a/utils/test/c/tmq_taosx_ci.c
+++ b/utils/test/c/tmq_taosx_ci.c
@@ -79,6 +79,7 @@ static void msg_process(TAOS_RES* msg) {
         } else {
           taosFprintfFile(g_fp, result);
           taosFprintfFile(g_fp, "\n");
+          taosFsyncFile(g_fp);
         }
       }
     }

From 0bcaf6f4e5324553b7beee173824a2c8ece4de03 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 9 Oct 2024 15:30:28 +0800
Subject: [PATCH 05/28] enh:[TS-5441] change poll flag to consumer for multi
 consumers

---
 source/client/src/clientTmq.c             | 16 +++++++++-------
 source/dnode/mnode/impl/src/mndConsumer.c |  1 +
 source/dnode/vnode/src/tq/tqRead.c        |  2 +-
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 975d14f3ee..4cbb808187 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -134,6 +134,7 @@ struct tmq_t {
   // poll info
   int64_t pollCnt;
   int64_t totalRows;
+  int8_t  pollFlag;
 
   // timer
   tmr_h       hbLiveTimer;
@@ -287,7 +288,6 @@ typedef struct {
 static   TdThreadOnce   tmqInit = PTHREAD_ONCE_INIT;  // initialize only once
 volatile int32_t        tmqInitRes = 0;               // initialize rsp code
 static   SMqMgmt        tmqMgmt = {0};
-static   int8_t         pollFlag = 0;
 
 tmq_conf_t* tmq_conf_new() {
   tmq_conf_t* conf = taosMemoryCalloc(1, sizeof(tmq_conf_t));
@@ -977,7 +977,8 @@ void tmqSendHbReq(void* param, void* tmrId) {
   SMqHbReq req = {0};
   req.consumerId = tmq->consumerId;
   req.epoch = tmq->epoch;
-  req.pollFlag = atomic_load_8(&pollFlag);
+  req.pollFlag = atomic_load_8(&tmq->pollFlag);
+  tqDebugC("consumer:0x%" PRIx64 " send hb, pollFlag:%d", tmq->consumerId, req.pollFlag);
   req.topics = taosArrayInit(taosArrayGetSize(tmq->clientTopics), sizeof(TopicOffsetRows));
   if (req.topics == NULL) {
     goto END;
@@ -1057,13 +1058,13 @@ void tmqSendHbReq(void* param, void* tmrId) {
   if (code != 0) {
     tqErrorC("tmqSendHbReq asyncSendMsgToServer failed");
   }
-  (void)atomic_val_compare_exchange_8(&pollFlag, 1, 0);
+  (void)atomic_val_compare_exchange_8(&tmq->pollFlag, 1, 0);
 
 END:
   tDestroySMqHbReq(&req);
   if (tmrId != NULL) {
     bool ret = taosTmrReset(tmqSendHbReq, tmq->heartBeatIntervalMs, param, tmqMgmt.timer, &tmq->hbLiveTimer);
-    tqDebugC("reset timer fo tmq hb:%d", ret);
+    tqDebugC("consumer:0x%" PRIx64 " reset timer for tmq hb:%d, pollFlag:%d", tmq->consumerId, ret, tmq->pollFlag);
   }
   int32_t ret = taosReleaseRef(tmqMgmt.rsetId, refId);
   if (ret != 0){
@@ -1422,7 +1423,7 @@ void tmqHandleAllDelayedTask(tmq_t* pTmq) {
       tqDebugC("consumer:0x%" PRIx64 " retrieve ep from mnode in 1s", pTmq->consumerId);
       bool ret = taosTmrReset(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(pTmq->refId), tmqMgmt.timer,
                               &pTmq->epTimer);
-      tqDebugC("reset timer fo tmq ask ep:%d", ret);
+      tqDebugC("reset timer for tmq ask ep:%d", ret);
     } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) {
       tmq_commit_cb* pCallbackFn = (pTmq->commitCb != NULL) ? pTmq->commitCb : defaultCommitCbFn;
       asyncCommitAllOffsets(pTmq, pCallbackFn, pTmq->commitCbUserParam);
@@ -1430,7 +1431,7 @@ void tmqHandleAllDelayedTask(tmq_t* pTmq) {
                pTmq->autoCommitInterval / 1000.0);
       bool ret = taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, (void*)(pTmq->refId), tmqMgmt.timer,
                               &pTmq->commitTimer);
-      tqDebugC("reset timer fo commit:%d", ret);
+      tqDebugC("reset timer for commit:%d", ret);
     } else {
       tqErrorC("consumer:0x%" PRIx64 " invalid task type:%d", pTmq->consumerId, *pTaskType);
     }
@@ -1640,6 +1641,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
   pTmq->status = TMQ_CONSUMER_STATUS__INIT;
   pTmq->pollCnt = 0;
   pTmq->epoch = 0;
+  pTmq->pollFlag = 0;
 
   // set conf
   tstrncpy(pTmq->clientId, conf->clientId, TSDB_CLIENT_ID_LEN);
@@ -2441,7 +2443,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
     return NULL;
   }
 
-  (void)atomic_val_compare_exchange_8(&pollFlag, 0, 1);
+  (void)atomic_val_compare_exchange_8(&tmq->pollFlag, 0, 1);
 
   while (1) {
     tmqHandleAllDelayedTask(tmq);
diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c
index a96b8b22f5..a91d2379ae 100644
--- a/source/dnode/mnode/impl/src/mndConsumer.c
+++ b/source/dnode/mnode/impl/src/mndConsumer.c
@@ -239,6 +239,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
   MND_TMQ_RETURN_CHECK(mndAcquireConsumer(pMnode, consumerId, &pConsumer));
   MND_TMQ_RETURN_CHECK(checkPrivilege(pMnode, pConsumer, &rsp, pMsg->info.conn.user));
   atomic_store_32(&pConsumer->hbStatus, 0);
+  mDebug("consumer:0x%" PRIx64 " receive hb pollFlag:%d %d", consumerId, req.pollFlag, pConsumer->pollStatus);
   if (req.pollFlag == 1){
     atomic_store_32(&pConsumer->pollStatus, 0);
   }
diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c
index 6dc5453d50..7ca749b5a3 100644
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@@ -953,7 +953,7 @@ END:
 }
 
 int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet) {
-  tqDebug("tq reader retrieve data block %p, %d", pReader->msg.msgStr, pReader->nextBlk);
+  tqTrace("tq reader retrieve data block %p, %d", pReader->msg.msgStr, pReader->nextBlk);
   SSDataBlock* block = NULL;
 
   SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);

From e6bf8dcfde498c8e73f3ce723d23230366e88c78 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 9 Oct 2024 18:10:43 +0800
Subject: [PATCH 06/28] enh:[TS-5441] cost too long in tmq write meta data by
 cache meta and vg info

---
 include/common/tmsg.h                   | 23 +++++-----
 include/libs/executor/executor.h        |  3 +-
 source/client/src/clientRawBlockWrite.c | 55 +++++++++++++++++++---
 source/common/src/tmsg.c                | 40 ++++++++++++++++
 source/dnode/vnode/src/tq/tqScan.c      | 61 ++++++++++++-------------
 source/dnode/vnode/src/tq/tqUtil.c      | 10 +++-
 source/libs/executor/inc/querytask.h    |  1 +
 source/libs/executor/src/executor.c     |  6 +++
 8 files changed, 145 insertions(+), 54 deletions(-)

diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index 1a10f02c96..fb118de907 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -4049,18 +4049,17 @@ void    tDeleteMqMetaRsp(SMqMetaRsp* pRsp);
 #define MQ_DATA_RSP_VERSION 100
 
 typedef struct {
-  struct {
-    SMqRspHead   head;
-    STqOffsetVal rspOffset;
-    STqOffsetVal reqOffset;
-    int32_t      blockNum;
-    int8_t       withTbName;
-    int8_t       withSchema;
-    SArray*      blockDataLen;
-    SArray*      blockData;
-    SArray*      blockTbName;
-    SArray*      blockSchema;
-  };
+  SMqRspHead   head;
+  STqOffsetVal rspOffset;
+  STqOffsetVal reqOffset;
+  int32_t      blockNum;
+  int8_t       withTbName;
+  int8_t       withSchema;
+  SArray*      blockDataLen;
+  SArray*      blockData;
+  SArray*      blockTbName;
+  SArray*      blockSchema;
+  SArray*      blockSuid;
 
   union{
     struct{
diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h
index ae26d5f2ae..6b02d8f985 100644
--- a/include/libs/executor/executor.h
+++ b/include/libs/executor/executor.h
@@ -209,7 +209,8 @@ SMqBatchMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo);
 
 const SSchemaWrapper* qExtractSchemaFromTask(qTaskInfo_t tinfo);
 
-const char* qExtractTbnameFromTask(qTaskInfo_t tinfo);
+const char*   qExtractTbnameFromTask(qTaskInfo_t tinfo);
+const int64_t qExtractSuidFromTask(qTaskInfo_t tinfo);
 
 void* qExtractReaderFromStreamScanner(void* scanner);
 
diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 064c3bdb2e..4b2e9e234d 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -1856,6 +1856,42 @@ static threadlocal SHashObj*     pCreateTbHash = NULL;
 static threadlocal SHashObj*     pNameHash = NULL;
 static threadlocal SHashObj*     pMetaHash = NULL;
 
+static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrapper* pSW){
+  char* p = (char*)rawData;
+  // | version | total length | total rows | blankFill | total columns | flag seg| block group id | column schema | each
+  // column length |
+  p += sizeof(int32_t);
+  p += sizeof(int32_t);
+  p += sizeof(int32_t);
+  p += sizeof(int32_t);
+  p += sizeof(int32_t);
+  p += sizeof(uint64_t);
+  int8_t* fields = p;
+
+  if (pSW->nCols != pTableMeta->tableInfo.numOfColumns) {
+    return true;
+  }
+  for (int i = 0; i < pSW->nCols; i++) {
+    int j = 0;
+    for (; j < pTableMeta->tableInfo.numOfColumns; j++) {
+      SSchema* pColSchema = &pTableMeta->schema[j];
+      char*    fieldName = pSW->pSchema[i].name;
+
+      if (strcmp(pColSchema->name, fieldName) == 0) {
+        if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) {
+          return true;
+        }
+        break;
+      }
+    }
+    fields += sizeof(int8_t) + sizeof(int32_t);
+
+    if (j == pTableMeta->tableInfo.numOfColumns)
+      return true;
+  }
+  return false;
+}
+
 static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t dataLen) {
   if (taos == NULL || data == NULL) {
     SET_ERROR_MSG("taos:%p or data:%p is NULL", taos, data);
@@ -1905,7 +1941,7 @@ static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t da
     RAW_NULL_CHECK(pNameHash);
   }
   if (pMetaHash == NULL){
-    pMetaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
+    pMetaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
     RAW_NULL_CHECK(pMetaHash);
     taosHashSetFreeFp(pMetaHash, taosMemoryFree);
   }
@@ -1931,6 +1967,9 @@ static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t da
     const char* tbName = (const char*)taosArrayGetP(rspObj.dataRsp.blockTbName, rspObj.resIter);
     RAW_NULL_CHECK(tbName);
 
+    int64_t* suid = taosArrayGet(rspObj.dataRsp.blockSuid, rspObj.resIter);
+    RAW_NULL_CHECK(suid);
+
     uDebug(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName);
     SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
     (void)strcpy(pName.dbname, pRequest->pDb);
@@ -1960,14 +1999,19 @@ static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t da
       vgId = vg->vgId;
     }
 
+    SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter);
+    RAW_NULL_CHECK(pSW);
+    void* rawData = getRawDataFromRes(pRetrieve);
+    RAW_NULL_CHECK(rawData);
+
     STableMeta* pTableMeta = NULL;
-    STableMeta** pTableMetaTmp = (STableMeta**)taosHashGet(pMetaHash, tbName, strlen(tbName));
-    if (pTableMetaTmp == NULL) {
+    STableMeta** pTableMetaTmp = (STableMeta**)taosHashGet(pMetaHash, suid, POINTER_BYTES);
+    if (pTableMetaTmp == NULL || needRefreshMeta(rawData, *pTableMetaTmp, pSW)) {
       if (pCreateReqDst) {  // change stable name to get meta
         (void)strcpy(pName.tname, pCreateReqDst->ctb.stbName);
       }
       RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta));
-      code = taosHashPut(pMetaHash, tbName, strlen(tbName), &pTableMeta, POINTER_BYTES);
+      code = taosHashPut(pMetaHash, suid, POINTER_BYTES, &pTableMeta, POINTER_BYTES);
       if (code != 0){
         taosMemoryFree(pTableMeta);
         goto end;
@@ -1981,9 +2025,6 @@ static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t da
       pTableMeta = *pTableMetaTmp;
     }
 
-    SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter);
-    RAW_NULL_CHECK(pSW);
-    void* rawData = getRawDataFromRes(pRetrieve);
     char  err[ERR_MSG_LEN] = {0};
     code = rawBlockBindData(pQuery, pTableMeta, rawData, pCreateReqDst, pSW, pSW->nCols, true, err, ERR_MSG_LEN, true);
     if (code != TSDB_CODE_SUCCESS) {
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index 986747fe58..41516d325a 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -10716,12 +10716,42 @@ _exit:
   return code;
 }
 
+int32_t tEncodeSuidArray(SEncoder *pEncoder, const SMqDataRsp *pRsp){
+  for (int32_t i = 0; i < pRsp->blockNum; i++) {
+    if (pRsp->withTbName) {
+      int64_t* suid = taosArrayGet(pRsp->blockSuid, i);
+      if (suid != NULL){
+        TAOS_CHECK_RETURN(tEncodeI64(pEncoder, *suid));
+      }
+    }
+  }
+  return 0;
+}
 int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
   TAOS_CHECK_RETURN(tEncodeMqDataRspCommon(pEncoder, pRsp));
   TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pRsp->sleepTime));
+  TAOS_CHECK_RETURN(tEncodeSuidArray(pEncoder, pRsp));
+
   return 0;
 }
+int32_t tDecodeSuidArray(SDecoder *pDecoder, SMqDataRsp *pRsp){
+  if (!tDecodeIsEnd(pDecoder)) {
+    if (pRsp->withTbName) {
+      if ((pRsp->blockSuid = taosArrayInit(pRsp->blockNum, sizeof(int64_t))) == NULL) {
+        TAOS_CHECK_RETURN(terrno);
+      }
+    }
 
+    for (int32_t i = 0; i < pRsp->blockNum; i++) {
+      int64_t suid = 0;
+      TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &suid));
+      if (taosArrayPush(pRsp->blockSuid, &suid) == NULL) {
+        TAOS_CHECK_RETURN(terrno);
+      }
+    }
+  }
+  return 0;
+}
 int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRsp *pRsp) {
   int32_t code = 0;
   int32_t lino;
@@ -10798,6 +10828,9 @@ int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) {
   if (!tDecodeIsEnd(pDecoder)) {
     TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pRsp->sleepTime));
   }
+  if (!tDecodeIsEnd(pDecoder)) {
+    TAOS_CHECK_RETURN(tDecodeSuidArray(pDecoder, pRsp));
+  }
 
   return 0;
 }
@@ -10811,6 +10844,8 @@ static void tDeleteMqDataRspCommon(SMqDataRsp *pRsp) {
   pRsp->blockSchema = NULL;
   taosArrayDestroyP(pRsp->blockTbName, (FDelete)taosMemoryFree);
   pRsp->blockTbName = NULL;
+  taosArrayDestroy(pRsp->blockSuid);
+  pRsp->blockSuid = NULL;
   tOffsetDestroy(&pRsp->reqOffset);
   tOffsetDestroy(&pRsp->rspOffset);
 }
@@ -10830,6 +10865,8 @@ int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
       TAOS_CHECK_EXIT(tEncodeBinary(pEncoder, createTableReq, createTableLen));
     }
   }
+  TAOS_CHECK_EXIT(tEncodeSuidArray(pEncoder, pRsp));
+
 _exit:
   return code;
 }
@@ -10860,6 +10897,9 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) {
       }
     }
   }
+  if (!tDecodeIsEnd(pDecoder)) {
+    TAOS_CHECK_EXIT(tDecodeSuidArray(pDecoder, pRsp));
+  }
 
 _exit:
   return code;
diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c
index dbc1b16cf5..a95437ab0d 100644
--- a/source/dnode/vnode/src/tq/tqScan.c
+++ b/source/dnode/vnode/src/tq/tqScan.c
@@ -77,6 +77,14 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, i
       tqError("failed to push tbName to blockTbName:%s", tbName);
       continue;
     }
+    int64_t suid = 0;
+    if(mr.me.type == TSDB_CHILD_TABLE){
+      suid = mr.me.ctbEntry.suid;
+    }
+    if(taosArrayPush(pRsp->blockSuid, &suid) == NULL){
+      tqError("failed to push suid to blockSuid:%"PRId64, suid);
+      continue;
+    }
   }
   metaReaderClear(&mr);
   return 0;
@@ -210,36 +218,26 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBat
 
     if (pDataBlock != NULL && pDataBlock->info.rows > 0) {
       if (pRsp->withTbName) {
-        if (pOffset->type == TMQ_OFFSET__LOG) {
-          int64_t uid = pExec->pTqReader->lastBlkUid;
-          if (tqAddTbNameToRsp(pTq, uid, pRsp, 1) < 0) {
-            tqError("vgId:%d, failed to add tbname to rsp msg", pTq->pVnode->config.vgId);
-            continue;
-          }
-        } else {
-          char* tbName = taosStrdup(qExtractTbnameFromTask(task));
-          if (tbName == NULL) {
-            tqError("vgId:%d, failed to add tbname to rsp msg, null", pTq->pVnode->config.vgId);
-            return terrno;
-          }
-          if (taosArrayPush(pRsp->blockTbName, &tbName) == NULL){
-            tqError("vgId:%d, failed to add tbname to rsp msg", pTq->pVnode->config.vgId);
-            continue;
-          }
+        char* tbName = taosStrdup(qExtractTbnameFromTask(task));
+        if (tbName == NULL) {
+          tqError("vgId:%d, failed to add tbname to rsp msg, null", pTq->pVnode->config.vgId);
+          return terrno;
+        }
+        if (taosArrayPush(pRsp->blockTbName, &tbName) == NULL){
+          tqError("vgId:%d, failed to add tbname to rsp msg", pTq->pVnode->config.vgId);
+          continue;
+        }
+        int64_t suid = qExtractSuidFromTask(task);
+        if (taosArrayPush(pRsp->blockSuid, &suid) == NULL){
+          tqError("vgId:%d, failed to add suid to rsp msg", pTq->pVnode->config.vgId);
+          continue;
         }
       }
       if (pRsp->withSchema) {
-        if (pOffset->type == TMQ_OFFSET__LOG) {
-          if (tqAddBlockSchemaToRsp(pExec, pRsp) != 0){
-            tqError("vgId:%d, failed to add schema to rsp msg", pTq->pVnode->config.vgId);
-            continue;
-          }
-        } else {
-          SSchemaWrapper* pSW = tCloneSSchemaWrapper(qExtractSchemaFromTask(task));
-          if(taosArrayPush(pRsp->blockSchema, &pSW) == NULL){
-            tqError("vgId:%d, failed to add schema to rsp msg", pTq->pVnode->config.vgId);
-            continue;
-          }
+        SSchemaWrapper* pSW = tCloneSSchemaWrapper(qExtractSchemaFromTask(task));
+        if(taosArrayPush(pRsp->blockSchema, &pSW) == NULL){
+          tqError("vgId:%d, failed to add schema to rsp msg", pTq->pVnode->config.vgId);
+          continue;
         }
       }
 
@@ -249,12 +247,9 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBat
         continue;
       }
       pRsp->blockNum++;
-      if (pOffset->type == TMQ_OFFSET__LOG) {
-        continue;
-      } else {
-        rowCnt += pDataBlock->info.rows;
-        if (rowCnt <= tmqRowSize) continue;
-      }
+      rowCnt += pDataBlock->info.rows;
+      if (rowCnt <= tmqRowSize) continue;
+
     }
 
     // get meta
diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c
index b4866b8c65..891e55d91d 100644
--- a/source/dnode/vnode/src/tq/tqUtil.c
+++ b/source/dnode/vnode/src/tq/tqUtil.c
@@ -50,8 +50,11 @@ static int32_t tqInitTaosxRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) {
   pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t));
   pRsp->blockTbName = taosArrayInit(0, sizeof(void*));
   pRsp->blockSchema = taosArrayInit(0, sizeof(void*));
+  pRsp->blockSuid = taosArrayInit(0, sizeof(int64_t));
 
-  if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL || pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) {
+  if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL ||
+      pRsp->blockTbName == NULL || pRsp->blockSchema == NULL ||
+      pRsp->blockSuid == NULL) {
     if (pRsp->blockData != NULL) {
       taosArrayDestroy(pRsp->blockData);
       pRsp->blockData = NULL;
@@ -71,6 +74,11 @@ static int32_t tqInitTaosxRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) {
       taosArrayDestroy(pRsp->blockSchema);
       pRsp->blockSchema = NULL;
     }
+
+    if (pRsp->blockSuid != NULL) {
+      taosArrayDestroy(pRsp->blockSuid);
+      pRsp->blockSuid = NULL;
+    }
     return terrno;
   }
 
diff --git a/source/libs/executor/inc/querytask.h b/source/libs/executor/inc/querytask.h
index e3bb9a1361..3c52f8080e 100644
--- a/source/libs/executor/inc/querytask.h
+++ b/source/libs/executor/inc/querytask.h
@@ -69,6 +69,7 @@ typedef struct {
   SVersionRange        fillHistoryVer;
   STimeWindow          fillHistoryWindow;
   SStreamState*        pState;
+  int64_t              suid;           // for tmq
 } SStreamTaskInfo;
 
 struct SExecTaskInfo {
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index cbf392f67e..08b7ba0e05 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -1212,6 +1212,11 @@ const char* qExtractTbnameFromTask(qTaskInfo_t tinfo) {
   return pTaskInfo->streamInfo.tbName;
 }
 
+const int64_t qExtractSuidFromTask(qTaskInfo_t tinfo) {
+  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
+  return pTaskInfo->streamInfo.suid;
+}
+
 SMqBatchMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo) {
   SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
   return &pTaskInfo->streamInfo.btMetaRsp;
@@ -1494,6 +1499,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
 
       cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
       tstrncpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName, TSDB_TABLE_NAME_LEN);
+      pTaskInfo->streamInfo.suid = mtInfo.suid;
       tDeleteSchemaWrapper(pTaskInfo->streamInfo.schema);
       pTaskInfo->streamInfo.schema = mtInfo.schema;
 

From 5f9de6ac70b4848c088226ccf2448c598527bf10 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 9 Oct 2024 18:34:14 +0800
Subject: [PATCH 07/28] enh:[TS-5441] cost too long in tmq write meta data by
 cache meta and vg info

---
 source/client/src/clientRawBlockWrite.c | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 4b2e9e234d..94162ff768 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -1892,6 +1892,11 @@ static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrappe
   return false;
 }
 
+static void tmqFreeMeta(void *data){
+  STableMeta* pTableMeta = *(STableMeta**)data;
+  taosMemoryFree(pTableMeta);
+}
+
 static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t dataLen) {
   if (taos == NULL || data == NULL) {
     SET_ERROR_MSG("taos:%p or data:%p is NULL", taos, data);
@@ -1943,7 +1948,7 @@ static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t da
   if (pMetaHash == NULL){
     pMetaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
     RAW_NULL_CHECK(pMetaHash);
-    taosHashSetFreeFp(pMetaHash, taosMemoryFree);
+    taosHashSetFreeFp(pMetaHash, tmqFreeMeta);
   }
   struct SCatalog* pCatalog = NULL;
   RAW_RETURN_CHECK(catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog));
@@ -2005,13 +2010,13 @@ static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t da
     RAW_NULL_CHECK(rawData);
 
     STableMeta* pTableMeta = NULL;
-    STableMeta** pTableMetaTmp = (STableMeta**)taosHashGet(pMetaHash, suid, POINTER_BYTES);
+    STableMeta** pTableMetaTmp = (STableMeta**)taosHashGet(pMetaHash, suid, LONG_BYTES);
     if (pTableMetaTmp == NULL || needRefreshMeta(rawData, *pTableMetaTmp, pSW)) {
       if (pCreateReqDst) {  // change stable name to get meta
         (void)strcpy(pName.tname, pCreateReqDst->ctb.stbName);
       }
       RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta));
-      code = taosHashPut(pMetaHash, suid, POINTER_BYTES, &pTableMeta, POINTER_BYTES);
+      code = taosHashPut(pMetaHash, suid, LONG_BYTES, &pTableMeta, POINTER_BYTES);
       if (code != 0){
         taosMemoryFree(pTableMeta);
         goto end;

From 42b7520d34356355d8d45c645a2f730c43a2c8fc Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Thu, 10 Oct 2024 18:27:42 +0800
Subject: [PATCH 08/28] enh:[TS-5441] cost too long in tmq write meta data by
 cache meta and vg info

---
 include/libs/executor/executor.h        |   2 +-
 source/client/src/clientRawBlockWrite.c | 271 ++++++++++++++----------
 source/client/src/clientTmq.c           |   2 +-
 source/common/src/tmsg.c                |  78 +++----
 source/dnode/vnode/src/tq/tqScan.c      |  28 +--
 source/dnode/vnode/src/tq/tqUtil.c      |  13 +-
 source/libs/executor/src/executor.c     |  10 +-
 source/libs/parser/src/parInsertUtil.c  |  15 +-
 tests/system-test/7-tmq/tmq_taosx.py    |   6 +-
 utils/test/c/tmq_taosx_ci.c             |  45 +++-
 10 files changed, 281 insertions(+), 189 deletions(-)

diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h
index 6b02d8f985..8501d88be0 100644
--- a/include/libs/executor/executor.h
+++ b/include/libs/executor/executor.h
@@ -210,7 +210,7 @@ SMqBatchMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo);
 const SSchemaWrapper* qExtractSchemaFromTask(qTaskInfo_t tinfo);
 
 const char*   qExtractTbnameFromTask(qTaskInfo_t tinfo);
-const int64_t qExtractSuidFromTask(qTaskInfo_t tinfo);
+//const int64_t qExtractSuidFromTask(qTaskInfo_t tinfo);
 
 void* qExtractReaderFromStreamScanner(void* scanner);
 
diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 94162ff768..578f8148da 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -53,9 +53,7 @@
 #define TMQ_META_VERSION "1.0"
 
 static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen);
-
 static tb_uid_t processSuid(tb_uid_t suid, char* db) { return suid + MurmurHash3_32(db, strlen(db)); }
-
 static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, int8_t t,
                                  SColCmprWrapper* pColCmprRow, cJSON** pJson) {
   int32_t code = TSDB_CODE_SUCCESS;
@@ -1856,6 +1854,12 @@ static threadlocal SHashObj*     pCreateTbHash = NULL;
 static threadlocal SHashObj*     pNameHash = NULL;
 static threadlocal SHashObj*     pMetaHash = NULL;
 
+typedef struct{
+  SVgroupInfo vgInfo;
+  int64_t     uid;
+  int64_t     suid;
+}tbInfo;
+
 static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrapper* pSW){
   char* p = (char*)rawData;
   // | version | total length | total rows | blankFill | total columns | flag seg| block group id | column schema | each
@@ -1878,7 +1882,7 @@ static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrappe
       char*    fieldName = pSW->pSchema[i].name;
 
       if (strcmp(pColSchema->name, fieldName) == 0) {
-        if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) {
+        if (*fields != pColSchema->type || *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) {
           return true;
         }
         break;
@@ -1892,11 +1896,6 @@ static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrappe
   return false;
 }
 
-static void tmqFreeMeta(void *data){
-  STableMeta* pTableMeta = *(STableMeta**)data;
-  taosMemoryFree(pTableMeta);
-}
-
 static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t dataLen) {
   if (taos == NULL || data == NULL) {
     SET_ERROR_MSG("taos:%p or data:%p is NULL", taos, data);
@@ -1933,23 +1932,6 @@ static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t da
     goto end;
   }
 
-  if (pVgHash == NULL){
-    pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
-    RAW_NULL_CHECK(pVgHash);
-  }
-  if (pCreateTbHash == NULL){
-    pCreateTbHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
-    RAW_NULL_CHECK(pCreateTbHash);
-  }
-  if (pNameHash == NULL){
-    pNameHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
-    RAW_NULL_CHECK(pNameHash);
-  }
-  if (pMetaHash == NULL){
-    pMetaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
-    RAW_NULL_CHECK(pMetaHash);
-    taosHashSetFreeFp(pMetaHash, tmqFreeMeta);
-  }
   struct SCatalog* pCatalog = NULL;
   RAW_RETURN_CHECK(catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog));
 
@@ -1959,90 +1941,119 @@ static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t da
   conn.requestObjRefId = pRequest->self;
   conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
 
-  RAW_RETURN_CHECK(smlInitHandle(&pQuery));
+  int retry = 0;
+  while(1){
+    RAW_RETURN_CHECK(smlInitHandle(&pQuery));
 
-  uDebug(LOG_ID_TAG " write raw data type:%d block num:%d", LOG_ID_VALUE, type, rspObj.dataRsp.blockNum);
-  while (++rspObj.resIter < rspObj.dataRsp.blockNum) {
-    void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter);
-    RAW_NULL_CHECK(pRetrieve);
-    if (!rspObj.dataRsp.withSchema) {
-      goto end;
-    }
-
-    const char* tbName = (const char*)taosArrayGetP(rspObj.dataRsp.blockTbName, rspObj.resIter);
-    RAW_NULL_CHECK(tbName);
-
-    int64_t* suid = taosArrayGet(rspObj.dataRsp.blockSuid, rspObj.resIter);
-    RAW_NULL_CHECK(suid);
-
-    uDebug(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName);
-    SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
-    (void)strcpy(pName.dbname, pRequest->pDb);
-    (void)strcpy(pName.tname, tbName);
-
-    // find schema data info
-    SVCreateTbReq* pCreateReqDst = NULL;
-    if (type == RES_TYPE__TMQ_METADATA){
-      pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, tbName, strlen(tbName));
-      if (pCreateReqDst == NULL) {
-        RAW_RETURN_CHECK(buildCreateTbMap(&rspObj.dataRsp, pCreateTbHash));
-        pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, tbName, strlen(tbName));
-      }
-    }
-
-    int32_t vgId = 0;
-    SVgroupInfo* vg = (SVgroupInfo*)taosHashGet(pNameHash, tbName, strlen(tbName));
-    if (vg == NULL) {
-      SVgroupInfo vgTmp = {0};
-      RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vgTmp));
-      RAW_RETURN_CHECK(taosHashPut(pNameHash, tbName, strlen(tbName), &vgTmp, sizeof(SVgroupInfo)));
-      code = taosHashPut(pVgHash, &vgTmp.vgId, sizeof(vgTmp.vgId), &vgTmp, sizeof(SVgroupInfo));
-      code = (code == TSDB_CODE_DUP_KEY) ? 0 : code;
-      RAW_RETURN_CHECK(code);
-      vgId = vgTmp.vgId;
-    } else {
-      vgId = vg->vgId;
-    }
-
-    SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter);
-    RAW_NULL_CHECK(pSW);
-    void* rawData = getRawDataFromRes(pRetrieve);
-    RAW_NULL_CHECK(rawData);
-
-    STableMeta* pTableMeta = NULL;
-    STableMeta** pTableMetaTmp = (STableMeta**)taosHashGet(pMetaHash, suid, LONG_BYTES);
-    if (pTableMetaTmp == NULL || needRefreshMeta(rawData, *pTableMetaTmp, pSW)) {
-      if (pCreateReqDst) {  // change stable name to get meta
-        (void)strcpy(pName.tname, pCreateReqDst->ctb.stbName);
-      }
-      RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta));
-      code = taosHashPut(pMetaHash, suid, LONG_BYTES, &pTableMeta, POINTER_BYTES);
-      if (code != 0){
-        taosMemoryFree(pTableMeta);
+    uDebug(LOG_ID_TAG " write raw data type:%d block num:%d", LOG_ID_VALUE, type, rspObj.dataRsp.blockNum);
+    while (++rspObj.resIter < rspObj.dataRsp.blockNum) {
+      void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter);
+      RAW_NULL_CHECK(pRetrieve);
+      if (!rspObj.dataRsp.withSchema) {
         goto end;
       }
-      if (pCreateReqDst) {
-        pTableMeta->vgId = vgId;
-        pTableMeta->uid = pCreateReqDst->uid;
-        pCreateReqDst->ctb.suid = pTableMeta->suid;
+
+      const char* tbName = (const char*)taosArrayGetP(rspObj.dataRsp.blockTbName, rspObj.resIter);
+      RAW_NULL_CHECK(tbName);
+
+//      int64_t* suid = taosArrayGet(rspObj.dataRsp.blockSuid, rspObj.resIter);
+//      RAW_NULL_CHECK(suid);
+
+      uDebug(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName);
+      SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
+      (void)strcpy(pName.dbname, pRequest->pDb);
+      (void)strcpy(pName.tname, tbName);
+
+      // find schema data info
+      SVCreateTbReq* pCreateReqDst = NULL;
+      if (type == RES_TYPE__TMQ_METADATA){
+        pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, tbName, strlen(tbName));
+        if (pCreateReqDst == NULL) {
+          RAW_RETURN_CHECK(buildCreateTbMap(&rspObj.dataRsp, pCreateTbHash));
+          pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, tbName, strlen(tbName));
+        }
       }
-    }else{
-      pTableMeta = *pTableMetaTmp;
-    }
+      STableMeta* pTableMeta = NULL;
+      tbInfo* tmpInfo = (tbInfo*)taosHashGet(pNameHash, tbName, strlen(tbName));
+      if (tmpInfo == NULL || retry > 0) {
+        tbInfo info = {0};
 
-    char  err[ERR_MSG_LEN] = {0};
-    code = rawBlockBindData(pQuery, pTableMeta, rawData, pCreateReqDst, pSW, pSW->nCols, true, err, ERR_MSG_LEN, true);
-    if (code != TSDB_CODE_SUCCESS) {
-      SET_ERROR_MSG("table:%s, err:%s", tbName, err);
-      goto end;
+        RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &info.vgInfo));
+        if (pCreateReqDst) {  // change stable name to get meta
+          (void)strcpy(pName.tname, pCreateReqDst->ctb.stbName);
+        }
+        RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta));
+        info.uid = pTableMeta->uid;
+        if (pTableMeta->tableType == TSDB_CHILD_TABLE){
+          info.suid = pTableMeta->suid;
+        } else {
+          info.suid = pTableMeta->uid;
+        }
+        code = taosHashPut(pMetaHash, &info.suid, LONG_BYTES, &pTableMeta, POINTER_BYTES);
+        if (code != 0){
+          taosMemoryFree(pTableMeta);
+          goto end;
+        }
+        if (pCreateReqDst) {
+          pTableMeta->vgId = info.vgInfo.vgId;
+          pTableMeta->uid = pCreateReqDst->uid;
+          pCreateReqDst->ctb.suid = pTableMeta->suid;
+        }
+
+        code = taosHashPut(pNameHash, pName.tname, strlen(pName.tname), &info, sizeof(tbInfo));
+        code = (code == TSDB_CODE_DUP_KEY) ? 0 : code;
+        RAW_RETURN_CHECK(code);
+        code = taosHashPut(pVgHash, &info.vgInfo.vgId, sizeof(info.vgInfo.vgId), &info.vgInfo, sizeof(SVgroupInfo));
+        code = (code == TSDB_CODE_DUP_KEY) ? 0 : code;
+        RAW_RETURN_CHECK(code);
+      }
+
+      SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter);
+      RAW_NULL_CHECK(pSW);
+      void* rawData = getRawDataFromRes(pRetrieve);
+      RAW_NULL_CHECK(rawData);
+
+      if (pTableMeta == NULL || retry > 0){
+        STableMeta** pTableMetaTmp = (STableMeta**)taosHashGet(pMetaHash, &tmpInfo->suid, LONG_BYTES);
+        if (pTableMetaTmp == NULL || retry > 0 || needRefreshMeta(rawData, *pTableMetaTmp, pSW)) {
+          RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta));
+          code = taosHashPut(pMetaHash, &tmpInfo->suid, LONG_BYTES, &pTableMeta, POINTER_BYTES);
+          if (code != 0){
+            taosMemoryFree(pTableMeta);
+            goto end;
+          }
+
+        }else{
+          pTableMeta = *pTableMetaTmp;
+          pTableMeta->uid = tmpInfo->uid;
+          pTableMeta->vgId = tmpInfo->vgInfo.vgId;
+        }
+      }
+
+      char  err[ERR_MSG_LEN] = {0};
+      code = rawBlockBindData(pQuery, pTableMeta, rawData, pCreateReqDst, pSW, pSW->nCols, true, err, ERR_MSG_LEN, true);
+      if (code != TSDB_CODE_SUCCESS) {
+        SET_ERROR_MSG("table:%s, err:%s", tbName, err);
+        goto end;
+      }
     }
+    RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
+    launchQueryImpl(pRequest, pQuery, true, NULL);
+    code = pRequest->code;
+
+    if (NEED_CLIENT_HANDLE_ERROR(code)) {
+      uInfo("write raw retry:%d/3 end code:%d, msg:%s", retry, code, tstrerror(code));
+      if (retry++ >= 3) {
+        break;
+      }
+      qDestroyQuery(pQuery);
+      pQuery = NULL;
+      rspObj.resIter = -1;
+      continue;
+    }
+    break;
   }
 
-  RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
-
-  launchQueryImpl(pRequest, pQuery, true, NULL);
-  code = pRequest->code;
-
   end:
   uDebug(LOG_ID_TAG " write raw metadata return, msg:%s", LOG_ID_VALUE, tstrerror(code));
   if (type == RES_TYPE__TMQ_METADATA){
@@ -2056,18 +2067,6 @@ static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t da
   return code;
 }
 
-void tmqClean() {
-  taosHashCleanup(pMetaHash);
-  taosHashCleanup(pNameHash);
-  void* pIter = taosHashIterate(pCreateTbHash, NULL);
-  while (pIter) {
-    tDestroySVCreateTbReq(pIter, TSDB_MSG_FLG_DECODE);
-    pIter = taosHashIterate(pCreateTbHash, pIter);
-  }
-  taosHashCleanup(pCreateTbHash);
-  taosHashCleanup(pVgHash);
-}
-
 static void processSimpleMeta(SMqMetaRsp* pMetaRsp, cJSON** meta) {
   if (pMetaRsp->resMsgType == TDMT_VND_CREATE_STB) {
     processCreateStb(pMetaRsp, meta);
@@ -2263,7 +2262,53 @@ void tmq_free_raw(tmq_raw_data raw) {
   (void)memset(terrMsg, 0, ERR_MSG_LEN);
 }
 
+static void tmqFreeMeta(void *data){
+  STableMeta* pTableMeta = *(STableMeta**)data;
+  taosMemoryFree(pTableMeta);
+}
+
+void freeHash() {
+  taosHashCleanup(pMetaHash);
+  taosHashCleanup(pNameHash);
+  void* pIter = taosHashIterate(pCreateTbHash, NULL);
+  while (pIter) {
+    tDestroySVCreateTbReq(pIter, TSDB_MSG_FLG_DECODE);
+    pIter = taosHashIterate(pCreateTbHash, pIter);
+  }
+  taosHashCleanup(pCreateTbHash);
+  taosHashCleanup(pVgHash);
+}
+
+static int32_t initHash(){
+  int32_t code = 0;
+  if (pVgHash == NULL){
+    pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
+    RAW_NULL_CHECK(pVgHash);
+  }
+  if (pCreateTbHash == NULL){
+    pCreateTbHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
+    RAW_NULL_CHECK(pCreateTbHash);
+  }
+  if (pNameHash == NULL){
+    pNameHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+    RAW_NULL_CHECK(pNameHash);
+  }
+  if (pMetaHash == NULL){
+    pMetaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
+    RAW_NULL_CHECK(pMetaHash);
+    taosHashSetFreeFp(pMetaHash, tmqFreeMeta);
+  }
+  return code;
+end:
+  freeHash();
+  return code;
+}
+
 static int32_t writeRawImpl(TAOS* taos, void* buf, uint32_t len, uint16_t type) {
+  int32_t code = initHash();
+  if (code != 0) {
+    return code;
+  }
   if (type == TDMT_VND_CREATE_STB) {
     return taosCreateStb(taos, buf, len);
   } else if (type == TDMT_VND_ALTER_STB) {
diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 4cbb808187..6d0323e20a 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -1064,7 +1064,7 @@ END:
   tDestroySMqHbReq(&req);
   if (tmrId != NULL) {
     bool ret = taosTmrReset(tmqSendHbReq, tmq->heartBeatIntervalMs, param, tmqMgmt.timer, &tmq->hbLiveTimer);
-    tqDebugC("consumer:0x%" PRIx64 " reset timer for tmq hb:%d, pollFlag:%d", tmq->consumerId, ret, tmq->pollFlag);
+    tqDebugC("consumer:0x%" PRIx64 " reset timer for tmq heartbeat:%d, pollFlag:%d", tmq->consumerId, ret, tmq->pollFlag);
   }
   int32_t ret = taosReleaseRef(tmqMgmt.rsetId, refId);
   if (ret != 0){
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index 41516d325a..e585212372 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -10716,42 +10716,42 @@ _exit:
   return code;
 }
 
-int32_t tEncodeSuidArray(SEncoder *pEncoder, const SMqDataRsp *pRsp){
-  for (int32_t i = 0; i < pRsp->blockNum; i++) {
-    if (pRsp->withTbName) {
-      int64_t* suid = taosArrayGet(pRsp->blockSuid, i);
-      if (suid != NULL){
-        TAOS_CHECK_RETURN(tEncodeI64(pEncoder, *suid));
-      }
-    }
-  }
-  return 0;
-}
+//int32_t tEncodeSuidArray(SEncoder *pEncoder, const SMqDataRsp *pRsp){
+//  for (int32_t i = 0; i < pRsp->blockNum; i++) {
+//    if (pRsp->withTbName) {
+//      int64_t* suid = taosArrayGet(pRsp->blockSuid, i);
+//      if (suid != NULL){
+//        TAOS_CHECK_RETURN(tEncodeI64(pEncoder, *suid));
+//      }
+//    }
+//  }
+//  return 0;
+//}
 int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
   TAOS_CHECK_RETURN(tEncodeMqDataRspCommon(pEncoder, pRsp));
   TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pRsp->sleepTime));
-  TAOS_CHECK_RETURN(tEncodeSuidArray(pEncoder, pRsp));
+//  TAOS_CHECK_RETURN(tEncodeSuidArray(pEncoder, pRsp));
 
   return 0;
 }
-int32_t tDecodeSuidArray(SDecoder *pDecoder, SMqDataRsp *pRsp){
-  if (!tDecodeIsEnd(pDecoder)) {
-    if (pRsp->withTbName) {
-      if ((pRsp->blockSuid = taosArrayInit(pRsp->blockNum, sizeof(int64_t))) == NULL) {
-        TAOS_CHECK_RETURN(terrno);
-      }
-    }
-
-    for (int32_t i = 0; i < pRsp->blockNum; i++) {
-      int64_t suid = 0;
-      TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &suid));
-      if (taosArrayPush(pRsp->blockSuid, &suid) == NULL) {
-        TAOS_CHECK_RETURN(terrno);
-      }
-    }
-  }
-  return 0;
-}
+//int32_t tDecodeSuidArray(SDecoder *pDecoder, SMqDataRsp *pRsp){
+//  if (!tDecodeIsEnd(pDecoder)) {
+//    if (pRsp->withTbName) {
+//      if ((pRsp->blockSuid = taosArrayInit(pRsp->blockNum, sizeof(int64_t))) == NULL) {
+//        TAOS_CHECK_RETURN(terrno);
+//      }
+//    }
+//
+//    for (int32_t i = 0; i < pRsp->blockNum; i++) {
+//      int64_t suid = 0;
+//      TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &suid));
+//      if (taosArrayPush(pRsp->blockSuid, &suid) == NULL) {
+//        TAOS_CHECK_RETURN(terrno);
+//      }
+//    }
+//  }
+//  return 0;
+//}
 int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRsp *pRsp) {
   int32_t code = 0;
   int32_t lino;
@@ -10828,9 +10828,9 @@ int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) {
   if (!tDecodeIsEnd(pDecoder)) {
     TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pRsp->sleepTime));
   }
-  if (!tDecodeIsEnd(pDecoder)) {
-    TAOS_CHECK_RETURN(tDecodeSuidArray(pDecoder, pRsp));
-  }
+//  if (!tDecodeIsEnd(pDecoder)) {
+//    TAOS_CHECK_RETURN(tDecodeSuidArray(pDecoder, pRsp));
+//  }
 
   return 0;
 }
@@ -10844,8 +10844,8 @@ static void tDeleteMqDataRspCommon(SMqDataRsp *pRsp) {
   pRsp->blockSchema = NULL;
   taosArrayDestroyP(pRsp->blockTbName, (FDelete)taosMemoryFree);
   pRsp->blockTbName = NULL;
-  taosArrayDestroy(pRsp->blockSuid);
-  pRsp->blockSuid = NULL;
+//  taosArrayDestroy(pRsp->blockSuid);
+//  pRsp->blockSuid = NULL;
   tOffsetDestroy(&pRsp->reqOffset);
   tOffsetDestroy(&pRsp->rspOffset);
 }
@@ -10865,7 +10865,7 @@ int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
       TAOS_CHECK_EXIT(tEncodeBinary(pEncoder, createTableReq, createTableLen));
     }
   }
-  TAOS_CHECK_EXIT(tEncodeSuidArray(pEncoder, pRsp));
+//  TAOS_CHECK_EXIT(tEncodeSuidArray(pEncoder, pRsp));
 
 _exit:
   return code;
@@ -10897,9 +10897,9 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) {
       }
     }
   }
-  if (!tDecodeIsEnd(pDecoder)) {
-    TAOS_CHECK_EXIT(tDecodeSuidArray(pDecoder, pRsp));
-  }
+//  if (!tDecodeIsEnd(pDecoder)) {
+//    TAOS_CHECK_EXIT(tDecodeSuidArray(pDecoder, pRsp));
+//  }
 
 _exit:
   return code;
diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c
index a95437ab0d..68702754aa 100644
--- a/source/dnode/vnode/src/tq/tqScan.c
+++ b/source/dnode/vnode/src/tq/tqScan.c
@@ -77,14 +77,16 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, i
       tqError("failed to push tbName to blockTbName:%s", tbName);
       continue;
     }
-    int64_t suid = 0;
-    if(mr.me.type == TSDB_CHILD_TABLE){
-      suid = mr.me.ctbEntry.suid;
-    }
-    if(taosArrayPush(pRsp->blockSuid, &suid) == NULL){
-      tqError("failed to push suid to blockSuid:%"PRId64, suid);
-      continue;
-    }
+//    int64_t suid = 0;
+//    if(mr.me.type == TSDB_CHILD_TABLE){
+//      suid = mr.me.ctbEntry.suid;
+//    }else{
+//      suid = mr.me.uid;
+//    }
+//    if(taosArrayPush(pRsp->blockSuid, &suid) == NULL){
+//      tqError("failed to push suid to blockSuid:%"PRId64, suid);
+//      continue;
+//    }
   }
   metaReaderClear(&mr);
   return 0;
@@ -227,11 +229,11 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBat
           tqError("vgId:%d, failed to add tbname to rsp msg", pTq->pVnode->config.vgId);
           continue;
         }
-        int64_t suid = qExtractSuidFromTask(task);
-        if (taosArrayPush(pRsp->blockSuid, &suid) == NULL){
-          tqError("vgId:%d, failed to add suid to rsp msg", pTq->pVnode->config.vgId);
-          continue;
-        }
+//        int64_t suid = qExtractSuidFromTask(task);
+//        if (taosArrayPush(pRsp->blockSuid, &suid) == NULL){
+//          tqError("vgId:%d, failed to add suid to rsp msg", pTq->pVnode->config.vgId);
+//          continue;
+//        }
       }
       if (pRsp->withSchema) {
         SSchemaWrapper* pSW = tCloneSSchemaWrapper(qExtractSchemaFromTask(task));
diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c
index 891e55d91d..6acfe6b074 100644
--- a/source/dnode/vnode/src/tq/tqUtil.c
+++ b/source/dnode/vnode/src/tq/tqUtil.c
@@ -50,11 +50,10 @@ static int32_t tqInitTaosxRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) {
   pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t));
   pRsp->blockTbName = taosArrayInit(0, sizeof(void*));
   pRsp->blockSchema = taosArrayInit(0, sizeof(void*));
-  pRsp->blockSuid = taosArrayInit(0, sizeof(int64_t));
+//  pRsp->blockSuid = taosArrayInit(0, sizeof(int64_t));
 
   if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL ||
-      pRsp->blockTbName == NULL || pRsp->blockSchema == NULL ||
-      pRsp->blockSuid == NULL) {
+      pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) {
     if (pRsp->blockData != NULL) {
       taosArrayDestroy(pRsp->blockData);
       pRsp->blockData = NULL;
@@ -75,10 +74,10 @@ static int32_t tqInitTaosxRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) {
       pRsp->blockSchema = NULL;
     }
 
-    if (pRsp->blockSuid != NULL) {
-      taosArrayDestroy(pRsp->blockSuid);
-      pRsp->blockSuid = NULL;
-    }
+//    if (pRsp->blockSuid != NULL) {
+//      taosArrayDestroy(pRsp->blockSuid);
+//      pRsp->blockSuid = NULL;
+//    }
     return terrno;
   }
 
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index 08b7ba0e05..b5af516f75 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -1212,10 +1212,10 @@ const char* qExtractTbnameFromTask(qTaskInfo_t tinfo) {
   return pTaskInfo->streamInfo.tbName;
 }
 
-const int64_t qExtractSuidFromTask(qTaskInfo_t tinfo) {
-  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
-  return pTaskInfo->streamInfo.suid;
-}
+//const int64_t qExtractSuidFromTask(qTaskInfo_t tinfo) {
+//  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
+//  return pTaskInfo->streamInfo.suid;
+//}
 
 SMqBatchMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo) {
   SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
@@ -1499,7 +1499,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
 
       cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
       tstrncpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName, TSDB_TABLE_NAME_LEN);
-      pTaskInfo->streamInfo.suid = mtInfo.suid;
+      pTaskInfo->streamInfo.suid = mtInfo.suid == 0 ? mtInfo.uid : mtInfo.suid;
       tDeleteSchemaWrapper(pTaskInfo->streamInfo.schema);
       pTaskInfo->streamInfo.schema = mtInfo.schema;
 
diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c
index f29ed79412..8adf32d2dd 100644
--- a/source/libs/parser/src/parInsertUtil.c
+++ b/source/libs/parser/src/parInsertUtil.c
@@ -963,11 +963,11 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
     ret = TSDB_CODE_INVALID_PARA;
     goto end;
   }
-  if (tFields != NULL && numFields > boundInfo->numOfBound) {
-    if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d bigger than num of bound cols:%d", numFields, boundInfo->numOfBound);
-    ret = TSDB_CODE_INVALID_PARA;
-    goto end;
-  }
+//  if (tFields != NULL && numFields > boundInfo->numOfBound) {
+//    if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d bigger than num of bound cols:%d", numFields, boundInfo->numOfBound);
+//    ret = TSDB_CODE_INVALID_PARA;
+//    goto end;
+//  }
   if (tFields == NULL && numOfCols != boundInfo->numOfBound) {
     if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d not equal to num of bound cols:%d", numOfCols, boundInfo->numOfBound);
     ret = TSDB_CODE_INVALID_PARA;
@@ -1037,6 +1037,11 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
             pStart += numOfRows * sizeof(int32_t);
           } else {
             pStart += BitmapLen(numOfRows);
+//            for(int k = 0; k < numOfRows; k++) {
+//              if(!colDataIsNull_f(offset, k) && pColSchema->type == TSDB_DATA_TYPE_INT){
+//                printf("colName:%s,val:%d", fieldName, *(int32_t*)(pStart + k * sizeof(int32_t)));
+//              }
+//            }
           }
           char* pData = pStart;
 
diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py
index 4e90aefe7c..5047ada1d1 100644
--- a/tests/system-test/7-tmq/tmq_taosx.py
+++ b/tests/system-test/7-tmq/tmq_taosx.py
@@ -131,14 +131,14 @@ class TDTestCase:
         tdSql.checkData(0, 2, 1)
 
         tdSql.query("select * from ct3 order by c1 desc")
-        tdSql.checkRows(2)
+        tdSql.checkRows(5)
         tdSql.checkData(0, 1, 51)
         tdSql.checkData(0, 4, 940)
         tdSql.checkData(1, 1, 23)
         tdSql.checkData(1, 4, None)
 
         tdSql.query("select * from st1 order by ts")
-        tdSql.checkRows(8)
+        tdSql.checkRows(14)
         tdSql.checkData(0, 1, 1)
         tdSql.checkData(1, 1, 3)
         tdSql.checkData(4, 1, 4)
@@ -180,7 +180,7 @@ class TDTestCase:
         tdSql.checkData(6, 8, None)
 
         tdSql.query("select * from ct1")
-        tdSql.checkRows(4)
+        tdSql.checkRows(7)
 
         tdSql.query("select * from ct2")
         tdSql.checkRows(0)
diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c
index d49c7e4ad4..cd70dd88f5 100644
--- a/utils/test/c/tmq_taosx_ci.c
+++ b/utils/test/c/tmq_taosx_ci.c
@@ -133,7 +133,7 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes) {
 
   pRes = taos_query(pConn, "create table if not exists ct0 using st1 tags(1000, \"ttt\", true)");
   if (taos_errno(pRes) != 0) {
-    printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes));
+    printf("failed to create child table ct0, reason:%s\n", taos_errstr(pRes));
     return -1;
   }
   taos_free_result(pRes);
@@ -176,7 +176,7 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes) {
   pRes = taos_query(
       pConn,
       "insert into ct3 values(1626006833600, 5, 6, 'c') ct1 values(1626006833601, 2, 3, 'sds') (1626006833602, 4, 5, "
-      "'ddd') ct0 values(1626006833603, 4, 3, 'hwj') ct1 values(now+5s, 23, 32, 's21ds')");
+      "'ddd') ct0 values(1626006833603, 4, 3, 'hwj') ct1 values(1626006833703, 23, 32, 's21ds')");
   if (taos_errno(pRes) != 0) {
     printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes));
     return -1;
@@ -190,6 +190,41 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes) {
   }
   taos_free_result(pRes);
 
+  pRes = taos_query(pConn, "insert into ct1 values(1736006813600, -32222, 43, 'ewb', 99)");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
+  pRes = taos_query(pConn, "alter table st1 drop column c4");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
+  pRes = taos_query(pConn, "insert into ct1 values(1736006833600, -4223, 344, 'bfs')");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
+  pRes = taos_query(pConn, "alter table st1 add column c4 bigint");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
+  pRes = taos_query(pConn, "insert into ct1 values(1766006833600, -4432, 4433, 'e23wb', 9349)");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
   pRes = taos_query(pConn, "alter table st1 modify column c3 binary(64)");
   if (taos_errno(pRes) != 0) {
     printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes));
@@ -597,6 +632,7 @@ tmq_t* build_consumer() {
   tmq_conf_set(conf, "enable.auto.commit", "true");
   tmq_conf_set(conf, "auto.offset.reset", "earliest");
   tmq_conf_set(conf, "msg.consume.excluded", "1");
+//  tmq_conf_set(conf, "session.timeout.ms", "1000000");
 //  tmq_conf_set(conf, "max.poll.interval.ms", "20000");
 
   if (g_conf.snapShot) {
@@ -637,6 +673,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
     TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 5000);
     if (tmqmessage) {
       cnt++;
+      printf("cnt:%d\n", cnt);
       msg_process(tmqmessage);
       taos_free_result(tmqmessage);
     } else {
@@ -845,6 +882,8 @@ void initLogFile() {
             "{\"name\":\"t1\",\"type\":4,\"value\":3000}],\"createList\":[]}",
             "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\","
             "\"colType\":5}",
+            "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":6,\"colName\":\"c4\"}",
+            "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\",\"colType\":5}",
             "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":7,\"colName\":\"c3\","
             "\"colType\":8,\"colLength\":64}",
             "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":1,\"colName\":\"t2\","
@@ -992,6 +1031,8 @@ void initLogFile() {
             "{\"name\":\"t1\",\"type\":4,\"value\":3000}],\"createList\":[]}",
             "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\","
             "\"colType\":5}",
+            "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":6,\"colName\":\"c4\"}",
+            "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\",\"colType\":5}",
             "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":7,\"colName\":\"c3\","
             "\"colType\":8,\"colLength\":64}",
             "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":1,\"colName\":\"t2\","

From 52f003e47408bb6aacdea7cfe0969ad7368a8b8e Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Fri, 11 Oct 2024 15:50:28 +0800
Subject: [PATCH 09/28] enh:[TS-5441] cost too long in tmq write meta data by
 cache meta and vg info

---
 source/client/src/clientRawBlockWrite.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 358df25481..94e0b9bec0 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -1821,6 +1821,7 @@ static int32_t buildCreateTbMap(SMqDataRsp* rsp, SHashObj* pHashObj) {
   int32_t       code = 0;
   SVCreateTbReq pCreateReq = {0};
   SDecoder      decoderTmp = {0};
+  SVCreateTbReq *pCreateReqTmp = NULL;
 
   for (int j = 0; j < rsp->createTableNum; j++) {
     void** dataTmp = taosArrayGet(rsp->createTableReq, j);
@@ -1836,8 +1837,9 @@ static int32_t buildCreateTbMap(SMqDataRsp* rsp, SHashObj* pHashObj) {
       goto end;
     }
     if (taosHashGet(pHashObj, pCreateReq.name, strlen(pCreateReq.name)) == NULL) {
-      RAW_RETURN_CHECK(
-          taosHashPut(pHashObj, pCreateReq.name, strlen(pCreateReq.name), &pCreateReq, sizeof(SVCreateTbReq)));
+      RAW_RETURN_CHECK(cloneSVreateTbReq(&pCreateReq, &pCreateReqTmp));
+      RAW_RETURN_CHECK(taosHashPut(pHashObj, pCreateReq.name, strlen(pCreateReq.name), pCreateReqTmp, sizeof(SVCreateTbReq)));
+      pCreateReqTmp = NULL;
     } else {
       tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE);
       pCreateReq = (SVCreateTbReq){0};
@@ -1850,6 +1852,7 @@ static int32_t buildCreateTbMap(SMqDataRsp* rsp, SHashObj* pHashObj) {
 end:
   tDecoderClear(&decoderTmp);
   tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE);
+  tDestroySVCreateTbReq(pCreateReqTmp, TSDB_MSG_FLG_DECODE);
   return code;
 }
 
@@ -1944,6 +1947,7 @@ static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t da
   conn.requestId = pRequest->requestId;
   conn.requestObjRefId = pRequest->self;
   conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
+  RAW_RETURN_CHECK(buildCreateTbMap(&rspObj.dataRsp, pCreateTbHash));
 
   int retry = 0;
   while(1){
@@ -1971,10 +1975,6 @@ static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t da
       SVCreateTbReq* pCreateReqDst = NULL;
       if (type == RES_TYPE__TMQ_METADATA){
         pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, tbName, strlen(tbName));
-        if (pCreateReqDst == NULL) {
-          RAW_RETURN_CHECK(buildCreateTbMap(&rspObj.dataRsp, pCreateTbHash));
-          pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, tbName, strlen(tbName));
-        }
       }
       STableMeta* pTableMeta = NULL;
       tbInfo* tmpInfo = (tbInfo*)taosHashGet(pNameHash, tbName, strlen(tbName));

From ddc3676752b46b156dccbb68bfd77c4aeb555fed Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Fri, 11 Oct 2024 16:26:48 +0800
Subject: [PATCH 10/28] enh:[TS-5441] cost too long in tmq write meta data by
 cache meta and vg info

---
 source/client/src/clientRawBlockWrite.c | 38 +++++++++++++------------
 1 file changed, 20 insertions(+), 18 deletions(-)

diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 94e0b9bec0..800973196a 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -1821,7 +1821,6 @@ static int32_t buildCreateTbMap(SMqDataRsp* rsp, SHashObj* pHashObj) {
   int32_t       code = 0;
   SVCreateTbReq pCreateReq = {0};
   SDecoder      decoderTmp = {0};
-  SVCreateTbReq *pCreateReqTmp = NULL;
 
   for (int j = 0; j < rsp->createTableNum; j++) {
     void** dataTmp = taosArrayGet(rsp->createTableReq, j);
@@ -1837,9 +1836,7 @@ static int32_t buildCreateTbMap(SMqDataRsp* rsp, SHashObj* pHashObj) {
       goto end;
     }
     if (taosHashGet(pHashObj, pCreateReq.name, strlen(pCreateReq.name)) == NULL) {
-      RAW_RETURN_CHECK(cloneSVreateTbReq(&pCreateReq, &pCreateReqTmp));
-      RAW_RETURN_CHECK(taosHashPut(pHashObj, pCreateReq.name, strlen(pCreateReq.name), pCreateReqTmp, sizeof(SVCreateTbReq)));
-      pCreateReqTmp = NULL;
+      RAW_RETURN_CHECK(taosHashPut(pHashObj, pCreateReq.name, strlen(pCreateReq.name), &pCreateReq, sizeof(SVCreateTbReq)));
     } else {
       tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE);
       pCreateReq = (SVCreateTbReq){0};
@@ -1852,12 +1849,11 @@ static int32_t buildCreateTbMap(SMqDataRsp* rsp, SHashObj* pHashObj) {
 end:
   tDecoderClear(&decoderTmp);
   tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE);
-  tDestroySVCreateTbReq(pCreateReqTmp, TSDB_MSG_FLG_DECODE);
   return code;
 }
 
 static threadlocal SHashObj*     pVgHash = NULL;
-static threadlocal SHashObj*     pCreateTbHash = NULL;
+//static threadlocal SHashObj*     pCreateTbHash = NULL;
 static threadlocal SHashObj*     pNameHash = NULL;
 static threadlocal SHashObj*     pMetaHash = NULL;
 
@@ -1912,7 +1908,8 @@ static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t da
   SQuery*        pQuery = NULL;
   SMqRspObj      rspObj = {0};
   SDecoder       decoder = {0};
-  SRequestObj* pRequest = NULL;
+  SHashObj*      pCreateTbHash = NULL;
+  SRequestObj*   pRequest = NULL;
   RAW_RETURN_CHECK(createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0, &pRequest));
 
   uDebug(LOG_ID_TAG " write raw metadata, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen);
@@ -1947,7 +1944,12 @@ static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t da
   conn.requestId = pRequest->requestId;
   conn.requestObjRefId = pRequest->self;
   conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
-  RAW_RETURN_CHECK(buildCreateTbMap(&rspObj.dataRsp, pCreateTbHash));
+
+  if (type == RES_TYPE__TMQ_METADATA) {
+    pCreateTbHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+    RAW_NULL_CHECK(pCreateTbHash);
+    RAW_RETURN_CHECK(buildCreateTbMap(&rspObj.dataRsp, pCreateTbHash));
+  }
 
   int retry = 0;
   while(1){
@@ -2064,6 +2066,12 @@ static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t da
   }else {
     tDeleteMqDataRsp(&rspObj.dataRsp);
   }
+  void* pIter = taosHashIterate(pCreateTbHash, NULL);
+  while (pIter) {
+    tDestroySVCreateTbReq(pIter, TSDB_MSG_FLG_DECODE);
+    pIter = taosHashIterate(pCreateTbHash, pIter);
+  }
+  taosHashCleanup(pCreateTbHash);
   tDecoderClear(&decoder);
   qDestroyQuery(pQuery);
   destroyRequest(pRequest);
@@ -2273,12 +2281,6 @@ static void tmqFreeMeta(void *data){
 void freeHash() {
   taosHashCleanup(pMetaHash);
   taosHashCleanup(pNameHash);
-  void* pIter = taosHashIterate(pCreateTbHash, NULL);
-  while (pIter) {
-    tDestroySVCreateTbReq(pIter, TSDB_MSG_FLG_DECODE);
-    pIter = taosHashIterate(pCreateTbHash, pIter);
-  }
-  taosHashCleanup(pCreateTbHash);
   taosHashCleanup(pVgHash);
 }
 
@@ -2288,10 +2290,10 @@ static int32_t initHash(){
     pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
     RAW_NULL_CHECK(pVgHash);
   }
-  if (pCreateTbHash == NULL){
-    pCreateTbHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
-    RAW_NULL_CHECK(pCreateTbHash);
-  }
+//  if (pCreateTbHash == NULL){
+//    pCreateTbHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
+//    RAW_NULL_CHECK(pCreateTbHash);
+//  }
   if (pNameHash == NULL){
     pNameHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
     RAW_NULL_CHECK(pNameHash);

From 471871bff53126f30af89d1ce5a8ee3eecee86bd Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Fri, 11 Oct 2024 17:02:22 +0800
Subject: [PATCH 11/28] enh:[TS-5441] cost too long in tmq write meta data by
 cache meta and vg info

---
 source/client/src/clientRawBlockWrite.c | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 800973196a..d3c2b53fb6 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -1984,7 +1984,7 @@ static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t da
         tbInfo info = {0};
 
         RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &info.vgInfo));
-        if (pCreateReqDst) {  // change stable name to get meta
+        if (pCreateReqDst && tmpInfo == NULL) {  // change stable name to get meta
           (void)strcpy(pName.tname, pCreateReqDst->ctb.stbName);
         }
         RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta));
@@ -2005,12 +2005,13 @@ static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t da
           pCreateReqDst->ctb.suid = pTableMeta->suid;
         }
 
-        code = taosHashPut(pNameHash, pName.tname, strlen(pName.tname), &info, sizeof(tbInfo));
-        code = (code == TSDB_CODE_DUP_KEY) ? 0 : code;
-        RAW_RETURN_CHECK(code);
-        code = taosHashPut(pVgHash, &info.vgInfo.vgId, sizeof(info.vgInfo.vgId), &info.vgInfo, sizeof(SVgroupInfo));
-        code = (code == TSDB_CODE_DUP_KEY) ? 0 : code;
-        RAW_RETURN_CHECK(code);
+        RAW_RETURN_CHECK(taosHashPut(pNameHash, pName.tname, strlen(pName.tname), &info, sizeof(tbInfo)));
+        tmpInfo = (tbInfo*)taosHashGet(pNameHash, pName.tname, strlen(pName.tname));
+//        code = (code == TSDB_CODE_DUP_KEY) ? 0 : code;
+//        RAW_RETURN_CHECK(code);
+        RAW_RETURN_CHECK(taosHashPut(pVgHash, &info.vgInfo.vgId, sizeof(info.vgInfo.vgId), &info.vgInfo, sizeof(SVgroupInfo)));
+//        code = (code == TSDB_CODE_DUP_KEY) ? 0 : code;
+//        RAW_RETURN_CHECK(code);
       }
 
       SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter);

From 3cb60211de79c8cad72c811e1bb60075936a2ae7 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Sat, 12 Oct 2024 15:43:58 +0800
Subject: [PATCH 12/28] enh:[TS-5441] cost too long in tmq write meta data by
 cache meta and vg info

---
 source/libs/executor/src/scanoperator.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index bae9926f63..3713a2c071 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -2877,6 +2877,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
     return 0;
   }
 
+  blockDataFreeRes((SSDataBlock*)pBlock);
   code = calBlockTbName(pInfo, pInfo->pRes, 0);
   QUERY_CHECK_CODE(code, lino, _end);
 

From 10436cacffbf07ddcccca22b392f521d6bada41b Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Tue, 15 Oct 2024 16:17:58 +0800
Subject: [PATCH 13/28] enh:[TS-5441] cost too long in tmq write meta data by
 cache meta and vg info

---
 source/client/src/clientTmq.c           | 10 ++++++----
 source/libs/executor/inc/querytask.h    |  2 +-
 source/libs/executor/src/executor.c     |  2 +-
 source/libs/executor/src/scanoperator.c |  1 -
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 020bf25e40..ce701a755c 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -831,8 +831,8 @@ static int32_t innerCommitAll(tmq_t* tmq, SMqCommitCbParamSet* pParamSet){
       }
 
       code = innerCommit(tmq, pTopic->topicName, &pVg->offsetInfo.endOffset, pVg, pParamSet);
-      if (code != 0){
-        tqDebugC("consumer:0x%" PRIx64 " topic:%s vgId:%d, no commit, code:%s, current offset version:%" PRId64 ", ordinal:%d/%d",
+      if (code != 0 && code != TSDB_CODE_TMQ_SAME_COMMITTED_VALUE){
+        tqErrorC("consumer:0x%" PRIx64 " topic:%s vgId:%d, no commit, code:%s, current offset version:%" PRId64 ", ordinal:%d/%d",
                  tmq->consumerId, pTopic->topicName, pVg->vgId, tstrerror(code), pVg->offsetInfo.endOffset.version, j + 1, numOfVgroups);
       }
     }
@@ -857,7 +857,7 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us
     return;
   }
   code = innerCommitAll(tmq, pParamSet);
-  if (code != 0){
+  if (code != 0 && code != TSDB_CODE_TMQ_SAME_COMMITTED_VALUE){
     tqErrorC("consumer:0x%" PRIx64 " innerCommitAll failed, code:%s", tmq->consumerId, tstrerror(code));
   }
 
@@ -1270,7 +1270,9 @@ static int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) {
   }
 
   if (code != TSDB_CODE_SUCCESS) {
-    tqErrorC("consumer:0x%" PRIx64 ", get topic endpoint error, code:%s", tmq->consumerId, tstrerror(code));
+    if (code != TSDB_CODE_MND_CONSUMER_NOT_READY){
+      tqErrorC("consumer:0x%" PRIx64 ", get topic endpoint error, code:%s", tmq->consumerId, tstrerror(code));
+    }
     goto END;
   }
 
diff --git a/source/libs/executor/inc/querytask.h b/source/libs/executor/inc/querytask.h
index 3c52f8080e..c9e65bacaf 100644
--- a/source/libs/executor/inc/querytask.h
+++ b/source/libs/executor/inc/querytask.h
@@ -69,7 +69,7 @@ typedef struct {
   SVersionRange        fillHistoryVer;
   STimeWindow          fillHistoryWindow;
   SStreamState*        pState;
-  int64_t              suid;           // for tmq
+//  int64_t              suid;           // for tmq
 } SStreamTaskInfo;
 
 struct SExecTaskInfo {
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index 4e5a1d2e1e..4cfa12be5b 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -1499,7 +1499,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
 
       cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
       tstrncpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName, TSDB_TABLE_NAME_LEN);
-      pTaskInfo->streamInfo.suid = mtInfo.suid == 0 ? mtInfo.uid : mtInfo.suid;
+//      pTaskInfo->streamInfo.suid = mtInfo.suid == 0 ? mtInfo.uid : mtInfo.suid;
       tDeleteSchemaWrapper(pTaskInfo->streamInfo.schema);
       pTaskInfo->streamInfo.schema = mtInfo.schema;
 
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 3713a2c071..bae9926f63 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -2877,7 +2877,6 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
     return 0;
   }
 
-  blockDataFreeRes((SSDataBlock*)pBlock);
   code = calBlockTbName(pInfo, pInfo->pRes, 0);
   QUERY_CHECK_CODE(code, lino, _end);
 

From 0002ede469c58c20a8064f44f42d2b0d2c388354 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 16 Oct 2024 10:19:40 +0800
Subject: [PATCH 14/28] enh:[TS-5441] cost too long in tmq write meta data by
 cache meta and vg info

---
 source/client/src/clientRawBlockWrite.c | 116 ++++++++++++++----------
 1 file changed, 70 insertions(+), 46 deletions(-)

diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 6764dbfea5..6c54c3aa69 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -1852,10 +1852,14 @@ end:
   return code;
 }
 
-static threadlocal SHashObj*     pVgHash = NULL;
-//static threadlocal SHashObj*     pCreateTbHash = NULL;
-static threadlocal SHashObj*     pNameHash = NULL;
-static threadlocal SHashObj*     pMetaHash = NULL;
+static SHashObj* writeRawCache = NULL;
+static int8_t    initFlag = 0;
+
+typedef struct{
+  SHashObj*     pVgHash;
+  SHashObj*     pNameHash;
+  SHashObj*     pMetaHash;
+}rawCacheInfo;
 
 typedef struct{
   SVgroupInfo vgInfo;
@@ -1863,6 +1867,29 @@ typedef struct{
   int64_t     suid;
 }tbInfo;
 
+static void tmqFreeMeta(void *data){
+  STableMeta* pTableMeta = *(STableMeta**)data;
+  taosMemoryFree(pTableMeta);
+}
+
+static void freeRawCache(void *data) {
+  rawCacheInfo* pRawCache = (rawCacheInfo*)data;
+  taosHashCleanup(pRawCache->pMetaHash);
+  taosHashCleanup(pRawCache->pNameHash);
+  taosHashCleanup(pRawCache->pVgHash);
+}
+
+static int32_t initRawCacheHash(){
+  if (writeRawCache == NULL){
+    writeRawCache = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
+    if (writeRawCache == NULL){
+      return terrno;
+    }
+    taosHashSetFreeFp(writeRawCache, freeRawCache);
+  }
+  return 0;
+}
+
 static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrapper* pSW){
   char* p = (char*)rawData;
   // | version | total length | total rows | blankFill | total columns | flag seg| block group id | column schema | each
@@ -1899,6 +1926,34 @@ static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrappe
   return false;
 }
 
+static int32_t getRawCache(SHashObj **pVgHash, SHashObj **pNameHash, SHashObj **pMetaHash, void *key) {
+  int32_t code = 0;
+  void* cacheInfo = taosHashGet(writeRawCache, &key, POINTER_BYTES);
+  if (cacheInfo == NULL){
+    *pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
+    RAW_NULL_CHECK(*pVgHash);
+    *pNameHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+    RAW_NULL_CHECK(*pNameHash);
+    *pMetaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
+    RAW_NULL_CHECK(*pMetaHash);
+    taosHashSetFreeFp(*pMetaHash, tmqFreeMeta);
+    rawCacheInfo info = {*pVgHash, *pNameHash, *pMetaHash};
+    RAW_RETURN_CHECK(taosHashPut(writeRawCache, &key, POINTER_BYTES, &info, sizeof(rawCacheInfo)));
+  } else {
+    rawCacheInfo *info = (rawCacheInfo *)cacheInfo;
+    *pVgHash = info->pVgHash;
+    *pNameHash = info->pNameHash;
+    *pMetaHash = info->pMetaHash;
+  }
+
+  return 0;
+end:
+  taosHashCleanup(*pMetaHash);
+  taosHashCleanup(*pNameHash);
+  taosHashCleanup(*pVgHash);
+  return code;
+}
+
 static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t dataLen) {
   if (taos == NULL || data == NULL) {
     SET_ERROR_MSG("taos:%p or data:%p is NULL", taos, data);
@@ -1951,6 +2006,10 @@ static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t da
     RAW_RETURN_CHECK(buildCreateTbMap(&rspObj.dataRsp, pCreateTbHash));
   }
 
+  SHashObj *pVgHash = NULL;
+  SHashObj *pNameHash = NULL;
+  SHashObj *pMetaHash = NULL;
+  RAW_RETURN_CHECK(getRawCache(&pVgHash, &pNameHash, &pMetaHash, taos));
   int retry = 0;
   while(1){
     RAW_RETURN_CHECK(smlInitHandle(&pQuery));
@@ -1965,9 +2024,6 @@ static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t da
       const char* tbName = (const char*)taosArrayGetP(rspObj.dataRsp.blockTbName, rspObj.resIter);
       RAW_NULL_CHECK(tbName);
 
-//      int64_t* suid = taosArrayGet(rspObj.dataRsp.blockSuid, rspObj.resIter);
-//      RAW_NULL_CHECK(suid);
-
       uDebug(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName);
       SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
       (void)strcpy(pName.dbname, pRequest->pDb);
@@ -2274,47 +2330,15 @@ void tmq_free_raw(tmq_raw_data raw) {
   (void)memset(terrMsg, 0, ERR_MSG_LEN);
 }
 
-static void tmqFreeMeta(void *data){
-  STableMeta* pTableMeta = *(STableMeta**)data;
-  taosMemoryFree(pTableMeta);
-}
-
-void freeHash() {
-  taosHashCleanup(pMetaHash);
-  taosHashCleanup(pNameHash);
-  taosHashCleanup(pVgHash);
-}
-
-static int32_t initHash(){
-  int32_t code = 0;
-  if (pVgHash == NULL){
-    pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
-    RAW_NULL_CHECK(pVgHash);
-  }
-//  if (pCreateTbHash == NULL){
-//    pCreateTbHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
-//    RAW_NULL_CHECK(pCreateTbHash);
-//  }
-  if (pNameHash == NULL){
-    pNameHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
-    RAW_NULL_CHECK(pNameHash);
-  }
-  if (pMetaHash == NULL){
-    pMetaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
-    RAW_NULL_CHECK(pMetaHash);
-    taosHashSetFreeFp(pMetaHash, tmqFreeMeta);
-  }
-  return code;
-end:
-  freeHash();
-  return code;
-}
-
 static int32_t writeRawImpl(TAOS* taos, void* buf, uint32_t len, uint16_t type) {
-  int32_t code = initHash();
-  if (code != 0) {
-    return code;
+  int8_t old = atomic_val_compare_exchange_8(&initFlag, 0, 1);
+  if (old == 0) {
+    int32_t code = initRawCacheHash();
+    if (code != 0) {
+      return code;
+    }
   }
+
   if (type == TDMT_VND_CREATE_STB) {
     return taosCreateStb(taos, buf, len);
   } else if (type == TDMT_VND_ALTER_STB) {

From 4f399749384955e256e0ecc08abd450b041beaf8 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 16 Oct 2024 17:17:44 +0800
Subject: [PATCH 15/28] enh:[TS-5441] cost too long in tmq write meta data by
 cache meta and vg info

---
 include/util/tlog.h                        |  2 +-
 source/client/src/clientTmq.c              | 11 +--
 source/common/src/tglobal.c                |  4 +-
 source/dnode/mnode/impl/src/mndConsumer.c  |  2 +-
 source/dnode/vnode/inc/vnode.h             |  2 +-
 source/dnode/vnode/src/inc/vnodeInt.h      |  2 +-
 source/dnode/vnode/src/meta/metaQuery.c    |  7 +-
 source/dnode/vnode/src/meta/metaSnapshot.c |  2 +-
 source/dnode/vnode/src/tq/tqRead.c         | 10 +--
 source/dnode/vnode/src/tq/tqScan.c         | 92 +++++++++++++---------
 source/dnode/vnode/src/vnd/vnodeQuery.c    |  2 +-
 source/util/src/tlog.c                     |  2 +-
 12 files changed, 76 insertions(+), 62 deletions(-)

diff --git a/include/util/tlog.h b/include/util/tlog.h
index e80e94de32..1a3b687e40 100644
--- a/include/util/tlog.h
+++ b/include/util/tlog.h
@@ -69,7 +69,7 @@ extern int32_t tdbDebugFlag;
 extern int32_t sndDebugFlag;
 extern int32_t simDebugFlag;
 
-extern int32_t tqClientDebug;
+extern int32_t tqClientDebugFlag;
 
 int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc);
 void    taosCloseLog();
diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index ce701a755c..0233372012 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -24,12 +24,9 @@
 #include "tref.h"
 #include "ttimer.h"
 
-#define tqFatalC(...) do { if (cDebugFlag & DEBUG_FATAL || tqClientDebug) { taosPrintLog("TQ  FATAL ", DEBUG_FATAL, tqDebugFlag, __VA_ARGS__); }}     while(0)
-#define tqErrorC(...) do { if (cDebugFlag & DEBUG_ERROR || tqClientDebug) { taosPrintLog("TQ  ERROR ", DEBUG_ERROR, tqDebugFlag, __VA_ARGS__); }}     while(0)
-#define tqWarnC(...)  do { if (cDebugFlag & DEBUG_WARN || tqClientDebug)  { taosPrintLog("TQ  WARN ", DEBUG_WARN, tqDebugFlag, __VA_ARGS__); }}       while(0)
-#define tqInfoC(...)  do { if (cDebugFlag & DEBUG_INFO || tqClientDebug)  { taosPrintLog("TQ  ", DEBUG_INFO, tqDebugFlag, __VA_ARGS__); }}            while(0)
-#define tqDebugC(...) do { if (cDebugFlag & DEBUG_DEBUG || tqClientDebug) { taosPrintLog("TQ  ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); }} while(0)
-#define tqTraceC(...) do { if (cDebugFlag & DEBUG_TRACE || tqClientDebug) { taosPrintLog("TQ  ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0)
+#define tqErrorC(...) do { if (cDebugFlag & DEBUG_ERROR || tqClientDebugFlag & DEBUG_ERROR) { taosPrintLog("TQ  ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}     while(0)
+#define tqInfoC(...)  do { if (cDebugFlag & DEBUG_INFO || tqClientDebugFlag & DEBUG_INFO)  { taosPrintLog("TQ  ", DEBUG_INFO, 255, __VA_ARGS__); }}            while(0)
+#define tqDebugC(...) do { if (cDebugFlag & DEBUG_DEBUG || tqClientDebugFlag & DEBUG_DEBUG) { taosPrintLog("TQ  ", DEBUG_DEBUG, tqClientDebugFlag|cDebugFlag, __VA_ARGS__); }} while(0)
 
 #define EMPTY_BLOCK_POLL_IDLE_DURATION 10
 #define DEFAULT_AUTO_COMMIT_INTERVAL   5000
@@ -957,7 +954,7 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) {
     }
   }
 
-  tqClientDebug = rsp.debugFlag;
+  tqClientDebugFlag = rsp.debugFlag;
   tDestroySMqHbRsp(&rsp);
 
 END:
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index b6fdc2c3c7..ea76adef1a 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -542,7 +542,7 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) {
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
-  TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqClientDebug", tqClientDebug, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
+  TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqClientDebugFlag", tqClientDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "udfDebugFlag", udfDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
@@ -1959,7 +1959,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) {
         {"smaDebugFlag", &smaDebugFlag}, {"idxDebugFlag", &idxDebugFlag}, {"tdbDebugFlag", &tdbDebugFlag},
         {"tmrDebugFlag", &tmrDebugFlag}, {"uDebugFlag", &uDebugFlag},     {"smaDebugFlag", &smaDebugFlag},
         {"rpcDebugFlag", &rpcDebugFlag}, {"qDebugFlag", &qDebugFlag},     {"metaDebugFlag", &metaDebugFlag},
-        {"stDebugFlag", &stDebugFlag},   {"sndDebugFlag", &sndDebugFlag}, {"tqClientDebug", &tqClientDebug},
+        {"stDebugFlag", &stDebugFlag},   {"sndDebugFlag", &sndDebugFlag}, {"tqClientDebugFlag", &tqClientDebugFlag},
     };
 
     static OptionNameAndVar options[] = {{"audit", &tsEnableAudit},
diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c
index e80654b65a..0538a5fe83 100644
--- a/source/dnode/mnode/impl/src/mndConsumer.c
+++ b/source/dnode/mnode/impl/src/mndConsumer.c
@@ -245,7 +245,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
   }
 
   storeOffsetRows(pMnode, &req, pConsumer);
-  rsp.debugFlag = tqClientDebug;
+  rsp.debugFlag = tqClientDebugFlag;
   code = buildMqHbRsp(pMsg, &rsp);
 
 END:
diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index 204311aa98..610ba43673 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -243,7 +243,7 @@ int32_t extractMsgFromWal(SWalReader *pReader, void **pItem, int64_t maxVer, con
 int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver);
 bool    tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids);
 int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char *idstr);
-int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet);
+int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet, int64_t *createTime);
 int32_t tqGetStreamExecInfo(SVnode *pVnode, int64_t streamId, int64_t *pDelay, bool *fhFinished);
 
 // sma
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 1bd4317234..4497651151 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -160,7 +160,7 @@ int32_t         metaDropTables(SMeta* pMeta, SArray* tbUids);
 int             metaTtlFindExpired(SMeta* pMeta, int64_t timePointMs, SArray* tbUids, int32_t ttlDropMaxCount);
 int             metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp);
 int             metaUpdateChangeTimeWithLock(SMeta* pMeta, tb_uid_t uid, int64_t changeTimeMs);
-SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock);
+SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, int64_t *createTime);
 int32_t         metaGetTbTSchemaNotNull(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, STSchema** ppTSchema);
 int32_t         metaGetTbTSchemaMaybeNull(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, STSchema** ppTSchema);
 STSchema*       metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock);
diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c
index 484c5c0a16..e2ba8d9ccb 100644
--- a/source/dnode/vnode/src/meta/metaQuery.c
+++ b/source/dnode/vnode/src/meta/metaQuery.c
@@ -371,7 +371,7 @@ int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType) {
   return 0;
 }
 
-SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock) {
+SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock, int64_t *createTime) {
   void           *pData = NULL;
   int             nData = 0;
   int64_t         version;
@@ -407,6 +407,9 @@ _query:
     }
   } else if (me.type == TSDB_CHILD_TABLE) {
     uid = me.ctbEntry.suid;
+    if (createTime != NULL){
+      *createTime = me.ctbEntry.btime;
+    }
     tDecoderClear(&dc);
     goto _query;
   } else {
@@ -617,7 +620,7 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock) {
   STSchema       *pTSchema = NULL;
   SSchemaWrapper *pSW = NULL;
 
-  pSW = metaGetTableSchema(pMeta, uid, sver, lock);
+  pSW = metaGetTableSchema(pMeta, uid, sver, lock, NULL);
   if (!pSW) return NULL;
 
   pTSchema = tBuildTSchema(pSW->pSchema, pSW->nCols, pSW->version);
diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c
index 0936d8f092..b2826ec45a 100644
--- a/source/dnode/vnode/src/meta/metaSnapshot.c
+++ b/source/dnode/vnode/src/meta/metaSnapshot.c
@@ -552,7 +552,7 @@ int32_t setForSnapShot(SSnapContext* ctx, int64_t uid) {
 
 void taosXSetTablePrimaryKey(SSnapContext* ctx, int64_t uid) {
   bool            ret = false;
-  SSchemaWrapper* schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1);
+  SSchemaWrapper* schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1, NULL);
   if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) {
     ret = true;
   }
diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c
index f594c2f229..fedcc0e82d 100644
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@@ -263,7 +263,7 @@ bool tqGetTablePrimaryKey(STqReader* pReader) { return pReader->hasPrimaryKey; }
 
 void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid) {
   bool            ret = false;
-  SSchemaWrapper* schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1);
+  SSchemaWrapper* schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1, NULL);
   if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) {
     ret = true;
   }
@@ -669,7 +669,7 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char*
       (pReader->cachedSchemaVer != sversion)) {
     tDeleteSchemaWrapper(pReader->pSchemaWrapper);
 
-    pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1);
+    pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1, NULL);
     if (pReader->pSchemaWrapper == NULL) {
       tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", uid:%" PRId64
              "version %d, possibly dropped table",
@@ -961,10 +961,8 @@ END:
   return code;
 }
 
-int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet) {
+int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet, int64_t *createTime) {
   tqTrace("tq reader retrieve data block %p, %d", pReader->msg.msgStr, pReader->nextBlk);
-  SSDataBlock* block = NULL;
-
   SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
   if (pSubmitTbData == NULL) {
     return terrno;
@@ -980,7 +978,7 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas
   pReader->lastBlkUid = uid;
 
   tDeleteSchemaWrapper(pReader->pSchemaWrapper);
-  pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1);
+  pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1, createTime);
   if (pReader->pSchemaWrapper == NULL) {
     tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table",
            pReader->pWalReader->pWal->cfg.vgId, uid, pReader->cachedSchemaVer);
diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c
index 68702754aa..3e4895378b 100644
--- a/source/dnode/vnode/src/tq/tqScan.c
+++ b/source/dnode/vnode/src/tq/tqScan.c
@@ -293,6 +293,54 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBat
   return code;
 }
 
+static int32_t buildCreateTbInfo(SMqDataRsp* pRsp, SVCreateTbReq* pCreateTbReq){
+  int32_t code = 0;
+  void*   createReq = NULL;
+  if (pRsp->createTableNum == 0) {
+    pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t));
+    if (pRsp->createTableLen == NULL) {
+      code = terrno;
+      goto END;
+    }
+    pRsp->createTableReq = taosArrayInit(0, sizeof(void*));
+    if (pRsp->createTableReq == NULL) {
+      code = terrno;
+      goto END;
+    }
+  }
+
+  uint32_t len = 0;
+  tEncodeSize(tEncodeSVCreateTbReq, pCreateTbReq, len, code);
+  if (TSDB_CODE_SUCCESS != code) {
+    goto END;
+  }
+  createReq = taosMemoryCalloc(1, len);
+  if (createReq == NULL){
+    code = terrno;
+    goto END;
+  }
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, createReq, len);
+  code = tEncodeSVCreateTbReq(&encoder, pCreateTbReq);
+  tEncoderClear(&encoder);
+  if (code < 0) {
+    goto END;
+  }
+  if (taosArrayPush(pRsp->createTableLen, &len) == NULL){
+    code = terrno;
+    goto END;
+  }
+  if (taosArrayPush(pRsp->createTableReq, &createReq) == NULL){
+    code = terrno;
+    goto END;
+  }
+  pRsp->createTableNum++;
+
+  return 0;
+END:
+  taosMemoryFree(createReq);
+  return code;
+}
 
 static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded){
   int32_t code = 0;
@@ -312,7 +360,8 @@ static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int
   }
 
   SSubmitTbData* pSubmitTbDataRet = NULL;
-  code = tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas, &pSubmitTbDataRet);
+  int64_t createTime = INT64_MAX;
+  code = tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas, &pSubmitTbDataRet, &createTime);
   if (code != 0) {
     tqError("vgId:%d, failed to retrieve block", pTq->pVnode->config.vgId);
     goto END;
@@ -330,46 +379,13 @@ static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int
     }
   }
   if (pHandle->fetchMeta != WITH_DATA && pSubmitTbDataRet->pCreateTbReq != NULL) {
-    if (pRsp->createTableNum == 0) {
-      pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t));
-      if (pRsp->createTableLen == NULL) {
-        code = terrno;
-        goto END;
-      }
-      pRsp->createTableReq = taosArrayInit(0, sizeof(void*));
-      if (pRsp->createTableReq == NULL) {
-        code = terrno;
+    if (pSubmitTbDataRet->ctimeMs - createTime <= 1000) {  // judge if table is already created to avoid sending crateTbReq
+      code = buildCreateTbInfo(pRsp, pSubmitTbDataRet->pCreateTbReq);
+      if (code != 0){
+        tqError("vgId:%d, failed to build create table info", pTq->pVnode->config.vgId);
         goto END;
       }
     }
-
-    uint32_t len = 0;
-    tEncodeSize(tEncodeSVCreateTbReq, pSubmitTbDataRet->pCreateTbReq, len, code);
-    if (TSDB_CODE_SUCCESS != code) {
-      goto END;
-    }
-    void*    createReq = taosMemoryCalloc(1, len);
-    if (createReq == NULL){
-      code = terrno;
-      goto END;
-    }
-    SEncoder encoder = {0};
-    tEncoderInit(&encoder, createReq, len);
-    code = tEncodeSVCreateTbReq(&encoder, pSubmitTbDataRet->pCreateTbReq);
-    tEncoderClear(&encoder);
-    if (code < 0) {
-      taosMemoryFree(createReq);
-      goto END;
-    }
-    if (taosArrayPush(pRsp->createTableLen, &len) == NULL){
-      taosMemoryFree(createReq);
-      goto END;
-    }
-    if (taosArrayPush(pRsp->createTableReq, &createReq) == NULL){
-      taosMemoryFree(createReq);
-      goto END;
-    }
-    pRsp->createTableNum++;
   }
   if (pHandle->fetchMeta == ONLY_META && pSubmitTbDataRet->pCreateTbReq == NULL) {
     goto END;
diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c
index 7c6a2e7313..0929953e1c 100644
--- a/source/dnode/vnode/src/vnd/vnodeQuery.c
+++ b/source/dnode/vnode/src/vnd/vnodeQuery.c
@@ -702,7 +702,7 @@ int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) {
 }
 
 int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) {
-  SSchemaWrapper *pSW = metaGetTableSchema(pVnode->pMeta, suid, -1, 0);
+  SSchemaWrapper *pSW = metaGetTableSchema(pVnode->pMeta, suid, -1, 0, NULL);
   if (pSW) {
     *num = pSW->nCols;
     tDeleteSchemaWrapper(pSW);
diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c
index 45c8a2f6c2..2b80eaf85d 100644
--- a/source/util/src/tlog.c
+++ b/source/util/src/tlog.c
@@ -125,7 +125,7 @@ int32_t idxDebugFlag = 131;
 int32_t sndDebugFlag = 131;
 int32_t simDebugFlag = 131;
 
-int32_t tqClientDebug = 0;
+int32_t tqClientDebugFlag = 0;
 
 int64_t dbgEmptyW = 0;
 int64_t dbgWN = 0;

From 47742ce5ed6e0a84229ddcb9c875607dccf7a09c Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 16 Oct 2024 17:54:53 +0800
Subject: [PATCH 16/28] enh:[TS-5441] cost too long in tmq write meta data by
 cache meta and vg info

---
 source/client/src/clientTmq.c | 6 ++++--
 source/util/src/tlog.c        | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 0233372012..72d709d962 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -24,8 +24,8 @@
 #include "tref.h"
 #include "ttimer.h"
 
-#define tqErrorC(...) do { if (cDebugFlag & DEBUG_ERROR || tqClientDebugFlag & DEBUG_ERROR) { taosPrintLog("TQ  ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}     while(0)
-#define tqInfoC(...)  do { if (cDebugFlag & DEBUG_INFO || tqClientDebugFlag & DEBUG_INFO)  { taosPrintLog("TQ  ", DEBUG_INFO, 255, __VA_ARGS__); }}            while(0)
+#define tqErrorC(...) do { if (cDebugFlag & DEBUG_ERROR || tqClientDebugFlag & DEBUG_ERROR) { taosPrintLog("TQ  ERROR ", DEBUG_ERROR, tqClientDebugFlag|cDebugFlag, __VA_ARGS__); }}     while(0)
+#define tqInfoC(...)  do { if (cDebugFlag & DEBUG_INFO || tqClientDebugFlag & DEBUG_INFO)  { taosPrintLog("TQ  ", DEBUG_INFO, tqClientDebugFlag|cDebugFlag, __VA_ARGS__); }}            while(0)
 #define tqDebugC(...) do { if (cDebugFlag & DEBUG_DEBUG || tqClientDebugFlag & DEBUG_DEBUG) { taosPrintLog("TQ  ", DEBUG_DEBUG, tqClientDebugFlag|cDebugFlag, __VA_ARGS__); }} while(0)
 
 #define EMPTY_BLOCK_POLL_IDLE_DURATION 10
@@ -955,6 +955,8 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) {
   }
 
   tqClientDebugFlag = rsp.debugFlag;
+  tqDebugC("consumer:0x%" PRIx64 ", tqClientDebugFlag:%d", tmq->consumerId, rsp.debugFlag);
+
   tDestroySMqHbRsp(&rsp);
 
 END:
diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c
index 2b80eaf85d..3b9d100da9 100644
--- a/source/util/src/tlog.c
+++ b/source/util/src/tlog.c
@@ -125,7 +125,7 @@ int32_t idxDebugFlag = 131;
 int32_t sndDebugFlag = 131;
 int32_t simDebugFlag = 131;
 
-int32_t tqClientDebugFlag = 0;
+int32_t tqClientDebugFlag = 131;
 
 int64_t dbgEmptyW = 0;
 int64_t dbgWN = 0;

From 4a240288172a56620e9e455af7fd6960f075e013 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 16 Oct 2024 19:34:19 +0800
Subject: [PATCH 17/28] enh:[TS-5441] cost too long in tmq write meta data by
 cache meta and vg info

---
 tests/system-test/2-query/db.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/system-test/2-query/db.py b/tests/system-test/2-query/db.py
index 588609e524..1964cea51f 100644
--- a/tests/system-test/2-query/db.py
+++ b/tests/system-test/2-query/db.py
@@ -57,7 +57,7 @@ class TDTestCase:
             tdSql.checkData(0, 2, 0)
 
         tdSql.query("show dnode 1 variables like '%debugFlag'")
-        tdSql.checkRows(23)
+        tdSql.checkRows(24)
 
         tdSql.query("show dnode 1 variables like '____debugFlag'")
         tdSql.checkRows(2)

From 3ae6f79290731acf6ef8048fe161bd661de0d7c9 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Thu, 17 Oct 2024 09:36:32 +0800
Subject: [PATCH 18/28] enh:[TS-5441] cost too long in tmq write meta data by
 cache meta and vg info

---
 source/client/src/clientTmq.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 72d709d962..950e0f7f34 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -955,7 +955,6 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) {
   }
 
   tqClientDebugFlag = rsp.debugFlag;
-  tqDebugC("consumer:0x%" PRIx64 ", tqClientDebugFlag:%d", tmq->consumerId, rsp.debugFlag);
 
   tDestroySMqHbRsp(&rsp);
 

From bc05289192b50ebda674fa77a510281bd2f497e5 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Fri, 18 Oct 2024 09:28:16 +0800
Subject: [PATCH 19/28] enh:[TS-5441] cost too long in tmq write meta data by
 cache meta and vg info

---
 include/common/tmsg.h                   |   1 -
 include/libs/executor/executor.h        |   1 -
 source/client/src/clientRawBlockWrite.c | 110 ------------------------
 source/common/src/tmsg.c                |  40 +--------
 source/dnode/vnode/src/tq/tqScan.c      |  15 ----
 source/dnode/vnode/src/tq/tqUtil.c      |   5 --
 source/libs/executor/inc/querytask.h    |   1 -
 source/libs/executor/src/executor.c     |   5 --
 8 files changed, 1 insertion(+), 177 deletions(-)

diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index eb7a08357b..9058dfe53f 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -4113,7 +4113,6 @@ typedef struct {
   SArray*      blockData;
   SArray*      blockTbName;
   SArray*      blockSchema;
-//  SArray*      blockSuid;
 
   union{
     struct{
diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h
index 8501d88be0..6d4b3df041 100644
--- a/include/libs/executor/executor.h
+++ b/include/libs/executor/executor.h
@@ -210,7 +210,6 @@ SMqBatchMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo);
 const SSchemaWrapper* qExtractSchemaFromTask(qTaskInfo_t tinfo);
 
 const char*   qExtractTbnameFromTask(qTaskInfo_t tinfo);
-//const int64_t qExtractSuidFromTask(qTaskInfo_t tinfo);
 
 void* qExtractReaderFromStreamScanner(void* scanner);
 
diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 6c54c3aa69..206bc63d19 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -1706,116 +1706,6 @@ static void* getRawDataFromRes(void* pRetrieve) {
   return rawData;
 }
 
-//static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
-//  if (taos == NULL || data == NULL) {
-//    SET_ERROR_MSG("taos:%p or data:%p is NULL", taos, data);
-//    return TSDB_CODE_INVALID_PARA;
-//  }
-//  int32_t     code = TSDB_CODE_SUCCESS;
-//  SHashObj*   pVgHash = NULL;
-//  SQuery*     pQuery = NULL;
-//  SMqRspObj   rspObj = {0};
-//  SDecoder    decoder = {0};
-//  STableMeta* pTableMeta = NULL;
-//
-//  SRequestObj* pRequest = NULL;
-//  RAW_RETURN_CHECK(createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0, &pRequest));
-//
-//  uDebug(LOG_ID_TAG " write raw data, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen);
-//  pRequest->syncQuery = true;
-//  rspObj.resIter = -1;
-//  rspObj.resType = RES_TYPE__TMQ;
-//
-//  int8_t dataVersion = *(int8_t*)data;
-//  if (dataVersion >= MQ_DATA_RSP_VERSION) {
-//    data = POINTER_SHIFT(data, sizeof(int8_t) + sizeof(int32_t));
-//    dataLen -= sizeof(int8_t) + sizeof(int32_t);
-//  }
-//  tDecoderInit(&decoder, data, dataLen);
-//  code = tDecodeMqDataRsp(&decoder, &rspObj.dataRsp);
-//  if (code != 0) {
-//    SET_ERROR_MSG("decode mq data rsp failed");
-//    code = TSDB_CODE_INVALID_MSG;
-//    goto end;
-//  }
-//
-//  if (!pRequest->pDb) {
-//    code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
-//    goto end;
-//  }
-//
-//  struct SCatalog* pCatalog = NULL;
-//  RAW_RETURN_CHECK(catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog));
-//
-//  SRequestConnInfo conn = {0};
-//  conn.pTrans = pRequest->pTscObj->pAppInfo->pTransporter;
-//  conn.requestId = pRequest->requestId;
-//  conn.requestObjRefId = pRequest->self;
-//  conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
-//
-//  RAW_RETURN_CHECK(smlInitHandle(&pQuery));
-//  pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
-//  RAW_NULL_CHECK(pVgHash);
-//  while (++rspObj.resIter < rspObj.dataRsp.blockNum) {
-//    void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter);
-//    RAW_NULL_CHECK(pRetrieve);
-//    if (!rspObj.dataRsp.withSchema) {
-//      goto end;
-//    }
-//
-//    const char* tbName = (const char*)taosArrayGetP(rspObj.dataRsp.blockTbName, rspObj.resIter);
-//    RAW_NULL_CHECK(tbName);
-//
-//    SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
-//    (void)strcpy(pName.dbname, pRequest->pDb);
-//    (void)strcpy(pName.tname, tbName);
-//
-//    RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta));
-//
-//    SVgroupInfo vg = {0};
-//    RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg));
-//
-//    void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId));
-//    if (hData == NULL) {
-//      RAW_RETURN_CHECK(taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)));
-//    }
-//
-//    SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter);
-//    RAW_NULL_CHECK(pSW);
-//    TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD));
-//    RAW_NULL_CHECK(fields);
-//    for (int i = 0; i < pSW->nCols; i++) {
-//      fields[i].type = pSW->pSchema[i].type;
-//      fields[i].bytes = pSW->pSchema[i].bytes;
-//      tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name));
-//    }
-//    void* rawData = getRawDataFromRes(pRetrieve);
-//    char  err[ERR_MSG_LEN] = {0};
-//    code = rawBlockBindData(pQuery, pTableMeta, rawData, NULL, fields, pSW->nCols, true, err, ERR_MSG_LEN);
-//    taosMemoryFree(fields);
-//    taosMemoryFreeClear(pTableMeta);
-//    if (code != TSDB_CODE_SUCCESS) {
-//      SET_ERROR_MSG("table:%s, err:%s", tbName, err);
-//      goto end;
-//    }
-//  }
-//
-//  RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
-//
-//  launchQueryImpl(pRequest, pQuery, true, NULL);
-//  code = pRequest->code;
-//
-//  end:
-//  uDebug(LOG_ID_TAG " write raw data return, msg:%s", LOG_ID_VALUE, tstrerror(code));
-//  tDeleteMqDataRsp(&rspObj.dataRsp);
-//  tDecoderClear(&decoder);
-//  qDestroyQuery(pQuery);
-//  destroyRequest(pRequest);
-//  taosHashCleanup(pVgHash);
-//  taosMemoryFreeClear(pTableMeta);
-//  return code;
-//}
-
 static int32_t buildCreateTbMap(SMqDataRsp* rsp, SHashObj* pHashObj) {
   // find schema data info
   int32_t       code = 0;
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index e0009177e5..9c8544fcd4 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -10980,42 +10980,13 @@ _exit:
   return code;
 }
 
-//int32_t tEncodeSuidArray(SEncoder *pEncoder, const SMqDataRsp *pRsp){
-//  for (int32_t i = 0; i < pRsp->blockNum; i++) {
-//    if (pRsp->withTbName) {
-//      int64_t* suid = taosArrayGet(pRsp->blockSuid, i);
-//      if (suid != NULL){
-//        TAOS_CHECK_RETURN(tEncodeI64(pEncoder, *suid));
-//      }
-//    }
-//  }
-//  return 0;
-//}
 int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
   TAOS_CHECK_RETURN(tEncodeMqDataRspCommon(pEncoder, pRsp));
   TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pRsp->sleepTime));
-//  TAOS_CHECK_RETURN(tEncodeSuidArray(pEncoder, pRsp));
 
   return 0;
 }
-//int32_t tDecodeSuidArray(SDecoder *pDecoder, SMqDataRsp *pRsp){
-//  if (!tDecodeIsEnd(pDecoder)) {
-//    if (pRsp->withTbName) {
-//      if ((pRsp->blockSuid = taosArrayInit(pRsp->blockNum, sizeof(int64_t))) == NULL) {
-//        TAOS_CHECK_RETURN(terrno);
-//      }
-//    }
-//
-//    for (int32_t i = 0; i < pRsp->blockNum; i++) {
-//      int64_t suid = 0;
-//      TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &suid));
-//      if (taosArrayPush(pRsp->blockSuid, &suid) == NULL) {
-//        TAOS_CHECK_RETURN(terrno);
-//      }
-//    }
-//  }
-//  return 0;
-//}
+
 int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRsp *pRsp) {
   int32_t code = 0;
   int32_t lino;
@@ -11092,9 +11063,6 @@ int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) {
   if (!tDecodeIsEnd(pDecoder)) {
     TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pRsp->sleepTime));
   }
-//  if (!tDecodeIsEnd(pDecoder)) {
-//    TAOS_CHECK_RETURN(tDecodeSuidArray(pDecoder, pRsp));
-//  }
 
   return 0;
 }
@@ -11108,8 +11076,6 @@ static void tDeleteMqDataRspCommon(SMqDataRsp *pRsp) {
   pRsp->blockSchema = NULL;
   taosArrayDestroyP(pRsp->blockTbName, (FDelete)taosMemoryFree);
   pRsp->blockTbName = NULL;
-//  taosArrayDestroy(pRsp->blockSuid);
-//  pRsp->blockSuid = NULL;
   tOffsetDestroy(&pRsp->reqOffset);
   tOffsetDestroy(&pRsp->rspOffset);
 }
@@ -11129,7 +11095,6 @@ int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
       TAOS_CHECK_EXIT(tEncodeBinary(pEncoder, createTableReq, createTableLen));
     }
   }
-//  TAOS_CHECK_EXIT(tEncodeSuidArray(pEncoder, pRsp));
 
 _exit:
   return code;
@@ -11161,9 +11126,6 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) {
       }
     }
   }
-//  if (!tDecodeIsEnd(pDecoder)) {
-//    TAOS_CHECK_EXIT(tDecodeSuidArray(pDecoder, pRsp));
-//  }
 
 _exit:
   return code;
diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c
index 3e4895378b..0e7e27fcff 100644
--- a/source/dnode/vnode/src/tq/tqScan.c
+++ b/source/dnode/vnode/src/tq/tqScan.c
@@ -77,16 +77,6 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, i
       tqError("failed to push tbName to blockTbName:%s", tbName);
       continue;
     }
-//    int64_t suid = 0;
-//    if(mr.me.type == TSDB_CHILD_TABLE){
-//      suid = mr.me.ctbEntry.suid;
-//    }else{
-//      suid = mr.me.uid;
-//    }
-//    if(taosArrayPush(pRsp->blockSuid, &suid) == NULL){
-//      tqError("failed to push suid to blockSuid:%"PRId64, suid);
-//      continue;
-//    }
   }
   metaReaderClear(&mr);
   return 0;
@@ -229,11 +219,6 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBat
           tqError("vgId:%d, failed to add tbname to rsp msg", pTq->pVnode->config.vgId);
           continue;
         }
-//        int64_t suid = qExtractSuidFromTask(task);
-//        if (taosArrayPush(pRsp->blockSuid, &suid) == NULL){
-//          tqError("vgId:%d, failed to add suid to rsp msg", pTq->pVnode->config.vgId);
-//          continue;
-//        }
       }
       if (pRsp->withSchema) {
         SSchemaWrapper* pSW = tCloneSSchemaWrapper(qExtractSchemaFromTask(task));
diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c
index 6acfe6b074..3f11937463 100644
--- a/source/dnode/vnode/src/tq/tqUtil.c
+++ b/source/dnode/vnode/src/tq/tqUtil.c
@@ -50,7 +50,6 @@ static int32_t tqInitTaosxRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) {
   pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t));
   pRsp->blockTbName = taosArrayInit(0, sizeof(void*));
   pRsp->blockSchema = taosArrayInit(0, sizeof(void*));
-//  pRsp->blockSuid = taosArrayInit(0, sizeof(int64_t));
 
   if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL ||
       pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) {
@@ -74,10 +73,6 @@ static int32_t tqInitTaosxRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) {
       pRsp->blockSchema = NULL;
     }
 
-//    if (pRsp->blockSuid != NULL) {
-//      taosArrayDestroy(pRsp->blockSuid);
-//      pRsp->blockSuid = NULL;
-//    }
     return terrno;
   }
 
diff --git a/source/libs/executor/inc/querytask.h b/source/libs/executor/inc/querytask.h
index c9e65bacaf..e3bb9a1361 100644
--- a/source/libs/executor/inc/querytask.h
+++ b/source/libs/executor/inc/querytask.h
@@ -69,7 +69,6 @@ typedef struct {
   SVersionRange        fillHistoryVer;
   STimeWindow          fillHistoryWindow;
   SStreamState*        pState;
-//  int64_t              suid;           // for tmq
 } SStreamTaskInfo;
 
 struct SExecTaskInfo {
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index 4cfa12be5b..b9e103ebca 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -1212,11 +1212,6 @@ const char* qExtractTbnameFromTask(qTaskInfo_t tinfo) {
   return pTaskInfo->streamInfo.tbName;
 }
 
-//const int64_t qExtractSuidFromTask(qTaskInfo_t tinfo) {
-//  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
-//  return pTaskInfo->streamInfo.suid;
-//}
-
 SMqBatchMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo) {
   SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
   return &pTaskInfo->streamInfo.btMetaRsp;

From 51c7f8b4f003804990d3bb02dfe09cb2858b5973 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Fri, 18 Oct 2024 17:36:57 +0800
Subject: [PATCH 20/28] enh:[TS-5441] cost too long in tmq write meta data by
 cache meta and vg info

---
 source/client/src/clientRawBlockWrite.c | 325 ++++++++++++++----------
 1 file changed, 192 insertions(+), 133 deletions(-)

diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 206bc63d19..eb4eed8b6d 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -888,9 +888,6 @@ end:
 }
 
 static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
-  if (taos == NULL || meta == NULL) {
-    return TSDB_CODE_INVALID_PARA;
-  }
   SVCreateStbReq req = {0};
   SDecoder       coder;
   SMCreateStbReq pReq = {0};
@@ -1001,9 +998,6 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
 }
 
 static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
-  if (taos == NULL || meta == NULL) {
-    return TSDB_CODE_INVALID_PARA;
-  }
   SVDropStbReq req = {0};
   SDecoder     coder = {0};
   SMDropStbReq pReq = {0};
@@ -1113,9 +1107,6 @@ static void destroyCreateTbReqBatch(void* data) {
 }
 
 static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
-  if (taos == NULL || meta == NULL) {
-    return TSDB_CODE_INVALID_PARA;
-  }
   SVCreateTbBatchReq req = {0};
   SDecoder           coder = {0};
   int32_t            code = TSDB_CODE_SUCCESS;
@@ -1302,9 +1293,6 @@ static void destroyDropTbReqBatch(void* data) {
 }
 
 static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
-  if (taos == NULL || meta == NULL) {
-    return TSDB_CODE_INVALID_PARA;
-  }
   SVDropTbBatchReq req = {0};
   SDecoder         coder = {0};
   int32_t          code = TSDB_CODE_SUCCESS;
@@ -1417,9 +1405,6 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
 }
 
 static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) {
-  if (taos == NULL || meta == NULL) {
-    return TSDB_CODE_INVALID_PARA;
-  }
   SDeleteRes req = {0};
   SDecoder   coder = {0};
   char       sql[256] = {0};
@@ -1455,9 +1440,6 @@ static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) {
 }
 
 static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
-  if (taos == NULL || meta == NULL) {
-    return TSDB_CODE_INVALID_PARA;
-  }
   SVAlterTbReq   req = {0};
   SDecoder       dcoder = {0};
   int32_t        code = TSDB_CODE_SUCCESS;
@@ -1771,7 +1753,7 @@ static void freeRawCache(void *data) {
 
 static int32_t initRawCacheHash(){
   if (writeRawCache == NULL){
-    writeRawCache = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
+    writeRawCache = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
     if (writeRawCache == NULL){
       return terrno;
     }
@@ -1844,57 +1826,113 @@ end:
   return code;
 }
 
-static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t dataLen) {
-  if (taos == NULL || data == NULL) {
-    SET_ERROR_MSG("taos:%p or data:%p is NULL", taos, data);
-    return TSDB_CODE_INVALID_PARA;
+static int32_t buildRawRequest(TAOS* taos, SRequestObj** pRequest, SCatalog**  pCatalog, SRequestConnInfo  *conn){
+  int32_t code = 0;
+  RAW_RETURN_CHECK(createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0, pRequest));
+  (*pRequest)->syncQuery = true;
+  if (!(*pRequest)->pDb) {
+    code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
+    goto end;
   }
-  int32_t        code = TSDB_CODE_SUCCESS;
-  SQuery*        pQuery = NULL;
-  SMqRspObj      rspObj = {0};
-  SDecoder       decoder = {0};
-  SHashObj*      pCreateTbHash = NULL;
-  SRequestObj*   pRequest = NULL;
-  RAW_RETURN_CHECK(createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0, &pRequest));
 
-  uDebug(LOG_ID_TAG " write raw metadata, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen);
-  pRequest->syncQuery = true;
-  rspObj.resIter = -1;
-//  rspObj.resType = RES_TYPE__TMQ_METADATA;
+  RAW_RETURN_CHECK(catalogGetHandle((*pRequest)->pTscObj->pAppInfo->clusterId, pCatalog));
+  conn->pTrans = (*pRequest)->pTscObj->pAppInfo->pTransporter;
+  conn->requestId = (*pRequest)->requestId;
+  conn->requestObjRefId = (*pRequest)->self;
+  conn->mgmtEps = getEpSet_s(&(*pRequest)->pTscObj->pAppInfo->mgmtEp);
 
+end:
+  return code;
+}
+
+typedef int32_t _raw_decode_func_(SDecoder *pDecoder, SMqDataRsp *pRsp);
+static int32_t decodeRawData(SDecoder *decoder, void* data, int32_t dataLen, _raw_decode_func_ func, SMqRspObj* rspObj){
   int8_t dataVersion = *(int8_t*)data;
   if (dataVersion >= MQ_DATA_RSP_VERSION) {
     data = POINTER_SHIFT(data, sizeof(int8_t) + sizeof(int32_t));
     dataLen -= sizeof(int8_t) + sizeof(int32_t);
   }
 
-  tDecoderInit(&decoder, data, dataLen);
-  code = (type == RES_TYPE__TMQ_METADATA) ? tDecodeSTaosxRsp(&decoder, &rspObj.dataRsp) : tDecodeMqDataRsp(&decoder, &rspObj.dataRsp);
+  rspObj->resIter = -1;
+  tDecoderInit(decoder, data, dataLen);
+  int32_t code = func(decoder, &rspObj->dataRsp);
   if (code != 0) {
     SET_ERROR_MSG("decode mq taosx data rsp failed");
-    code = TSDB_CODE_INVALID_MSG;
-    goto end;
+  }
+  return code;
+}
+
+static int32_t processCacheMeta(SHashObj *pVgHash, SHashObj *pNameHash, SHashObj *pMetaHash, SVCreateTbReq* pCreateReqDst,
+                 SCatalog* pCatalog, SRequestConnInfo* conn, SName* pName,
+                 STableMeta** pMeta, SSchemaWrapper* pSW, void* rawData, int32_t retry){
+  int32_t code = 0;
+  STableMeta* pTableMeta = NULL;
+  tbInfo* tmpInfo = (tbInfo*)taosHashGet(pNameHash, pName->tname, strlen(pName->tname));
+  if (tmpInfo == NULL || retry > 0) {
+    tbInfo info = {0};
+
+    RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, conn, pName, &info.vgInfo));
+    if (pCreateReqDst && tmpInfo == NULL) {  // change stable name to get meta
+      tstrncpy(pName->tname, pCreateReqDst->ctb.stbName, TSDB_TABLE_NAME_LEN);
+    }
+    RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, conn, pName, &pTableMeta));
+    info.uid = pTableMeta->uid;
+    if (pTableMeta->tableType == TSDB_CHILD_TABLE){
+      info.suid = pTableMeta->suid;
+    } else {
+      info.suid = pTableMeta->uid;
+    }
+    code = taosHashPut(pMetaHash, &info.suid, LONG_BYTES, &pTableMeta, POINTER_BYTES);
+    if (code != 0){
+      taosMemoryFree(pTableMeta);
+      goto end;
+    }
+    if (pCreateReqDst) {
+      pTableMeta->vgId = info.vgInfo.vgId;
+      pTableMeta->uid = pCreateReqDst->uid;
+      pCreateReqDst->ctb.suid = pTableMeta->suid;
+    }
+
+    RAW_RETURN_CHECK(taosHashPut(pNameHash, pName->tname, strlen(pName->tname), &info, sizeof(tbInfo)));
+    tmpInfo = (tbInfo*)taosHashGet(pNameHash, pName->tname, strlen(pName->tname));
+    RAW_RETURN_CHECK(taosHashPut(pVgHash, &info.vgInfo.vgId, sizeof(info.vgInfo.vgId), &info.vgInfo, sizeof(SVgroupInfo)));
   }
 
-  if (!pRequest->pDb) {
-    code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
-    goto end;
+  if (pTableMeta == NULL || retry > 0){
+    STableMeta** pTableMetaTmp = (STableMeta**)taosHashGet(pMetaHash, &tmpInfo->suid, LONG_BYTES);
+    if (pTableMetaTmp == NULL || retry > 0 || needRefreshMeta(rawData, *pTableMetaTmp, pSW)) {
+      RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, conn, pName, &pTableMeta));
+      code = taosHashPut(pMetaHash, &tmpInfo->suid, LONG_BYTES, &pTableMeta, POINTER_BYTES);
+      if (code != 0){
+        taosMemoryFree(pTableMeta);
+        goto end;
+      }
+
+    }else{
+      pTableMeta = *pTableMetaTmp;
+      pTableMeta->uid = tmpInfo->uid;
+      pTableMeta->vgId = tmpInfo->vgInfo.vgId;
+    }
   }
+  *pMeta = pTableMeta;
 
-  struct SCatalog* pCatalog = NULL;
-  RAW_RETURN_CHECK(catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog));
+end:
+  return code;
+}
 
-  SRequestConnInfo conn = {0};
-  conn.pTrans = pRequest->pTscObj->pAppInfo->pTransporter;
-  conn.requestId = pRequest->requestId;
-  conn.requestObjRefId = pRequest->self;
-  conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
+static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen){
+  int32_t        code = TSDB_CODE_SUCCESS;
+  SQuery*        pQuery = NULL;
+  SMqRspObj      rspObj = {0};
+  SDecoder       decoder = {0};
 
-  if (type == RES_TYPE__TMQ_METADATA) {
-    pCreateTbHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
-    RAW_NULL_CHECK(pCreateTbHash);
-    RAW_RETURN_CHECK(buildCreateTbMap(&rspObj.dataRsp, pCreateTbHash));
-  }
+  SRequestObj*      pRequest = NULL;
+  SCatalog*         pCatalog = NULL;
+  SRequestConnInfo  conn = {0};
+
+  uDebug(LOG_ID_TAG " write raw data, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen);
+  RAW_RETURN_CHECK(buildRawRequest(taos, &pRequest, &pCatalog, &conn));
+  RAW_RETURN_CHECK(decodeRawData(&decoder, data, dataLen, tDecodeMqDataRsp, &rspObj));
 
   SHashObj *pVgHash = NULL;
   SHashObj *pNameHash = NULL;
@@ -1903,89 +1941,33 @@ static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t da
   int retry = 0;
   while(1){
     RAW_RETURN_CHECK(smlInitHandle(&pQuery));
-    uDebug(LOG_ID_TAG " write raw data type:%d block num:%d", LOG_ID_VALUE, type, rspObj.dataRsp.blockNum);
+    uDebug(LOG_ID_TAG " write raw meta data block num:%d", LOG_ID_VALUE, rspObj.dataRsp.blockNum);
     while (++rspObj.resIter < rspObj.dataRsp.blockNum) {
-      void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter);
-      RAW_NULL_CHECK(pRetrieve);
       if (!rspObj.dataRsp.withSchema) {
         goto end;
       }
 
       const char* tbName = (const char*)taosArrayGetP(rspObj.dataRsp.blockTbName, rspObj.resIter);
       RAW_NULL_CHECK(tbName);
-
-      uDebug(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName);
-      SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
-      (void)strcpy(pName.dbname, pRequest->pDb);
-      (void)strcpy(pName.tname, tbName);
-
-      // find schema data info
-      SVCreateTbReq* pCreateReqDst = NULL;
-      if (type == RES_TYPE__TMQ_METADATA){
-        pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, tbName, strlen(tbName));
-      }
-      STableMeta* pTableMeta = NULL;
-      tbInfo* tmpInfo = (tbInfo*)taosHashGet(pNameHash, tbName, strlen(tbName));
-      if (tmpInfo == NULL || retry > 0) {
-        tbInfo info = {0};
-
-        RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &info.vgInfo));
-        if (pCreateReqDst && tmpInfo == NULL) {  // change stable name to get meta
-          (void)strcpy(pName.tname, pCreateReqDst->ctb.stbName);
-        }
-        RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta));
-        info.uid = pTableMeta->uid;
-        if (pTableMeta->tableType == TSDB_CHILD_TABLE){
-          info.suid = pTableMeta->suid;
-        } else {
-          info.suid = pTableMeta->uid;
-        }
-        code = taosHashPut(pMetaHash, &info.suid, LONG_BYTES, &pTableMeta, POINTER_BYTES);
-        if (code != 0){
-          taosMemoryFree(pTableMeta);
-          goto end;
-        }
-        if (pCreateReqDst) {
-          pTableMeta->vgId = info.vgInfo.vgId;
-          pTableMeta->uid = pCreateReqDst->uid;
-          pCreateReqDst->ctb.suid = pTableMeta->suid;
-        }
-
-        RAW_RETURN_CHECK(taosHashPut(pNameHash, pName.tname, strlen(pName.tname), &info, sizeof(tbInfo)));
-        tmpInfo = (tbInfo*)taosHashGet(pNameHash, pName.tname, strlen(pName.tname));
-//        code = (code == TSDB_CODE_DUP_KEY) ? 0 : code;
-//        RAW_RETURN_CHECK(code);
-        RAW_RETURN_CHECK(taosHashPut(pVgHash, &info.vgInfo.vgId, sizeof(info.vgInfo.vgId), &info.vgInfo, sizeof(SVgroupInfo)));
-//        code = (code == TSDB_CODE_DUP_KEY) ? 0 : code;
-//        RAW_RETURN_CHECK(code);
-      }
-
       SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter);
       RAW_NULL_CHECK(pSW);
+      void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter);
+      RAW_NULL_CHECK(pRetrieve);
       void* rawData = getRawDataFromRes(pRetrieve);
       RAW_NULL_CHECK(rawData);
 
-      if (pTableMeta == NULL || retry > 0){
-        STableMeta** pTableMetaTmp = (STableMeta**)taosHashGet(pMetaHash, &tmpInfo->suid, LONG_BYTES);
-        if (pTableMetaTmp == NULL || retry > 0 || needRefreshMeta(rawData, *pTableMetaTmp, pSW)) {
-          RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta));
-          code = taosHashPut(pMetaHash, &tmpInfo->suid, LONG_BYTES, &pTableMeta, POINTER_BYTES);
-          if (code != 0){
-            taosMemoryFree(pTableMeta);
-            goto end;
-          }
-
-        }else{
-          pTableMeta = *pTableMetaTmp;
-          pTableMeta->uid = tmpInfo->uid;
-          pTableMeta->vgId = tmpInfo->vgInfo.vgId;
-        }
-      }
+      uDebug(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName);
+      SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
+      tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN);
+      tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN);
 
+      STableMeta* pTableMeta = NULL;
+      processCacheMeta(pVgHash, pNameHash, pMetaHash, NULL, pCatalog, &conn,
+                       &pName, &pTableMeta, pSW, rawData, retry);
       char  err[ERR_MSG_LEN] = {0};
-      code = rawBlockBindData(pQuery, pTableMeta, rawData, pCreateReqDst, pSW, pSW->nCols, true, err, ERR_MSG_LEN, true);
+      code = rawBlockBindData(pQuery, pTableMeta, rawData, NULL, pSW, pSW->nCols, true, err, ERR_MSG_LEN, true);
       if (code != TSDB_CODE_SUCCESS) {
-        SET_ERROR_MSG("table:%s, err:%s", tbName, err);
+        SET_ERROR_MSG("table:%s, err:%s", pName.tname, err);
         goto end;
       }
     }
@@ -1993,11 +1975,8 @@ static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t da
     launchQueryImpl(pRequest, pQuery, true, NULL);
     code = pRequest->code;
 
-    if (NEED_CLIENT_HANDLE_ERROR(code)) {
+    if (NEED_CLIENT_HANDLE_ERROR(code) && retry++ < 3) {
       uInfo("write raw retry:%d/3 end code:%d, msg:%s", retry, code, tstrerror(code));
-      if (retry++ >= 3) {
-        break;
-      }
       qDestroyQuery(pQuery);
       pQuery = NULL;
       rspObj.resIter = -1;
@@ -2007,12 +1986,89 @@ static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t da
   }
 
   end:
-  uDebug(LOG_ID_TAG " write raw metadata return, msg:%s", LOG_ID_VALUE, tstrerror(code));
-  if (type == RES_TYPE__TMQ_METADATA){
-    tDeleteSTaosxRsp(&rspObj.dataRsp);
-  }else {
-    tDeleteMqDataRsp(&rspObj.dataRsp);
+  uDebug(LOG_ID_TAG " write raw data return, msg:%s", LOG_ID_VALUE, tstrerror(code));
+  tDeleteMqDataRsp(&rspObj.dataRsp);
+  tDecoderClear(&decoder);
+  qDestroyQuery(pQuery);
+  destroyRequest(pRequest);
+  return code;
+}
+
+static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) {
+  int32_t        code = TSDB_CODE_SUCCESS;
+  SQuery*        pQuery = NULL;
+  SMqRspObj      rspObj = {0};
+  SDecoder       decoder = {0};
+  SHashObj*      pCreateTbHash = NULL;
+
+  SRequestObj*      pRequest = NULL;
+  SCatalog*         pCatalog = NULL;
+  SRequestConnInfo  conn = {0};
+
+  uDebug(LOG_ID_TAG " write raw metadata, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen);
+  RAW_RETURN_CHECK(buildRawRequest(taos, &pRequest, &pCatalog, &conn));
+  RAW_RETURN_CHECK(decodeRawData(&decoder, data, dataLen, tDecodeSTaosxRsp, &rspObj));
+
+  pCreateTbHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+  RAW_NULL_CHECK(pCreateTbHash);
+  RAW_RETURN_CHECK(buildCreateTbMap(&rspObj.dataRsp, pCreateTbHash));
+
+  SHashObj *pVgHash = NULL;
+  SHashObj *pNameHash = NULL;
+  SHashObj *pMetaHash = NULL;
+  RAW_RETURN_CHECK(getRawCache(&pVgHash, &pNameHash, &pMetaHash, taos));
+  int retry = 0;
+  while(1){
+    RAW_RETURN_CHECK(smlInitHandle(&pQuery));
+    uDebug(LOG_ID_TAG " write raw meta data block num:%d", LOG_ID_VALUE, rspObj.dataRsp.blockNum);
+    while (++rspObj.resIter < rspObj.dataRsp.blockNum) {
+      if (!rspObj.dataRsp.withSchema) {
+        goto end;
+      }
+
+      const char* tbName = (const char*)taosArrayGetP(rspObj.dataRsp.blockTbName, rspObj.resIter);
+      RAW_NULL_CHECK(tbName);
+      SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter);
+      RAW_NULL_CHECK(pSW);
+      void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter);
+      RAW_NULL_CHECK(pRetrieve);
+      void* rawData = getRawDataFromRes(pRetrieve);
+      RAW_NULL_CHECK(rawData);
+
+      uDebug(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName);
+      SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
+      tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN);
+      tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN);
+
+      // find schema data info
+      SVCreateTbReq* pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, pName.tname, strlen(pName.tname));
+      STableMeta* pTableMeta = NULL;
+      processCacheMeta(pVgHash, pNameHash, pMetaHash, pCreateReqDst, pCatalog, &conn,
+                       &pName, &pTableMeta, pSW, rawData, retry);
+      char  err[ERR_MSG_LEN] = {0};
+      code = rawBlockBindData(pQuery, pTableMeta, rawData, pCreateReqDst, pSW, pSW->nCols, true, err, ERR_MSG_LEN, true);
+      if (code != TSDB_CODE_SUCCESS) {
+        SET_ERROR_MSG("table:%s, err:%s", pName.tname, err);
+        goto end;
+      }
+    }
+    RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
+    launchQueryImpl(pRequest, pQuery, true, NULL);
+    code = pRequest->code;
+
+    if (NEED_CLIENT_HANDLE_ERROR(code) && retry++ < 3) {
+      uInfo("write raw retry:%d/3 end code:%d, msg:%s", retry, code, tstrerror(code));
+      qDestroyQuery(pQuery);
+      pQuery = NULL;
+      rspObj.resIter = -1;
+      continue;
+    }
+    break;
   }
+
+end:
+  uDebug(LOG_ID_TAG " write raw metadata return, msg:%s", LOG_ID_VALUE, tstrerror(code));
+  tDeleteSTaosxRsp(&rspObj.dataRsp);
   void* pIter = taosHashIterate(pCreateTbHash, NULL);
   while (pIter) {
     tDestroySVCreateTbReq(pIter, TSDB_MSG_FLG_DECODE);
@@ -2243,8 +2299,10 @@ static int32_t writeRawImpl(TAOS* taos, void* buf, uint32_t len, uint16_t type)
     return taosDropTable(taos, buf, len);
   } else if (type == TDMT_VND_DELETE) {
     return taosDeleteData(taos, buf, len);
-  } else if (type == RES_TYPE__TMQ_METADATA || type == RES_TYPE__TMQ) {
-    return tmqWriteRawImpl(taos, type, buf, len);
+  } else if (type == RES_TYPE__TMQ_METADATA){
+    return tmqWriteRawMetaDataImpl(taos, buf, len);
+  } else if (type == RES_TYPE__TMQ) {
+    return tmqWriteRawDataImpl(taos, buf, len);
   } else if (type == RES_TYPE__TMQ_BATCH_META) {
     return tmqWriteBatchMetaDataImpl(taos, buf, len);
   }
@@ -2252,7 +2310,8 @@ static int32_t writeRawImpl(TAOS* taos, void* buf, uint32_t len, uint16_t type)
 }
 
 int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) {
-  if (!taos) {
+  if (taos == NULL || raw.raw == NULL || raw.raw_len <= 0) {
+    SET_ERROR_MSG("taos:%p or data:%p is NULL or raw_len <= 0", taos, raw.raw);
     return TSDB_CODE_INVALID_PARA;
   }
 

From ccfe5246365487f3158a8ced9770bbea25a93b5e Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Fri, 18 Oct 2024 18:28:09 +0800
Subject: [PATCH 21/28] enh:[TS-5441] cost too long in tmq write meta data by
 cache meta and vg info

---
 source/client/src/clientRawBlockWrite.c | 35 ++++++++++++++++++++-----
 1 file changed, 29 insertions(+), 6 deletions(-)

diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index eb4eed8b6d..d955fad150 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -1724,8 +1724,15 @@ end:
   return code;
 }
 
+typedef enum {
+  WRITE_RAW_INIT_START = 0,
+  WRITE_RAW_INIT_OK,
+  WRITE_RAW_INIT_FAIL,
+};
+
 static SHashObj* writeRawCache = NULL;
 static int8_t    initFlag = 0;
+static int8_t    initedFlag = WRITE_RAW_INIT_START;
 
 typedef struct{
   SHashObj*     pVgHash;
@@ -2276,15 +2283,31 @@ void tmq_free_raw(tmq_raw_data raw) {
   (void)memset(terrMsg, 0, ERR_MSG_LEN);
 }
 
-static int32_t writeRawImpl(TAOS* taos, void* buf, uint32_t len, uint16_t type) {
-  int8_t old = atomic_val_compare_exchange_8(&initFlag, 0, 1);
-  if (old == 0) {
-    int32_t code = initRawCacheHash();
-    if (code != 0) {
-      return code;
+static int32_t writeRawInit(){
+  while (atomic_load_8(&initedFlag) == WRITE_RAW_INIT_OK) {
+    int8_t old = atomic_val_compare_exchange_8(&initFlag, 0, 1);
+    if (old == 0) {
+      int32_t code = initRawCacheHash();
+      if (code != 0) {
+        uError("tmq writeRawImpl init error:%d", code);
+        atomic_store_8(&initedFlag, WRITE_RAW_INIT_FAIL);
+        return code;
+      }
+      atomic_store_8(&initedFlag, WRITE_RAW_INIT_OK);
     }
   }
 
+  if (atomic_load_8(&initedFlag) == WRITE_RAW_INIT_FAIL){
+    return TSDB_CODE_INTERNAL_ERROR;
+  }
+  return 0;
+}
+
+static int32_t writeRawImpl(TAOS* taos, void* buf, uint32_t len, uint16_t type) {
+  if (writeRawInit() != 0) {
+    return TSDB_CODE_INTERNAL_ERROR;
+  }
+
   if (type == TDMT_VND_CREATE_STB) {
     return taosCreateStb(taos, buf, len);
   } else if (type == TDMT_VND_ALTER_STB) {

From a6f6d41297c1de50effe63a4003a92cd099f2a35 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Mon, 21 Oct 2024 09:25:22 +0800
Subject: [PATCH 22/28] enh:[TS-5441] cost too long in tmq write meta data by
 cache meta and vg info

---
 source/client/src/clientRawBlockWrite.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index d955fad150..59dcbdf292 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -1728,7 +1728,7 @@ typedef enum {
   WRITE_RAW_INIT_START = 0,
   WRITE_RAW_INIT_OK,
   WRITE_RAW_INIT_FAIL,
-};
+} WRITE_RAW_INIT_STATUS;
 
 static SHashObj* writeRawCache = NULL;
 static int8_t    initFlag = 0;

From 02c16bddc6dd76f3efc8f0043bae29d720b32725 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Mon, 21 Oct 2024 14:40:49 +0800
Subject: [PATCH 23/28] enh:[TS-5441] cost too long in tmq write meta data by
 cache meta and vg info

---
 source/client/src/clientRawBlockWrite.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 59dcbdf292..22de6e4a75 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -1969,8 +1969,8 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen){
       tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN);
 
       STableMeta* pTableMeta = NULL;
-      processCacheMeta(pVgHash, pNameHash, pMetaHash, NULL, pCatalog, &conn,
-                       &pName, &pTableMeta, pSW, rawData, retry);
+      RAW_RETURN_CHECK(processCacheMeta(pVgHash, pNameHash, pMetaHash, NULL, pCatalog, &conn,
+                       &pName, &pTableMeta, pSW, rawData, retry));
       char  err[ERR_MSG_LEN] = {0};
       code = rawBlockBindData(pQuery, pTableMeta, rawData, NULL, pSW, pSW->nCols, true, err, ERR_MSG_LEN, true);
       if (code != TSDB_CODE_SUCCESS) {
@@ -2050,8 +2050,8 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
       // find schema data info
       SVCreateTbReq* pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, pName.tname, strlen(pName.tname));
       STableMeta* pTableMeta = NULL;
-      processCacheMeta(pVgHash, pNameHash, pMetaHash, pCreateReqDst, pCatalog, &conn,
-                       &pName, &pTableMeta, pSW, rawData, retry);
+      RAW_RETURN_CHECK(processCacheMeta(pVgHash, pNameHash, pMetaHash, pCreateReqDst, pCatalog, &conn,
+                       &pName, &pTableMeta, pSW, rawData, retry));
       char  err[ERR_MSG_LEN] = {0};
       code = rawBlockBindData(pQuery, pTableMeta, rawData, pCreateReqDst, pSW, pSW->nCols, true, err, ERR_MSG_LEN, true);
       if (code != TSDB_CODE_SUCCESS) {

From 4d26469ac51651e8f8abbbd47c326be40439ff07 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Mon, 21 Oct 2024 18:44:22 +0800
Subject: [PATCH 24/28] enh:[TS-5441] cost too long in tmq write meta data by
 cache meta and vg info

---
 source/client/src/clientRawBlockWrite.c | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 22de6e4a75..681d6e5fdb 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -1936,9 +1936,8 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen){
   SRequestObj*      pRequest = NULL;
   SCatalog*         pCatalog = NULL;
   SRequestConnInfo  conn = {0};
-
-  uDebug(LOG_ID_TAG " write raw data, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen);
   RAW_RETURN_CHECK(buildRawRequest(taos, &pRequest, &pCatalog, &conn));
+  uDebug(LOG_ID_TAG " write raw data, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen);
   RAW_RETURN_CHECK(decodeRawData(&decoder, data, dataLen, tDecodeMqDataRsp, &rspObj));
 
   SHashObj *pVgHash = NULL;
@@ -2012,8 +2011,8 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
   SCatalog*         pCatalog = NULL;
   SRequestConnInfo  conn = {0};
 
-  uDebug(LOG_ID_TAG " write raw metadata, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen);
   RAW_RETURN_CHECK(buildRawRequest(taos, &pRequest, &pCatalog, &conn));
+  uDebug(LOG_ID_TAG " write raw metadata, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen);
   RAW_RETURN_CHECK(decodeRawData(&decoder, data, dataLen, tDecodeSTaosxRsp, &rspObj));
 
   pCreateTbHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
@@ -2284,7 +2283,7 @@ void tmq_free_raw(tmq_raw_data raw) {
 }
 
 static int32_t writeRawInit(){
-  while (atomic_load_8(&initedFlag) == WRITE_RAW_INIT_OK) {
+  while (atomic_load_8(&initedFlag) == WRITE_RAW_INIT_START) {
     int8_t old = atomic_val_compare_exchange_8(&initFlag, 0, 1);
     if (old == 0) {
       int32_t code = initRawCacheHash();

From 94a70c9389cd930a3f80dbfe66edbd272a67246c Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Tue, 22 Oct 2024 23:24:39 +0800
Subject: [PATCH 25/28] enh:[TS-5441] cost too long in tmq write meta data by
 cache meta and vg info

---
 source/client/src/clientRawBlockWrite.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 681d6e5fdb..2bd815b460 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -2344,9 +2344,9 @@ static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen
   if (taos == NULL || meta == NULL) {
     return TSDB_CODE_INVALID_PARA;
   }
-  SMqBatchMetaRsp rsp = {0};
+  SMqBatchMetaRsp rsp   = {0};
   SDecoder        coder = {0};
-  int32_t         code = TSDB_CODE_SUCCESS;
+  int32_t         code  = TSDB_CODE_SUCCESS;
 
   // decode and process req
   tDecoderInit(&coder, meta, metaLen);

From 0100d383eead8c110c8f21b31154ab7d6573c297 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Mon, 28 Oct 2024 15:02:03 +0800
Subject: [PATCH 26/28] enh:[TS-5441] cost too long in tmq write meta data by
 cache meta and vg info

---
 source/client/src/clientRawBlockWrite.c | 315 ++++++++++++------------
 1 file changed, 159 insertions(+), 156 deletions(-)

diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 2bd815b460..80403986aa 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -23,12 +23,12 @@
 #include "tglobal.h"
 #include "tmsgtype.h"
 
-#define RAW_NULL_CHECK(c)             \
-  do {                                \
-    if (c == NULL) {                  \
-      code = terrno; \
-      goto end;                       \
-    }                                 \
+#define RAW_NULL_CHECK(c) \
+  do {                    \
+    if (c == NULL) {      \
+      code = terrno;      \
+      goto end;           \
+    }                     \
   } while (0)
 
 #define RAW_FALSE_CHECK(c)           \
@@ -52,7 +52,7 @@
 
 #define TMQ_META_VERSION "1.0"
 
-static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen);
+static int32_t  tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen);
 static tb_uid_t processSuid(tb_uid_t suid, char* db) { return suid + MurmurHash3_32(db, strlen(db)); }
 static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, int8_t t,
                                  SColCmprWrapper* pColCmprRow, cJSON** pJson) {
@@ -163,7 +163,7 @@ static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sche
   }
   RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tags", tags));
 
-  end:
+end:
   *pJson = json;
 }
 
@@ -197,7 +197,7 @@ static int32_t setCompressOption(cJSON* json, uint32_t para) {
     return code;
   }
 
-  end:
+end:
   return code;
 }
 static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON** pJson) {
@@ -338,7 +338,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON**
       break;
   }
 
-  end:
+end:
   tFreeSMAltertbReq(&req);
   *pJson = json;
 }
@@ -455,7 +455,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
 
     cJSON* tvalue = NULL;
     if (IS_VAR_DATA_TYPE(pTagVal->type)) {
-      char* buf = NULL;
+      char*   buf = NULL;
       int64_t bufSize = 0;
       if (pTagVal->type == TSDB_DATA_TYPE_VARBINARY) {
         bufSize = pTagVal->nData * 2 + 2 + 3;
@@ -485,7 +485,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
     RAW_FALSE_CHECK(cJSON_AddItemToArray(tags, tag));
   }
 
-  end:
+end:
   RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tags", tags));
   taosArrayDestroy(pTagVals);
 }
@@ -514,7 +514,7 @@ static void buildCreateCTableJson(SVCreateTbReq* pCreateReq, int32_t nReqs, cJSO
   }
   RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "createList", createList));
 
-  end:
+end:
   *pJson = json;
 }
 
@@ -585,7 +585,7 @@ static void processAutoCreateTable(SMqDataRsp* rsp, char** string) {
   *string = cJSON_PrintUnformatted(pJson);
   cJSON_Delete(pJson);
 
-  end:
+end:
   uDebug("auto created table return, sql json:%s", *string);
   for (int i = 0; decoder && pCreateReq && i < rsp->createTableNum; i++) {
     tDecoderClear(&decoder[i]);
@@ -989,7 +989,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
 
   code = pRequest->code;
 
-  end:
+end:
   uDebug(LOG_ID_TAG " create stable return, msg:%s", LOG_ID_VALUE, tstrerror(code));
   destroyRequest(pRequest);
   tFreeSMCreateStbReq(&pReq);
@@ -1023,9 +1023,9 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
   SCatalog* pCatalog = NULL;
   RAW_RETURN_CHECK(catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog));
   SRequestConnInfo conn = {.pTrans = pRequest->pTscObj->pAppInfo->pTransporter,
-      .requestId = pRequest->requestId,
-      .requestObjRefId = pRequest->self,
-      .mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)};
+                           .requestId = pRequest->requestId,
+                           .requestObjRefId = pRequest->self,
+                           .mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)};
   SName            pName = {0};
   toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, &pName);
   STableMeta* pTableMeta = NULL;
@@ -1088,7 +1088,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
 
   code = pRequest->code;
 
-  end:
+end:
   uDebug(LOG_ID_TAG " drop stable return, msg:%s", LOG_ID_VALUE, tstrerror(code));
   destroyRequest(pRequest);
   tDecoderClear(&coder);
@@ -1142,9 +1142,9 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
   taosHashSetFreeFp(pVgroupHashmap, destroyCreateTbReqBatch);
 
   SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter,
-      .requestId = pRequest->requestId,
-      .requestObjRefId = pRequest->self,
-      .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)};
+                           .requestId = pRequest->requestId,
+                           .requestObjRefId = pRequest->self,
+                           .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)};
 
   pRequest->tableList = taosArrayInit(req.nReqs, sizeof(SName));
   RAW_NULL_CHECK(pRequest->tableList);
@@ -1269,7 +1269,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
 
   code = pRequest->code;
 
-  end:
+end:
   uDebug(LOG_ID_TAG " create table return, msg:%s", LOG_ID_VALUE, tstrerror(code));
   tDeleteSVCreateTbBatchReq(&req);
 
@@ -1328,9 +1328,9 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
   taosHashSetFreeFp(pVgroupHashmap, destroyDropTbReqBatch);
 
   SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter,
-      .requestId = pRequest->requestId,
-      .requestObjRefId = pRequest->self,
-      .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)};
+                           .requestId = pRequest->requestId,
+                           .requestObjRefId = pRequest->self,
+                           .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)};
   pRequest->tableList = taosArrayInit(req.nReqs, sizeof(SName));
   RAW_NULL_CHECK(pRequest->tableList);
   // loop to create table
@@ -1395,7 +1395,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
   }
   code = pRequest->code;
 
-  end:
+end:
   uDebug(LOG_ID_TAG " drop table return, msg:%s", LOG_ID_VALUE, tstrerror(code));
   taosHashCleanup(pVgroupHashmap);
   destroyRequest(pRequest);
@@ -1433,7 +1433,7 @@ static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) {
   }
   taos_free_result(res);
 
-  end:
+end:
   uDebug("connId:0x%" PRIx64 " delete data sql:%s, code:%s", *(int64_t*)taos, sql, tstrerror(code));
   tDecoderClear(&coder);
   return code;
@@ -1473,9 +1473,9 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
   SCatalog* pCatalog = NULL;
   RAW_RETURN_CHECK(catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog));
   SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter,
-      .requestId = pRequest->requestId,
-      .requestObjRefId = pRequest->self,
-      .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)};
+                           .requestId = pRequest->requestId,
+                           .requestObjRefId = pRequest->self,
+                           .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)};
 
   SVgroupInfo pInfo = {0};
   SName       pName = {0};
@@ -1543,7 +1543,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
       code = handleAlterTbExecRes(pRes->res, pCatalog);
     }
   }
-  end:
+end:
   uDebug(LOG_ID_TAG " alter table return, meta:%p, len:%d, msg:%s", LOG_ID_VALUE, meta, metaLen, tstrerror(code));
   taosArrayDestroy(pArray);
   if (pVgData) taosMemoryFreeClear(pVgData->pData);
@@ -1608,7 +1608,7 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pDat
   launchQueryImpl(pRequest, pQuery, true, NULL);
   code = pRequest->code;
 
-  end:
+end:
   uDebug(LOG_ID_TAG " write raw block with field return, msg:%s", LOG_ID_VALUE, tstrerror(code));
   taosMemoryFreeClear(pTableMeta);
   qDestroyQuery(pQuery);
@@ -1668,7 +1668,7 @@ int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const cha
   launchQueryImpl(pRequest, pQuery, true, NULL);
   code = pRequest->code;
 
-  end:
+end:
   uDebug(LOG_ID_TAG " write raw block return, msg:%s", LOG_ID_VALUE, tstrerror(code));
   taosMemoryFreeClear(pTableMeta);
   qDestroyQuery(pQuery);
@@ -1708,7 +1708,8 @@ static int32_t buildCreateTbMap(SMqDataRsp* rsp, SHashObj* pHashObj) {
       goto end;
     }
     if (taosHashGet(pHashObj, pCreateReq.name, strlen(pCreateReq.name)) == NULL) {
-      RAW_RETURN_CHECK(taosHashPut(pHashObj, pCreateReq.name, strlen(pCreateReq.name), &pCreateReq, sizeof(SVCreateTbReq)));
+      RAW_RETURN_CHECK(
+          taosHashPut(pHashObj, pCreateReq.name, strlen(pCreateReq.name), &pCreateReq, sizeof(SVCreateTbReq)));
     } else {
       tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE);
       pCreateReq = (SVCreateTbReq){0};
@@ -1734,34 +1735,34 @@ static SHashObj* writeRawCache = NULL;
 static int8_t    initFlag = 0;
 static int8_t    initedFlag = WRITE_RAW_INIT_START;
 
-typedef struct{
-  SHashObj*     pVgHash;
-  SHashObj*     pNameHash;
-  SHashObj*     pMetaHash;
-}rawCacheInfo;
+typedef struct {
+  SHashObj* pVgHash;
+  SHashObj* pNameHash;
+  SHashObj* pMetaHash;
+} rawCacheInfo;
 
-typedef struct{
+typedef struct {
   SVgroupInfo vgInfo;
   int64_t     uid;
   int64_t     suid;
-}tbInfo;
+} tbInfo;
 
-static void tmqFreeMeta(void *data){
+static void tmqFreeMeta(void* data) {
   STableMeta* pTableMeta = *(STableMeta**)data;
   taosMemoryFree(pTableMeta);
 }
 
-static void freeRawCache(void *data) {
+static void freeRawCache(void* data) {
   rawCacheInfo* pRawCache = (rawCacheInfo*)data;
   taosHashCleanup(pRawCache->pMetaHash);
   taosHashCleanup(pRawCache->pNameHash);
   taosHashCleanup(pRawCache->pVgHash);
 }
 
-static int32_t initRawCacheHash(){
-  if (writeRawCache == NULL){
+static int32_t initRawCacheHash() {
+  if (writeRawCache == NULL) {
     writeRawCache = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
-    if (writeRawCache == NULL){
+    if (writeRawCache == NULL) {
       return terrno;
     }
     taosHashSetFreeFp(writeRawCache, freeRawCache);
@@ -1769,7 +1770,7 @@ static int32_t initRawCacheHash(){
   return 0;
 }
 
-static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrapper* pSW){
+static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrapper* pSW) {
   char* p = (char*)rawData;
   // | version | total length | total rows | blankFill | total columns | flag seg| block group id | column schema | each
   // column length |
@@ -1799,16 +1800,15 @@ static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrappe
     }
     fields += sizeof(int8_t) + sizeof(int32_t);
 
-    if (j == pTableMeta->tableInfo.numOfColumns)
-      return true;
+    if (j == pTableMeta->tableInfo.numOfColumns) return true;
   }
   return false;
 }
 
-static int32_t getRawCache(SHashObj **pVgHash, SHashObj **pNameHash, SHashObj **pMetaHash, void *key) {
+static int32_t getRawCache(SHashObj** pVgHash, SHashObj** pNameHash, SHashObj** pMetaHash, void* key) {
   int32_t code = 0;
-  void* cacheInfo = taosHashGet(writeRawCache, &key, POINTER_BYTES);
-  if (cacheInfo == NULL){
+  void*   cacheInfo = taosHashGet(writeRawCache, &key, POINTER_BYTES);
+  if (cacheInfo == NULL) {
     *pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
     RAW_NULL_CHECK(*pVgHash);
     *pNameHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
@@ -1819,7 +1819,7 @@ static int32_t getRawCache(SHashObj **pVgHash, SHashObj **pNameHash, SHashObj **
     rawCacheInfo info = {*pVgHash, *pNameHash, *pMetaHash};
     RAW_RETURN_CHECK(taosHashPut(writeRawCache, &key, POINTER_BYTES, &info, sizeof(rawCacheInfo)));
   } else {
-    rawCacheInfo *info = (rawCacheInfo *)cacheInfo;
+    rawCacheInfo* info = (rawCacheInfo*)cacheInfo;
     *pVgHash = info->pVgHash;
     *pNameHash = info->pNameHash;
     *pMetaHash = info->pMetaHash;
@@ -1833,7 +1833,7 @@ end:
   return code;
 }
 
-static int32_t buildRawRequest(TAOS* taos, SRequestObj** pRequest, SCatalog**  pCatalog, SRequestConnInfo  *conn){
+static int32_t buildRawRequest(TAOS* taos, SRequestObj** pRequest, SCatalog** pCatalog, SRequestConnInfo* conn) {
   int32_t code = 0;
   RAW_RETURN_CHECK(createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0, pRequest));
   (*pRequest)->syncQuery = true;
@@ -1852,29 +1852,30 @@ end:
   return code;
 }
 
-typedef int32_t _raw_decode_func_(SDecoder *pDecoder, SMqDataRsp *pRsp);
-static int32_t decodeRawData(SDecoder *decoder, void* data, int32_t dataLen, _raw_decode_func_ func, SMqRspObj* rspObj){
-  int8_t dataVersion = *(int8_t*)data;
-  if (dataVersion >= MQ_DATA_RSP_VERSION) {
-    data = POINTER_SHIFT(data, sizeof(int8_t) + sizeof(int32_t));
-    dataLen -= sizeof(int8_t) + sizeof(int32_t);
+typedef int32_t _raw_decode_func_(SDecoder* pDecoder, SMqDataRsp* pRsp);
+static int32_t  decodeRawData(SDecoder* decoder, void* data, int32_t dataLen, _raw_decode_func_ func,
+                              SMqRspObj* rspObj) {
+   int8_t dataVersion = *(int8_t*)data;
+   if (dataVersion >= MQ_DATA_RSP_VERSION) {
+     data = POINTER_SHIFT(data, sizeof(int8_t) + sizeof(int32_t));
+     dataLen -= sizeof(int8_t) + sizeof(int32_t);
   }
 
-  rspObj->resIter = -1;
-  tDecoderInit(decoder, data, dataLen);
-  int32_t code = func(decoder, &rspObj->dataRsp);
-  if (code != 0) {
-    SET_ERROR_MSG("decode mq taosx data rsp failed");
+   rspObj->resIter = -1;
+   tDecoderInit(decoder, data, dataLen);
+   int32_t code = func(decoder, &rspObj->dataRsp);
+   if (code != 0) {
+     SET_ERROR_MSG("decode mq taosx data rsp failed");
   }
-  return code;
+   return code;
 }
 
-static int32_t processCacheMeta(SHashObj *pVgHash, SHashObj *pNameHash, SHashObj *pMetaHash, SVCreateTbReq* pCreateReqDst,
-                 SCatalog* pCatalog, SRequestConnInfo* conn, SName* pName,
-                 STableMeta** pMeta, SSchemaWrapper* pSW, void* rawData, int32_t retry){
-  int32_t code = 0;
+static int32_t processCacheMeta(SHashObj* pVgHash, SHashObj* pNameHash, SHashObj* pMetaHash,
+                                SVCreateTbReq* pCreateReqDst, SCatalog* pCatalog, SRequestConnInfo* conn, SName* pName,
+                                STableMeta** pMeta, SSchemaWrapper* pSW, void* rawData, int32_t retry) {
+  int32_t     code = 0;
   STableMeta* pTableMeta = NULL;
-  tbInfo* tmpInfo = (tbInfo*)taosHashGet(pNameHash, pName->tname, strlen(pName->tname));
+  tbInfo*     tmpInfo = (tbInfo*)taosHashGet(pNameHash, pName->tname, strlen(pName->tname));
   if (tmpInfo == NULL || retry > 0) {
     tbInfo info = {0};
 
@@ -1884,13 +1885,13 @@ static int32_t processCacheMeta(SHashObj *pVgHash, SHashObj *pNameHash, SHashObj
     }
     RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, conn, pName, &pTableMeta));
     info.uid = pTableMeta->uid;
-    if (pTableMeta->tableType == TSDB_CHILD_TABLE){
+    if (pTableMeta->tableType == TSDB_CHILD_TABLE) {
       info.suid = pTableMeta->suid;
     } else {
       info.suid = pTableMeta->uid;
     }
     code = taosHashPut(pMetaHash, &info.suid, LONG_BYTES, &pTableMeta, POINTER_BYTES);
-    if (code != 0){
+    if (code != 0) {
       taosMemoryFree(pTableMeta);
       goto end;
     }
@@ -1902,20 +1903,21 @@ static int32_t processCacheMeta(SHashObj *pVgHash, SHashObj *pNameHash, SHashObj
 
     RAW_RETURN_CHECK(taosHashPut(pNameHash, pName->tname, strlen(pName->tname), &info, sizeof(tbInfo)));
     tmpInfo = (tbInfo*)taosHashGet(pNameHash, pName->tname, strlen(pName->tname));
-    RAW_RETURN_CHECK(taosHashPut(pVgHash, &info.vgInfo.vgId, sizeof(info.vgInfo.vgId), &info.vgInfo, sizeof(SVgroupInfo)));
+    RAW_RETURN_CHECK(
+        taosHashPut(pVgHash, &info.vgInfo.vgId, sizeof(info.vgInfo.vgId), &info.vgInfo, sizeof(SVgroupInfo)));
   }
 
-  if (pTableMeta == NULL || retry > 0){
+  if (pTableMeta == NULL || retry > 0) {
     STableMeta** pTableMetaTmp = (STableMeta**)taosHashGet(pMetaHash, &tmpInfo->suid, LONG_BYTES);
     if (pTableMetaTmp == NULL || retry > 0 || needRefreshMeta(rawData, *pTableMetaTmp, pSW)) {
       RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, conn, pName, &pTableMeta));
       code = taosHashPut(pMetaHash, &tmpInfo->suid, LONG_BYTES, &pTableMeta, POINTER_BYTES);
-      if (code != 0){
+      if (code != 0) {
         taosMemoryFree(pTableMeta);
         goto end;
       }
 
-    }else{
+    } else {
       pTableMeta = *pTableMetaTmp;
       pTableMeta->uid = tmpInfo->uid;
       pTableMeta->vgId = tmpInfo->vgInfo.vgId;
@@ -1927,25 +1929,25 @@ end:
   return code;
 }
 
-static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen){
-  int32_t        code = TSDB_CODE_SUCCESS;
-  SQuery*        pQuery = NULL;
-  SMqRspObj      rspObj = {0};
-  SDecoder       decoder = {0};
+static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
+  int32_t   code = TSDB_CODE_SUCCESS;
+  SQuery*   pQuery = NULL;
+  SMqRspObj rspObj = {0};
+  SDecoder  decoder = {0};
 
-  SRequestObj*      pRequest = NULL;
-  SCatalog*         pCatalog = NULL;
-  SRequestConnInfo  conn = {0};
+  SRequestObj*     pRequest = NULL;
+  SCatalog*        pCatalog = NULL;
+  SRequestConnInfo conn = {0};
   RAW_RETURN_CHECK(buildRawRequest(taos, &pRequest, &pCatalog, &conn));
   uDebug(LOG_ID_TAG " write raw data, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen);
   RAW_RETURN_CHECK(decodeRawData(&decoder, data, dataLen, tDecodeMqDataRsp, &rspObj));
 
-  SHashObj *pVgHash = NULL;
-  SHashObj *pNameHash = NULL;
-  SHashObj *pMetaHash = NULL;
+  SHashObj* pVgHash = NULL;
+  SHashObj* pNameHash = NULL;
+  SHashObj* pMetaHash = NULL;
   RAW_RETURN_CHECK(getRawCache(&pVgHash, &pNameHash, &pMetaHash, taos));
   int retry = 0;
-  while(1){
+  while (1) {
     RAW_RETURN_CHECK(smlInitHandle(&pQuery));
     uDebug(LOG_ID_TAG " write raw meta data block num:%d", LOG_ID_VALUE, rspObj.dataRsp.blockNum);
     while (++rspObj.resIter < rspObj.dataRsp.blockNum) {
@@ -1968,9 +1970,9 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen){
       tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN);
 
       STableMeta* pTableMeta = NULL;
-      RAW_RETURN_CHECK(processCacheMeta(pVgHash, pNameHash, pMetaHash, NULL, pCatalog, &conn,
-                       &pName, &pTableMeta, pSW, rawData, retry));
-      char  err[ERR_MSG_LEN] = {0};
+      RAW_RETURN_CHECK(processCacheMeta(pVgHash, pNameHash, pMetaHash, NULL, pCatalog, &conn, &pName, &pTableMeta, pSW,
+                                        rawData, retry));
+      char err[ERR_MSG_LEN] = {0};
       code = rawBlockBindData(pQuery, pTableMeta, rawData, NULL, pSW, pSW->nCols, true, err, ERR_MSG_LEN, true);
       if (code != TSDB_CODE_SUCCESS) {
         SET_ERROR_MSG("table:%s, err:%s", pName.tname, err);
@@ -1991,7 +1993,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen){
     break;
   }
 
-  end:
+end:
   uDebug(LOG_ID_TAG " write raw data return, msg:%s", LOG_ID_VALUE, tstrerror(code));
   tDeleteMqDataRsp(&rspObj.dataRsp);
   tDecoderClear(&decoder);
@@ -2001,15 +2003,15 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen){
 }
 
 static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) {
-  int32_t        code = TSDB_CODE_SUCCESS;
-  SQuery*        pQuery = NULL;
-  SMqRspObj      rspObj = {0};
-  SDecoder       decoder = {0};
-  SHashObj*      pCreateTbHash = NULL;
+  int32_t   code = TSDB_CODE_SUCCESS;
+  SQuery*   pQuery = NULL;
+  SMqRspObj rspObj = {0};
+  SDecoder  decoder = {0};
+  SHashObj* pCreateTbHash = NULL;
 
-  SRequestObj*      pRequest = NULL;
-  SCatalog*         pCatalog = NULL;
-  SRequestConnInfo  conn = {0};
+  SRequestObj*     pRequest = NULL;
+  SCatalog*        pCatalog = NULL;
+  SRequestConnInfo conn = {0};
 
   RAW_RETURN_CHECK(buildRawRequest(taos, &pRequest, &pCatalog, &conn));
   uDebug(LOG_ID_TAG " write raw metadata, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen);
@@ -2019,12 +2021,12 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
   RAW_NULL_CHECK(pCreateTbHash);
   RAW_RETURN_CHECK(buildCreateTbMap(&rspObj.dataRsp, pCreateTbHash));
 
-  SHashObj *pVgHash = NULL;
-  SHashObj *pNameHash = NULL;
-  SHashObj *pMetaHash = NULL;
+  SHashObj* pVgHash = NULL;
+  SHashObj* pNameHash = NULL;
+  SHashObj* pMetaHash = NULL;
   RAW_RETURN_CHECK(getRawCache(&pVgHash, &pNameHash, &pMetaHash, taos));
   int retry = 0;
-  while(1){
+  while (1) {
     RAW_RETURN_CHECK(smlInitHandle(&pQuery));
     uDebug(LOG_ID_TAG " write raw meta data block num:%d", LOG_ID_VALUE, rspObj.dataRsp.blockNum);
     while (++rspObj.resIter < rspObj.dataRsp.blockNum) {
@@ -2048,11 +2050,12 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
 
       // find schema data info
       SVCreateTbReq* pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, pName.tname, strlen(pName.tname));
-      STableMeta* pTableMeta = NULL;
-      RAW_RETURN_CHECK(processCacheMeta(pVgHash, pNameHash, pMetaHash, pCreateReqDst, pCatalog, &conn,
-                       &pName, &pTableMeta, pSW, rawData, retry));
-      char  err[ERR_MSG_LEN] = {0};
-      code = rawBlockBindData(pQuery, pTableMeta, rawData, pCreateReqDst, pSW, pSW->nCols, true, err, ERR_MSG_LEN, true);
+      STableMeta*    pTableMeta = NULL;
+      RAW_RETURN_CHECK(processCacheMeta(pVgHash, pNameHash, pMetaHash, pCreateReqDst, pCatalog, &conn, &pName,
+                                        &pTableMeta, pSW, rawData, retry));
+      char err[ERR_MSG_LEN] = {0};
+      code =
+          rawBlockBindData(pQuery, pTableMeta, rawData, pCreateReqDst, pSW, pSW->nCols, true, err, ERR_MSG_LEN, true);
       if (code != TSDB_CODE_SUCCESS) {
         SET_ERROR_MSG("table:%s, err:%s", pName.tname, err);
         goto end;
@@ -2147,7 +2150,7 @@ static void processBatchMetaToJson(SMqBatchMetaRsp* pMsgRsp, char** string) {
   *string = fullStr;
   return;
 
-  end:
+end:
   cJSON_Delete(pJson);
   tDeleteMqBatchMetaRsp(&rsp);
 }
@@ -2159,18 +2162,18 @@ char* tmq_get_json_meta(TAOS_RES* res) {
     return NULL;
   }
 
-  char*           string = NULL;
+  char*      string = NULL;
   SMqRspObj* rspObj = (SMqRspObj*)res;
   if (TD_RES_TMQ_METADATA(res)) {
     processAutoCreateTable(&rspObj->dataRsp, &string);
   } else if (TD_RES_TMQ_BATCH_META(res)) {
     processBatchMetaToJson(&rspObj->batchMetaRsp, &string);
   } else if (TD_RES_TMQ_META(res)) {
-    cJSON*         pJson = NULL;
+    cJSON* pJson = NULL;
     processSimpleMeta(&rspObj->metaRsp, &pJson);
     string = cJSON_PrintUnformatted(pJson);
     cJSON_Delete(pJson);
-  } else{
+  } else {
     uError("tmq_get_json_meta res:%d, invalid type", *(int8_t*)res);
   }
 
@@ -2181,7 +2184,7 @@ char* tmq_get_json_meta(TAOS_RES* res) {
 void tmq_free_json_meta(char* jsonMeta) { taosMemoryFreeClear(jsonMeta); }
 
 static int32_t getOffSetLen(const SMqDataRsp* pRsp) {
-  SEncoder                coder = {0};
+  SEncoder coder = {0};
   tEncoderInit(&coder, NULL, 0);
   if (tEncodeSTqOffsetVal(&coder, &pRsp->reqOffset) < 0) return -1;
   if (tEncodeSTqOffsetVal(&coder, &pRsp->rspOffset) < 0) return -1;
@@ -2191,46 +2194,46 @@ static int32_t getOffSetLen(const SMqDataRsp* pRsp) {
 }
 
 typedef int32_t __encode_func__(SEncoder* pEncoder, const SMqDataRsp* pRsp);
-static int32_t encodeMqDataRsp(__encode_func__* encodeFunc, SMqDataRsp* rspObj, tmq_raw_data* raw) {
-  int32_t  len = 0;
-  int32_t  code = 0;
-  SEncoder encoder = {0};
-  void*    buf = NULL;
-  tEncodeSize(encodeFunc, rspObj, len, code);
-  if (code < 0) {
-    code = TSDB_CODE_INVALID_MSG;
-    goto FAILED;
+static int32_t  encodeMqDataRsp(__encode_func__* encodeFunc, SMqDataRsp* rspObj, tmq_raw_data* raw) {
+   int32_t  len = 0;
+   int32_t  code = 0;
+   SEncoder encoder = {0};
+   void*    buf = NULL;
+   tEncodeSize(encodeFunc, rspObj, len, code);
+   if (code < 0) {
+     code = TSDB_CODE_INVALID_MSG;
+     goto FAILED;
   }
-  len += sizeof(int8_t) + sizeof(int32_t);
-  buf = taosMemoryCalloc(1, len);
-  if (buf == NULL) {
-    code = terrno;
-    goto FAILED;
+   len += sizeof(int8_t) + sizeof(int32_t);
+   buf = taosMemoryCalloc(1, len);
+   if (buf == NULL) {
+     code = terrno;
+     goto FAILED;
   }
-  tEncoderInit(&encoder, buf, len);
-  if (tEncodeI8(&encoder, MQ_DATA_RSP_VERSION) < 0) {
-    code = TSDB_CODE_INVALID_MSG;
-    goto FAILED;
+   tEncoderInit(&encoder, buf, len);
+   if (tEncodeI8(&encoder, MQ_DATA_RSP_VERSION) < 0) {
+     code = TSDB_CODE_INVALID_MSG;
+     goto FAILED;
   }
-  int32_t offsetLen = getOffSetLen(rspObj);
-  if (offsetLen <= 0) {
-    code = TSDB_CODE_INVALID_MSG;
-    goto FAILED;
+   int32_t offsetLen = getOffSetLen(rspObj);
+   if (offsetLen <= 0) {
+     code = TSDB_CODE_INVALID_MSG;
+     goto FAILED;
   }
-  if (tEncodeI32(&encoder, offsetLen) < 0) {
-    code = TSDB_CODE_INVALID_MSG;
-    goto FAILED;
+   if (tEncodeI32(&encoder, offsetLen) < 0) {
+     code = TSDB_CODE_INVALID_MSG;
+     goto FAILED;
   }
-  if (encodeFunc(&encoder, rspObj) < 0) {
-    code = TSDB_CODE_INVALID_MSG;
-    goto FAILED;
+   if (encodeFunc(&encoder, rspObj) < 0) {
+     code = TSDB_CODE_INVALID_MSG;
+     goto FAILED;
   }
-  tEncoderClear(&encoder);
+   tEncoderClear(&encoder);
 
-  raw->raw = buf;
-  raw->raw_len = len;
-  return code;
-  FAILED:
+   raw->raw = buf;
+   raw->raw_len = len;
+   return code;
+FAILED:
   tEncoderClear(&encoder);
   taosMemoryFree(buf);
   return code;
@@ -2247,7 +2250,7 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) {
     raw->raw_type = rspObj->metaRsp.resMsgType;
     uDebug("tmq get raw type meta:%p", raw);
   } else if (TD_RES_TMQ(res)) {
-    int32_t    code = encodeMqDataRsp(tEncodeMqDataRsp, &rspObj->dataRsp, raw);
+    int32_t code = encodeMqDataRsp(tEncodeMqDataRsp, &rspObj->dataRsp, raw);
     if (code != 0) {
       uError("tmq get raw type error:%d", terrno);
       return code;
@@ -2282,7 +2285,7 @@ void tmq_free_raw(tmq_raw_data raw) {
   (void)memset(terrMsg, 0, ERR_MSG_LEN);
 }
 
-static int32_t writeRawInit(){
+static int32_t writeRawInit() {
   while (atomic_load_8(&initedFlag) == WRITE_RAW_INIT_START) {
     int8_t old = atomic_val_compare_exchange_8(&initFlag, 0, 1);
     if (old == 0) {
@@ -2296,7 +2299,7 @@ static int32_t writeRawInit(){
     }
   }
 
-  if (atomic_load_8(&initedFlag) == WRITE_RAW_INIT_FAIL){
+  if (atomic_load_8(&initedFlag) == WRITE_RAW_INIT_FAIL) {
     return TSDB_CODE_INTERNAL_ERROR;
   }
   return 0;
@@ -2321,7 +2324,7 @@ static int32_t writeRawImpl(TAOS* taos, void* buf, uint32_t len, uint16_t type)
     return taosDropTable(taos, buf, len);
   } else if (type == TDMT_VND_DELETE) {
     return taosDeleteData(taos, buf, len);
-  } else if (type == RES_TYPE__TMQ_METADATA){
+  } else if (type == RES_TYPE__TMQ_METADATA) {
     return tmqWriteRawMetaDataImpl(taos, buf, len);
   } else if (type == RES_TYPE__TMQ) {
     return tmqWriteRawDataImpl(taos, buf, len);
@@ -2344,9 +2347,9 @@ static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen
   if (taos == NULL || meta == NULL) {
     return TSDB_CODE_INVALID_PARA;
   }
-  SMqBatchMetaRsp rsp   = {0};
+  SMqBatchMetaRsp rsp = {0};
   SDecoder        coder = {0};
-  int32_t         code  = TSDB_CODE_SUCCESS;
+  int32_t         code = TSDB_CODE_SUCCESS;
 
   // decode and process req
   tDecoderInit(&coder, meta, metaLen);
@@ -2374,7 +2377,7 @@ static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen
     }
   }
 
-  end:
+end:
   tDeleteMqBatchMetaRsp(&rsp);
   return code;
 }

From af0a41979e59c3846833b66e1ff9350fe4d04479 Mon Sep 17 00:00:00 2001
From: yihaoDeng <yhdeng@taosdata.com>
Date: Mon, 28 Oct 2024 18:11:39 +0800
Subject: [PATCH 27/28] fix:conflicts from 3.0

---
 source/common/src/tglobal.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index 56af9a1c47..7ea6b01bf3 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -2000,7 +2000,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) {
         {"tdbDebugFlag", &tdbDebugFlag},   {"tmrDebugFlag", &tmrDebugFlag}, {"uDebugFlag", &uDebugFlag},
         {"smaDebugFlag", &smaDebugFlag},   {"rpcDebugFlag", &rpcDebugFlag}, {"qDebugFlag", &qDebugFlag},
         {"metaDebugFlag", &metaDebugFlag}, {"stDebugFlag", &stDebugFlag},   {"sndDebugFlag", &sndDebugFlag},
-        {"tqClientDebugFlag", &}tqClientDebugFlag,
+        {"tqClientDebugFlag", &tqClientDebugFlag},
     };
 
     static OptionNameAndVar options[] = {{"audit", &tsEnableAudit},

From 6c155d949e43457561ce02147b99dca0ab263ae9 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Mon, 28 Oct 2024 23:13:31 +0800
Subject: [PATCH 28/28] fix:[TS-5441] test case error

---
 tests/system-test/2-query/db.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/system-test/2-query/db.py b/tests/system-test/2-query/db.py
index cd7c5bd26e..d4e5f89aa3 100644
--- a/tests/system-test/2-query/db.py
+++ b/tests/system-test/2-query/db.py
@@ -57,7 +57,7 @@ class TDTestCase:
             tdSql.checkData(0, 2, 0)
 
         tdSql.query("show dnode 1 variables like '%debugFlag'")
-        tdSql.checkRows(24)
+        tdSql.checkRows(25)
 
         tdSql.query("show dnode 1 variables like '____debugFlag'")
         tdSql.checkRows(2)