From 25e75f20ad97f22b84b8c212279f5a00663a2572 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Tue, 13 Sep 2022 13:53:23 +0800
Subject: [PATCH 01/57] fix:add test cases for sml

---
 tests/system-test/fulltest.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh
index 8987ba3bbd..138633868a 100755
--- a/tests/system-test/fulltest.sh
+++ b/tests/system-test/fulltest.sh
@@ -156,8 +156,8 @@ python3 ./test.py -f 2-query/sin.py
 python3 ./test.py -f 2-query/sin.py -R
 python3 ./test.py -f 2-query/smaTest.py
 python3 ./test.py -f 2-query/smaTest.py -R
-#python3 ./test.py -f 2-query/sml.py
-#python3 ./test.py -f 2-query/sml.py -R
+python3 ./test.py -f 2-query/sml.py
+python3 ./test.py -f 2-query/sml.py -R
 python3 ./test.py -f 2-query/spread.py
 python3 ./test.py -f 2-query/spread.py -R
 python3 ./test.py -f 2-query/sqrt.py
@@ -512,6 +512,6 @@ python3 ./test.py -f 2-query/count_partition.py -Q 3
 python3 ./test.py -f 2-query/max_partition.py -Q 3
 python3 ./test.py -f 2-query/last_row.py -Q 3
 python3 ./test.py -f 2-query/tsbsQuery.py -Q 3
-#python3 ./test.py -f 2-query/sml.py -Q 3
+python3 ./test.py -f 2-query/sml.py -Q 3
 python3 ./test.py -f 2-query/interp.py -Q 3
 

From a3161bf9c19eb1bb8adb4c1639b530a28b648ba2 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Tue, 13 Sep 2022 19:16:26 +0800
Subject: [PATCH 02/57] fix:add logic for auto create table in taosX

---
 include/client/taos.h                   |   2 +-
 source/client/inc/clientInt.h           |   6 +-
 source/client/src/clientMain.c          |   2 +-
 source/client/src/clientRawBlockWrite.c | 392 ++++++++++++++++++++++--
 source/client/src/clientTmq.c           |  10 +-
 5 files changed, 378 insertions(+), 34 deletions(-)

diff --git a/include/client/taos.h b/include/client/taos.h
index 49cfbb52b8..270b647a77 100644
--- a/include/client/taos.h
+++ b/include/client/taos.h
@@ -254,7 +254,7 @@ enum tmq_res_t {
   TMQ_RES_INVALID = -1,
   TMQ_RES_DATA = 1,
   TMQ_RES_TABLE_META = 2,
-  TMQ_RES_TAOSX = 3,
+  TMQ_RES_METADATA = 3,
 };
 
 typedef struct tmq_raw_data {
diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h
index b8fa9580e7..574d8188fe 100644
--- a/source/client/inc/clientInt.h
+++ b/source/client/inc/clientInt.h
@@ -52,7 +52,7 @@ enum {
   RES_TYPE__QUERY = 1,
   RES_TYPE__TMQ,
   RES_TYPE__TMQ_META,
-  RES_TYPE__TAOSX,
+  RES_TYPE__TMQ_METADATA,
 };
 
 #define SHOW_VARIABLES_RESULT_COLS       2
@@ -60,9 +60,9 @@ enum {
 #define SHOW_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE)
 
 #define TD_RES_QUERY(res)     (*(int8_t*)res == RES_TYPE__QUERY)
-#define TD_RES_TMQ(res)       (*(int8_t*)res == RES_TYPE__TMQ || *(int8_t*)res == RES_TYPE__TAOSX)
+#define TD_RES_TMQ(res)       (*(int8_t*)res == RES_TYPE__TMQ)
 #define TD_RES_TMQ_META(res)  (*(int8_t*)res == RES_TYPE__TMQ_META)
-#define TD_RES_TMQ_TAOSX(res) (*(int8_t*)res == RES_TYPE__TAOSX)
+#define TD_RES_TMQ_METADATA(res) (*(int8_t*)res == RES_TYPE__TMQ_METADATA)
 
 typedef struct SAppInstInfo SAppInstInfo;
 
diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index 3086078080..273bbcf7ac 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -184,7 +184,7 @@ void taos_free_result(TAOS_RES *res) {
     SRequestObj *pRequest = (SRequestObj *)res;
     tscDebug("0x%" PRIx64 " taos_free_result start to free query", pRequest->requestId);
     destroyRequest(pRequest);
-  } else if (TD_RES_TMQ_TAOSX(res)) {
+  } else if (TD_RES_TMQ_METADATA(res)) {
     SMqTaosxRspObj *pRsp = (SMqTaosxRspObj *)res;
     if (pRsp->rsp.blockData) taosArrayDestroyP(pRsp->rsp.blockData, taosMemoryFree);
     if (pRsp->rsp.blockDataLen) taosArrayDestroy(pRsp->rsp.blockDataLen);
diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index c135965f07..7c78928333 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -25,12 +25,11 @@
 #include "tref.h"
 #include "ttimer.h"
 
-static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id,
-                                  int8_t t) {
-  char*  string = NULL;
+static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id,
+                                  int8_t t, cJSON* tables) {
   cJSON* json = cJSON_CreateObject();
   if (json == NULL) {
-    return string;
+    return;
   }
   cJSON* type = cJSON_CreateString("create");
   cJSON_AddItemToObject(json, "type", type);
@@ -87,10 +86,7 @@ static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sch
     cJSON_AddItemToArray(tags, tag);
   }
   cJSON_AddItemToObject(json, "tags", tags);
-
-  string = cJSON_PrintUnformatted(json);
-  cJSON_Delete(json);
-  return string;
+  cJSON_AddItemToArray(tables, json);
 }
 
 static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) {
@@ -189,6 +185,7 @@ static char* processCreateStb(SMqMetaRsp* metaRsp) {
   SVCreateStbReq req = {0};
   SDecoder       coder;
   char*          string = NULL;
+  cJSON*         tables = cJSON_CreateArray();
 
   // decode and process req
   void*   data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead));
@@ -198,11 +195,11 @@ static char* processCreateStb(SMqMetaRsp* metaRsp) {
   if (tDecodeSVCreateStbReq(&coder, &req) < 0) {
     goto _err;
   }
-  string = buildCreateTableJson(&req.schemaRow, &req.schemaTag, req.name, req.suid, TSDB_SUPER_TABLE);
-  tDecoderClear(&coder);
-  return string;
+  buildCreateTableJson(&req.schemaRow, &req.schemaTag, req.name, req.suid, TSDB_SUPER_TABLE, tables);
+  string = cJSON_PrintUnformatted(tables);
 
 _err:
+  cJSON_Delete(tables);
   tDecoderClear(&coder);
   return string;
 }
@@ -229,12 +226,11 @@ _err:
   return string;
 }
 
-static char* buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* tagName, int64_t id, uint8_t tagNum) {
-  char*   string = NULL;
+static void buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* tagName, int64_t id, uint8_t tagNum, cJSON* tables) {
   SArray* pTagVals = NULL;
   cJSON*  json = cJSON_CreateObject();
   if (json == NULL) {
-    return string;
+    return;
   }
   cJSON* type = cJSON_CreateString("create");
   cJSON_AddItemToObject(json, "type", type);
@@ -315,10 +311,8 @@ static char* buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray*
 
 end:
   cJSON_AddItemToObject(json, "tags", tags);
-  string = cJSON_PrintUnformatted(json);
-  cJSON_Delete(json);
   taosArrayDestroy(pTagVals);
-  return string;
+  cJSON_AddItemToArray(tables, json);
 }
 
 static char* processCreateTable(SMqMetaRsp* metaRsp) {
@@ -335,24 +329,57 @@ static char* processCreateTable(SMqMetaRsp* metaRsp) {
   }
 
   // loop to create table
+  cJSON*  tables = cJSON_CreateArray();
   for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
     pCreateReq = req.pReqs + iReq;
     if (pCreateReq->type == TSDB_CHILD_TABLE) {
-      string = buildCreateCTableJson((STag*)pCreateReq->ctb.pTag, pCreateReq->ctb.name, pCreateReq->name,
-                                     pCreateReq->ctb.tagName, pCreateReq->uid, pCreateReq->ctb.tagNum);
+      buildCreateCTableJson((STag*)pCreateReq->ctb.pTag, pCreateReq->ctb.name, pCreateReq->name,
+                                     pCreateReq->ctb.tagName, pCreateReq->uid, pCreateReq->ctb.tagNum, tables);
     } else if (pCreateReq->type == TSDB_NORMAL_TABLE) {
-      string =
-          buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE);
+      buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE, tables);
     }
   }
 
-  tDecoderClear(&decoder);
-
+  string = cJSON_PrintUnformatted(tables);
+  cJSON_Delete(tables);
 _exit:
   tDecoderClear(&decoder);
   return string;
 }
 
+static char* processAutoCreateTable(STaosxRsp* rsp) {
+  SDecoder           decoder = {0};
+  SVCreateTbReq*     pCreateReq;
+  char*              string = NULL;
+
+
+  // loop to create table
+  cJSON*  tables = cJSON_CreateArray();
+  for (int32_t iReq = 0; iReq < rsp->createTableNum; iReq++) {
+    // decode
+    void**   data = taosArrayGet(rsp->createTableReq, iReq);
+    int32_t *len = taosArrayGet(rsp->createTableLen, iReq);
+    tDecoderInit(&decoder, *data, *len);
+    if (tDecodeSVCreateTbReq(&decoder, pCreateReq) < 0) {
+      tDecoderClear(&decoder);
+      goto _exit;
+    }
+
+    if (pCreateReq->type == TSDB_CHILD_TABLE) {
+      buildCreateCTableJson((STag*)pCreateReq->ctb.pTag, pCreateReq->ctb.name, pCreateReq->name,
+                            pCreateReq->ctb.tagName, pCreateReq->uid, pCreateReq->ctb.tagNum, tables);
+    } else if (pCreateReq->type == TSDB_NORMAL_TABLE) {
+      buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE, tables);
+    }
+    tDecoderClear(&decoder);
+  }
+
+  string = cJSON_PrintUnformatted(tables);
+_exit:
+  cJSON_Delete(tables);
+  return string;
+}
+
 static char* processAlterTable(SMqMetaRsp* metaRsp) {
   SDecoder     decoder = {0};
   SVAlterTbReq vAlterTbReq = {0};
@@ -1586,11 +1613,307 @@ end:
   return code;
 }
 
+static int32_t tmqWriteRaqMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) {
+  int32_t     code = TSDB_CODE_SUCCESS;
+  SHashObj*   pVgHash = NULL;
+  SQuery*     pQuery = NULL;
+  SMqTaosxRspObj   rspObj = {0};
+  SDecoder    decoder = {0};
+  STableMeta* pTableMeta = NULL;
+
+  terrno = TSDB_CODE_SUCCESS;
+  SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT);
+  if (!pRequest) {
+    uError("WriteRaw:createRequest error request is null");
+    return terrno;
+  }
+
+  rspObj.resIter = -1;
+  rspObj.resType = RES_TYPE__TMQ_METADATA;
+
+  tDecoderInit(&decoder, data, dataLen);
+  code = tDecodeSTaosxRsp(&decoder, &rspObj.rsp);
+  if (code != 0) {
+    uError("WriteRaw:decode smqDataRsp error");
+    code = TSDB_CODE_INVALID_MSG;
+    goto end;
+  }
+
+  if (!pRequest->pDb) {
+    uError("WriteRaw:not use db");
+    code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
+    goto end;
+  }
+
+  pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
+  taosHashSetFreeFp(pVgHash, destroyVgHash);
+  struct SCatalog* pCatalog = NULL;
+  code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
+  if (code != TSDB_CODE_SUCCESS) {
+    uError("WriteRaw: get gatlog error");
+    goto end;
+  }
+
+  SRequestConnInfo conn = {0};
+  conn.pTrans = pRequest->pTscObj->pAppInfo->pTransporter;
+  conn.requestId = pRequest->requestId;
+  conn.requestObjRefId = pRequest->self;
+  conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
+
+  printf("raw data block num:%d\n", rspObj.rsp.blockNum);
+  while (++rspObj.resIter < rspObj.rsp.blockNum) {
+    SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter);
+    if (!rspObj.rsp.withSchema) {
+      uError("WriteRaw:no schema, iter:%d", rspObj.resIter);
+      goto end;
+    }
+    SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.blockSchema, rspObj.resIter);
+    setResSchemaInfo(&rspObj.resInfo, pSW->pSchema, pSW->nCols);
+
+    code = setQueryResultFromRsp(&rspObj.resInfo, pRetrieve, false, false);
+    if (code != TSDB_CODE_SUCCESS) {
+      uError("WriteRaw: setQueryResultFromRsp error");
+      goto end;
+    }
+
+    const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.blockTbName, rspObj.resIter);
+    if (!tbName) {
+      uError("WriteRaw: tbname is null");
+      code = TSDB_CODE_TMQ_INVALID_MSG;
+      goto end;
+    }
+
+    printf("raw data tbname:%s\n", tbName);
+    SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
+    strcpy(pName.dbname, pRequest->pDb);
+    strcpy(pName.tname, tbName);
+
+    VgData vgData = {0};
+    code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &(vgData.vg));
+    if (code != TSDB_CODE_SUCCESS) {
+      uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbName);
+      goto end;
+    }
+
+    code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta);
+    if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
+      uError("WriteRaw:catalogGetTableMeta table not exist. table name: %s", tbName);
+      code = TSDB_CODE_SUCCESS;
+      continue;
+    }
+    if (code != TSDB_CODE_SUCCESS) {
+      uError("WriteRaw:catalogGetTableMeta failed. table name: %s", tbName);
+      goto end;
+    }
+
+    uint16_t fLen = 0;
+    int32_t  rowSize = 0;
+    int16_t  nVar = 0;
+    for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) {
+      SSchema* schema = &pTableMeta->schema[i];
+      fLen += TYPE_BYTES[schema->type];
+      rowSize += schema->bytes;
+      if (IS_VAR_DATA_TYPE(schema->type)) {
+        nVar++;
+      }
+    }
+
+    int32_t rows = rspObj.resInfo.numOfRows;
+    int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) +
+                              (int32_t)TD_BITMAP_BYTES(pTableMeta->tableInfo.numOfColumns - 1);
+
+    // find schema data info
+    int32_t schemaLen = 0;
+    void* schemaData = NULL;
+    for(int j = 0; j < rspObj.rsp.createTableNum; j++){
+      void** dataTmp = taosArrayGet(rspObj.rsp.createTableReq, j);
+      int32_t* lenTmp = taosArrayGet(rspObj.rsp.createTableLen, j);
+
+      SDecoder           decoderTmp = {0};
+      SVCreateTbReq*     pCreateReq;
+
+      tDecoderInit(&decoderTmp, *dataTmp, *lenTmp);
+      if (tDecodeSVCreateTbReq(&decoderTmp, pCreateReq) < 0) {
+        tDecoderClear(&decoderTmp);
+        goto end;
+      }
+
+      ASSERT (pCreateReq->type == TSDB_CHILD_TABLE);
+      if(strcmp(tbName, pCreateReq->name) == 0){
+        schemaLen = *lenTmp;
+        schemaData = *dataTmp;
+        tDecoderClear(&decoderTmp);
+        break;
+      }
+      tDecoderClear(&decoderTmp);
+    }
+
+    int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize;
+
+    SSubmitReq* subReq = NULL;
+    SSubmitBlk* blk = NULL;
+    void*       hData = taosHashGet(pVgHash, &vgData.vg.vgId, sizeof(vgData.vg.vgId));
+    if (hData) {
+      vgData = *(VgData*)hData;
+
+      int32_t totalLen = ((SSubmitReq*)(vgData.data))->length + submitLen;
+      void*   tmp = taosMemoryRealloc(vgData.data, totalLen);
+      if (tmp == NULL) {
+        code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+        goto end;
+      }
+      vgData.data = tmp;
+      ((VgData*)hData)->data = tmp;
+      subReq = (SSubmitReq*)(vgData.data);
+      blk = POINTER_SHIFT(vgData.data, subReq->length);
+    } else {
+      int32_t totalLen = sizeof(SSubmitReq) + submitLen;
+      void*   tmp = taosMemoryCalloc(1, totalLen);
+      if (tmp == NULL) {
+        code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+        goto end;
+      }
+      vgData.data = tmp;
+      taosHashPut(pVgHash, (const char*)&vgData.vg.vgId, sizeof(vgData.vg.vgId), (char*)&vgData, sizeof(vgData));
+      subReq = (SSubmitReq*)(vgData.data);
+      subReq->length = sizeof(SSubmitReq);
+      subReq->numOfBlocks = 0;
+
+      blk = POINTER_SHIFT(vgData.data, sizeof(SSubmitReq));
+    }
+
+    // pSW->pSchema should be same as pTableMeta->schema
+    //    ASSERT(pSW->nCols == pTableMeta->tableInfo.numOfColumns);
+    uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid);
+    uint64_t uid = pTableMeta->uid;
+    int16_t  sver = pTableMeta->sversion;
+
+    void*   blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk));
+    if(schemaData){
+      memcpy(blkSchema, schemaData, schemaLen);
+    }
+    STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen);
+
+    SRowBuilder rb = {0};
+    tdSRowInit(&rb, sver);
+    tdSRowSetTpInfo(&rb, pTableMeta->tableInfo.numOfColumns, fLen);
+    int32_t totalLen = 0;
+
+    SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
+    for (int i = 0; i < pSW->nCols; i++) {
+      SSchema* schema = &pSW->pSchema[i];
+      taosHashPut(schemaHash, schema->name, strlen(schema->name), &i, sizeof(int32_t));
+    }
+
+    for (int32_t j = 0; j < rows; j++) {
+      tdSRowResetBuf(&rb, rowData);
+
+      doSetOneRowPtr(&rspObj.resInfo);
+      rspObj.resInfo.current += 1;
+
+      int32_t offset = 0;
+      for (int32_t k = 0; k < pTableMeta->tableInfo.numOfColumns; k++) {
+        const SSchema* pColumn = &pTableMeta->schema[k];
+        int32_t*       index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name));
+        if (!index) {
+          tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k);
+        } else {
+          char* colData = rspObj.resInfo.row[*index];
+          if (!colData) {
+            tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k);
+          } else {
+            if (IS_VAR_DATA_TYPE(pColumn->type)) {
+              colData -= VARSTR_HEADER_SIZE;
+            }
+            tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k);
+          }
+        }
+
+        offset += TYPE_BYTES[pColumn->type];
+      }
+      tdSRowEnd(&rb);
+      int32_t rowLen = TD_ROW_LEN(rowData);
+      rowData = POINTER_SHIFT(rowData, rowLen);
+      totalLen += rowLen;
+    }
+
+    taosHashCleanup(schemaHash);
+    blk->uid = htobe64(uid);
+    blk->suid = htobe64(suid);
+    blk->sversion = htonl(sver);
+    blk->schemaLen = htonl(schemaLen);
+    blk->numOfRows = htonl(rows);
+    blk->dataLen = htonl(totalLen);
+    subReq->length += sizeof(SSubmitBlk) + schemaLen + totalLen;
+    subReq->numOfBlocks++;
+    taosMemoryFreeClear(pTableMeta);
+  }
+
+  pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY);
+  if (NULL == pQuery) {
+    uError("create SQuery error");
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto end;
+  }
+  pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
+  pQuery->haveResultSet = false;
+  pQuery->msgType = TDMT_VND_SUBMIT;
+  pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT);
+  if (NULL == pQuery->pRoot) {
+    uError("create pQuery->pRoot error");
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto end;
+  }
+  SVnodeModifOpStmt* nodeStmt = (SVnodeModifOpStmt*)(pQuery->pRoot);
+  nodeStmt->payloadType = PAYLOAD_TYPE_KV;
+
+  int32_t numOfVg = taosHashGetSize(pVgHash);
+  nodeStmt->pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES);
+
+  VgData* vData = (VgData*)taosHashIterate(pVgHash, NULL);
+  while (vData) {
+    SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks));
+    if (NULL == dst) {
+      code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+      goto end;
+    }
+    dst->vg = vData->vg;
+    SSubmitReq* subReq = (SSubmitReq*)(vData->data);
+    dst->numOfTables = subReq->numOfBlocks;
+    dst->size = subReq->length;
+    dst->pData = (char*)subReq;
+    vData->data = NULL;  // no need free
+    subReq->header.vgId = htonl(dst->vg.vgId);
+    subReq->version = htonl(1);
+    subReq->header.contLen = htonl(subReq->length);
+    subReq->length = htonl(subReq->length);
+    subReq->numOfBlocks = htonl(subReq->numOfBlocks);
+    taosArrayPush(nodeStmt->pDataBlocks, &dst);
+    vData = (VgData*)taosHashIterate(pVgHash, vData);
+  }
+
+  launchQueryImpl(pRequest, pQuery, true, NULL);
+  code = pRequest->code;
+
+  end:
+  tDecoderClear(&decoder);
+  qDestroyQuery(pQuery);
+  destroyRequest(pRequest);
+  taosHashCleanup(pVgHash);
+  taosMemoryFreeClear(pTableMeta);
+  return code;
+}
+
 char* tmq_get_json_meta(TAOS_RES* res) {
-  if (!TD_RES_TMQ_META(res)) {
+  if (!TD_RES_TMQ_META(res) && !TD_RES_TMQ_METADATA(res)) {
     return NULL;
   }
 
+  if(TD_RES_TMQ_METADATA(res)){
+    SMqTaosxRspObj* pMetaDataRspObj = (SMqTaosxRspObj*)res;
+    return processAutoCreateTable(&pMetaDataRspObj->rsp);
+  }
+
   SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
   if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_STB) {
     return processCreateStb(&pMetaRspObj->metaRsp);
@@ -1638,6 +1961,25 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) {
     raw->raw = buf;
     raw->raw_len = len;
     raw->raw_type = RES_TYPE__TMQ;
+  } else if (TD_RES_TMQ_METADATA(res)) {
+    SMqTaosxRspObj* rspObj = ((SMqTaosxRspObj*)res);
+
+    int32_t len = 0;
+    int32_t code = 0;
+    tEncodeSize(tEncodeSTaosxRsp, &rspObj->rsp, len, code);
+    if (code < 0) {
+      return -1;
+    }
+
+    void*    buf = taosMemoryCalloc(1, len);
+    SEncoder encoder = {0};
+    tEncoderInit(&encoder, buf, len);
+    tEncodeSTaosxRsp(&encoder, &rspObj->rsp);
+    tEncoderClear(&encoder);
+
+    raw->raw = buf;
+    raw->raw_len = len;
+    raw->raw_type = RES_TYPE__TMQ_METADATA;
   } else {
     return TSDB_CODE_TMQ_INVALID_MSG;
   }
@@ -1671,6 +2013,8 @@ int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) {
     return taosDeleteData(taos, raw.raw, raw.raw_len);
   } else if (raw.raw_type == RES_TYPE__TMQ) {
     return tmqWriteRawDataImpl(taos, raw.raw, raw.raw_len);
+  } else if (raw.raw_type == RES_TYPE__TMQ_METADATA) {
+    return tmqWriteRaqMetaDataImpl(taos, raw.raw, raw.raw_len);
   }
   return TSDB_CODE_INVALID_PARA;
 }
diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 9f9a14952e..a84c36790a 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -1465,7 +1465,7 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) {
 
 SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper) {
   SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj));
-  pRspObj->resType = RES_TYPE__TAOSX;
+  pRspObj->resType = RES_TYPE__TMQ_METADATA;
   tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN);
   tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN);
   pRspObj->vgId = pWrapper->vgHandle->vgId;
@@ -1649,7 +1649,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
           continue;
         }
         // build rsp
-        SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper);
+        SMqTaosxRspObj* pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper);
         taosFreeQitem(pollRspWrapper);
         return pRsp;
       } else {
@@ -1769,11 +1769,11 @@ tmq_res_t tmq_get_res_type(TAOS_RES* res) {
   } else if (TD_RES_TMQ_META(res)) {
     SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
     if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DELETE) {
-      return TMQ_RES_TAOSX;
+      return TMQ_RES_DATA;
     }
     return TMQ_RES_TABLE_META;
-  } else if (TD_RES_TMQ_TAOSX(res)) {
-    return TMQ_RES_DATA;
+  } else if (TD_RES_TMQ_METADATA(res)) {
+    return TMQ_RES_METADATA;
   } else {
     return TMQ_RES_INVALID;
   }

From faeb5297b14984947769bb93c95b3b8b275477a5 Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Wed, 14 Sep 2022 11:44:54 +0800
Subject: [PATCH 03/57] feat: update taos-tools 9f4c01e for 3.0

---
 cmake/taostools_CMakeLists.txt.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in
index 0f7a450920..c13aa5f4b1 100644
--- a/cmake/taostools_CMakeLists.txt.in
+++ b/cmake/taostools_CMakeLists.txt.in
@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG 3588b3d
+        GIT_TAG 9f4c01e
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

From 8a8423d66d63df0e726e1ac31754388dfb1bf7ad Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 14 Sep 2022 16:44:41 +0800
Subject: [PATCH 04/57] fix:add logic for auto create table in taosX

---
 source/client/src/clientRawBlockWrite.c | 112 ++++++++++++++----------
 source/client/src/clientTmq.c           |  13 ++-
 utils/test/c/tmq_taosx_ci.c             |   4 +-
 3 files changed, 77 insertions(+), 52 deletions(-)

diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 7c78928333..82f7285357 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -25,11 +25,12 @@
 #include "tref.h"
 #include "ttimer.h"
 
-static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id,
-                                  int8_t t, cJSON* tables) {
+static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id,
+                                  int8_t t) {
+  char*  string = NULL;
   cJSON* json = cJSON_CreateObject();
   if (json == NULL) {
-    return;
+    return NULL;
   }
   cJSON* type = cJSON_CreateString("create");
   cJSON_AddItemToObject(json, "type", type);
@@ -86,7 +87,10 @@ static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sche
     cJSON_AddItemToArray(tags, tag);
   }
   cJSON_AddItemToObject(json, "tags", tags);
-  cJSON_AddItemToArray(tables, json);
+
+  string = cJSON_PrintUnformatted(json);
+  cJSON_Delete(json);
+  return string;
 }
 
 static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) {
@@ -185,7 +189,6 @@ static char* processCreateStb(SMqMetaRsp* metaRsp) {
   SVCreateStbReq req = {0};
   SDecoder       coder;
   char*          string = NULL;
-  cJSON*         tables = cJSON_CreateArray();
 
   // decode and process req
   void*   data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead));
@@ -195,11 +198,9 @@ static char* processCreateStb(SMqMetaRsp* metaRsp) {
   if (tDecodeSVCreateStbReq(&coder, &req) < 0) {
     goto _err;
   }
-  buildCreateTableJson(&req.schemaRow, &req.schemaTag, req.name, req.suid, TSDB_SUPER_TABLE, tables);
-  string = cJSON_PrintUnformatted(tables);
+  string = buildCreateTableJson(&req.schemaRow, &req.schemaTag, req.name, req.suid, TSDB_SUPER_TABLE);
 
 _err:
-  cJSON_Delete(tables);
   tDecoderClear(&coder);
   return string;
 }
@@ -226,23 +227,16 @@ _err:
   return string;
 }
 
-static void buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* tagName, int64_t id, uint8_t tagNum, cJSON* tables) {
-  SArray* pTagVals = NULL;
-  cJSON*  json = cJSON_CreateObject();
-  if (json == NULL) {
-    return;
-  }
-  cJSON* type = cJSON_CreateString("create");
-  cJSON_AddItemToObject(json, "type", type);
-  //  char cid[32] = {0};
-  //  sprintf(cid, "%"PRIi64, id);
-  //  cJSON* cid_ = cJSON_CreateString(cid);
-  //  cJSON_AddItemToObject(json, "id", cid_);
+static void buildChildElement(cJSON*  json, SVCreateTbReq* pCreateReq){
+  STag* pTag = (STag*)pCreateReq->ctb.pTag;
+  char* sname = pCreateReq->ctb.name;
+  char* name = pCreateReq->name;
+  SArray* tagName = pCreateReq->ctb.tagName;
+  int64_t id = pCreateReq->uid;
+  uint8_t tagNum = pCreateReq->ctb.tagNum;
 
   cJSON* tableName = cJSON_CreateString(name);
   cJSON_AddItemToObject(json, "tableName", tableName);
-  cJSON* tableType = cJSON_CreateString("child");
-  cJSON_AddItemToObject(json, "tableType", tableType);
   cJSON* using = cJSON_CreateString(sname);
   cJSON_AddItemToObject(json, "using", using);
   cJSON* tagNumJson = cJSON_CreateNumber(tagNum);
@@ -251,6 +245,7 @@ static void buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* t
   //  cJSON_AddItemToObject(json, "version", version);
 
   cJSON*  tags = cJSON_CreateArray();
+  SArray* pTagVals = NULL;
   int32_t code = tTagToValArray(pTag, &pTagVals);
   if (code) {
     goto end;
@@ -309,10 +304,38 @@ static void buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* t
     cJSON_AddItemToArray(tags, tag);
   }
 
-end:
+  end:
   cJSON_AddItemToObject(json, "tags", tags);
   taosArrayDestroy(pTagVals);
-  cJSON_AddItemToArray(tables, json);
+}
+
+static char* buildCreateCTableJson(SVCreateTbReq* pCreateReq, int32_t nReqs) {
+  char*   string = NULL;
+  cJSON*  json = cJSON_CreateObject();
+  if (json == NULL) {
+    return NULL;
+  }
+  cJSON* type = cJSON_CreateString("create");
+  cJSON_AddItemToObject(json, "type", type);
+  //  char cid[32] = {0};
+  //  sprintf(cid, "%"PRIi64, id);
+  //  cJSON* cid_ = cJSON_CreateString(cid);
+  //  cJSON_AddItemToObject(json, "id", cid_);
+
+  cJSON* tableType = cJSON_CreateString("child");
+  cJSON_AddItemToObject(json, "tableType", tableType);
+
+  buildChildElement(json, pCreateReq);
+  cJSON* createList = cJSON_CreateArray();
+  for(int i = 0; nReqs > 1 && i < nReqs; i++){
+    cJSON* create = cJSON_CreateObject();
+    buildChildElement(create, pCreateReq + i);
+    cJSON_AddItemToArray(createList, create);
+  }
+  cJSON_AddItemToObject(json, "createList", createList);
+  string = cJSON_PrintUnformatted(json);
+  cJSON_Delete(json);
+  return string;
 }
 
 static char* processCreateTable(SMqMetaRsp* metaRsp) {
@@ -329,54 +352,47 @@ static char* processCreateTable(SMqMetaRsp* metaRsp) {
   }
 
   // loop to create table
-  cJSON*  tables = cJSON_CreateArray();
-  for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
-    pCreateReq = req.pReqs + iReq;
+  if (req.nReqs > 0) {
+    pCreateReq = req.pReqs;
     if (pCreateReq->type == TSDB_CHILD_TABLE) {
-      buildCreateCTableJson((STag*)pCreateReq->ctb.pTag, pCreateReq->ctb.name, pCreateReq->name,
-                                     pCreateReq->ctb.tagName, pCreateReq->uid, pCreateReq->ctb.tagNum, tables);
+      string = buildCreateCTableJson(req.pReqs, req.nReqs);
     } else if (pCreateReq->type == TSDB_NORMAL_TABLE) {
-      buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE, tables);
+      string = buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE);
     }
   }
 
-  string = cJSON_PrintUnformatted(tables);
-  cJSON_Delete(tables);
 _exit:
   tDecoderClear(&decoder);
   return string;
 }
 
 static char* processAutoCreateTable(STaosxRsp* rsp) {
-  SDecoder           decoder = {0};
-  SVCreateTbReq*     pCreateReq;
+  if(rsp->createTableNum == 0) return NULL;
+
+  SDecoder*          decoder = taosMemoryCalloc(rsp->createTableNum, sizeof(SDecoder));
+  SVCreateTbReq*     pCreateReq = taosMemoryCalloc(rsp->createTableNum, sizeof(SVCreateTbReq));
   char*              string = NULL;
 
-
   // loop to create table
-  cJSON*  tables = cJSON_CreateArray();
   for (int32_t iReq = 0; iReq < rsp->createTableNum; iReq++) {
     // decode
     void**   data = taosArrayGet(rsp->createTableReq, iReq);
     int32_t *len = taosArrayGet(rsp->createTableLen, iReq);
-    tDecoderInit(&decoder, *data, *len);
-    if (tDecodeSVCreateTbReq(&decoder, pCreateReq) < 0) {
-      tDecoderClear(&decoder);
+    tDecoderInit(&decoder[iReq], *data, *len);
+    if (tDecodeSVCreateTbReq(&decoder[iReq], pCreateReq + iReq) < 0) {
       goto _exit;
     }
 
-    if (pCreateReq->type == TSDB_CHILD_TABLE) {
-      buildCreateCTableJson((STag*)pCreateReq->ctb.pTag, pCreateReq->ctb.name, pCreateReq->name,
-                            pCreateReq->ctb.tagName, pCreateReq->uid, pCreateReq->ctb.tagNum, tables);
-    } else if (pCreateReq->type == TSDB_NORMAL_TABLE) {
-      buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE, tables);
-    }
-    tDecoderClear(&decoder);
+    ASSERT(pCreateReq[iReq].type == TSDB_CHILD_TABLE);
   }
+  string = buildCreateCTableJson(pCreateReq, rsp->createTableNum);
 
-  string = cJSON_PrintUnformatted(tables);
 _exit:
-  cJSON_Delete(tables);
+  for(int i = 0; i < rsp->createTableNum; i++){
+    tDecoderClear(&decoder[i]);
+  }
+  taosMemoryFree(decoder);
+  taosMemoryFree(pCreateReq);
   return string;
 }
 
diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index a84c36790a..047191ec27 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -1470,11 +1470,11 @@ SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper) {
   tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN);
   pRspObj->vgId = pWrapper->vgHandle->vgId;
   pRspObj->resIter = -1;
-  memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqTaosxRspObj));
+  memcpy(&pRspObj->rsp, &pWrapper->taosxRsp, sizeof(STaosxRsp));
 
   pRspObj->resInfo.totalRows = 0;
   pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI;
-  if (!pWrapper->dataRsp.withSchema) {
+  if (!pWrapper->taosxRsp.withSchema) {
     setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols);
   }
 
@@ -1786,6 +1786,9 @@ const char* tmq_get_topic_name(TAOS_RES* res) {
   } else if (TD_RES_TMQ_META(res)) {
     SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
     return strchr(pMetaRspObj->topic, '.') + 1;
+  } else if (TD_RES_TMQ_METADATA(res)) {
+    SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)res;
+    return strchr(pRspObj->topic, '.') + 1;
   } else {
     return NULL;
   }
@@ -1798,6 +1801,9 @@ const char* tmq_get_db_name(TAOS_RES* res) {
   } else if (TD_RES_TMQ_META(res)) {
     SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
     return strchr(pMetaRspObj->db, '.') + 1;
+  } else if (TD_RES_TMQ_METADATA(res)) {
+    SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)res;
+    return strchr(pRspObj->db, '.') + 1;
   } else {
     return NULL;
   }
@@ -1810,6 +1816,9 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) {
   } else if (TD_RES_TMQ_META(res)) {
     SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
     return pMetaRspObj->vgId;
+  } else if (TD_RES_TMQ_META(res)) {
+    SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)res;
+    return pRspObj->vgId;
   } else {
     return -1;
   }
diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c
index f917b9159e..4458a70748 100644
--- a/utils/test/c/tmq_taosx_ci.c
+++ b/utils/test/c/tmq_taosx_ci.c
@@ -54,12 +54,12 @@ static void msg_process(TAOS_RES* msg) {
   printf("db: %s\n", tmq_get_db_name(msg));
   printf("vg: %d\n", tmq_get_vgroup_id(msg));
   TAOS *pConn = use_db();
-  if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) {
+  if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META || tmq_get_res_type(msg) == TMQ_RES_METADATA) {
     char* result = tmq_get_json_meta(msg);
     if (result) {
       printf("meta result: %s\n", result);
     }
-    if(g_fp){
+    if(g_fp && result){
       taosFprintfFile(g_fp, result);
       taosFprintfFile(g_fp, "\n");
     }

From 0afef12ce012b5866f9962ac55a88cc5bc8fded9 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 14 Sep 2022 18:52:56 +0800
Subject: [PATCH 05/57] fix:add createTableList element in tmq_get_json

---
 source/client/src/clientRawBlockWrite.c | 16 ++---
 tests/system-test/7-tmq/tmq_taosx.py    |  9 ++-
 utils/test/c/tmq_taosx_ci.c             | 90 +++++++++++++++++--------
 3 files changed, 75 insertions(+), 40 deletions(-)

diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 82f7285357..a87343be6e 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -39,10 +39,10 @@ static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sch
   //  sprintf(uid, "%"PRIi64, id);
   //  cJSON* id_ = cJSON_CreateString(uid);
   //  cJSON_AddItemToObject(json, "id", id_);
-  cJSON* tableName = cJSON_CreateString(name);
-  cJSON_AddItemToObject(json, "tableName", tableName);
   cJSON* tableType = cJSON_CreateString(t == TSDB_NORMAL_TABLE ? "normal" : "super");
   cJSON_AddItemToObject(json, "tableType", tableType);
+  cJSON* tableName = cJSON_CreateString(name);
+  cJSON_AddItemToObject(json, "tableName", tableName);
   //  cJSON* version = cJSON_CreateNumber(1);
   //  cJSON_AddItemToObject(json, "version", version);
 
@@ -112,10 +112,10 @@ static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) {
   //  cJSON_AddItemToObject(json, "uid", uid);
   SName name = {0};
   tNameFromString(&name, req.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
-  cJSON* tableName = cJSON_CreateString(name.tname);
-  cJSON_AddItemToObject(json, "tableName", tableName);
   cJSON* tableType = cJSON_CreateString("super");
   cJSON_AddItemToObject(json, "tableType", tableType);
+  cJSON* tableName = cJSON_CreateString(name.tname);
+  cJSON_AddItemToObject(json, "tableName", tableName);
 
   cJSON* alterType = cJSON_CreateNumber(req.alterType);
   cJSON_AddItemToObject(json, "alterType", alterType);
@@ -417,10 +417,10 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) {
   cJSON_AddItemToObject(json, "type", type);
   //  cJSON* uid = cJSON_CreateNumber(id);
   //  cJSON_AddItemToObject(json, "uid", uid);
-  cJSON* tableName = cJSON_CreateString(vAlterTbReq.tbName);
-  cJSON_AddItemToObject(json, "tableName", tableName);
   cJSON* tableType = cJSON_CreateString(vAlterTbReq.action == TSDB_ALTER_TABLE_UPDATE_TAG_VAL ? "child" : "normal");
   cJSON_AddItemToObject(json, "tableType", tableType);
+  cJSON* tableName = cJSON_CreateString(vAlterTbReq.tbName);
+  cJSON_AddItemToObject(json, "tableName", tableName);
   cJSON* alterType = cJSON_CreateNumber(vAlterTbReq.action);
   cJSON_AddItemToObject(json, "alterType", alterType);
 
@@ -528,10 +528,10 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) {
   }
   cJSON* type = cJSON_CreateString("drop");
   cJSON_AddItemToObject(json, "type", type);
-  cJSON* tableName = cJSON_CreateString(req.name);
-  cJSON_AddItemToObject(json, "tableName", tableName);
   cJSON* tableType = cJSON_CreateString("super");
   cJSON_AddItemToObject(json, "tableType", tableType);
+  cJSON* tableName = cJSON_CreateString(req.name);
+  cJSON_AddItemToObject(json, "tableName", tableName);
 
   string = cJSON_PrintUnformatted(json);
 
diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py
index 07602ec29f..9bd23b422d 100644
--- a/tests/system-test/7-tmq/tmq_taosx.py
+++ b/tests/system-test/7-tmq/tmq_taosx.py
@@ -31,10 +31,9 @@ class TDTestCase:
         while True:
             dst = queryFile.readline()
             src = consumeFile.readline()
-
-            if dst:
+            if src:
                 if dst != src:
-                    tdLog.exit("compare error: %s != %s"%src, dst)
+                    tdLog.exit("compare error: %s != %s"%(src, dst))
             else:
                 break
         return
@@ -42,7 +41,7 @@ class TDTestCase:
     def checkDropData(self):
         tdSql.execute('use db_taosx')
         tdSql.query("show tables")
-        tdSql.checkRows(2)
+        tdSql.checkRows(6)
         tdSql.query("select * from jt order by i")
         tdSql.checkRows(2)
         tdSql.checkData(0, 1, 1)
@@ -52,7 +51,7 @@ class TDTestCase:
 
         tdSql.execute('use abc1')
         tdSql.query("show tables")
-        tdSql.checkRows(2)
+        tdSql.checkRows(6)
         tdSql.query("select * from jt order by i")
         tdSql.checkRows(2)
         tdSql.checkData(0, 1, 1)
diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c
index 4458a70748..337a3cbd29 100644
--- a/utils/test/c/tmq_taosx_ci.c
+++ b/utils/test/c/tmq_taosx_ci.c
@@ -309,6 +309,33 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes){
     }
     taos_free_result(pRes);
   }
+
+  pRes = taos_query(pConn,
+                    "create stable if not exists stt (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 "
+                    "nchar(8), t4 bool)");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create super table stt, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
+  pRes = taos_query(pConn,
+                    "create stable if not exists sttb (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 "
+                    "nchar(8), t4 bool)");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create super table sttb, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
+  pRes = taos_query(pConn, "create table if not exists stt1 using stt tags(2, \"stt1\", true) sttb1 using sttb tags(4, \"sttb1\", true) "
+                    "stt2 using stt tags(43, \"stt2\", false) sttb2 using sttb tags(54, \"sttb2\", true)");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
   return 0;
 }
 
@@ -543,15 +570,21 @@ void initLogFile() {
 
   if(g_conf.snapShot){
     char *result[] = {
-        "{\"type\":\"create\",\"tableName\":\"st1\",\"tableType\":\"super\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":64},{\"name\":\"c4\",\"type\":5}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1},{\"name\":\"t2\",\"type\":8,\"length\":64}]}",
-        "{\"type\":\"create\",\"tableName\":\"ct0\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}",
-        "{\"type\":\"create\",\"tableName\":\"ct1\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}]}",
-        "{\"type\":\"create\",\"tableName\":\"ct2\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[]}",
-        "{\"type\":\"create\",\"tableName\":\"ct3\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":5000}]}",
-        "{\"type\":\"create\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c2\",\"type\":10,\"length\":8},{\"name\":\"cc3\",\"type\":5}],\"tags\":[]}",
-        "{\"type\":\"create\",\"tableName\":\"jt\",\"tableType\":\"super\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}",
-        "{\"type\":\"create\",\"tableName\":\"jt1\",\"tableType\":\"child\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}]}",
-        "{\"type\":\"create\",\"tableName\":\"jt2\",\"tableType\":\"child\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[]}",
+        "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":64},{\"name\":\"c4\",\"type\":5}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1},{\"name\":\"t2\",\"type\":8,\"length\":64}]}",
+        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}",
+        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}],\"createList\":[]}",
+        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[],\"createList\":[]}",
+        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":5000}],\"createList\":[]}",
+        "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c2\",\"type\":10,\"length\":8},{\"name\":\"cc3\",\"type\":5}],\"tags\":[]}",
+        "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"jt\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}",
+        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt1\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}],\"createList\":[]}",
+        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt2\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[],\"createList\":[]}",
+        "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"stt\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}",
+        "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"sttb\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}",
+        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt1\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}",
+        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"sttb1\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}",
+        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt2\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":43},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}],\"createList\":[]}",
+        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"sttb2\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":54},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}",
     };
 
     for(int i = 0; i < sizeof(result)/sizeof(result[0]); i++){
@@ -560,24 +593,27 @@ void initLogFile() {
     }
   }else{
     char *result[] = {
-        "{\"type\":\"create\",\"tableName\":\"st1\",\"tableType\":\"super\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}",
-        "{\"type\":\"create\",\"tableName\":\"ct0\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}",
-        "{\"type\":\"create\",\"tableName\":\"ct1\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}]}",
-        "{\"type\":\"create\",\"tableName\":\"ct2\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[]}",
-        "{\"type\":\"create\",\"tableName\":\"ct3\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":3000}]}",
-        "{\"type\":\"alter\",\"tableName\":\"st1\",\"tableType\":\"super\",\"alterType\":5,\"colName\":\"c4\",\"colType\":5}",
-        "{\"type\":\"alter\",\"tableName\":\"st1\",\"tableType\":\"super\",\"alterType\":7,\"colName\":\"c3\",\"colType\":8,\"colLength\":64}",
-        "{\"type\":\"alter\",\"tableName\":\"st1\",\"tableType\":\"super\",\"alterType\":1,\"colName\":\"t2\",\"colType\":8,\"colLength\":64}",
-        "{\"type\":\"alter\",\"tableName\":\"ct3\",\"tableType\":\"child\",\"alterType\":4,\"colName\":\"t1\",\"colValue\":\"5000\",\"colValueNull\":false}",
-        "{\"type\":\"create\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":10,\"length\":4}],\"tags\":[]}",
-        "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":5,\"colName\":\"c3\",\"colType\":5}",
-        "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":7,\"colName\":\"c2\",\"colType\":10,\"colLength\":8}",
-        "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":10,\"colName\":\"c3\",\"colNewName\":\"cc3\"}",
-        "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":9}",
-        "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":6,\"colName\":\"c1\"}",
-        "{\"type\":\"create\",\"tableName\":\"jt\",\"tableType\":\"super\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}",
-        "{\"type\":\"create\",\"tableName\":\"jt1\",\"tableType\":\"child\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}]}",
-        "{\"type\":\"create\",\"tableName\":\"jt2\",\"tableType\":\"child\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[]}"
+        "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}",
+        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}",
+        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}],\"createList\":[]}",
+        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[],\"createList\":[]}",
+        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":3000}],\"createList\":[]}",
+        "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\",\"colType\":5}",
+        "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":7,\"colName\":\"c3\",\"colType\":8,\"colLength\":64}",
+        "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":1,\"colName\":\"t2\",\"colType\":8,\"colLength\":64}",
+        "{\"type\":\"alter\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"alterType\":4,\"colName\":\"t1\",\"colValue\":\"5000\",\"colValueNull\":false}",
+        "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":10,\"length\":4}],\"tags\":[]}",
+        "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":5,\"colName\":\"c3\",\"colType\":5}",
+        "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":7,\"colName\":\"c2\",\"colType\":10,\"colLength\":8}",
+        "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":10,\"colName\":\"c3\",\"colNewName\":\"cc3\"}",
+        "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":9}",
+        "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":6,\"colName\":\"c1\"}",
+        "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"jt\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}",
+        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt1\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}],\"createList\":[]}",
+        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt2\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[],\"createList\":[]}",
+        "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"stt\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}",
+        "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"sttb\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}",
+        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt1\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"stt1\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"sttb1\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"stt2\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":43},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}]},{\"tableName\":\"sttb2\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":54},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}"
     };
 
     for(int i = 0; i < sizeof(result)/sizeof(result[0]); i++){

From 817eed740ffe27082bb662eec531f8f94d251196 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Thu, 15 Sep 2022 10:01:24 +0800
Subject: [PATCH 06/57] fix: prototype to verify stream + udf as task framework

---
 source/libs/function/src/udfd.c  | 19 ++++++++++++++++---
 source/libs/function/test/udf1.c |  4 ++++
 tests/script/tsim/query/udf.sim  | 26 +++++++++++++-------------
 3 files changed, 33 insertions(+), 16 deletions(-)

diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c
index a45e4585e8..6777dc0299 100644
--- a/source/libs/function/src/udfd.c
+++ b/source/libs/function/src/udfd.c
@@ -29,6 +29,8 @@
 #include "trpc.h"
 // clang-foramt on
 
+SArray* udfdResidentFuncs;
+
 typedef struct SUdfdContext {
   uv_loop_t * loop;
   uv_pipe_t   ctrlPipe;
@@ -576,9 +578,9 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) {
     uv_dlsym(&udf->lib, finishFuncName, (void **)(&udf->aggFinishFunc));
     char mergeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0};
     char *mergeSuffix = "_merge";
-    strncpy(finishFuncName, processFuncName, sizeof(finishFuncName));
-    strncat(finishFuncName, mergeSuffix, strlen(mergeSuffix));
-    uv_dlsym(&udf->lib, finishFuncName, (void **)(&udf->aggMergeFunc));
+    strncpy(mergeFuncName, processFuncName, sizeof(mergeFuncName));
+    strncat(mergeFuncName, mergeSuffix, strlen(mergeSuffix));
+    uv_dlsym(&udf->lib, mergeFuncName, (void **)(&udf->aggMergeFunc));
   }
   return 0;
 }
@@ -941,6 +943,14 @@ void udfdConnectMnodeThreadFunc(void *args) {
   }
 }
 
+int32_t udfdInitResidentFuncs() {
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t udfdDeinitResidentFuncs() {
+  return TSDB_CODE_SUCCESS;
+}
+
 int main(int argc, char *argv[]) {
   if (!taosCheckSystemIsLittleEnd()) {
     printf("failed to start since on non-little-end machines\n");
@@ -978,6 +988,8 @@ int main(int argc, char *argv[]) {
     return -5;
   }
 
+  udfdInitResidentFuncs();
+
   uv_thread_t mnodeConnectThread;
   uv_thread_create(&mnodeConnectThread, udfdConnectMnodeThreadFunc, NULL);
 
@@ -986,5 +998,6 @@ int main(int argc, char *argv[]) {
   removeListeningPipe();
   udfdCloseClientRpc();
 
+  udfdDeinitResidentFuncs();
   return 0;
 }
diff --git a/source/libs/function/test/udf1.c b/source/libs/function/test/udf1.c
index 5be18af553..620a0653aa 100644
--- a/source/libs/function/test/udf1.c
+++ b/source/libs/function/test/udf1.c
@@ -40,6 +40,10 @@ DLL_EXPORT int32_t udf1(SUdfDataBlock* block, SUdfColumn *resultCol) {
       udfColDataSet(resultCol, i, (char *)&luckyNum, false);
     }
   }
+  TAOS* taos = taos_connect("127.0.0.1", "root", "taosdata", "gpd", 6030);
+  taos_query(taos, "create st (ts timestamp, f int) tags(t int)");
+  taos_query(taos, "insert into t using st tags(1) values(now, 1) ");
+  taos_query(taos, "select * from gpd.t");
   //to simulate actual processing delay by udf
 #ifdef LINUX
   usleep(1 * 1000);   // usleep takes sleep time in us (1 millionth of a second)
diff --git a/tests/script/tsim/query/udf.sim b/tests/script/tsim/query/udf.sim
index 7f8b1044ef..0b48a815e2 100644
--- a/tests/script/tsim/query/udf.sim
+++ b/tests/script/tsim/query/udf.sim
@@ -144,18 +144,18 @@ if $data20 != 8.000000000 then
   return -1
 endi
 
-sql drop function bit_and;
-sql show functions;
-if $rows != 1 then
-  return -1
-endi
-if $data00 != @l2norm@ then
-  return -1
-  endi
-sql drop function l2norm;
-sql show functions;
-if $rows != 0 then
-  return -1
-endi
+#sql drop function bit_and;
+#sql show functions;
+#if $rows != 1 then
+#  return -1
+#endi
+#if $data00 != @l2norm@ then
+#  return -1
+#  endi
+#sql drop function l2norm;
+#sql show functions;
+#if $rows != 0 then
+#  return -1
+#endi
 
 system sh/exec.sh -n dnode1 -s stop -x SIGINT

From 8859a3efa35828c6f8ea21ca01afd0ebfc64fed0 Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Thu, 15 Sep 2022 14:00:10 +0800
Subject: [PATCH 07/57] feat: update taos-tools e7270c9 for 3.0

---
 cmake/taostools_CMakeLists.txt.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in
index c13aa5f4b1..6fb2a69847 100644
--- a/cmake/taostools_CMakeLists.txt.in
+++ b/cmake/taostools_CMakeLists.txt.in
@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG 9f4c01e
+        GIT_TAG e7270c9
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

From d156bff782a05e3802728b8b70c8dd9bad105c38 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Thu, 15 Sep 2022 17:41:17 +0800
Subject: [PATCH 08/57] enhance: support resident functions

---
 source/libs/function/src/udfd.c  | 31 ++++++++++++-
 source/libs/function/test/udf1.c |  4 --
 tests/script/sh/gpd.c            | 77 ++++++++++++++++++++++++++++++++
 3 files changed, 106 insertions(+), 6 deletions(-)
 create mode 100644 tests/script/sh/gpd.c

diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c
index 6777dc0299..23a95320fb 100644
--- a/source/libs/function/src/udfd.c
+++ b/source/libs/function/src/udfd.c
@@ -29,7 +29,7 @@
 #include "trpc.h"
 // clang-foramt on
 
-SArray* udfdResidentFuncs;
+SArray* udfdResidentFuncs = NULL;
 
 typedef struct SUdfdContext {
   uv_loop_t * loop;
@@ -69,6 +69,7 @@ typedef struct SUdf {
   EUdfState  state;
   uv_mutex_t lock;
   uv_cond_t  condReady;
+  bool resident;
 
   char    name[TSDB_FUNC_NAME_LEN];
   int8_t  funcType;
@@ -202,6 +203,14 @@ void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
     if (udf->initFunc) {
       udf->initFunc();
     }
+    udf->resident = false;
+    for (int32_t i = 0; i < taosArrayGetSize(udfdResidentFuncs); ++i) {
+      char* funcName = taosArrayGet(udfdResidentFuncs, i);
+      if (strcmp(setup->udfName, funcName) == 0) {
+        udf->resident = true;
+        break;
+      }
+    }
     udf->state = UDF_STATE_READY;
     uv_cond_broadcast(&udf->condReady);
     uv_mutex_unlock(&udf->lock);
@@ -347,7 +356,7 @@ void udfdProcessTeardownRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
 
   uv_mutex_lock(&global.udfsMutex);
   udf->refCount--;
-  if (udf->refCount == 0) {
+  if (udf->refCount == 0 && !udf->resident) {
     unloadUdf = true;
     taosHashRemove(global.udfsHash, udf->name, strlen(udf->name));
   }
@@ -944,10 +953,28 @@ void udfdConnectMnodeThreadFunc(void *args) {
 }
 
 int32_t udfdInitResidentFuncs() {
+  udfdResidentFuncs = taosArrayInit(2, TSDB_FUNC_NAME_LEN);
+  char gpd[TSDB_FUNC_NAME_LEN] = "gpd";
+  taosArrayPush(udfdResidentFuncs, gpd);
+  char gpdBatch[TSDB_FUNC_NAME_LEN] = "gpdbatch";
+  taosArrayPush(udfdResidentFuncs, gpdBatch);
   return TSDB_CODE_SUCCESS;
 }
 
 int32_t udfdDeinitResidentFuncs() {
+  for (int32_t i = 0; i < taosArrayGetSize(udfdResidentFuncs); ++i) {
+    char* funcName = taosArrayGet(udfdResidentFuncs, i);
+    SUdf** udfInHash =  taosHashGet(global.udfsHash, funcName, strlen(funcName));
+    if (udfInHash) {
+      taosHashRemove(global.udfsHash, funcName, strlen(funcName));
+      SUdf* udf = *udfInHash;
+      if (udf->destroyFunc) {
+        (udf->destroyFunc)();
+      }
+      uv_dlclose(&udf->lib);
+      taosMemoryFree(udf);
+    }
+  }
   return TSDB_CODE_SUCCESS;
 }
 
diff --git a/source/libs/function/test/udf1.c b/source/libs/function/test/udf1.c
index 620a0653aa..5be18af553 100644
--- a/source/libs/function/test/udf1.c
+++ b/source/libs/function/test/udf1.c
@@ -40,10 +40,6 @@ DLL_EXPORT int32_t udf1(SUdfDataBlock* block, SUdfColumn *resultCol) {
       udfColDataSet(resultCol, i, (char *)&luckyNum, false);
     }
   }
-  TAOS* taos = taos_connect("127.0.0.1", "root", "taosdata", "gpd", 6030);
-  taos_query(taos, "create st (ts timestamp, f int) tags(t int)");
-  taos_query(taos, "insert into t using st tags(1) values(now, 1) ");
-  taos_query(taos, "select * from gpd.t");
   //to simulate actual processing delay by udf
 #ifdef LINUX
   usleep(1 * 1000);   // usleep takes sleep time in us (1 millionth of a second)
diff --git a/tests/script/sh/gpd.c b/tests/script/sh/gpd.c
new file mode 100644
index 0000000000..56c3388f3e
--- /dev/null
+++ b/tests/script/sh/gpd.c
@@ -0,0 +1,77 @@
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+#ifdef LINUX
+#include <unistd.h>
+#endif
+#ifdef WINDOWS
+#include <windows.h>
+#endif
+#include "taosudf.h"
+
+
+DLL_EXPORT int32_t gpd_init() {
+  return 0;
+}
+
+DLL_EXPORT int32_t gpd_destroy() {
+  return 0;
+}
+
+DLL_EXPORT int32_t gpd(SUdfDataBlock* block, SUdfColumn *resultCol) {
+  SUdfColumnMeta *meta = &resultCol->colMeta;
+  meta->bytes = 4;
+  meta->type = TSDB_DATA_TYPE_INT;
+  meta->scale = 0;
+  meta->precision = 0;
+
+  SUdfColumnData *resultData = &resultCol->colData;
+  resultData->numOfRows = block->numOfRows;
+  for (int32_t i = 0; i < resultData->numOfRows; ++i) {
+    int j = 0;
+    for (; j < block->numOfCols; ++j) {
+      if (udfColDataIsNull(block->udfCols[j], i)) {
+        udfColDataSetNull(resultCol, i);
+        break;
+      }
+    }
+    if ( j == block->numOfCols) {
+      int32_t luckyNum = 88;
+      udfColDataSet(resultCol, i, (char *)&luckyNum, false);
+    }
+  }
+  taos_init();
+  TAOS* taos = taos_connect("localhost", "root", "taosdata", "", 7100);
+  if (taos == NULL) {
+    char* errstr = "can not connect";
+  }
+  TAOS_RES* res = taos_query(taos, "create database if not exists gpd");
+  if (taos_errno(res) != 0) {
+    char* errstr = taos_errstr(res);
+  } 
+  res = taos_query(taos, "create table gpd.st (ts timestamp, f int) tags(t int)");
+  if (taos_errno(res) != 0) {
+    char* errstr = taos_errstr(res);
+  } 
+
+  taos_query(taos, "insert into gpd.t using gpd.st tags(1) values(now, 1) ");
+  if (taos_errno(res) != 0) {
+    char* errstr = taos_errstr(res);
+  } 
+
+  taos_query(taos, "select * from gpd.t");
+  if (taos_errno(res) != 0) {
+    char* errstr = taos_errstr(res);
+  } 
+
+  taos_close(taos);
+  taos_cleanup();
+  //to simulate actual processing delay by udf
+#ifdef LINUX
+  usleep(1 * 1000);   // usleep takes sleep time in us (1 millionth of a second)
+#endif
+#ifdef WINDOWS
+  Sleep(1);
+#endif
+  return 0;
+}

From 414021a8e0d6962ed8cd4c1534e653e81751e084 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Thu, 15 Sep 2022 19:23:12 +0800
Subject: [PATCH 09/57] fix:error in auto create table for taosX

---
 include/common/tmsg.h                     |   2 +-
 source/client/src/clientRawBlockWrite.c   |  82 ++++++----
 source/dnode/vnode/src/inc/vnodeInt.h     |   2 +-
 source/dnode/vnode/src/sma/smaTimeRange.c |   2 +-
 source/dnode/vnode/src/tq/tqExec.c        |  18 +-
 source/dnode/vnode/src/tq/tqSink.c        |  32 +++-
 source/libs/parser/src/parInsert.c        |   4 +-
 source/libs/parser/src/parTranslater.c    |   7 +-
 source/libs/parser/src/parUtil.c          |   6 +
 source/libs/scalar/src/sclfunc.c          |   3 +-
 tests/system-test/7-tmq/tmq_taosx.py      | 101 +++++++++++-
 utils/test/c/tmq_taosx_ci.c               | 191 +++++++++++++---------
 12 files changed, 315 insertions(+), 135 deletions(-)

diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index 9f919fa250..405acf8d99 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -2954,7 +2954,7 @@ static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicE
 }
 
 static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) {
-  // taosMemoryFree(pSubTopicEp->schema.pSchema);
+  taosMemoryFreeClear(pSubTopicEp->schema.pSchema);
   taosArrayDestroy(pSubTopicEp->vgs);
 }
 
diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index a87343be6e..00c46dcb8a 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -362,12 +362,19 @@ static char* processCreateTable(SMqMetaRsp* metaRsp) {
   }
 
 _exit:
+  for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
+    pCreateReq = req.pReqs + iReq;
+    taosMemoryFreeClear(pCreateReq->comment);
+    if (pCreateReq->type == TSDB_CHILD_TABLE) {
+      taosArrayDestroy(pCreateReq->ctb.tagName);
+    }
+  }
   tDecoderClear(&decoder);
   return string;
 }
 
 static char* processAutoCreateTable(STaosxRsp* rsp) {
-  if(rsp->createTableNum == 0) return NULL;
+  if(rsp->createTableNum == 0) return strdup("");
 
   SDecoder*          decoder = taosMemoryCalloc(rsp->createTableNum, sizeof(SDecoder));
   SVCreateTbReq*     pCreateReq = taosMemoryCalloc(rsp->createTableNum, sizeof(SVCreateTbReq));
@@ -873,6 +880,14 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
   code = pRequest->code;
 
 end:
+  for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
+    pCreateReq = req.pReqs + iReq;
+    taosMemoryFreeClear(pCreateReq->comment);
+    if (pCreateReq->type == TSDB_CHILD_TABLE) {
+      taosArrayDestroy(pCreateReq->ctb.tagName);
+    }
+  }
+
   taosHashCleanup(pVgroupHashmap);
   destroyRequest(pRequest);
   tDecoderClear(&coder);
@@ -1621,6 +1636,9 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
   code = pRequest->code;
 
 end:
+  tDeleteSMqDataRsp(&rspObj.rsp);
+  rspObj.resInfo.pRspMsg = NULL;
+  doFreeReqResultInfo(&rspObj.resInfo);
   tDecoderClear(&decoder);
   qDestroyQuery(pQuery);
   destroyRequest(pRequest);
@@ -1629,7 +1647,7 @@ end:
   return code;
 }
 
-static int32_t tmqWriteRaqMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) {
+static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) {
   int32_t     code = TSDB_CODE_SUCCESS;
   SHashObj*   pVgHash = NULL;
   SQuery*     pQuery = NULL;
@@ -1711,6 +1729,33 @@ static int32_t tmqWriteRaqMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
       goto end;
     }
 
+    // find schema data info
+    int32_t schemaLen = 0;
+    void* schemaData = NULL;
+    for(int j = 0; j < rspObj.rsp.createTableNum; j++){
+      void** dataTmp = taosArrayGet(rspObj.rsp.createTableReq, j);
+      int32_t* lenTmp = taosArrayGet(rspObj.rsp.createTableLen, j);
+
+      SDecoder           decoderTmp = {0};
+      SVCreateTbReq*     pCreateReq;
+
+      tDecoderInit(&decoderTmp, *dataTmp, *lenTmp);
+      if (tDecodeSVCreateTbReq(&decoderTmp, pCreateReq) < 0) {
+        tDecoderClear(&decoderTmp);
+        goto end;
+      }
+
+      ASSERT (pCreateReq->type == TSDB_CHILD_TABLE);
+      if(strcmp(tbName, pCreateReq->name) == 0){
+        schemaLen = *lenTmp;
+        schemaData = *dataTmp;
+        strcpy(pName.tname, pCreateReq->ctb.name);
+        tDecoderClear(&decoderTmp);
+        break;
+      }
+      tDecoderClear(&decoderTmp);
+    }
+
     code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta);
     if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
       uError("WriteRaw:catalogGetTableMeta table not exist. table name: %s", tbName);
@@ -1738,32 +1783,6 @@ static int32_t tmqWriteRaqMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
     int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) +
                               (int32_t)TD_BITMAP_BYTES(pTableMeta->tableInfo.numOfColumns - 1);
 
-    // find schema data info
-    int32_t schemaLen = 0;
-    void* schemaData = NULL;
-    for(int j = 0; j < rspObj.rsp.createTableNum; j++){
-      void** dataTmp = taosArrayGet(rspObj.rsp.createTableReq, j);
-      int32_t* lenTmp = taosArrayGet(rspObj.rsp.createTableLen, j);
-
-      SDecoder           decoderTmp = {0};
-      SVCreateTbReq*     pCreateReq;
-
-      tDecoderInit(&decoderTmp, *dataTmp, *lenTmp);
-      if (tDecodeSVCreateTbReq(&decoderTmp, pCreateReq) < 0) {
-        tDecoderClear(&decoderTmp);
-        goto end;
-      }
-
-      ASSERT (pCreateReq->type == TSDB_CHILD_TABLE);
-      if(strcmp(tbName, pCreateReq->name) == 0){
-        schemaLen = *lenTmp;
-        schemaData = *dataTmp;
-        tDecoderClear(&decoderTmp);
-        break;
-      }
-      tDecoderClear(&decoderTmp);
-    }
-
     int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize;
 
     SSubmitReq* subReq = NULL;
@@ -1912,6 +1931,9 @@ static int32_t tmqWriteRaqMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
   code = pRequest->code;
 
   end:
+  tDeleteSTaosxRsp(&rspObj.rsp);
+  rspObj.resInfo.pRspMsg = NULL;
+  doFreeReqResultInfo(&rspObj.resInfo);
   tDecoderClear(&decoder);
   qDestroyQuery(pQuery);
   destroyRequest(pRequest);
@@ -2003,7 +2025,7 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) {
 }
 
 void tmq_free_raw(tmq_raw_data raw) {
-  if (raw.raw_type == RES_TYPE__TMQ) {
+  if (raw.raw_type == RES_TYPE__TMQ || raw.raw_type == RES_TYPE__TMQ_METADATA) {
     taosMemoryFree(raw.raw);
   }
 }
@@ -2030,7 +2052,7 @@ int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) {
   } else if (raw.raw_type == RES_TYPE__TMQ) {
     return tmqWriteRawDataImpl(taos, raw.raw, raw.raw_len);
   } else if (raw.raw_type == RES_TYPE__TMQ_METADATA) {
-    return tmqWriteRaqMetaDataImpl(taos, raw.raw, raw.raw_len);
+    return tmqWriteRawMetaDataImpl(taos, raw.raw, raw.raw_len);
   }
   return TSDB_CODE_INVALID_PARA;
 }
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 0e85e7bfb6..4c8045d651 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -184,7 +184,7 @@ int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg);
 
-SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pSchema, bool createTb, int64_t suid,
+SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pSchema, SSchemaWrapper* pTagSchemaWrapper, bool createTb, int64_t suid,
                             const char* stbFullName, SBatchDeleteReq* pDeleteReq);
 
 // sma
diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c
index e2cb51f586..6c32fbbc84 100644
--- a/source/dnode/vnode/src/sma/smaTimeRange.c
+++ b/source/dnode/vnode/src/sma/smaTimeRange.c
@@ -204,7 +204,7 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char
   }
 
   SBatchDeleteReq deleteReq;
-  SSubmitReq     *pSubmitReq = tqBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, true,
+  SSubmitReq     *pSubmitReq = tqBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, &pTsmaStat->pTSma->schemaTag, true,
                                                pTsmaStat->pTSma->dstTbUid, pTsmaStat->pTSma->dstTbName, &deleteReq);
 
   if (!pSubmitReq) {
diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c
index d00907f677..a24f920235 100644
--- a/source/dnode/vnode/src/tq/tqExec.c
+++ b/source/dnode/vnode/src/tq/tqExec.c
@@ -243,14 +243,15 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp
       }
       if (pHandle->fetchMeta) {
         SSubmitBlk* pBlk = pReader->pBlock;
-        if (pBlk->schemaLen > 0) {
+        int32_t schemaLen = htonl(pBlk->schemaLen);
+        if (schemaLen > 0) {
           if (pRsp->createTableNum == 0) {
             pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t));
             pRsp->createTableReq = taosArrayInit(0, sizeof(void*));
           }
-          void* createReq = taosMemoryCalloc(1, pBlk->schemaLen);
-          memcpy(createReq, pBlk->data, pBlk->schemaLen);
-          taosArrayPush(pRsp->createTableLen, &pBlk->schemaLen);
+          void* createReq = taosMemoryCalloc(1, schemaLen);
+          memcpy(createReq, pBlk->data, schemaLen);
+          taosArrayPush(pRsp->createTableLen, &schemaLen);
           taosArrayPush(pRsp->createTableReq, &createReq);
           pRsp->createTableNum++;
         }
@@ -277,14 +278,15 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp
       }
       if (pHandle->fetchMeta) {
         SSubmitBlk* pBlk = pReader->pBlock;
-        if (pBlk->schemaLen > 0) {
+        int32_t schemaLen = htonl(pBlk->schemaLen);
+        if (schemaLen > 0) {
           if (pRsp->createTableNum == 0) {
             pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t));
             pRsp->createTableReq = taosArrayInit(0, sizeof(void*));
           }
-          void* createReq = taosMemoryCalloc(1, pBlk->schemaLen);
-          memcpy(createReq, pBlk->data, pBlk->schemaLen);
-          taosArrayPush(pRsp->createTableLen, &pBlk->schemaLen);
+          void* createReq = taosMemoryCalloc(1, schemaLen);
+          memcpy(createReq, pBlk->data, schemaLen);
+          taosArrayPush(pRsp->createTableLen, &schemaLen);
           taosArrayPush(pRsp->createTableReq, &createReq);
           pRsp->createTableNum++;
         }
diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c
index 522bf46aa1..f1538494f9 100644
--- a/source/dnode/vnode/src/tq/tqSink.c
+++ b/source/dnode/vnode/src/tq/tqSink.c
@@ -48,7 +48,7 @@ int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBl
   return 0;
 }
 
-SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pTSchema, bool createTb,
+SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pTSchema, SSchemaWrapper* pTagSchemaWrapper, bool createTb,
                             int64_t suid, const char* stbFullName, SBatchDeleteReq* pDeleteReq) {
   SSubmitReq* ret = NULL;
   SArray*     schemaReqs = NULL;
@@ -89,6 +89,30 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem
         return NULL;
       }
 
+      SArray *tagName = taosArrayInit(1, TSDB_COL_NAME_LEN);
+      taosArrayPush(tagName, "group_id");
+
+//      STag* pTag = NULL;
+//      taosArrayClear(tagArray);
+//      SArray *tagName = taosArrayInit(1, TSDB_COL_NAME_LEN);
+//      for(int j = 0; j < pTagSchemaWrapper->nCols; j++){
+//        STagVal tagVal = {
+//            .cid = pTagSchemaWrapper->pSchema[j].colId,
+//            .type = pTagSchemaWrapper->pSchema[j].type,
+//            .i64 = (int64_t)pDataBlock->info.groupId,
+//        };
+//        taosArrayPush(tagArray, &tagVal);
+//        taosArrayPush(tagName, pTagSchemaWrapper->pSchema[j].name);
+//      }
+//
+//      tTagNew(tagArray, 1, false, &pTag);
+//      if (pTag == NULL) {
+//        terrno = TSDB_CODE_OUT_OF_MEMORY;
+//        taosArrayDestroy(tagArray);
+//        taosArrayDestroy(tagName);
+//        return NULL;
+//      }
+
       SVCreateTbReq createTbReq = {0};
       SName         name = {0};
       tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
@@ -99,6 +123,8 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem
       createTbReq.type = TSDB_CHILD_TABLE;
       createTbReq.ctb.suid = suid;
       createTbReq.ctb.pTag = (uint8_t*)pTag;
+      createTbReq.ctb.tagNum = taosArrayGetSize(tagArray);
+      createTbReq.ctb.tagName = tagName;
 
       int32_t code;
       int32_t schemaLen;
@@ -113,6 +139,7 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem
       void* schemaStr = taosMemoryMalloc(schemaLen);
       if (schemaStr == NULL) {
         terrno = TSDB_CODE_OUT_OF_MEMORY;
+        tdDestroySVCreateTbReq(&createTbReq);
         return NULL;
       }
       taosArrayPush(schemaReqs, &schemaStr);
@@ -123,6 +150,7 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem
       code = tEncodeSVCreateTbReq(&encoder, &createTbReq);
       if (code < 0) {
         terrno = TSDB_CODE_OUT_OF_MEMORY;
+        tdDestroySVCreateTbReq(&createTbReq);
         return NULL;
       }
       tEncoderClear(&encoder);
@@ -231,7 +259,7 @@ void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) {
 
   ASSERT(pTask->tbSink.pTSchema);
   deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq));
-  SSubmitReq* submitReq = tqBlockToSubmit(pVnode, pRes, pTask->tbSink.pTSchema, true, pTask->tbSink.stbUid,
+  SSubmitReq* submitReq = tqBlockToSubmit(pVnode, pRes, pTask->tbSink.pTSchema, pTask->tbSink.pSchemaWrapper, true, pTask->tbSink.stbUid,
                                           pTask->tbSink.stbFullName, &deleteReq);
 
   tqDebug("vgId:%d, task %d convert blocks over, put into write-queue", TD_VID(pVnode), pTask->taskId);
diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c
index 162161b67a..8eb8526178 100644
--- a/source/libs/parser/src/parInsert.c
+++ b/source/libs/parser/src/parInsert.c
@@ -1059,7 +1059,7 @@ end:
   for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) {
     STagVal* p = (STagVal*)taosArrayGet(pTagVals, i);
     if (IS_VAR_DATA_TYPE(p->type)) {
-      taosMemoryFree(p->pData);
+      taosMemoryFreeClear(p->pData);
     }
   }
   taosArrayDestroy(pTagVals);
@@ -2039,7 +2039,7 @@ end:
   for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) {
     STagVal* p = (STagVal*)taosArrayGet(pTagArray, i);
     if (p->type == TSDB_DATA_TYPE_NCHAR) {
-      taosMemoryFree(p->pData);
+      taosMemoryFreeClear(p->pData);
     }
   }
   taosArrayDestroy(pTagArray);
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 91787f5a72..d9d011514c 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -6633,12 +6633,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS
         break;
       }
     } while (0);
-    for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) {
-      STagVal* p = (STagVal*)taosArrayGet(pTagVals, i);
-      if (IS_VAR_DATA_TYPE(p->type)) {
-        taosMemoryFree(p->pData);
-      }
-    }
+
     taosArrayDestroy(pTagVals);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c
index 32513fd0b6..84d2f1d46b 100644
--- a/source/libs/parser/src/parUtil.c
+++ b/source/libs/parser/src/parUtil.c
@@ -410,6 +410,12 @@ end:
   if (retCode == TSDB_CODE_SUCCESS) {
     tTagNew(pTagVals, 1, true, ppTag);
   }
+  for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) {
+    STagVal* p = (STagVal*)taosArrayGet(pTagVals, i);
+    if (IS_VAR_DATA_TYPE(p->type)) {
+      taosMemoryFreeClear(p->pData);
+    }
+  }
   cJSON_Delete(root);
   return retCode;
 }
diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c
index 4ead1147e4..4bc2ce771d 100644
--- a/source/libs/scalar/src/sclfunc.c
+++ b/source/libs/scalar/src/sclfunc.c
@@ -1153,7 +1153,8 @@ int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
       memcpy(tmp, varDataVal(input), varDataLen(input));
       tmp[varDataLen(input)] = 0;
       if(parseJsontoTagData(tmp, pTagVals, &pTag, NULL)){
-        tTagNew(pTagVals, 1, true, &pTag);
+        taosArrayDestroy(pTagVals);
+        return TSDB_CODE_FAILED;
       }
     }
 
diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py
index 9bd23b422d..626e935733 100644
--- a/tests/system-test/7-tmq/tmq_taosx.py
+++ b/tests/system-test/7-tmq/tmq_taosx.py
@@ -38,10 +38,13 @@ class TDTestCase:
                 break
         return
 
-    def checkDropData(self):
+    def checkDropData(self, drop):
         tdSql.execute('use db_taosx')
         tdSql.query("show tables")
-        tdSql.checkRows(6)
+        if drop:
+            tdSql.checkRows(10)
+        else:
+            tdSql.checkRows(15)
         tdSql.query("select * from jt order by i")
         tdSql.checkRows(2)
         tdSql.checkData(0, 1, 1)
@@ -49,15 +52,72 @@ class TDTestCase:
         tdSql.checkData(0, 2, '{"k1":1,"k2":"hello"}')
         tdSql.checkData(1, 2, None)
 
+        tdSql.query("select * from sttb order by ts")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 13)
+        tdSql.checkData(1, 1, 16)
+        tdSql.checkData(0, 2, 22)
+        tdSql.checkData(1, 2, 25)
+        tdSql.checkData(0, 5, "sttb3")
+        tdSql.checkData(1, 5, "sttb4")
+
+        tdSql.query("select * from stt order by ts")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 1)
+        tdSql.checkData(1, 1, 21)
+        tdSql.checkData(0, 2, 2)
+        tdSql.checkData(1, 2, 21)
+        tdSql.checkData(0, 5, "stt3")
+        tdSql.checkData(1, 5, "stt4")
+
         tdSql.execute('use abc1')
         tdSql.query("show tables")
-        tdSql.checkRows(6)
+        if drop:
+            tdSql.checkRows(10)
+        else:
+            tdSql.checkRows(15)
         tdSql.query("select * from jt order by i")
         tdSql.checkRows(2)
         tdSql.checkData(0, 1, 1)
         tdSql.checkData(1, 1, 11)
         tdSql.checkData(0, 2, '{"k1":1,"k2":"hello"}')
         tdSql.checkData(1, 2, None)
+
+        tdSql.query("select * from sttb order by ts")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 13)
+        tdSql.checkData(1, 1, 16)
+        tdSql.checkData(0, 2, 22)
+        tdSql.checkData(1, 2, 25)
+        tdSql.checkData(0, 5, "sttb3")
+        tdSql.checkData(1, 5, "sttb4")
+
+        tdSql.query("select * from stt order by ts")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 1)
+        tdSql.checkData(1, 1, 21)
+        tdSql.checkData(0, 2, 2)
+        tdSql.checkData(1, 2, 21)
+        tdSql.checkData(0, 5, "stt3")
+        tdSql.checkData(1, 5, "stt4")
+
+        return
+
+    def checkDataTable(self):
+        tdSql.execute('use db_taosx')
+        tdSql.query("select * from meters_summary")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 1, 120)
+        tdSql.checkData(0, 2, 1)
+        tdSql.checkData(0, 3, "San Francisco")
+
+        tdSql.execute('use abc1')
+        tdSql.query("select * from meters_summary")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 1, 120)
+        tdSql.checkData(0, 2, 1)
+        tdSql.checkData(0, 3, "San Francisco")
+
         return
 
     def checkData(self):
@@ -144,6 +204,19 @@ class TDTestCase:
 
         self.checkJson(cfgPath, "tmq_taosx_tmp")
         self.checkData()
+        self.checkDropData(False)
+
+        return
+
+    def checkWal1VgroupTable(self):
+        buildPath = tdCom.getBuildPath()
+        cfgPath = tdCom.getClientCfgPath()
+        cmdStr = '%s/build/bin/tmq_taosx_ci -c %s -sv 1 -dv 1 -t'%(buildPath, cfgPath)
+        tdLog.info(cmdStr)
+        os.system(cmdStr)
+
+        self.checkJson(cfgPath, "tmq_taosx_tmp")
+        self.checkDataTable()
 
         return
 
@@ -154,6 +227,7 @@ class TDTestCase:
         os.system(cmdStr)
 
         self.checkData()
+        self.checkDropData(False)
 
         return
 
@@ -163,7 +237,7 @@ class TDTestCase:
         tdLog.info(cmdStr)
         os.system(cmdStr)
 
-        self.checkDropData()
+        self.checkDropData(True)
 
         return
 
@@ -176,6 +250,19 @@ class TDTestCase:
 
         self.checkJson(cfgPath, "tmq_taosx_tmp_snapshot")
         self.checkData()
+        self.checkDropData(False)
+
+        return
+
+    def checkSnapshot1VgroupTable(self):
+        buildPath = tdCom.getBuildPath()
+        cfgPath = tdCom.getClientCfgPath()
+        cmdStr = '%s/build/bin/tmq_taosx_ci -c %s -sv 1 -dv 1 -s -t'%(buildPath, cfgPath)
+        tdLog.info(cmdStr)
+        os.system(cmdStr)
+
+        self.checkJson(cfgPath, "tmq_taosx_tmp_snapshot")
+        self.checkDataTable()
 
         return
 
@@ -186,6 +273,7 @@ class TDTestCase:
         os.system(cmdStr)
 
         self.checkData()
+        self.checkDropData(False)
 
         return
 
@@ -195,7 +283,7 @@ class TDTestCase:
         tdLog.info(cmdStr)
         os.system(cmdStr)
 
-        self.checkDropData()
+        self.checkDropData(True)
 
         return
 
@@ -204,6 +292,9 @@ class TDTestCase:
         self.checkWal1Vgroup()
         self.checkSnapshot1Vgroup()
 
+        self.checkWal1VgroupTable()
+        self.checkSnapshot1VgroupTable()
+
         self.checkWalMultiVgroups()
         self.checkSnapshotMultiVgroups()
 
diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c
index 337a3cbd29..223c02de43 100644
--- a/utils/test/c/tmq_taosx_ci.c
+++ b/utils/test/c/tmq_taosx_ci.c
@@ -58,12 +58,11 @@ static void msg_process(TAOS_RES* msg) {
     char* result = tmq_get_json_meta(msg);
     if (result) {
       printf("meta result: %s\n", result);
+      if(g_fp && strcmp(result, "") != 0){
+        taosFprintfFile(g_fp, result);
+        taosFprintfFile(g_fp, "\n");
+      }
     }
-    if(g_fp && result){
-      taosFprintfFile(g_fp, result);
-      taosFprintfFile(g_fp, "\n");
-    }
-
     tmq_free_json_meta(result);
   }
 
@@ -71,7 +70,7 @@ static void msg_process(TAOS_RES* msg) {
   tmq_get_raw(msg, &raw);
   int32_t ret = tmq_write_raw(pConn, raw);
   printf("write raw data: %s\n", tmq_err2str(ret));
-
+  tmq_free_raw(raw);
   taos_close(pConn);
 }
 
@@ -310,31 +309,39 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes){
     taos_free_result(pRes);
   }
 
-  pRes = taos_query(pConn,
-                    "create stable if not exists stt (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 "
-                    "nchar(8), t4 bool)");
-  if (taos_errno(pRes) != 0) {
-    printf("failed to create super table stt, reason:%s\n", taos_errstr(pRes));
-    return -1;
-  }
-  taos_free_result(pRes);
-
-  pRes = taos_query(pConn,
-                    "create stable if not exists sttb (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 "
-                    "nchar(8), t4 bool)");
-  if (taos_errno(pRes) != 0) {
-    printf("failed to create super table sttb, reason:%s\n", taos_errstr(pRes));
-    return -1;
-  }
-  taos_free_result(pRes);
-
-  pRes = taos_query(pConn, "create table if not exists stt1 using stt tags(2, \"stt1\", true) sttb1 using sttb tags(4, \"sttb1\", true) "
-                    "stt2 using stt tags(43, \"stt2\", false) sttb2 using sttb tags(54, \"sttb2\", true)");
-  if (taos_errno(pRes) != 0) {
-    printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes));
-    return -1;
-  }
-  taos_free_result(pRes);
+//  pRes = taos_query(pConn,
+//                    "create stable if not exists stt (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 "
+//                    "nchar(8), t4 bool)");
+//  if (taos_errno(pRes) != 0) {
+//    printf("failed to create super table stt, reason:%s\n", taos_errstr(pRes));
+//    return -1;
+//  }
+//  taos_free_result(pRes);
+//
+//  pRes = taos_query(pConn,
+//                    "create stable if not exists sttb (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 "
+//                    "nchar(8), t4 bool)");
+//  if (taos_errno(pRes) != 0) {
+//    printf("failed to create super table sttb, reason:%s\n", taos_errstr(pRes));
+//    return -1;
+//  }
+//  taos_free_result(pRes);
+//
+//  pRes = taos_query(pConn, "create table if not exists stt1 using stt tags(2, \"stt1\", true) sttb1 using sttb tags(4, \"sttb1\", true) "
+//                    "stt2 using stt tags(43, \"stt2\", false) sttb2 using sttb tags(54, \"sttb2\", true)");
+//  if (taos_errno(pRes) != 0) {
+//    printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes));
+//    return -1;
+//  }
+//  taos_free_result(pRes);
+//
+//  pRes = taos_query(pConn, "insert into stt3 using stt tags(23, \"stt3\", true) values(now + 1s, 1, 2, 'stt3') sttb3 using sttb tags(4, \"sttb3\", true) values(now + 2s, 13, 22, 'sttb3') "
+//                           "stt4 using stt tags(433, \"stt4\", false) values(now + 3s, 21, 21, 'stt4') sttb4 using sttb tags(543, \"sttb4\", true) values(now + 4s, 16, 25, 'sttb4')");
+//  if (taos_errno(pRes) != 0) {
+//    printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes));
+//    return -1;
+//  }
+//  taos_free_result(pRes);
 
   return 0;
 }
@@ -569,56 +576,83 @@ void initLogFile() {
   }
 
   if(g_conf.snapShot){
-    char *result[] = {
-        "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":64},{\"name\":\"c4\",\"type\":5}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1},{\"name\":\"t2\",\"type\":8,\"length\":64}]}",
-        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}",
-        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}],\"createList\":[]}",
-        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[],\"createList\":[]}",
-        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":5000}],\"createList\":[]}",
-        "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c2\",\"type\":10,\"length\":8},{\"name\":\"cc3\",\"type\":5}],\"tags\":[]}",
-        "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"jt\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}",
-        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt1\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}],\"createList\":[]}",
-        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt2\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[],\"createList\":[]}",
-        "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"stt\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}",
-        "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"sttb\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}",
-        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt1\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}",
-        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"sttb1\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}",
-        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt2\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":43},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}],\"createList\":[]}",
-        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"sttb2\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":54},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}",
-    };
-
-    for(int i = 0; i < sizeof(result)/sizeof(result[0]); i++){
-      taosFprintfFile(pFile2, result[i]);
-      taosFprintfFile(pFile2, "\n");
+    if(g_conf.subTable){
+      char *result[] = {
+          "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"meters_summary\",\"columns\":[{\"name\":\"_wstart\",\"type\":9},{\"name\":\"current\",\"type\":6},{\"name\":\"groupid\",\"type\":4},{\"name\":\"location\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"group_id\",\"type\":14}]}",
+          "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"t_d2a450ee819dcf7576f0282d9ac22dbc\",\"using\":\"meters_summary\",\"tagNum\":1,\"tags\":[{\"name\":\"group_id\",\"type\":14,\"value\":1.313555008277358e+19}],\"createList\":[]}"
+      };
+      for(int i = 0; i < sizeof(result)/sizeof(result[0]); i++){
+        taosFprintfFile(pFile2, result[i]);
+        taosFprintfFile(pFile2, "\n");
+      }
+    }else{
+      char *result[] = {
+          "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":64},{\"name\":\"c4\",\"type\":5}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1},{\"name\":\"t2\",\"type\":8,\"length\":64}]}",
+          "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}",
+          "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}],\"createList\":[]}",
+          "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[],\"createList\":[]}",
+          "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":5000}],\"createList\":[]}",
+          "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c2\",\"type\":10,\"length\":8},{\"name\":\"cc3\",\"type\":5}],\"tags\":[]}",
+          "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"jt\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}",
+          "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt1\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}],\"createList\":[]}",
+          "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt2\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[],\"createList\":[]}",
+          "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"stt\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}",
+          "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"sttb\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}",
+          "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt1\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}",
+          "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"sttb1\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}",
+          "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt2\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":43},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}],\"createList\":[]}",
+          "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"sttb2\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":54},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}",
+          "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt3\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":23},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}",
+          "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"sttb3\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}",
+          "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt4\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":433},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}],\"createList\":[]}",
+          "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"sttb4\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":543},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}"
+      };
+      for(int i = 0; i < sizeof(result)/sizeof(result[0]); i++){
+        taosFprintfFile(pFile2, result[i]);
+        taosFprintfFile(pFile2, "\n");
+      }
     }
   }else{
-    char *result[] = {
-        "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}",
-        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}",
-        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}],\"createList\":[]}",
-        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[],\"createList\":[]}",
-        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":3000}],\"createList\":[]}",
-        "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\",\"colType\":5}",
-        "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":7,\"colName\":\"c3\",\"colType\":8,\"colLength\":64}",
-        "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":1,\"colName\":\"t2\",\"colType\":8,\"colLength\":64}",
-        "{\"type\":\"alter\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"alterType\":4,\"colName\":\"t1\",\"colValue\":\"5000\",\"colValueNull\":false}",
-        "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":10,\"length\":4}],\"tags\":[]}",
-        "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":5,\"colName\":\"c3\",\"colType\":5}",
-        "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":7,\"colName\":\"c2\",\"colType\":10,\"colLength\":8}",
-        "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":10,\"colName\":\"c3\",\"colNewName\":\"cc3\"}",
-        "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":9}",
-        "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":6,\"colName\":\"c1\"}",
-        "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"jt\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}",
-        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt1\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}],\"createList\":[]}",
-        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt2\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[],\"createList\":[]}",
-        "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"stt\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}",
-        "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"sttb\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}",
-        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt1\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"stt1\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"sttb1\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"stt2\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":43},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}]},{\"tableName\":\"sttb2\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":54},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}"
-    };
+    if(g_conf.subTable){
+      char *result[] = {
+        "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"meters_summary\",\"columns\":[{\"name\":\"_wstart\",\"type\":9},{\"name\":\"current\",\"type\":6},{\"name\":\"groupid\",\"type\":4},{\"name\":\"location\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"group_id\",\"type\":14}]}",
+        "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"t_d2a450ee819dcf7576f0282d9ac22dbc\",\"using\":\"meters_summary\",\"tagNum\":1,\"tags\":[{\"name\":\"group_id\",\"type\":14,\"value\":1.313555008277358e+19}],\"createList\":[]}"
+      };
 
-    for(int i = 0; i < sizeof(result)/sizeof(result[0]); i++){
-      taosFprintfFile(pFile2, result[i]);
-      taosFprintfFile(pFile2, "\n");
+      for(int i = 0; i < sizeof(result)/sizeof(result[0]); i++){
+        taosFprintfFile(pFile2, result[i]);
+        taosFprintfFile(pFile2, "\n");
+      }
+    }else{
+      char *result[] = {
+          "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}",
+          "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}",
+          "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}],\"createList\":[]}",
+          "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[],\"createList\":[]}",
+          "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":3000}],\"createList\":[]}",
+          "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\",\"colType\":5}",
+          "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":7,\"colName\":\"c3\",\"colType\":8,\"colLength\":64}",
+          "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":1,\"colName\":\"t2\",\"colType\":8,\"colLength\":64}",
+          "{\"type\":\"alter\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"alterType\":4,\"colName\":\"t1\",\"colValue\":\"5000\",\"colValueNull\":false}",
+          "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":10,\"length\":4}],\"tags\":[]}",
+          "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":5,\"colName\":\"c3\",\"colType\":5}",
+          "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":7,\"colName\":\"c2\",\"colType\":10,\"colLength\":8}",
+          "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":10,\"colName\":\"c3\",\"colNewName\":\"cc3\"}",
+          "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":9}",
+          "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":6,\"colName\":\"c1\"}",
+          "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"jt\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}",
+          "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt1\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}],\"createList\":[]}",
+          "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt2\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[],\"createList\":[]}",
+          "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"stt\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}",
+          "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"sttb\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}",
+          "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt1\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"stt1\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"sttb1\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"stt2\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":43},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}]},{\"tableName\":\"sttb2\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":54},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}",
+          "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt3\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":23},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"stt3\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":23},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"sttb3\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"stt4\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":433},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}]},{\"tableName\":\"sttb4\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":543},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}"
+      };
+
+      for(int i = 0; i < sizeof(result)/sizeof(result[0]); i++){
+        taosFprintfFile(pFile2, result[i]);
+        taosFprintfFile(pFile2, "\n");
+      }
     }
   }
 
@@ -655,5 +689,6 @@ int main(int argc, char* argv[]) {
   tmq_t*      tmq = build_consumer();
   tmq_list_t* topic_list = build_topic_list();
   basic_consume_loop(tmq, topic_list);
+  tmq_list_destroy(topic_list);
   taosCloseFile(&g_fp);
 }

From e0445e0698adcf2dbdaeda77ebff9d30d60e9e2a Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Fri, 16 Sep 2022 09:56:15 +0800
Subject: [PATCH 10/57] fix: taos_connect in gpd_init method and taos_close in
 gpd_destroy method

---
 source/libs/function/src/udfd.c | 28 +++++++++++++++++-----------
 tests/script/sh/gpd.c           | 11 ++++-------
 2 files changed, 21 insertions(+), 18 deletions(-)

diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c
index 23a95320fb..f8f44a9816 100644
--- a/source/libs/function/src/udfd.c
+++ b/source/libs/function/src/udfd.c
@@ -29,8 +29,6 @@
 #include "trpc.h"
 // clang-foramt on
 
-SArray* udfdResidentFuncs = NULL;
-
 typedef struct SUdfdContext {
   uv_loop_t * loop;
   uv_pipe_t   ctrlPipe;
@@ -43,6 +41,8 @@ typedef struct SUdfdContext {
   uv_mutex_t udfsMutex;
   SHashObj * udfsHash;
 
+  SArray* residentFuncs;
+
   bool printVersion;
 } SUdfdContext;
 
@@ -204,8 +204,8 @@ void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
       udf->initFunc();
     }
     udf->resident = false;
-    for (int32_t i = 0; i < taosArrayGetSize(udfdResidentFuncs); ++i) {
-      char* funcName = taosArrayGet(udfdResidentFuncs, i);
+    for (int32_t i = 0; i < taosArrayGetSize(global.residentFuncs); ++i) {
+      char* funcName = taosArrayGet(global.residentFuncs, i);
       if (strcmp(setup->udfName, funcName) == 0) {
         udf->resident = true;
         break;
@@ -930,8 +930,6 @@ static int32_t udfdRun() {
   uv_run(global.loop, UV_RUN_DEFAULT);
   uv_loop_close(global.loop);
 
-  uv_mutex_destroy(&global.udfsMutex);
-  taosHashCleanup(global.udfsHash);
   return 0;
 }
 
@@ -953,17 +951,17 @@ void udfdConnectMnodeThreadFunc(void *args) {
 }
 
 int32_t udfdInitResidentFuncs() {
-  udfdResidentFuncs = taosArrayInit(2, TSDB_FUNC_NAME_LEN);
+  global.residentFuncs = taosArrayInit(2, TSDB_FUNC_NAME_LEN);
   char gpd[TSDB_FUNC_NAME_LEN] = "gpd";
-  taosArrayPush(udfdResidentFuncs, gpd);
+  taosArrayPush(global.residentFuncs, gpd);
   char gpdBatch[TSDB_FUNC_NAME_LEN] = "gpdbatch";
-  taosArrayPush(udfdResidentFuncs, gpdBatch);
+  taosArrayPush(global.residentFuncs, gpdBatch);
   return TSDB_CODE_SUCCESS;
 }
 
 int32_t udfdDeinitResidentFuncs() {
-  for (int32_t i = 0; i < taosArrayGetSize(udfdResidentFuncs); ++i) {
-    char* funcName = taosArrayGet(udfdResidentFuncs, i);
+  for (int32_t i = 0; i < taosArrayGetSize(global.residentFuncs); ++i) {
+    char* funcName = taosArrayGet(global.residentFuncs, i);
     SUdf** udfInHash =  taosHashGet(global.udfsHash, funcName, strlen(funcName));
     if (udfInHash) {
       taosHashRemove(global.udfsHash, funcName, strlen(funcName));
@@ -975,9 +973,16 @@ int32_t udfdDeinitResidentFuncs() {
       taosMemoryFree(udf);
     }
   }
+  taosArrayDestroy(global.residentFuncs);
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t udfdCleanup() {
+  uv_mutex_destroy(&global.udfsMutex);
+  taosHashCleanup(global.udfsHash);
+  return 0;
+}
+
 int main(int argc, char *argv[]) {
   if (!taosCheckSystemIsLittleEnd()) {
     printf("failed to start since on non-little-end machines\n");
@@ -1026,5 +1031,6 @@ int main(int argc, char *argv[]) {
   udfdCloseClientRpc();
 
   udfdDeinitResidentFuncs();
+  udfdCleanup();
   return 0;
 }
diff --git a/tests/script/sh/gpd.c b/tests/script/sh/gpd.c
index 56c3388f3e..8d69bacb5e 100644
--- a/tests/script/sh/gpd.c
+++ b/tests/script/sh/gpd.c
@@ -9,12 +9,16 @@
 #endif
 #include "taosudf.h"
 
+TAOS* taos = NULL;
 
 DLL_EXPORT int32_t gpd_init() {
+  taos = taos_connect("localhost", "root", "taosdata", "", 7100);
   return 0;
 }
 
 DLL_EXPORT int32_t gpd_destroy() {
+  taos_close(taos);
+  taos_cleanup();
   return 0;
 }
 
@@ -40,11 +44,6 @@ DLL_EXPORT int32_t gpd(SUdfDataBlock* block, SUdfColumn *resultCol) {
       udfColDataSet(resultCol, i, (char *)&luckyNum, false);
     }
   }
-  taos_init();
-  TAOS* taos = taos_connect("localhost", "root", "taosdata", "", 7100);
-  if (taos == NULL) {
-    char* errstr = "can not connect";
-  }
   TAOS_RES* res = taos_query(taos, "create database if not exists gpd");
   if (taos_errno(res) != 0) {
     char* errstr = taos_errstr(res);
@@ -64,8 +63,6 @@ DLL_EXPORT int32_t gpd(SUdfDataBlock* block, SUdfColumn *resultCol) {
     char* errstr = taos_errstr(res);
   } 
 
-  taos_close(taos);
-  taos_cleanup();
   //to simulate actual processing delay by udf
 #ifdef LINUX
   usleep(1 * 1000);   // usleep takes sleep time in us (1 millionth of a second)

From 13059c40bcc5599367e75ac8b4db0683e871aaa6 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Fri, 16 Sep 2022 10:45:32 +0800
Subject: [PATCH 11/57] fix:error in auto create table for taosX

---
 source/client/src/clientImpl.c          |  1 +
 source/client/src/clientRawBlockWrite.c |  2 +
 utils/test/c/tmq_taosx_ci.c             | 66 ++++++++++++-------------
 3 files changed, 36 insertions(+), 33 deletions(-)

diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index 5ebc2729f8..6cb0660dc8 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -189,6 +189,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
     tscError("%d failed to add to request container, reqId:0x%" PRIx64 ", conn:%d, %s", (*pRequest)->self,
              (*pRequest)->requestId, pTscObj->id, sql);
 
+    taosMemoryFree(param);
     destroyRequest(*pRequest);
     *pRequest = NULL;
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 00c46dcb8a..a77567dca3 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -599,6 +599,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
     goto end;
   }
 
+  pRequest->syncQuery = true;
   if (!pRequest->pDb) {
     code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
     goto end;
@@ -767,6 +768,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
     goto end;
   }
 
+  pRequest->syncQuery = true;
   if (!pRequest->pDb) {
     code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
     goto end;
diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c
index 223c02de43..038723955f 100644
--- a/utils/test/c/tmq_taosx_ci.c
+++ b/utils/test/c/tmq_taosx_ci.c
@@ -309,39 +309,39 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes){
     taos_free_result(pRes);
   }
 
-//  pRes = taos_query(pConn,
-//                    "create stable if not exists stt (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 "
-//                    "nchar(8), t4 bool)");
-//  if (taos_errno(pRes) != 0) {
-//    printf("failed to create super table stt, reason:%s\n", taos_errstr(pRes));
-//    return -1;
-//  }
-//  taos_free_result(pRes);
-//
-//  pRes = taos_query(pConn,
-//                    "create stable if not exists sttb (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 "
-//                    "nchar(8), t4 bool)");
-//  if (taos_errno(pRes) != 0) {
-//    printf("failed to create super table sttb, reason:%s\n", taos_errstr(pRes));
-//    return -1;
-//  }
-//  taos_free_result(pRes);
-//
-//  pRes = taos_query(pConn, "create table if not exists stt1 using stt tags(2, \"stt1\", true) sttb1 using sttb tags(4, \"sttb1\", true) "
-//                    "stt2 using stt tags(43, \"stt2\", false) sttb2 using sttb tags(54, \"sttb2\", true)");
-//  if (taos_errno(pRes) != 0) {
-//    printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes));
-//    return -1;
-//  }
-//  taos_free_result(pRes);
-//
-//  pRes = taos_query(pConn, "insert into stt3 using stt tags(23, \"stt3\", true) values(now + 1s, 1, 2, 'stt3') sttb3 using sttb tags(4, \"sttb3\", true) values(now + 2s, 13, 22, 'sttb3') "
-//                           "stt4 using stt tags(433, \"stt4\", false) values(now + 3s, 21, 21, 'stt4') sttb4 using sttb tags(543, \"sttb4\", true) values(now + 4s, 16, 25, 'sttb4')");
-//  if (taos_errno(pRes) != 0) {
-//    printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes));
-//    return -1;
-//  }
-//  taos_free_result(pRes);
+  pRes = taos_query(pConn,
+                    "create stable if not exists stt (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 "
+                    "nchar(8), t4 bool)");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create super table stt, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
+  pRes = taos_query(pConn,
+                    "create stable if not exists sttb (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 "
+                    "nchar(8), t4 bool)");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create super table sttb, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
+  pRes = taos_query(pConn, "create table if not exists stt1 using stt tags(2, \"stt1\", true) sttb1 using sttb tags(4, \"sttb1\", true) "
+                    "stt2 using stt tags(43, \"stt2\", false) sttb2 using sttb tags(54, \"sttb2\", true)");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
+  pRes = taos_query(pConn, "insert into stt3 using stt tags(23, \"stt3\", true) values(now + 1s, 1, 2, 'stt3') sttb3 using sttb tags(4, \"sttb3\", true) values(now + 2s, 13, 22, 'sttb3') "
+                           "stt4 using stt tags(433, \"stt4\", false) values(now + 3s, 21, 21, 'stt4') sttb4 using sttb tags(543, \"sttb4\", true) values(now + 4s, 16, 25, 'sttb4')");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
 
   return 0;
 }

From 9b600d6f8f01ae8f8bf0e98f1784ffab4790b05d Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Fri, 16 Sep 2022 11:53:58 +0800
Subject: [PATCH 12/57] mroe code

---
 source/dnode/vnode/src/tsdb/tsdbCompress.c | 179 ++++++++++++++++++---
 1 file changed, 159 insertions(+), 20 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbCompress.c b/source/dnode/vnode/src/tsdb/tsdbCompress.c
index 76be7c1070..68cf6ef0ca 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCompress.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCompress.c
@@ -13,45 +13,129 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "lz4.h"
 #include "tsdb.h"
 
-// Integer =====================================================
 typedef struct {
-  int8_t   rawCopy;
-  int64_t  prevVal;
-  int32_t  nVal;
-  int32_t  nBuf;
-  uint8_t *pBuf;
-} SIntCompressor;
+  int8_t   type;
+  int8_t   cmprAlg;
+  uint8_t *aBuf[2];
+  int64_t  nBuf[2];
+  union {
+    // Timestamp ----
+    struct {
+      /* data */
+    };
+    // Integer ----
+    struct {
+      /* data */
+    };
+    // Binary ----
+    struct {
+      /* data */
+    };
+    // Float ----
+    struct {
+      /* data */
+    };
+    // Bool ----
+    struct {
+      int32_t bool_n;
+      uint8_t bool_b;
+    };
+  };
+} SCompressor;
 
+// Timestamp =====================================================
+static int32_t tCompTimestamp(SCompressor *pCmprsor, TSKEY ts) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+// Integer =====================================================
 #define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (b)) || ((a) < 0 && (b) >= INT64_MIN - (a)))
 #define SIMPLE8B_MAX       ((uint64_t)1152921504606846974LL)
 
-static int32_t tsdbCmprI64(SIntCompressor *pCompressor, int64_t val) {
+static int32_t tCompI64(SCompressor *pCmprsor, int64_t val) {
   int32_t code = 0;
+#if 0
 
   // raw copy
-  if (pCompressor->rawCopy) {
-    memcpy(pCompressor->pBuf + pCompressor->nBuf, &val, sizeof(val));
-    pCompressor->nBuf += sizeof(val);
-    pCompressor->nVal++;
+  if (pCmprsor->rawCopy) {
+    memcpy(pCmprsor->pBuf + pCmprsor->nBuf, &val, sizeof(val));
+    pCmprsor->nBuf += sizeof(val);
+    pCmprsor->nVal++;
     goto _exit;
   }
 
-  if (!I64_SAFE_ADD(val, pCompressor->prevVal)) {
-    pCompressor->rawCopy = 1;
+  if (!I64_SAFE_ADD(val, pCmprsor->prevVal)) {
+    pCmprsor->rawCopy = 1;
     // TODO: decompress and copy
-    pCompressor->nVal++;
+    pCmprsor->nVal++;
     goto _exit;
   }
 
-  int64_t diff = val - pCompressor->prevVal;
+  int64_t diff = val - pCmprsor->prevVal;
   uint8_t zigzag = ZIGZAGE(int64_t, diff);
 
   if (zigzag >= SIMPLE8B_MAX) {
-    pCompressor->rawCopy = 1;
+    pCmprsor->rawCopy = 1;
     // TODO: decompress and copy
-    pCompressor->nVal++;
+    pCmprsor->nVal++;
+    goto _exit;
+  }
+
+_exit:
+#endif
+  return code;
+}
+
+// Float =====================================================
+static int32_t tCompFloat() {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+// Binary =====================================================
+static int32_t tCompBinary(SCompressor *pCmprsor, const uint8_t *pData, int32_t nData) {
+  int32_t code = 0;
+
+  if (nData) {
+    memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], pData, nData);
+    pCmprsor->nBuf[0] += nData;
+  }
+
+  return code;
+}
+
+// Bool =====================================================
+static uint8_t BOOL_CMPR_TABLE[] = {0b01, 0b0100, 0b010000, 0b01000000};
+static int32_t tCompBool(SCompressor *pCmprsor, bool vBool) {
+  int32_t code = 0;
+
+  if (vBool) {
+    pCmprsor->bool_b |= BOOL_CMPR_TABLE[pCmprsor->bool_n % 4];
+  }
+  pCmprsor->bool_n++;
+
+  if (pCmprsor->bool_n % 4 == 0) {
+    pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = pCmprsor->bool_b;
+    pCmprsor->nBuf[0]++;
+    pCmprsor->bool_b = 0;
+  }
+
+  return code;
+}
+
+// SCompressor =====================================================
+int32_t tCompressorCreate(SCompressor **ppCmprsor) {
+  int32_t code = 0;
+
+  *ppCmprsor = (SCompressor *)taosMemoryCalloc(1, sizeof(SCompressor));
+  if ((*ppCmprsor) == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
     goto _exit;
   }
 
@@ -59,6 +143,61 @@ _exit:
   return code;
 }
 
-// Timestamp =====================================================
+int32_t tCompressorDestroy(SCompressor *pCmprsor) {
+  int32_t code = 0;
 
-// Float =====================================================
\ No newline at end of file
+  if (pCmprsor) {
+    for (int32_t iBuf = 0; iBuf < sizeof(pCmprsor->aBuf) / sizeof(pCmprsor->aBuf[0]); iBuf++) {
+      tFree(pCmprsor->aBuf[iBuf]);
+    }
+  }
+
+  return code;
+}
+
+int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) {
+  int32_t code = 0;
+
+  pCmprsor->type = type;
+  pCmprsor->cmprAlg = cmprAlg;
+
+  switch (type) {
+    case TSDB_DATA_TYPE_BOOL:
+      pCmprsor->bool_n = 0;
+      pCmprsor->bool_b = 0;
+      break;
+
+    default:
+      break;
+  }
+
+  return code;
+}
+
+int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int64_t *nData) {
+  int32_t code = 0;
+
+  if (pCmprsor->cmprAlg == TWO_STAGE_COMP || IS_VAR_DATA_TYPE(pCmprsor->type)) {
+    code = tRealloc(&pCmprsor->aBuf[1], pCmprsor->nBuf[0] + 1);
+    if (code) goto _exit;
+
+    int64_t ret = LZ4_compress_default(pCmprsor->aBuf[0], pCmprsor->aBuf[1] + 1, pCmprsor->nBuf[0], pCmprsor->nBuf[0]);
+    if (ret) {
+      pCmprsor->aBuf[1][0] = 0;
+      pCmprsor->nBuf[1] = ret + 1;
+    } else {
+      pCmprsor->aBuf[1][0] = 1;
+      memcpy(pCmprsor->aBuf[1] + 1, pCmprsor->aBuf[0], pCmprsor->nBuf[0]);
+      pCmprsor->nBuf[1] = pCmprsor->nBuf[0] + 1;
+    }
+
+    *ppData = pCmprsor->aBuf[1];
+    *nData = pCmprsor->nBuf[1];
+  } else {
+    *ppData = pCmprsor->aBuf[0];
+    *nData = pCmprsor->nBuf[0];
+  }
+
+_exit:
+  return code;
+}

From a34b4ec749bd00f179d1bfefd32b11c7089eff0c Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Fri, 16 Sep 2022 13:33:36 +0800
Subject: [PATCH 13/57] more code

---
 source/dnode/vnode/src/tsdb/tsdbCompress.c | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbCompress.c b/source/dnode/vnode/src/tsdb/tsdbCompress.c
index 68cf6ef0ca..2d402a8976 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCompress.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCompress.c
@@ -41,7 +41,6 @@ typedef struct {
     // Bool ----
     struct {
       int32_t bool_n;
-      uint8_t bool_b;
     };
   };
 } SCompressor;
@@ -115,17 +114,17 @@ static uint8_t BOOL_CMPR_TABLE[] = {0b01, 0b0100, 0b010000, 0b01000000};
 static int32_t tCompBool(SCompressor *pCmprsor, bool vBool) {
   int32_t code = 0;
 
+  int32_t mod4 = pCmprsor->bool_n & 3;
   if (vBool) {
-    pCmprsor->bool_b |= BOOL_CMPR_TABLE[pCmprsor->bool_n % 4];
+    pCmprsor->aBuf[0][pCmprsor->nBuf[0]] |= BOOL_CMPR_TABLE[mod4];
   }
   pCmprsor->bool_n++;
-
-  if (pCmprsor->bool_n % 4 == 0) {
-    pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = pCmprsor->bool_b;
+  if (mod4 == 3) {
     pCmprsor->nBuf[0]++;
-    pCmprsor->bool_b = 0;
+    pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = 0;
   }
 
+_exit:
   return code;
 }
 

From 832f2fb7a397584e9594ff59818ac2d028f08944 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Fri, 16 Sep 2022 15:49:43 +0800
Subject: [PATCH 14/57] more code

---
 source/dnode/vnode/src/tsdb/tsdbCompress.c | 164 ++++++++++++++++++++-
 1 file changed, 156 insertions(+), 8 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbCompress.c b/source/dnode/vnode/src/tsdb/tsdbCompress.c
index 2d402a8976..0b1a70fba7 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCompress.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCompress.c
@@ -16,6 +16,8 @@
 #include "lz4.h"
 #include "tsdb.h"
 
+#define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (b)) || ((a) < 0 && (b) >= INT64_MIN - (a)))
+
 typedef struct {
   int8_t   type;
   int8_t   cmprAlg;
@@ -24,7 +26,11 @@ typedef struct {
   union {
     // Timestamp ----
     struct {
-      /* data */
+      int8_t   ts_copy;
+      int32_t  ts_n;
+      int64_t  ts_prev_val;
+      int64_t  ts_prev_delta;
+      uint8_t *ts_flag_p;
     };
     // Integer ----
     struct {
@@ -46,15 +52,133 @@ typedef struct {
 } SCompressor;
 
 // Timestamp =====================================================
+static int32_t tCompSetCopyMode(SCompressor *pCmprsor) {
+  int32_t code = 0;
+
+  if (pCmprsor->ts_n) {
+    code = tRealloc(&pCmprsor->aBuf[1], sizeof(int64_t) * (pCmprsor->ts_n + 1));
+    if (code) return code;
+    pCmprsor->nBuf[1] = 1;
+
+    int64_t  n = 1;
+    int64_t  valPrev;
+    int64_t  delPrev;
+    uint64_t vZigzag;
+    while (n < pCmprsor->nBuf[0]) {
+      uint8_t n1 = pCmprsor->aBuf[0][0] & 0xf;
+      uint8_t n2 = pCmprsor->aBuf[0][0] >> 4;
+
+      n++;
+
+      vZigzag = 0;
+      for (uint8_t i = 0; i < n1; i++) {
+        vZigzag |= (((uint64_t)pCmprsor->aBuf[0][n]) << (sizeof(int64_t) * i));
+        n++;
+      }
+      int64_t delta_of_delta = ZIGZAGD(int64_t, vZigzag);
+      if (n == 2) {
+        delPrev = 0;
+        valPrev = delta_of_delta;
+      } else {
+        delPrev = delta_of_delta + delPrev;
+        valPrev = delPrev + valPrev;
+      }
+
+      memcpy(pCmprsor->aBuf[1] + pCmprsor->nBuf[1], &valPrev, sizeof(int64_t));
+      pCmprsor->nBuf[1] += sizeof(int64_t);
+
+      if (n >= pCmprsor->nBuf[0]) break;
+
+      vZigzag = 0;
+      for (uint8_t i = 0; i < n2; i++) {
+        vZigzag |= (((uint64_t)pCmprsor->aBuf[0][n]) << (sizeof(int64_t) * i));
+        n++;
+      }
+      int64_t delta_of_delta = ZIGZAGD(int64_t, vZigzag);
+      delPrev = delta_of_delta + delPrev;
+      valPrev = delPrev + valPrev;
+    }
+
+    uint8_t *pBuf = pCmprsor->aBuf[0];
+    pCmprsor->aBuf[0] = pCmprsor->aBuf[1];
+    pCmprsor->aBuf[1] = pBuf;
+    pCmprsor->nBuf[0] = pCmprsor->nBuf[1];
+  } else {
+    // TODO
+  }
+
+  pCmprsor->aBuf[0][0] = 0;
+  pCmprsor->ts_copy = 1;
+
+  return code;
+}
 static int32_t tCompTimestamp(SCompressor *pCmprsor, TSKEY ts) {
   int32_t code = 0;
-  // TODO
+
+  if (pCmprsor->ts_n == 0) {
+    pCmprsor->ts_prev_val = ts;
+    pCmprsor->ts_prev_delta = -ts;
+  }
+
+  if (pCmprsor->ts_copy) goto _copy_exit;
+
+  if (!I64_SAFE_ADD(ts, -pCmprsor->ts_prev_val)) {
+    code = tCompSetCopyMode(pCmprsor);
+    if (code) return code;
+    goto _copy_exit;
+  }
+
+  int64_t delta = ts - pCmprsor->ts_prev_val;
+
+  if (!I64_SAFE_ADD(delta, -pCmprsor->ts_prev_delta)) {
+    code = tCompSetCopyMode(pCmprsor);
+    if (code) return code;
+    goto _copy_exit;
+  }
+
+  int64_t  delta_of_delta = delta - pCmprsor->ts_prev_delta;
+  uint64_t zigzag_value = ZIGZAGE(int64_t, delta_of_delta);
+
+  pCmprsor->ts_prev_val = ts;
+  pCmprsor->ts_prev_delta = delta;
+
+  if (pCmprsor->ts_n & 0x1 == 0) {
+    code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17 /*sizeof(int64_t) * 2 + 1*/);
+    if (code) return code;
+
+    pCmprsor->ts_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]];
+    pCmprsor->nBuf[0]++;
+    pCmprsor->ts_flag_p[0] = 0;
+
+    while (zigzag_value) {
+      pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (zigzag_value & 0xff);
+      pCmprsor->nBuf[0]++;
+      pCmprsor->ts_flag_p[0]++;
+    }
+  } else {
+    while (zigzag_value) {
+      pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (zigzag_value & 0xff);
+      pCmprsor->nBuf[0]++;
+      pCmprsor->ts_flag_p += (uint8_t)0x10;
+    }
+  }
+
+  pCmprsor->ts_n++;
+  return code;
+
+_copy_exit:
+  code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(int64_t));
+  if (code) return code;
+
+  memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], &ts, sizeof(ts));
+  pCmprsor->nBuf[0] += sizeof(ts);
+
+  pCmprsor->ts_n++;
   return code;
 }
 
 // Integer =====================================================
-#define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (b)) || ((a) < 0 && (b) >= INT64_MIN - (a)))
-#define SIMPLE8B_MAX       ((uint64_t)1152921504606846974LL)
+#define SIMPLE8B_MAX ((uint64_t)1152921504606846974LL)
 
 static int32_t tCompI64(SCompressor *pCmprsor, int64_t val) {
   int32_t code = 0;
@@ -110,7 +234,8 @@ static int32_t tCompBinary(SCompressor *pCmprsor, const uint8_t *pData, int32_t
 }
 
 // Bool =====================================================
-static uint8_t BOOL_CMPR_TABLE[] = {0b01, 0b0100, 0b010000, 0b01000000};
+static const uint8_t BOOL_CMPR_TABLE[] = {0b01, 0b0100, 0b010000, 0b01000000};
+
 static int32_t tCompBool(SCompressor *pCmprsor, bool vBool) {
   int32_t code = 0;
 
@@ -122,6 +247,9 @@ static int32_t tCompBool(SCompressor *pCmprsor, bool vBool) {
   if (mod4 == 3) {
     pCmprsor->nBuf[0]++;
     pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = 0;
+
+    code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0]);
+    if (code) goto _exit;
   }
 
 _exit:
@@ -138,6 +266,13 @@ int32_t tCompressorCreate(SCompressor **ppCmprsor) {
     goto _exit;
   }
 
+  code = tRealloc(&(*ppCmprsor)->aBuf[0], 1024);
+  if (code) {
+    taosMemoryFree(*ppCmprsor);
+    *ppCmprsor = NULL;
+    goto _exit;
+  }
+
 _exit:
   return code;
 }
@@ -146,9 +281,12 @@ int32_t tCompressorDestroy(SCompressor *pCmprsor) {
   int32_t code = 0;
 
   if (pCmprsor) {
-    for (int32_t iBuf = 0; iBuf < sizeof(pCmprsor->aBuf) / sizeof(pCmprsor->aBuf[0]); iBuf++) {
+    int32_t nBuf = sizeof(pCmprsor->aBuf) / sizeof(pCmprsor->aBuf[0]);
+    for (int32_t iBuf = 0; iBuf < nBuf; iBuf++) {
       tFree(pCmprsor->aBuf[iBuf]);
     }
+
+    taosMemoryFree(pCmprsor);
   }
 
   return code;
@@ -159,13 +297,17 @@ int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) {
 
   pCmprsor->type = type;
   pCmprsor->cmprAlg = cmprAlg;
+  pCmprsor->nBuf[0] = 0;  // (todo) may or may not +/- 1
 
   switch (type) {
+    case TSDB_DATA_TYPE_TIMESTAMP:
+      pCmprsor->ts_copy = 0;
+      pCmprsor->ts_n = 0;
+      break;
     case TSDB_DATA_TYPE_BOOL:
       pCmprsor->bool_n = 0;
-      pCmprsor->bool_b = 0;
+      pCmprsor->aBuf[0][0] = 0;
       break;
-
     default:
       break;
   }
@@ -200,3 +342,9 @@ int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int64_t *nData)
 _exit:
   return code;
 }
+
+int32_t tCompress(SCompressor *pCmprsor, void *pData, int64_t nData) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
\ No newline at end of file

From dfc5ce23a0910b1d598b0f92a9d572a056ee7b54 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Fri, 16 Sep 2022 17:01:41 +0800
Subject: [PATCH 15/57] fix:error in auto create table for taosX

---
 source/client/src/clientRawBlockWrite.c    | 34 +++++++++++++++++-----
 source/client/src/clientTmq.c              | 11 +++++--
 source/common/src/tmsg.c                   |  6 ++++
 source/dnode/mnode/impl/inc/mndDef.h       |  2 +-
 source/dnode/mnode/impl/src/mndConsumer.c  | 33 +++++++++++----------
 source/dnode/mnode/impl/src/mndDef.c       |  4 +--
 source/dnode/vnode/src/meta/metaQuery.c    | 10 +++++--
 source/dnode/vnode/src/meta/metaSnapshot.c |  3 ++
 source/dnode/vnode/src/meta/metaTable.c    | 14 +++++----
 source/dnode/vnode/src/tq/tq.c             | 20 +++++++++++++
 source/dnode/vnode/src/tq/tqOffset.c       |  2 ++
 source/libs/executor/src/executorimpl.c    |  6 ++++
 source/libs/executor/src/scanoperator.c    |  1 +
 source/libs/qworker/src/qwMsg.c            |  1 +
 utils/test/c/tmq_taosx_ci.c                |  1 +
 15 files changed, 112 insertions(+), 36 deletions(-)

diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index a77567dca3..514f99d463 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -219,8 +219,6 @@ static char* processAlterStb(SMqMetaRsp* metaRsp) {
     goto _err;
   }
   string = buildAlterSTableJson(req.alterOriData, req.alterOriDataLen);
-  tDecoderClear(&coder);
-  return string;
 
 _err:
   tDecoderClear(&coder);
@@ -397,6 +395,10 @@ static char* processAutoCreateTable(STaosxRsp* rsp) {
 _exit:
   for(int i = 0; i < rsp->createTableNum; i++){
     tDecoderClear(&decoder[i]);
+    taosMemoryFreeClear(pCreateReq[i].comment);
+    if (pCreateReq[i].type == TSDB_CHILD_TABLE) {
+      taosArrayDestroy(pCreateReq[i].ctb.tagName);
+    }
   }
   taosMemoryFree(decoder);
   taosMemoryFree(pCreateReq);
@@ -512,6 +514,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) {
   string = cJSON_PrintUnformatted(json);
 
 _exit:
+  cJSON_Delete(json);
   tDecoderClear(&decoder);
   return string;
 }
@@ -543,6 +546,7 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) {
   string = cJSON_PrintUnformatted(json);
 
 _exit:
+  cJSON_Delete(json);
   tDecoderClear(&decoder);
   return string;
 }
@@ -583,6 +587,7 @@ static char* processDropTable(SMqMetaRsp* metaRsp) {
   string = cJSON_PrintUnformatted(json);
 
 _exit:
+  cJSON_Delete(json);
   tDecoderClear(&decoder);
   return string;
 }
@@ -688,6 +693,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
     goto end;
   }
 
+  pRequest->syncQuery = true;
   if (!pRequest->pDb) {
     code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
     goto end;
@@ -920,7 +926,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
   if (code != TSDB_CODE_SUCCESS) {
     goto end;
   }
-
+  pRequest->syncQuery = true;
   if (!pRequest->pDb) {
     code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
     goto end;
@@ -1093,6 +1099,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
     goto end;
   }
 
+  pRequest->syncQuery = true;
   if (!pRequest->pDb) {
     code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
     goto end;
@@ -1212,6 +1219,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname)
     goto end;
   }
 
+  pRequest->syncQuery = true;
   if (!pRequest->pDb) {
     uError("WriteRaw:not use db");
     code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
@@ -1399,6 +1407,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
     return terrno;
   }
 
+  pRequest->syncQuery = true;
   rspObj.resIter = -1;
   rspObj.resType = RES_TYPE__TMQ;
 
@@ -1664,6 +1673,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
     return terrno;
   }
 
+  pRequest->syncQuery = true;
   rspObj.resIter = -1;
   rspObj.resType = RES_TYPE__TMQ_METADATA;
 
@@ -1739,23 +1749,29 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
       int32_t* lenTmp = taosArrayGet(rspObj.rsp.createTableLen, j);
 
       SDecoder           decoderTmp = {0};
-      SVCreateTbReq*     pCreateReq;
+      SVCreateTbReq      pCreateReq = {0};
 
       tDecoderInit(&decoderTmp, *dataTmp, *lenTmp);
-      if (tDecodeSVCreateTbReq(&decoderTmp, pCreateReq) < 0) {
+      if (tDecodeSVCreateTbReq(&decoderTmp, &pCreateReq) < 0) {
         tDecoderClear(&decoderTmp);
+        taosMemoryFreeClear(pCreateReq.comment);
+        taosArrayDestroy(pCreateReq.ctb.tagName);
         goto end;
       }
 
-      ASSERT (pCreateReq->type == TSDB_CHILD_TABLE);
-      if(strcmp(tbName, pCreateReq->name) == 0){
+      ASSERT (pCreateReq.type == TSDB_CHILD_TABLE);
+      if(strcmp(tbName, pCreateReq.name) == 0){
         schemaLen = *lenTmp;
         schemaData = *dataTmp;
-        strcpy(pName.tname, pCreateReq->ctb.name);
+        strcpy(pName.tname, pCreateReq.ctb.name);
         tDecoderClear(&decoderTmp);
+        taosMemoryFreeClear(pCreateReq.comment);
+        taosArrayDestroy(pCreateReq.ctb.tagName);
         break;
       }
       tDecoderClear(&decoderTmp);
+      taosMemoryFreeClear(pCreateReq.comment);
+      taosArrayDestroy(pCreateReq.ctb.tagName);
     }
 
     code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta);
@@ -1884,6 +1900,8 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
     subReq->length += sizeof(SSubmitBlk) + schemaLen + totalLen;
     subReq->numOfBlocks++;
     taosMemoryFreeClear(pTableMeta);
+    rspObj.resInfo.pRspMsg = NULL;
+    doFreeReqResultInfo(&rspObj.resInfo);
   }
 
   pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY);
diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 047191ec27..c67556faa0 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -1816,7 +1816,7 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) {
   } else if (TD_RES_TMQ_META(res)) {
     SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
     return pMetaRspObj->vgId;
-  } else if (TD_RES_TMQ_META(res)) {
+  } else if (TD_RES_TMQ_METADATA(res)) {
     SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)res;
     return pRspObj->vgId;
   } else {
@@ -1832,7 +1832,14 @@ const char* tmq_get_table_name(TAOS_RES* res) {
       return NULL;
     }
     return (const char*)taosArrayGetP(pRspObj->rsp.blockTbName, pRspObj->resIter);
-  }
+  } else if (TD_RES_TMQ_METADATA(res)) {
+    SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)res;
+      if (!pRspObj->rsp.withTbName || pRspObj->rsp.blockTbName == NULL || pRspObj->resIter < 0 ||
+          pRspObj->resIter >= pRspObj->rsp.blockNum) {
+        return NULL;
+      }
+      return (const char*)taosArrayGetP(pRspObj->rsp.blockTbName, pRspObj->resIter);
+    }
   return NULL;
 }
 
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index ea25094d10..111c8da8f6 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -6007,12 +6007,18 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp) {
 
 void tDeleteSTaosxRsp(STaosxRsp *pRsp) {
   taosArrayDestroy(pRsp->blockDataLen);
+  pRsp->blockDataLen = NULL;
   taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree);
+  pRsp->blockData = NULL;
   taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSSchemaWrapper);
+  pRsp->blockSchema = NULL;
   taosArrayDestroyP(pRsp->blockTbName, (FDelete)taosMemoryFree);
+  pRsp->blockTbName = NULL;
 
   taosArrayDestroy(pRsp->createTableLen);
+  pRsp->createTableLen = NULL;
   taosArrayDestroyP(pRsp->createTableReq, (FDelete)taosMemoryFree);
+  pRsp->createTableReq = NULL;
 }
 
 int32_t tEncodeSSingleDeleteReq(SEncoder *pEncoder, const SSingleDeleteReq *pReq) {
diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h
index 9632be1b24..4625cbfb09 100644
--- a/source/dnode/mnode/impl/inc/mndDef.h
+++ b/source/dnode/mnode/impl/inc/mndDef.h
@@ -539,7 +539,7 @@ typedef struct {
 } SMqConsumerEp;
 
 SMqConsumerEp* tCloneSMqConsumerEp(const SMqConsumerEp* pEp);
-void           tDeleteSMqConsumerEp(SMqConsumerEp* pEp);
+void           tDeleteSMqConsumerEp(void* pEp);
 int32_t        tEncodeSMqConsumerEp(void** buf, const SMqConsumerEp* pEp);
 void*          tDecodeSMqConsumerEp(const void* buf, SMqConsumerEp* pEp);
 
diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c
index 614348c209..168f3cd79b 100644
--- a/source/dnode/mnode/impl/src/mndConsumer.c
+++ b/source/dnode/mnode/impl/src/mndConsumer.c
@@ -197,11 +197,12 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) {
       SMqConsumerLostMsg *pLostMsg = rpcMallocCont(sizeof(SMqConsumerLostMsg));
 
       pLostMsg->consumerId = pConsumer->consumerId;
-      SRpcMsg *pRpcMsg = taosMemoryCalloc(1, sizeof(SRpcMsg));
-      pRpcMsg->msgType = TDMT_MND_MQ_CONSUMER_LOST;
-      pRpcMsg->pCont = pLostMsg;
-      pRpcMsg->contLen = sizeof(SMqConsumerLostMsg);
-      tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, pRpcMsg);
+      SRpcMsg pRpcMsg = {
+          .msgType = TDMT_MND_MQ_CONSUMER_LOST,
+          .pCont = pLostMsg,
+          .contLen = sizeof(SMqConsumerLostMsg),
+      };
+      tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &pRpcMsg);
     }
     if (status == MQ_CONSUMER_STATUS__LOST_REBD || status == MQ_CONSUMER_STATUS__READY) {
       // do nothing
@@ -280,11 +281,12 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
     SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg));
 
     pRecoverMsg->consumerId = consumerId;
-    SRpcMsg *pRpcMsg = taosMemoryCalloc(1, sizeof(SRpcMsg));
-    pRpcMsg->msgType = TDMT_MND_MQ_CONSUMER_RECOVER;
-    pRpcMsg->pCont = pRecoverMsg;
-    pRpcMsg->contLen = sizeof(SMqConsumerRecoverMsg);
-    tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, pRpcMsg);
+    SRpcMsg pRpcMsg = {
+        .msgType = TDMT_MND_MQ_CONSUMER_RECOVER,
+        .pCont = pRecoverMsg,
+        .contLen = sizeof(SMqConsumerRecoverMsg),
+    };
+    tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &pRpcMsg);
   }
 
   mndReleaseConsumer(pMnode, pConsumer);
@@ -318,11 +320,12 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
     SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg));
 
     pRecoverMsg->consumerId = consumerId;
-    SRpcMsg *pRpcMsg = taosMemoryCalloc(1, sizeof(SRpcMsg));
-    pRpcMsg->msgType = TDMT_MND_MQ_CONSUMER_RECOVER;
-    pRpcMsg->pCont = pRecoverMsg;
-    pRpcMsg->contLen = sizeof(SMqConsumerRecoverMsg);
-    tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, pRpcMsg);
+    SRpcMsg pRpcMsg = {
+        .msgType = TDMT_MND_MQ_CONSUMER_RECOVER,
+        .pCont = pRecoverMsg,
+        .contLen = sizeof(SMqConsumerRecoverMsg),
+    };
+    tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &pRpcMsg);
   }
 #endif
 
diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c
index e6f1a40993..67e599cfc7 100644
--- a/source/dnode/mnode/impl/src/mndDef.c
+++ b/source/dnode/mnode/impl/src/mndDef.c
@@ -343,8 +343,8 @@ SMqConsumerEp *tCloneSMqConsumerEp(const SMqConsumerEp *pConsumerEpOld) {
   return pConsumerEpNew;
 }
 
-void tDeleteSMqConsumerEp(SMqConsumerEp *pConsumerEp) {
-  //
+void tDeleteSMqConsumerEp(void *data) {
+  SMqConsumerEp *pConsumerEp = (SMqConsumerEp*)data;
   taosArrayDestroyP(pConsumerEp->vgs, (FDelete)tDeleteSMqVgEp);
 }
 
diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c
index a34569b08e..9b199ecd37 100644
--- a/source/dnode/vnode/src/meta/metaQuery.c
+++ b/source/dnode/vnode/src/meta/metaQuery.c
@@ -319,8 +319,12 @@ _query:
         pSchema = tCloneSSchemaWrapper(&meNew.stbEntry.schemaRow);
         tDecoderClear(&dcNew);
         tdbTbcClose(pCur);
+        tdbFree(pKey);
+        tdbFree(pVal);
         goto _exit;
       }
+      tdbFree(pKey);
+      tdbFree(pVal);
       tdbTbcClose(pCur);
     }
   } else if (me.type == TSDB_CHILD_TABLE) {
@@ -347,11 +351,13 @@ _query:
   tDecoderClear(&dc);
 
 _exit:
+  tDecoderClear(&dc);
   metaULock(pMeta);
   tdbFree(pData);
   return pSchema;
 
 _err:
+  tDecoderClear(&dc);
   metaULock(pMeta);
   tdbFree(pData);
   return NULL;
@@ -382,11 +388,9 @@ int metaTtlSmaller(SMeta *pMeta, uint64_t ttl, SArray *uidList) {
     }
     ttlKey = *(STtlIdxKey *)pKey;
     taosArrayPush(uidList, &ttlKey.uid);
+    tdbFree(pKey);
   }
   tdbTbcClose(pCur);
-
-  tdbFree(pKey);
-
   return 0;
 }
 
diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c
index 0edbd092e6..9fdbe50f88 100644
--- a/source/dnode/vnode/src/meta/metaSnapshot.c
+++ b/source/dnode/vnode/src/meta/metaSnapshot.c
@@ -353,6 +353,8 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t
     metaDebug("tmqsnap init idVersion uid:%" PRIi64 " version:%" PRIi64 " index:%d", *uid, idData->version, idData->index);
   }
 
+  tdbFree(pKey);
+  tdbFree(pVal);
   return TDB_CODE_SUCCESS;
 }
 
@@ -528,6 +530,7 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void **pBuf, int32_t *contLen, in
           }
         }
       }
+      taosArrayDestroy(pTagVals);
     }
 //    SIdInfo* sidInfo = (SIdInfo*)taosHashGet(ctx->idVersion, &me.ctbEntry.suid, sizeof(tb_uid_t));
 //    if(sidInfo->version >= idInfo->version){
diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index 92cf90d328..7768b484e9 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -1191,10 +1191,11 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
   const void    *pTagData = NULL;  //
   int32_t        nTagData = 0;
   SDecoder       dc = {0};
-
+  int32_t        ret = 0;
   // get super table
   if (tdbTbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData) != 0) {
-    return -1;
+    ret = -1;
+    goto end;
   }
   tbDbKey.uid = pCtbEntry->ctbEntry.suid;
   tbDbKey.version = ((SUidIdxVal *)pData)[0].version;
@@ -1220,17 +1221,20 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
     // nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len;
     pTagData = pCtbEntry->ctbEntry.pTags;
     nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len;
-    return metaSaveJsonVarToIdx(pMeta, pCtbEntry, pTagColumn);
+    ret = metaSaveJsonVarToIdx(pMeta, pCtbEntry, pTagColumn);
+    goto end;
   }
   if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type,
                           pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) {
-    return -1;
+    ret = -1;
+    goto end;
   }
   tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn);
+end:
   metaDestroyTagIdxKey(pTagIdxKey);
   tDecoderClear(&dc);
   tdbFree(pData);
-  return 0;
+  return ret;
 }
 
 static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) {
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index f9c2757c37..73f9e2f679 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -51,6 +51,20 @@ void tqCleanUp() {
   }
 }
 
+static void destroySTqHandle(void* data) {
+  STqHandle* pData = (STqHandle*)data;
+  qDestroyTask(pData->execHandle.task);
+  if (pData->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
+  } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__DB) {
+    tqCloseReader(pData->execHandle.pExecReader);
+    walCloseReader(pData->pWalReader);
+    taosHashCleanup(pData->execHandle.execDb.pFilterOutTbUid);
+  } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__TABLE){
+    walCloseReader(pData->pWalReader);
+    tqCloseReader(pData->execHandle.pExecReader);
+  }
+}
+
 STQ* tqOpen(const char* path, SVnode* pVnode) {
   STQ* pTq = taosMemoryCalloc(1, sizeof(STQ));
   if (pTq == NULL) {
@@ -62,6 +76,8 @@ STQ* tqOpen(const char* path, SVnode* pVnode) {
 
   pTq->pHandle = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
 
+  taosHashSetFreeFp(pTq->pHandle, destroySTqHandle);
+
   pTq->pPushMgr = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
 
   pTq->pCheckInfo = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
@@ -520,6 +536,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
     int64_t fetchVer = fetchOffsetNew.version + 1;
     pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048);
     if (pCkHead == NULL) {
+      tDeleteSTaosxRsp(&taosxRsp);
       return -1;
     }
 
@@ -580,14 +597,17 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
         if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) {
           code = -1;
           taosMemoryFree(pCkHead);
+          tDeleteSTaosxRsp(&taosxRsp);
           return code;
         }
         code = 0;
         if (pCkHead) taosMemoryFree(pCkHead);
+        tDeleteSTaosxRsp(&taosxRsp);
         return code;
       }
     }
   }
+  tDeleteSTaosxRsp(&taosxRsp);
   return 0;
 }
 
diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c
index 5c1d5d65b4..7097591c35 100644
--- a/source/dnode/vnode/src/tq/tqOffset.c
+++ b/source/dnode/vnode/src/tq/tqOffset.c
@@ -145,8 +145,10 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore) {
       ASSERT(0);
       tqError("write offset incomplete, len %d, write len %" PRId64, bodyLen, writeLen);
       taosHashCancelIterate(pStore->pHash, pIter);
+      taosMemoryFree(buf);
       return -1;
     }
+    taosMemoryFree(buf);
   }
   // close and rename file
   taosCloseFile(&pFile);
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 815d6a71ef..397d8460c7 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -3645,6 +3645,11 @@ static void cleanupTableSchemaInfo(SSchemaInfo* pSchemaInfo) {
   tDeleteSSchemaWrapper(pSchemaInfo->qsw);
 }
 
+static void cleanupStreamInfo(SStreamTaskInfo* pStreamInfo) {
+  tDeleteSSchemaWrapper(pStreamInfo->schema);
+  cleanupQueryTableDataCond(&pStreamInfo->tableCond);
+}
+
 static int32_t sortTableGroup(STableListInfo* pTableListInfo) {
   taosArrayClear(pTableListInfo->pGroupList);
   SArray* sortSupport = taosArrayInit(16, sizeof(uint64_t));
@@ -4338,6 +4343,7 @@ void doDestroyTask(SExecTaskInfo* pTaskInfo) {
   doDestroyTableList(&pTaskInfo->tableqinfoList);
   destroyOperatorInfo(pTaskInfo->pRoot);
   cleanupTableSchemaInfo(&pTaskInfo->schemaInfo);
+  cleanupStreamInfo(&pTaskInfo->streamInfo);
 
   nodesDestroyNode((SNode*)pTaskInfo->pSubplan);
 
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index ad9cd1ffe7..0df70b4818 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -1769,6 +1769,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
       qDebug("tmqsnap change get data uid:%ld", mtInfo.uid);
       qStreamPrepareScan(pTaskInfo, &pTaskInfo->streamInfo.prepareStatus, pInfo->sContext->subType);
     }
+    tDeleteSSchemaWrapper(mtInfo.schema);
     qDebug("tmqsnap stream scan tsdb return null");
     return NULL;
   } else if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_META) {
diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c
index ecc25e9c2d..c38c9ac1cf 100644
--- a/source/libs/qworker/src/qwMsg.c
+++ b/source/libs/qworker/src/qwMsg.c
@@ -632,6 +632,7 @@ int32_t qWorkerProcessDeleteMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, SD
 
   QW_ERR_JRET(qwProcessDelete(QW_FPARAMS(), &qwMsg, pRes));
 
+  taosMemoryFreeClear(req.msg);
   QW_SCH_TASK_DLOG("processDelete end, node:%p", node);
 
 _return:
diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c
index 038723955f..3a598ba98b 100644
--- a/utils/test/c/tmq_taosx_ci.c
+++ b/utils/test/c/tmq_taosx_ci.c
@@ -68,6 +68,7 @@ static void msg_process(TAOS_RES* msg) {
 
   tmq_raw_data raw = {0};
   tmq_get_raw(msg, &raw);
+  printf("write raw data type: %d\n", raw.raw_type);
   int32_t ret = tmq_write_raw(pConn, raw);
   printf("write raw data: %s\n", tmq_err2str(ret));
   tmq_free_raw(raw);

From 512aa3d8d71dddbe48e5d50b219389ca85c3a1f6 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Fri, 16 Sep 2022 17:36:31 +0800
Subject: [PATCH 16/57] more code

---
 source/dnode/vnode/src/tsdb/tsdbCompress.c | 127 +++++++++++++++++++--
 1 file changed, 119 insertions(+), 8 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbCompress.c b/source/dnode/vnode/src/tsdb/tsdbCompress.c
index 0b1a70fba7..55d94e3b37 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCompress.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCompress.c
@@ -36,13 +36,17 @@ typedef struct {
     struct {
       /* data */
     };
-    // Binary ----
-    struct {
-      /* data */
-    };
     // Float ----
     struct {
-      /* data */
+      int32_t  f_n;
+      uint32_t f_prev;
+      uint8_t *f_flag_p;
+    };
+    // Double ----
+    struct {
+      int32_t  d_n;
+      uint64_t d_prev;
+      uint8_t *d_flag_p;
     };
     // Bool ----
     struct {
@@ -94,7 +98,7 @@ static int32_t tCompSetCopyMode(SCompressor *pCmprsor) {
         vZigzag |= (((uint64_t)pCmprsor->aBuf[0][n]) << (sizeof(int64_t) * i));
         n++;
       }
-      int64_t delta_of_delta = ZIGZAGD(int64_t, vZigzag);
+      delta_of_delta = ZIGZAGD(int64_t, vZigzag);
       delPrev = delta_of_delta + delPrev;
       valPrev = delPrev + valPrev;
     }
@@ -215,9 +219,116 @@ _exit:
 }
 
 // Float =====================================================
-static int32_t tCompFloat() {
+static int32_t tCompFloat(SCompressor *pCmprsor, float f) {
   int32_t code = 0;
-  // TODO
+
+  union {
+    float    f;
+    uint32_t u;
+  } val = {.f = f};
+
+  uint32_t diff = val.u ^ pCmprsor->f_prev;
+  pCmprsor->f_prev = val.u;
+
+  int32_t clz, ctz;
+  if (diff) {
+    clz = BUILDIN_CLZ(diff);
+    ctz = BUILDIN_CTZ(diff);
+  } else {
+    clz = sizeof(uint32_t);
+    ctz = sizeof(uint32_t);
+  }
+
+  if (pCmprsor->f_n & 0x1 == 0) {
+    code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 9 /* sizeof(float) * 2 + 1 */);
+    if (code) return code;
+
+    pCmprsor->f_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]];
+    pCmprsor->nBuf[0]++;
+
+    if (clz < ctz) {
+      uint8_t nBytes = sizeof(uint32_t) - ctz / BITS_PER_BYTE;
+      pCmprsor->f_flag_p[0] = (0x08 | (nBytes - 1));
+      diff >>= (32 - nBytes * BITS_PER_BYTE);
+    } else {
+      uint8_t nBytes = sizeof(uint32_t) - clz / BITS_PER_BYTE;
+      pCmprsor->f_flag_p[0] = nBytes - 1;
+    }
+  } else {
+    if (clz < ctz) {
+      uint8_t nBytes = sizeof(uint32_t) - ctz / BITS_PER_BYTE;
+      pCmprsor->f_flag_p[0] |= ((0x08 | (nBytes - 1)) << 4);
+      diff >>= (32 - nBytes * BITS_PER_BYTE);
+    } else {
+      uint8_t nBytes = sizeof(uint32_t) - clz / BITS_PER_BYTE;
+      pCmprsor->f_flag_p[0] |= ((nBytes - 1) << 4);
+    }
+  }
+
+  while (diff) {
+    pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (diff & 0xff);
+    pCmprsor->nBuf[0]++;
+    diff >>= BITS_PER_BYTE;
+  }
+
+  pCmprsor->f_n++;
+  return code;
+}
+
+// Double =====================================================
+static int32_t tCompDouble(SCompressor *pCmprsor, double d) {
+  int32_t code = 0;
+
+  union {
+    double   d;
+    uint64_t u;
+  } val = {.d = d};
+
+  uint64_t diff = val.u ^ pCmprsor->d_prev;
+  pCmprsor->d_prev = val.u;
+
+  int32_t clz, ctz;
+  if (diff) {
+    clz = BUILDIN_CLZ(diff);
+    ctz = BUILDIN_CTZ(diff);
+  } else {
+    clz = sizeof(uint64_t);
+    ctz = sizeof(uint64_t);
+  }
+
+  if (pCmprsor->d_n & 0x1 == 0) {
+    code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17 /* sizeof(double) * 2 + 1 */);
+    if (code) return code;
+
+    pCmprsor->d_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]];
+    pCmprsor->nBuf[0]++;
+
+    if (clz < ctz) {
+      uint8_t nBytes = sizeof(uint64_t) - ctz / BITS_PER_BYTE;
+      pCmprsor->d_flag_p[0] = (0x08 | (nBytes - 1));
+      diff >>= (64 - nBytes * BITS_PER_BYTE);
+    } else {
+      uint8_t nBytes = sizeof(uint64_t) - clz / BITS_PER_BYTE;
+      pCmprsor->d_flag_p[0] = nBytes - 1;
+    }
+  } else {
+    if (clz < ctz) {
+      uint8_t nBytes = sizeof(uint64_t) - ctz / BITS_PER_BYTE;
+      pCmprsor->d_flag_p[0] |= ((0x08 | (nBytes - 1)) << 4);
+      diff >>= (64 - nBytes * BITS_PER_BYTE);
+    } else {
+      uint8_t nBytes = sizeof(uint64_t) - clz / BITS_PER_BYTE;
+      pCmprsor->d_flag_p[0] |= ((nBytes - 1) << 4);
+    }
+  }
+
+  while (diff) {
+    pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (diff & 0xff);
+    pCmprsor->nBuf[0]++;
+    diff >>= BITS_PER_BYTE;
+  }
+
+  pCmprsor->d_n++;
   return code;
 }
 

From 5ffaaebad6aa82f2729a3875ba374c809de46b4b Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Fri, 16 Sep 2022 17:46:50 +0800
Subject: [PATCH 17/57] more code

---
 source/dnode/vnode/src/tsdb/tsdbCompress.c | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbCompress.c b/source/dnode/vnode/src/tsdb/tsdbCompress.c
index 55d94e3b37..616da35f1f 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCompress.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCompress.c
@@ -34,7 +34,8 @@ typedef struct {
     };
     // Integer ----
     struct {
-      /* data */
+      int8_t  i_copy;
+      int32_t i_n;
     };
     // Float ----
     struct {
@@ -52,6 +53,10 @@ typedef struct {
     struct {
       int32_t bool_n;
     };
+    // Binary ----
+    struct {
+      int32_t b_n;
+    };
   };
 } SCompressor;
 
@@ -337,9 +342,13 @@ static int32_t tCompBinary(SCompressor *pCmprsor, const uint8_t *pData, int32_t
   int32_t code = 0;
 
   if (nData) {
+    code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + nData);
+    if (code) return code;
+
     memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], pData, nData);
     pCmprsor->nBuf[0] += nData;
   }
+  pCmprsor->b_n++;
 
   return code;
 }
@@ -419,6 +428,9 @@ int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) {
       pCmprsor->bool_n = 0;
       pCmprsor->aBuf[0][0] = 0;
       break;
+    case TSDB_DATA_TYPE_BINARY:
+      pCmprsor->b_n = 0;
+      break;
     default:
       break;
   }

From 24615a08da417f56929b64a272431538ef8f8d86 Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Fri, 16 Sep 2022 18:21:24 +0800
Subject: [PATCH 18/57] feat: update taos-tools 7d7b3ce for 3.0

---
 cmake/taostools_CMakeLists.txt.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in
index 6fb2a69847..980a15e462 100644
--- a/cmake/taostools_CMakeLists.txt.in
+++ b/cmake/taostools_CMakeLists.txt.in
@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG e7270c9
+        GIT_TAG 7d7b3ce
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

From 0ff451dcf3dff8cffd1758ae1d6533680cfa9509 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Fri, 16 Sep 2022 18:50:49 +0800
Subject: [PATCH 19/57] more code

---
 source/dnode/vnode/src/tsdb/tsdbCompress.c | 85 ++++++++++++++++------
 1 file changed, 61 insertions(+), 24 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbCompress.c b/source/dnode/vnode/src/tsdb/tsdbCompress.c
index 616da35f1f..31e24ca751 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCompress.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCompress.c
@@ -36,6 +36,9 @@ typedef struct {
     struct {
       int8_t  i_copy;
       int32_t i_n;
+      int64_t i_prev;
+      int32_t i_selector;
+      int32_t i_nele;
     };
     // Float ----
     struct {
@@ -188,38 +191,72 @@ _copy_exit:
 
 // Integer =====================================================
 #define SIMPLE8B_MAX ((uint64_t)1152921504606846974LL)
+static const char    bit_per_integer[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60};
+static const int32_t selector_to_elems[] = {240, 120, 60, 30, 20, 15, 12, 10, 8, 7, 6, 5, 4, 3, 2, 1};
+static const char    bit_to_selector[] = {0,  2,  3,  4,  5,  6,  7,  8,  9,  10, 10, 11, 11, 12, 12, 12,
+                                          13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 15,
+                                          15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
+                                          15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15};
 
 static int32_t tCompI64(SCompressor *pCmprsor, int64_t val) {
   int32_t code = 0;
-#if 0
 
-  // raw copy
-  if (pCmprsor->rawCopy) {
-    memcpy(pCmprsor->pBuf + pCmprsor->nBuf, &val, sizeof(val));
-    pCmprsor->nBuf += sizeof(val);
-    pCmprsor->nVal++;
-    goto _exit;
+  if (pCmprsor->i_copy == 1) goto _copy_cmpr;
+
+  if (!I64_SAFE_ADD(val, pCmprsor->i_prev)) {
+    // TODO
+    goto _copy_cmpr;
   }
 
-  if (!I64_SAFE_ADD(val, pCmprsor->prevVal)) {
-    pCmprsor->rawCopy = 1;
-    // TODO: decompress and copy
-    pCmprsor->nVal++;
-    goto _exit;
+  int64_t  diff = val - pCmprsor->i_prev;
+  uint64_t vZigzag = ZIGZAGE(int64_t, diff);
+
+  if (vZigzag >= SIMPLE8B_MAX) {
+    // TODO
+    goto _copy_cmpr;
   }
 
-  int64_t diff = val - pCmprsor->prevVal;
-  uint8_t zigzag = ZIGZAGE(int64_t, diff);
-
-  if (zigzag >= SIMPLE8B_MAX) {
-    pCmprsor->rawCopy = 1;
-    // TODO: decompress and copy
-    pCmprsor->nVal++;
-    goto _exit;
+  int64_t nBit;
+  if (vZigzag) {
+    nBit = 64 - BUILDIN_CLZL(vZigzag);
+  } else {
+    nBit = 0;
   }
 
-_exit:
-#endif
+  if (pCmprsor->i_nele + 1 <= selector_to_elems[pCmprsor->i_selector] &&
+      pCmprsor->i_nele + 1 <= selector_to_elems[bit_to_selector[nBit]]) {
+    if (pCmprsor->i_selector < bit_to_selector[nBit]) {
+      pCmprsor->i_selector = bit_to_selector[nBit];
+    }
+    pCmprsor->i_nele++;
+  } else {
+    while (pCmprsor->i_nele < selector_to_elems[pCmprsor->i_selector]) {
+      pCmprsor->i_selector++;
+    }
+    pCmprsor->i_nele = selector_to_elems[pCmprsor->i_selector];
+
+    code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(uint64_t));
+    if (code) return code;
+
+    uint64_t *bp = pCmprsor->aBuf[0] + pCmprsor->nBuf[0];
+    pCmprsor->nBuf[0] += sizeof(uint64_t);
+    bp[0] = pCmprsor->i_selector;
+    for (int32_t iVal = 0; iVal < pCmprsor->i_nele; iVal++) {
+      /* code */
+    }
+
+    // reset and continue
+  }
+
+  return code;
+
+_copy_cmpr:
+  code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + tDataTypes[pCmprsor->type].bytes);
+  if (code) return code;
+
+  memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], NULL /* todo */, tDataTypes[pCmprsor->type].bytes);
+  pCmprsor->nBuf[0] += tDataTypes[pCmprsor->type].bytes;
+
   return code;
 }
 
@@ -294,8 +331,8 @@ static int32_t tCompDouble(SCompressor *pCmprsor, double d) {
 
   int32_t clz, ctz;
   if (diff) {
-    clz = BUILDIN_CLZ(diff);
-    ctz = BUILDIN_CTZ(diff);
+    clz = BUILDIN_CLZL(diff);
+    ctz = BUILDIN_CTZL(diff);
   } else {
     clz = sizeof(uint64_t);
     ctz = sizeof(uint64_t);

From a54f71a03c3a46bf68525c0731d260659dedb53e Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Fri, 16 Sep 2022 18:57:33 +0800
Subject: [PATCH 20/57] fix:error in auto create table for taosX

---
 source/dnode/vnode/src/tq/tqRead.c      | 15 +++++++++++++++
 source/dnode/vnode/src/tq/tqSink.c      |  4 +++-
 source/libs/executor/src/executil.c     |  2 +-
 source/libs/executor/src/executor.c     |  1 +
 source/libs/executor/src/executorimpl.c |  1 -
 5 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c
index 375130fa2c..8e5f328efa 100644
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@@ -80,6 +80,13 @@ bool isValValidForTable(STqHandle* pHandle, SWalCont *pHead){
       void* buf = taosMemoryMalloc(tlen);
       if (NULL == buf) {
         taosArrayDestroy(reqNew.pArray);
+        for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
+          pCreateReq = req.pReqs + iReq;
+          taosMemoryFreeClear(pCreateReq->comment);
+          if (pCreateReq->type == TSDB_CHILD_TABLE) {
+            taosArrayDestroy(pCreateReq->ctb.tagName);
+          }
+        }
         goto end;
       }
       SEncoder coderNew = {0};
@@ -91,6 +98,14 @@ bool isValValidForTable(STqHandle* pHandle, SWalCont *pHead){
       taosMemoryFree(buf);
       taosArrayDestroy(reqNew.pArray);
     }
+
+    for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
+      pCreateReq = req.pReqs + iReq;
+      taosMemoryFreeClear(pCreateReq->comment);
+      if (pCreateReq->type == TSDB_CHILD_TABLE) {
+        taosArrayDestroy(pCreateReq->ctb.tagName);
+      }
+    }
   } else if (msgType == TDMT_VND_ALTER_TABLE) {
     SVAlterTbReq   req = {0};
 
diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c
index f1538494f9..5ddca1f37b 100644
--- a/source/dnode/vnode/src/tq/tqSink.c
+++ b/source/dnode/vnode/src/tq/tqSink.c
@@ -90,7 +90,9 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem
       }
 
       SArray *tagName = taosArrayInit(1, TSDB_COL_NAME_LEN);
-      taosArrayPush(tagName, "group_id");
+      char     tagNameStr[TSDB_COL_NAME_LEN] = {0};
+      strcpy(tagNameStr, "group_id");
+      taosArrayPush(tagName, tagNameStr);
 
 //      STag* pTag = NULL;
 //      taosArrayClear(tagArray);
diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index 0c1728386b..23c4857ed1 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -1324,7 +1324,7 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi
   return TSDB_CODE_SUCCESS;
 }
 
-void cleanupQueryTableDataCond(SQueryTableDataCond* pCond) { taosMemoryFree(pCond->colList); }
+void cleanupQueryTableDataCond(SQueryTableDataCond* pCond) { taosMemoryFreeClear(pCond->colList); }
 
 int32_t convertFillType(int32_t mode) {
   int32_t type = TSDB_FILL_NONE;
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index 95415e1113..9b48b2865e 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -858,6 +858,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
     tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pTaskInfo->tableqinfoList.pTableList,
                    &pInfo->dataReader, NULL);
 
+    cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
     strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName);
     tDeleteSSchemaWrapper(pTaskInfo->streamInfo.schema);
     pTaskInfo->streamInfo.schema = mtInfo.schema;
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 397d8460c7..e55654f6ca 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -3647,7 +3647,6 @@ static void cleanupTableSchemaInfo(SSchemaInfo* pSchemaInfo) {
 
 static void cleanupStreamInfo(SStreamTaskInfo* pStreamInfo) {
   tDeleteSSchemaWrapper(pStreamInfo->schema);
-  cleanupQueryTableDataCond(&pStreamInfo->tableCond);
 }
 
 static int32_t sortTableGroup(STableListInfo* pTableListInfo) {

From 0446d7075aa731be76591a7c73c54c39ee88e8a4 Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Sun, 18 Sep 2022 02:36:07 +0000
Subject: [PATCH 21/57] feat: update taos-tools 125c77a for 3.0

---
 cmake/taostools_CMakeLists.txt.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in
index 980a15e462..6a3439ada9 100644
--- a/cmake/taostools_CMakeLists.txt.in
+++ b/cmake/taostools_CMakeLists.txt.in
@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG 7d7b3ce
+        GIT_TAG 125c77a
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

From 7a21781a5581eac7dab679402e9614035cc51e39 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 19 Sep 2022 10:08:08 +0800
Subject: [PATCH 22/57] refact code

---
 source/dnode/vnode/CMakeLists.txt          |   1 -
 source/dnode/vnode/src/tsdb/tsdbCompress.c | 510 ---------------------
 source/util/src/tcompression.c             | 497 ++++++++++++++++++++
 3 files changed, 497 insertions(+), 511 deletions(-)
 delete mode 100644 source/dnode/vnode/src/tsdb/tsdbCompress.c

diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt
index 7a99d26683..1f7a059ffc 100644
--- a/source/dnode/vnode/CMakeLists.txt
+++ b/source/dnode/vnode/CMakeLists.txt
@@ -51,7 +51,6 @@ target_sources(
     "src/tsdb/tsdbCacheRead.c"
     "src/tsdb/tsdbRetention.c"
     "src/tsdb/tsdbDiskData.c"
-    "src/tsdb/tsdbCompress.c"
     "src/tsdb/tsdbCompact.c"
     "src/tsdb/tsdbMergeTree.c"
 
diff --git a/source/dnode/vnode/src/tsdb/tsdbCompress.c b/source/dnode/vnode/src/tsdb/tsdbCompress.c
deleted file mode 100644
index 31e24ca751..0000000000
--- a/source/dnode/vnode/src/tsdb/tsdbCompress.c
+++ /dev/null
@@ -1,510 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "lz4.h"
-#include "tsdb.h"
-
-#define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (b)) || ((a) < 0 && (b) >= INT64_MIN - (a)))
-
-typedef struct {
-  int8_t   type;
-  int8_t   cmprAlg;
-  uint8_t *aBuf[2];
-  int64_t  nBuf[2];
-  union {
-    // Timestamp ----
-    struct {
-      int8_t   ts_copy;
-      int32_t  ts_n;
-      int64_t  ts_prev_val;
-      int64_t  ts_prev_delta;
-      uint8_t *ts_flag_p;
-    };
-    // Integer ----
-    struct {
-      int8_t  i_copy;
-      int32_t i_n;
-      int64_t i_prev;
-      int32_t i_selector;
-      int32_t i_nele;
-    };
-    // Float ----
-    struct {
-      int32_t  f_n;
-      uint32_t f_prev;
-      uint8_t *f_flag_p;
-    };
-    // Double ----
-    struct {
-      int32_t  d_n;
-      uint64_t d_prev;
-      uint8_t *d_flag_p;
-    };
-    // Bool ----
-    struct {
-      int32_t bool_n;
-    };
-    // Binary ----
-    struct {
-      int32_t b_n;
-    };
-  };
-} SCompressor;
-
-// Timestamp =====================================================
-static int32_t tCompSetCopyMode(SCompressor *pCmprsor) {
-  int32_t code = 0;
-
-  if (pCmprsor->ts_n) {
-    code = tRealloc(&pCmprsor->aBuf[1], sizeof(int64_t) * (pCmprsor->ts_n + 1));
-    if (code) return code;
-    pCmprsor->nBuf[1] = 1;
-
-    int64_t  n = 1;
-    int64_t  valPrev;
-    int64_t  delPrev;
-    uint64_t vZigzag;
-    while (n < pCmprsor->nBuf[0]) {
-      uint8_t n1 = pCmprsor->aBuf[0][0] & 0xf;
-      uint8_t n2 = pCmprsor->aBuf[0][0] >> 4;
-
-      n++;
-
-      vZigzag = 0;
-      for (uint8_t i = 0; i < n1; i++) {
-        vZigzag |= (((uint64_t)pCmprsor->aBuf[0][n]) << (sizeof(int64_t) * i));
-        n++;
-      }
-      int64_t delta_of_delta = ZIGZAGD(int64_t, vZigzag);
-      if (n == 2) {
-        delPrev = 0;
-        valPrev = delta_of_delta;
-      } else {
-        delPrev = delta_of_delta + delPrev;
-        valPrev = delPrev + valPrev;
-      }
-
-      memcpy(pCmprsor->aBuf[1] + pCmprsor->nBuf[1], &valPrev, sizeof(int64_t));
-      pCmprsor->nBuf[1] += sizeof(int64_t);
-
-      if (n >= pCmprsor->nBuf[0]) break;
-
-      vZigzag = 0;
-      for (uint8_t i = 0; i < n2; i++) {
-        vZigzag |= (((uint64_t)pCmprsor->aBuf[0][n]) << (sizeof(int64_t) * i));
-        n++;
-      }
-      delta_of_delta = ZIGZAGD(int64_t, vZigzag);
-      delPrev = delta_of_delta + delPrev;
-      valPrev = delPrev + valPrev;
-    }
-
-    uint8_t *pBuf = pCmprsor->aBuf[0];
-    pCmprsor->aBuf[0] = pCmprsor->aBuf[1];
-    pCmprsor->aBuf[1] = pBuf;
-    pCmprsor->nBuf[0] = pCmprsor->nBuf[1];
-  } else {
-    // TODO
-  }
-
-  pCmprsor->aBuf[0][0] = 0;
-  pCmprsor->ts_copy = 1;
-
-  return code;
-}
-static int32_t tCompTimestamp(SCompressor *pCmprsor, TSKEY ts) {
-  int32_t code = 0;
-
-  if (pCmprsor->ts_n == 0) {
-    pCmprsor->ts_prev_val = ts;
-    pCmprsor->ts_prev_delta = -ts;
-  }
-
-  if (pCmprsor->ts_copy) goto _copy_exit;
-
-  if (!I64_SAFE_ADD(ts, -pCmprsor->ts_prev_val)) {
-    code = tCompSetCopyMode(pCmprsor);
-    if (code) return code;
-    goto _copy_exit;
-  }
-
-  int64_t delta = ts - pCmprsor->ts_prev_val;
-
-  if (!I64_SAFE_ADD(delta, -pCmprsor->ts_prev_delta)) {
-    code = tCompSetCopyMode(pCmprsor);
-    if (code) return code;
-    goto _copy_exit;
-  }
-
-  int64_t  delta_of_delta = delta - pCmprsor->ts_prev_delta;
-  uint64_t zigzag_value = ZIGZAGE(int64_t, delta_of_delta);
-
-  pCmprsor->ts_prev_val = ts;
-  pCmprsor->ts_prev_delta = delta;
-
-  if (pCmprsor->ts_n & 0x1 == 0) {
-    code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17 /*sizeof(int64_t) * 2 + 1*/);
-    if (code) return code;
-
-    pCmprsor->ts_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]];
-    pCmprsor->nBuf[0]++;
-    pCmprsor->ts_flag_p[0] = 0;
-
-    while (zigzag_value) {
-      pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (zigzag_value & 0xff);
-      pCmprsor->nBuf[0]++;
-      pCmprsor->ts_flag_p[0]++;
-    }
-  } else {
-    while (zigzag_value) {
-      pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (zigzag_value & 0xff);
-      pCmprsor->nBuf[0]++;
-      pCmprsor->ts_flag_p += (uint8_t)0x10;
-    }
-  }
-
-  pCmprsor->ts_n++;
-  return code;
-
-_copy_exit:
-  code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(int64_t));
-  if (code) return code;
-
-  memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], &ts, sizeof(ts));
-  pCmprsor->nBuf[0] += sizeof(ts);
-
-  pCmprsor->ts_n++;
-  return code;
-}
-
-// Integer =====================================================
-#define SIMPLE8B_MAX ((uint64_t)1152921504606846974LL)
-static const char    bit_per_integer[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60};
-static const int32_t selector_to_elems[] = {240, 120, 60, 30, 20, 15, 12, 10, 8, 7, 6, 5, 4, 3, 2, 1};
-static const char    bit_to_selector[] = {0,  2,  3,  4,  5,  6,  7,  8,  9,  10, 10, 11, 11, 12, 12, 12,
-                                          13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 15,
-                                          15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
-                                          15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15};
-
-static int32_t tCompI64(SCompressor *pCmprsor, int64_t val) {
-  int32_t code = 0;
-
-  if (pCmprsor->i_copy == 1) goto _copy_cmpr;
-
-  if (!I64_SAFE_ADD(val, pCmprsor->i_prev)) {
-    // TODO
-    goto _copy_cmpr;
-  }
-
-  int64_t  diff = val - pCmprsor->i_prev;
-  uint64_t vZigzag = ZIGZAGE(int64_t, diff);
-
-  if (vZigzag >= SIMPLE8B_MAX) {
-    // TODO
-    goto _copy_cmpr;
-  }
-
-  int64_t nBit;
-  if (vZigzag) {
-    nBit = 64 - BUILDIN_CLZL(vZigzag);
-  } else {
-    nBit = 0;
-  }
-
-  if (pCmprsor->i_nele + 1 <= selector_to_elems[pCmprsor->i_selector] &&
-      pCmprsor->i_nele + 1 <= selector_to_elems[bit_to_selector[nBit]]) {
-    if (pCmprsor->i_selector < bit_to_selector[nBit]) {
-      pCmprsor->i_selector = bit_to_selector[nBit];
-    }
-    pCmprsor->i_nele++;
-  } else {
-    while (pCmprsor->i_nele < selector_to_elems[pCmprsor->i_selector]) {
-      pCmprsor->i_selector++;
-    }
-    pCmprsor->i_nele = selector_to_elems[pCmprsor->i_selector];
-
-    code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(uint64_t));
-    if (code) return code;
-
-    uint64_t *bp = pCmprsor->aBuf[0] + pCmprsor->nBuf[0];
-    pCmprsor->nBuf[0] += sizeof(uint64_t);
-    bp[0] = pCmprsor->i_selector;
-    for (int32_t iVal = 0; iVal < pCmprsor->i_nele; iVal++) {
-      /* code */
-    }
-
-    // reset and continue
-  }
-
-  return code;
-
-_copy_cmpr:
-  code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + tDataTypes[pCmprsor->type].bytes);
-  if (code) return code;
-
-  memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], NULL /* todo */, tDataTypes[pCmprsor->type].bytes);
-  pCmprsor->nBuf[0] += tDataTypes[pCmprsor->type].bytes;
-
-  return code;
-}
-
-// Float =====================================================
-static int32_t tCompFloat(SCompressor *pCmprsor, float f) {
-  int32_t code = 0;
-
-  union {
-    float    f;
-    uint32_t u;
-  } val = {.f = f};
-
-  uint32_t diff = val.u ^ pCmprsor->f_prev;
-  pCmprsor->f_prev = val.u;
-
-  int32_t clz, ctz;
-  if (diff) {
-    clz = BUILDIN_CLZ(diff);
-    ctz = BUILDIN_CTZ(diff);
-  } else {
-    clz = sizeof(uint32_t);
-    ctz = sizeof(uint32_t);
-  }
-
-  if (pCmprsor->f_n & 0x1 == 0) {
-    code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 9 /* sizeof(float) * 2 + 1 */);
-    if (code) return code;
-
-    pCmprsor->f_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]];
-    pCmprsor->nBuf[0]++;
-
-    if (clz < ctz) {
-      uint8_t nBytes = sizeof(uint32_t) - ctz / BITS_PER_BYTE;
-      pCmprsor->f_flag_p[0] = (0x08 | (nBytes - 1));
-      diff >>= (32 - nBytes * BITS_PER_BYTE);
-    } else {
-      uint8_t nBytes = sizeof(uint32_t) - clz / BITS_PER_BYTE;
-      pCmprsor->f_flag_p[0] = nBytes - 1;
-    }
-  } else {
-    if (clz < ctz) {
-      uint8_t nBytes = sizeof(uint32_t) - ctz / BITS_PER_BYTE;
-      pCmprsor->f_flag_p[0] |= ((0x08 | (nBytes - 1)) << 4);
-      diff >>= (32 - nBytes * BITS_PER_BYTE);
-    } else {
-      uint8_t nBytes = sizeof(uint32_t) - clz / BITS_PER_BYTE;
-      pCmprsor->f_flag_p[0] |= ((nBytes - 1) << 4);
-    }
-  }
-
-  while (diff) {
-    pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (diff & 0xff);
-    pCmprsor->nBuf[0]++;
-    diff >>= BITS_PER_BYTE;
-  }
-
-  pCmprsor->f_n++;
-  return code;
-}
-
-// Double =====================================================
-static int32_t tCompDouble(SCompressor *pCmprsor, double d) {
-  int32_t code = 0;
-
-  union {
-    double   d;
-    uint64_t u;
-  } val = {.d = d};
-
-  uint64_t diff = val.u ^ pCmprsor->d_prev;
-  pCmprsor->d_prev = val.u;
-
-  int32_t clz, ctz;
-  if (diff) {
-    clz = BUILDIN_CLZL(diff);
-    ctz = BUILDIN_CTZL(diff);
-  } else {
-    clz = sizeof(uint64_t);
-    ctz = sizeof(uint64_t);
-  }
-
-  if (pCmprsor->d_n & 0x1 == 0) {
-    code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17 /* sizeof(double) * 2 + 1 */);
-    if (code) return code;
-
-    pCmprsor->d_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]];
-    pCmprsor->nBuf[0]++;
-
-    if (clz < ctz) {
-      uint8_t nBytes = sizeof(uint64_t) - ctz / BITS_PER_BYTE;
-      pCmprsor->d_flag_p[0] = (0x08 | (nBytes - 1));
-      diff >>= (64 - nBytes * BITS_PER_BYTE);
-    } else {
-      uint8_t nBytes = sizeof(uint64_t) - clz / BITS_PER_BYTE;
-      pCmprsor->d_flag_p[0] = nBytes - 1;
-    }
-  } else {
-    if (clz < ctz) {
-      uint8_t nBytes = sizeof(uint64_t) - ctz / BITS_PER_BYTE;
-      pCmprsor->d_flag_p[0] |= ((0x08 | (nBytes - 1)) << 4);
-      diff >>= (64 - nBytes * BITS_PER_BYTE);
-    } else {
-      uint8_t nBytes = sizeof(uint64_t) - clz / BITS_PER_BYTE;
-      pCmprsor->d_flag_p[0] |= ((nBytes - 1) << 4);
-    }
-  }
-
-  while (diff) {
-    pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (diff & 0xff);
-    pCmprsor->nBuf[0]++;
-    diff >>= BITS_PER_BYTE;
-  }
-
-  pCmprsor->d_n++;
-  return code;
-}
-
-// Binary =====================================================
-static int32_t tCompBinary(SCompressor *pCmprsor, const uint8_t *pData, int32_t nData) {
-  int32_t code = 0;
-
-  if (nData) {
-    code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + nData);
-    if (code) return code;
-
-    memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], pData, nData);
-    pCmprsor->nBuf[0] += nData;
-  }
-  pCmprsor->b_n++;
-
-  return code;
-}
-
-// Bool =====================================================
-static const uint8_t BOOL_CMPR_TABLE[] = {0b01, 0b0100, 0b010000, 0b01000000};
-
-static int32_t tCompBool(SCompressor *pCmprsor, bool vBool) {
-  int32_t code = 0;
-
-  int32_t mod4 = pCmprsor->bool_n & 3;
-  if (vBool) {
-    pCmprsor->aBuf[0][pCmprsor->nBuf[0]] |= BOOL_CMPR_TABLE[mod4];
-  }
-  pCmprsor->bool_n++;
-  if (mod4 == 3) {
-    pCmprsor->nBuf[0]++;
-    pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = 0;
-
-    code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0]);
-    if (code) goto _exit;
-  }
-
-_exit:
-  return code;
-}
-
-// SCompressor =====================================================
-int32_t tCompressorCreate(SCompressor **ppCmprsor) {
-  int32_t code = 0;
-
-  *ppCmprsor = (SCompressor *)taosMemoryCalloc(1, sizeof(SCompressor));
-  if ((*ppCmprsor) == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _exit;
-  }
-
-  code = tRealloc(&(*ppCmprsor)->aBuf[0], 1024);
-  if (code) {
-    taosMemoryFree(*ppCmprsor);
-    *ppCmprsor = NULL;
-    goto _exit;
-  }
-
-_exit:
-  return code;
-}
-
-int32_t tCompressorDestroy(SCompressor *pCmprsor) {
-  int32_t code = 0;
-
-  if (pCmprsor) {
-    int32_t nBuf = sizeof(pCmprsor->aBuf) / sizeof(pCmprsor->aBuf[0]);
-    for (int32_t iBuf = 0; iBuf < nBuf; iBuf++) {
-      tFree(pCmprsor->aBuf[iBuf]);
-    }
-
-    taosMemoryFree(pCmprsor);
-  }
-
-  return code;
-}
-
-int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) {
-  int32_t code = 0;
-
-  pCmprsor->type = type;
-  pCmprsor->cmprAlg = cmprAlg;
-  pCmprsor->nBuf[0] = 0;  // (todo) may or may not +/- 1
-
-  switch (type) {
-    case TSDB_DATA_TYPE_TIMESTAMP:
-      pCmprsor->ts_copy = 0;
-      pCmprsor->ts_n = 0;
-      break;
-    case TSDB_DATA_TYPE_BOOL:
-      pCmprsor->bool_n = 0;
-      pCmprsor->aBuf[0][0] = 0;
-      break;
-    case TSDB_DATA_TYPE_BINARY:
-      pCmprsor->b_n = 0;
-      break;
-    default:
-      break;
-  }
-
-  return code;
-}
-
-int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int64_t *nData) {
-  int32_t code = 0;
-
-  if (pCmprsor->cmprAlg == TWO_STAGE_COMP || IS_VAR_DATA_TYPE(pCmprsor->type)) {
-    code = tRealloc(&pCmprsor->aBuf[1], pCmprsor->nBuf[0] + 1);
-    if (code) goto _exit;
-
-    int64_t ret = LZ4_compress_default(pCmprsor->aBuf[0], pCmprsor->aBuf[1] + 1, pCmprsor->nBuf[0], pCmprsor->nBuf[0]);
-    if (ret) {
-      pCmprsor->aBuf[1][0] = 0;
-      pCmprsor->nBuf[1] = ret + 1;
-    } else {
-      pCmprsor->aBuf[1][0] = 1;
-      memcpy(pCmprsor->aBuf[1] + 1, pCmprsor->aBuf[0], pCmprsor->nBuf[0]);
-      pCmprsor->nBuf[1] = pCmprsor->nBuf[0] + 1;
-    }
-
-    *ppData = pCmprsor->aBuf[1];
-    *nData = pCmprsor->nBuf[1];
-  } else {
-    *ppData = pCmprsor->aBuf[0];
-    *nData = pCmprsor->nBuf[0];
-  }
-
-_exit:
-  return code;
-}
-
-int32_t tCompress(SCompressor *pCmprsor, void *pData, int64_t nData) {
-  int32_t code = 0;
-  // TODO
-  return code;
-}
\ No newline at end of file
diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c
index ba877915b1..a0f5c45df2 100644
--- a/source/util/src/tcompression.c
+++ b/source/util/src/tcompression.c
@@ -50,6 +50,7 @@
 #define _DEFAULT_SOURCE
 #include "tcompression.h"
 #include "lz4.h"
+#include "tRealloc.h"
 #include "tlog.h"
 
 #ifdef TD_TSZ
@@ -994,3 +995,499 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co
   return tdszDecompress(SZ_DOUBLE, input + 1, compressedSize - 1, nelements, output);
 }
 #endif
+
+/*************************************************************************
+ *                  STREAM COMPRESSION
+ *************************************************************************/
+#define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (b)) || ((a) < 0 && (b) >= INT64_MIN - (a)))
+
+typedef struct {
+  int8_t   type;
+  int8_t   cmprAlg;
+  uint8_t *aBuf[2];
+  int64_t  nBuf[2];
+  union {
+    // Timestamp ----
+    struct {
+      int8_t   ts_copy;
+      int32_t  ts_n;
+      int64_t  ts_prev_val;
+      int64_t  ts_prev_delta;
+      uint8_t *ts_flag_p;
+    };
+    // Integer ----
+    struct {
+      int8_t  i_copy;
+      int32_t i_n;
+      int64_t i_prev;
+      int32_t i_selector;
+      int32_t i_nele;
+    };
+    // Float ----
+    struct {
+      int32_t  f_n;
+      uint32_t f_prev;
+      uint8_t *f_flag_p;
+    };
+    // Double ----
+    struct {
+      int32_t  d_n;
+      uint64_t d_prev;
+      uint8_t *d_flag_p;
+    };
+    // Bool ----
+    struct {
+      int32_t bool_n;
+    };
+    // Binary ----
+    struct {
+      int32_t b_n;
+    };
+  };
+} SCompressor;
+
+// Timestamp =====================================================
+static int32_t tCompSetCopyMode(SCompressor *pCmprsor) {
+  int32_t code = 0;
+
+  if (pCmprsor->ts_n) {
+    code = tRealloc(&pCmprsor->aBuf[1], sizeof(int64_t) * (pCmprsor->ts_n + 1));
+    if (code) return code;
+    pCmprsor->nBuf[1] = 1;
+
+    int64_t  n = 1;
+    int64_t  valPrev;
+    int64_t  delPrev;
+    uint64_t vZigzag;
+    while (n < pCmprsor->nBuf[0]) {
+      uint8_t n1 = pCmprsor->aBuf[0][0] & 0xf;
+      uint8_t n2 = pCmprsor->aBuf[0][0] >> 4;
+
+      n++;
+
+      vZigzag = 0;
+      for (uint8_t i = 0; i < n1; i++) {
+        vZigzag |= (((uint64_t)pCmprsor->aBuf[0][n]) << (sizeof(int64_t) * i));
+        n++;
+      }
+      int64_t delta_of_delta = ZIGZAG_DECODE(int64_t, vZigzag);
+      if (n == 2) {
+        delPrev = 0;
+        valPrev = delta_of_delta;
+      } else {
+        delPrev = delta_of_delta + delPrev;
+        valPrev = delPrev + valPrev;
+      }
+
+      memcpy(pCmprsor->aBuf[1] + pCmprsor->nBuf[1], &valPrev, sizeof(int64_t));
+      pCmprsor->nBuf[1] += sizeof(int64_t);
+
+      if (n >= pCmprsor->nBuf[0]) break;
+
+      vZigzag = 0;
+      for (uint8_t i = 0; i < n2; i++) {
+        vZigzag |= (((uint64_t)pCmprsor->aBuf[0][n]) << (sizeof(int64_t) * i));
+        n++;
+      }
+      delta_of_delta = ZIGZAG_DECODE(int64_t, vZigzag);
+      delPrev = delta_of_delta + delPrev;
+      valPrev = delPrev + valPrev;
+    }
+
+    uint8_t *pBuf = pCmprsor->aBuf[0];
+    pCmprsor->aBuf[0] = pCmprsor->aBuf[1];
+    pCmprsor->aBuf[1] = pBuf;
+    pCmprsor->nBuf[0] = pCmprsor->nBuf[1];
+  } else {
+    // TODO
+  }
+
+  pCmprsor->aBuf[0][0] = 0;
+  pCmprsor->ts_copy = 1;
+
+  return code;
+}
+static int32_t tCompTimestamp(SCompressor *pCmprsor, TSKEY ts) {
+  int32_t code = 0;
+
+  if (pCmprsor->ts_n == 0) {
+    pCmprsor->ts_prev_val = ts;
+    pCmprsor->ts_prev_delta = -ts;
+  }
+
+  if (pCmprsor->ts_copy) goto _copy_exit;
+
+  if (!I64_SAFE_ADD(ts, -pCmprsor->ts_prev_val)) {
+    code = tCompSetCopyMode(pCmprsor);
+    if (code) return code;
+    goto _copy_exit;
+  }
+
+  int64_t delta = ts - pCmprsor->ts_prev_val;
+
+  if (!I64_SAFE_ADD(delta, -pCmprsor->ts_prev_delta)) {
+    code = tCompSetCopyMode(pCmprsor);
+    if (code) return code;
+    goto _copy_exit;
+  }
+
+  int64_t  delta_of_delta = delta - pCmprsor->ts_prev_delta;
+  uint64_t zigzag_value = ZIGZAG_ENCODE(int64_t, delta_of_delta);
+
+  pCmprsor->ts_prev_val = ts;
+  pCmprsor->ts_prev_delta = delta;
+
+  if (pCmprsor->ts_n & 0x1 == 0) {
+    code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17 /*sizeof(int64_t) * 2 + 1*/);
+    if (code) return code;
+
+    pCmprsor->ts_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]];
+    pCmprsor->nBuf[0]++;
+    pCmprsor->ts_flag_p[0] = 0;
+
+    while (zigzag_value) {
+      pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (zigzag_value & 0xff);
+      pCmprsor->nBuf[0]++;
+      pCmprsor->ts_flag_p[0]++;
+    }
+  } else {
+    while (zigzag_value) {
+      pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (zigzag_value & 0xff);
+      pCmprsor->nBuf[0]++;
+      pCmprsor->ts_flag_p += (uint8_t)0x10;
+    }
+  }
+
+  pCmprsor->ts_n++;
+  return code;
+
+_copy_exit:
+  code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(int64_t));
+  if (code) return code;
+
+  memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], &ts, sizeof(ts));
+  pCmprsor->nBuf[0] += sizeof(ts);
+
+  pCmprsor->ts_n++;
+  return code;
+}
+
+// Integer =====================================================
+#define SIMPLE8B_MAX ((uint64_t)1152921504606846974LL)
+static const char    bit_per_integer[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60};
+static const int32_t selector_to_elems[] = {240, 120, 60, 30, 20, 15, 12, 10, 8, 7, 6, 5, 4, 3, 2, 1};
+static const char    bit_to_selector[] = {0,  2,  3,  4,  5,  6,  7,  8,  9,  10, 10, 11, 11, 12, 12, 12,
+                                          13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 15,
+                                          15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
+                                          15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15};
+
+static int32_t tCompI64(SCompressor *pCmprsor, int64_t val) {
+  int32_t code = 0;
+
+  if (pCmprsor->i_copy == 1) goto _copy_cmpr;
+
+  if (!I64_SAFE_ADD(val, pCmprsor->i_prev)) {
+    // TODO
+    goto _copy_cmpr;
+  }
+
+  int64_t  diff = val - pCmprsor->i_prev;
+  uint64_t vZigzag = ZIGZAG_ENCODE(int64_t, diff);
+
+  if (vZigzag >= SIMPLE8B_MAX) {
+    // TODO
+    goto _copy_cmpr;
+  }
+
+  int64_t nBit;
+  if (vZigzag) {
+    nBit = 64 - BUILDIN_CLZL(vZigzag);
+  } else {
+    nBit = 0;
+  }
+
+  if (pCmprsor->i_nele + 1 <= selector_to_elems[pCmprsor->i_selector] &&
+      pCmprsor->i_nele + 1 <= selector_to_elems[bit_to_selector[nBit]]) {
+    if (pCmprsor->i_selector < bit_to_selector[nBit]) {
+      pCmprsor->i_selector = bit_to_selector[nBit];
+    }
+    pCmprsor->i_nele++;
+  } else {
+    while (pCmprsor->i_nele < selector_to_elems[pCmprsor->i_selector]) {
+      pCmprsor->i_selector++;
+    }
+    pCmprsor->i_nele = selector_to_elems[pCmprsor->i_selector];
+
+    code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(uint64_t));
+    if (code) return code;
+
+    uint64_t *bp = (uint64_t *)(pCmprsor->aBuf[0] + pCmprsor->nBuf[0]);
+    pCmprsor->nBuf[0] += sizeof(uint64_t);
+    bp[0] = pCmprsor->i_selector;
+    for (int32_t iVal = 0; iVal < pCmprsor->i_nele; iVal++) {
+      /* code */
+    }
+
+    // reset and continue
+  }
+
+  return code;
+
+_copy_cmpr:
+  code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 0 /*tDataTypes[pCmprsor->type].bytes (todo)*/);
+  if (code) return code;
+
+  // memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], NULL /* todo */, 0 /*tDataTypes[pCmprsor->type].bytes (todo)*/);
+  // pCmprsor->nBuf[0] += tDataTypes[pCmprsor->type].bytes;
+
+  return code;
+}
+
+// Float =====================================================
+static int32_t tCompFloat(SCompressor *pCmprsor, float f) {
+  int32_t code = 0;
+
+  union {
+    float    f;
+    uint32_t u;
+  } val = {.f = f};
+
+  uint32_t diff = val.u ^ pCmprsor->f_prev;
+  pCmprsor->f_prev = val.u;
+
+  int32_t clz, ctz;
+  if (diff) {
+    clz = BUILDIN_CLZ(diff);
+    ctz = BUILDIN_CTZ(diff);
+  } else {
+    clz = sizeof(uint32_t);
+    ctz = sizeof(uint32_t);
+  }
+
+  if (pCmprsor->f_n & 0x1 == 0) {
+    code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 9 /* sizeof(float) * 2 + 1 */);
+    if (code) return code;
+
+    pCmprsor->f_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]];
+    pCmprsor->nBuf[0]++;
+
+    if (clz < ctz) {
+      uint8_t nBytes = sizeof(uint32_t) - ctz / BITS_PER_BYTE;
+      pCmprsor->f_flag_p[0] = (0x08 | (nBytes - 1));
+      diff >>= (32 - nBytes * BITS_PER_BYTE);
+    } else {
+      uint8_t nBytes = sizeof(uint32_t) - clz / BITS_PER_BYTE;
+      pCmprsor->f_flag_p[0] = nBytes - 1;
+    }
+  } else {
+    if (clz < ctz) {
+      uint8_t nBytes = sizeof(uint32_t) - ctz / BITS_PER_BYTE;
+      pCmprsor->f_flag_p[0] |= ((0x08 | (nBytes - 1)) << 4);
+      diff >>= (32 - nBytes * BITS_PER_BYTE);
+    } else {
+      uint8_t nBytes = sizeof(uint32_t) - clz / BITS_PER_BYTE;
+      pCmprsor->f_flag_p[0] |= ((nBytes - 1) << 4);
+    }
+  }
+
+  while (diff) {
+    pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (diff & 0xff);
+    pCmprsor->nBuf[0]++;
+    diff >>= BITS_PER_BYTE;
+  }
+
+  pCmprsor->f_n++;
+  return code;
+}
+
+// Double =====================================================
+static int32_t tCompDouble(SCompressor *pCmprsor, double d) {
+  int32_t code = 0;
+
+  union {
+    double   d;
+    uint64_t u;
+  } val = {.d = d};
+
+  uint64_t diff = val.u ^ pCmprsor->d_prev;
+  pCmprsor->d_prev = val.u;
+
+  int32_t clz, ctz;
+  if (diff) {
+    clz = BUILDIN_CLZL(diff);
+    ctz = BUILDIN_CTZL(diff);
+  } else {
+    clz = sizeof(uint64_t);
+    ctz = sizeof(uint64_t);
+  }
+
+  if (pCmprsor->d_n & 0x1 == 0) {
+    code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17 /* sizeof(double) * 2 + 1 */);
+    if (code) return code;
+
+    pCmprsor->d_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]];
+    pCmprsor->nBuf[0]++;
+
+    if (clz < ctz) {
+      uint8_t nBytes = sizeof(uint64_t) - ctz / BITS_PER_BYTE;
+      pCmprsor->d_flag_p[0] = (0x08 | (nBytes - 1));
+      diff >>= (64 - nBytes * BITS_PER_BYTE);
+    } else {
+      uint8_t nBytes = sizeof(uint64_t) - clz / BITS_PER_BYTE;
+      pCmprsor->d_flag_p[0] = nBytes - 1;
+    }
+  } else {
+    if (clz < ctz) {
+      uint8_t nBytes = sizeof(uint64_t) - ctz / BITS_PER_BYTE;
+      pCmprsor->d_flag_p[0] |= ((0x08 | (nBytes - 1)) << 4);
+      diff >>= (64 - nBytes * BITS_PER_BYTE);
+    } else {
+      uint8_t nBytes = sizeof(uint64_t) - clz / BITS_PER_BYTE;
+      pCmprsor->d_flag_p[0] |= ((nBytes - 1) << 4);
+    }
+  }
+
+  while (diff) {
+    pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (diff & 0xff);
+    pCmprsor->nBuf[0]++;
+    diff >>= BITS_PER_BYTE;
+  }
+
+  pCmprsor->d_n++;
+  return code;
+}
+
+// Binary =====================================================
+static int32_t tCompBinary(SCompressor *pCmprsor, const uint8_t *pData, int32_t nData) {
+  int32_t code = 0;
+
+  if (nData) {
+    code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + nData);
+    if (code) return code;
+
+    memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], pData, nData);
+    pCmprsor->nBuf[0] += nData;
+  }
+  pCmprsor->b_n++;
+
+  return code;
+}
+
+// Bool =====================================================
+static const uint8_t BOOL_CMPR_TABLE[] = {0b01, 0b0100, 0b010000, 0b01000000};
+
+static int32_t tCompBool(SCompressor *pCmprsor, bool vBool) {
+  int32_t code = 0;
+
+  int32_t mod4 = pCmprsor->bool_n & 3;
+  if (vBool) {
+    pCmprsor->aBuf[0][pCmprsor->nBuf[0]] |= BOOL_CMPR_TABLE[mod4];
+  }
+  pCmprsor->bool_n++;
+  if (mod4 == 3) {
+    pCmprsor->nBuf[0]++;
+    pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = 0;
+
+    code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0]);
+    if (code) goto _exit;
+  }
+
+_exit:
+  return code;
+}
+
+// SCompressor =====================================================
+int32_t tCompressorCreate(SCompressor **ppCmprsor) {
+  int32_t code = 0;
+
+  *ppCmprsor = (SCompressor *)taosMemoryCalloc(1, sizeof(SCompressor));
+  if ((*ppCmprsor) == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _exit;
+  }
+
+  code = tRealloc(&(*ppCmprsor)->aBuf[0], 1024);
+  if (code) {
+    taosMemoryFree(*ppCmprsor);
+    *ppCmprsor = NULL;
+    goto _exit;
+  }
+
+_exit:
+  return code;
+}
+
+int32_t tCompressorDestroy(SCompressor *pCmprsor) {
+  int32_t code = 0;
+
+  if (pCmprsor) {
+    int32_t nBuf = sizeof(pCmprsor->aBuf) / sizeof(pCmprsor->aBuf[0]);
+    for (int32_t iBuf = 0; iBuf < nBuf; iBuf++) {
+      tFree(pCmprsor->aBuf[iBuf]);
+    }
+
+    taosMemoryFree(pCmprsor);
+  }
+
+  return code;
+}
+
+int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) {
+  int32_t code = 0;
+
+  pCmprsor->type = type;
+  pCmprsor->cmprAlg = cmprAlg;
+  pCmprsor->nBuf[0] = 0;  // (todo) may or may not +/- 1
+
+  switch (type) {
+    case TSDB_DATA_TYPE_TIMESTAMP:
+      pCmprsor->ts_copy = 0;
+      pCmprsor->ts_n = 0;
+      break;
+    case TSDB_DATA_TYPE_BOOL:
+      pCmprsor->bool_n = 0;
+      pCmprsor->aBuf[0][0] = 0;
+      break;
+    case TSDB_DATA_TYPE_BINARY:
+      pCmprsor->b_n = 0;
+      break;
+    default:
+      break;
+  }
+
+  return code;
+}
+
+int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int64_t *nData) {
+  int32_t code = 0;
+
+  if (pCmprsor->cmprAlg == TWO_STAGE_COMP /*|| IS_VAR_DATA_TYPE(pCmprsor->type)*/) {
+    code = tRealloc(&pCmprsor->aBuf[1], pCmprsor->nBuf[0] + 1);
+    if (code) goto _exit;
+
+    int64_t ret = LZ4_compress_default(pCmprsor->aBuf[0], pCmprsor->aBuf[1] + 1, pCmprsor->nBuf[0], pCmprsor->nBuf[0]);
+    if (ret) {
+      pCmprsor->aBuf[1][0] = 0;
+      pCmprsor->nBuf[1] = ret + 1;
+    } else {
+      pCmprsor->aBuf[1][0] = 1;
+      memcpy(pCmprsor->aBuf[1] + 1, pCmprsor->aBuf[0], pCmprsor->nBuf[0]);
+      pCmprsor->nBuf[1] = pCmprsor->nBuf[0] + 1;
+    }
+
+    *ppData = pCmprsor->aBuf[1];
+    *nData = pCmprsor->nBuf[1];
+  } else {
+    *ppData = pCmprsor->aBuf[0];
+    *nData = pCmprsor->nBuf[0];
+  }
+
+_exit:
+  return code;
+}
+
+int32_t tCompress(SCompressor *pCmprsor, void *pData, int64_t nData) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
\ No newline at end of file

From f2cd319df22f845ab001b8f56f2e049d79ef5fff Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 19 Sep 2022 12:00:55 +0800
Subject: [PATCH 23/57] more code

---
 source/util/src/tcompression.c | 190 +++++++++++++++++----------------
 1 file changed, 97 insertions(+), 93 deletions(-)

diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c
index a0f5c45df2..75c1997428 100644
--- a/source/util/src/tcompression.c
+++ b/source/util/src/tcompression.c
@@ -1004,12 +1004,12 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co
 typedef struct {
   int8_t   type;
   int8_t   cmprAlg;
+  int8_t   autoAlloc;
   uint8_t *aBuf[2];
   int64_t  nBuf[2];
   union {
     // Timestamp ----
     struct {
-      int8_t   ts_copy;
       int32_t  ts_n;
       int64_t  ts_prev_val;
       int64_t  ts_prev_delta;
@@ -1051,124 +1051,123 @@ static int32_t tCompSetCopyMode(SCompressor *pCmprsor) {
   int32_t code = 0;
 
   if (pCmprsor->ts_n) {
-    code = tRealloc(&pCmprsor->aBuf[1], sizeof(int64_t) * (pCmprsor->ts_n + 1));
-    if (code) return code;
-    pCmprsor->nBuf[1] = 1;
+    if (pCmprsor->autoAlloc) {
+      code = tRealloc(&pCmprsor->aBuf[1], sizeof(int64_t) * pCmprsor->ts_n);
+      if (code) return code;
+    }
+    pCmprsor->nBuf[1] = 0;
 
     int64_t  n = 1;
-    int64_t  valPrev;
-    int64_t  delPrev;
+    int64_t  value;
+    int64_t  delta;
     uint64_t vZigzag;
     while (n < pCmprsor->nBuf[0]) {
-      uint8_t n1 = pCmprsor->aBuf[0][0] & 0xf;
-      uint8_t n2 = pCmprsor->aBuf[0][0] >> 4;
+      uint8_t aN[2];
+      aN[0] = pCmprsor->aBuf[0][n] & 0xf;
+      aN[1] = pCmprsor->aBuf[0][n] >> 4;
 
       n++;
 
-      vZigzag = 0;
-      for (uint8_t i = 0; i < n1; i++) {
-        vZigzag |= (((uint64_t)pCmprsor->aBuf[0][n]) << (sizeof(int64_t) * i));
-        n++;
-      }
-      int64_t delta_of_delta = ZIGZAG_DECODE(int64_t, vZigzag);
-      if (n == 2) {
-        delPrev = 0;
-        valPrev = delta_of_delta;
-      } else {
-        delPrev = delta_of_delta + delPrev;
-        valPrev = delPrev + valPrev;
-      }
+      for (int32_t i = 0; i < 2; i++) {
+        vZigzag = 0;
+        for (uint8_t j = 0; j < aN[i]; j++) {
+          vZigzag |= (((uint64_t)pCmprsor->aBuf[0][n]) << (8 * j));
+          n++;
+        }
 
-      memcpy(pCmprsor->aBuf[1] + pCmprsor->nBuf[1], &valPrev, sizeof(int64_t));
-      pCmprsor->nBuf[1] += sizeof(int64_t);
+        int64_t delta_of_delta = ZIGZAG_DECODE(int64_t, vZigzag);
+        if (pCmprsor->nBuf[1] == 0) {
+          delta = 0;
+          value = delta_of_delta;
+        } else {
+          delta = delta_of_delta + delta;
+          value = delta + value;
+        }
 
-      if (n >= pCmprsor->nBuf[0]) break;
+        memcpy(pCmprsor->aBuf[1] + pCmprsor->nBuf[1], &value, sizeof(int64_t));
+        pCmprsor->nBuf[1] += sizeof(int64_t);
 
-      vZigzag = 0;
-      for (uint8_t i = 0; i < n2; i++) {
-        vZigzag |= (((uint64_t)pCmprsor->aBuf[0][n]) << (sizeof(int64_t) * i));
-        n++;
+        if (n >= pCmprsor->nBuf[0]) break;
       }
-      delta_of_delta = ZIGZAG_DECODE(int64_t, vZigzag);
-      delPrev = delta_of_delta + delPrev;
-      valPrev = delPrev + valPrev;
     }
 
-    uint8_t *pBuf = pCmprsor->aBuf[0];
-    pCmprsor->aBuf[0] = pCmprsor->aBuf[1];
-    pCmprsor->aBuf[1] = pBuf;
-    pCmprsor->nBuf[0] = pCmprsor->nBuf[1];
-  } else {
-    // TODO
-  }
+    ASSERT(n == pCmprsor->nBuf[0]);
 
-  pCmprsor->aBuf[0][0] = 0;
-  pCmprsor->ts_copy = 1;
+    if (pCmprsor->autoAlloc) {
+      code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[1] + 1);
+      if (code) return code;
+    }
+    memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->aBuf[1], pCmprsor->nBuf[1]);
+    pCmprsor->nBuf[0] = 1 + pCmprsor->nBuf[1];
+  }
+  pCmprsor->aBuf[0][0] = MODE_NOCOMPRESS;
 
   return code;
 }
-static int32_t tCompTimestamp(SCompressor *pCmprsor, TSKEY ts) {
+static int32_t tCompTimestamp(SCompressor *pCmprsor, int64_t ts) {
   int32_t code = 0;
 
-  if (pCmprsor->ts_n == 0) {
+  ASSERT(pCmprsor->type == TSDB_DATA_TYPE_TIMESTAMP);
+
+  if (pCmprsor->aBuf[0][0] == MODE_COMPRESS) {
+    if (pCmprsor->ts_n == 0) {
+      pCmprsor->ts_prev_val = ts;
+      pCmprsor->ts_prev_delta = -ts;
+    }
+
+    if (!I64_SAFE_ADD(ts, -pCmprsor->ts_prev_val)) {
+      code = tCompSetCopyMode(pCmprsor);
+      if (code) return code;
+      goto _copy_cmpr;
+    }
+    int64_t delta = ts - pCmprsor->ts_prev_val;
+
+    if (!I64_SAFE_ADD(delta, -pCmprsor->ts_prev_delta)) {
+      code = tCompSetCopyMode(pCmprsor);
+      if (code) return code;
+      goto _copy_cmpr;
+    }
+    int64_t  delta_of_delta = delta - pCmprsor->ts_prev_delta;
+    uint64_t vZigzag = ZIGZAG_ENCODE(int64_t, delta_of_delta);
+
     pCmprsor->ts_prev_val = ts;
-    pCmprsor->ts_prev_delta = -ts;
-  }
+    pCmprsor->ts_prev_delta = delta;
 
-  if (pCmprsor->ts_copy) goto _copy_exit;
+    if ((pCmprsor->ts_n & 0x1) == 0) {
+      if (pCmprsor->autoAlloc) {
+        code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17);
+        if (code) return code;
+      }
 
-  if (!I64_SAFE_ADD(ts, -pCmprsor->ts_prev_val)) {
-    code = tCompSetCopyMode(pCmprsor);
-    if (code) return code;
-    goto _copy_exit;
-  }
-
-  int64_t delta = ts - pCmprsor->ts_prev_val;
-
-  if (!I64_SAFE_ADD(delta, -pCmprsor->ts_prev_delta)) {
-    code = tCompSetCopyMode(pCmprsor);
-    if (code) return code;
-    goto _copy_exit;
-  }
-
-  int64_t  delta_of_delta = delta - pCmprsor->ts_prev_delta;
-  uint64_t zigzag_value = ZIGZAG_ENCODE(int64_t, delta_of_delta);
-
-  pCmprsor->ts_prev_val = ts;
-  pCmprsor->ts_prev_delta = delta;
-
-  if (pCmprsor->ts_n & 0x1 == 0) {
-    code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17 /*sizeof(int64_t) * 2 + 1*/);
-    if (code) return code;
-
-    pCmprsor->ts_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]];
-    pCmprsor->nBuf[0]++;
-    pCmprsor->ts_flag_p[0] = 0;
-
-    while (zigzag_value) {
-      pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (zigzag_value & 0xff);
+      pCmprsor->ts_flag_p = pCmprsor->aBuf[0] + pCmprsor->nBuf[0];
       pCmprsor->nBuf[0]++;
-      pCmprsor->ts_flag_p[0]++;
+      pCmprsor->ts_flag_p[0] = 0;
+      while (vZigzag) {
+        pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (vZigzag & 0xff);
+        pCmprsor->nBuf[0]++;
+        pCmprsor->ts_flag_p[0]++;
+        vZigzag >>= 8;
+      }
+    } else {
+      while (vZigzag) {
+        pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (vZigzag & 0xff);
+        pCmprsor->nBuf[0]++;
+        pCmprsor->ts_flag_p[0] += 0x10;
+        vZigzag >>= 8;
+      }
     }
   } else {
-    while (zigzag_value) {
-      pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (zigzag_value & 0xff);
-      pCmprsor->nBuf[0]++;
-      pCmprsor->ts_flag_p += (uint8_t)0x10;
+  _copy_cmpr:
+    if (pCmprsor->autoAlloc) {
+      code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(ts));
+      if (code) return code;
     }
+
+    memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], &ts, sizeof(ts));
+    pCmprsor->nBuf[0] += sizeof(ts);
   }
-
   pCmprsor->ts_n++;
-  return code;
 
-_copy_exit:
-  code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(int64_t));
-  if (code) return code;
-
-  memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], &ts, sizeof(ts));
-  pCmprsor->nBuf[0] += sizeof(ts);
-
-  pCmprsor->ts_n++;
   return code;
 }
 
@@ -1432,17 +1431,22 @@ int32_t tCompressorDestroy(SCompressor *pCmprsor) {
   return code;
 }
 
-int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) {
+int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg, int8_t autoAlloc) {
   int32_t code = 0;
 
   pCmprsor->type = type;
   pCmprsor->cmprAlg = cmprAlg;
-  pCmprsor->nBuf[0] = 0;  // (todo) may or may not +/- 1
+  pCmprsor->autoAlloc = autoAlloc;
 
   switch (type) {
     case TSDB_DATA_TYPE_TIMESTAMP:
-      pCmprsor->ts_copy = 0;
       pCmprsor->ts_n = 0;
+      pCmprsor->ts_prev_val = 0;
+      pCmprsor->ts_prev_delta = 0;
+      pCmprsor->ts_flag_p = NULL;
+      pCmprsor->aBuf[0][0] = MODE_COMPRESS;
+      pCmprsor->nBuf[0] = 1;
+      pCmprsor->nBuf[1] = 0;
       break;
     case TSDB_DATA_TYPE_BOOL:
       pCmprsor->bool_n = 0;

From 6bfb85314e554438f30b844dfaa645f9deaf355a Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Mon, 19 Sep 2022 13:49:55 +0800
Subject: [PATCH 24/57] fix:error in auto create table for taosX

---
 utils/test/c/tmqSim.c | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c
index 71b31ba107..1e8a7350f1 100644
--- a/utils/test/c/tmqSim.c
+++ b/utils/test/c/tmqSim.c
@@ -665,11 +665,11 @@ static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn
     }
 	
     char* result = tmq_get_json_meta(msg);
-    if(result){
+    if(result && strcmp(result, "") != 0){
   	  //printf("meta result: %s\n", result);
   	  taosFprintfFile(pInfo->pConsumeMetaFile, "%s\n", result);
-  	  taosMemoryFree(result);
     }
+    tmq_free_json_meta(result);
   }
 
   totalRows++;
@@ -818,8 +818,12 @@ void loop_consume(SThreadInfo* pInfo) {
 	  	tmq_res_t msgType = tmq_get_res_type(tmqMsg);
 		if (msgType == TMQ_RES_TABLE_META) {
  		  totalRows += meta_msg_process(tmqMsg, pInfo, totalMsgs);
-		} else if (msgType == TMQ_RES_DATA)
-          totalRows += data_msg_process(tmqMsg, pInfo, totalMsgs);
+		} else if (msgType == TMQ_RES_DATA){
+                  totalRows += data_msg_process(tmqMsg, pInfo, totalMsgs);
+                } else if (msgType == TMQ_RES_METADATA){
+                  meta_msg_process(tmqMsg, pInfo, totalMsgs);
+                  totalRows += data_msg_process(tmqMsg, pInfo, totalMsgs);
+                }
       }
 
       taos_free_result(tmqMsg);

From e5e21bf6357cdba1e69d75bb0dafb8038b3d0f7a Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 19 Sep 2022 14:08:30 +0800
Subject: [PATCH 25/57] fix(query): add null ptr check.

---
 source/libs/executor/src/executil.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index d75f0580e1..b88589740d 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -989,7 +989,8 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod
 
     if (pNode->output) {
       (*numOfOutputCols) += 1;
-    } else {
+    } else if (info != NULL) {
+      // select distinct tbname from stb where tbname='abc';
       info->output = false;
     }
   }

From a1257090e236e79ac642234f18c47c69f000b5ff Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Mon, 19 Sep 2022 14:28:03 +0800
Subject: [PATCH 26/57] fix:error in auto create table for taosX

---
 source/client/src/clientMain.c | 16 ++++++++--------
 source/client/src/clientTmq.c  |  4 ++++
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index 8fdc57f413..13cbaa0e22 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -148,7 +148,7 @@ int taos_errno(TAOS_RES *res) {
     return terrno;
   }
 
-  if (TD_RES_TMQ(res)) {
+  if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
     return 0;
   }
 
@@ -162,7 +162,7 @@ const char *taos_errstr(TAOS_RES *res) {
     return (const char *)tstrerror(terrno);
   }
 
-  if (TD_RES_TMQ(res)) {
+  if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
     return "success";
   }
 
@@ -264,7 +264,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
     return doFetchRows(pRequest, true, true);
 #endif
 
-  } else if (TD_RES_TMQ(res)) {
+  } else if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
     SMqRspObj      *msg = ((SMqRspObj *)res);
     SReqResultInfo *pResultInfo;
     if (msg->resIter == -1) {
@@ -437,7 +437,7 @@ const char *taos_data_type(int type) {
 const char *taos_get_client_info() { return version; }
 
 int taos_affected_rows(TAOS_RES *res) {
-  if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res)) {
+  if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res)) {
     return 0;
   }
 
@@ -454,7 +454,7 @@ int taos_result_precision(TAOS_RES *res) {
   if (TD_RES_QUERY(res)) {
     SRequestObj *pRequest = (SRequestObj *)res;
     return pRequest->body.resInfo.precision;
-  } else if (TD_RES_TMQ(res)) {
+  } else if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
     SReqResultInfo *info = tmqGetCurResInfo(res);
     return info->precision;
   }
@@ -487,7 +487,7 @@ int taos_select_db(TAOS *taos, const char *db) {
 }
 
 void taos_stop_query(TAOS_RES *res) {
-  if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res)) {
+  if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res)) {
     return;
   }
 
@@ -559,7 +559,7 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
     (*rows) = pResultInfo->row;
     (*numOfRows) = pResultInfo->numOfRows;
     return pRequest->code;
-  } else if (TD_RES_TMQ(res)) {
+  } else if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
     SReqResultInfo *pResultInfo = tmqGetNextResInfo(res, true);
     if (pResultInfo == NULL) return -1;
 
@@ -578,7 +578,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
     return 0;
   }
 
-  if (TD_RES_TMQ(res)) {
+  if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
     SReqResultInfo *pResultInfo = tmqGetNextResInfo(res, false);
     if (pResultInfo == NULL) {
       (*numOfRows) = 0;
diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index c67556faa0..94c1d16790 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -515,6 +515,10 @@ int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_comm
     SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)msg;
     topic = pMetaRspObj->topic;
     vgId = pMetaRspObj->vgId;
+  } else if(TD_RES_TMQ_METADATA(msg)) {
+    SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)msg;
+    topic = pRspObj->topic;
+    vgId = pRspObj->vgId;
   } else {
     return TSDB_CODE_TMQ_INVALID_MSG;
   }

From 01adee2f53306f12913936798aa8938173af2084 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 19 Sep 2022 14:36:55 +0800
Subject: [PATCH 27/57] fix(query): add an assert

---
 source/common/src/tdatablock.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index 16b8e55cf7..8a8e6a83a5 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -1446,6 +1446,7 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) {
   int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(numOfCols);
   int32_t rowSize = pBlock->info.rowSize;
   int32_t nRows = payloadSize / rowSize;
+  ASSERT(nRows >= 1);
 
   // the true value must be less than the value of nRows
   int32_t additional = 0;

From 61496324e7cfb86ab06d64af42d741ff6f994e91 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 19 Sep 2022 14:39:21 +0800
Subject: [PATCH 28/57] more code

---
 source/util/src/tcompression.c | 134 ++++++++++++++++++++-------------
 1 file changed, 80 insertions(+), 54 deletions(-)

diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c
index 75c1997428..4bec2f16ed 100644
--- a/source/util/src/tcompression.c
+++ b/source/util/src/tcompression.c
@@ -815,24 +815,24 @@ int32_t tsCompressFloatImp(const char *const input, const int32_t nelements, cha
     uint32_t predicted = prev_value;
     uint32_t diff = curr.bits ^ predicted;
 
-    int32_t leading_zeros = FLOAT_BYTES * BITS_PER_BYTE;
-    int32_t trailing_zeros = leading_zeros;
+    int32_t clz = FLOAT_BYTES * BITS_PER_BYTE;
+    int32_t ctz = clz;
 
     if (diff) {
-      trailing_zeros = BUILDIN_CTZ(diff);
-      leading_zeros = BUILDIN_CLZ(diff);
+      ctz = BUILDIN_CTZ(diff);
+      clz = BUILDIN_CLZ(diff);
     }
 
     uint8_t nbytes = 0;
     uint8_t flag;
 
-    if (trailing_zeros > leading_zeros) {
-      nbytes = (uint8_t)(FLOAT_BYTES - trailing_zeros / BITS_PER_BYTE);
+    if (ctz > clz) {
+      nbytes = (uint8_t)(FLOAT_BYTES - ctz / BITS_PER_BYTE);
 
       if (nbytes > 0) nbytes--;
       flag = ((uint8_t)1 << 3) | nbytes;
     } else {
-      nbytes = (uint8_t)(FLOAT_BYTES - leading_zeros / BITS_PER_BYTE);
+      nbytes = (uint8_t)(FLOAT_BYTES - clz / BITS_PER_BYTE);
       if (nbytes > 0) nbytes--;
       flag = nbytes;
     }
@@ -1041,7 +1041,7 @@ typedef struct {
     };
     // Binary ----
     struct {
-      int32_t b_n;
+      int32_t binary_n;
     };
   };
 } SCompressor;
@@ -1100,7 +1100,7 @@ static int32_t tCompSetCopyMode(SCompressor *pCmprsor) {
     memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->aBuf[1], pCmprsor->nBuf[1]);
     pCmprsor->nBuf[0] = 1 + pCmprsor->nBuf[1];
   }
-  pCmprsor->aBuf[0][0] = MODE_NOCOMPRESS;
+  pCmprsor->aBuf[0][0] = 0;
 
   return code;
 }
@@ -1109,7 +1109,7 @@ static int32_t tCompTimestamp(SCompressor *pCmprsor, int64_t ts) {
 
   ASSERT(pCmprsor->type == TSDB_DATA_TYPE_TIMESTAMP);
 
-  if (pCmprsor->aBuf[0][0] == MODE_COMPRESS) {
+  if (pCmprsor->aBuf[0][0] == 1) {
     if (pCmprsor->ts_n == 0) {
       pCmprsor->ts_prev_val = ts;
       pCmprsor->ts_prev_delta = -ts;
@@ -1259,43 +1259,47 @@ static int32_t tCompFloat(SCompressor *pCmprsor, float f) {
     clz = BUILDIN_CLZ(diff);
     ctz = BUILDIN_CTZ(diff);
   } else {
-    clz = sizeof(uint32_t);
-    ctz = sizeof(uint32_t);
+    clz = 32;
+    ctz = 32;
   }
 
-  if (pCmprsor->f_n & 0x1 == 0) {
-    code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 9 /* sizeof(float) * 2 + 1 */);
-    if (code) return code;
+  uint8_t nBytes;
+  if (clz < ctz) {
+    nBytes = sizeof(uint32_t) - ctz / BITS_PER_BYTE;
+    if (nBytes) diff >>= (32 - nBytes * BITS_PER_BYTE);
+  } else {
+    nBytes = sizeof(uint32_t) - clz / BITS_PER_BYTE;
+  }
+  if (nBytes == 0) nBytes++;
+
+  if ((pCmprsor->f_n & 0x1) == 0) {
+    if (pCmprsor->autoAlloc) {
+      code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 9);
+      if (code) return code;
+    }
 
     pCmprsor->f_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]];
     pCmprsor->nBuf[0]++;
 
     if (clz < ctz) {
-      uint8_t nBytes = sizeof(uint32_t) - ctz / BITS_PER_BYTE;
       pCmprsor->f_flag_p[0] = (0x08 | (nBytes - 1));
-      diff >>= (32 - nBytes * BITS_PER_BYTE);
     } else {
-      uint8_t nBytes = sizeof(uint32_t) - clz / BITS_PER_BYTE;
       pCmprsor->f_flag_p[0] = nBytes - 1;
     }
   } else {
     if (clz < ctz) {
-      uint8_t nBytes = sizeof(uint32_t) - ctz / BITS_PER_BYTE;
       pCmprsor->f_flag_p[0] |= ((0x08 | (nBytes - 1)) << 4);
-      diff >>= (32 - nBytes * BITS_PER_BYTE);
     } else {
-      uint8_t nBytes = sizeof(uint32_t) - clz / BITS_PER_BYTE;
       pCmprsor->f_flag_p[0] |= ((nBytes - 1) << 4);
     }
   }
-
-  while (diff) {
+  for (; nBytes; nBytes--) {
     pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (diff & 0xff);
     pCmprsor->nBuf[0]++;
     diff >>= BITS_PER_BYTE;
   }
-
   pCmprsor->f_n++;
+
   return code;
 }
 
@@ -1316,43 +1320,47 @@ static int32_t tCompDouble(SCompressor *pCmprsor, double d) {
     clz = BUILDIN_CLZL(diff);
     ctz = BUILDIN_CTZL(diff);
   } else {
-    clz = sizeof(uint64_t);
-    ctz = sizeof(uint64_t);
+    clz = 64;
+    ctz = 64;
   }
 
-  if (pCmprsor->d_n & 0x1 == 0) {
-    code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17 /* sizeof(double) * 2 + 1 */);
-    if (code) return code;
+  uint8_t nBytes;
+  if (clz < ctz) {
+    nBytes = sizeof(uint64_t) - ctz / BITS_PER_BYTE;
+    if (nBytes) diff >>= (64 - nBytes * BITS_PER_BYTE);
+  } else {
+    nBytes = sizeof(uint64_t) - clz / BITS_PER_BYTE;
+  }
+  if (nBytes == 0) nBytes++;
+
+  if ((pCmprsor->d_n & 0x1) == 0) {
+    if (pCmprsor->autoAlloc) {
+      code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17);
+      if (code) return code;
+    }
 
     pCmprsor->d_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]];
     pCmprsor->nBuf[0]++;
 
     if (clz < ctz) {
-      uint8_t nBytes = sizeof(uint64_t) - ctz / BITS_PER_BYTE;
       pCmprsor->d_flag_p[0] = (0x08 | (nBytes - 1));
-      diff >>= (64 - nBytes * BITS_PER_BYTE);
     } else {
-      uint8_t nBytes = sizeof(uint64_t) - clz / BITS_PER_BYTE;
       pCmprsor->d_flag_p[0] = nBytes - 1;
     }
   } else {
     if (clz < ctz) {
-      uint8_t nBytes = sizeof(uint64_t) - ctz / BITS_PER_BYTE;
       pCmprsor->d_flag_p[0] |= ((0x08 | (nBytes - 1)) << 4);
-      diff >>= (64 - nBytes * BITS_PER_BYTE);
     } else {
-      uint8_t nBytes = sizeof(uint64_t) - clz / BITS_PER_BYTE;
       pCmprsor->d_flag_p[0] |= ((nBytes - 1) << 4);
     }
   }
-
-  while (diff) {
+  for (; nBytes; nBytes--) {
     pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (diff & 0xff);
     pCmprsor->nBuf[0]++;
     diff >>= BITS_PER_BYTE;
   }
-
   pCmprsor->d_n++;
+
   return code;
 }
 
@@ -1361,13 +1369,15 @@ static int32_t tCompBinary(SCompressor *pCmprsor, const uint8_t *pData, int32_t
   int32_t code = 0;
 
   if (nData) {
-    code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + nData);
-    if (code) return code;
+    if (pCmprsor->autoAlloc) {
+      code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + nData);
+      if (code) return code;
+    }
 
     memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], pData, nData);
     pCmprsor->nBuf[0] += nData;
   }
-  pCmprsor->b_n++;
+  pCmprsor->binary_n++;
 
   return code;
 }
@@ -1379,19 +1389,21 @@ static int32_t tCompBool(SCompressor *pCmprsor, bool vBool) {
   int32_t code = 0;
 
   int32_t mod4 = pCmprsor->bool_n & 3;
+  if (mod4 == 0) {
+    pCmprsor->nBuf[0]++;
+
+    if (pCmprsor->autoAlloc) {
+      code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0]);
+      if (code) return code;
+    }
+
+    pCmprsor->aBuf[0][pCmprsor->nBuf[0] - 1] = 0;
+  }
   if (vBool) {
-    pCmprsor->aBuf[0][pCmprsor->nBuf[0]] |= BOOL_CMPR_TABLE[mod4];
+    pCmprsor->aBuf[0][pCmprsor->nBuf[0] - 1] |= BOOL_CMPR_TABLE[mod4];
   }
   pCmprsor->bool_n++;
-  if (mod4 == 3) {
-    pCmprsor->nBuf[0]++;
-    pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = 0;
 
-    code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0]);
-    if (code) goto _exit;
-  }
-
-_exit:
   return code;
 }
 
@@ -1444,16 +1456,30 @@ int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg, int
       pCmprsor->ts_prev_val = 0;
       pCmprsor->ts_prev_delta = 0;
       pCmprsor->ts_flag_p = NULL;
-      pCmprsor->aBuf[0][0] = MODE_COMPRESS;
+      pCmprsor->aBuf[0][0] = 1;  // For timestamp, 1 means compressed, 0 otherwise
       pCmprsor->nBuf[0] = 1;
-      pCmprsor->nBuf[1] = 0;
       break;
     case TSDB_DATA_TYPE_BOOL:
       pCmprsor->bool_n = 0;
-      pCmprsor->aBuf[0][0] = 0;
+      pCmprsor->nBuf[0] = 0;
       break;
     case TSDB_DATA_TYPE_BINARY:
-      pCmprsor->b_n = 0;
+      pCmprsor->binary_n = 0;
+      pCmprsor->nBuf[0] = 0;
+      break;
+    case TSDB_DATA_TYPE_FLOAT:
+      pCmprsor->f_n = 0;
+      pCmprsor->f_prev = 0;
+      pCmprsor->f_flag_p = NULL;
+      pCmprsor->aBuf[0][0] = 0;  // 0 means compressed, 1 otherwise (for backward compatibility)
+      pCmprsor->nBuf[0] = 1;
+      break;
+    case TSDB_DATA_TYPE_DOUBLE:
+      pCmprsor->d_n = 0;
+      pCmprsor->d_prev = 0;
+      pCmprsor->d_flag_p = NULL;
+      pCmprsor->aBuf[0][0] = 0;  // 0 means compressed, 1 otherwise (for backward compatibility)
+      pCmprsor->nBuf[0] = 1;
       break;
     default:
       break;

From 4e36fe0c1bbdb8ea5829cea8d852feb6b60d5c2e Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Mon, 19 Sep 2022 14:54:31 +0800
Subject: [PATCH 29/57] feat: update taos-tools 318246c for 3.0

---
 cmake/taostools_CMakeLists.txt.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in
index 6a3439ada9..bcf8392052 100644
--- a/cmake/taostools_CMakeLists.txt.in
+++ b/cmake/taostools_CMakeLists.txt.in
@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG 125c77a
+        GIT_TAG 318246c
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

From 0a497f6f213e64cda9682d55ef1cbb45cf6c9a22 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Mon, 19 Sep 2022 15:07:18 +0800
Subject: [PATCH 30/57] fix:error in auto create table for taosX

---
 utils/test/c/tmqSim.c | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c
index 1e8a7350f1..a043aa7a6d 100644
--- a/utils/test/c/tmqSim.c
+++ b/utils/test/c/tmqSim.c
@@ -653,15 +653,15 @@ static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn
     int32_t code = tmq_get_raw(msg, &raw);
 	
     if(code == TSDB_CODE_SUCCESS){
-	  int retCode = queryDB(pInfo->taos, "use metadb");
-	  if (retCode != 0) {
-		taosFprintfFile(g_fp, "error when use metadb\n");
-		taosCloseFile(&g_fp);
-		exit(-1);
-	  }	  
-	  taosFprintfFile(g_fp, "raw:%p\n", &raw);
-	
-      tmq_write_raw(pInfo->taos, raw);
+//	  int retCode = queryDB(pInfo->taos, "use metadb");
+//	  if (retCode != 0) {
+//		taosFprintfFile(g_fp, "error when use metadb\n");
+//		taosCloseFile(&g_fp);
+//		exit(-1);
+//	  }
+//	  taosFprintfFile(g_fp, "raw:%p\n", &raw);
+//
+//      tmq_write_raw(pInfo->taos, raw);
     }
 	
     char* result = tmq_get_json_meta(msg);

From d48a56326019f9444a84efc1dad280944dd7e774 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 19 Sep 2022 15:42:03 +0800
Subject: [PATCH 31/57] fix(query): assign index back.

---
 source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
index 45fe29f0fa..5f03a82bc0 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
@@ -320,6 +320,7 @@ void tLDataIterNextBlock(SLDataIter *pIter) {
 
   pIter->pSttBlk = NULL;
   if (index != -1) {
+    pIter->iSttBlk = index;
     pIter->pSttBlk = (SSttBlk *)taosArrayGet(pIter->pBlockLoadInfo->aSttBlk, pIter->iSttBlk);
   }
 }

From e33c255ad189fda15d0c9ae451ba432546894ea5 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 19 Sep 2022 15:57:58 +0800
Subject: [PATCH 32/57] more code

---
 source/util/src/tcompression.c | 243 +++++++++++++++++++++------------
 1 file changed, 156 insertions(+), 87 deletions(-)

diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c
index 4bec2f16ed..748ab8c975 100644
--- a/source/util/src/tcompression.c
+++ b/source/util/src/tcompression.c
@@ -1000,59 +1000,76 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co
  *                  STREAM COMPRESSION
  *************************************************************************/
 #define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (b)) || ((a) < 0 && (b) >= INT64_MIN - (a)))
+typedef struct SCompressor SCompressor;
 
-typedef struct {
+static struct {
+  int8_t  type;
+  int32_t bytes;
+  int8_t  isVarLen;
+  int32_t (*cmprFn)(SCompressor *, const void *, int32_t nData);
+} DATA_TYPE_INFO[] = {
+    {TSDB_DATA_TYPE_NULL, 0, 0},        // TSDB_DATA_TYPE_NULL
+    {TSDB_DATA_TYPE_BOOL, 1, 0},        // TSDB_DATA_TYPE_BOOL
+    {TSDB_DATA_TYPE_TINYINT, 1, 0},     // TSDB_DATA_TYPE_TINYINT
+    {TSDB_DATA_TYPE_SMALLINT, 2, 0},    // TSDB_DATA_TYPE_SMALLINT
+    {TSDB_DATA_TYPE_INT, 4, 0},         // TSDB_DATA_TYPE_INT
+    {TSDB_DATA_TYPE_BIGINT, 8, 0},      // TSDB_DATA_TYPE_BIGINT
+    {TSDB_DATA_TYPE_FLOAT, 4, 0},       // TSDB_DATA_TYPE_FLOAT
+    {TSDB_DATA_TYPE_DOUBLE, 8, 0},      // TSDB_DATA_TYPE_DOUBLE
+    {TSDB_DATA_TYPE_VARCHAR, 1, 1},     // TSDB_DATA_TYPE_VARCHAR
+    {TSDB_DATA_TYPE_TIMESTAMP, 8, 0},   // pTSDB_DATA_TYPE_TIMESTAMP
+    {TSDB_DATA_TYPE_NCHAR, 1, 1},       // TSDB_DATA_TYPE_NCHAR
+    {TSDB_DATA_TYPE_UTINYINT, 1, 0},    // TSDB_DATA_TYPE_UTINYINT
+    {TSDB_DATA_TYPE_USMALLINT, 2, 0},   // TSDB_DATA_TYPE_USMALLINT
+    {TSDB_DATA_TYPE_UINT, 4, 0},        // TSDB_DATA_TYPE_UINT
+    {TSDB_DATA_TYPE_UBIGINT, 8, 0},     // TSDB_DATA_TYPE_UBIGINT
+    {TSDB_DATA_TYPE_JSON, 1, 1},        // TSDB_DATA_TYPE_JSON
+    {TSDB_DATA_TYPE_VARBINARY, 1, 1},   // TSDB_DATA_TYPE_VARBINARY
+    {TSDB_DATA_TYPE_DECIMAL, 1, 1},     // TSDB_DATA_TYPE_DECIMAL
+    {TSDB_DATA_TYPE_BLOB, 1, 1},        // TSDB_DATA_TYPE_BLOB
+    {TSDB_DATA_TYPE_MEDIUMBLOB, 1, 1},  // TSDB_DATA_TYPE_MEDIUMBLOB
+};
+
+struct SCompressor {
   int8_t   type;
   int8_t   cmprAlg;
   int8_t   autoAlloc;
+  int32_t  nVal;
   uint8_t *aBuf[2];
   int64_t  nBuf[2];
   union {
     // Timestamp ----
     struct {
-      int32_t  ts_n;
       int64_t  ts_prev_val;
       int64_t  ts_prev_delta;
       uint8_t *ts_flag_p;
     };
     // Integer ----
     struct {
-      int8_t  i_copy;
-      int32_t i_n;
       int64_t i_prev;
       int32_t i_selector;
       int32_t i_nele;
     };
     // Float ----
     struct {
-      int32_t  f_n;
       uint32_t f_prev;
       uint8_t *f_flag_p;
     };
     // Double ----
     struct {
-      int32_t  d_n;
       uint64_t d_prev;
       uint8_t *d_flag_p;
     };
-    // Bool ----
-    struct {
-      int32_t bool_n;
-    };
-    // Binary ----
-    struct {
-      int32_t binary_n;
-    };
   };
-} SCompressor;
+};
 
 // Timestamp =====================================================
 static int32_t tCompSetCopyMode(SCompressor *pCmprsor) {
   int32_t code = 0;
 
-  if (pCmprsor->ts_n) {
+  if (pCmprsor->nVal) {
     if (pCmprsor->autoAlloc) {
-      code = tRealloc(&pCmprsor->aBuf[1], sizeof(int64_t) * pCmprsor->ts_n);
+      code = tRealloc(&pCmprsor->aBuf[1], sizeof(int64_t) * pCmprsor->nVal);
       if (code) return code;
     }
     pCmprsor->nBuf[1] = 0;
@@ -1110,7 +1127,7 @@ static int32_t tCompTimestamp(SCompressor *pCmprsor, int64_t ts) {
   ASSERT(pCmprsor->type == TSDB_DATA_TYPE_TIMESTAMP);
 
   if (pCmprsor->aBuf[0][0] == 1) {
-    if (pCmprsor->ts_n == 0) {
+    if (pCmprsor->nVal == 0) {
       pCmprsor->ts_prev_val = ts;
       pCmprsor->ts_prev_delta = -ts;
     }
@@ -1133,7 +1150,7 @@ static int32_t tCompTimestamp(SCompressor *pCmprsor, int64_t ts) {
     pCmprsor->ts_prev_val = ts;
     pCmprsor->ts_prev_delta = delta;
 
-    if ((pCmprsor->ts_n & 0x1) == 0) {
+    if ((pCmprsor->nVal & 0x1) == 0) {
       if (pCmprsor->autoAlloc) {
         code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17);
         if (code) return code;
@@ -1166,7 +1183,7 @@ static int32_t tCompTimestamp(SCompressor *pCmprsor, int64_t ts) {
     memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], &ts, sizeof(ts));
     pCmprsor->nBuf[0] += sizeof(ts);
   }
-  pCmprsor->ts_n++;
+  pCmprsor->nVal++;
 
   return code;
 }
@@ -1180,76 +1197,114 @@ static const char    bit_to_selector[] = {0,  2,  3,  4,  5,  6,  7,  8,  9,  10
                                           15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
                                           15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15};
 
-static int32_t tCompI64(SCompressor *pCmprsor, int64_t val) {
+static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) {
   int32_t code = 0;
 
-  if (pCmprsor->i_copy == 1) goto _copy_cmpr;
+  ASSERT(nData == DATA_TYPE_INFO[pCmprsor->type].bytes);
 
-  if (!I64_SAFE_ADD(val, pCmprsor->i_prev)) {
-    // TODO
-    goto _copy_cmpr;
-  }
+  if (pCmprsor->aBuf[0][0] == 0) {
+    int64_t val;
 
-  int64_t  diff = val - pCmprsor->i_prev;
-  uint64_t vZigzag = ZIGZAG_ENCODE(int64_t, diff);
-
-  if (vZigzag >= SIMPLE8B_MAX) {
-    // TODO
-    goto _copy_cmpr;
-  }
-
-  int64_t nBit;
-  if (vZigzag) {
-    nBit = 64 - BUILDIN_CLZL(vZigzag);
-  } else {
-    nBit = 0;
-  }
-
-  if (pCmprsor->i_nele + 1 <= selector_to_elems[pCmprsor->i_selector] &&
-      pCmprsor->i_nele + 1 <= selector_to_elems[bit_to_selector[nBit]]) {
-    if (pCmprsor->i_selector < bit_to_selector[nBit]) {
-      pCmprsor->i_selector = bit_to_selector[nBit];
+    switch (pCmprsor->type) {
+      case TSDB_DATA_TYPE_TINYINT:
+        val = *(int8_t *)pData;
+        break;
+      case TSDB_DATA_TYPE_SMALLINT:
+        val = *(int16_t *)pData;
+        break;
+      case TSDB_DATA_TYPE_INT:
+        val = *(int32_t *)pData;
+        break;
+      case TSDB_DATA_TYPE_BIGINT:
+        val = *(int64_t *)pData;
+        break;
+      case TSDB_DATA_TYPE_UTINYINT:
+        val = *(uint8_t *)pData;
+        break;
+      case TSDB_DATA_TYPE_USMALLINT:
+        val = *(uint16_t *)pData;
+        break;
+      case TSDB_DATA_TYPE_UINT:
+        val = *(uint32_t *)pData;
+        break;
+      // case TSDB_DATA_TYPE_UBIGINT:
+      //   val = *(int64_t *)pData;
+      //   break;
+      default:
+        ASSERT(0);
+        break;
     }
-    pCmprsor->i_nele++;
-  } else {
-    while (pCmprsor->i_nele < selector_to_elems[pCmprsor->i_selector]) {
-      pCmprsor->i_selector++;
-    }
-    pCmprsor->i_nele = selector_to_elems[pCmprsor->i_selector];
 
-    code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(uint64_t));
+    if (!I64_SAFE_ADD(val, pCmprsor->i_prev)) {
+      // TODO
+      goto _copy_cmpr;
+    }
+
+    int64_t  diff = val - pCmprsor->i_prev;
+    uint64_t vZigzag = ZIGZAG_ENCODE(int64_t, diff);
+
+    if (vZigzag >= SIMPLE8B_MAX) {
+      // TODO
+      goto _copy_cmpr;
+    }
+
+    int64_t nBit;
+    if (vZigzag) {
+      nBit = 64 - BUILDIN_CLZL(vZigzag);
+    } else {
+      nBit = 0;
+    }
+
+    if (pCmprsor->i_nele + 1 <= selector_to_elems[pCmprsor->i_selector] &&
+        pCmprsor->i_nele + 1 <= selector_to_elems[bit_to_selector[nBit]]) {
+      if (pCmprsor->i_selector < bit_to_selector[nBit]) {
+        pCmprsor->i_selector = bit_to_selector[nBit];
+      }
+      pCmprsor->i_nele++;
+      pCmprsor->i_prev = val;
+    } else {
+      while (pCmprsor->i_nele < selector_to_elems[pCmprsor->i_selector]) {
+        pCmprsor->i_selector++;
+      }
+      pCmprsor->i_nele = selector_to_elems[pCmprsor->i_selector];
+
+      code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(uint64_t));
+      if (code) return code;
+
+      uint64_t *bp = (uint64_t *)(pCmprsor->aBuf[0] + pCmprsor->nBuf[0]);
+      pCmprsor->nBuf[0] += sizeof(uint64_t);
+      bp[0] = pCmprsor->i_selector;
+      for (int32_t iVal = 0; iVal < pCmprsor->i_nele; iVal++) {
+        /* code */
+      }
+
+      // reset and continue
+      pCmprsor->i_nele = 0;
+      pCmprsor->i_selector = 0;
+    }
+  } else {
+  _copy_cmpr:
+    code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + nData);
     if (code) return code;
 
-    uint64_t *bp = (uint64_t *)(pCmprsor->aBuf[0] + pCmprsor->nBuf[0]);
-    pCmprsor->nBuf[0] += sizeof(uint64_t);
-    bp[0] = pCmprsor->i_selector;
-    for (int32_t iVal = 0; iVal < pCmprsor->i_nele; iVal++) {
-      /* code */
-    }
-
-    // reset and continue
+    memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], pData, nData);
+    pCmprsor->nBuf[0] += nData;
   }
-
-  return code;
-
-_copy_cmpr:
-  code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 0 /*tDataTypes[pCmprsor->type].bytes (todo)*/);
-  if (code) return code;
-
-  // memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], NULL /* todo */, 0 /*tDataTypes[pCmprsor->type].bytes (todo)*/);
-  // pCmprsor->nBuf[0] += tDataTypes[pCmprsor->type].bytes;
+  pCmprsor->nVal++;
 
   return code;
 }
 
 // Float =====================================================
-static int32_t tCompFloat(SCompressor *pCmprsor, float f) {
+static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nData) {
   int32_t code = 0;
 
+  ASSERT(nData == sizeof(float));
+
   union {
     float    f;
     uint32_t u;
-  } val = {.f = f};
+  } val = {.f = *(float *)pData};
 
   uint32_t diff = val.u ^ pCmprsor->f_prev;
   pCmprsor->f_prev = val.u;
@@ -1272,7 +1327,7 @@ static int32_t tCompFloat(SCompressor *pCmprsor, float f) {
   }
   if (nBytes == 0) nBytes++;
 
-  if ((pCmprsor->f_n & 0x1) == 0) {
+  if ((pCmprsor->nVal & 0x1) == 0) {
     if (pCmprsor->autoAlloc) {
       code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 9);
       if (code) return code;
@@ -1298,19 +1353,21 @@ static int32_t tCompFloat(SCompressor *pCmprsor, float f) {
     pCmprsor->nBuf[0]++;
     diff >>= BITS_PER_BYTE;
   }
-  pCmprsor->f_n++;
+  pCmprsor->nVal++;
 
   return code;
 }
 
 // Double =====================================================
-static int32_t tCompDouble(SCompressor *pCmprsor, double d) {
+static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nData) {
   int32_t code = 0;
 
+  ASSERT(nData == sizeof(double));
+
   union {
     double   d;
     uint64_t u;
-  } val = {.d = d};
+  } val = {.d = *(double *)pData};
 
   uint64_t diff = val.u ^ pCmprsor->d_prev;
   pCmprsor->d_prev = val.u;
@@ -1333,7 +1390,7 @@ static int32_t tCompDouble(SCompressor *pCmprsor, double d) {
   }
   if (nBytes == 0) nBytes++;
 
-  if ((pCmprsor->d_n & 0x1) == 0) {
+  if ((pCmprsor->nVal & 0x1) == 0) {
     if (pCmprsor->autoAlloc) {
       code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17);
       if (code) return code;
@@ -1359,13 +1416,13 @@ static int32_t tCompDouble(SCompressor *pCmprsor, double d) {
     pCmprsor->nBuf[0]++;
     diff >>= BITS_PER_BYTE;
   }
-  pCmprsor->d_n++;
+  pCmprsor->nVal++;
 
   return code;
 }
 
 // Binary =====================================================
-static int32_t tCompBinary(SCompressor *pCmprsor, const uint8_t *pData, int32_t nData) {
+static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData) {
   int32_t code = 0;
 
   if (nData) {
@@ -1377,7 +1434,7 @@ static int32_t tCompBinary(SCompressor *pCmprsor, const uint8_t *pData, int32_t
     memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], pData, nData);
     pCmprsor->nBuf[0] += nData;
   }
-  pCmprsor->binary_n++;
+  pCmprsor->nVal++;
 
   return code;
 }
@@ -1385,10 +1442,12 @@ static int32_t tCompBinary(SCompressor *pCmprsor, const uint8_t *pData, int32_t
 // Bool =====================================================
 static const uint8_t BOOL_CMPR_TABLE[] = {0b01, 0b0100, 0b010000, 0b01000000};
 
-static int32_t tCompBool(SCompressor *pCmprsor, bool vBool) {
+static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData) {
   int32_t code = 0;
 
-  int32_t mod4 = pCmprsor->bool_n & 3;
+  bool vBool = *(int8_t *)pData;
+
+  int32_t mod4 = pCmprsor->nVal & 3;
   if (mod4 == 0) {
     pCmprsor->nBuf[0]++;
 
@@ -1402,7 +1461,7 @@ static int32_t tCompBool(SCompressor *pCmprsor, bool vBool) {
   if (vBool) {
     pCmprsor->aBuf[0][pCmprsor->nBuf[0] - 1] |= BOOL_CMPR_TABLE[mod4];
   }
-  pCmprsor->bool_n++;
+  pCmprsor->nVal++;
 
   return code;
 }
@@ -1449,10 +1508,10 @@ int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg, int
   pCmprsor->type = type;
   pCmprsor->cmprAlg = cmprAlg;
   pCmprsor->autoAlloc = autoAlloc;
+  pCmprsor->nVal = 0;
 
   switch (type) {
     case TSDB_DATA_TYPE_TIMESTAMP:
-      pCmprsor->ts_n = 0;
       pCmprsor->ts_prev_val = 0;
       pCmprsor->ts_prev_delta = 0;
       pCmprsor->ts_flag_p = NULL;
@@ -1460,27 +1519,37 @@ int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg, int
       pCmprsor->nBuf[0] = 1;
       break;
     case TSDB_DATA_TYPE_BOOL:
-      pCmprsor->bool_n = 0;
       pCmprsor->nBuf[0] = 0;
       break;
     case TSDB_DATA_TYPE_BINARY:
-      pCmprsor->binary_n = 0;
       pCmprsor->nBuf[0] = 0;
       break;
     case TSDB_DATA_TYPE_FLOAT:
-      pCmprsor->f_n = 0;
       pCmprsor->f_prev = 0;
       pCmprsor->f_flag_p = NULL;
       pCmprsor->aBuf[0][0] = 0;  // 0 means compressed, 1 otherwise (for backward compatibility)
       pCmprsor->nBuf[0] = 1;
       break;
     case TSDB_DATA_TYPE_DOUBLE:
-      pCmprsor->d_n = 0;
       pCmprsor->d_prev = 0;
       pCmprsor->d_flag_p = NULL;
       pCmprsor->aBuf[0][0] = 0;  // 0 means compressed, 1 otherwise (for backward compatibility)
       pCmprsor->nBuf[0] = 1;
       break;
+    case TSDB_DATA_TYPE_TINYINT:
+    case TSDB_DATA_TYPE_SMALLINT:
+    case TSDB_DATA_TYPE_INT:
+    case TSDB_DATA_TYPE_BIGINT:
+    case TSDB_DATA_TYPE_UTINYINT:
+    case TSDB_DATA_TYPE_USMALLINT:
+    case TSDB_DATA_TYPE_UINT:
+    case TSDB_DATA_TYPE_UBIGINT:
+      pCmprsor->i_prev = 0;
+      pCmprsor->i_selector = 0;
+      pCmprsor->i_nele = 0;
+      pCmprsor->aBuf[0][0] = 0;  // 0 means compressed, 1 otherwise (for backward compatibility)
+      pCmprsor->nBuf[0] = 1;
+      break;
     default:
       break;
   }

From 7b58a94a3591213cec03793f35c596686849435f Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Mon, 19 Sep 2022 16:17:27 +0800
Subject: [PATCH 33/57] feat: update taos-tools 509ec72 for 3.0

---
 cmake/taostools_CMakeLists.txt.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in
index bcf8392052..5cc580a9c6 100644
--- a/cmake/taostools_CMakeLists.txt.in
+++ b/cmake/taostools_CMakeLists.txt.in
@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG 318246c
+        GIT_TAG 509ec72
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

From c379a35fb3e0ff99d7f13102259c72971970dc63 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Mon, 19 Sep 2022 16:34:07 +0800
Subject: [PATCH 34/57] fix:error in CI

---
 source/dnode/vnode/src/meta/metaQuery.c | 2 +-
 source/libs/scalar/src/sclfunc.c        | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c
index 9b199ecd37..24ade017d3 100644
--- a/source/dnode/vnode/src/meta/metaQuery.c
+++ b/source/dnode/vnode/src/meta/metaQuery.c
@@ -388,8 +388,8 @@ int metaTtlSmaller(SMeta *pMeta, uint64_t ttl, SArray *uidList) {
     }
     ttlKey = *(STtlIdxKey *)pKey;
     taosArrayPush(uidList, &ttlKey.uid);
-    tdbFree(pKey);
   }
+  tdbFree(pKey);
   tdbTbcClose(pCur);
   return 0;
 }
diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c
index 4bc2ce771d..4ead1147e4 100644
--- a/source/libs/scalar/src/sclfunc.c
+++ b/source/libs/scalar/src/sclfunc.c
@@ -1153,8 +1153,7 @@ int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
       memcpy(tmp, varDataVal(input), varDataLen(input));
       tmp[varDataLen(input)] = 0;
       if(parseJsontoTagData(tmp, pTagVals, &pTag, NULL)){
-        taosArrayDestroy(pTagVals);
-        return TSDB_CODE_FAILED;
+        tTagNew(pTagVals, 1, true, &pTag);
       }
     }
 

From 2a494b267157f4a4b12d362390fe157d3131af72 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Mon, 19 Sep 2022 16:56:35 +0800
Subject: [PATCH 35/57] fix(tsc): fix taosdump failure

---
 source/client/src/clientHb.c | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c
index 7ce80553a0..75ccd44977 100644
--- a/source/client/src/clientHb.c
+++ b/source/client/src/clientHb.c
@@ -878,12 +878,18 @@ int hbMgrInit() {
   clientHbMgr.appHbMgrs = taosArrayInit(0, sizeof(void *));
 
   TdThreadMutexAttr attr = {0};
-  taosThreadMutexAttrSetType(&attr, PTHREAD_MUTEX_RECURSIVE);
+
   int ret = taosThreadMutexAttrInit(&attr);
   assert(ret == 0);
 
-  taosThreadMutexInit(&clientHbMgr.lock, &attr);
-  taosThreadMutexAttrDestroy(&attr);
+  ret = taosThreadMutexAttrSetType(&attr, PTHREAD_MUTEX_RECURSIVE);
+  assert(ret == 0);
+
+  ret = taosThreadMutexInit(&clientHbMgr.lock, &attr);
+  assert(ret == 0);
+
+  ret = taosThreadMutexAttrDestroy(&attr);
+  assert(ret == 0);
 
   // init handle funcs
   hbMgrInitHandle();

From b08288df489f2e4d9d6dbd9ca09e688dc98ba3f2 Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Mon, 19 Sep 2022 17:07:22 +0800
Subject: [PATCH 36/57] fix(query): fix invalid read in doModeAdd

TD-19124
---
 source/libs/function/src/builtinsimpl.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c
index 9b502eded7..a23f58a732 100644
--- a/source/libs/function/src/builtinsimpl.c
+++ b/source/libs/function/src/builtinsimpl.c
@@ -5297,12 +5297,12 @@ bool modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) {
 }
 
 static void doModeAdd(SModeInfo* pInfo, char* data) {
-  int32_t     hashKeyBytes = IS_VAR_DATA_TYPE(pInfo->colType) ? varDataTLen(data) : pInfo->colBytes;
+  int32_t     hashKeyBytes = IS_STR_DATA_TYPE(pInfo->colType) ? varDataTLen(data) : pInfo->colBytes;
   SModeItem** pHashItem = taosHashGet(pInfo->pHash, data, hashKeyBytes);
   if (pHashItem == NULL) {
     int32_t    size = sizeof(SModeItem) + pInfo->colBytes;
     SModeItem* pItem = (SModeItem*)(pInfo->pItems + pInfo->numOfPoints * size);
-    memcpy(pItem->data, data, pInfo->colBytes);
+    memcpy(pItem->data, data, hashKeyBytes);
     pItem->count += 1;
 
     taosHashPut(pInfo->pHash, data, hashKeyBytes, &pItem, sizeof(SModeItem*));

From 955838d1f6a9a7f60762807e7b370a88de6f8101 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao@163.com>
Date: Mon, 19 Sep 2022 15:51:35 +0800
Subject: [PATCH 37/57] feat(stream): optimize disc buff

---
 source/libs/executor/inc/executorimpl.h       |   3 +-
 source/libs/executor/src/executorimpl.c       |  10 +-
 source/libs/executor/src/timewindowoperator.c | 144 ++++++++++++++----
 source/libs/stream/src/streamState.c          |  22 +--
 4 files changed, 136 insertions(+), 43 deletions(-)

diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index 18f6d3ad2c..a516a9d11f 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -462,6 +462,7 @@ typedef struct SPartitionDataInfo {
 typedef struct STimeWindowAggSupp {
   int8_t          calTrigger;
   int64_t         waterMark;
+  int64_t         deleteMark;
   TSKEY           maxTs;
   TSKEY           minTs;
   SColumnInfoData timeWindowData;  // query time window info for scalar function execution.
@@ -1090,7 +1091,7 @@ int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupI
                            int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup,
                            SExecTaskInfo* pTaskInfo);
 int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult);
-int32_t saveOutput(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize);
+int32_t saveOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize);
 
 #ifdef __cplusplus
 }
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index e0d417093c..f6bb1ffe42 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -4166,9 +4166,8 @@ int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupI
   };
   char*   value = NULL;
   int32_t size = pAggSup->resultRowSize;
-  /*if (streamStateGet(pTaskInfo->streamInfo.pState, &key, (void**)&value, &size) < 0) {*/
-  /*value = taosMemoryCalloc(1, size);*/
-  /*}*/
+
+  tSimpleHashPut(pAggSup->pResultRowHashTable, &key, sizeof(SWinKey), NULL, 0);
   if (streamStateAddIfNotExist(pTaskInfo->streamInfo.pState, &key, (void**)&value, &size) < 0) {
     return TSDB_CODE_QRY_OUT_OF_MEMORY;
   }
@@ -4186,7 +4185,7 @@ int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pR
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t saveOutput(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize) {
+int32_t saveOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize) {
   streamStatePut(pTaskInfo->streamInfo.pState, pKey, pResult, resSize);
   return TSDB_CODE_SUCCESS;
 }
@@ -4259,8 +4258,9 @@ int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock
         }
       }
     }
-    releaseOutputBuf(pTaskInfo, &key, pRow);
+
     pBlock->info.rows += pRow->numOfRows;
+    releaseOutputBuf(pTaskInfo, &key, pRow);
   }
   blockDataUpdateTsWindow(pBlock, 0);
   return TSDB_CODE_SUCCESS;
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index c221aaf4fd..81c0f9cd0c 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -867,6 +867,10 @@ static int32_t saveWinResultRow(SResultRow* result, uint64_t groupId, SHashObj*
   return saveWinResult(result->win.skey, result->pageId, result->offset, groupId, pUpdatedMap);
 }
 
+static int32_t saveWinResultInfo(TSKEY ts, uint64_t groupId, SHashObj* pUpdatedMap) {
+  return saveWinResult(ts, -1, -1, groupId, pUpdatedMap);
+}
+
 static int32_t saveResultRow(SResultRow* result, uint64_t groupId, SArray* pUpdated) {
   return saveResult(result->win.skey, result->pageId, result->offset, groupId, pUpdated);
 }
@@ -918,12 +922,16 @@ static void removeDeleteResults(SHashObj* pUpdatedMap, SArray* pDelWins) {
   }
 }
 
-bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup) {
-  ASSERT(pSup->maxTs == INT64_MIN || pSup->maxTs > 0);
-  return pSup->maxTs != INT64_MIN && ts < pSup->maxTs - pSup->waterMark;
+bool isOverdue(TSKEY ekey, STimeWindowAggSupp* pTwSup) {
+  ASSERT(pTwSup->maxTs == INT64_MIN || pTwSup->maxTs > 0);
+  return pTwSup->maxTs != INT64_MIN && ekey < pTwSup->maxTs - pTwSup->waterMark;
 }
 
-bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup) { return isOverdue(pWin->ekey, pSup); }
+bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pTwSup) { return isOverdue(pWin->ekey, pTwSup); }
+
+bool needDeleteWindowBuf(STimeWindow* pWin, STimeWindowAggSupp* pTwSup) {
+  return pTwSup->maxTs != INT64_MIN && pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark;
+}
 
 static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
                             int32_t scanFlag) {
@@ -1374,6 +1382,41 @@ static bool doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData,
   return true;
 }
 
+static bool doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, int32_t numOfOutput) {
+  SStreamIntervalOperatorInfo* pInfo = pOperator->info;
+  SWinKey                      key = {.ts = ts, .groupId = groupId};
+  tSimpleHashRemove(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey));
+  streamStateDel(pOperator->pTaskInfo->streamInfo.pState, &key);
+  return true;
+}
+
+static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, int32_t numOfOutput, SSDataBlock* pBlock,
+                            SArray* pUpWins, SHashObj* pUpdatedMap) {
+  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
+  TSKEY*           startTsCols = (TSKEY*)pStartTsCol->pData;
+  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
+  TSKEY*           endTsCols = (TSKEY*)pEndTsCol->pData;
+  SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
+  uint64_t*        pGpDatas = (uint64_t*)pGpCol->pData;
+  for (int32_t i = 0; i < pBlock->info.rows; i++) {
+    SResultRowInfo dumyInfo;
+    dumyInfo.cur.pageId = -1;
+    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC);
+    while (win.ekey <= endTsCols[i]) {
+      uint64_t winGpId = pGpDatas[i];
+      bool     res = doDeleteWindow(pOperator, win.skey, winGpId, numOfOutput);
+      SWinKey  winRes = {.ts = win.skey, .groupId = winGpId};
+      if (pUpWins && res) {
+        taosArrayPush(pUpWins, &winRes);
+      }
+      if (pUpdatedMap) {
+        taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey));
+      }
+      getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win);
+    }
+  }
+}
+
 bool doDeleteIntervalWindow(SAggSupporter* pAggSup, TSKEY ts, uint64_t groupId) {
   size_t bytes = sizeof(TSKEY);
   SET_RES_WINDOW_KEY(pAggSup->keyBuf, &ts, bytes, groupId);
@@ -1383,8 +1426,6 @@ bool doDeleteIntervalWindow(SAggSupporter* pAggSup, TSKEY ts, uint64_t groupId)
     // window has been closed
     return false;
   }
-  // SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, p1->pageId);
-  // dBufSetBufPageRecycled(pAggSup->pResultBuf, bufPage);
   tSimpleHashRemove(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
   return true;
 }
@@ -1512,6 +1553,49 @@ static int32_t closeIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pSup
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t closeStreamIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SInterval* pInterval,
+                                         SHashObj* pPullDataMap, SHashObj* closeWins, SOperatorInfo* pOperator) {
+  qDebug("===stream===close interval window");
+  void*   pIte = NULL;
+  size_t  keyLen = 0;
+  int32_t iter = 0;
+  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {
+    void*       key = tSimpleHashGetKey(pIte, &keyLen);
+    SWinKey*    pWinKey = (SWinKey*)key;
+    void*       chIds = taosHashGet(pPullDataMap, pWinKey, sizeof(SWinKey));
+    STimeWindow win = {
+        .skey = pWinKey->ts,
+        .ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1,
+    };
+    if (isCloseWindow(&win, pTwSup)) {
+      if (chIds && pPullDataMap) {
+        SArray* chAy = *(SArray**)chIds;
+        int32_t size = taosArrayGetSize(chAy);
+        qDebug("===stream===window %" PRId64 " wait child size:%d", pWinKey->ts, size);
+        for (int32_t i = 0; i < size; i++) {
+          qDebug("===stream===window %" PRId64 " wait child id:%d", pWinKey->ts, *(int32_t*)taosArrayGet(chAy, i));
+        }
+        continue;
+      } else if (pPullDataMap) {
+        qDebug("===stream===close window %" PRId64, pWinKey->ts);
+      }
+
+      if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
+        int32_t code = saveWinResultInfo(pWinKey->ts, pWinKey->groupId, closeWins);
+        if (code != TSDB_CODE_SUCCESS) {
+          return code;
+        }
+      }
+      tSimpleHashIterateRemove(pHashMap, pWinKey, sizeof(SWinKey), &pIte, &iter);
+
+      if (needDeleteWindowBuf(&win, pTwSup)) {
+        streamStateDel(pOperator->pTaskInfo->streamInfo.pState, pWinKey);
+      }
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 static void closeChildIntervalWindow(SArray* pChildren, TSKEY maxTs) {
   int32_t size = taosArrayGetSize(pChildren);
   for (int32_t i = 0; i < size; i++) {
@@ -4918,8 +5002,8 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR
   SExprSupp*     pSup = &pOperatorInfo->exprSupp;
   SInterval*     pInterval = &iaInfo->interval;
 
-  int32_t        startPos = 0;
-  int64_t*       tsCols = extractTsCol(pBlock, iaInfo);
+  int32_t  startPos = 0;
+  int64_t* tsCols = extractTsCol(pBlock, iaInfo);
 
   TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols);
 
@@ -4938,7 +5022,7 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR
   win.skey = miaInfo->curTs;
   win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
 
-  int32_t ret = setSingleOutputTupleBuf(pResultRowInfo, &win,  &miaInfo->pResultRow, pSup, &iaInfo->aggSup);
+  int32_t ret = setSingleOutputTupleBuf(pResultRowInfo, &win, &miaInfo->pResultRow, pSup, &iaInfo->aggSup);
   if (ret != TSDB_CODE_SUCCESS || miaInfo->pResultRow == NULL) {
     T_LONG_JMP(pTaskInfo->env, ret);
   }
@@ -4963,7 +5047,7 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR
     currWin.ekey = taosTimeAdd(currWin.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
 
     startPos = currPos;
-    ret = setSingleOutputTupleBuf(pResultRowInfo, &win,  &miaInfo->pResultRow, pSup, &iaInfo->aggSup);
+    ret = setSingleOutputTupleBuf(pResultRowInfo, &win, &miaInfo->pResultRow, pSup, &iaInfo->aggSup);
     if (ret != TSDB_CODE_SUCCESS || miaInfo->pResultRow == NULL) {
       T_LONG_JMP(pTaskInfo->env, ret);
     }
@@ -5032,7 +5116,7 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
         pMiaInfo->prefetchedBlock = pBlock;
         cleanupAfterGroupResultGen(pMiaInfo, pRes);
         break;
-      } else  {
+      } else {
         // continue
       }
     }
@@ -5197,7 +5281,7 @@ static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t table
   SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(
       iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
   ASSERT(p1 != NULL);
-//  finalizeResultRows(iaInfo->aggSup.pResultBuf, p1, pResultBlock, pTaskInfo);
+  //  finalizeResultRows(iaInfo->aggSup.pResultBuf, p1, pResultBlock, pTaskInfo);
   tSimpleHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
   return TSDB_CODE_SUCCESS;
 }
@@ -5222,7 +5306,7 @@ static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t t
 
     STimeWindow* prevWin = &prevGrpWin->window;
     if ((ascScan && newWin->skey > prevWin->ekey) || ((!ascScan) && newWin->skey < prevWin->ekey)) {
-//      finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock);
+      //      finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock);
       tdListPopNode(miaInfo->groupIntervals, listNode);
     }
   }
@@ -5382,7 +5466,7 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) {
 
     if (listNode != NULL) {
       SGroupTimeWindow* grpWin = (SGroupTimeWindow*)(listNode->data);
-//      finalizeWindowResult(pOperator, grpWin->groupId, &grpWin->window, pRes);
+      //      finalizeWindowResult(pOperator, grpWin->groupId, &grpWin->window, pRes);
       pRes->info.groupId = grpWin->groupId;
     }
   }
@@ -5591,7 +5675,7 @@ static void doStreamIntervalAggImpl2(SOperatorInfo* pOperatorInfo, SSDataBlock*
     forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL,
                                            TSDB_ORDER_ASC);
     if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) {
-      saveWinResultRow(pResult, tableGroupId, pUpdatedMap);
+      saveWinResultInfo(pResult->win.skey, tableGroupId, pUpdatedMap);
     }
     updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
     doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
@@ -5600,7 +5684,7 @@ static void doStreamIntervalAggImpl2(SOperatorInfo* pOperatorInfo, SSDataBlock*
         .ts = nextWin.skey,
         .groupId = tableGroupId,
     };
-    saveOutput(pTaskInfo, &key, pResult, pInfo->aggSup.resultRowSize);
+    saveOutputBuf(pTaskInfo, &key, pResult, pInfo->aggSup.resultRowSize);
     releaseOutputBuf(pTaskInfo, &key, pResult);
     int32_t prevEndPos = (forwardRows - 1) * step + startPos;
     ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0);
@@ -5645,7 +5729,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
       return pInfo->pDelRes;
     }
 
-    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
+    doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo);
+    // doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
     if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) {
       pOperator->status = OP_EXEC_DONE;
       qDebug("===stream===single interval is done");
@@ -5671,13 +5756,14 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
     printDataBlock(pBlock, "single interval recv");
 
     if (pBlock->info.type == STREAM_CLEAR) {
-      doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock,
-                     NULL);
+      doDeleteWindows(pOperator, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, NULL, NULL);
       qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo));
       continue;
     } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
-      doDeleteSpecifyIntervalWindow(&pInfo->aggSup, &pInfo->twAggSup, pBlock, pInfo->pDelWins, &pInfo->interval,
-                                    pUpdatedMap);
+      // doDeleteSpecifyIntervalWindow(&pInfo->aggSup, &pInfo->twAggSup, pBlock, pInfo->pDelWins, &pInfo->interval,
+      //                               pUpdatedMap);
+      doDeleteWindows(pOperator, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pInfo->pDelWins,
+                      pUpdatedMap);
       continue;
     } else if (pBlock->info.type == STREAM_GET_ALL) {
       getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap);
@@ -5704,9 +5790,9 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
 
     maxTs = TMAX(maxTs, pBlock->info.window.ekey);
     minTs = TMIN(minTs, pBlock->info.window.skey);
-    doStreamIntervalAggImpl(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdatedMap);
+    // doStreamIntervalAggImpl(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdatedMap);
     // new disc buf
-    /*doStreamIntervalAggImpl2(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap);*/
+    doStreamIntervalAggImpl2(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap);
   }
   pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
   pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, minTs);
@@ -5741,8 +5827,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
 #endif
 
   pOperator->status = OP_RES_TO_RETURN;
-  closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdatedMap,
-                      pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
+  closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdatedMap,
+                            pOperator);
 
   void* pIte = NULL;
   while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) {
@@ -5751,7 +5837,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
   taosArraySort(pUpdated, resultrowComparAsc);
 
   // new disc buf
-  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
+  // finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated,
+  // pSup->rowEntryInfoOffset);
   initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
   blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
   removeDeleteResults(pUpdatedMap, pInfo->pDelWins);
@@ -5762,9 +5849,9 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
     return pInfo->pDelRes;
   }
 
-  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
+  // doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
   // new disc buf
-  // doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo);
+  doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo);
   printDataBlock(pInfo->binfo.pRes, "single interval");
   return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
 }
@@ -5809,6 +5896,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
       .calTrigger = pIntervalPhyNode->window.triggerType,
       .maxTs = INT64_MIN,
       .minTs = INT64_MAX,
+      .deleteMark = INT64_MAX,
   };
   ASSERT(twAggSupp.calTrigger != STREAM_TRIGGER_MAX_DELAY);
   pOperator->pTaskInfo = pTaskInfo;
diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c
index 6cd5132bb9..5234f09175 100644
--- a/source/libs/stream/src/streamState.c
+++ b/source/libs/stream/src/streamState.c
@@ -135,15 +135,9 @@ int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void*
   if (streamStateGet(pState, key, pVal, pVLen) == 0) {
     return 0;
   }
-  void* tmp = taosMemoryCalloc(1, size);
-  if (streamStatePut(pState, key, &tmp, size) == 0) {
-    taosMemoryFree(tmp);
-    int32_t code = streamStateGet(pState, key, pVal, pVLen);
-    ASSERT(code == 0);
-    return code;
-  }
-  taosMemoryFree(tmp);
-  return -1;
+  *pVal = tdbRealloc(NULL, size);
+  memset(*pVal, 0, size);
+  return 0;
 }
 
 int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal) {
@@ -191,9 +185,14 @@ SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key
   if (pCur == NULL) {
     return NULL;
   }
+  if (tdbTbcOpen(pState->pStateDb, &pCur->pCur, NULL) < 0) {
+    taosMemoryFree(pCur);
+    return NULL;
+  }
 
   int32_t c;
   if (tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c) < 0) {
+    tdbTbcClose(pCur->pCur);
     taosMemoryFree(pCur);
     return NULL;
   }
@@ -212,9 +211,14 @@ SStreamStateCur* streamStateSeekKeyPrev(SStreamState* pState, const SWinKey* key
   if (pCur == NULL) {
     return NULL;
   }
+  if (tdbTbcOpen(pState->pStateDb, &pCur->pCur, NULL) < 0) {
+    taosMemoryFree(pCur);
+    return NULL;
+  }
 
   int32_t c;
   if (tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c) < 0) {
+    tdbTbcClose(pCur->pCur);
     taosMemoryFree(pCur);
     return NULL;
   }

From 86c0fb56e0b29be59a09eed2e435e3c5ae1cd422 Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Mon, 19 Sep 2022 17:27:25 +0800
Subject: [PATCH 38/57] enhance stream backend for sma

---
 include/libs/stream/streamState.h    | 2 +-
 source/dnode/vnode/src/tq/tq.c       | 4 ++--
 source/libs/stream/src/streamState.c | 9 +++++++--
 source/libs/wal/src/walMeta.c        | 2 +-
 4 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h
index 80fa7a7218..849d83a58b 100644
--- a/include/libs/stream/streamState.h
+++ b/include/libs/stream/streamState.h
@@ -34,7 +34,7 @@ typedef struct {
   TXN          txn;
 } SStreamState;
 
-SStreamState* streamStateOpen(char* path, SStreamTask* pTask);
+SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath);
 void          streamStateClose(SStreamState* pState);
 int32_t       streamStateBegin(SStreamState* pState);
 int32_t       streamStateCommit(SStreamState* pState);
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index c8841e5e16..db3076e1bd 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -760,7 +760,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask) {
 
   // expand executor
   if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
-    pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask);
+    pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false);
     if (pTask->pState == NULL) {
       return -1;
     }
@@ -774,7 +774,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask) {
     pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle);
     ASSERT(pTask->exec.executor);
   } else if (pTask->taskLevel == TASK_LEVEL__AGG) {
-    pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask);
+    pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false);
     if (pTask->pState == NULL) {
       return -1;
     }
diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c
index 6cd5132bb9..5c7c5e7a6d 100644
--- a/source/libs/stream/src/streamState.c
+++ b/source/libs/stream/src/streamState.c
@@ -18,14 +18,19 @@
 #include "tcommon.h"
 #include "ttimer.h"
 
-SStreamState* streamStateOpen(char* path, SStreamTask* pTask) {
+SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath) {
   SStreamState* pState = taosMemoryCalloc(1, sizeof(SStreamState));
   if (pState == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
+
   char statePath[300];
-  sprintf(statePath, "%s/%d", path, pTask->taskId);
+  if (!specPath) {
+    sprintf(statePath, "%s/%d", path, pTask->taskId);
+  } else {
+    memcpy(statePath, path, 300);
+  }
   if (tdbOpen(statePath, 4096, 256, &pState->db) < 0) {
     goto _err;
   }
diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c
index 93ced912f8..95a861177b 100644
--- a/source/libs/wal/src/walMeta.c
+++ b/source/libs/wal/src/walMeta.c
@@ -268,7 +268,7 @@ int walRollFileInfo(SWal* pWal) {
 char* walMetaSerialize(SWal* pWal) {
   char buf[30];
   ASSERT(pWal->fileInfoSet);
-  int    sz = pWal->fileInfoSet->size;
+  int    sz = taosArrayGetSize(pWal->fileInfoSet);
   cJSON* pRoot = cJSON_CreateObject();
   cJSON* pMeta = cJSON_CreateObject();
   cJSON* pFiles = cJSON_CreateArray();

From 8ed3bf3ef156de2f330bb7153344d3da098ea9b1 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Mon, 19 Sep 2022 17:28:13 +0800
Subject: [PATCH 39/57] feat: add resident funcs to udfd

---
 include/common/tglobal.h        |  1 +
 source/common/src/tglobal.c     |  3 +++
 source/libs/function/src/udfd.c | 16 ++++++++++++----
 3 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/include/common/tglobal.h b/include/common/tglobal.h
index 2de4ffdc17..66bae5ad3b 100644
--- a/include/common/tglobal.h
+++ b/include/common/tglobal.h
@@ -120,6 +120,7 @@ extern SDiskCfg tsDiskCfg[];
 
 // udf
 extern bool tsStartUdfd;
+extern char tsUdfdResFuncs[];
 
 // schemaless
 extern char tsSmlChildTableName[];
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index ddda8f8c9a..97e8a05024 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -163,6 +163,7 @@ int32_t tsTtlUnit = 86400;
 int32_t tsTtlPushInterval = 86400;
 int32_t tsGrantHBInterval = 60;
 int32_t tsUptimeInterval = 300;  // seconds
+char    tsUdfdResFuncs[1024] = ""; // udfd resident funcs that teardown when udfd exits
 
 #ifndef _STORAGE
 int32_t taosSetTfsCfg(SConfig *pCfg) {
@@ -421,6 +422,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, 1) != 0) return -1;
 
   if (cfgAddBool(pCfg, "udf", tsStartUdfd, 0) != 0) return -1;
+  if (cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, 0) != 0) return -1;
   GRANT_CFG_ADD;
   return 0;
 }
@@ -717,6 +719,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsUptimeInterval = cfgGetItem(pCfg, "uptimeInterval")->i32;
 
   tsStartUdfd = cfgGetItem(pCfg, "udf")->bval;
+  tstrncpy(tsUdfdResFuncs, cfgGetItem(pCfg, "udfdResFuncs")->str, sizeof(tsUdfdResFuncs));
 
   if (tsQueryBufferSize >= 0) {
     tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL;
diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c
index f8f44a9816..636f006d6e 100644
--- a/source/libs/function/src/udfd.c
+++ b/source/libs/function/src/udfd.c
@@ -951,11 +951,19 @@ void udfdConnectMnodeThreadFunc(void *args) {
 }
 
 int32_t udfdInitResidentFuncs() {
+  if (strlen(tsUdfdResFuncs) == 0) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   global.residentFuncs = taosArrayInit(2, TSDB_FUNC_NAME_LEN);
-  char gpd[TSDB_FUNC_NAME_LEN] = "gpd";
-  taosArrayPush(global.residentFuncs, gpd);
-  char gpdBatch[TSDB_FUNC_NAME_LEN] = "gpdbatch";
-  taosArrayPush(global.residentFuncs, gpdBatch);
+  char* pSave = tsUdfdResFuncs;
+  char* token;
+  while ((token = strtok_r(pSave, ",", &pSave)) != NULL) {
+    char func[TSDB_FUNC_NAME_LEN] = {0};
+    strncpy(func, token, strlen(token));
+    taosArrayPush(global.residentFuncs, func);
+  }
+
   return TSDB_CODE_SUCCESS;
 }
 

From 1604d729672c8c710f83c0c99ed61ea2786c9107 Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Mon, 19 Sep 2022 17:40:38 +0800
Subject: [PATCH 40/57] add log for wal

---
 source/libs/wal/src/walMeta.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c
index 95a861177b..b7783d0629 100644
--- a/source/libs/wal/src/walMeta.c
+++ b/source/libs/wal/src/walMeta.c
@@ -384,8 +384,10 @@ static int walFindCurMetaVer(SWal* pWal) {
     int   code = regexec(&walMetaRegexPattern, name, 0, NULL, 0);
     if (code == 0) {
       sscanf(name, "meta-ver%d", &metaVer);
+      wDebug("vgId:%d, wal find current meta: %s is the meta file, ver %d", pWal->cfg.vgId, name, metaVer);
       break;
     }
+    wDebug("vgId:%d, wal find current meta: %s is not meta file", pWal->cfg.vgId, name);
   }
   taosCloseDir(&pDir);
   regfree(&walMetaRegexPattern);

From 1aa9624ea33927f84021f6fb23641026fe3e8230 Mon Sep 17 00:00:00 2001
From: Xiaoyu Wang <xiaoyuwang@taosdata.com>
Date: Mon, 19 Sep 2022 17:47:07 +0800
Subject: [PATCH 41/57] enh: the expression in project is pushed down to scan

---
 source/libs/planner/src/planSpliter.c | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c
index beb938b161..cdfbc3fdf4 100644
--- a/source/libs/planner/src/planSpliter.c
+++ b/source/libs/planner/src/planSpliter.c
@@ -915,20 +915,28 @@ static int32_t stbSplSplitSortNode(SSplitContext* pCxt, SStableSplitInfo* pInfo)
 }
 
 static int32_t stbSplSplitScanNodeWithoutPartTags(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
-  int32_t code = splCreateExchangeNodeForSubplan(pCxt, pInfo->pSubplan, pInfo->pSplitNode, SUBPLAN_TYPE_MERGE);
+  SLogicNode* pSplitNode = pInfo->pSplitNode;
+  if (NULL != pInfo->pSplitNode->pParent && QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pInfo->pSplitNode->pParent)) {
+    pSplitNode = pInfo->pSplitNode->pParent;
+  }
+  int32_t code = splCreateExchangeNodeForSubplan(pCxt, pInfo->pSubplan, pSplitNode, SUBPLAN_TYPE_MERGE);
   if (TSDB_CODE_SUCCESS == code) {
     code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren,
-                                     (SNode*)splCreateScanSubplan(pCxt, pInfo->pSplitNode, SPLIT_FLAG_STABLE_SPLIT));
+                                     (SNode*)splCreateScanSubplan(pCxt, pSplitNode, SPLIT_FLAG_STABLE_SPLIT));
   }
   ++(pCxt->groupId);
   return code;
 }
 
 static int32_t stbSplSplitScanNodeWithPartTags(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
-  int32_t code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, NULL, pInfo->pSplitNode, true);
+  SLogicNode* pSplitNode = pInfo->pSplitNode;
+  if (NULL != pInfo->pSplitNode->pParent && QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pInfo->pSplitNode->pParent)) {
+    pSplitNode = pInfo->pSplitNode->pParent;
+  }
+  int32_t code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pSplitNode, NULL, pSplitNode, true);
   if (TSDB_CODE_SUCCESS == code) {
     code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren,
-                                     (SNode*)splCreateScanSubplan(pCxt, pInfo->pSplitNode, SPLIT_FLAG_STABLE_SPLIT));
+                                     (SNode*)splCreateScanSubplan(pCxt, pSplitNode, SPLIT_FLAG_STABLE_SPLIT));
   }
   pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE;
   ++(pCxt->groupId);

From 747ce081b3f2982c70686052d6945de900fd1f96 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 19 Sep 2022 17:59:05 +0800
Subject: [PATCH 42/57] more code

---
 source/util/src/tcompression.c | 94 ++++++++++++++++++----------------
 1 file changed, 51 insertions(+), 43 deletions(-)

diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c
index 748ab8c975..375f9db734 100644
--- a/source/util/src/tcompression.c
+++ b/source/util/src/tcompression.c
@@ -1046,9 +1046,12 @@ struct SCompressor {
     };
     // Integer ----
     struct {
-      int64_t i_prev;
-      int32_t i_selector;
-      int32_t i_nele;
+      int64_t  i_prev;
+      int32_t  i_selector;
+      int32_t  i_start;
+      int32_t  i_end;
+      uint64_t i_aZigzag[241];
+      int8_t   i_aBitN[241];
     };
     // Float ----
     struct {
@@ -1190,9 +1193,9 @@ static int32_t tCompTimestamp(SCompressor *pCmprsor, int64_t ts) {
 
 // Integer =====================================================
 #define SIMPLE8B_MAX ((uint64_t)1152921504606846974LL)
-static const char    bit_per_integer[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60};
-static const int32_t selector_to_elems[] = {240, 120, 60, 30, 20, 15, 12, 10, 8, 7, 6, 5, 4, 3, 2, 1};
-static const char    bit_to_selector[] = {0,  2,  3,  4,  5,  6,  7,  8,  9,  10, 10, 11, 11, 12, 12, 12,
+static const char    BIT_PER_INTEGER[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60};
+static const int32_t SELECTOR_TO_ELEMS[] = {240, 120, 60, 30, 20, 15, 12, 10, 8, 7, 6, 5, 4, 3, 2, 1};
+static const char    BIT_TO_SELECTOR[] = {0,  2,  3,  4,  5,  6,  7,  8,  9,  10, 10, 11, 11, 12, 12, 12,
                                           13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 15,
                                           15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
                                           15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15};
@@ -1227,60 +1230,64 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData)
       case TSDB_DATA_TYPE_UINT:
         val = *(uint32_t *)pData;
         break;
-      // case TSDB_DATA_TYPE_UBIGINT:
-      //   val = *(int64_t *)pData;
-      //   break;
+      case TSDB_DATA_TYPE_UBIGINT:
+        val = *(int64_t *)pData;
+        break;
       default:
         ASSERT(0);
         break;
     }
 
-    if (!I64_SAFE_ADD(val, pCmprsor->i_prev)) {
+    if (!I64_SAFE_ADD(val, -pCmprsor->i_prev)) {
       // TODO
       goto _copy_cmpr;
     }
 
     int64_t  diff = val - pCmprsor->i_prev;
     uint64_t vZigzag = ZIGZAG_ENCODE(int64_t, diff);
-
     if (vZigzag >= SIMPLE8B_MAX) {
       // TODO
       goto _copy_cmpr;
     }
 
-    int64_t nBit;
-    if (vZigzag) {
-      nBit = 64 - BUILDIN_CLZL(vZigzag);
-    } else {
-      nBit = 0;
-    }
+    int8_t nBit = (vZigzag) ? (64 - BUILDIN_CLZL(vZigzag)) : 0;
+    pCmprsor->i_prev = val;
 
-    if (pCmprsor->i_nele + 1 <= selector_to_elems[pCmprsor->i_selector] &&
-        pCmprsor->i_nele + 1 <= selector_to_elems[bit_to_selector[nBit]]) {
-      if (pCmprsor->i_selector < bit_to_selector[nBit]) {
-        pCmprsor->i_selector = bit_to_selector[nBit];
+    while (1) {
+      int32_t nEle = pCmprsor->i_end - pCmprsor->i_start;
+
+      if (nEle + 1 <= SELECTOR_TO_ELEMS[pCmprsor->i_selector] && nEle + 1 <= SELECTOR_TO_ELEMS[BIT_TO_SELECTOR[nBit]]) {
+        if (pCmprsor->i_selector < BIT_TO_SELECTOR[nBit]) {
+          pCmprsor->i_selector = BIT_TO_SELECTOR[nBit];
+        }
+        pCmprsor->i_end = (pCmprsor->i_end + 1) % 241;
+        pCmprsor->i_aZigzag[pCmprsor->i_end] = vZigzag;
+        pCmprsor->i_aBitN[pCmprsor->i_end] = nBit;
+        break;
+      } else {
+        while (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) {
+          pCmprsor->i_selector++;
+        }
+        nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector];
+
+        code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(uint64_t));
+        if (code) return code;
+
+        // uint64_t *bp = (uint64_t *)(pCmprsor->aBuf[0] + pCmprsor->nBuf[0]);
+        // pCmprsor->nBuf[0] += sizeof(uint64_t);
+        // bp[0] = pCmprsor->i_selector;
+        // for (int32_t iVal = 0; iVal < pCmprsor->i_nele; iVal++) {
+        //   /* code */
+        // }
+
+        // reset and continue
+        pCmprsor->i_selector = 0;
+        for (int32_t iVal = pCmprsor->i_start; iVal < pCmprsor->i_end; iVal++) {
+          if (pCmprsor->i_selector < BIT_TO_SELECTOR[pCmprsor->i_aBitN[iVal]]) {
+            pCmprsor->i_selector = BIT_TO_SELECTOR[pCmprsor->i_aBitN[iVal]];
+          }
+        }
       }
-      pCmprsor->i_nele++;
-      pCmprsor->i_prev = val;
-    } else {
-      while (pCmprsor->i_nele < selector_to_elems[pCmprsor->i_selector]) {
-        pCmprsor->i_selector++;
-      }
-      pCmprsor->i_nele = selector_to_elems[pCmprsor->i_selector];
-
-      code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(uint64_t));
-      if (code) return code;
-
-      uint64_t *bp = (uint64_t *)(pCmprsor->aBuf[0] + pCmprsor->nBuf[0]);
-      pCmprsor->nBuf[0] += sizeof(uint64_t);
-      bp[0] = pCmprsor->i_selector;
-      for (int32_t iVal = 0; iVal < pCmprsor->i_nele; iVal++) {
-        /* code */
-      }
-
-      // reset and continue
-      pCmprsor->i_nele = 0;
-      pCmprsor->i_selector = 0;
     }
   } else {
   _copy_cmpr:
@@ -1546,7 +1553,8 @@ int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg, int
     case TSDB_DATA_TYPE_UBIGINT:
       pCmprsor->i_prev = 0;
       pCmprsor->i_selector = 0;
-      pCmprsor->i_nele = 0;
+      pCmprsor->i_start = 0;
+      pCmprsor->i_end = 0;
       pCmprsor->aBuf[0][0] = 0;  // 0 means compressed, 1 otherwise (for backward compatibility)
       pCmprsor->nBuf[0] = 1;
       break;

From c90123c0d79171879d17f06c61d83c92d8a8e570 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 19 Sep 2022 18:19:41 +0800
Subject: [PATCH 43/57] more code

---
 source/util/src/tcompression.c | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c
index 375f9db734..febd54820b 100644
--- a/source/util/src/tcompression.c
+++ b/source/util/src/tcompression.c
@@ -1193,9 +1193,9 @@ static int32_t tCompTimestamp(SCompressor *pCmprsor, int64_t ts) {
 
 // Integer =====================================================
 #define SIMPLE8B_MAX ((uint64_t)1152921504606846974LL)
-static const char    BIT_PER_INTEGER[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60};
+static const uint8_t BIT_PER_INTEGER[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60};
 static const int32_t SELECTOR_TO_ELEMS[] = {240, 120, 60, 30, 20, 15, 12, 10, 8, 7, 6, 5, 4, 3, 2, 1};
-static const char    BIT_TO_SELECTOR[] = {0,  2,  3,  4,  5,  6,  7,  8,  9,  10, 10, 11, 11, 12, 12, 12,
+static const uint8_t BIT_TO_SELECTOR[] = {0,  2,  3,  4,  5,  6,  7,  8,  9,  10, 10, 11, 11, 12, 12, 12,
                                           13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 15,
                                           15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
                                           15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15};
@@ -1254,7 +1254,7 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData)
     pCmprsor->i_prev = val;
 
     while (1) {
-      int32_t nEle = pCmprsor->i_end - pCmprsor->i_start;
+      int32_t nEle = (pCmprsor->i_end + 241 - pCmprsor->i_start) % 241;
 
       if (nEle + 1 <= SELECTOR_TO_ELEMS[pCmprsor->i_selector] && nEle + 1 <= SELECTOR_TO_ELEMS[BIT_TO_SELECTOR[nBit]]) {
         if (pCmprsor->i_selector < BIT_TO_SELECTOR[nBit]) {
@@ -1270,19 +1270,23 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData)
         }
         nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector];
 
-        code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(uint64_t));
-        if (code) return code;
+        if (pCmprsor->autoAlloc) {
+          code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(uint64_t));
+          if (code) return code;
+        }
 
-        // uint64_t *bp = (uint64_t *)(pCmprsor->aBuf[0] + pCmprsor->nBuf[0]);
-        // pCmprsor->nBuf[0] += sizeof(uint64_t);
-        // bp[0] = pCmprsor->i_selector;
-        // for (int32_t iVal = 0; iVal < pCmprsor->i_nele; iVal++) {
-        //   /* code */
-        // }
+        uint64_t *bp = (uint64_t *)(pCmprsor->aBuf[0] + pCmprsor->nBuf[0]);
+        pCmprsor->nBuf[0] += sizeof(uint64_t);
+        bp[0] = pCmprsor->i_selector;
+        uint8_t bits = BIT_PER_INTEGER[pCmprsor->i_selector];
+        for (int32_t iVal = 0; iVal < nEle; iVal++) {
+          bp[0] |= ((pCmprsor->i_aZigzag[pCmprsor->i_start] & ((((uint64_t)1) << bits) - 1)) << (bits * iVal + 4));
+          pCmprsor->i_start = (pCmprsor->i_start + 1) % 241;
+        }
 
         // reset and continue
         pCmprsor->i_selector = 0;
-        for (int32_t iVal = pCmprsor->i_start; iVal < pCmprsor->i_end; iVal++) {
+        for (int32_t iVal = pCmprsor->i_start; iVal < pCmprsor->i_end; iVal = (iVal + 1) % 241) {
           if (pCmprsor->i_selector < BIT_TO_SELECTOR[pCmprsor->i_aBitN[iVal]]) {
             pCmprsor->i_selector = BIT_TO_SELECTOR[pCmprsor->i_aBitN[iVal]];
           }

From e43a4df8984d3329b90243e3693c9199d0e5e0dd Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 19 Sep 2022 18:47:39 +0800
Subject: [PATCH 44/57] more code

---
 source/util/src/tcompression.c | 65 ++++++++++++++++++++--------------
 1 file changed, 38 insertions(+), 27 deletions(-)

diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c
index febd54820b..62b9d87628 100644
--- a/source/util/src/tcompression.c
+++ b/source/util/src/tcompression.c
@@ -1002,32 +1002,38 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co
 #define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (b)) || ((a) < 0 && (b) >= INT64_MIN - (a)))
 typedef struct SCompressor SCompressor;
 
+static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData);
+static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData);
+static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nData);
+static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nData);
+static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t nData);
+static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData);
 static struct {
   int8_t  type;
   int32_t bytes;
   int8_t  isVarLen;
   int32_t (*cmprFn)(SCompressor *, const void *, int32_t nData);
 } DATA_TYPE_INFO[] = {
-    {TSDB_DATA_TYPE_NULL, 0, 0},        // TSDB_DATA_TYPE_NULL
-    {TSDB_DATA_TYPE_BOOL, 1, 0},        // TSDB_DATA_TYPE_BOOL
-    {TSDB_DATA_TYPE_TINYINT, 1, 0},     // TSDB_DATA_TYPE_TINYINT
-    {TSDB_DATA_TYPE_SMALLINT, 2, 0},    // TSDB_DATA_TYPE_SMALLINT
-    {TSDB_DATA_TYPE_INT, 4, 0},         // TSDB_DATA_TYPE_INT
-    {TSDB_DATA_TYPE_BIGINT, 8, 0},      // TSDB_DATA_TYPE_BIGINT
-    {TSDB_DATA_TYPE_FLOAT, 4, 0},       // TSDB_DATA_TYPE_FLOAT
-    {TSDB_DATA_TYPE_DOUBLE, 8, 0},      // TSDB_DATA_TYPE_DOUBLE
-    {TSDB_DATA_TYPE_VARCHAR, 1, 1},     // TSDB_DATA_TYPE_VARCHAR
-    {TSDB_DATA_TYPE_TIMESTAMP, 8, 0},   // pTSDB_DATA_TYPE_TIMESTAMP
-    {TSDB_DATA_TYPE_NCHAR, 1, 1},       // TSDB_DATA_TYPE_NCHAR
-    {TSDB_DATA_TYPE_UTINYINT, 1, 0},    // TSDB_DATA_TYPE_UTINYINT
-    {TSDB_DATA_TYPE_USMALLINT, 2, 0},   // TSDB_DATA_TYPE_USMALLINT
-    {TSDB_DATA_TYPE_UINT, 4, 0},        // TSDB_DATA_TYPE_UINT
-    {TSDB_DATA_TYPE_UBIGINT, 8, 0},     // TSDB_DATA_TYPE_UBIGINT
-    {TSDB_DATA_TYPE_JSON, 1, 1},        // TSDB_DATA_TYPE_JSON
-    {TSDB_DATA_TYPE_VARBINARY, 1, 1},   // TSDB_DATA_TYPE_VARBINARY
-    {TSDB_DATA_TYPE_DECIMAL, 1, 1},     // TSDB_DATA_TYPE_DECIMAL
-    {TSDB_DATA_TYPE_BLOB, 1, 1},        // TSDB_DATA_TYPE_BLOB
-    {TSDB_DATA_TYPE_MEDIUMBLOB, 1, 1},  // TSDB_DATA_TYPE_MEDIUMBLOB
+    {TSDB_DATA_TYPE_NULL, 0, 0, NULL},                 // TSDB_DATA_TYPE_NULL
+    {TSDB_DATA_TYPE_BOOL, 1, 0, tCompBool},            // TSDB_DATA_TYPE_BOOL
+    {TSDB_DATA_TYPE_TINYINT, 1, 0, tCompInt},          // TSDB_DATA_TYPE_TINYINT
+    {TSDB_DATA_TYPE_SMALLINT, 2, 0, tCompInt},         // TSDB_DATA_TYPE_SMALLINT
+    {TSDB_DATA_TYPE_INT, 4, 0, tCompInt},              // TSDB_DATA_TYPE_INT
+    {TSDB_DATA_TYPE_BIGINT, 8, 0, tCompInt},           // TSDB_DATA_TYPE_BIGINT
+    {TSDB_DATA_TYPE_FLOAT, 4, 0, tCompFloat},          // TSDB_DATA_TYPE_FLOAT
+    {TSDB_DATA_TYPE_DOUBLE, 8, 0, tCompDouble},        // TSDB_DATA_TYPE_DOUBLE
+    {TSDB_DATA_TYPE_VARCHAR, 1, 1, tCompBinary},       // TSDB_DATA_TYPE_VARCHAR
+    {TSDB_DATA_TYPE_TIMESTAMP, 8, 0, tCompTimestamp},  // pTSDB_DATA_TYPE_TIMESTAMP
+    {TSDB_DATA_TYPE_NCHAR, 1, 1, tCompBinary},         // TSDB_DATA_TYPE_NCHAR
+    {TSDB_DATA_TYPE_UTINYINT, 1, 0, tCompInt},         // TSDB_DATA_TYPE_UTINYINT
+    {TSDB_DATA_TYPE_USMALLINT, 2, 0, tCompInt},        // TSDB_DATA_TYPE_USMALLINT
+    {TSDB_DATA_TYPE_UINT, 4, 0, tCompInt},             // TSDB_DATA_TYPE_UINT
+    {TSDB_DATA_TYPE_UBIGINT, 8, 0, tCompInt},          // TSDB_DATA_TYPE_UBIGINT
+    {TSDB_DATA_TYPE_JSON, 1, 1, tCompBinary},          // TSDB_DATA_TYPE_JSON
+    {TSDB_DATA_TYPE_VARBINARY, 1, 1, tCompBinary},     // TSDB_DATA_TYPE_VARBINARY
+    {TSDB_DATA_TYPE_DECIMAL, 1, 1, tCompBinary},       // TSDB_DATA_TYPE_DECIMAL
+    {TSDB_DATA_TYPE_BLOB, 1, 1, tCompBinary},          // TSDB_DATA_TYPE_BLOB
+    {TSDB_DATA_TYPE_MEDIUMBLOB, 1, 1, tCompBinary},    // TSDB_DATA_TYPE_MEDIUMBLOB
 };
 
 struct SCompressor {
@@ -1124,10 +1130,12 @@ static int32_t tCompSetCopyMode(SCompressor *pCmprsor) {
 
   return code;
 }
-static int32_t tCompTimestamp(SCompressor *pCmprsor, int64_t ts) {
+static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t nData) {
   int32_t code = 0;
 
+  int64_t ts = *(int64_t *)pData;
   ASSERT(pCmprsor->type == TSDB_DATA_TYPE_TIMESTAMP);
+  ASSERT(nData == 8);
 
   if (pCmprsor->aBuf[0][0] == 1) {
     if (pCmprsor->nVal == 0) {
@@ -1572,9 +1580,15 @@ int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg, int
 int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int64_t *nData) {
   int32_t code = 0;
 
+  if (pCmprsor->nVal == 0) {
+    *ppData = NULL;
+    *nData = 0;
+    return code;
+  }
+
   if (pCmprsor->cmprAlg == TWO_STAGE_COMP /*|| IS_VAR_DATA_TYPE(pCmprsor->type)*/) {
     code = tRealloc(&pCmprsor->aBuf[1], pCmprsor->nBuf[0] + 1);
-    if (code) goto _exit;
+    if (code) return code;
 
     int64_t ret = LZ4_compress_default(pCmprsor->aBuf[0], pCmprsor->aBuf[1] + 1, pCmprsor->nBuf[0], pCmprsor->nBuf[0]);
     if (ret) {
@@ -1593,12 +1607,9 @@ int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int64_t *nData)
     *nData = pCmprsor->nBuf[0];
   }
 
-_exit:
   return code;
 }
 
-int32_t tCompress(SCompressor *pCmprsor, void *pData, int64_t nData) {
-  int32_t code = 0;
-  // TODO
-  return code;
+int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData) {
+  return DATA_TYPE_INFO[pCmprsor->type].cmprFn(pCmprsor, pData, nData);
 }
\ No newline at end of file

From cc8738d251f737368af64907801b98b9ad68ad3b Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Mon, 19 Sep 2022 19:00:21 +0800
Subject: [PATCH 45/57] fix:error in CI

---
 source/client/src/clientRawBlockWrite.c | 2 +-
 source/client/src/clientTmq.c           | 8 +++++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 514f99d463..6bd0d51999 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -372,7 +372,7 @@ _exit:
 }
 
 static char* processAutoCreateTable(STaosxRsp* rsp) {
-  if(rsp->createTableNum == 0) return strdup("");
+  ASSERT(rsp->createTableNum != 0);
 
   SDecoder*          decoder = taosMemoryCalloc(rsp->createTableNum, sizeof(SDecoder));
   SVCreateTbReq*     pCreateReq = taosMemoryCalloc(rsp->createTableNum, sizeof(SVCreateTbReq));
diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index e1c9257920..c9c02a77e1 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -1658,8 +1658,14 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
           rspWrapper = NULL;
           continue;
         }
+
         // build rsp
-        SMqTaosxRspObj* pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper);
+        void* pRsp = NULL;
+        if(pollRspWrapper->taosxRsp.createTableNum == 0){
+          pRsp = tmqBuildRspFromWrapper(pollRspWrapper);
+        }else{
+          pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper);
+        }
         taosFreeQitem(pollRspWrapper);
         return pRsp;
       } else {

From a5c92370d50e0e367bcf7c25c2721c1c79ceb3a8 Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Mon, 19 Sep 2022 19:01:15 +0800
Subject: [PATCH 46/57] fix: func declarition for arm32 (#16930)

---
 source/client/src/TMQConnector.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/client/src/TMQConnector.c b/source/client/src/TMQConnector.c
index fcf6957df9..26bf55055f 100644
--- a/source/client/src/TMQConnector.c
+++ b/source/client/src/TMQConnector.c
@@ -212,7 +212,7 @@ JNIEXPORT void JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqCommitAsync(JN
   tmq_commit_async(tmq, res, commit_cb, consumer);
 }
 
-JNIEXPORT int JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqUnsubscribeImp(JNIEnv *env, jobject jobj, jlong jtmq) {
+JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqUnsubscribeImp(JNIEnv *env, jobject jobj, jlong jtmq) {
   tmq_t *tmq = (tmq_t *)jtmq;
   if (tmq == NULL) {
     jniError("jobj:%p, tmq is closed", jobj);
@@ -222,7 +222,7 @@ JNIEXPORT int JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqUnsubscribeImp(
   return tmq_unsubscribe((tmq_t *)tmq);
 }
 
-JNIEXPORT int JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqConsumerCloseImp(JNIEnv *env, jobject jobj,
+JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqConsumerCloseImp(JNIEnv *env, jobject jobj,
                                                                                   jlong jtmq) {
   tmq_t *tmq = (tmq_t *)jtmq;
   if (tmq == NULL) {

From f2b02318f90732d80c1ff46f5d15f9132bbcf24f Mon Sep 17 00:00:00 2001
From: Xiaoyu Wang <xiaoyuwang@taosdata.com>
Date: Mon, 19 Sep 2022 19:03:35 +0800
Subject: [PATCH 47/57] enh: the expression in project is pushed down to scan

---
 source/libs/planner/src/planSpliter.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c
index cdfbc3fdf4..a3cb0c2654 100644
--- a/source/libs/planner/src/planSpliter.c
+++ b/source/libs/planner/src/planSpliter.c
@@ -916,7 +916,8 @@ static int32_t stbSplSplitSortNode(SSplitContext* pCxt, SStableSplitInfo* pInfo)
 
 static int32_t stbSplSplitScanNodeWithoutPartTags(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
   SLogicNode* pSplitNode = pInfo->pSplitNode;
-  if (NULL != pInfo->pSplitNode->pParent && QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pInfo->pSplitNode->pParent)) {
+  if (NULL != pInfo->pSplitNode->pParent && QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pInfo->pSplitNode->pParent) &&
+      NULL == pInfo->pSplitNode->pParent->pLimit && NULL == pInfo->pSplitNode->pParent->pSlimit) {
     pSplitNode = pInfo->pSplitNode->pParent;
   }
   int32_t code = splCreateExchangeNodeForSubplan(pCxt, pInfo->pSubplan, pSplitNode, SUBPLAN_TYPE_MERGE);
@@ -930,7 +931,8 @@ static int32_t stbSplSplitScanNodeWithoutPartTags(SSplitContext* pCxt, SStableSp
 
 static int32_t stbSplSplitScanNodeWithPartTags(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
   SLogicNode* pSplitNode = pInfo->pSplitNode;
-  if (NULL != pInfo->pSplitNode->pParent && QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pInfo->pSplitNode->pParent)) {
+  if (NULL != pInfo->pSplitNode->pParent && QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pInfo->pSplitNode->pParent) &&
+      NULL == pInfo->pSplitNode->pParent->pLimit && NULL == pInfo->pSplitNode->pParent->pSlimit) {
     pSplitNode = pInfo->pSplitNode->pParent;
   }
   int32_t code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pSplitNode, NULL, pSplitNode, true);

From 777fadee254c7c0efa58127105307426bd33941a Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Mon, 19 Sep 2022 19:27:23 +0800
Subject: [PATCH 48/57] feat: stream state optimization for rsma

---
 include/util/taoserror.h               |  2 +
 source/dnode/vnode/src/inc/sma.h       |  7 +-
 source/dnode/vnode/src/sma/smaRollup.c | 94 ++++++++++++++++++++++++--
 source/util/src/terror.c               |  2 +
 4 files changed, 98 insertions(+), 7 deletions(-)

diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index 01d42d6950..840e7309fe 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -619,6 +619,8 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_RSMA_EMPTY_INFO                TAOS_DEF_ERROR_CODE(0, 0x3156)
 #define TSDB_CODE_RSMA_INVALID_SCHEMA            TAOS_DEF_ERROR_CODE(0, 0x3157)
 #define TSDB_CODE_RSMA_REGEX_MATCH               TAOS_DEF_ERROR_CODE(0, 0x3158)
+#define TSDB_CODE_RSMA_STREAM_STATE_OPEN         TAOS_DEF_ERROR_CODE(0, 0x3159)
+#define TSDB_CODE_RSMA_STREAM_STATE_COMMIT       TAOS_DEF_ERROR_CODE(0, 0x3160)
 
 //index
 #define TSDB_CODE_INDEX_REBUILDING               TAOS_DEF_ERROR_CODE(0, 0x3200)
diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h
index 9931462e5f..dade85b12d 100644
--- a/source/dnode/vnode/src/inc/sma.h
+++ b/source/dnode/vnode/src/inc/sma.h
@@ -146,6 +146,7 @@ struct SRSmaInfoItem {
   uint16_t nScanned;
   int32_t  maxDelay;  // ms
   tmr_h    tmrId;
+  void    *pStreamState;
 };
 
 struct SRSmaInfo {
@@ -224,8 +225,10 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
 int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type);
 int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash);
 int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer);
-void    tdRSmaQTaskInfoGetFileName(int32_t vid, int64_t version, char *outputName);
-void    tdRSmaQTaskInfoGetFullName(int32_t vid, int64_t version, const char *path, char *outputName);
+void    tdRSmaQTaskInfoGetFileName(int32_t vgId, int64_t version, char *outputName);
+void    tdRSmaQTaskInfoGetFullName(int32_t vgId, int64_t version, const char *path, char *outputName);
+void    tdRSmaQTaskInfoGetFullPath(int32_t vgId, int8_t level, const char *path, char *outputName);
+void    tdRSmaQTaskInfoGetFullPathEx(int32_t vgId, tb_uid_t suid, int8_t level, const char *path, char *outputName);
 
 static FORCE_INLINE void tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) {
   int32_t ref = T_REF_INC(pRSmaInfo);
diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c
index ec8fcb2932..412def646f 100644
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@@ -92,6 +92,18 @@ void tdRSmaQTaskInfoGetFullName(int32_t vgId, int64_t version, const char *path,
   tdGetVndFileName(vgId, path, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, version, outputName);
 }
 
+void tdRSmaQTaskInfoGetFullPath(int32_t vgId, int8_t level, const char *path, char *outputName) {
+  tdGetVndDirName(vgId, path, VNODE_RSMA_DIR, true, outputName);
+  int32_t rsmaLen = strlen(outputName);
+  snprintf(outputName + rsmaLen, TSDB_FILENAME_LEN - rsmaLen, "%" PRIi8, level);
+}
+
+void tdRSmaQTaskInfoGetFullPathEx(int32_t vgId, tb_uid_t suid, int8_t level, const char *path, char *outputName) {
+  tdGetVndDirName(vgId, path, VNODE_RSMA_DIR, true, outputName);
+  int32_t rsmaLen = strlen(outputName);
+  snprintf(outputName + rsmaLen, TSDB_FILENAME_LEN - rsmaLen, "%" PRIi64 "%s%" PRIi8, suid, TD_DIRSEP, level);
+}
+
 static FORCE_INLINE int32_t tdRSmaQTaskInfoContLen(int32_t lenWithHead) {
   return lenWithHead - RSMA_QTASKINFO_HEAD_LEN;
 }
@@ -130,6 +142,10 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) {
         taosTmrStopA(&pItem->tmrId);
       }
 
+      if (isDeepFree && pItem->pStreamState) {
+        streamStateClose(pItem->pStreamState);
+      }
+
       if (isDeepFree && pInfo->taskInfo[i]) {
         tdRSmaQTaskInfoFree(&pInfo->taskInfo[i], SMA_VID(pSma), i + 1);
       } else {
@@ -290,12 +306,33 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat
     SRetention *pRetention = SMA_RETENTION(pSma);
     STsdbCfg   *pTsdbCfg = SMA_TSDB_CFG(pSma);
     SVnode     *pVnode = pSma->pVnode;
+    char        taskInfDir[TSDB_FILENAME_LEN] = {0};
+    void       *pStreamState = NULL;
+
+    // set the backend of stream state
+    tdRSmaQTaskInfoGetFullPathEx(TD_VID(pVnode), pRSmaInfo->suid, idx + 1, tfsGetPrimaryPath(pVnode->pTfs), taskInfDir);
+    if (!taosCheckExistFile(taskInfDir)) {
+      char *s = strdup(taskInfDir);
+      if (taosMulMkDir(taosDirName(s)) != 0) {
+        terrno = TAOS_SYSTEM_ERROR(errno);
+        taosMemoryFree(s);
+        return TSDB_CODE_FAILED;
+      }
+      taosMemoryFree(s);
+    }
+    pStreamState = streamStateOpen(taskInfDir, NULL, true);
+    if (!pStreamState) {
+      terrno = TSDB_CODE_RSMA_STREAM_STATE_OPEN;
+      return TSDB_CODE_FAILED;
+    }
+    
+
     SReadHandle handle = {
         .meta = pVnode->pMeta,
         .vnode = pVnode,
         .initTqReader = 1,
+        .pStateBackend = pStreamState,
     };
-
     pRSmaInfo->taskInfo[idx] = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle);
     if (!pRSmaInfo->taskInfo[idx]) {
       terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE;
@@ -303,6 +340,7 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat
     }
     SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]);
     pItem->triggerStat = TASK_TRIGGER_STAT_ACTIVE;  // fetch the data when reboot
+    pItem->pStreamState = pStreamState;
     if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) {
       int64_t msInterval =
           convertTimeFromPrecisionToUnit(pRetention[idx + 1].freq, pTsdbCfg->precision, TIME_UNIT_MILLISECOND);
@@ -322,7 +360,6 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat
 
     pItem->fetchLevel = pItem->level;
     taosTmrReset(tdRSmaFetchTrigger, RSMA_FETCH_INTERVAL, pItem, smaMgmt.tmrHandle, &pItem->tmrId);
-    
 
     smaInfo("vgId:%d, item:%p table:%" PRIi64 " level:%" PRIi8 " maxdelay:%" PRIi64 " watermark:%" PRIi64
             ", finally maxdelay:%" PRIi32,
@@ -1226,16 +1263,17 @@ int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer)
   if (tdRSmaRestoreQTaskInfoInit(pSma, &nTables) < 0) {
     goto _err;
   }
-
   if (nTables <= 0) {
     smaDebug("vgId:%d, no need to restore rsma task %" PRIi8 " since no tables", SMA_VID(pSma), type);
     return TSDB_CODE_SUCCESS;
   }
 
+#if 0
   // step 2: retrieve qtaskinfo items from the persistence file(rsma/qtaskinfo) and restore
   if (tdRSmaRestoreQTaskInfoReload(pSma, type, qtaskFileVer) < 0) {
     goto _err;
   }
+#endif
 
   // step 3: reload ts data from checkpoint
   if (tdRSmaRestoreTSDataReload(pSma) < 0) {
@@ -1440,6 +1478,50 @@ static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, int8_t type, SRSmaQTaskInfoIte
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) {
+  SSma   *pSma = pRSmaStat->pSma;
+  SVnode *pVnode = pSma->pVnode;
+  int32_t vid = SMA_VID(pSma);
+
+  if (taosHashGetSize(pInfoHash) <= 0) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  int64_t fsMaxVer = tdRSmaFSMaxVer(pSma, pRSmaStat);
+  if (pRSmaStat->commitAppliedVer <= fsMaxVer) {
+    smaDebug("vgId:%d, rsma persist, no need as applied %" PRIi64 " not larger than fsMaxVer %" PRIi64, vid,
+             pRSmaStat->commitAppliedVer, fsMaxVer);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  void *infoHash = NULL;
+  while ((infoHash = taosHashIterate(pInfoHash, infoHash))) {
+    SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash;
+
+    if (RSMA_INFO_IS_DEL(pRSmaInfo)) {
+      continue;
+    }
+
+    for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
+      SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pRSmaInfo, i);
+      if (pItem && pItem->pStreamState) {
+        if (streamStateCommit(pItem->pStreamState) < 0) {
+          terrno = TSDB_CODE_RSMA_STREAM_STATE_COMMIT;
+          goto _err;
+        }
+        smaDebug("vgId:%d, rsma persist, stream state commit success, table %" PRIi64 " level %d", vid, pRSmaInfo->suid,
+                 i + 1);
+      }
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+_err:
+  smaError("vgId:%d, rsma persist failed since %s", vid, terrstr());
+  return TSDB_CODE_FAILED;
+}
+
+#if 0
 int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) {
   SSma   *pSma = pRSmaStat->pSma;
   SVnode *pVnode = pSma->pVnode;
@@ -1459,7 +1541,7 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) {
   int64_t fsMaxVer = tdRSmaFSMaxVer(pSma, pRSmaStat);
   if (pRSmaStat->commitAppliedVer <= fsMaxVer) {
     smaDebug("vgId:%d, rsma persist, no need as applied %" PRIi64 " not larger than fsMaxVer %" PRIi64, vid,
-            pRSmaStat->commitAppliedVer, fsMaxVer);
+             pRSmaStat->commitAppliedVer, fsMaxVer);
     return TSDB_CODE_SUCCESS;
   }
 
@@ -1579,6 +1661,8 @@ _err:
   return TSDB_CODE_FAILED;
 }
 
+#endif
+
 /**
  * @brief trigger to get rsma result in async mode
  *
@@ -1926,7 +2010,7 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) {
 
       if ((pEnv->flag & SMA_ENV_FLG_CLOSE) && (atomic_load_64(&pRSmaStat->nBufItems) <= 0)) {
         smaDebug("vgId:%d, exec task end, flag:%" PRIi8 ", nBufItems:%" PRIi64, SMA_VID(pSma), pEnv->flag,
-                atomic_load_64(&pRSmaStat->nBufItems));
+                 atomic_load_64(&pRSmaStat->nBufItems));
         break;
       }
     }
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index 50c42ff170..1906a77127 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -621,6 +621,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FETCH_MSG_MSSED_UP,        "Rsma fetch msg is m
 TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_EMPTY_INFO,                "Rsma info is empty")
 TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_SCHEMA,            "Rsma invalid schema")
 TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_REGEX_MATCH,               "Rsma regex match")
+TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_STREAM_STATE_OPEN,         "Rsma stream state open")
+TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_STREAM_STATE_COMMIT,       "Rsma stream state commit")
 
 //index
 TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_REBUILDING,               "Index is rebuilding")

From 756a2d3d5d2189f2715b5b433c99cecd2d4eb132 Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Mon, 19 Sep 2022 19:29:20 +0800
Subject: [PATCH 49/57] merge 3.0

---
 source/common/src/tglobal.c    | 39 +++++++++++++++++-----------------
 source/dnode/vnode/src/tq/tq.c |  7 ++----
 source/libs/wal/src/walMeta.c  |  1 +
 3 files changed, 22 insertions(+), 25 deletions(-)

diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index ddda8f8c9a..64d376d265 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -58,7 +58,7 @@ int32_t tsNumOfMnodeFetchThreads = 1;
 int32_t tsNumOfMnodeReadThreads = 1;
 int32_t tsNumOfVnodeQueryThreads = 4;
 int32_t tsNumOfVnodeStreamThreads = 2;
-int32_t tsNumOfVnodeFetchThreads = 4;
+int32_t tsNumOfVnodeFetchThreads = 1;
 int32_t tsNumOfVnodeWriteThreads = 2;
 int32_t tsNumOfVnodeSyncThreads = 2;
 int32_t tsNumOfVnodeRsmaThreads = 2;
@@ -365,8 +365,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   tsNumOfVnodeStreamThreads = TMAX(tsNumOfVnodeStreamThreads, 4);
   if (cfgAddInt32(pCfg, "numOfVnodeStreamThreads", tsNumOfVnodeStreamThreads, 4, 1024, 0) != 0) return -1;
 
-  tsNumOfVnodeFetchThreads = tsNumOfCores / 4;
-  tsNumOfVnodeFetchThreads = TMAX(tsNumOfVnodeFetchThreads, 4);
+  tsNumOfVnodeFetchThreads = 1;
   if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 4, 1024, 0) != 0) return -1;
 
   tsNumOfVnodeWriteThreads = tsNumOfCores;
@@ -385,9 +384,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   tsNumOfQnodeQueryThreads = TMAX(tsNumOfQnodeQueryThreads, 4);
   if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 1, 1024, 0) != 0) return -1;
 
-//  tsNumOfQnodeFetchThreads = tsNumOfCores / 2;
-//  tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4);
-//  if (cfgAddInt32(pCfg, "numOfQnodeFetchThreads", tsNumOfQnodeFetchThreads, 1, 1024, 0) != 0) return -1;
+  //  tsNumOfQnodeFetchThreads = tsNumOfCores / 2;
+  //  tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4);
+  //  if (cfgAddInt32(pCfg, "numOfQnodeFetchThreads", tsNumOfQnodeFetchThreads, 1, 1024, 0) != 0) return -1;
 
   tsNumOfSnodeSharedThreads = tsNumOfCores / 4;
   tsNumOfSnodeSharedThreads = TRANGE(tsNumOfSnodeSharedThreads, 2, 4);
@@ -527,15 +526,15 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
     pItem->stype = stype;
   }
 
-/*
-  pItem = cfgGetItem(tsCfg, "numOfQnodeFetchThreads");
-  if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
-    tsNumOfQnodeFetchThreads = numOfCores / 2;
-    tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4);
-    pItem->i32 = tsNumOfQnodeFetchThreads;
-    pItem->stype = stype;
-  }
-*/
+  /*
+    pItem = cfgGetItem(tsCfg, "numOfQnodeFetchThreads");
+    if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
+      tsNumOfQnodeFetchThreads = numOfCores / 2;
+      tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4);
+      pItem->i32 = tsNumOfQnodeFetchThreads;
+      pItem->stype = stype;
+    }
+  */
 
   pItem = cfgGetItem(tsCfg, "numOfSnodeSharedThreads");
   if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
@@ -693,7 +692,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsNumOfVnodeSyncThreads = cfgGetItem(pCfg, "numOfVnodeSyncThreads")->i32;
   tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32;
   tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32;
-//  tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32;
+  //  tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32;
   tsNumOfSnodeSharedThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32;
   tsNumOfSnodeUniqueThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32;
   tsRpcQueueMemoryAllowed = cfgGetItem(pCfg, "rpcQueueMemoryAllowed")->i64;
@@ -941,10 +940,10 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) {
         tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32;
       } else if (strcasecmp("numOfQnodeQueryThreads", name) == 0) {
         tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32;
-/*        
-      } else if (strcasecmp("numOfQnodeFetchThreads", name) == 0) {
-        tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32;
-*/
+        /*
+              } else if (strcasecmp("numOfQnodeFetchThreads", name) == 0) {
+                tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32;
+        */
       } else if (strcasecmp("numOfSnodeSharedThreads", name) == 0) {
         tsNumOfSnodeSharedThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32;
       } else if (strcasecmp("numOfSnodeUniqueThreads", name) == 0) {
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index db3076e1bd..cbe916d0f7 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -141,11 +141,8 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con
   ASSERT(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum);
   ASSERT(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum);
 
-  if (pRsp->withSchema) {
-    ASSERT(taosArrayGetSize(pRsp->blockSchema) == pRsp->blockNum);
-  } else {
-    ASSERT(taosArrayGetSize(pRsp->blockSchema) == 0);
-  }
+  ASSERT(!pRsp->withSchema);
+  ASSERT(taosArrayGetSize(pRsp->blockSchema) == 0);
 
   if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) {
     if (pRsp->blockNum > 0) {
diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c
index b7783d0629..5284aeff77 100644
--- a/source/libs/wal/src/walMeta.c
+++ b/source/libs/wal/src/walMeta.c
@@ -424,6 +424,7 @@ int walLoadMeta(SWal* pWal) {
   // find existing meta file
   int metaVer = walFindCurMetaVer(pWal);
   if (metaVer == -1) {
+    wDebug("vgId:%d wal find meta ver %d", pWal->cfg.vgId, metaVer);
     return -1;
   }
   char fnameStr[WAL_FILE_LEN];

From 2e5e25549df3402109123033ca58a161881f5902 Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Mon, 19 Sep 2022 19:46:13 +0800
Subject: [PATCH 50/57] fix config

---
 source/common/src/tglobal.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index 64d376d265..f1c1e79a86 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -58,7 +58,7 @@ int32_t tsNumOfMnodeFetchThreads = 1;
 int32_t tsNumOfMnodeReadThreads = 1;
 int32_t tsNumOfVnodeQueryThreads = 4;
 int32_t tsNumOfVnodeStreamThreads = 2;
-int32_t tsNumOfVnodeFetchThreads = 1;
+int32_t tsNumOfVnodeFetchThreads = 4;
 int32_t tsNumOfVnodeWriteThreads = 2;
 int32_t tsNumOfVnodeSyncThreads = 2;
 int32_t tsNumOfVnodeRsmaThreads = 2;
@@ -365,7 +365,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   tsNumOfVnodeStreamThreads = TMAX(tsNumOfVnodeStreamThreads, 4);
   if (cfgAddInt32(pCfg, "numOfVnodeStreamThreads", tsNumOfVnodeStreamThreads, 4, 1024, 0) != 0) return -1;
 
-  tsNumOfVnodeFetchThreads = 1;
+  tsNumOfVnodeFetchThreads = tsNumOfCores / 4;
+  tsNumOfVnodeFetchThreads = TMAX(tsNumOfVnodeFetchThreads, 4);
   if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 4, 1024, 0) != 0) return -1;
 
   tsNumOfVnodeWriteThreads = tsNumOfCores;

From 5a5b9f4f707d8c7f36bb59e249c9c1f3aabb5ff2 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Mon, 19 Sep 2022 21:29:32 +0800
Subject: [PATCH 51/57] fix: update dbOrigSize before releasing dirty pages

---
 source/libs/tdb/src/db/tdbPager.c | 33 ++++++++++---------------------
 1 file changed, 10 insertions(+), 23 deletions(-)

diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c
index 543ffc55b6..04711eb6a0 100644
--- a/source/libs/tdb/src/db/tdbPager.c
+++ b/source/libs/tdb/src/db/tdbPager.c
@@ -248,6 +248,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
     return 0;
   }
 
+  // loop to write the dirty pages to file
   SRBTreeIter  iter = tRBTreeIterCreate(&pPager->rbt, 1);
   SRBTreeNode *pNode = NULL;
   while ((pNode = tRBTreeIterNext(&iter)) != NULL) {
@@ -257,6 +258,15 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
       ASSERT(0);
       return -1;
     }
+  }
+
+  tdbTrace("tdbttl commit:%p, %d/%d", pPager, pPager->dbOrigSize, pPager->dbFileSize);
+  pPager->dbOrigSize = pPager->dbFileSize;
+
+  // release the page
+  iter = tRBTreeIterCreate(&pPager->rbt, 1);
+  while ((pNode = tRBTreeIterNext(&iter)) != NULL) {
+    pPage = (SPage *)pNode;
 
     pPage->isDirty = 0;
 
@@ -265,29 +275,6 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
   }
 
   tRBTreeCreate(&pPager->rbt, pageCmpFn);
-  /*
-  // loop to write the dirty pages to file
-  for (pPage = pPager->pDirty; pPage; pPage = pPage->pDirtyNext) {
-    // TODO: update the page footer
-    ret = tdbPagerWritePageToDB(pPager, pPage);
-    if (ret < 0) {
-      ASSERT(0);
-      return -1;
-    }
-  }
-
-  // release the page
-  for (pPage = pPager->pDirty; pPage; pPage = pPager->pDirty) {
-    pPager->pDirty = pPage->pDirtyNext;
-    pPage->pDirtyNext = NULL;
-
-    pPage->isDirty = 0;
-
-    tdbPCacheRelease(pPager->pCache, pPage, pTxn);
-  }
-  */
-  tdbTrace("tdbttl commit:%p, %d", pPager, pPager->dbOrigSize);
-  pPager->dbOrigSize = pPager->dbFileSize;
 
   // sync the db file
   tdbOsFSync(pPager->fd);

From 82684bb4179ca3aa918ddec9c76f4810bc9135b2 Mon Sep 17 00:00:00 2001
From: gccgdb1234 <wxzhang@taosdata.com>
Date: Tue, 20 Sep 2022 09:39:55 +0800
Subject: [PATCH 52/57] doc: refine a few points in 06-select.md

---
 docs/en/12-taos-sql/06-select.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md
index 118a248373..e409973173 100644
--- a/docs/en/12-taos-sql/06-select.md
+++ b/docs/en/12-taos-sql/06-select.md
@@ -66,7 +66,7 @@ order_expr:
 
 A query can be performed on some or all columns. Data and tag columns can all be included in the SELECT list.
 
-## Wildcards
+### Wildcards
 
 You can use an asterisk (\*) as a wildcard character to indicate all columns. For standard tables, the asterisk indicates only data columns. For supertables and subtables, tag columns are also included.
 
@@ -136,6 +136,8 @@ taos> SELECT ts, ts AS primary_key_ts FROM d1001;
 
 ### Pseudocolumns
 
+**Pseudocolumn:** A pseudo-column behaves like a table column but is not actually stored in the table. You can select from pseudo-columns, but you cannot insert, update, or delete their values. A pseudo-column is also similar to a function without arguments. This section describes these pseudo-columns:
+
 **TBNAME**
 The TBNAME pseudocolumn in a supertable contains the names of subtables within the supertable.
 

From bc9c0dfd92f033edb69016684a115aeb1b70298f Mon Sep 17 00:00:00 2001
From: gccgdb1234 <wxzhang@taosdata.com>
Date: Tue, 20 Sep 2022 09:42:54 +0800
Subject: [PATCH 53/57] doc: refine pseudo columns

---
 docs/zh/12-taos-sql/06-select.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md
index c49351b0c9..5841904e61 100644
--- a/docs/zh/12-taos-sql/06-select.md
+++ b/docs/zh/12-taos-sql/06-select.md
@@ -137,6 +137,8 @@ taos> SELECT ts, ts AS primary_key_ts FROM d1001;
 
 ### 伪列
 
+**伪列**: 伪列的行为表现与普通数据列相似但其并不实际存储在表中。可以查询伪列，但不能对其做插入、更新和删除的操作。伪列有点像没有参数的函数。下面介绍是可用的伪列：
+
 **TBNAME**
 `TBNAME` 可以视为超级表中一个特殊的标签，代表子表的表名。
 

From ee2f5abf822a585a1ae9672a77c8615a9e7845b4 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Tue, 20 Sep 2022 09:52:10 +0800
Subject: [PATCH 54/57] fix:error in CI

---
 source/client/src/clientRawBlockWrite.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 6bd0d51999..eb7b45cc05 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -1598,6 +1598,8 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
     subReq->length += sizeof(SSubmitBlk) + schemaLen + totalLen;
     subReq->numOfBlocks++;
     taosMemoryFreeClear(pTableMeta);
+    rspObj.resInfo.pRspMsg = NULL;
+    doFreeReqResultInfo(&rspObj.resInfo);
   }
 
   pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY);

From 13114a6275eb10da324b521b9ca0d59b07361796 Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Tue, 20 Sep 2022 09:58:59 +0800
Subject: [PATCH 55/57] feat: update taos-tools 285b5e0 for 3.0

---
 cmake/taostools_CMakeLists.txt.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in
index 5cc580a9c6..b75d4607b8 100644
--- a/cmake/taostools_CMakeLists.txt.in
+++ b/cmake/taostools_CMakeLists.txt.in
@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG 509ec72
+        GIT_TAG 285b5e0
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

From 7eb75fea47ec28e0deea2b2bbfdf552e47921bb7 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao@163.com>
Date: Tue, 20 Sep 2022 09:59:01 +0800
Subject: [PATCH 56/57] feat(steam):optimize window close

---
 source/libs/executor/inc/executorimpl.h       |  1 +
 source/libs/executor/src/scanoperator.c       |  8 +----
 source/libs/executor/src/timewindowoperator.c | 29 ++++++++++++-------
 3 files changed, 21 insertions(+), 17 deletions(-)

diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index a516a9d11f..6484b6cb31 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -1064,6 +1064,7 @@ bool               functionNeedToExecute(SqlFunctionCtx* pCtx);
 bool               isOverdue(TSKEY ts, STimeWindowAggSupp* pSup);
 bool               isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup);
 bool               isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup);
+bool               isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SOperatorInfo* pOperator, STimeWindowAggSupp* pTwSup);
 void               appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp);
 void               printDataBlock(SSDataBlock* pBlock, const char* flag);
 uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId);
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index edc7ddb92e..0c51f95284 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -1331,7 +1331,7 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock
     // must check update info first.
     bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, tsCol[rowId]);
     bool closedWin = isClosed && isSignleIntervalWindow(pInfo) &&
-                     isDeletedWindow(&win, pBlock->info.groupId, pInfo->windowSup.pIntervalAggSup);
+                     isDeletedStreamWindow(&win, pBlock->info.groupId, pInfo->pTableScanOp, &pInfo->twAggSup);
     if ((update || closedWin) && out) {
       qDebug("stream update check not pass, update %d, closedWin %d", update, closedWin);
       uint64_t gpId = closedWin && pInfo->partitionSup.needCalc
@@ -1931,11 +1931,6 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
 
   pInfo->pTagCond = pTagCond;
   pInfo->pGroupTags = pTableScanNode->pGroupTags;
-  pInfo->twAggSup = (STimeWindowAggSupp){
-      .waterMark = pTableScanNode->watermark,
-      .calTrigger = pTableScanNode->triggerType,
-      .maxTs = INT64_MIN,
-  };
 
   int32_t numOfCols = 0;
   pInfo->pColMatchInfo = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID);
@@ -1985,7 +1980,6 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
 
     pInfo->pUpdateInfo = NULL;
     pInfo->pTableScanOp = pTableScanOp;
-    pInfo->interval = pTSInfo->pdInfo.interval;
 
     pInfo->readHandle = *pHandle;
     pInfo->tableUid = pScanPhyNode->uid;
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index 81c0f9cd0c..aed4ae9b26 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -1753,16 +1753,17 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt
 }
 
 void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSupporter* pSup, SInterval* pInterval,
-                            int64_t waterMark) {
+                            STimeWindowAggSupp* pTwSup) {
   if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
-    initIntervalDownStream(downstream->pDownstream[0], type, pSup, pInterval, waterMark);
+    initIntervalDownStream(downstream->pDownstream[0], type, pSup, pInterval, pTwSup);
     return;
   }
   SStreamScanInfo* pScanInfo = downstream->info;
   pScanInfo->windowSup.parentType = type;
   pScanInfo->windowSup.pIntervalAggSup = pSup;
-  pScanInfo->pUpdateInfo = updateInfoInitP(pInterval, waterMark);
+  pScanInfo->pUpdateInfo = updateInfoInitP(pInterval, pTwSup->waterMark);
   pScanInfo->interval = *pInterval;
+  pScanInfo->twAggSup = *pTwSup;
 }
 
 void initStreamFunciton(SqlFunctionCtx* pCtx, int32_t numOfExpr) {
@@ -1847,11 +1848,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo*
   pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, NULL,
                                          destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
 
-  if (nodeType(pPhyNode) == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL) {
-    initIntervalDownStream(downstream, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, &pInfo->aggSup, &pInfo->interval,
-                           pInfo->twAggSup.waterMark);
-  }
-
   code = appendDownstream(pOperator, &downstream, 1);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
@@ -2868,6 +2864,19 @@ bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) {
   return p1 == NULL;
 }
 
+bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SOperatorInfo* pOperator, STimeWindowAggSupp* pTwSup) {
+  if (pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark) {
+    SWinKey key = {.ts = pWin->skey, .groupId = groupId};
+    void*   pVal = NULL;
+    int32_t size = 0;
+    if (streamStateGet(pOperator->pTaskInfo->streamInfo.pState, &key, &pVal, &size) < 0) {
+      return false;
+    }
+    streamStateReleaseBuf(pOperator->pTaskInfo->streamInfo.pState, &key, pVal);
+  }
+  return false;
+}
+
 int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols, int32_t startPos, TSKEY eKey,
                         STimeWindow* pNextWin) {
   int32_t forwardRows =
@@ -3425,7 +3434,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
       createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo,
                           aggEncodeResultRow, aggDecodeResultRow, NULL);
   if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) {
-    initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, pInfo->twAggSup.waterMark);
+    initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup);
   }
   code = appendDownstream(pOperator, &downstream, 1);
   if (code != TSDB_CODE_SUCCESS) {
@@ -5944,7 +5953,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
       createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, NULL, destroyStreamIntervalOperatorInfo,
                           aggEncodeResultRow, aggDecodeResultRow, NULL);
 
-  initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, pInfo->twAggSup.waterMark);
+  initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup);
   code = appendDownstream(pOperator, &downstream, 1);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;

From 5f4d6e1c9954619e2139f90e3aaa3c9c0f719e7d Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Tue, 20 Sep 2022 10:08:18 +0800
Subject: [PATCH 57/57] fix: remove double free msg's pCont

---
 source/dnode/vnode/src/tq/tqSink.c | 1 -
 source/libs/function/src/tudf.c    | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c
index 522bf46aa1..875a5e9346 100644
--- a/source/dnode/vnode/src/tq/tqSink.c
+++ b/source/dnode/vnode/src/tq/tqSink.c
@@ -274,7 +274,6 @@ void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) {
   };
 
   if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) {
-    rpcFreeCont(submitReq);
     tqDebug("failed to put into write-queue since %s", terrstr());
   }
 }
diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c
index 3f472b53a0..62427e0488 100644
--- a/source/libs/function/src/tudf.c
+++ b/source/libs/function/src/tudf.c
@@ -961,6 +961,7 @@ void releaseUdfFuncHandle(char* udfName) {
   strcpy(key.udfName, udfName);
   SUdfcFuncStub *foundStub = taosArraySearch(gUdfdProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ);
   if (!foundStub) {
+    uv_mutex_unlock(&gUdfdProxy.udfStubsMutex);
     return;
   }
   if (foundStub->refCount > 0) {