feat:merge from 3.0

examples/c/tmq.c

@@ -26,6 +26,14 @@ static void msg_process(TAOS_RES* msg) {
   printf("topic: %s\n", tmq_get_topic_name(msg));
   printf("db: %s\n", tmq_get_db_name(msg));
   printf("vg: %d\n", tmq_get_vgroup_id(msg));
+  if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) {
+    void*   meta;
+    int32_t metaLen;
+    tmq_get_raw_meta(msg, &meta, &metaLen);
+
+    printf("meta, len is %d\n", metaLen);
+    return;
+  }
   while (1) {
     TAOS_ROW row = taos_fetch_row(msg);
     if (row == NULL) break;
@@ -76,19 +84,41 @@ int32_t init_env() {
   }
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "create table if not exists ct1 using st1 tags(2000)");
+  pRes = taos_query(pConn, "insert into ct0 values(now, 1, 2, 'a')");
   if (taos_errno(pRes) != 0) {
-    printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes));
+    printf("failed to insert into ct0, reason:%s\n", taos_errstr(pRes));
     return -1;
   }
+  taos_free_result(pRes);
+
+  pRes = taos_query(pConn, "create table if not exists ct1 using st1 tags(2000)");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create child table ct1, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
+  pRes = taos_query(pConn, "insert into ct1 values(now, 3, 4, 'b')");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
 
   pRes = taos_query(pConn, "create table if not exists ct3 using st1 tags(3000)");
   if (taos_errno(pRes) != 0) {
-    printf("failed to create child table tu3, reason:%s\n", taos_errstr(pRes));
+    printf("failed to create child table ct3, reason:%s\n", taos_errstr(pRes));
     return -1;
   }
-
   taos_free_result(pRes);
+
+  pRes = taos_query(pConn, "insert into ct3 values(now, 5, 6, 'c')");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
   return 0;
 }
 
@@ -107,8 +137,8 @@ int32_t create_topic() {
   }
   taos_free_result(pRes);
 
-  /*pRes = taos_query(pConn, "create topic topic_ctb_column as database abc1");*/
+  pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1");
-  pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");
+  /*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");*/
   if (taos_errno(pRes) != 0) {
     printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes));
     return -1;
@@ -168,6 +198,9 @@ tmq_t* build_consumer() {
   tmq_conf_set(conf, "td.connect.pass", "taosdata");
   tmq_conf_set(conf, "msg.with.table.name", "true");
   tmq_conf_set(conf, "enable.auto.commit", "true");
+
+  tmq_conf_set(conf, "experiment.use.snapshot", "false");
+
   tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
   tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
   assert(tmq);

include/client/taos.h

@@ -261,7 +261,7 @@ enum tmq_res_t {
 typedef enum tmq_res_t tmq_res_t;
 
 DLL_EXPORT tmq_res_t   tmq_get_res_type(TAOS_RES *res);
-DLL_EXPORT int32_t     tmq_get_raw_meta(TAOS_RES *res, const void **raw_meta, int32_t *raw_meta_len);
+DLL_EXPORT int32_t     tmq_get_raw_meta(TAOS_RES *res, void **raw_meta, int32_t *raw_meta_len);
 DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res);
 DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res);
 DLL_EXPORT int32_t     tmq_get_vgroup_id(TAOS_RES *res);

include/common/tcommon.h

@@ -56,6 +56,7 @@ typedef struct {
   bool      needSortTableByGroupId;
   void*     pTagCond;
   void*     pTagIndexCond;
+  uint64_t  suid;
 } STableListInfo;
 
 typedef struct SColumnDataAgg {
@@ -70,14 +71,13 @@ typedef struct SColumnDataAgg {
 
 typedef struct SDataBlockInfo {
   STimeWindow window;
-  int32_t     rows;
+  int32_t     rows;     // todo hide this attribute
   int32_t     rowSize;
-  int64_t     uid;      // the uid of table, from which current data block comes
+  uint64_t    uid;      // the uid of table, from which current data block comes
-  int64_t     blockId;  // block id, generated by physical planner
+  uint16_t    blockId;  // block id, generated by physical planner
   uint64_t    groupId;  // no need to serialize
-  int16_t     numOfCols;
   int16_t     hasVarCol;
-  int32_t     capacity;
+  uint32_t    capacity;
   // TODO: optimize and remove following
   int32_t     childId;  // used for stream, do not serialize
   EStreamType type;     // used for stream, do not serialize
@@ -98,7 +98,7 @@ typedef struct SVarColAttr {
 // pBlockAgg->numOfNull == info.rows, all data are null
 // pBlockAgg->numOfNull == 0, no data are null.
 typedef struct SColumnInfoData {
-  SColumnInfo info;     // TODO filter info needs to be removed
+  SColumnInfo info;     // column info
   bool        hasNull;  // if current column data has null value.
   char*       pData;    // the corresponding block data in memory
   union {
@@ -117,8 +117,8 @@ typedef struct SQueryTableDataCond {
   int32_t      type;              // data block load type:
   int32_t      numOfTWindows;
   STimeWindow* twindows;
-  int32_t      startVersion;
+  int64_t      startVersion;
-  int32_t      endVersion;
+  int64_t      endVersion;
 } SQueryTableDataCond;
 
 void*   blockDataDestroy(SSDataBlock* pBlock);

include/common/tdatablock.h

@@ -184,9 +184,9 @@ static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, u
 int32_t getJsonValueLen(const char* data);
 
 int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull);
-int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity,
+int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, uint32_t* capacity,
                         const SColumnInfoData* pSource, uint32_t numOfRow2);
-int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows);
+int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, const SDataBlockInfo* pBlockInfo);
 int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex);
 
 int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows);
@@ -212,7 +212,7 @@ size_t blockDataGetSerialMetaSize(uint32_t numOfCols);
 int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo);
 int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst);
 
-int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, uint32_t numOfRows);
+int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows);
 int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows);
 
 void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows);
@@ -223,6 +223,11 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize);
 int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n);
 
 SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData);
+SSDataBlock* createDataBlock();
+int32_t      blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData);
+
+SColumnInfoData  createColumnInfoData(int16_t type, int32_t bytes, int16_t colId);
+SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index);
 
 void        blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols,
                                 int8_t needCompress);
@@ -236,7 +241,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
 char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId);
 
 static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) {
-  return blockDataGetSerialMetaSize(pBlock->info.numOfCols) + blockDataGetSize(pBlock);
+  return blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)) + blockDataGetSize(pBlock);
 }
 
 static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data,

include/common/tmsg.h

@@ -629,11 +629,6 @@ typedef struct {
   uint8_t scale;
 } SColumnInfo;
 
-typedef struct {
-  int64_t uid;
-  TSKEY   key;  // last accessed ts, for subscription
-} STableIdInfo;
-
 typedef struct STimeWindow {
   TSKEY skey;
   TSKEY ekey;
@@ -696,12 +691,12 @@ typedef struct {
 
 typedef STableCfg STableCfgRsp;
 
-int32_t tSerializeSTableCfgReq(void *buf, int32_t bufLen, STableCfgReq *pReq);
+int32_t tSerializeSTableCfgReq(void* buf, int32_t bufLen, STableCfgReq* pReq);
-int32_t tDeserializeSTableCfgReq(void *buf, int32_t bufLen, STableCfgReq *pReq);
+int32_t tDeserializeSTableCfgReq(void* buf, int32_t bufLen, STableCfgReq* pReq);
 
-int32_t tSerializeSTableCfgRsp(void *buf, int32_t bufLen, STableCfgRsp *pRsp);
+int32_t tSerializeSTableCfgRsp(void* buf, int32_t bufLen, STableCfgRsp* pRsp);
-int32_t tDeserializeSTableCfgRsp(void *buf, int32_t bufLen, STableCfgRsp *pRsp);
+int32_t tDeserializeSTableCfgRsp(void* buf, int32_t bufLen, STableCfgRsp* pRsp);
-void tFreeSTableCfgRsp(STableCfgRsp *pRsp);
+void    tFreeSTableCfgRsp(STableCfgRsp* pRsp);
 
 typedef struct {
   char    db[TSDB_DB_FNAME_LEN];
@@ -1346,11 +1341,19 @@ typedef struct {
   int32_t dnodeId;
   char    config[TSDB_DNODE_CONFIG_LEN];
   char    value[TSDB_DNODE_VALUE_LEN];
-} SMCfgDnodeReq, SDCfgDnodeReq;
+} SMCfgDnodeReq;
 
 int32_t tSerializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq);
 int32_t tDeserializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq);
 
+typedef struct {
+  char config[TSDB_DNODE_CONFIG_LEN];
+  char value[TSDB_DNODE_VALUE_LEN];
+} SDCfgDnodeReq;
+
+int32_t tSerializeSDCfgDnodeReq(void* buf, int32_t bufLen, SDCfgDnodeReq* pReq);
+int32_t tDeserializeSDCfgDnodeReq(void* buf, int32_t bufLen, SDCfgDnodeReq* pReq);
+
 typedef struct {
   int32_t dnodeId;
 } SMCreateMnodeReq, SMDropMnodeReq, SDDropMnodeReq, SMCreateQnodeReq, SMDropQnodeReq, SDCreateQnodeReq, SDDropQnodeReq,
@@ -2669,6 +2672,7 @@ typedef struct {
   SMsgHead head;
   char     subKey[TSDB_SUBSCRIBE_KEY_LEN];
   int8_t   withTbName;
+  int8_t   useSnapshot;
   int32_t  epoch;
   uint64_t reqId;
   int64_t  consumerId;

include/libs/executor/dataSinkMgt.h

@@ -33,7 +33,7 @@ struct SDataSink;
 struct SSDataBlock;
 
 typedef struct SDeleterRes {
-  uint64_t uid;
+  uint64_t suid;
   SArray*  uidList;
   int64_t  skey;
   int64_t  ekey;
@@ -41,7 +41,8 @@ typedef struct SDeleterRes {
 } SDeleterRes;
 
 typedef struct SDeleterParam {
-  SArray* pUidList;
+  uint64_t suid;
+  SArray*  pUidList;
 } SDeleterParam;
 
 typedef struct SDataSinkStat {

include/libs/executor/executor.h

@@ -36,11 +36,13 @@ typedef struct SReadHandle {
   void*   vnode;
   void*   mnd;
   SMsgCb* pMsgCb;
+  int8_t  initTsdbReader;
 } SReadHandle;
 
 enum {
   STREAM_DATA_TYPE_SUBMIT_BLOCK = 1,
   STREAM_DATA_TYPE_SSDATA_BLOCK = 2,
+  STREAM_DATA_TYPE_FROM_SNAPSHOT = 3,
 };
 
 typedef enum {
@@ -56,6 +58,13 @@ typedef enum {
  */
 qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle);
 
+/**
+ * Switch the stream scan to snapshot mode
+ * @param tinfo
+ * @return
+ */
+int32_t qStreamScanSnapshot(qTaskInfo_t tinfo);
+
 /**
  * Set the input data block for the stream scan.
  * @param tinfo

include/libs/function/function.h

@@ -172,23 +172,6 @@ typedef struct tExprNode {
 
 void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *));
 
-typedef struct SAggFunctionInfo {
-  char      name[FUNCTIONS_NAME_MAX_LENGTH];
-  int8_t    type;         // Scalar function or aggregation function
-  uint32_t  functionId;   // Function Id
-  int8_t    sFunctionId;  // Transfer function for super table query
-  uint16_t  status;
-
-  bool (*init)(SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);  // setup the execute environment
-  void (*addInput)(SqlFunctionCtx *pCtx);
-
-  // finalizer must be called after all exec has been executed to generated final result.
-  void (*finalize)(SqlFunctionCtx *pCtx);
-  void (*combine)(SqlFunctionCtx *pCtx);
-
-  int32_t (*dataReqFunc)(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId);
-} SAggFunctionInfo;
-
 struct SScalarParam {
   SColumnInfoData *columnData;
   SHashObj        *pHashFilter;

include/libs/monitor/monitor.h

@@ -109,6 +109,9 @@ typedef struct {
   char    version[MON_VER_LEN];
   float   master_uptime;     // day
   int32_t monitor_interval;  // sec
+  int32_t dbs_total;
+  int32_t stbs_total;
+  int64_t tbs_total;
   int32_t vgroups_total;
   int32_t vgroups_alive;
   int32_t vnodes_total;

include/libs/qworker/qworker.h

@@ -32,7 +32,7 @@ enum {
 };
 
 typedef struct SDeleteRes {
-  uint64_t uid;
+  uint64_t suid;
   SArray*  uidList;
   int64_t  skey;
   int64_t  ekey;

include/util/taoserror.h

@@ -71,6 +71,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE       TAOS_DEF_ERROR_CODE(0, 0x0029)
 #define TSDB_CODE_INVALID_TIMESTAMP             TAOS_DEF_ERROR_CODE(0, 0x0030)
 #define TSDB_CODE_MSG_DECODE_ERROR              TAOS_DEF_ERROR_CODE(0, 0x0031)
+#define TSDB_CODE_NO_AVAIL_DISK                 TAOS_DEF_ERROR_CODE(0, 0x0032)
 
 #define TSDB_CODE_REF_NO_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0040)
 #define TSDB_CODE_REF_FULL                      TAOS_DEF_ERROR_CODE(0, 0x0041)

source/client/src/clientMain.c

@@ -290,6 +290,8 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
       pResultInfo->current += 1;
       return pResultInfo->row;
     }
+  } else if (TD_RES_TMQ_META(res)) {
+    return NULL;
   } else {
     // assert to avoid un-initialization error
     ASSERT(0);

source/client/src/tmq.c

@@ -54,6 +54,7 @@ struct tmq_conf_t {
   int8_t   autoCommit;
   int8_t   resetOffset;
   int8_t   withTbName;
+  int8_t   useSnapshot;
   uint16_t port;
   int32_t  autoCommitInterval;
   char*    ip;
@@ -69,6 +70,7 @@ struct tmq_t {
   char           groupId[TSDB_CGROUP_LEN];
   char           clientId[256];
   int8_t         withTbName;
+  int8_t         useSnapshot;
   int8_t         autoCommit;
   int32_t        autoCommitInterval;
   int32_t        resetOffsetCfg;
@@ -282,6 +284,18 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
     }
   }
 
+  if (strcmp(key, "experiment.use.snapshot") == 0) {
+    if (strcmp(value, "true") == 0) {
+      conf->useSnapshot = true;
+      return TMQ_CONF_OK;
+    } else if (strcmp(value, "false") == 0) {
+      conf->useSnapshot = false;
+      return TMQ_CONF_OK;
+    } else {
+      return TMQ_CONF_INVALID;
+    }
+  }
+
   if (strcmp(key, "td.connect.ip") == 0) {
     conf->ip = strdup(value);
     return TMQ_CONF_OK;
@@ -953,6 +967,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
   strcpy(pTmq->clientId, conf->clientId);
   strcpy(pTmq->groupId, conf->groupId);
   pTmq->withTbName = conf->withTbName;
+  pTmq->useSnapshot = conf->useSnapshot;
   pTmq->autoCommit = conf->autoCommit;
   pTmq->autoCommitInterval = conf->autoCommitInterval;
   pTmq->commitCb = conf->commitCb;
@@ -1145,8 +1160,6 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
 
   // handle meta rsp
   int8_t rspType = ((SMqRspHead*)pMsg->pData)->mqMsgType;
-  if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) {
-  }
 
   SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM);
   if (pRspWrapper == NULL) {
@@ -1159,19 +1172,19 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   pRspWrapper->vgHandle = pVg;
   pRspWrapper->topicHandle = pTopic;
 
-  memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead));
-
   if (rspType == TMQ_MSG_TYPE__POLL_RSP) {
+    memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead));
     tDecodeSMqDataBlkRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->dataRsp);
   } else {
     ASSERT(rspType == TMQ_MSG_TYPE__POLL_META_RSP);
+    memcpy(&pRspWrapper->metaRsp, pMsg->pData, sizeof(SMqRspHead));
     tDecodeSMqMetaRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->metaRsp);
   }
 
   taosMemoryFree(pMsg->pData);
 
-  tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pVg->vgId,
+  tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld, type %d", tmq->consumerId, pVg->vgId,
-           pRspWrapper->dataRsp.reqOffset, pRspWrapper->dataRsp.rspOffset);
+           pRspWrapper->dataRsp.reqOffset, pRspWrapper->dataRsp.rspOffset, rspType);
 
   taosWriteQitem(tmq->mqueue, pRspWrapper);
   tsem_post(&tmq->rspSem);
@@ -1534,6 +1547,8 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic*
   pReq->currentOffset = reqOffset;
   pReq->reqId = generateRequestId();
 
+  pReq->useSnapshot = tmq->useSnapshot;
+
   pReq->head.vgId = htonl(pVg->vgId);
   pReq->head.contLen = htonl(sizeof(SMqPollReq));
   return pReq;
@@ -1541,7 +1556,7 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic*
 
 SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
   SMqMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqMetaRspObj));
-  pRspObj->resType = RES_TYPE__TMQ;
+  pRspObj->resType = RES_TYPE__TMQ_META;
   tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN);
   tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN);
   pRspObj->vgId = pWrapper->vgHandle->vgId;
@@ -1659,7 +1674,7 @@ int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset)
   return 0;
 }
 
-SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
+void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
   while (1) {
     SMqRspWrapper* rspWrapper = NULL;
     taosGetQitem(tmq->qall, (void**)&rspWrapper);
@@ -1699,18 +1714,18 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
     } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) {
       SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper;
       int32_t            consumerEpoch = atomic_load_32(&tmq->epoch);
-      if (pollRspWrapper->dataRsp.head.epoch == consumerEpoch) {
+      if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) {
         SMqClientVg* pVg = pollRspWrapper->vgHandle;
         /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/
-        pVg->currentOffset = pollRspWrapper->dataRsp.rspOffset;
+        pVg->currentOffset = pollRspWrapper->metaRsp.rspOffset;
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
         // build rsp
-        SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper);
+        SMqMetaRspObj* pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper);
         taosFreeQitem(pollRspWrapper);
         return pRsp;
       } else {
         tscDebug("msg discard since epoch mismatch: msg epoch %d, consumer epoch %d\n",
-                 pollRspWrapper->dataRsp.head.epoch, consumerEpoch);
+                 pollRspWrapper->metaRsp.head.epoch, consumerEpoch);
         taosFreeQitem(pollRspWrapper);
       }
     } else {
@@ -1727,8 +1742,8 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
 }
 
 TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
-  SMqRspObj* rspObj;
+  void*   rspObj;
-  int64_t    startTime = taosGetTimestampMs();
+  int64_t startTime = taosGetTimestampMs();
 
 #if 0
   tmqHandleAllDelayedTask(tmq);
@@ -1856,7 +1871,7 @@ const char* tmq_get_table_name(TAOS_RES* res) {
   return NULL;
 }
 
-int32_t tmq_get_raw_meta(TAOS_RES* res, const void** raw_meta, int32_t* raw_meta_len) {
+int32_t tmq_get_raw_meta(TAOS_RES* res, void** raw_meta, int32_t* raw_meta_len) {
   if (TD_RES_TMQ_META(res)) {
     SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
     *raw_meta = pMetaRspObj->metaRsp.metaRsp;

source/common/src/tdatablock.c

@@ -214,7 +214,7 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c
   }
 }
 
-int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity,
+int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, uint32_t* capacity,
                         const SColumnInfoData* pSource, uint32_t numOfRow2) {
   ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type);
   if (numOfRow2 == 0) {
@@ -293,22 +293,18 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, in
   return numOfRow1 + numOfRow2;
 }
 
-int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows) {
+int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, const SDataBlockInfo* pBlockInfo) {
   ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type);
   if (numOfRows <= 0) {
     return numOfRows;
   }
 
+  if (pBlockInfo != NULL) {
+    ASSERT(pBlockInfo->capacity >= numOfRows);
+  }
+
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
-    // Handle the bitmap
-    char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * numOfRows);
-    if (p == NULL) {
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-
-    pColumnInfoData->varmeta.offset = (int32_t*)p;
     memcpy(pColumnInfoData->varmeta.offset, pSource->varmeta.offset, sizeof(int32_t) * numOfRows);
-
     if (pColumnInfoData->varmeta.allocLen < pSource->varmeta.length) {
       char* tmp = taosMemoryRealloc(pColumnInfoData->pData, pSource->varmeta.length);
       if (tmp == NULL) {
@@ -319,27 +315,11 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p
       pColumnInfoData->varmeta.allocLen = pSource->varmeta.length;
     }
 
-    memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length);
     pColumnInfoData->varmeta.length = pSource->varmeta.length;
+    memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length);
   } else {
-    char* tmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(numOfRows));
-    if (tmp == NULL) {
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-
-    pColumnInfoData->nullbitmap = tmp;
     memcpy(pColumnInfoData->nullbitmap, pSource->nullbitmap, BitmapLen(numOfRows));
-
+    memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows);
-    if (pSource->pData) {
-      int32_t newSize = numOfRows * pColumnInfoData->info.bytes;
-      tmp = taosMemoryRealloc(pColumnInfoData->pData, newSize);
-      if (tmp == NULL) {
-        return TSDB_CODE_OUT_OF_MEMORY;
-      }
-
-      pColumnInfoData->pData = tmp;
-      memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows);
-    }
   }
 
   pColumnInfoData->hasNull = pSource->hasNull;
@@ -348,8 +328,7 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p
 }
 
 size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) {
-  ASSERT(pBlock && pBlock->info.numOfCols == taosArrayGetSize(pBlock->pDataBlock));
+  return taosArrayGetSize(pBlock->pDataBlock);
-  return pBlock->info.numOfCols;
 }
 
 size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { return pBlock->info.rows; }
@@ -359,7 +338,8 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex)
     return 0;
   }
 
-  if (pDataBlock->info.numOfCols <= 0) {
+  size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
+  if (numOfCols <= 0) {
     return -1;
   }
 
@@ -383,7 +363,8 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) {
   assert(pSrc != NULL && pDest != NULL);
   int32_t capacity = pDest->info.capacity;
 
-  for (int32_t i = 0; i < pDest->info.numOfCols; ++i) {
+  size_t numOfCols = taosArrayGetSize(pDest->pDataBlock);
+  for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i);
     SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i);
 
@@ -399,9 +380,8 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) {
 size_t blockDataGetSize(const SSDataBlock* pBlock) {
   assert(pBlock != NULL);
 
-  size_t  total = 0;
+  size_t total = 0;
-  int32_t numOfCols = pBlock->info.numOfCols;
+  size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
-
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
     total += colDataGetFullLength(pColInfoData, pBlock->info.rows);
@@ -416,7 +396,7 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd
                            int32_t pageSize) {
   ASSERT(pBlock != NULL && stopIndex != NULL);
 
-  int32_t numOfCols = pBlock->info.numOfCols;
+  size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
   int32_t numOfRows = pBlock->info.rows;
 
   int32_t bitmapChar = 1;
@@ -477,33 +457,24 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3
     return NULL;
   }
 
-  SSDataBlock* pDst = taosMemoryCalloc(1, sizeof(SSDataBlock));
+  SSDataBlock* pDst = createDataBlock();
   if (pDst == NULL) {
     return NULL;
   }
 
   pDst->info = pBlock->info;
-
   pDst->info.rows = 0;
-  pDst->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData));
+  size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
-
+  for (int32_t i = 0; i < numOfCols; ++i) {
-  for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
     SColumnInfoData  colInfo = {0};
     SColumnInfoData* pSrcCol = taosArrayGet(pBlock->pDataBlock, i);
     colInfo.info = pSrcCol->info;
-
+    blockDataAppendColInfo(pDst, &colInfo);
-    if (IS_VAR_DATA_TYPE(pSrcCol->info.type)) {
-      SVarColAttr* pAttr = &colInfo.varmeta;
-      pAttr->offset = taosMemoryCalloc(rowCount, sizeof(int32_t));
-    } else {
-      colInfo.nullbitmap = taosMemoryCalloc(1, BitmapLen(rowCount));
-      colInfo.pData = taosMemoryCalloc(rowCount, colInfo.info.bytes);
-    }
-
-    taosArrayPush(pDst->pDataBlock, &colInfo);
   }
 
-  for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
+  blockDataEnsureCapacity(pDst, rowCount);
+
+  for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i);
     SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i);
 
@@ -541,7 +512,7 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) {
   // write the number of rows
   *(uint32_t*)buf = pBlock->info.rows;
 
-  int32_t numOfCols = pBlock->info.numOfCols;
+  size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
   int32_t numOfRows = pBlock->info.rows;
 
   char* pStart = buf + sizeof(uint32_t);
@@ -571,7 +542,7 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) {
 int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) {
   pBlock->info.rows = *(int32_t*)buf;
 
-  int32_t     numOfCols = pBlock->info.numOfCols;
+  size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
   const char* pStart = buf + sizeof(uint32_t);
 
   for (int32_t i = 0; i < numOfCols; ++i) {
@@ -620,7 +591,8 @@ int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity)
   pBlock->info.rows = *(int32_t*)buf;
   pBlock->info.groupId = *(uint64_t*)(buf + sizeof(int32_t));
 
-  int32_t     numOfCols = pBlock->info.numOfCols;
+  size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
+
   const char* pStart = buf + sizeof(uint32_t) + sizeof(uint64_t);
 
   for (int32_t i = 0; i < numOfCols; ++i) {
@@ -666,7 +638,7 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) {
   if (pBlock->info.rowSize == 0) {
     size_t rowSize = 0;
 
-    size_t numOfCols = pBlock->info.numOfCols;
+    size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
     for (int32_t i = 0; i < numOfCols; ++i) {
       SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
       rowSize += pColInfo->info.bytes;
@@ -693,7 +665,7 @@ double blockDataGetSerialRowSize(const SSDataBlock* pBlock) {
   ASSERT(pBlock != NULL);
   double rowSize = 0;
 
-  size_t numOfCols = pBlock->info.numOfCols;
+  size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
     rowSize += pColInfo->info.bytes;
@@ -762,7 +734,7 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) {
 static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, const SSDataBlock* pSrcBlock,
                                 int32_t tupleIndex) {
   int32_t code = 0;
-  int32_t numOfCols = pSrcBlock->info.numOfCols;
+  size_t numOfCols = taosArrayGetSize(pSrcBlock->pDataBlock);
 
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* pDst = &pDstCols[i];
@@ -785,7 +757,7 @@ static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, co
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataBlock, int32_t* index) {
+static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataBlock, const int32_t* index) {
 #if 0
   for (int32_t i = 0; i < pDataBlock->info.rows; ++i) {
     int32_t code = doAssignOneTuple(pCols, i, pDataBlock, index[i]);
@@ -794,7 +766,8 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB
     }
   }
 #else
-  for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) {
+  size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
+  for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* pDst = &pCols[i];
     SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i);
 
@@ -821,7 +794,7 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB
 
 static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) {
   int32_t rows = pDataBlock->info.rows;
-  int32_t numOfCols = pDataBlock->info.numOfCols;
+  size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
 
   SColumnInfoData* pCols = taosMemoryCalloc(numOfCols, sizeof(SColumnInfoData));
   if (pCols == NULL) {
@@ -848,7 +821,7 @@ static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) {
 }
 
 static void copyBackToBlock(SSDataBlock* pDataBlock, SColumnInfoData* pCols) {
-  int32_t numOfCols = pDataBlock->info.numOfCols;
+  size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
 
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, i);
@@ -909,8 +882,10 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
     }
   }
 
+  size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
+
   if (taosArrayGetSize(pOrderInfo) == 1 && (!sortColumnHasNull)) {
-    if (pDataBlock->info.numOfCols == 1) {
+    if (numOfCols == 1) {
       if (!varTypeSort) {
         SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, 0);
         SBlockOrderInfo* pOrder = taosArrayGet(pOrderInfo, 0);
@@ -926,7 +901,8 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
         return TSDB_CODE_SUCCESS;
       } else {  // var data type
       }
-    } else if (pDataBlock->info.numOfCols == 2) {
+    } else if (numOfCols == 2) {
+
     }
   }
 
@@ -1120,34 +1096,39 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF
 
 void blockDataCleanup(SSDataBlock* pDataBlock) {
   pDataBlock->info.rows = 0;
-  for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) {
+  size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
+  for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
     colInfoDataCleanup(p, pDataBlock->info.capacity);
   }
 }
 
-int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, uint32_t numOfRows) {
+static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo *pBlockInfo, uint32_t numOfRows) {
-  ASSERT(numOfRows);
+  ASSERT(numOfRows > 0 && pBlockInfo->capacity >= pBlockInfo->rows);
-  if (0 == numOfRows || numOfRows <= existRows) {
+  if (numOfRows < pBlockInfo->capacity) {
     return TSDB_CODE_SUCCESS;
   }
 
+  // todo temp disable it
+//  ASSERT(pColumn->info.bytes != 0);
+
+  int32_t existedRows = pBlockInfo->rows;
+
   if (IS_VAR_DATA_TYPE(pColumn->info.type)) {
     char* tmp = taosMemoryRealloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows);
-
     if (tmp == NULL) {
       return TSDB_CODE_OUT_OF_MEMORY;
     }
 
     pColumn->varmeta.offset = (int32_t*)tmp;
-    memset(&pColumn->varmeta.offset[existRows], 0, sizeof(int32_t) * (numOfRows - existRows));
+    memset(&pColumn->varmeta.offset[existedRows], 0, sizeof(int32_t) * (numOfRows - existedRows));
   } else {
     char* tmp = taosMemoryRealloc(pColumn->nullbitmap, BitmapLen(numOfRows));
     if (tmp == NULL) {
       return TSDB_CODE_OUT_OF_MEMORY;
     }
 
-    int32_t oldLen = BitmapLen(existRows);
+    int32_t oldLen = BitmapLen(existedRows);
     pColumn->nullbitmap = tmp;
     memset(&pColumn->nullbitmap[oldLen], 0, BitmapLen(numOfRows) - oldLen);
 
@@ -1160,8 +1141,8 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, ui
     if (tmp == NULL) {
       return TSDB_CODE_OUT_OF_MEMORY;
     }
-    memset(tmp + pColumn->info.bytes * existRows, 0, pColumn->info.bytes * (numOfRows - existRows));
     
+    memset(tmp + pColumn->info.bytes * existedRows, 0, pColumn->info.bytes * (numOfRows - existedRows));
     pColumn->pData = tmp;
   }
 
@@ -1178,6 +1159,11 @@ void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows) {
   }
 }
 
+int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) {
+  SDataBlockInfo info = {0};
+  return doEnsureCapacity(pColumn, &info, numOfRows);
+}
+
 int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) {
   int32_t code = 0;
   ASSERT(numOfRows > 0);
@@ -1186,11 +1172,14 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) {
     return TSDB_CODE_SUCCESS;
   }
 
-  pDataBlock->info.capacity = numOfRows;
+  if (pDataBlock->info.capacity < numOfRows) {
+    pDataBlock->info.capacity = numOfRows;
+  }
 
-  for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) {
+  size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
+  for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
-    code = colInfoDataEnsureCapacity(p, pDataBlock->info.rows, numOfRows);
+    code = doEnsureCapacity(p, &pDataBlock->info, numOfRows);
     if (code) {
       return code;
     }
@@ -1214,40 +1203,34 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
     return NULL;
   }
 
-  int32_t numOfCols = pDataBlock->info.numOfCols;
+  SSDataBlock* pBlock = createDataBlock();
-
+  pBlock->info = pDataBlock->info;
-  SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
+  pBlock->info.rows = 0;
-  pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
+  pBlock->info.capacity = 0;
-
-  pBlock->info.numOfCols = numOfCols;
-  pBlock->info.hasVarCol = pDataBlock->info.hasVarCol;
-  pBlock->info.rowSize = pDataBlock->info.rowSize;
-  pBlock->info.groupId = pDataBlock->info.groupId;
-  pBlock->info.childId = pDataBlock->info.childId;
-  pBlock->info.type = pDataBlock->info.type;
 
+  size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
   for (int32_t i = 0; i < numOfCols; ++i) {
-    SColumnInfoData  colInfo = {0};
     SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
-    colInfo.info = p->info;
+    SColumnInfoData  colInfo = {.hasNull = true, .info = p->info};
-    colInfo.hasNull = true;
+    blockDataAppendColInfo(pBlock, &colInfo);
-    taosArrayPush(pBlock->pDataBlock, &colInfo);
   }
 
   if (copyData) {
+    int32_t code = blockDataEnsureCapacity(pBlock, pDataBlock->info.rows);
+    if (code != TSDB_CODE_SUCCESS) {
+      terrno = code;
+      blockDataDestroy(pBlock);
+      return NULL;
+    }
+
     for (int32_t i = 0; i < numOfCols; ++i) {
       SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i);
       SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i);
-
-      int32_t code = colInfoDataEnsureCapacity(pDst, 0, pDataBlock->info.rows);
-      if (code != TSDB_CODE_SUCCESS) {
-        return NULL;
-      }
-
       if (pSrc->pData == NULL) {
         continue;
       }
-      colDataAssign(pDst, pSrc, pDataBlock->info.rows);
+
+      colDataAssign(pDst, pSrc, pDataBlock->info.rows, &pDataBlock->info);
     }
 
     pBlock->info.rows = pDataBlock->info.rows;
@@ -1257,16 +1240,75 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
   return pBlock;
 }
 
+SSDataBlock* createDataBlock() {
+  SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
+  if (pBlock == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
+  if (pBlock->pDataBlock == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    taosMemoryFree(pBlock);
+  }
+
+  return pBlock;
+}
+
+int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData) {
+  ASSERT(pBlock != NULL && pColInfoData != NULL);
+  if (pBlock->pDataBlock == NULL) {
+    pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
+    if (pBlock->pDataBlock == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return terrno;
+    }
+  }
+
+  void* p = taosArrayPush(pBlock->pDataBlock, pColInfoData);
+  if (p == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return terrno;
+  }
+
+  // todo disable it temporarily
+//  ASSERT(pColInfoData->info.type != 0);
+  if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
+    pBlock->info.hasVarCol = true;
+  }
+
+  pBlock->info.rowSize += pColInfoData->info.bytes;
+  return TSDB_CODE_SUCCESS;
+}
+
+SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId) {
+  SColumnInfoData col = {.hasNull = true};
+  col.info.colId = colId;
+  col.info.type  = type;
+  col.info.bytes = bytes;
+
+  return col;
+}
+
+SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index) {
+  ASSERT(pBlock != NULL);
+  if (index >= taosArrayGetSize(pBlock->pDataBlock)) {
+    return NULL;
+  }
+
+  return taosArrayGet(pBlock->pDataBlock, index);
+}
+
 size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) {
-  int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(pBlock->info.numOfCols);
+  size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
 
+  int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(numOfCols);
   int32_t rowSize = pBlock->info.rowSize;
-
   int32_t nRows = payloadSize / rowSize;
 
   // the true value must be less than the value of nRows
   int32_t additional = 0;
-  for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
+  for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i);
     if (IS_VAR_DATA_TYPE(pCol->info.type)) {
       additional += nRows * sizeof(int32_t);
@@ -1350,7 +1392,8 @@ int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) {
   if (pBlock->info.rows <= n) {
     blockDataCleanup(pBlock);
   } else {
-    for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
+    size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
+    for (int32_t i = 0; i < numOfCols; ++i) {
       SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
       colDataTrimFirstNRows(pColInfoData, n, pBlock->info.rows);
     }
@@ -1362,7 +1405,7 @@ int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) {
 
 int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) {
   int64_t tbUid = pBlock->info.uid;
-  int16_t numOfCols = pBlock->info.numOfCols;
+  int16_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
   int16_t hasVarCol = pBlock->info.hasVarCol;
   int32_t rows = pBlock->info.rows;
   int32_t sz = taosArrayGetSize(pBlock->pDataBlock);
@@ -1397,8 +1440,10 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) {
 void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) {
   int32_t sz;
 
-  buf = taosDecodeFixedI64(buf, &pBlock->info.uid);
+  int16_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
-  buf = taosDecodeFixedI16(buf, &pBlock->info.numOfCols);
+
+  buf = taosDecodeFixedU64(buf, &pBlock->info.uid);
+  buf = taosDecodeFixedI16(buf, &numOfCols);
   buf = taosDecodeFixedI16(buf, &pBlock->info.hasVarCol);
   buf = taosDecodeFixedI32(buf, &pBlock->info.rows);
   buf = taosDecodeFixedI32(buf, &sz);
@@ -1507,12 +1552,13 @@ void blockDebugShowData(const SArray* dataBlocks, const char* flag) {
   int32_t sz = taosArrayGetSize(dataBlocks);
   for (int32_t i = 0; i < sz; i++) {
     SSDataBlock* pDataBlock = taosArrayGet(dataBlocks, i);
-    int32_t      colNum = pDataBlock->info.numOfCols;
+    size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
+
     int32_t      rows = pDataBlock->info.rows;
     printf("%s |block type %d |child id %d|\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId);
     for (int32_t j = 0; j < rows; j++) {
       printf("%s |", flag);
-      for (int32_t k = 0; k < colNum; k++) {
+      for (int32_t k = 0; k < numOfCols; k++) {
         SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
         void*            var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
         if (colDataIsNull(pColInfoData, rows, j, NULL)) {
@@ -1565,7 +1611,9 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
   int32_t bufSize = sizeof(SSubmitReq);
   for (int32_t i = 0; i < sz; ++i) {
     SDataBlockInfo* pBlkInfo = &((SSDataBlock*)taosArrayGet(pDataBlocks, i))->info;
-    bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(pBlkInfo->numOfCols));
+
+    int32_t numOfCols = taosArrayGetSize(pDataBlocks);
+    bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(numOfCols));
     bufSize += sizeof(SSubmitBlk);
   }
 
@@ -1583,10 +1631,10 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
 
   for (int32_t i = 0; i < sz; ++i) {
     SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i);
-    int32_t      colNum = pDataBlock->info.numOfCols;
+    int32_t      colNum = taosArrayGetSize(pDataBlock->pDataBlock);
     int32_t      rows = pDataBlock->info.rows;
-    int32_t      rowSize = pDataBlock->info.rowSize;
+//    int32_t      rowSize = pDataBlock->info.rowSize;
-    int64_t      groupId = pDataBlock->info.groupId;
+//    int64_t      groupId = pDataBlock->info.groupId;
 
     if (colNum <= 1) {
       // invalid if only with TS col
@@ -1804,8 +1852,6 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen
 }
 
 const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) {
-  blockDataEnsureCapacity(pBlock, numOfRows);
-
   const char* pStart = pData;
 
   int32_t dataLen = *(int32_t*)pStart;
@@ -1819,9 +1865,6 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t
     taosArraySetSize(pBlock->pDataBlock, numOfCols);
   }
 
-  pBlock->info.numOfCols = taosArrayGetSize(pBlock->pDataBlock);
-  ASSERT(pBlock->pDataBlock->size >= numOfCols);
-
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
     pColInfoData->info.type = *(int16_t*)pStart;
@@ -1846,28 +1889,26 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t
 
     SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
     if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
-      pColInfoData->varmeta.length = colLen[i];
-      pColInfoData->varmeta.allocLen = colLen[i];
-
       memcpy(pColInfoData->varmeta.offset, pStart, sizeof(int32_t) * numOfRows);
       pStart += sizeof(int32_t) * numOfRows;
 
-      if (colLen[i] > 0) {
+      if (colLen[i] > 0 && pColInfoData->varmeta.allocLen < colLen[i]) {
-        taosMemoryFreeClear(pColInfoData->pData);
+        char* tmp = taosMemoryRealloc(pColInfoData->pData, colLen[i]);
-        pColInfoData->pData = taosMemoryMalloc(colLen[i]);
+        if (tmp == NULL) {
+          return NULL;
+        }
+
+        pColInfoData->pData = tmp;
+        pColInfoData->varmeta.allocLen = colLen[i];
       }
+
+      pColInfoData->varmeta.length = colLen[i];
     } else {
-      if (pColInfoData->nullbitmap == NULL) {
-        pColInfoData->nullbitmap = taosMemoryCalloc(1, BitmapLen(numOfRows));
-      }
       memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows));
       pStart += BitmapLen(numOfRows);
     }
 
     if (colLen[i] > 0) {
-      if (pColInfoData->pData == NULL) {
-        pColInfoData->pData = taosMemoryCalloc(1, colLen[i]);
-      }
       memcpy(pColInfoData->pData, pStart, colLen[i]);
     }
 

source/common/src/tglobal.c

@@ -760,9 +760,9 @@ void taosCleanupCfg() {
 }
 
 void taosCfgDynamicOptions(const char *option, const char *value) {
-  if (strcasecmp(option, "debugFlag") == 0) {
+  if (strncasecmp(option, "debugFlag", 9) == 0) {
-    int32_t debugFlag = atoi(value);
+    int32_t flag = atoi(value);
-    taosSetAllDebugFlag(debugFlag);
+    taosSetAllDebugFlag(flag);
   }
 
   if (strcasecmp(option, "resetlog") == 0) {

source/common/src/tmsg.c

@@ -1500,6 +1500,33 @@ int32_t tDeserializeSMCfgDnodeReq(void *buf, int32_t bufLen, SMCfgDnodeReq *pReq
   return 0;
 }
 
+int32_t tSerializeSDCfgDnodeReq(void *buf, int32_t bufLen, SDCfgDnodeReq *pReq) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->config) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->value) < 0) return -1;
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSDCfgDnodeReq(void *buf, int32_t bufLen, SDCfgDnodeReq *pReq) {
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, buf, bufLen);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->config) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->value) < 0) return -1;
+  tEndDecode(&decoder);
+
+  tDecoderClear(&decoder);
+  return 0;
+}
+
 int32_t tSerializeSCreateDnodeReq(void *buf, int32_t bufLen, SCreateDnodeReq *pReq) {
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, bufLen);

source/common/src/ttime.c

@@ -76,22 +76,28 @@ void           deltaToUtcInitOnce() {
 
 static int64_t parseFraction(char* str, char** end, int32_t timePrec);
 static int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim);
-static int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec);
+static int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim);
-static int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec);
+static int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim);
 static char*   forwardToTimeStringEnd(char* str);
 static bool    checkTzPresent(const char* str, int32_t len);
 
-static int32_t (*parseLocaltimeFp[])(char* timestr, int32_t len, int64_t* utime, int32_t timePrec) = {parseLocaltime,
+static int32_t (*parseLocaltimeFp[])(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) = {parseLocaltime,
-                                                                                        parseLocaltimeDst};
+                                                                                                                  parseLocaltimeDst};
 
 int32_t taosParseTime(const char* timestr, int64_t* utime, int32_t len, int32_t timePrec, int8_t day_light) {
   /* parse datatime string in with tz */
   if (strnchr(timestr, 'T', len, false) != NULL) {
-    return parseTimeWithTz(timestr, utime, timePrec, 'T');
+    if (checkTzPresent(timestr, len)) {
-  } else if (checkTzPresent(timestr, len)) {
+      return parseTimeWithTz(timestr, utime, timePrec, 'T');
-    return parseTimeWithTz(timestr, utime, timePrec, 0);
+    } else {
+      return (*parseLocaltimeFp[day_light])((char*)timestr, len, utime, timePrec, 'T');
+    }
   } else {
-    return (*parseLocaltimeFp[day_light])((char*)timestr, len, utime, timePrec);
+    if (checkTzPresent(timestr, len)) {
+      return parseTimeWithTz(timestr, utime, timePrec, 0);
+    } else {
+      return (*parseLocaltimeFp[day_light])((char*)timestr, len, utime, timePrec, 0);
+    }
   }
 }
 
@@ -333,13 +339,25 @@ static FORCE_INLINE bool validateTm(struct tm* pTm) {
   return  true;
 }
 
-int32_t parseLocaltime(char* timestr, int32_t len, int64_t* time, int32_t timePrec) {
+int32_t parseLocaltime(char* timestr, int32_t len, int64_t* time, int32_t timePrec, char delim) {
   *time = 0;
   struct tm tm = {0};
 
-  char* str = taosStrpTime(timestr, "%Y-%m-%d %H:%M:%S", &tm);
+  char *str;
+  if (delim == 'T') {
+    str = taosStrpTime(timestr, "%Y-%m-%dT%H:%M:%S", &tm);
+  } else if (delim == 0) {
+    str = taosStrpTime(timestr, "%Y-%m-%d %H:%M:%S", &tm);
+  } else {
+    str = NULL;
+  }
+
   if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) {
-    return -1;
+    //if parse failed, try "%Y-%m-%d" format
+    str = taosStrpTime(timestr, "%Y-%m-%d", &tm);
+    if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) {
+      return -1;
+    }
   }
 
 #ifdef _MSC_VER
@@ -367,14 +385,26 @@ int32_t parseLocaltime(char* timestr, int32_t len, int64_t* time, int32_t timePr
   return 0;
 }
 
-int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* time, int32_t timePrec) {
+int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* time, int32_t timePrec, char delim) {
   *time = 0;
   struct tm tm = {0};
   tm.tm_isdst = -1;
 
-  char* str = taosStrpTime(timestr, "%Y-%m-%d %H:%M:%S", &tm);
+  char *str;
+  if (delim == 'T') {
+    str = taosStrpTime(timestr, "%Y-%m-%dT%H:%M:%S", &tm);
+  } else if (delim == 0) {
+    str = taosStrpTime(timestr, "%Y-%m-%d %H:%M:%S", &tm);
+  } else {
+    str = NULL;
+  }
+
   if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) {
-    return -1;
+    //if parse failed, try "%Y-%m-%d" format
+    str = taosStrpTime(timestr, "%Y-%m-%d", &tm);
+    if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) {
+      return -1;
+    }
   }
 
   /* mktime will be affected by TZ, set by using taos_options */

source/common/test/commonTests.cpp

@@ -88,26 +88,15 @@ TEST(testCase, toInteger_test) {
 }
 
 TEST(testCase, Datablock_test) {
-  SSDataBlock* b = static_cast<SSDataBlock*>(taosMemoryCalloc(1, sizeof(SSDataBlock)));
+  SSDataBlock* b = createDataBlock();
-  b->info.numOfCols = 2;
-  b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
 
-  SColumnInfoData infoData = {0};
+  SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 1);
-  infoData.info.bytes = 4;
-  infoData.info.type = TSDB_DATA_TYPE_INT;
-  infoData.info.colId = 1;
-
-  infoData.pData = (char*)taosMemoryCalloc(40, infoData.info.bytes);
-  infoData.nullbitmap = (char*)taosMemoryCalloc(1, sizeof(char) * (40 / 8));
   taosArrayPush(b->pDataBlock, &infoData);
+  blockDataAppendColInfo(b, &infoData);
 
-  SColumnInfoData infoData1 = {0};
+  SColumnInfoData infoData1 = createColumnInfoData(TSDB_DATA_TYPE_BINARY, 40, 2);
-  infoData1.info.bytes = 40;
+  blockDataAppendColInfo(b, &infoData1);
-  infoData1.info.type = TSDB_DATA_TYPE_BINARY;
+  blockDataEnsureCapacity(b, 40);
-  infoData1.info.colId = 2;
-
-  infoData1.varmeta.offset = (int32_t*)taosMemoryCalloc(40, sizeof(uint32_t));
-  taosArrayPush(b->pDataBlock, &infoData1);
 
   char* str = "the value of: %d";
   char  buf[128] = {0};
@@ -222,28 +211,17 @@ TEST(testCase, non_var_dataBlock_split_test) {
 #endif
 
 TEST(testCase, var_dataBlock_split_test) {
-  SSDataBlock* b = static_cast<SSDataBlock*>(taosMemoryCalloc(1, sizeof(SSDataBlock)));
-  b->info.numOfCols = 2;
-  b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
-
   int32_t numOfRows = 1000000;
 
-  SColumnInfoData infoData = {0};
+  SSDataBlock* b = createDataBlock();
-  infoData.info.bytes = 4;
-  infoData.info.type = TSDB_DATA_TYPE_INT;
-  infoData.info.colId = 1;
 
-  infoData.pData = (char*)taosMemoryCalloc(numOfRows, infoData.info.bytes);
+  SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 1);
-  infoData.nullbitmap = (char*)taosMemoryCalloc(1, sizeof(char) * (numOfRows / 8));
+  blockDataAppendColInfo(b, &infoData);
-  taosArrayPush(b->pDataBlock, &infoData);
 
-  SColumnInfoData infoData1 = {0};
+  SColumnInfoData infoData1 = createColumnInfoData(TSDB_DATA_TYPE_BINARY, 40, 2);
-  infoData1.info.bytes = 40;
+  blockDataAppendColInfo(b, &infoData1);
-  infoData1.info.type = TSDB_DATA_TYPE_BINARY;
-  infoData1.info.colId = 2;
 
-  infoData1.varmeta.offset = (int32_t*)taosMemoryCalloc(numOfRows, sizeof(uint32_t));
+  blockDataEnsureCapacity(b, numOfRows);
-  taosArrayPush(b->pDataBlock, &infoData1);
 
   char buf[41] = {0};
   char buf1[100] = {0};

source/dnode/mgmt/mgmt_dnode/src/dmHandle.c

@@ -123,8 +123,15 @@ int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
 }
 
 int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
-  dError("config req is received, but not supported yet");
+  SDCfgDnodeReq cfgReq = {0};
-  return TSDB_CODE_OPS_NOT_SUPPORT;
+  if (tDeserializeSDCfgDnodeReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  dInfo("start to config, option:%s, value:%s", cfgReq.config, cfgReq.value);
+  taosCfgDynamicOptions(cfgReq.config, cfgReq.value);
+  return 0;
 }
 
 static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) {
@@ -203,7 +210,6 @@ SSDataBlock* dmBuildVariablesBlock(void) {
     taosArrayPush(pBlock->pDataBlock, &colInfoData);
   }
 
-  pBlock->info.numOfCols = pMeta[index].colNum;
   pBlock->info.hasVarCol = true;
 
   return pBlock;
@@ -262,8 +268,9 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
 
   dmAppendVariablesToBlock(pBlock, pMgmt->pData->dnodeId);
 
-  size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pBlock->info.numOfCols +
+  size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
-         blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(pBlock->info.numOfCols);
+  size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * numOfCols +
+         blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(numOfCols);
 
   SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size);
   if (pRsp == NULL) {
@@ -274,10 +281,10 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   }
 
   char    *pStart = pRsp->data;
-  *(int32_t *)pStart = htonl(pBlock->info.numOfCols);
+  *(int32_t *)pStart = htonl(numOfCols);
   pStart += sizeof(int32_t);  // number of columns
 
-  for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
+  for (int32_t i = 0; i < numOfCols; ++i) {
     SSysTableSchema *pSchema = (SSysTableSchema *)pStart;
     SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, i);
     
@@ -289,7 +296,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   }
 
   int32_t len = 0;
-  blockCompressEncode(pBlock, pStart, &len, pBlock->info.numOfCols, false);
+  blockCompressEncode(pBlock, pStart, &len, numOfCols, false);
 
   pRsp->numOfRows = htonl(pBlock->info.rows);
   pRsp->precision = TSDB_TIME_PRECISION_MILLI;  // millisecond time precision

source/dnode/mgmt/node_util/src/dmFile.c

@@ -127,14 +127,24 @@ TdFilePtr dmCheckRunning(const char *dataDir) {
     return NULL;
   }
 
-  int32_t ret = taosLockFile(pFile);
+  int32_t retryTimes = 0;
-  if (ret != 0) {
+  int32_t ret = 0;
+  do {
+    ret = taosLockFile(pFile);
+    if (ret == 0) break;
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    taosMsleep(100);
+    retryTimes++;
+    dError("failed to lock file:%s since %s, retryTimes:%d", filepath, terrstr(), retryTimes);
+  } while (retryTimes < 120);
+
+  if (ret < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
-    dError("failed to lock file:%s since %s", filepath, terrstr());
     taosCloseFile(&pFile);
     return NULL;
   }
 
+  terrno = 0;
   dDebug("lock file:%s to prevent repeated starts", filepath);
   return pFile;
 }

source/dnode/mnode/impl/src/mndDnode.c

@@ -502,12 +502,12 @@ _OVER:
 }
 
 static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) {
-  SMnode *pMnode = pReq->info.node;
+  SMnode       *pMnode = pReq->info.node;
-  SSdb   *pSdb = pMnode->pSdb;
+  SSdb         *pSdb = pMnode->pSdb;
-  SDnodeObj *pObj = NULL;
+  SDnodeObj    *pObj = NULL;
-  void *pIter = NULL;
+  void         *pIter = NULL;
   SDnodeListRsp rsp = {0};
-  int32_t code = -1;
+  int32_t       code = -1;
 
   rsp.dnodeList = taosArrayInit(5, sizeof(SEpSet));
   if (NULL == rsp.dnodeList) {
@@ -554,7 +554,6 @@ _OVER:
   return code;
 }
 
-
 static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
   SMnode         *pMnode = pReq->info.node;
   int32_t         code = -1;
@@ -648,30 +647,30 @@ _OVER:
 }
 
 static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
-  SMnode        *pMnode = pReq->info.node;
+  SMnode       *pMnode = pReq->info.node;
-  int32_t        code = -1;
+  int32_t       code = -1;
-  SDnodeObj     *pDnode = NULL;
+  SDnodeObj    *pDnode = NULL;
-  SMnodeObj     *pMObj = NULL;
+  SMnodeObj    *pMObj = NULL;
-  SQnodeObj     *pQObj = NULL;
+  SQnodeObj    *pQObj = NULL;
-  SSnodeObj     *pSObj = NULL;
+  SSnodeObj    *pSObj = NULL;
-  SMDropMnodeReq dropReq = {0};
+  SDropDnodeReq dropReq = {0};
 
-  if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
+  if (tDeserializeSDropDnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
     goto _OVER;
   }
 
-  mInfo("dnode:%d, start to drop", dropReq.dnodeId);
+  mInfo("dnode:%d, start to drop, ep:%s:%d", dropReq.dnodeId, dropReq.fqdn, dropReq.port);
-
-  if (dropReq.dnodeId <= 0) {
-    terrno = TSDB_CODE_MND_INVALID_DNODE_ID;
-    goto _OVER;
-  }
 
   pDnode = mndAcquireDnode(pMnode, dropReq.dnodeId);
   if (pDnode == NULL) {
-    terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
+    char ep[TSDB_EP_LEN + 1] = {0};
-    goto _OVER;
+    snprintf(ep, sizeof(ep), dropReq.fqdn, dropReq.port);
+    pDnode = mndAcquireDnodeByEp(pMnode, ep);
+    if (pDnode == NULL) {
+      terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
+      goto _OVER;
+    }
   }
 
   pQObj = mndAcquireQnode(pMnode, dropReq.dnodeId);
@@ -726,23 +725,46 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
     return -1;
   }
 
+  mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value);
   SDnodeObj *pDnode = mndAcquireDnode(pMnode, cfgReq.dnodeId);
   if (pDnode == NULL) {
     mError("dnode:%d, failed to config since %s ", cfgReq.dnodeId, terrstr());
     return -1;
   }
-
   SEpSet epSet = mndGetDnodeEpset(pDnode);
   mndReleaseDnode(pMnode, pDnode);
 
-  int32_t bufLen = tSerializeSMCfgDnodeReq(NULL, 0, &cfgReq);
+  SDCfgDnodeReq dcfgReq = {0};
+  if (strncasecmp(cfgReq.config, "debugFlag", 9) == 0) {
+    const char *value = cfgReq.value;
+    int32_t     flag = atoi(value);
+    if (flag <= 0) {
+      flag = atoi(cfgReq.config + 10);
+    }
+    if (flag <= 0 || flag > 255) {
+      mError("dnode:%d, failed to config debugFlag since value:%d", cfgReq.dnodeId, flag);
+      terrno = TSDB_CODE_INVALID_CFG;
+      return -1;
+    }
+
+    strcpy(dcfgReq.config, "debugFlag");
+    snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
+  } else if (strcasecmp(cfgReq.config, "resetlog") == 0) {
+    strcpy(dcfgReq.config, "resetlog");
+  } else {
+    terrno = TSDB_CODE_INVALID_CFG;
+    mError("dnode:%d, failed to config since %s", cfgReq.dnodeId, terrstr());
+    return -1;
+  }
+
+  int32_t bufLen = tSerializeSDCfgDnodeReq(NULL, 0, &dcfgReq);
   void   *pBuf = rpcMallocCont(bufLen);
 
   if (pBuf == NULL) return -1;
-  tSerializeSMCfgDnodeReq(pBuf, bufLen, &cfgReq);
+  tSerializeSDCfgDnodeReq(pBuf, bufLen, &dcfgReq);
 
   mDebug("dnode:%d, send config req to dnode, app:%p", cfgReq.dnodeId, pReq->info.ahandle);
-  SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen, .info = pReq->info};
+  SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen};
   return tmsgSendReq(&epSet, &rpcMsg);
 }
 

source/dnode/mnode/impl/src/mndMain.c

@@ -632,6 +632,8 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
   tstrncpy(pClusterInfo->version, version, sizeof(pClusterInfo->version));
   pClusterInfo->monitor_interval = tsMonitorInterval;
   pClusterInfo->connections_total = mndGetNumOfConnections(pMnode);
+  pClusterInfo->dbs_total = sdbGetSize(pSdb, SDB_DB);
+  pClusterInfo->stbs_total = sdbGetSize(pSdb, SDB_STB);
 
   void *pIter = NULL;
   while (1) {
@@ -681,6 +683,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
     if (pIter == NULL) break;
 
     pClusterInfo->vgroups_total++;
+    pClusterInfo->tbs_total += pVgroup->numOfTables;
 
     SMonVgroupDesc desc = {0};
     desc.vgroup_id = pVgroup->vgId;

source/dnode/mnode/impl/src/mndScheduler.c

@@ -509,7 +509,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
       SStreamTask* pTask = tNewSStreamTask(pStream->uid);
       mndAddTaskToTaskSet(taskOneLevel, pTask);
 
-      // input
+      // source
       pTask->isDataScan = 1;
 
       // trigger

source/dnode/mnode/impl/src/mndShow.c

@@ -236,7 +236,6 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
   int32_t      numOfCols = pShow->pMeta->numOfColumns;
   SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
   pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
-  pBlock->info.numOfCols = numOfCols;
 
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData idata = {0};
@@ -271,7 +270,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
   }
 
   size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pShow->pMeta->numOfColumns +
-         blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(pBlock->info.numOfCols);
+         blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock));
 
   SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size);
   if (pRsp == NULL) {

source/dnode/vnode/CMakeLists.txt

@@ -29,6 +29,7 @@ target_sources(
     # sma
     "src/sma/sma.c"
     "src/sma/smaEnv.c"
+    "src/sma/smaUtil.c"
     "src/sma/smaOpen.c"
     "src/sma/smaRollup.c"
     "src/sma/smaTimeRange.c"

source/dnode/vnode/inc/vnode.h

@@ -150,8 +150,8 @@ int32_t tqReadHandleRemoveTbUidList(STqReadHandle *pHandle, const SArray *tbUidL
 int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver);
 bool    tqNextDataBlock(STqReadHandle *pHandle);
 bool    tqNextDataBlockFilterOut(STqReadHandle *pHandle, SHashObj *filterOutUids);
-int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t *pUid,
+int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t *pUid,
-                            int32_t *pNumOfRows, int16_t *pNumOfCols);
+                            int32_t *pNumOfRows);
 
 // sma
 int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);

source/dnode/vnode/src/inc/sma.h

@@ -34,7 +34,8 @@ extern "C" {
 
 typedef struct SSmaEnv       SSmaEnv;
 typedef struct SSmaStat      SSmaStat;
-typedef struct SSmaStatItem  SSmaStatItem;
+typedef struct STSmaStat     STSmaStat;
+typedef struct SRSmaStat     SRSmaStat;
 typedef struct SSmaKey       SSmaKey;
 typedef struct SRSmaInfo     SRSmaInfo;
 typedef struct SRSmaInfoItem SRSmaInfoItem;
@@ -45,26 +46,38 @@ struct SSmaEnv {
   SSmaStat      *pStat;
 };
 
-#define SMA_ENV_LOCK(env)       ((env)->lock)
+#define SMA_ENV_LOCK(env) ((env)->lock)
-#define SMA_ENV_TYPE(env)       ((env)->type)
+#define SMA_ENV_TYPE(env) ((env)->type)
-#define SMA_ENV_STAT(env)       ((env)->pStat)
+#define SMA_ENV_STAT(env) ((env)->pStat)
-#define SMA_ENV_STAT_ITEM(env) ((env)->pStat->tsmaStatItem)
 
-struct SSmaStatItem {
+struct STSmaStat {
   int8_t    state;  // ETsdbSmaStat
   STSma    *pTSma;  // cache schema
   STSchema *pTSchema;
 };
 
+struct SRSmaStat {
+  SSma     *pSma;
+  void     *tmrHandle;
+  tmr_h     tmrId;
+  int8_t    tmrStat;
+  int32_t   tmrSeconds;
+  SHashObj *rsmaInfoHash;  // key: stbUid, value: SRSmaInfo;
+};
+
 struct SSmaStat {
   union {
-    SSmaStatItem tsmaStatItem;
+    STSmaStat tsmaStat;  // time-range-wise sma
-    SHashObj    *rsmaInfoHash;  // key: stbUid, value: SRSmaInfo;
+    SRSmaStat rsmaStat;  // rollup sma
   };
   T_REF_DECLARE()
 };
-#define SMA_STAT_ITEM(s)      ((s)->tsmaStatItem)
+#define SMA_TSMA_STAT(s)       (&(s)->tsmaStat)
-#define SMA_STAT_INFO_HASH(s) ((s)->rsmaInfoHash)
+#define SMA_RSMA_STAT(s)       (&(s)->rsmaStat)
+#define SMA_RSMA_INFO_HASH(s)  ((s)->rsmaStat.rsmaInfoHash)
+#define SMA_RSMA_TMR_HANDLE(s) ((s)->rsmaStat.tmrHandle)
+#define SMA_RSMA_TMR_STAT(s)   ((s)->rsmaStat.tmrStat)
+#define RSMA_INFO_HASH(r)      ((r)->rsmaInfoHash)
 
 void  tdDestroySmaEnv(SSmaEnv *pSmaEnv);
 void *tdFreeSmaEnv(SSmaEnv *pSmaEnv);
@@ -107,53 +120,51 @@ static FORCE_INLINE int32_t tdUnLockSmaEnv(SSmaEnv *pEnv) {
   return 0;
 }
 
-static FORCE_INLINE int8_t tdSmaStat(SSmaStatItem *pStatItem) {
+static FORCE_INLINE int8_t tdSmaStat(STSmaStat *pTStat) {
-  if (pStatItem) {
+  if (pTStat) {
-    return atomic_load_8(&pStatItem->state);
+    return atomic_load_8(&pTStat->state);
   }
   return TSDB_SMA_STAT_UNKNOWN;
 }
 
-static FORCE_INLINE bool tdSmaStatIsOK(SSmaStatItem *pStatItem, int8_t *state) {
+static FORCE_INLINE bool tdSmaStatIsOK(STSmaStat *pTStat, int8_t *state) {
-  if (!pStatItem) {
+  if (!pTStat) {
     return false;
   }
 
   if (state) {
-    *state = atomic_load_8(&pStatItem->state);
+    *state = atomic_load_8(&pTStat->state);
     return *state == TSDB_SMA_STAT_OK;
   }
-  return atomic_load_8(&pStatItem->state) == TSDB_SMA_STAT_OK;
+  return atomic_load_8(&pTStat->state) == TSDB_SMA_STAT_OK;
 }
 
-static FORCE_INLINE bool tdSmaStatIsExpired(SSmaStatItem *pStatItem) {
+static FORCE_INLINE bool tdSmaStatIsExpired(STSmaStat *pTStat) {
-  return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_EXPIRED) : true;
+  return pTStat ? (atomic_load_8(&pTStat->state) & TSDB_SMA_STAT_EXPIRED) : true;
 }
 
-static FORCE_INLINE bool tdSmaStatIsDropped(SSmaStatItem *pStatItem) {
+static FORCE_INLINE bool tdSmaStatIsDropped(STSmaStat *pTStat) {
-  return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_DROPPED) : true;
+  return pTStat ? (atomic_load_8(&pTStat->state) & TSDB_SMA_STAT_DROPPED) : true;
 }
 
-static FORCE_INLINE void tdSmaStatSetOK(SSmaStatItem *pStatItem) {
+static FORCE_INLINE void tdSmaStatSetOK(STSmaStat *pTStat) {
-  if (pStatItem) {
+  if (pTStat) {
-    atomic_store_8(&pStatItem->state, TSDB_SMA_STAT_OK);
+    atomic_store_8(&pTStat->state, TSDB_SMA_STAT_OK);
   }
 }
 
-static FORCE_INLINE void tdSmaStatSetExpired(SSmaStatItem *pStatItem) {
+static FORCE_INLINE void tdSmaStatSetExpired(STSmaStat *pTStat) {
-  if (pStatItem) {
+  if (pTStat) {
-    atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_EXPIRED);
+    atomic_or_fetch_8(&pTStat->state, TSDB_SMA_STAT_EXPIRED);
   }
 }
 
-static FORCE_INLINE void tdSmaStatSetDropped(SSmaStatItem *pStatItem) {
+static FORCE_INLINE void tdSmaStatSetDropped(STSmaStat *pTStat) {
-  if (pStatItem) {
+  if (pTStat) {
-    atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_DROPPED);
+    atomic_or_fetch_8(&pTStat->state, TSDB_SMA_STAT_DROPPED);
   }
 }
 
-static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType);
-void          *tdFreeSmaStatItem(SSmaStatItem *pSmaStatItem);
 static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType);
 void          *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType);
 
@@ -163,6 +174,51 @@ int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg);
 int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg);
 int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);
 
+typedef struct STFInfo STFInfo;
+typedef struct STFile  STFile;
+
+struct STFInfo {
+  uint32_t magic;
+  uint32_t ftype;
+  uint32_t fver;
+  uint64_t fsize;
+};
+
+struct STFile {
+  STFInfo   info;
+  STfsFile  f;
+  TdFilePtr pFile;
+  uint8_t   state;
+};
+
+#define TD_FILE_F(tf)            (&((tf)->f))
+#define TD_FILE_PFILE(tf)        ((tf)->pFile)
+#define TD_FILE_OPENED(tf)       (TD_FILE_PFILE(tf) != NULL)
+#define TD_FILE_FULL_NAME(tf)    (TD_FILE_F(tf)->aname)
+#define TD_FILE_REL_NAME(tf)     (TD_FILE_F(tf)->rname)
+#define TD_FILE_OPENED(tf)       (TD_FILE_PFILE(tf) != NULL)
+#define TD_FILE_CLOSED(tf)       (!TD_FILE_OPENED(tf))
+#define TD_FILE_SET_CLOSED(f)    (TD_FILE_PFILE(f) = NULL)
+#define TD_FILE_STATE(tf)        ((tf)->state)
+#define TD_FILE_SET_STATE(tf, s) ((tf)->state = (s))
+#define TD_FILE_DID(tf)          (TD_FILE_F(tf)->did)
+#define TD_FILE_IS_OK(tf)        (TD_FILE_STATE(tf) == TD_FILE_STATE_OK)
+#define TD_FILE_IS_BAD(tf)       (TD_FILE_STATE(tf) == TD_FILE_STATE_BAD)
+
+int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname);
+int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fType);
+int32_t tdOpenTFile(STFile *pTFile, int flags);
+int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte);
+int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence);
+int64_t tdWriteTFile(STFile *pTFile, void *buf, int64_t nbyte);
+int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset);
+int32_t tdRemoveTFile(STFile *pTFile);
+int32_t tdLoadTFileHeader(STFile *pTFile, STFInfo *pInfo);
+int32_t tdUpdateTFileHeader(STFile *pTFile);
+void    tdUpdateTFileMagic(STFile *pTFile, void *pCksm);
+void    tdCloseTFile(STFile *pTFile);
+void    tdGetVndFileName(int32_t vid, const char *dname, const char *fname, char *outputName);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/inc/tq.h

@@ -150,6 +150,7 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalHead*
 
 // tqExec
 int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkRsp* pRsp, int32_t workerId);
+int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp, int32_t workerId);
 int32_t tqSendPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataBlkRsp* pRsp);
 
 // tqMeta

source/dnode/vnode/src/inc/vnodeInt.h

@@ -247,7 +247,6 @@ struct SVnode {
 
 struct STbUidStore {
   tb_uid_t  suid;
-  tb_uid_t  uid;  // TODO: just for debugging, remove when uid provided in SSDataBlock
   SArray*   tbUids;
   SHashObj* uidHash;
 };

source/dnode/vnode/src/meta/metaQuery.c

@@ -264,8 +264,8 @@ struct SMCtbCursor {
 SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) {
   SMCtbCursor *pCtbCur = NULL;
   SCtbIdxKey   ctbIdxKey;
-  int          ret;
+  int          ret = 0;
-  int          c;
+  int          c = 0;
 
   pCtbCur = (SMCtbCursor *)taosMemoryCalloc(1, sizeof(*pCtbCur));
   if (pCtbCur == NULL) {

source/dnode/vnode/src/sma/smaEnv.c

@@ -21,9 +21,10 @@ typedef struct SSmaStat SSmaStat;
 
 // declaration of static functions
 
-static int32_t  tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType);
+static int32_t  tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pSma);
 static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path);
 static int32_t  tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SSmaEnv **pEnv);
+static void    *tdFreeTSmaStat(STSmaStat *pStat);
 
 // implementation
 
@@ -45,7 +46,7 @@ static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path)
     return NULL;
   }
 
-  if (tdInitSmaStat(&SMA_ENV_STAT(pEnv), smaType) != TSDB_CODE_SUCCESS) {
+  if (tdInitSmaStat(&SMA_ENV_STAT(pEnv), smaType, pSma) != TSDB_CODE_SUCCESS) {
     tdFreeSmaEnv(pEnv);
     return NULL;
   }
@@ -105,7 +106,7 @@ int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) {
   return 0;
 }
 
-static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType) {
+static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pSma) {
   ASSERT(pSmaStat != NULL);
 
   if (*pSmaStat) {  // no lock
@@ -125,10 +126,23 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType) {
     }
 
     if (smaType == TSDB_SMA_TYPE_ROLLUP) {
-      SMA_STAT_INFO_HASH(*pSmaStat) = taosHashInit(
+      SMA_RSMA_STAT(*pSmaStat)->pSma = (SSma*)pSma;
-          RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
+      // init timer
+      SMA_RSMA_TMR_HANDLE(*pSmaStat) = taosTmrInit(10000, 100, 10000, "RSMA_G");
+      if (!SMA_RSMA_TMR_HANDLE(*pSmaStat)) {
+        taosMemoryFreeClear(*pSmaStat);
+        return TSDB_CODE_FAILED;
+      }
       
-      if (!SMA_STAT_INFO_HASH(*pSmaStat)) {
+      atomic_store_8(&SMA_RSMA_TMR_STAT(*pSmaStat), TASK_TRIGGER_STATUS__ACTIVE);
+
+      // init hash
+      SMA_RSMA_INFO_HASH(*pSmaStat) = taosHashInit(
+          RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
+      if (!SMA_RSMA_INFO_HASH(*pSmaStat)) {
+        if (SMA_RSMA_TMR_HANDLE(*pSmaStat)) {
+          taosTmrCleanUp(SMA_RSMA_TMR_HANDLE(*pSmaStat));
+        }
         taosMemoryFreeClear(*pSmaStat);
         return TSDB_CODE_FAILED;
       }
@@ -141,16 +155,16 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType) {
   return TSDB_CODE_SUCCESS;
 }
 
-void *tdFreeSmaStatItem(SSmaStatItem *pSmaStatItem) {
+static void *tdFreeTSmaStat(STSmaStat *pStat) {
-  if (pSmaStatItem) {
+  if (pStat) {
-    tDestroyTSma(pSmaStatItem->pTSma);
+    tDestroyTSma(pStat->pTSma);
-    taosMemoryFreeClear(pSmaStatItem->pTSma);
+    taosMemoryFreeClear(pStat->pTSma);
-    taosMemoryFreeClear(pSmaStatItem);
+    taosMemoryFreeClear(pStat);
   }
   return NULL;
 }
 
-void* tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) {
+void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) {
   tdDestroySmaState(pSmaStat, smaType);
   taosMemoryFreeClear(pSmaStat);
   return NULL;
@@ -165,16 +179,19 @@ void* tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) {
 int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) {
   if (pSmaStat) {
     if (smaType == TSDB_SMA_TYPE_TIME_RANGE) {
-      tdFreeSmaStatItem(&pSmaStat->tsmaStatItem);
+      tdFreeTSmaStat(&pSmaStat->tsmaStat);
     } else if (smaType == TSDB_SMA_TYPE_ROLLUP) {
+      if (SMA_RSMA_TMR_HANDLE(pSmaStat)) {
+        taosTmrCleanUp(SMA_RSMA_TMR_HANDLE(pSmaStat));
+      }
       // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready.
-      void *infoHash = taosHashIterate(SMA_STAT_INFO_HASH(pSmaStat), NULL);
+      void *infoHash = taosHashIterate(SMA_RSMA_INFO_HASH(pSmaStat), NULL);
       while (infoHash) {
         SRSmaInfo *pInfoHash = *(SRSmaInfo **)infoHash;
         tdFreeRSmaInfo(pInfoHash);
-        infoHash = taosHashIterate(SMA_STAT_INFO_HASH(pSmaStat), infoHash);
+        infoHash = taosHashIterate(SMA_RSMA_INFO_HASH(pSmaStat), infoHash);
       }
-      taosHashCleanup(SMA_STAT_INFO_HASH(pSmaStat));
+      taosHashCleanup(SMA_RSMA_INFO_HASH(pSmaStat));
     } else {
       ASSERT(0);
     }

source/dnode/vnode/src/sma/smaOpen.c

@@ -138,3 +138,16 @@ int32_t smaClose(SSma *pSma) {
   }
   return 0;
 }
+
+/**
+ * @brief rsma env restore
+ * 
+ * @param pSma 
+ * @return int32_t 
+ */
+int32_t smaRestore(SSma *pSma) {
+  if (!pSma) return 0;
+  // iterate all stables to restore the rsma env
+  
+  return TSDB_CODE_SUCCESS;
+}

source/dnode/vnode/src/sma/smaRollup.c

@@ -16,35 +16,17 @@
 #include "sma.h"
 #include "tstream.h"
 
-static FORCE_INLINE int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
+typedef enum { TD_QTASK_TMP_FILE = 0, TD_QTASK_CUR_FILE } TD_QTASK_FILE_T;
-static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids);
+static const char *tdQTaskInfoFname[] = {"qtaskinfo.t", "qtaskinfo"};
-static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *rsmaItem,
-                                              tb_uid_t suid, int8_t level);
 
-#define SET_RSMA_INFO_ITEM_PARAMS(__idx, __level)                                                               \
+static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
-  if (param->qmsg[__idx]) {                                                                                     \
+static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids);
-    pRSmaInfo->items[__idx].pRsmaInfo = pRSmaInfo;                                                              \
+static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo *pRSmaInfo, SReadHandle *handle,
-    pRSmaInfo->items[__idx].taskInfo = qCreateStreamExecTaskInfo(param->qmsg[0], &handle);                      \
+                                       int8_t idx);
-    if (!pRSmaInfo->items[__idx].taskInfo) {                                                                    \
+static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *rsmaItem,
-      goto _err;                                                                                                \
+                                 tb_uid_t suid, int8_t level);
-    }                                                                                                           \
+static void    tdRSmaFetchTrigger(void *param, void *tmrId);
-    pRSmaInfo->items[__idx].triggerStatus = TASK_TRIGGER_STATUS__IN_ACTIVE;                                     \
+static void    tdRSmaPersistTrigger(void *param, void *tmrId);
-    if (param->maxdelay[__idx] < 1) {                                                                           \
-      int64_t msInterval =                                                                                      \
-          convertTimeFromPrecisionToUnit(pRetention[__level].freq, pTsdbCfg->precision, TIME_UNIT_MILLISECOND); \
-      pRSmaInfo->items[__idx].maxDelay = msInterval;                                                            \
-    } else {                                                                                                    \
-      pRSmaInfo->items[__idx].maxDelay = param->maxdelay[__idx];                                                \
-    }                                                                                                           \
-    if (pRSmaInfo->items[__idx].maxDelay > TSDB_MAX_ROLLUP_MAX_DELAY) {                                         \
-      pRSmaInfo->items[__idx].maxDelay = TSDB_MAX_ROLLUP_MAX_DELAY;                                             \
-    }                                                                                                           \
-    pRSmaInfo->items[__idx].level = TSDB_RETENTION_L##__level;                                                  \
-    pRSmaInfo->items[__idx].tmrHandle = taosTmrInit(10000, 100, 10000, "RSMA");                                 \
-    if (!pRSmaInfo->items[__idx].tmrHandle) {                                                                   \
-      goto _err;                                                                                                \
-    }                                                                                                           \
-  }
 
 struct SRSmaInfoItem {
   SRSmaInfo *pRsmaInfo;
@@ -56,14 +38,6 @@ struct SRSmaInfoItem {
   int8_t     triggerStatus;  // TASK_TRIGGER_STATUS__IN_ACTIVE/TASK_TRIGGER_STATUS__ACTIVE
   int32_t    maxDelay;
 };
-
-typedef struct {
-  int64_t        suid;
-  SRSmaInfoItem *pItem;
-  SSma          *pSma;
-  STSchema      *pTSchema;
-} SRSmaTriggerParam;
-
 struct SRSmaInfo {
   STSchema     *pTSchema;
   SSma         *pSma;
@@ -81,7 +55,7 @@ static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle) {
 
 void *tdFreeRSmaInfo(SRSmaInfo *pInfo) {
   if (pInfo) {
-    for (int32_t i = 0; i < TSDB_RETENTION_MAX; ++i) {
+    for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
       SRSmaInfoItem *pItem = &pInfo->items[i];
       if (pItem->taskInfo) {
         tdFreeTaskHandle(pItem->taskInfo);
@@ -118,7 +92,7 @@ static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SA
     return TSDB_CODE_FAILED;
   }
 
-  pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), suid, sizeof(tb_uid_t));
+  pRSmaInfo = taosHashGet(SMA_RSMA_INFO_HASH(pStat), suid, sizeof(tb_uid_t));
   if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
     smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64, SMA_VID(pSma), *suid);
     terrno = TSDB_CODE_RSMA_INVALID_STAT;
@@ -187,7 +161,7 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui
 
   SSmaStat *pStat = SMA_ENV_STAT(pEnv);
   SHashObj *infoHash = NULL;
-  if (!pStat || !(infoHash = SMA_STAT_INFO_HASH(pStat))) {
+  if (!pStat || !(infoHash = SMA_RSMA_INFO_HASH(pStat))) {
     terrno = TSDB_CODE_RSMA_INVALID_STAT;
     return TSDB_CODE_FAILED;
   }
@@ -213,6 +187,40 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo *pRSmaInfo, SReadHandle *pReadHandle,
+                                       int8_t idx) {
+  SRetention *pRetention = SMA_RETENTION(pSma);
+  STsdbCfg   *pTsdbCfg = SMA_TSDB_CFG(pSma);
+
+  if (param->qmsg[idx]) {
+    SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]);
+    pItem->pRsmaInfo = pRSmaInfo;
+    pItem->taskInfo = qCreateStreamExecTaskInfo(param->qmsg[0], pReadHandle);
+    if (!pItem->taskInfo) {
+      goto _err;
+    }
+    pItem->triggerStatus = TASK_TRIGGER_STATUS__IN_ACTIVE;
+    if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) {
+      int64_t msInterval =
+          convertTimeFromPrecisionToUnit(pRetention[idx + 1].freq, pTsdbCfg->precision, TIME_UNIT_MILLISECOND);
+      pItem->maxDelay = (int32_t)msInterval;
+    } else {
+      pItem->maxDelay = (int32_t)param->maxdelay[idx];
+    }
+    if (pItem->maxDelay > TSDB_MAX_ROLLUP_MAX_DELAY) {
+      pItem->maxDelay = TSDB_MAX_ROLLUP_MAX_DELAY;
+    }
+    pItem->level = (idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2);
+    pItem->tmrHandle = taosTmrInit(10000, 100, 10000, "RSMA");
+    if (!pItem->tmrHandle) {
+      goto _err;
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+_err:
+  return TSDB_CODE_FAILED;
+}
+
 /**
  * @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam.
  *
@@ -246,7 +254,7 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) {
   SSmaStat  *pStat = SMA_ENV_STAT(pEnv);
   SRSmaInfo *pRSmaInfo = NULL;
 
-  pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t));
+  pRSmaInfo = taosHashGet(SMA_RSMA_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t));
   if (pRSmaInfo) {
     ASSERT(0);  // TODO: free original pRSmaInfo is exists abnormally
     smaWarn("vgId:%d, rsma info already exists for stb: %s, %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid);
@@ -282,14 +290,14 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) {
   pRSmaInfo->pSma = pSma;
   pRSmaInfo->suid = pReq->suid;
 
-  SRetention *pRetention = SMA_RETENTION(pSma);
+  if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 0) < 0) {
-  STsdbCfg   *pTsdbCfg = SMA_TSDB_CFG(pSma);
+    goto _err;
+  }
+  if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 1) < 0) {
+    goto _err;
+  }
 
-  SET_RSMA_INFO_ITEM_PARAMS(0, 1);
+  if (taosHashPut(SMA_RSMA_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) {
-  SET_RSMA_INFO_ITEM_PARAMS(1, 2);
-
-  if (taosHashPut(SMA_STAT_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) !=
-      TSDB_CODE_SUCCESS) {
     goto _err;
   } else {
     smaDebug("vgId:%d, register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), pReq->suid);
@@ -418,7 +426,6 @@ static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) {
 
     if (!pBlock) break;
     tdUidStorePut(pStore, msgIter.suid, NULL);
-    pStore->uid = msgIter.uid;  // TODO: remove, just for debugging
   }
 
   if (terrno != TSDB_CODE_SUCCESS) return -1;
@@ -439,8 +446,9 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType)
     if (!output) {
       break;
     }
+
     if (!pResult) {
-      pResult = taosArrayInit(0, sizeof(SSDataBlock));
+      pResult = taosArrayInit(1, sizeof(SSDataBlock));
       if (!pResult) {
         terrno = TSDB_CODE_OUT_OF_MEMORY;
         return TSDB_CODE_FAILED;
@@ -451,7 +459,7 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType)
   }
 
   if (taosArrayGetSize(pResult) > 0) {
-#if 0
+#if 1
     char flag[10] = {0};
     snprintf(flag, 10, "level %" PRIi8, pItem->level);
     blockDebugShowData(pResult, flag);
@@ -459,14 +467,12 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType)
     STsdb      *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb1 : pSma->pRSmaTsdb2);
     SSubmitReq *pReq = NULL;
     if (buildSubmitReqFromDataBlock(&pReq, pResult, pRSmaInfo->pTSchema, SMA_VID(pSma), pRSmaInfo->suid) < 0) {
-      taosArrayDestroy(pResult);
+      goto _err;
-      return TSDB_CODE_FAILED;
     }
 
     if (pReq && tdProcessSubmitReq(sinkTsdb, INT64_MAX, pReq) < 0) {
-      taosArrayDestroy(pResult);
       taosMemoryFreeClear(pReq);
-      return TSDB_CODE_FAILED;
+      goto _err;
     }
 
     taosMemoryFreeClear(pReq);
@@ -479,7 +485,10 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType)
   }
 
   taosArrayDestroy(pResult);
-  return 0;
+  return TSDB_CODE_SUCCESS;
+_err:
+  taosArrayDestroy(pResult);
+  return TSDB_CODE_FAILED;
 }
 
 /**
@@ -488,13 +497,12 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType)
  * @param param
  * @param tmrId
  */
-static void rsmaTriggerByTimer(void *param, void *tmrId) {
+static void tdRSmaFetchTrigger(void *param, void *tmrId) {
-  // SRSmaTriggerParam *pParam = (SRSmaTriggerParam *)param;
-  // SRSmaInfoItem     *pItem = pParam->pItem;
   SRSmaInfoItem *pItem = param;
 
   if (atomic_load_8(&pItem->triggerStatus) == TASK_TRIGGER_STATUS__ACTIVE) {
-    smaTrace("level %" PRIi8 " status is active for tb suid:%" PRIi64, pItem->level, pItem->pRsmaInfo->suid);
+    smaWarn("%s:%d THREAD:%" PRIi64 " level %" PRIi8 " status is active for tb suid:%" PRIi64, __func__, __LINE__,
+            taosGetSelfPthreadId(), pItem->level, pItem->pRsmaInfo->suid);
     SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL};
 
     atomic_store_8(&pItem->triggerStatus, TASK_TRIGGER_STATUS__IN_ACTIVE);
@@ -502,10 +510,11 @@ static void rsmaTriggerByTimer(void *param, void *tmrId) {
 
     tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SSDATA_BLOCK);
   } else {
-    smaTrace("level %" PRIi8 " status is inactive for tb suid:%" PRIi64, pItem->level, pItem->pRsmaInfo->suid);
+    smaWarn("%s:%d THREAD:%" PRIi64 " level %" PRIi8 " status is inactive for tb suid:%" PRIi64, __func__, __LINE__,
+            taosGetSelfPthreadId(), pItem->level, pItem->pRsmaInfo->suid);
   }
 
-  // taosTmrReset(rsmaTriggerByTimer, pItem->maxDelay, pItem, pItem->tmrHandle, &pItem->tmrId);
+  // taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, pItem->tmrHandle, &pItem->tmrId);
 }
 
 static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem,
@@ -518,16 +527,20 @@ static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int3
   smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level,
            pItem->taskInfo, suid);
 
-  // inputType = STREAM_DATA_TYPE_SUBMIT_BLOCK(1)
+  if (qSetStreamInput(pItem->taskInfo, pMsg, inputType, true) < 0) {  // STREAM_DATA_TYPE_SUBMIT_BLOCK
-  if (qSetStreamInput(pItem->taskInfo, pMsg, inputType, true) < 0) {
     smaError("vgId:%d, rsma % " PRIi8 " qSetStreamInput failed since %s", SMA_VID(pSma), level, tstrerror(terrno));
     return TSDB_CODE_FAILED;
   }
 
-  // SRSmaTriggerParam triggerParam = {.suid = suid, .pItem = pItem, .pSma = pSma, .pTSchema = pTSchema};
   tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SUBMIT_BLOCK);
   atomic_store_8(&pItem->triggerStatus, TASK_TRIGGER_STATUS__ACTIVE);
-  taosTmrReset(rsmaTriggerByTimer, pItem->maxDelay, pItem, pItem->tmrHandle, &pItem->tmrId);
+  smaWarn("%s:%d THREAD:%" PRIi64 " process rsma insert", __func__, __LINE__, taosGetSelfPthreadId());
+
+  SSmaEnv   *pEnv = SMA_RSMA_ENV(pSma);
+  SRSmaStat *pStat = SMA_RSMA_STAT(pEnv->pStat);
+
+  taosTmrStart(tdRSmaPersistTrigger, 5000, pStat, pStat->tmrHandle);
+  taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, pItem->tmrHandle, &pItem->tmrId);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -542,7 +555,7 @@ static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb
   SSmaStat  *pStat = SMA_ENV_STAT(pEnv);
   SRSmaInfo *pRSmaInfo = NULL;
 
-  pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
+  pRSmaInfo = taosHashGet(SMA_RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
 
   if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
     smaDebug("vgId:%d, return as no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid);
@@ -594,3 +607,106 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) {
   }
   return TSDB_CODE_SUCCESS;
 }
+
+void tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char* outputName) {
+  tdGetVndFileName(vid, "rsma", tdQTaskInfoFname[ftype], outputName);
+}
+
+static void *tdRSmaPersistExec(void *param) {
+  setThreadName("rsma-task-persist");
+  SRSmaStat *pRSmaStat = param;
+  SSma      *pSma = pRSmaStat->pSma;
+  STfs      *pTfs = pSma->pVnode->pTfs;
+  int64_t    toffset = 0;
+
+  void *infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL);
+  if (!infoHash) {
+    goto _end;
+  }
+
+  STFile  tFile = {0};
+  int32_t vid = 2;
+  char    qTaskInfoFName[TSDB_FILENAME_LEN];
+  tdRSmaQTaskGetFName(vid, TD_QTASK_TMP_FILE, qTaskInfoFName);
+  tdInitTFile(&tFile, pTfs, qTaskInfoFName);
+  tdCreateTFile(&tFile, pTfs, true, -1);
+
+  while (infoHash) {
+    SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash;
+    char      *pOutput = NULL;
+    int32_t    len = 0;
+    if (qSerializeTaskStatus(pRSmaInfo->items[0].taskInfo, &pOutput, &len) < 0) {
+      smaError("serialize rsma task for table %" PRIi64 " failed since %s", pRSmaInfo->items[0].pRsmaInfo->suid,
+               terrstr(terrno));
+    } else {
+      smaWarn("serialize rsma task for table %" PRIi64 " success and len is %d", pRSmaInfo->items[0].pRsmaInfo->suid,
+              len);
+    }
+    tdAppendTFile(&tFile, &len, sizeof(len), &toffset);
+    tdAppendTFile(&tFile, pOutput, len, &toffset);
+
+    taosMemoryFree(pOutput);
+    infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), infoHash);
+  }
+_end:
+
+  if (tdUpdateTFileHeader(&tFile) < 0) {
+    smaError("vgId:%d, failed to update tfile %s header since %s", vid, TD_FILE_FULL_NAME(&tFile), tstrerror(terrno));
+    tdCloseTFile(&tFile);
+    tdRemoveTFile(&tFile);
+    return NULL;
+  }
+  
+  tdCloseTFile(&tFile);
+
+  char newFName[TSDB_FILENAME_LEN];
+  strncpy(newFName, TD_FILE_FULL_NAME(&tFile), TSDB_FILENAME_LEN);
+  char *pos = strstr(newFName, tdQTaskInfoFname[TD_QTASK_TMP_FILE]);
+  strncpy(pos, tdQTaskInfoFname[TD_QTASK_CUR_FILE], TSDB_FILENAME_LEN - POINTER_DISTANCE(pos, newFName));
+  taosRenameFile(TD_FILE_FULL_NAME(&tFile), newFName);
+
+  atomic_store_8(&pRSmaStat->tmrStat, TASK_TRIGGER_STATUS__ACTIVE);
+  return NULL;
+_err:
+  atomic_store_8(&pRSmaStat->tmrStat, TASK_TRIGGER_STATUS__ACTIVE);
+  // remove the .tmp file
+  return NULL;
+}
+
+static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) {
+  smaWarn("%s:%d entry ", __func__, __LINE__);
+  TdThread     threadId;
+  TdThreadAttr thAttr;
+  taosThreadAttrInit(&thAttr);
+  taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_DETACHED);
+
+  if (taosThreadCreate(&threadId, &thAttr, tdRSmaPersistExec, pRSmaStat) != 0) {
+    smaError("failed to create thread to persist rsma qTaskInfo since %s", strerror(errno));
+  }
+
+  taosThreadAttrDestroy(&thAttr);
+  smaWarn("%s:%d end ", __func__, __LINE__);
+}
+
+/**
+ * @brief trigger to persist rsma qTaskInfo
+ *
+ * @param param
+ * @param tmrId
+ */
+static void tdRSmaPersistTrigger(void *param, void *tmrId) {
+  SRSmaStat *pRSmaStat = param;
+
+  if (atomic_load_8(&pRSmaStat->tmrStat) == TASK_TRIGGER_STATUS__ACTIVE) {
+    smaWarn("%s:%d THREAD:%" PRIi64 " rsma persistence start since active", __func__, __LINE__, taosGetSelfPthreadId());
+    atomic_store_8(&pRSmaStat->tmrStat, TASK_TRIGGER_STATUS__IN_ACTIVE);
+
+    // execution
+    tdRSmaPersistTask(pRSmaStat);
+  } else {
+    smaWarn("%s:%d THREAD:%" PRIi64 " rsma persistence not start since inactive", __func__, __LINE__,
+            taosGetSelfPthreadId());
+  }
+
+  taosTmrReset(tdRSmaPersistTrigger, 3600000, pRSmaStat, pRSmaStat->tmrHandle, &pRSmaStat->tmrId);
+}

source/dnode/vnode/src/sma/smaTimeRange.c

@@ -129,7 +129,7 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
 
   SSmaEnv      *pEnv = SMA_TSMA_ENV(pSma);
   SSmaStat     *pStat = NULL;
-  SSmaStatItem *pItem = NULL;
+  STSmaStat    *pItem = NULL;
 
   if (!pEnv || !(pStat = SMA_ENV_STAT(pEnv))) {
     terrno = TSDB_CODE_TSMA_INVALID_STAT;
@@ -137,7 +137,7 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
   }
 
   tdRefSmaStat(pSma, pStat);
-  pItem = &pStat->tsmaStatItem;
+  pItem = &pStat->tsmaStat;
 
   ASSERT(pItem);
 

source/dnode/vnode/src/sma/smaUtil.c

@@ -0,0 +1,238 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "sma.h"
+
+#define TD_FILE_HEAD_SIZE 512
+
+#define TD_FILE_STATE_OK  0
+#define TD_FILE_STATE_BAD 1
+
+#define TD_FILE_INIT_MAGIC 0xFFFFFFFF
+
+
+static int32_t tdEncodeTFInfo(void **buf, STFInfo *pInfo);
+static void   *tdDecodeTFInfo(void *buf, STFInfo *pInfo);
+
+static int32_t tdEncodeTFInfo(void **buf, STFInfo *pInfo) {
+  int32_t tlen = 0;
+
+  tlen += taosEncodeFixedU32(buf, pInfo->magic);
+  tlen += taosEncodeFixedU32(buf, pInfo->ftype);
+  tlen += taosEncodeFixedU32(buf, pInfo->fver);
+  tlen += taosEncodeFixedU64(buf, pInfo->fsize);
+
+  return tlen;
+}
+
+static void *tdDecodeTFInfo(void *buf, STFInfo *pInfo) {
+  buf = taosDecodeFixedU32(buf, &(pInfo->magic));
+  buf = taosDecodeFixedU32(buf, &(pInfo->ftype));
+  buf = taosDecodeFixedU32(buf, &(pInfo->fver));
+  buf = taosDecodeFixedU64(buf, &(pInfo->fsize));
+  return buf;
+}
+
+int64_t tdWriteTFile(STFile *pTFile, void *buf, int64_t nbyte) {
+  ASSERT(TD_FILE_OPENED(pTFile));
+
+  int64_t nwrite = taosWriteFile(pTFile->pFile, buf, nbyte);
+  if (nwrite < nbyte) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nwrite;
+}
+
+int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence) {
+  ASSERT(TD_FILE_OPENED(pTFile));
+
+  int64_t loffset = taosLSeekFile(TD_FILE_PFILE(pTFile), offset, whence);
+  if (loffset < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return loffset;
+}
+
+int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte) {
+  ASSERT(TD_FILE_OPENED(pTFile));
+
+  int64_t nread = taosReadFile(pTFile->pFile, buf, nbyte);
+  if (nread < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nread;
+}
+
+int32_t tdUpdateTFileHeader(STFile *pTFile) {
+  char buf[TD_FILE_HEAD_SIZE] = "\0";
+
+  if (tdSeekTFile(pTFile, 0, SEEK_SET) < 0) {
+    return -1;
+  }
+
+  void *ptr = buf;
+  tdEncodeTFInfo(&ptr, &(pTFile->info));
+
+  taosCalcChecksumAppend(0, (uint8_t *)buf, TD_FILE_HEAD_SIZE);
+  if (tdWriteTFile(pTFile, buf, TD_FILE_HEAD_SIZE) < 0) {
+    return -1;
+  }
+
+  return 0;
+}
+
+int32_t tdLoadTFileHeader(STFile *pTFile, STFInfo *pInfo) {
+  char     buf[TD_FILE_HEAD_SIZE] = "\0";
+  uint32_t _version;
+
+  ASSERT(TD_FILE_OPENED(pTFile));
+
+  if (tdSeekTFile(pTFile, 0, SEEK_SET) < 0) {
+    return -1;
+  }
+
+  if (tdReadTFile(pTFile, buf, TD_FILE_HEAD_SIZE) < 0) {
+    return -1;
+  }
+
+  if (!taosCheckChecksumWhole((uint8_t *)buf, TD_FILE_HEAD_SIZE)) {
+    terrno = TSDB_CODE_FILE_CORRUPTED;
+    return -1;
+  }
+
+  void *pBuf = buf;
+  pBuf = tdDecodeTFInfo(pBuf, pInfo);
+  return 0;
+}
+
+void tdUpdateTFileMagic(STFile *pTFile, void *pCksm) {
+  pTFile->info.magic = taosCalcChecksum(pTFile->info.magic, (uint8_t *)(pCksm), sizeof(TSCKSUM));
+}
+
+int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset) {
+  ASSERT(TD_FILE_OPENED(pTFile));
+
+  int64_t toffset;
+
+  if ((toffset = tdSeekTFile(pTFile, 0, SEEK_END)) < 0) {
+    return -1;
+  }
+
+  ASSERT(pTFile->info.fsize == toffset);
+
+  if (offset) {
+    *offset = toffset;
+  }
+
+  if (tdWriteTFile(pTFile, buf, nbyte) < 0) {
+    return -1;
+  }
+
+  pTFile->info.fsize += nbyte;
+
+  return nbyte;
+}
+
+int32_t tdOpenTFile(STFile *pTFile, int flags) {
+  ASSERT(!TD_FILE_OPENED(pTFile));
+
+  pTFile->pFile = taosOpenFile(TD_FILE_FULL_NAME(pTFile), flags);
+  if (pTFile->pFile == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return 0;
+}
+
+void tdCloseTFile(STFile *pTFile) {
+  if (TD_FILE_OPENED(pTFile)) {
+    taosCloseFile(&pTFile->pFile);
+    TD_FILE_SET_CLOSED(pTFile);
+  }
+}
+
+void tdGetVndFileName(int32_t vid, const char *dname, const char *fname, char *outputName) {
+  snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/%s", vid, dname, fname);
+}
+
+int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname) {
+  char    fullname[TSDB_FILENAME_LEN];
+  SDiskID did = {0};
+
+  TD_FILE_SET_STATE(pTFile, TD_FILE_STATE_OK);
+  TD_FILE_SET_CLOSED(pTFile);
+
+  memset(&(pTFile->info), 0, sizeof(pTFile->info));
+  pTFile->info.magic = TD_FILE_INIT_MAGIC;
+
+  if (tfsAllocDisk(pTfs, 0, &did) < 0) {
+    terrno = TSDB_CODE_NO_AVAIL_DISK;
+    return -1;
+  }
+
+  tfsInitFile(pTfs, &(pTFile->f), did, fname);
+
+  return 0;
+}
+
+int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fType) {
+  ASSERT(pTFile->info.fsize == 0 && pTFile->info.magic == TD_FILE_INIT_MAGIC);
+
+  pTFile->pFile = taosOpenFile(TD_FILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pTFile->pFile == NULL) {
+    if (errno == ENOENT) {
+      // Try to create directory recursively
+      char *s = strdup(TD_FILE_REL_NAME(pTFile));
+      if (tfsMkdirRecurAt(pTfs, taosDirName(s), TD_FILE_DID(pTFile)) < 0) {
+        taosMemoryFreeClear(s);
+        return -1;
+      }
+      taosMemoryFreeClear(s);
+
+      pTFile->pFile = taosOpenFile(TD_FILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+      if (pTFile->pFile == NULL) {
+        terrno = TAOS_SYSTEM_ERROR(errno);
+        return -1;
+      }
+    } else {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      return -1;
+    }
+  }
+
+  if (!updateHeader) {
+    return 0;
+  }
+
+  pTFile->info.fsize += TD_FILE_HEAD_SIZE;
+  pTFile->info.fver = 0;
+
+  if (tdUpdateTFileHeader(pTFile) < 0) {
+    tdCloseTFile(pTFile);
+    tdRemoveTFile(pTFile);
+    return -1;
+  }
+
+  return 0;
+}
+
+int32_t tdRemoveTFile(STFile *pTFile) { return tfsRemoveFile(TD_FILE_F(pTFile)); }

source/dnode/vnode/src/tq/tq.c

@@ -227,19 +227,16 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
     consumerEpoch = atomic_val_compare_exchange_32(&pHandle->epoch, consumerEpoch, reqEpoch);
   }
 
-  SWalHead* pHeadWithCkSum = taosMemoryMalloc(sizeof(SWalHead) + 2048);
-  if (pHeadWithCkSum == NULL) {
-    return -1;
-  }
-
-  walSetReaderCapacity(pHandle->pWalReader, 2048);
-
   SMqDataBlkRsp rsp = {0};
   rsp.reqOffset = pReq->currentOffset;
 
   rsp.blockData = taosArrayInit(0, sizeof(void*));
   rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
 
+  if (rsp.blockData == NULL || rsp.blockDataLen == NULL) {
+    return -1;
+  }
+
   rsp.withTbName = pReq->withTbName;
   if (rsp.withTbName) {
     rsp.blockTbName = taosArrayInit(0, sizeof(void*));
@@ -253,6 +250,32 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
     rsp.blockSchema = taosArrayInit(0, sizeof(void*));
   }
 
+#if 1
+  if (pReq->useSnapshot) {
+    // TODO set ver into snapshot
+    int64_t lastVer = walGetCommittedVer(pTq->pWal);
+    if (rsp.reqOffset < lastVer) {
+      tqInfo("retrieve using snapshot req offset %ld last ver %ld", rsp.reqOffset, lastVer);
+      tqScanSnapshot(pTq, &pHandle->execHandle, &rsp, workerId);
+
+      if (rsp.blockNum != 0) {
+        rsp.withTbName = false;
+        rsp.rspOffset = lastVer;
+        tqInfo("direct send by snapshot req offset %ld rsp offset %ld", rsp.reqOffset, rsp.rspOffset);
+        fetchOffset = lastVer;
+        goto SEND_RSP;
+      }
+    }
+  }
+#endif
+
+  SWalHead* pHeadWithCkSum = taosMemoryMalloc(sizeof(SWalHead) + 2048);
+  if (pHeadWithCkSum == NULL) {
+    return -1;
+  }
+
+  walSetReaderCapacity(pHandle->pWalReader, 2048);
+
   while (1) {
     consumerEpoch = atomic_load_32(&pHandle->epoch);
     if (consumerEpoch > reqEpoch) {
@@ -283,7 +306,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
              pHead->msgType == TDMT_VND_DROP_STB || pHead->msgType == TDMT_VND_CREATE_TABLE ||
              pHead->msgType == TDMT_VND_ALTER_TABLE || pHead->msgType == TDMT_VND_DROP_TABLE ||
              pHead->msgType == TDMT_VND_DROP_TTL_TABLE);
-      // return
+      tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType);
       SMqMetaRsp metaRsp = {0};
       metaRsp.reqOffset = pReq->currentOffset;
       metaRsp.rspOffset = fetchOffset;
@@ -292,6 +315,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
       metaRsp.metaRsp = pHead->body;
       if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) {
         code = -1;
+        goto OVER;
       }
       code = 0;
       goto OVER;
@@ -308,6 +332,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
 
   taosMemoryFree(pHeadWithCkSum);
 
+SEND_RSP:
   ASSERT(taosArrayGetSize(rsp.blockData) == rsp.blockNum);
   ASSERT(taosArrayGetSize(rsp.blockDataLen) == rsp.blockNum);
   if (rsp.withSchema) {
@@ -364,6 +389,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
     pHandle->epoch = -1;
 
     pHandle->execHandle.subType = req.subType;
+    pHandle->fetchMeta = req.withMeta;
 
     pHandle->pWalReader = walOpenReadHandle(pTq->pVnode->pWal);
     for (int32_t i = 0; i < 5; i++) {
@@ -376,6 +402,8 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
         SReadHandle handle = {
             .reader = pHandle->execHandle.pExecReader[i],
             .meta = pTq->pVnode->pMeta,
+            .vnode = pTq->pVnode,
+            .initTsdbReader = 1,
         };
         pHandle->execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle);
         ASSERT(pHandle->execHandle.execCol.task[i]);
@@ -448,6 +476,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) {
              .reader = pStreamReader,
              .meta = pTq->pVnode->pMeta,
              .vnode = pTq->pVnode,
+             .initTsdbReader = 1,
       };
       /*pTask->exec.inputHandle = pStreamReader;*/
       pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle);

source/dnode/vnode/src/tq/tqExec.c

@@ -29,7 +29,7 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataBlkRsp* pRs
 
   // TODO enable compress
   int32_t actualLen = 0;
-  blockCompressEncode(pBlock, pRetrieve->data, &actualLen, pBlock->info.numOfCols, false);
+  blockCompressEncode(pBlock, pRetrieve->data, &actualLen, taosArrayGetSize(pBlock->pDataBlock), false);
   actualLen += sizeof(SRetrieveTableRsp);
   ASSERT(actualLen <= dataStrLen);
   taosArrayPush(pRsp->blockDataLen, &actualLen);
@@ -60,6 +60,30 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, const STqExecHandle* pExec, SMqD
   return 0;
 }
 
+int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp, int32_t workerId) {
+  ASSERT(pExec->subType == TOPIC_SUB_TYPE__COLUMN);
+  qTaskInfo_t task = pExec->execCol.task[workerId];
+  if (qStreamScanSnapshot(task) < 0) {
+    ASSERT(0);
+  }
+  while (1) {
+    SSDataBlock* pDataBlock = NULL;
+    uint64_t     ts = 0;
+    if (qExecTask(task, &pDataBlock, &ts) < 0) {
+      ASSERT(0);
+    }
+    if (pDataBlock == NULL) break;
+
+    ASSERT(pDataBlock->info.rows != 0);
+    ASSERT(taosArrayGetSize(pDataBlock->pDataBlock) != 0);
+
+    tqAddBlockDataToRsp(pDataBlock, pRsp);
+    pRsp->blockNum++;
+  }
+
+  return 0;
+}
+
 int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkRsp* pRsp, int32_t workerId) {
   if (pExec->subType == TOPIC_SUB_TYPE__COLUMN) {
     qTaskInfo_t task = pExec->execCol.task[workerId];
@@ -74,7 +98,6 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR
       if (pDataBlock == NULL) break;
 
       ASSERT(pDataBlock->info.rows != 0);
-      ASSERT(pDataBlock->info.numOfCols != 0);
 
       tqAddBlockDataToRsp(pDataBlock, pRsp);
       if (pRsp->withTbName) {
@@ -88,8 +111,7 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR
     tqReadHandleSetMsg(pReader, pReq, 0);
     while (tqNextDataBlock(pReader)) {
       SSDataBlock block = {0};
-      if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.uid, &block.info.rows,
+      if (tqRetrieveDataBlock(&block, pReader, &block.info.groupId, &block.info.uid, &block.info.rows) < 0) {
-                              &block.info.numOfCols) < 0) {
         if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;
         ASSERT(0);
       }
@@ -106,8 +128,7 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR
     tqReadHandleSetMsg(pReader, pReq, 0);
     while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) {
       SSDataBlock block = {0};
-      if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.uid, &block.info.rows,
+      if (tqRetrieveDataBlock(&block, pReader, &block.info.groupId, &block.info.uid, &block.info.rows) < 0) {
-                              &block.info.numOfCols) < 0) {
         if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;
         ASSERT(0);
       }

source/dnode/vnode/src/tq/tqRead.c

@@ -142,8 +142,8 @@ bool tqNextDataBlockFilterOut(STqReadHandle* pHandle, SHashObj* filterOutUids) {
   return false;
 }
 
-int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* pGroupId, uint64_t* pUid,
+int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_t* pGroupId, uint64_t* pUid,
-                            int32_t* pNumOfRows, int16_t* pNumOfCols) {
+                            int32_t* pNumOfRows) {
   *pUid = 0;
 
   // TODO set to real sversion
@@ -180,23 +180,14 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
   int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList);
 
   if (colNumNeed == 0) {
-    *ppCols = taosArrayInit(pSchemaWrapper->nCols, sizeof(SColumnInfoData));
-    if (*ppCols == NULL) {
-      return -1;
-    }
-
     int32_t colMeta = 0;
     while (colMeta < pSchemaWrapper->nCols) {
       SSchema*        pColSchema = &pSchemaWrapper->pSchema[colMeta];
-      SColumnInfoData colInfo = {0};
+      SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId);
-      colInfo.info.bytes = pColSchema->bytes;
+      int32_t code = blockDataAppendColInfo(pBlock, &colInfo);
-      colInfo.info.colId = pColSchema->colId;
+      if (code != TSDB_CODE_SUCCESS) {
-      colInfo.info.type = pColSchema->type;
-
-      if (colInfoDataEnsureCapacity(&colInfo, 0, *pNumOfRows) < 0) {
         goto FAIL;
       }
-      taosArrayPush(*ppCols, &colInfo);
       colMeta++;
     }
   } else {
@@ -204,11 +195,6 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
       colNumNeed = pSchemaWrapper->nCols;
     }
 
-    *ppCols = taosArrayInit(colNumNeed, sizeof(SColumnInfoData));
-    if (*ppCols == NULL) {
-      return -1;
-    }
-
     int32_t colMeta = 0;
     int32_t colNeed = 0;
     while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) {
@@ -220,23 +206,22 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
       } else if (colIdSchema > colIdNeed) {
         colNeed++;
       } else {
-        SColumnInfoData colInfo = {0};
+        SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId);
-        colInfo.info.bytes = pColSchema->bytes;
+        int32_t code = blockDataAppendColInfo(pBlock, &colInfo);
-        colInfo.info.colId = pColSchema->colId;
+        if (code != TSDB_CODE_SUCCESS) {
-        colInfo.info.type = pColSchema->type;
-
-        if (colInfoDataEnsureCapacity(&colInfo, 0, *pNumOfRows) < 0) {
           goto FAIL;
         }
-        taosArrayPush(*ppCols, &colInfo);
         colMeta++;
         colNeed++;
       }
     }
   }
 
-  int32_t colActual = taosArrayGetSize(*ppCols);
+  if (blockDataEnsureCapacity(pBlock, *pNumOfRows) < 0) {
-  *pNumOfCols = colActual;
+    goto FAIL;
+  }
+
+  int32_t colActual = blockDataGetNumOfCols(pBlock);
 
   // TODO in stream shuffle case, fetch groupId
   *pGroupId = 0;
@@ -253,7 +238,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
     tdSTSRowIterReset(&iter, row);
     // get all wanted col of that block
     for (int32_t i = 0; i < colActual; i++) {
-      SColumnInfoData* pColData = taosArrayGet(*ppCols, i);
+      SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i);
       SCellVal         sVal = {0};
       if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) {
         break;
@@ -265,8 +250,9 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
     curRow++;
   }
   return 0;
-FAIL:
+
-  if (*ppCols) taosArrayDestroy(*ppCols);
+FAIL: // todo refactor here
+//  if (*ppCols) taosArrayDestroy(*ppCols);
   return -1;
 }
 

source/dnode/vnode/src/tq/tqSink.c

@@ -44,7 +44,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
       createTbReq.ctb.suid = suid;
 
       STagVal tagVal = {
-          .cid = pDataBlock->info.numOfCols + 1,
+          .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
           .type = TSDB_DATA_TYPE_UBIGINT,
           .i64 = (int64_t)pDataBlock->info.groupId,
       };
@@ -107,7 +107,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
       createTbReq.ctb.suid = suid;
 
       STagVal tagVal = {
-          .cid = pDataBlock->info.numOfCols + 1,
+          .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
           .type = TSDB_DATA_TYPE_UBIGINT,
           .i64 = (int64_t)pDataBlock->info.groupId,
       };

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -16,13 +16,11 @@
 #include "tsdb.h"
 #include "vnode.h"
 
-#define EXTRA_BYTES                2
 #define ASCENDING_TRAVERSE(o)      (o == TSDB_ORDER_ASC)
-#define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns)))
+#define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pResBlock->pDataBlock)))
 
 #define GET_FILE_DATA_BLOCK_INFO(_checkInfo, _block)                                      \
   ((SDataBlockInfo){.window = {.skey = (_block)->minKey.ts, .ekey = (_block)->maxKey.ts}, \
-                    .numOfCols = (_block)->numOfCols,                                     \
                     .rows = (_block)->numOfRows,                                          \
                     .uid = (_checkInfo)->tableId})
 
@@ -115,7 +113,8 @@ typedef struct STsdbReadHandle {
   //  SColumnDataAgg* statis;  // query level statistics, only one table block statistics info exists at any time
   //  SColumnDataAgg** pstatis;// the ptr array list to return to caller
   int32_t numOfBlocks;
-  SArray* pColumns;  // column list, SColumnInfoData array list
+  SSDataBlock* pResBlock;
+//  SArray* pColumns;  // column list, SColumnInfoData array list
   bool    locateStart;
   int32_t outputCapacity;
   int32_t realNumOfRows;
@@ -177,7 +176,7 @@ static SArray* getColumnIdList(STsdbReadHandle* pTsdbReadHandle) {
 
   SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t));
   for (int32_t i = 0; i < numOfCols; ++i) {
-    SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pColumns, i);
+    SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
     taosArrayPush(pIdList, &pCol->info.colId);
   }
 
@@ -422,23 +421,21 @@ static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond*
     }
 
     // todo: use list instead of array?
-    pReadHandle->pColumns = taosArrayInit(pCond->numOfCols, sizeof(SColumnInfoData));
+    pReadHandle->pResBlock = createDataBlock();
-    if (pReadHandle->pColumns == NULL) {
+    if (pReadHandle->pResBlock == NULL) {
       goto _end;
     }
 
     for (int32_t i = 0; i < pCond->numOfCols; ++i) {
-      SColumnInfoData colInfo = {{0}, 0};
+      SColumnInfoData colInfo = {.info = pCond->colList[i], 0};
-      colInfo.info = pCond->colList[i];
+      int32_t code = blockDataAppendColInfo(pReadHandle->pResBlock, &colInfo);
-
+      if (code != TSDB_CODE_SUCCESS){
-      int32_t code = colInfoDataEnsureCapacity(&colInfo, 0, pReadHandle->outputCapacity);
-      if (code != TSDB_CODE_SUCCESS) {
         goto _end;
       }
-
-      taosArrayPush(pReadHandle->pColumns, &colInfo);
     }
 
+    blockDataEnsureCapacity(pReadHandle->pResBlock, pReadHandle->outputCapacity);
+
     pReadHandle->suppInfo.defaultLoadColumn = getDefaultLoadColumns(pReadHandle, true);
 
     size_t size = taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn);
@@ -1546,12 +1543,12 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
   int32_t trueEnd = ascScan ? end : start;
   int32_t step = ascScan ? 1 : -1;
 
-  int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns);
+  int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pResBlock->pDataBlock);
 
   // data in buffer has greater timestamp, copy data in file block
   int32_t i = 0, j = 0;
   while (i < requiredNumOfCols && j < pCols->numOfCols) {
-    SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
+    SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
 
     SDataCol* src = &pCols->cols[j];
     if (src->colId < pColInfo->info.colId) {
@@ -1602,7 +1599,7 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
   }
 
   while (i < requiredNumOfCols) {  // the remain columns are all null data
-    SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
+    SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
     colDataAppendNNULL(pColInfo, numOfRows, num);
     i++;
   }
@@ -1684,7 +1681,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa
 
   int32_t i = 0, j = 0, k = 0;
   while (i < numOfCols && (j < numOfColsOfRow1 || k < numOfColsOfRow2)) {
-    SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
+    SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
 
     int32_t colIdOfRow1;
     if (j >= numOfColsOfRow1) {
@@ -1815,7 +1812,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa
 
   if (mergeOption == 1) {
     while (i < numOfCols) {  // the remain columns are all null data
-      SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
+      SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
       colDataAppend(pColInfo, *curRow, NULL, true);
       ++i;
     }
@@ -1871,7 +1868,7 @@ static void doCheckGeneratedBlockRange(STsdbReadHandle* pTsdbReadHandle) {
       assert(cur->win.skey >= pTsdbReadHandle->window.ekey && cur->win.ekey <= pTsdbReadHandle->window.skey);
     }
 
-    SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, 0);
+    SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, 0);
     assert(cur->win.skey == ((TSKEY*)pColInfoData->pData)[0] &&
            cur->win.ekey == ((TSKEY*)pColInfoData->pData)[cur->rows - 1]);
   } else {
@@ -2781,7 +2778,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
                                  STsdbReadHandle* pTsdbReadHandle) {
   int       numOfRows = 0;
   int       curRows = 0;
-  int32_t   numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns);
+  int32_t   numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pResBlock->pDataBlock);
   STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb);
   win->skey = TSKEY_INITIAL_VAL;
 
@@ -2911,7 +2908,7 @@ static bool loadBlockOfActiveTable(STsdbReadHandle* pTsdbReadHandle) {
     if (exists) {
       tsdbRetrieveDataBlock((tsdbReaderT*)pTsdbReadHandle, NULL);
       if (pTsdbReadHandle->currentLoadExternalRows && pTsdbReadHandle->window.skey == pTsdbReadHandle->window.ekey) {
-        SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, 0);
+        SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, 0);
         assert(*(int64_t*)pColInfo->pData == pTsdbReadHandle->window.skey);
       }
 
@@ -3013,7 +3010,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) {
 //
 //    int32_t i = 0, j = 0;
 //    while(i < tgNumOfCols && j < numOfCols) {
-//      pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
+//      pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
 //      if (pTable->lastCols[j].colId < pColInfo->info.colId) {
 //        j++;
 //        continue;
@@ -3076,7 +3073,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) {
 //            continue;
 //          }
 //
-//          pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, n);
+//          pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, n);
 //          pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;;
 //
 //          if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
@@ -3101,7 +3098,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) {
 //
 //    // leave the real ts column as the last row, because last function only (not stable) use the last row as res
 //    if (priKey != TSKEY_INITIAL_VAL) {
-//      pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, priIdx);
+//      pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, priIdx);
 //      pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;
 //
 //      *(TSKEY *)pData = priKey;
@@ -3111,7 +3108,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) {
 //          continue;
 //        }
 //
-//        pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, n);
+//        pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, n);
 //        pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;;
 //
 //        assert (pColInfo->info.colId != PRIMARYKEY_TIMESTAMP_COL_ID);
@@ -3171,9 +3168,9 @@ static bool loadDataBlockFromTableSeq(STsdbReadHandle* pTsdbReadHandle) {
 bool tsdbNextDataBlock(tsdbReaderT pHandle) {
   STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)pHandle;
 
-  size_t numOfCols = taosArrayGetSize(pTsdbReadHandle->pColumns);
+  size_t numOfCols = taosArrayGetSize(pTsdbReadHandle->pResBlock->pDataBlock);
   for (int32_t i = 0; i < numOfCols; ++i) {
-    SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
+    SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
     colInfoDataCleanup(pColInfo, pTsdbReadHandle->outputCapacity);
   }
 
@@ -3260,7 +3257,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) {
 //  SArray* row = (type == TSDB_PREV_ROW)? pTsdbReadHandle->prev : pTsdbReadHandle->next;
 //
 //  for (int32_t i = 0; i < numOfCols; ++i) {
-//    SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pColumns, i);
+//    SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
 //
 //    SColumnInfoData colInfo = {{0}, 0};
 //    colInfo.info = pCol->info;
@@ -3290,7 +3287,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) {
 //  }
 //
 //  for (int32_t i = 0; i < cond.numOfCols; ++i) {
-//    SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, i);
+//    SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
 //    memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo));
 //  }
 //
@@ -3352,7 +3349,7 @@ bool tsdbGetExternalRow(tsdbReaderT pHandle) {
 
   int32_t numOfCols = (int32_t)QH_GET_NUM_OF_COLS(pTsdbReadHandle);
   for (int32_t i = 0; i < numOfCols; ++i) {
-    SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, i);
+    SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
     SColumnInfoData* first = taosArrayGet(pTsdbReadHandle->prev, i);
 
     memcpy(pColInfoData->pData, first->pData, pColInfoData->info.bytes);
@@ -3627,13 +3624,13 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) {
    */
   STsdbReadHandle* pHandle = (STsdbReadHandle*)pTsdbReadHandle;
   if (pHandle->cur.fid == INT32_MIN) {
-    return pHandle->pColumns;
+    return pHandle->pResBlock->pDataBlock;
   } else {
     STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[pHandle->cur.slot];
     STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo;
 
     if (pHandle->cur.mixBlock) {
-      return pHandle->pColumns;
+      return pHandle->pResBlock->pDataBlock;
     } else {
       SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlockInfo->compBlock);
       assert(pHandle->realNumOfRows <= binfo.rows);
@@ -3643,7 +3640,7 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) {
 
       if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fid == pHandle->cur.fid &&
           pBlockLoadInfo->uid == pCheckInfo->tableId) {
-        return pHandle->pColumns;
+        return pHandle->pResBlock->pDataBlock;
       } else {  // only load the file block
         SBlock* pBlock = pBlockInfo->compBlock;
         if (doLoadFileDataBlock(pHandle, pBlock, pCheckInfo, pHandle->cur.slot) != TSDB_CODE_SUCCESS) {
@@ -3651,7 +3648,7 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) {
         }
 
         int32_t numOfRows = doCopyRowsFromFileBlock(pHandle, pHandle->outputCapacity, 0, 0, pBlock->numOfRows - 1);
-        return pHandle->pColumns;
+        return pHandle->pResBlock->pDataBlock;
       }
     }
   }
@@ -3691,7 +3688,7 @@ void tsdbCleanupReadHandle(tsdbReaderT queryHandle) {
     return;
   }
 
-  pTsdbReadHandle->pColumns = doFreeColumnInfoData(pTsdbReadHandle->pColumns);
+  pTsdbReadHandle->pResBlock->pDataBlock = doFreeColumnInfoData(pTsdbReadHandle->pResBlock->pDataBlock);
 
   taosArrayDestroy(pTsdbReadHandle->suppInfo.defaultLoadColumn);
   taosMemoryFreeClear(pTsdbReadHandle->pDataBlockInfo);

source/dnode/vnode/test/tsdbSmaTest.cpp

@@ -435,7 +435,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
     SSDataBlock *pDataBlock = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock));
     EXPECT_NE(pDataBlock, nullptr);
     pDataBlock->pBlockAgg = NULL;
-    pDataBlock->info.numOfCols = tSmaNumOfCols;
+    taosArrayGetSize(pDataBlock->pDataBlock) = tSmaNumOfCols;
     pDataBlock->info.rows = tSmaNumOfRows;
     pDataBlock->info.groupId = tSmaGroupId + g;
 

source/libs/command/src/command.c

@@ -32,31 +32,19 @@ static int32_t getSchemaBytes(const SSchema* pSchema) {
 }
 
 static SSDataBlock* buildDescResultDataBlock() {
-  SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
+  SSDataBlock* pBlock = createDataBlock();
-  pBlock->info.numOfCols = DESCRIBE_RESULT_COLS;
-  pBlock->info.hasVarCol = true;
 
-  pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
+  SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_FIELD_LEN, 1);
+  blockDataAppendColInfo(pBlock, &infoData);
 
-  SColumnInfoData infoData = {0};
+  infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_TYPE_LEN, 2);
-  infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
+  blockDataAppendColInfo(pBlock, &infoData);
-  infoData.info.bytes = DESCRIBE_RESULT_FIELD_LEN;
 
-  taosArrayPush(pBlock->pDataBlock, &infoData);
+  infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, tDataTypes[TSDB_DATA_TYPE_INT].bytes, 3);
-
+  blockDataAppendColInfo(pBlock, &infoData);
-  infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
-  infoData.info.bytes = DESCRIBE_RESULT_TYPE_LEN;
-  taosArrayPush(pBlock->pDataBlock, &infoData);
-
-  infoData.info.type = TSDB_DATA_TYPE_INT;
-  infoData.info.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes;
-
-  taosArrayPush(pBlock->pDataBlock, &infoData);
-
-  infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
-  infoData.info.bytes = DESCRIBE_RESULT_NOTE_LEN;
-  taosArrayPush(pBlock->pDataBlock, &infoData);
 
+  infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_NOTE_LEN, 4);
+  blockDataAppendColInfo(pBlock, &infoData);
   return pBlock;
 }
 
@@ -127,22 +115,12 @@ static int32_t execResetQueryCache() { return catalogClearCache(); }
 
 
 static SSDataBlock* buildCreateDBResultDataBlock() {
-  SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
+  SSDataBlock* pBlock = createDataBlock();
-  pBlock->info.numOfCols = SHOW_CREATE_DB_RESULT_COLS;
+  SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_COLS, 1);
-  pBlock->info.hasVarCol = true;
+  blockDataAppendColInfo(pBlock, &infoData);
-
-  pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData));
-
-  SColumnInfoData infoData = {0};
-  infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
-  infoData.info.bytes = SHOW_CREATE_DB_RESULT_FIELD1_LEN;
-
-  taosArrayPush(pBlock->pDataBlock, &infoData);
-
-  infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
-  infoData.info.bytes = SHOW_CREATE_DB_RESULT_FIELD2_LEN;
-  taosArrayPush(pBlock->pDataBlock, &infoData);
 
+  infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_FIELD2_LEN, 2);
+  blockDataAppendColInfo(pBlock, &infoData);
   return pBlock;
 }
 
@@ -284,21 +262,13 @@ static int32_t execShowCreateDatabase(SShowCreateDatabaseStmt* pStmt, SRetrieveT
 }
 
 static SSDataBlock* buildCreateTbResultDataBlock() {
-  SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
+  SSDataBlock* pBlock = createDataBlock();
-  pBlock->info.numOfCols = SHOW_CREATE_TB_RESULT_COLS;
-  pBlock->info.hasVarCol = true;
 
-  pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData));
+  SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_TB_RESULT_FIELD1_LEN, 1);
+  blockDataAppendColInfo(pBlock, &infoData);
 
-  SColumnInfoData infoData = {0};
+  infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_TB_RESULT_FIELD2_LEN, 2);
-  infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
+  blockDataAppendColInfo(pBlock, &infoData);
-  infoData.info.bytes = SHOW_CREATE_TB_RESULT_FIELD1_LEN;
-
-  taosArrayPush(pBlock->pDataBlock, &infoData);
-
-  infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
-  infoData.info.bytes = SHOW_CREATE_TB_RESULT_FIELD2_LEN;
-  taosArrayPush(pBlock->pDataBlock, &infoData);
 
   return pBlock;
 }
@@ -554,10 +524,9 @@ static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { return TSDB_CODE_FAILED;
 
 static SSDataBlock* buildLocalVariablesResultDataBlock() {
   SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
-  pBlock->info.numOfCols = SHOW_LOCAL_VARIABLES_RESULT_COLS;
   pBlock->info.hasVarCol = true;
 
-  pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData));
+  pBlock->pDataBlock = taosArrayInit(SHOW_LOCAL_VARIABLES_RESULT_COLS, sizeof(SColumnInfoData));
 
   SColumnInfoData infoData = {0};
   infoData.info.type = TSDB_DATA_TYPE_VARCHAR;

source/libs/command/src/explain.c

@@ -1190,17 +1190,12 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
     QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
   }
 
-  SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
+  SSDataBlock *pBlock = createDataBlock();
-  SColumnInfoData infoData = {0};
+  SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, TSDB_EXPLAIN_RESULT_ROW_SIZE, 1);
-  infoData.info.type  = TSDB_DATA_TYPE_VARCHAR;
+  blockDataAppendColInfo(pBlock, &infoData);
-  infoData.info.bytes = TSDB_EXPLAIN_RESULT_ROW_SIZE;
+  blockDataEnsureCapacity(pBlock, rowNum);
-
-  pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
-  taosArrayPush(pBlock->pDataBlock, &infoData);
 
   SColumnInfoData* pInfoData = taosArrayGet(pBlock->pDataBlock, 0);
-  pInfoData->hasNull = false;
-  colInfoDataEnsureCapacity(pInfoData, 0, rowNum);
 
   char buf[1024] = {0};
   for (int32_t i = 0; i < rowNum; ++i) {
@@ -1210,9 +1205,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
     colDataAppend(pInfoData, i, buf, false);
   }
 
-  pBlock->info.numOfCols = 1;
   pBlock->info.rows = rowNum;
-  pBlock->info.hasVarCol = true;
 
   int32_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
 
@@ -1226,7 +1219,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
   rsp->numOfRows = htonl(rowNum);
 
   int32_t len = 0;
-  blockCompressEncode(pBlock, rsp->data, &len, pBlock->info.numOfCols, 0);
+  blockCompressEncode(pBlock, rsp->data, &len, taosArrayGetSize(pBlock->pDataBlock), 0);
   ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
 
   rsp->compLen = htonl(len);

source/libs/executor/inc/executorimpl.h

@@ -343,7 +343,7 @@ typedef struct SStreamBlockScanInfo {
   SReadHandle     readHandle;
   uint64_t        tableUid;         // queried super table uid
   EStreamScanMode scanMode;
-  SOperatorInfo* pOperatorDumy;
+  SOperatorInfo* pSnapshotReadOp;
   SInterval      interval;     // if the upstream is an interval operator, the interval info is also kept here.
   SArray*        childIds;
   SessionWindowSupporter sessionSup;

source/libs/executor/src/dataDeleter.c

@@ -75,7 +75,7 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp
   SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData;
   pEntry->compressed = 0;
   pEntry->numOfRows = pInput->pData->info.rows;
-  pEntry->numOfCols = pInput->pData->info.numOfCols;
+  pEntry->numOfCols = taosArrayGetSize(pInput->pData->pDataBlock);
   pEntry->dataLen = sizeof(SDeleterRes);
 
   ASSERT(1 == pEntry->numOfRows);
@@ -86,7 +86,7 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp
   SColumnInfoData* pColRes = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 0);
 
   SDeleterRes* pRes = (SDeleterRes*)pEntry->data;
-  pRes->uid = pHandle->pDeleter->tableId;
+  pRes->suid = pHandle->pParam->suid;
   pRes->uidList = pHandle->pParam->pUidList;
   pRes->skey = pHandle->pDeleter->deleteTimeRange.skey;
   pRes->ekey = pHandle->pDeleter->deleteTimeRange.ekey;

source/libs/executor/src/executil.c

@@ -183,34 +183,24 @@ SArray* createSortInfo(SNodeList* pNodeList) {
 SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) {
   int32_t numOfCols = LIST_LENGTH(pNode->pSlots);
 
-  SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
+  SSDataBlock* pBlock = createDataBlock();
-  pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
 
   pBlock->info.blockId = pNode->dataBlockId;
-  pBlock->info.rowSize = pNode->totalRowSize;  // todo ??
   pBlock->info.type = STREAM_INVALID;
 
   for (int32_t i = 0; i < numOfCols; ++i) {
-    SColumnInfoData idata = {{0}};
     SSlotDescNode*  pDescNode = (SSlotDescNode*)nodesListGetNode(pNode->pSlots, i);
 //    if (!pDescNode->output) {  // todo disable it temporarily
 //      continue;
 //    }
 
-    idata.info.type = pDescNode->dataType.type;
+    SColumnInfoData idata = createColumnInfoData(pDescNode->dataType.type, pDescNode->dataType.bytes, pDescNode->slotId);
-    idata.info.bytes = pDescNode->dataType.bytes;
     idata.info.scale = pDescNode->dataType.scale;
-    idata.info.slotId = pDescNode->slotId;
     idata.info.precision = pDescNode->dataType.precision;
 
-    if (IS_VAR_DATA_TYPE(idata.info.type)) {
+    blockDataAppendColInfo(pBlock, &idata);
-      pBlock->info.hasVarCol = true;
-    }
-
-    taosArrayPush(pBlock->pDataBlock, &idata);
   }
 
-  pBlock->info.numOfCols = taosArrayGetSize(pBlock->pDataBlock);
   return pBlock;
 }
 
@@ -301,6 +291,8 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo
 
   uint64_t tableUid = pScanNode->uid;
 
+  pListInfo->suid = pScanNode->suid;
+  
   SNode* pTagCond = (SNode*)pListInfo->pTagCond;
   SNode* pTagIndexCond = (SNode*)pListInfo->pTagIndexCond;
   if (pScanNode->tableType == TSDB_SUPER_TABLE) {
@@ -702,7 +694,7 @@ void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray
 
     if (p->info.colId == pmInfo->colId) {
       SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, pmInfo->targetSlotId);
-      colDataAssign(pDst, p, pBlock->info.rows);
+      colDataAssign(pDst, p, pBlock->info.rows, &pBlock->info);
       i++;
       j++;
     } else if (p->info.colId < pmInfo->colId) {

source/libs/executor/src/executor.c

@@ -67,6 +67,9 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
         taosArrayAddAll(p->pDataBlock, pDataBlock->pDataBlock);
         taosArrayPush(pInfo->pBlockLists, &p);
       }
+    } else if (type == STREAM_DATA_TYPE_FROM_SNAPSHOT) {
+      // do nothing
+      ASSERT(pInfo->blockType == STREAM_DATA_TYPE_FROM_SNAPSHOT);
     } else {
       ASSERT(0);
     }
@@ -75,6 +78,14 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
   }
 }
 
+int32_t qStreamScanSnapshot(qTaskInfo_t tinfo) {
+  if (tinfo == NULL) {
+    return TSDB_CODE_QRY_APP_ERROR;
+  }
+  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
+  return doSetStreamBlock(pTaskInfo->pRoot, NULL, 0, STREAM_DATA_TYPE_FROM_SNAPSHOT, 0, NULL);
+}
+
 int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type, bool assignUid) {
   return qSetMultiStreamInput(tinfo, input, 1, type, assignUid);
 }
@@ -106,14 +117,6 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) {
     return NULL;
   }
 
-  // print those info into log
-#if 0
-  pMsg->sId = pMsg->sId;
-  pMsg->queryId = pMsg->queryId;
-  pMsg->taskId = pMsg->taskId;
-  pMsg->contentLen = pMsg->contentLen;
-#endif
-
   /*qDebugL("stream task string %s", (const char*)msg);*/
 
   struct SSubplan* plan = NULL;

source/libs/executor/src/executorimpl.c

@@ -356,7 +356,7 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow
   pColData->info.type = TSDB_DATA_TYPE_TIMESTAMP;
   pColData->info.bytes = sizeof(int64_t);
 
-  colInfoDataEnsureCapacity(pColData, 0, 5);
+  colInfoDataEnsureCapacity(pColData, 5);
   colDataAppendInt64(pColData, 0, &pQueryWindow->skey);
   colDataAppendInt64(pColData, 1, &pQueryWindow->ekey);
 
@@ -458,7 +458,7 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc
     pColInfo = pInput->pData[paramIndex];
   }
 
-  colInfoDataEnsureCapacity(pColInfo, 0, numOfRows);
+  colInfoDataEnsureCapacity(pColInfo, numOfRows);
 
   int8_t type = pFuncParam->param.nType;
   if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_UBIGINT) {
@@ -579,7 +579,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc
         colDataMergeCol(pColInfoData, pResult->info.rows, &pResult->info.capacity, pfCtx->input.pData[0],
                         pfCtx->input.numOfRows);
       } else {
-        colDataAssign(pColInfoData, pfCtx->input.pData[0], pfCtx->input.numOfRows);
+        colDataAssign(pColInfoData, pfCtx->input.pData[0], pfCtx->input.numOfRows, &pResult->info);
       }
 
       numOfRows = pfCtx->input.numOfRows;
@@ -1186,7 +1186,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc
 //    }
 
 //    if (pQueryAttr->pFilters != NULL) {
-//      filterSetColFieldData(pQueryAttr->pFilters, pBlock->info.numOfCols, pBlock->pDataBlock);
+//      filterSetColFieldData(pQueryAttr->pFilters, taosArrayGetSize(pBlock->pDataBlock), pBlock->pDataBlock);
 //    }
 
 //    if (pQueryAttr->pFilters != NULL || pRuntimeEnv->pTsBuf != NULL) {
@@ -1321,7 +1321,8 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) {
   // todo move to the initialization function
   int32_t code = filterInitFromNode((SNode*)pFilterNode, &filter, 0);
 
-  SFilterColumnParam param1 = {.numOfCols = pBlock->info.numOfCols, .pDataBlock = pBlock->pDataBlock};
+  size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
+  SFilterColumnParam param1 = {.numOfCols = numOfCols, .pDataBlock = pBlock->pDataBlock};
   code = filterSetDataFromSlotId(filter, &param1);
 
   int8_t* rowRes = NULL;
@@ -1343,7 +1344,8 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR
     int32_t totalRows = pBlock->info.rows;
     SSDataBlock* px = createOneDataBlock(pBlock, true);
 
-    for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
+    size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
+    for (int32_t i = 0; i < numOfCols; ++i) {
       SColumnInfoData* pSrc = taosArrayGet(px->pDataBlock, i);
       SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i);
       // it is a reserved column for scalar function, and no data in this column yet.
@@ -2045,8 +2047,9 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo
                                      int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total,
                                      SArray* pColList) {
   if (pColList == NULL) {  // data from other sources
+    blockDataCleanup(pRes);
+//    blockDataEnsureCapacity(pRes, numOfRows);
     blockCompressDecode(pRes, numOfOutput, numOfRows, pData);
-    pRes->info.rows = numOfRows;
   } else {  // extract data according to pColList
     ASSERT(numOfOutput == taosArrayGetSize(pColList));
     char* pStart = pData;
@@ -2064,32 +2067,19 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo
       pStart += sizeof(SSysTableSchema);
     }
 
-    SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
+    SSDataBlock* pBlock = createDataBlock();
-    pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
-    pBlock->info.numOfCols = numOfCols;
-
     for (int32_t i = 0; i < numOfCols; ++i) {
-      SColumnInfoData idata = {0};
+      SColumnInfoData idata = createColumnInfoData(pSchema[i].type, pSchema[i].bytes, pSchema[i].colId);
-
+      blockDataAppendColInfo(pBlock, &idata);
-      idata.info.type = pSchema[i].type;
-      idata.info.bytes = pSchema[i].bytes;
-      idata.info.colId = pSchema[i].colId;
-      idata.hasNull = true;
-
-      taosArrayPush(pBlock->pDataBlock, &idata);
-      if (IS_VAR_DATA_TYPE(idata.info.type)) {
-        pBlock->info.hasVarCol = true;
-      }
     }
 
     blockCompressDecode(pBlock, numOfCols, numOfRows, pStart);
+    blockDataEnsureCapacity(pRes, numOfRows);
 
     // data from mnode
     pRes->info.rows = numOfRows;
     relocateColumnData(pRes, pColList, pBlock->pDataBlock, false);
-    taosArrayDestroy(pBlock->pDataBlock);
+    blockDataDestroy(pBlock);
-    taosMemoryFree(pBlock);
-    //    blockDataDestroy(pBlock);
   }
 
   // todo move this to time window aggregator, since the primary timestamp may not be known by exchange operator.
@@ -2412,15 +2402,15 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode
 
   tsem_init(&pInfo->ready, 0, 0);
 
-  pInfo->seqLoadData = false;
+  pInfo->seqLoadData  = false;
   pInfo->pTransporter = pTransporter;
-  pInfo->pResult = createResDataBlock(pExNode->node.pOutputDataBlockDesc);
+  pInfo->pResult      = createResDataBlock(pExNode->node.pOutputDataBlockDesc);
-  pOperator->name = "ExchangeOperator";
+  pOperator->name     = "ExchangeOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE;
   pOperator->blocking = false;
-  pOperator->status = OP_NOT_OPENED;
+  pOperator->status   = OP_NOT_OPENED;
-  pOperator->info = pInfo;
+  pOperator->info     = pInfo;
-  pOperator->exprSupp.numOfExprs = pInfo->pResult->info.numOfCols;
+  pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pResult->pDataBlock);
   pOperator->pTaskInfo = pTaskInfo;
 
   pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL,
@@ -2542,9 +2532,6 @@ static void doMergeImpl(SOperatorInfo* pOperator, int32_t numOfExpr, SSDataBlock
   SSortedMergeOperatorInfo* pInfo = pOperator->info;
 
   SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx;
-  for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
-    //    pCtx[i].size = 1;
-  }
 
   for (int32_t i = 0; i < pBlock->info.rows; ++i) {
     if (!pInfo->hasGroupVal) {
@@ -2653,7 +2640,7 @@ SSDataBlock* getSortedMergeBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlo
 
       SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
       SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId);
-      colDataAssign(pDst, pSrc, p->info.rows);
+      colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info);
     }
 
     pDataBlock->info.rows = p->info.rows;
@@ -4085,7 +4072,6 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) {
       return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, (SExchangePhysiNode*)pPhyNode, pTaskInfo);
     } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) {
-      SScanPhysiNode*      pScanPhyNode = (SScanPhysiNode*)pPhyNode;  // simple child table.
       STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
       STimeWindowAggSupp   twSup = {
             .waterMark = pTableScanNode->watermark,
@@ -4464,6 +4450,7 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pT
         return TSDB_CODE_OUT_OF_MEMORY;
       }
       int32_t tbNum = taosArrayGetSize(pTask->tableqinfoList.pTableList);
+      pDeleterParam->suid = pTask->tableqinfoList.suid;
       pDeleterParam->pUidList = taosArrayInit(tbNum, sizeof(uint64_t));
       if (NULL == pDeleterParam->pUidList) {
         taosMemoryFree(pDeleterParam);

source/libs/executor/src/groupoperator.c

@@ -559,8 +559,8 @@ uint64_t calcGroupId(char* pData, int32_t len) {
 }
 
 int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity) {
-  size_t numOfCols = pBlock->info.numOfCols;
+  size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
-  int32_t* offset = taosMemoryCalloc(pBlock->info.numOfCols, sizeof(int32_t));
+  int32_t* offset = taosMemoryCalloc(numOfCols, sizeof(int32_t));
 
   offset[0] = sizeof(int32_t) + sizeof(uint64_t);  // the number of rows in current page, ref to SSDataBlock paged serialization format
 

source/libs/executor/src/scanoperator.c

@@ -203,7 +203,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
     pCost->skipBlocks += 1;
 
     // clear all data in pBlock that are set when handing the previous block
-    for (int32_t i = 0; i < pBlockInfo->numOfCols; ++i) {
+    for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
       SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i);
       pcol->pData = NULL;
     }
@@ -217,7 +217,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
     tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg);
 
     if (allColumnsHaveAgg == true) {
-      int32_t numOfCols = pBlock->info.numOfCols;
+      int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
 
       // todo create this buffer during creating operator
       if (pBlock->pBlockAgg == NULL) {
@@ -316,7 +316,7 @@ void addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_
 
     SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotId);
 
-    colInfoDataEnsureCapacity(pColInfoData, 0, pBlock->info.rows);
+    colInfoDataEnsureCapacity(pColInfoData, pBlock->info.rows);
     colInfoDataCleanup(pColInfoData, pBlock->info.rows);
 
     int32_t functionId = pExpr->pExpr->_function.functionId;
@@ -337,7 +337,8 @@ void addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_
       }
 
       for (int32_t i = 0; i < pBlock->info.rows; ++i) {
-        colDataAppend(pColInfoData, i, data, (data == NULL) || (pColInfoData->info.type == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(data)));
+        colDataAppend(pColInfoData, i, data,
+                      (data == NULL) || (pColInfoData->info.type == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(data)));
       }
 
       if (data && (pColInfoData->info.type != TSDB_DATA_TYPE_JSON) && p != NULL &&
@@ -354,10 +355,8 @@ void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* p
   struct SScalarFuncExecFuncs fpSet = {0};
   fmGetScalarFuncExecFuncs(functionId, &fpSet);
 
-  SColumnInfoData infoData = {0};
+  SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_BIGINT, sizeof(uint64_t), 1);
-  infoData.info.type = TSDB_DATA_TYPE_BIGINT;
+  colInfoDataEnsureCapacity(&infoData, 1);
-  infoData.info.bytes = sizeof(uint64_t);
-  colInfoDataEnsureCapacity(&infoData, 0, 1);
 
   colDataAppendInt64(&infoData, 0, (int64_t*)&pBlock->info.uid);
   SScalarParam srcParam = {.numOfRows = pBlock->info.rows, .param = pMeta, .columnData = &infoData};
@@ -819,7 +818,7 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) {
   if (!needRead) {
     return false;
   }
-  STableScanInfo* pTableScanInfo = pInfo->pOperatorDumy->info;
+  STableScanInfo* pTableScanInfo = pInfo->pSnapshotReadOp->info;
   pTableScanInfo->cond.twindows[0] = win;
   pTableScanInfo->curTWinIdx = 0;
   tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
@@ -828,7 +827,7 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) {
 }
 
 static void copyOneRow(SSDataBlock* dest, SSDataBlock* source, int32_t sourceRowId) {
-  for (int32_t j = 0; j < source->info.numOfCols; j++) {
+  for (int32_t j = 0; j < taosArrayGetSize(source->pDataBlock); j++) {
     SColumnInfoData* pDestCol = (SColumnInfoData*)taosArrayGet(dest->pDataBlock, j);
     SColumnInfoData* pSourceCol = (SColumnInfoData*)taosArrayGet(source->pDataBlock, j);
     if (colDataIsNull_s(pSourceCol, sourceRowId)) {
@@ -864,11 +863,11 @@ static uint64_t getGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_
 static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo) {
   while (1) {
     SSDataBlock* pResult = NULL;
-    pResult = doTableScan(pInfo->pOperatorDumy);
+    pResult = doTableScan(pInfo->pSnapshotReadOp);
     if (pResult == NULL) {
       if (prepareDataScan(pInfo)) {
         // scan next window data
-        pResult = doTableScan(pInfo->pOperatorDumy);
+        pResult = doTableScan(pInfo->pSnapshotReadOp);
       }
     }
     if (!pResult) {
@@ -900,14 +899,13 @@ static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDa
     SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pUpdateBlock->pDataBlock, pInfo->primaryTsIndex);
     ASSERT(pCol->info.type == TSDB_DATA_TYPE_TIMESTAMP);
     blockDataEnsureCapacity(pUpdateBlock, size);
-    ASSERT(pBlock->info.numOfCols == pUpdateBlock->info.numOfCols);
 
     int32_t rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, pInfo->tsArrayIndex);
-    pInfo->groupId = getGroupId(pInfo->pOperatorDumy, pBlock, rowId);
+    pInfo->groupId = getGroupId(pInfo->pSnapshotReadOp, pBlock, rowId);
     int32_t i = 0;
     for (; i < size; i++) {
       rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, i + pInfo->tsArrayIndex);
-      uint64_t id = getGroupId(pInfo->pOperatorDumy, pBlock, rowId);
+      uint64_t id = getGroupId(pInfo->pSnapshotReadOp, pBlock, rowId);
       if (pInfo->groupId != id) {
         break;
       }
@@ -971,7 +969,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
     SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current);
     blockDataUpdateTsWindow(pBlock, 0);
     return pBlock;
-  } else {
+  } else if (pInfo->blockType == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
     if (pInfo->scanMode == STREAM_SCAN_FROM_RES) {
       blockDataDestroy(pInfo->pUpdateRes);
       pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
@@ -1011,13 +1009,13 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
     blockDataCleanup(pInfo->pRes);
 
     while (tqNextDataBlock(pInfo->streamBlockReader)) {
-      SArray*  pCols = NULL;
+      SSDataBlock block = {0};
       uint64_t groupId = 0;
       uint64_t uid = 0;
       int32_t  numOfRows = 0;
-      int16_t  outputCol = 0;
 
-      int32_t code = tqRetrieveDataBlock(&pCols, pInfo->streamBlockReader, &groupId, &uid, &numOfRows, &outputCol);
+      // todo refactor
+      int32_t code = tqRetrieveDataBlock(&block, pInfo->streamBlockReader, &groupId, &uid, &numOfRows);
 
       if (code != TSDB_CODE_SUCCESS || numOfRows == 0) {
         pTaskInfo->code = code;
@@ -1028,6 +1026,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
       pInfo->pRes->info.rows = numOfRows;
       pInfo->pRes->info.uid = uid;
       pInfo->pRes->info.type = STREAM_NORMAL;
+      pInfo->pRes->info.capacity = numOfRows;
 
       // for generating rollup SMA result, each time is an independent time serie.
       // TODO temporarily used, when the statement of "partition by tbname" is ready, remove this
@@ -1042,6 +1041,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
         pInfo->pRes->info.groupId = *groupIdPre;
       }
 
+      // todo extract method
       for (int32_t i = 0; i < taosArrayGetSize(pInfo->pColMatchInfo); ++i) {
         SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i);
         if (!pColMatchInfo->output) {
@@ -1049,8 +1049,8 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
         }
 
         bool colExists = false;
-        for (int32_t j = 0; j < taosArrayGetSize(pCols); ++j) {
+        for (int32_t j = 0; j < blockDataGetNumOfCols(&block); ++j) {
-          SColumnInfoData* pResCol = taosArrayGet(pCols, j);
+          SColumnInfoData* pResCol = bdGetColumnInfoData(&block, j);
           if (pResCol->info.colId == pColMatchInfo->colId) {
             taosArraySet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId, pResCol);
             colExists = true;
@@ -1061,7 +1061,6 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
         // the required column does not exists in submit block, let's set it to be all null value
         if (!colExists) {
           SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId);
-          colInfoDataEnsureCapacity(pDst, 0, pBlockInfo->rows);
           colDataAppendNNULL(pDst, 0, pBlockInfo->rows);
         }
       }
@@ -1105,6 +1104,15 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
     }
 
     return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes;
+  } else if (pInfo->blockType == STREAM_DATA_TYPE_FROM_SNAPSHOT) {
+    SSDataBlock* pResult = doTableScan(pInfo->pSnapshotReadOp);
+    if (pResult) {
+      return pResult->info.rows > 0 ? pResult : NULL;
+    }
+    return NULL;
+  } else {
+    ASSERT(0);
+    return NULL;
   }
 }
 
@@ -1169,7 +1177,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle,
     } else {
       pInfo->pUpdateInfo = NULL;
     }
-    pInfo->pOperatorDumy = pTableScanDummy;
+    pInfo->pSnapshotReadOp = pTableScanDummy;
     pInfo->interval = pSTInfo->interval;
 
     pInfo->readHandle = *pHandle;
@@ -1205,7 +1213,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle,
   pOperator->blocking = false;
   pOperator->status = OP_NOT_OPENED;
   pOperator->info = pInfo;
-  pOperator->exprSupp.numOfExprs = pInfo->pRes->info.numOfCols;
+  pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock);
   pOperator->pTaskInfo = pTaskInfo;
 
   pOperator->fpSet =
@@ -1333,7 +1341,7 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) {
     SColumnInfoData* pSrc = taosArrayGet(pInfo->pRes->pDataBlock, i);
 
     if (keep) {
-      colDataAssign(pDest, pSrc, pInfo->pRes->info.rows);
+      colDataAssign(pDest, pSrc, pInfo->pRes->info.rows, &px->info);
       numOfRow = pInfo->pRes->info.rows;
     } else if (NULL != rowRes) {
       numOfRow = 0;
@@ -1363,8 +1371,6 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) {
 }
 
 static SSDataBlock* buildSysTableMetaBlock() {
-  SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
-
   size_t               size = 0;
   const SSysTableMeta* pMeta = NULL;
   getInfosDbMeta(&pMeta, &size);
@@ -1377,19 +1383,12 @@ static SSDataBlock* buildSysTableMetaBlock() {
     }
   }
 
-  pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData));
+  SSDataBlock* pBlock = createDataBlock();
-
   for (int32_t i = 0; i < pMeta[index].colNum; ++i) {
-    SColumnInfoData colInfoData = {0};
+    SColumnInfoData colInfoData = createColumnInfoData(pMeta[index].schema[i].type, pMeta[index].schema[i].bytes, i + 1);
-    colInfoData.info.colId = i + 1;
+    blockDataAppendColInfo(pBlock, &colInfoData);
-    colInfoData.info.type = pMeta[index].schema[i].type;
-    colInfoData.info.bytes = pMeta[index].schema[i].bytes;
-    taosArrayPush(pBlock->pDataBlock, &colInfoData);
   }
 
-  pBlock->info.numOfCols = pMeta[index].colNum;
-  pBlock->info.hasVarCol = true;
-
   return pBlock;
 }
 
@@ -1420,7 +1419,6 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
       }
 
       blockDataCleanup(pInfo->pRes);
-
       int32_t numOfRows = 0;
 
       const char* db = NULL;
@@ -1590,7 +1588,8 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
         return NULL;
       }
 
-      int32_t msgType = (strcasecmp(name, TSDB_INS_TABLE_DNODE_VARIABLES) == 0) ? TDMT_DND_SYSTABLE_RETRIEVE : TDMT_MND_SYSTABLE_RETRIEVE;
+      int32_t msgType = (strcasecmp(name, TSDB_INS_TABLE_DNODE_VARIABLES) == 0) ? TDMT_DND_SYSTABLE_RETRIEVE
+                                                                                : TDMT_MND_SYSTABLE_RETRIEVE;
 
       pMsgSendInfo->param = pOperator;
       pMsgSendInfo->msgInfo.pData = buf1;
@@ -1615,7 +1614,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
 
       if (pRsp->numOfRows == 0 || pRsp->completed) {
         pOperator->status = OP_EXEC_DONE;
-        qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64 " ", GET_TASKID(pTaskInfo),
+        qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64, GET_TASKID(pTaskInfo),
                pRsp->numOfRows, pInfo->loadInfo.totalRows);
 
         if (pRsp->numOfRows == 0) {
@@ -1742,7 +1741,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan
   pOperator->blocking = false;
   pOperator->status = OP_NOT_OPENED;
   pOperator->info = pInfo;
-  pOperator->exprSupp.numOfExprs = pResBlock->info.numOfCols;
+  pOperator->exprSupp.numOfExprs = taosArrayGetSize(pResBlock->pDataBlock);
   pOperator->pTaskInfo = pTaskInfo;
 
   pOperator->fpSet =
@@ -1876,7 +1875,8 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
         } else {
           data = (char*)p;
         }
-        colDataAppend(pDst, count, data, (data == NULL) || (pDst->info.type == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(data)));
+        colDataAppend(pDst, count, data,
+                      (data == NULL) || (pDst->info.type == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(data)));
 
         if (pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL && IS_VAR_DATA_TYPE(((const STagVal*)p)->type) &&
             data != NULL) {
@@ -2025,6 +2025,7 @@ int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle
   return TSDB_CODE_SUCCESS;
 }
 
+// todo refactor
 static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo,
                                          int32_t readerIdx, SSDataBlock* pBlock, uint32_t* status) {
   SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
@@ -2055,7 +2056,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc
     pCost->skipBlocks += 1;
 
     // clear all data in pBlock that are set when handing the previous block
-    for (int32_t i = 0; i < pBlockInfo->numOfCols; ++i) {
+    for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
       SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i);
       pcol->pData = NULL;
     }
@@ -2070,7 +2071,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc
     tsdbRetrieveDataBlockStatisInfo(reader, &pColAgg, &allColumnsHaveAgg);
 
     if (allColumnsHaveAgg == true) {
-      int32_t numOfCols = pBlock->info.numOfCols;
+      int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
 
       // todo create this buffer during creating operator
       if (pBlock->pBlockAgg == NULL) {
@@ -2165,7 +2166,13 @@ static SSDataBlock* getTableDataBlock(void* param) {
       continue;
     }
 
-    tsdbRetrieveDataBlockInfo(reader, &pBlock->info);
+    blockDataCleanup(pBlock);
+    SDataBlockInfo binfo = pBlock->info;
+    tsdbRetrieveDataBlockInfo(reader, &binfo);
+
+    binfo.capacity = binfo.rows;
+    blockDataEnsureCapacity(pBlock, binfo.capacity);
+    pBlock->info = binfo;
 
     uint32_t status = 0;
     int32_t  code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, readerIdx, pBlock, &status);

source/libs/executor/src/sortoperator.c

@@ -80,7 +80,7 @@ _error:
 }
 
 void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
-  for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
+  for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
     SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
     bool             isNull = tsortIsNullVal(pTupleHandle, i);
     if (isNull) {
@@ -120,6 +120,9 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i
   }
 
   if (p->info.rows > 0) {
+    blockDataEnsureCapacity(pDataBlock, capacity);
+
+    // todo extract function to handle this
     int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
     for (int32_t i = 0; i < numOfCols; ++i) {
       SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i);
@@ -127,11 +130,10 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i
 
       SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
       SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId);
-      colDataAssign(pDst, pSrc, p->info.rows);
+      colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info);
     }
 
     pDataBlock->info.rows = p->info.rows;
-    pDataBlock->info.capacity = p->info.rows;
   }
 
   blockDataDestroy(p);
@@ -257,6 +259,7 @@ typedef struct SGroupSortOperatorInfo {
 SSDataBlock* getGroupSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity,
                                      SArray* pColMatchInfo, SGroupSortOperatorInfo* pInfo) {
   blockDataCleanup(pDataBlock);
+  blockDataEnsureCapacity(pDataBlock, capacity);
 
   SSDataBlock* p = tsortGetSortedDataBlock(pHandle);
   if (p == NULL) {
@@ -285,7 +288,7 @@ SSDataBlock* getGroupSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlo
 
       SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
       SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId);
-      colDataAssign(pDst, pSrc, p->info.rows);
+      colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info);
     }
 
     pDataBlock->info.rows = p->info.rows;
@@ -387,6 +390,10 @@ SSDataBlock* doGroupSort(SOperatorInfo* pOperator) {
     pInfo->hasGroupId = true;
 
     pInfo->prefetchedSortInput = pOperator->pDownstream[0]->fpSet.getNextFn(pOperator->pDownstream[0]);
+    if (pInfo->prefetchedSortInput == NULL) {
+      doSetOperatorCompleted(pOperator);
+      return NULL;
+    }
     pInfo->currGroupId = pInfo->prefetchedSortInput->info.groupId;
     pInfo->childOpStatus = CHILD_OP_NEW_GROUP;
     beginSortGroup(pOperator);
@@ -460,7 +467,7 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort
   pInfo->pColMatchInfo = pColMatchColInfo;
   pOperator->name = "GroupSortOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT;
-  pOperator->blocking = true;
+  pOperator->blocking = false;
   pOperator->status = OP_NOT_OPENED;
   pOperator->info = pInfo;
   pOperator->exprSupp.pExprInfo = pExprInfo;
@@ -564,7 +571,8 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
     }
   }
 
-  if (p->info.rows > 0) {
+  if (p->info.rows > 0) {// todo extract method
+    blockDataEnsureCapacity(pDataBlock, p->info.rows);
     int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
     for (int32_t i = 0; i < numOfCols; ++i) {
       SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i);
@@ -572,11 +580,10 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
 
       SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
       SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId);
-      colDataAssign(pDst, pSrc, p->info.rows);
+      colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info);
     }
 
     pDataBlock->info.rows = p->info.rows;
-    pDataBlock->info.capacity = p->info.rows;
     pDataBlock->info.groupId = pInfo->groupId;
   }
 

source/libs/executor/src/tfill.c

@@ -50,7 +50,7 @@ static void setTagsValue(SFillInfo* pFillInfo, void** data, int32_t genRows) {
 
 static void setNullRow(SSDataBlock* pBlock, int32_t numOfCol, int32_t rowIndex) {
   // the first are always the timestamp column, so start from the second column.
-  for (int32_t i = 1; i < pBlock->info.numOfCols; ++i) {
+  for (int32_t i = 1; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
     SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, i);
     colDataAppendNULL(p, rowIndex);
   }

source/libs/executor/src/timewindowoperator.c

@@ -1814,7 +1814,7 @@ static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
-  int32_t numOfCols = pBlock->info.numOfCols;
+  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
 
@@ -2231,23 +2231,23 @@ static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo)
 }
 
 static void clearUpdateDataBlock(SSDataBlock* pBlock) {
-  if (pBlock->info.rows <= 0) {
-    return;
-  }
   blockDataCleanup(pBlock);
 }
 
 void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) {
-  ASSERT(pDest->info.capacity >= pSource->info.rows);
+  // ASSERT(pDest->info.capacity >= pSource->info.rows);
+  blockDataEnsureCapacity(pDest, pSource->info.rows);
   clearUpdateDataBlock(pDest);
   SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0);
   SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex);
+
   // copy timestamp column
-  colDataAssign(pDestCol, pSourceCol, pSource->info.rows);
+  colDataAssign(pDestCol, pSourceCol, pSource->info.rows, &pDest->info);
-  for (int32_t i = 1; i < pDest->info.numOfCols; i++) {
+  for (int32_t i = 1; i < taosArrayGetSize(pDest->pDataBlock); i++) {
     SColumnInfoData* pCol = taosArrayGet(pDest->pDataBlock, i);
     colDataAppendNNULL(pCol, 0, pSource->info.rows);
   }
+
   pDest->info.rows = pSource->info.rows;
   pDest->info.groupId = pSource->info.groupId;
   pDest->info.type = pSource->info.type;
@@ -2917,7 +2917,7 @@ void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** It
   while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) {
     SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0);
     colDataAppend(pColInfoData, pBlock->info.rows, *Ite, false);
-    for (int32_t i = 1; i < pBlock->info.numOfCols; i++) {
+    for (int32_t i = 1; i < taosArrayGetSize(pBlock->pDataBlock); i++) {
       pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
       colDataAppendNULL(pColInfoData, pBlock->info.rows);
     }
@@ -4117,6 +4117,8 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprI
     goto _error;
   }
 
+  miaInfo->groupIntervalHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_NO_LOCK);
+
   SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo;
 
   iaInfo->win = pTaskInfo->window;

source/libs/executor/src/tsort.c

@@ -147,7 +147,7 @@ static int32_t doAddNewExternalMemSource(SDiskbasedBuf *pBuf, SArray* pAllSource
   int32_t rowSize = blockDataGetSerialRowSize(pSource->src.pBlock);
 
   // The value of numOfRows must be greater than 0, which is guaranteed by the previous memory allocation
-  int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(pBlock->info.numOfCols))/rowSize;
+  int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)))/rowSize;
   ASSERT(numOfRows > 0);
   return blockDataEnsureCapacity(pSource->src.pBlock, numOfRows);
 }
@@ -178,7 +178,7 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) {
       return terrno;
     }
 
-    int32_t size = blockDataGetSize(p) + sizeof(int32_t)  + p->info.numOfCols * sizeof(int32_t);
+    int32_t size = blockDataGetSize(p) + sizeof(int32_t)  + taosArrayGetSize(p->pDataBlock) * sizeof(int32_t);
     assert(size <= getBufPageSize(pHandle->pBuf));
 
     blockDataToBuf(pPage, p);
@@ -252,7 +252,7 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int
 }
 
 static void appendOneRowToDataBlock(SSDataBlock *pBlock, const SSDataBlock* pSource, int32_t* rowIndex) {
-  for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
+  for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
     SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
 
     SColumnInfoData* pSrcColInfo = taosArrayGet(pSource->pDataBlock, i);
@@ -500,7 +500,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) {
           return terrno;
         }
 
-        int32_t size = blockDataGetSize(pDataBlock) + sizeof(int32_t) + pDataBlock->info.numOfCols * sizeof(int32_t);
+        int32_t size = blockDataGetSize(pDataBlock) + sizeof(int32_t) + taosArrayGetSize(pDataBlock->pDataBlock) * sizeof(int32_t);
         assert(size <= getBufPageSize(pHandle->pBuf));
 
         blockDataToBuf(pPage, pDataBlock);

source/libs/executor/test/executorTests.cpp

@@ -62,18 +62,11 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator) {
   }
 
   if (pInfo->pBlock == NULL) {
-    pInfo->pBlock = static_cast<SSDataBlock*>(taosMemoryCalloc(1, sizeof(SSDataBlock)));
+    pInfo->pBlock = createDataBlock();
 
-    pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
+    SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1);
-
+    blockDataAppendColInfo(pInfo->pBlock, &colInfo);
-    SColumnInfoData colInfo = {0};
+    blockDataEnsureCapacity(pInfo->pBlock, pInfo->numOfRowsPerPage);
-    colInfo.info.type = TSDB_DATA_TYPE_INT;
-    colInfo.info.bytes = sizeof(int32_t);
-    colInfo.info.colId = 1;
-    colInfo.pData = static_cast<char*>(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int32_t)));
-    colInfo.nullbitmap = static_cast<char*>(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8));
-
-    taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo);
 
     //    SColumnInfoData colInfo1 = {0};
     //    colInfo1.info.type = TSDB_DATA_TYPE_BINARY;
@@ -115,7 +108,6 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator) {
   }
 
   pBlock->info.rows = pInfo->numOfRowsPerPage;
-  pBlock->info.numOfCols = 1;
 
   pInfo->current += 1;
   return pBlock;
@@ -128,28 +120,15 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) {
   }
 
   if (pInfo->pBlock == NULL) {
-    pInfo->pBlock = static_cast<SSDataBlock*>(taosMemoryCalloc(1, sizeof(SSDataBlock)));
+    pInfo->pBlock = createDataBlock();
 
-    pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
+    SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1);
+    blockDataAppendColInfo(pInfo->pBlock, &colInfo);
 
-    SColumnInfoData colInfo = {0};
+    SColumnInfoData colInfo1 = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 2);
-    colInfo.info.type = TSDB_DATA_TYPE_TIMESTAMP;
+    blockDataAppendColInfo(pInfo->pBlock, &colInfo1);
-    colInfo.info.bytes = sizeof(int64_t);
-    colInfo.info.colId = 1;
-    colInfo.pData = static_cast<char*>(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int64_t)));
-    //    colInfo.nullbitmap = static_cast<char*>(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8));
 
-    taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo);
+    blockDataEnsureCapacity(pInfo->pBlock, pInfo->numOfRowsPerPage);
-
-    SColumnInfoData colInfo1 = {0};
-    colInfo1.info.type = TSDB_DATA_TYPE_INT;
-    colInfo1.info.bytes = 4;
-    colInfo1.info.colId = 2;
-
-    colInfo1.pData = static_cast<char*>(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int32_t)));
-    colInfo1.nullbitmap = static_cast<char*>(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8));
-
-    taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo1);
   } else {
     blockDataCleanup(pInfo->pBlock);
   }
@@ -185,7 +164,6 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) {
   }
 
   pBlock->info.rows = pInfo->numOfRowsPerPage;
-  pBlock->info.numOfCols = 1;
 
   pInfo->current += 1;
 

source/libs/executor/test/sortTests.cpp

@@ -62,25 +62,21 @@ SSDataBlock* getSingleColDummyBlock(void* param) {
     return NULL;
   }
 
-  SSDataBlock* pBlock = static_cast<SSDataBlock*>(taosMemoryCalloc(1, sizeof(SSDataBlock)));
+  SSDataBlock* pBlock = createDataBlock();
-  pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
 
   SColumnInfoData colInfo = {0};
   colInfo.info.type = pInfo->type;
   if (pInfo->type == TSDB_DATA_TYPE_NCHAR){
     colInfo.info.bytes = TSDB_NCHAR_SIZE * VARCOUNT + VARSTR_HEADER_SIZE;
-    colInfo.varmeta.offset = static_cast<int32_t *>(taosMemoryCalloc(pInfo->pageRows, sizeof(int32_t)));
   } else if(pInfo->type == TSDB_DATA_TYPE_BINARY) {
     colInfo.info.bytes = VARCOUNT + VARSTR_HEADER_SIZE;
-    colInfo.varmeta.offset = static_cast<int32_t *>(taosMemoryCalloc(pInfo->pageRows, sizeof(int32_t)));
   } else{
     colInfo.info.bytes = tDataTypes[pInfo->type].bytes;
-    colInfo.pData = static_cast<char*>(taosMemoryCalloc(pInfo->pageRows, colInfo.info.bytes));
-    colInfo.nullbitmap = static_cast<char*>(taosMemoryCalloc(1, (pInfo->pageRows + 7) / 8));
   }
   colInfo.info.colId = 1;
 
-  taosArrayPush(pBlock->pDataBlock, &colInfo);
+  blockDataAppendColInfo(pBlock, &colInfo);
+  blockDataEnsureCapacity(pBlock, pInfo->pageRows);
 
   for (int32_t i = 0; i < pInfo->pageRows; ++i) {
     SColumnInfoData* pColInfo = static_cast<SColumnInfoData*>(TARRAY_GET_ELEM(pBlock->pDataBlock, 0));
@@ -128,7 +124,6 @@ SSDataBlock* getSingleColDummyBlock(void* param) {
   }
 
   pBlock->info.rows = pInfo->pageRows;
-  pBlock->info.numOfCols = 1;
   return pBlock;
 }
 
@@ -354,15 +349,10 @@ TEST(testCase, ordered_merge_sort_Test) {
   SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo));
   taosArrayPush(orderInfo, &oi);
 
-  SSDataBlock* pBlock = static_cast<SSDataBlock*>(taosMemoryCalloc(1, sizeof(SSDataBlock)));
+  SSDataBlock* pBlock = createDataBlock();
-  pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
+  for (int32_t i = 0; i < 1; ++i) {
-  pBlock->info.numOfCols = 1;
+    SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1);
-  for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
+    blockDataAppendColInfo(pBlock, &colInfo);
-    SColumnInfoData colInfo = {0};
-    colInfo.info.type = TSDB_DATA_TYPE_INT;
-    colInfo.info.bytes = sizeof(int32_t);
-    colInfo.info.colId = 1;
-    taosArrayPush(pBlock->pDataBlock, &colInfo);
   }
 
   SSortHandle* phandle = tsortCreateSortHandle(orderInfo, SORT_MULTISOURCE_MERGE, 1024, 5, pBlock,"test_abc");

source/libs/function/inc/taggfunction.h

@@ -28,19 +28,6 @@ extern "C" {
 #include "function.h"
 #include "tudf.h"
 
-
-extern SAggFunctionInfo aggFunc[35];
-
-#define FUNCSTATE_SO           0x0u
-#define FUNCSTATE_MO           0x1u    // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM
-#define FUNCSTATE_STREAM       0x2u    // function avail for stream
-#define FUNCSTATE_STABLE       0x4u    // function avail for super table
-#define FUNCSTATE_NEED_TS      0x8u    // timestamp is required during query processing
-#define FUNCSTATE_SELECTIVITY  0x10u   // selectivity functions, can exists along with tag columns
-
-#define BASIC_FUNC_SO FUNCSTATE_SO | FUNCSTATE_STREAM | FUNCSTATE_STABLE
-#define BASIC_FUNC_MO FUNCSTATE_MO | FUNCSTATE_STREAM | FUNCSTATE_STABLE
-
 #define AVG_FUNCTION_INTER_BUFFER_SIZE 50
 
 #define DATA_SET_FLAG ','  // to denote the output area has data, not null value

source/libs/function/src/builtins.c

@@ -1267,6 +1267,29 @@ static bool validateMinuteRange(int8_t hour, int8_t minute, char sign) {
   return false;
 }
 
+static bool validateTimestampDigits(const SValueNode* pVal) {
+  if (!IS_INTEGER_TYPE(pVal->node.resType.type)) {
+    return false;
+  }
+
+  int64_t tsVal = pVal->datum.i;
+  char fraction[20] = {0};
+  NUM_TO_STRING(pVal->node.resType.type, &tsVal, sizeof(fraction), fraction);
+  int32_t tsDigits = (int32_t)strlen(fraction);
+
+  if (tsDigits > TSDB_TIME_PRECISION_SEC_DIGITS) {
+    if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS ||
+        tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS ||
+        tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  return true;
+}
+
 static bool validateTimezoneFormat(const SValueNode* pVal) {
   if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) {
     return false;
@@ -1379,6 +1402,15 @@ static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t l
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
+  if (QUERY_NODE_VALUE == nodeType(nodesListGetNode(pFunc->pParameterList, 0))) {
+    SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0);
+
+    if (!validateTimestampDigits(pValue)) {
+      pFunc->node.resType = (SDataType){.bytes = 0, .type = TSDB_DATA_TYPE_BINARY};
+      return TSDB_CODE_SUCCESS;
+    }
+  }
+
   // param1
   if (numOfParams == 2) {
     SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1);

source/libs/function/src/builtinsimpl.c

@@ -561,7 +561,7 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) {
 
   // check for overflow
   if (IS_FLOAT_TYPE(type) && (isinf(pSumRes->dsum) || isnan(pSumRes->dsum))) {
-    GET_RES_INFO(pCtx)->isNullRes = 1;
+    numOfElem = 0;
   }
 
 _sum_over:
@@ -927,7 +927,7 @@ int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
 
   // check for overflow
   if (isinf(pAvgRes->result) || isnan(pAvgRes->result)) {
-    GET_RES_INFO(pCtx)->isNullRes = 1;
+    GET_RES_INFO(pCtx)->numOfRes = 0;
   }
 
   return functionFinalize(pCtx, pBlock);
@@ -1449,11 +1449,13 @@ void setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t
 void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuplePos* pTuplePos, int32_t rowIndex) {
   int32_t pageId = pTuplePos->pageId;
   int32_t offset = pTuplePos->offset;
+
   if (pTuplePos->pageId != -1) {
+    int32_t numOfCols = taosArrayGetSize(pCtx->pSrcBlock->pDataBlock);
     SFilePage* pPage = getBufPage(pCtx->pBuf, pageId);
 
     bool* nullList = (bool*)((char*)pPage + offset);
-    char* pStart = (char*)(nullList + pCtx->pSrcBlock->info.numOfCols * sizeof(bool));
+    char* pStart = (char*)(nullList + numOfCols * sizeof(bool));
 
     // todo set the offset value to optimize the performance.
     for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) {
@@ -1774,6 +1776,11 @@ int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
     pStddevRes->result = sqrt(fabs(pStddevRes->quadraticDSum / ((double)pStddevRes->count) - avg * avg));
   }
 
+  // check for overflow
+  if (isinf(pStddevRes->result) || isnan(pStddevRes->result)) {
+    GET_RES_INFO(pCtx)->numOfRes = 0;
+  }
+
   return functionFinalize(pCtx, pBlock);
 }
 
@@ -3038,7 +3045,7 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData
 void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) {
   SFilePage* pPage = NULL;
 
-  int32_t completeRowSize = pSrcBlock->info.rowSize + pSrcBlock->info.numOfCols * sizeof(bool);
+  int32_t completeRowSize = pSrcBlock->info.rowSize + (int32_t) taosArrayGetSize(pSrcBlock->pDataBlock) * sizeof(bool);
 
   if (pCtx->curBufPage == -1) {
     pPage = getNewBufPage(pCtx->pBuf, 0, &pCtx->curBufPage);
@@ -3056,8 +3063,8 @@ void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS
   // keep the current row data, extract method
   int32_t offset = 0;
   bool*   nullList = (bool*)((char*)pPage + pPage->num);
-  char*   pStart = (char*)(nullList + sizeof(bool) * pSrcBlock->info.numOfCols);
+  char*   pStart = (char*)(nullList + sizeof(bool) * (int32_t) taosArrayGetSize(pSrcBlock->pDataBlock));
-  for (int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) {
+  for (int32_t i = 0; i < (int32_t) taosArrayGetSize(pSrcBlock->pDataBlock); ++i) {
     SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i);
     bool             isNull = colDataIsNull_s(pCol, rowIndex);
     if (isNull) {
@@ -3086,11 +3093,13 @@ void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS
 void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) {
   SFilePage* pPage = getBufPage(pCtx->pBuf, pPos->pageId);
 
+  int32_t numOfCols = taosArrayGetSize(pSrcBlock->pDataBlock);
+
   bool* nullList = (bool*)((char*)pPage + pPos->offset);
-  char* pStart = (char*)(nullList + pSrcBlock->info.numOfCols * sizeof(bool));
+  char* pStart = (char*)(nullList + numOfCols * sizeof(bool));
 
   int32_t offset = 0;
-  for (int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) {
+  for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i);
     if ((nullList[i] = colDataIsNull_s(pCol, rowIndex)) == true) {
       offset += pCol->info.bytes;

source/libs/function/src/tudf.c

@@ -763,8 +763,8 @@ void freeUdfInterBuf(SUdfInterBuf *buf) {
 
 int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlock) {
   udfBlock->numOfRows = block->info.rows;
-  udfBlock->numOfCols = block->info.numOfCols;
+  udfBlock->numOfCols = taosArrayGetSize(block->pDataBlock);
-  udfBlock->udfCols = taosMemoryCalloc(udfBlock->numOfCols, sizeof(SUdfColumn*));
+  udfBlock->udfCols = taosMemoryCalloc(taosArrayGetSize(block->pDataBlock), sizeof(SUdfColumn*));
   for (int32_t i = 0; i < udfBlock->numOfCols; ++i) {
     udfBlock->udfCols[i] = taosMemoryCalloc(1, sizeof(SUdfColumn));
     SColumnInfoData *col= (SColumnInfoData*)taosArrayGet(block->pDataBlock, i);
@@ -799,7 +799,6 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo
 }
 
 int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) {
-  block->info.numOfCols = 1;
   block->info.rows = udfCol->colData.numOfRows;
   block->info.hasVarCol = IS_VAR_DATA_TYPE(udfCol->colMeta.type);
 
@@ -830,25 +829,19 @@ int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) {
 
 int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SSDataBlock *output) {
   output->info.rows = input->numOfRows;
-  output->info.numOfCols = numOfCols;
-  bool hasVarCol = false;
-  for (int32_t i = 0; i < numOfCols; ++i) {
-    if (IS_VAR_DATA_TYPE((input+i)->columnData->info.type)) {
-      hasVarCol = true;
-      break;
-    }
-  }
-  output->info.hasVarCol = hasVarCol;
-
   output->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
   for (int32_t i = 0; i < numOfCols; ++i) {
     taosArrayPush(output->pDataBlock, (input + i)->columnData);
+
+    if (IS_VAR_DATA_TYPE((input+i)->columnData->info.type)) {
+      output->info.hasVarCol = true;
+    }
   }
   return 0;
 }
 
 int32_t convertDataBlockToScalarParm(SSDataBlock *input, SScalarParam *output) {
-  if (input->info.numOfCols != 1) {
+  if (taosArrayGetSize(input->pDataBlock) != 1) {
     fnError("scalar function only support one column");
     return -1;
   }
@@ -1089,24 +1082,14 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
   int32_t start = pInput->startRowIndex;
   int32_t numOfRows = pInput->numOfRows;
 
-
+  SSDataBlock* pTempBlock = createDataBlock();
-  SSDataBlock tempBlock = {0};
+  pTempBlock->info.rows = pInput->totalRows;
-  tempBlock.info.numOfCols = numOfCols;
+  pTempBlock->info.uid = pInput->uid;
-  tempBlock.info.rows = pInput->totalRows;
-  tempBlock.info.uid = pInput->uid;
-  bool hasVarCol = false;
-  tempBlock.pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
-
   for (int32_t i = 0; i < numOfCols; ++i) {
-    SColumnInfoData *col = pInput->pData[i];
+    blockDataAppendColInfo(pTempBlock, pInput->pData[i]);
-    if (IS_VAR_DATA_TYPE(col->info.type)) {
-      hasVarCol = true;
-    }
-    taosArrayPush(tempBlock.pDataBlock, col);
   }
-  tempBlock.info.hasVarCol = hasVarCol;
 
-  SSDataBlock *inputBlock = blockDataExtractBlock(&tempBlock, start, numOfRows);
+  SSDataBlock *inputBlock = blockDataExtractBlock(pTempBlock, start, numOfRows);
 
   SUdfInterBuf state = {.buf = udfRes->interResBuf,
                         .bufLen = session->bufSize,
@@ -1131,7 +1114,9 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
   }
 
   blockDataDestroy(inputBlock);
-  taosArrayDestroy(tempBlock.pDataBlock);
+
+  taosArrayDestroy(pTempBlock->pDataBlock);
+  taosMemoryFree(pTempBlock);
 
   releaseUdfFuncHandle(pCtx->udfName);
   freeUdfInterBuf(&newState);

source/libs/function/src/udfd.c

@@ -613,7 +613,9 @@ int32_t udfdOpenClientRpc() {
 }
 
 int32_t udfdCloseClientRpc() {
+  fnInfo("udfd begin closing rpc");
   rpcClose(global.clientRpc);
+  fnInfo("udfd finish closing rpc");
   return 0;
 }
 
@@ -939,7 +941,6 @@ int main(int argc, char *argv[]) {
   udfdRun();
   
   removeListeningPipe();
-  uv_thread_join(&mnodeConnectThread);
   udfdCloseClientRpc();
 
   return 0;

source/libs/function/test/runUdf.c

@@ -45,27 +45,27 @@ int scalarFuncTest() {
   for (int k = 0; k < 1; ++k) {
     SSDataBlock  block = {0};
     SSDataBlock *pBlock = &block;
-    pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
+    for (int32_t i = 0; i < 1; ++i) {
-    pBlock->info.numOfCols = 1;
+      SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1);
+      blockDataAppendColInfo(pBlock, &colInfo);
+    }
+
+    blockDataEnsureCapacity(pBlock, 1024);
     pBlock->info.rows = 1024;
-    for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
+
-      SColumnInfoData colInfo = {0};
+    SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, 0);
-      colInfo.info.type = TSDB_DATA_TYPE_INT;
+    for (int32_t j = 0; j < pBlock->info.rows; ++j) {
-      colInfo.info.bytes = sizeof(int32_t);
+      colDataAppendInt32(pCol, j, &j);
-      colInfo.info.colId = 1;
-      colInfoDataEnsureCapacity(&colInfo, 0, pBlock->info.rows);
-      for (int32_t j = 0; j < pBlock->info.rows; ++j) {
-        colDataAppendInt32(&colInfo, j, &j);
-      }
-      taosArrayPush(pBlock->pDataBlock, &colInfo);
     }
 
     SScalarParam input = {0};
     input.numOfRows = pBlock->info.rows;
     input.columnData = taosArrayGet(pBlock->pDataBlock, 0);
+
     SScalarParam output = {0};
     doCallUdfScalarFunc(handle, &input, 1, &output);
     taosArrayDestroy(pBlock->pDataBlock);
+
     SColumnInfoData *col = output.columnData;
     for (int32_t i = 0; i < output.numOfRows; ++i) {
       if (i % 100 == 0)
@@ -89,21 +89,18 @@ int aggregateFuncTest() {
     return -1;
   }
 
-  SSDataBlock  block = {0};
+  SSDataBlock *pBlock = createDataBlock();
-  SSDataBlock *pBlock = &block;
+  for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
-  pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
+    SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1);
-  pBlock->info.numOfCols = 1;
+     blockDataAppendColInfo(pBlock, &colInfo);
+  }
+
+  blockDataEnsureCapacity(pBlock, 1024);
   pBlock->info.rows = 1024;
-  for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
+
-    SColumnInfoData colInfo = {0};
+  SColumnInfoData* pColInfo = bdGetColumnInfoData(pBlock, 0);
-    colInfo.info.type = TSDB_DATA_TYPE_INT;
+  for (int32_t j = 0; j < pBlock->info.rows; ++j) {
-    colInfo.info.bytes = sizeof(int32_t);
+    colDataAppendInt32(pColInfo, j, &j);
-    colInfo.info.colId = 1;
-    colInfoDataEnsureCapacity(&colInfo, 0, pBlock->info.rows);
-    for (int32_t j = 0; j < pBlock->info.rows; ++j) {
-      colDataAppendInt32(&colInfo, j, &j);
-    }
-    taosArrayPush(pBlock->pDataBlock, &colInfo);
   }
 
   SUdfInterBuf buf = {0};
@@ -121,6 +118,7 @@ int aggregateFuncTest() {
   freeUdfInterBuf(&resultBuf);
   doTeardownUdf(handle);
 
+  blockDataDestroy(pBlock);
   return 0;
 }
 

source/libs/index/test/index_executor_tests.cpp

@@ -86,7 +86,7 @@ void sifAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *sl
     SColumnInfoData idata = {0};
     idata.info = *colInfo;
 
-    colInfoDataEnsureCapacity(&idata, 0, rows);
+    colInfoDataEnsureCapacity(&idata, rows);
 
     taosArrayPush(res->pDataBlock, &idata);
 

source/libs/monitor/src/monMain.c

@@ -201,6 +201,9 @@ static void monGenClusterJson(SMonInfo *pMonitor) {
   tjsonAddStringToObject(pJson, "version", pInfo->version);
   tjsonAddDoubleToObject(pJson, "master_uptime", pInfo->master_uptime);
   tjsonAddDoubleToObject(pJson, "monitor_interval", pInfo->monitor_interval);
+  tjsonAddDoubleToObject(pJson, "dbs_total", pInfo->dbs_total);
+  tjsonAddDoubleToObject(pJson, "tbs_total", pInfo->tbs_total);
+  tjsonAddDoubleToObject(pJson, "stbs_total", pInfo->stbs_total);
   tjsonAddDoubleToObject(pJson, "vgroups_total", pInfo->vgroups_total);
   tjsonAddDoubleToObject(pJson, "vgroups_alive", pInfo->vgroups_alive);
   tjsonAddDoubleToObject(pJson, "vnodes_total", pInfo->vnodes_total);

source/libs/monitor/src/monMsg.c

@@ -102,6 +102,9 @@ int32_t tEncodeSMonClusterInfo(SEncoder *encoder, const SMonClusterInfo *pInfo)
   if (tEncodeCStr(encoder, pInfo->version) < 0) return -1;
   if (tEncodeFloat(encoder, pInfo->master_uptime) < 0) return -1;
   if (tEncodeI32(encoder, pInfo->monitor_interval) < 0) return -1;
+  if (tEncodeI32(encoder, pInfo->dbs_total) < 0) return -1;
+  if (tEncodeI32(encoder, pInfo->stbs_total) < 0) return -1;
+  if (tEncodeI64(encoder, pInfo->tbs_total) < 0) return -1;
   if (tEncodeI32(encoder, pInfo->vgroups_total) < 0) return -1;
   if (tEncodeI32(encoder, pInfo->vgroups_alive) < 0) return -1;
   if (tEncodeI32(encoder, pInfo->vnodes_total) < 0) return -1;
@@ -130,6 +133,9 @@ int32_t tDecodeSMonClusterInfo(SDecoder *decoder, SMonClusterInfo *pInfo) {
   if (tDecodeCStrTo(decoder, pInfo->version) < 0) return -1;
   if (tDecodeFloat(decoder, &pInfo->master_uptime) < 0) return -1;
   if (tDecodeI32(decoder, &pInfo->monitor_interval) < 0) return -1;
+  if (tDecodeI32(decoder, &pInfo->dbs_total) < 0) return -1;
+  if (tDecodeI32(decoder, &pInfo->stbs_total) < 0) return -1;
+  if (tDecodeI64(decoder, &pInfo->tbs_total) < 0) return -1;
   if (tDecodeI32(decoder, &pInfo->vgroups_total) < 0) return -1;
   if (tDecodeI32(decoder, &pInfo->vgroups_alive) < 0) return -1;
   if (tDecodeI32(decoder, &pInfo->vnodes_total) < 0) return -1;

source/libs/qworker/src/qworker.c

@@ -271,7 +271,7 @@ int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen
   SDeleterRes* pDelRes = (SDeleterRes*)output.pData;
   
   rsp.affectedRows = pDelRes->affectedRows;
-  pRes->uid = pDelRes->uid;
+  pRes->suid = pDelRes->suid;
   pRes->uidList = pDelRes->uidList;
   pRes->skey = pDelRes->skey;
   pRes->ekey = pDelRes->ekey;

source/libs/scalar/inc/sclInt.h

@@ -57,7 +57,7 @@ typedef struct SScalarCtx {
 #define SCL_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
 
 int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out);
-SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows);
+SColumnInfoData* sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows);
 int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode);
 
 #define GET_PARAM_TYPE(_c)      ((_c)->columnData ? (_c)->columnData->info.type : (_c)->hashValueType)

source/libs/scalar/src/filter.c

@@ -3821,7 +3821,7 @@ bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnData
     SScalarParam output = {0};
 
     SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)};
-    output.columnData = createColumnInfoData(&type, pSrc->info.rows);
+    output.columnData = sclCreateColumnInfoData(&type, pSrc->info.rows);
 
     SArray *pList = taosArrayInit(1, POINTER_BYTES);
     taosArrayPush(pList, &pSrc);

source/libs/scalar/src/scalar.c

@@ -36,7 +36,7 @@ int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode) {
 }
 
 
-SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) {
+SColumnInfoData* sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows) {
   SColumnInfoData* pColumnData = taosMemoryCalloc(1, sizeof(SColumnInfoData));
   if (pColumnData == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -48,7 +48,7 @@ SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) {
   pColumnData->info.scale     = pType->scale;
   pColumnData->info.precision = pType->precision;
 
-  int32_t code = colInfoDataEnsureCapacity(pColumnData, 0, numOfRows);
+  int32_t code = colInfoDataEnsureCapacity(pColumnData, numOfRows);
   if (code != TSDB_CODE_SUCCESS) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     taosMemoryFree(pColumnData);
@@ -60,10 +60,10 @@ SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) {
 
 int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out) {
   SScalarParam in = {.numOfRows = 1};
-  in.columnData = createColumnInfoData(&pValueNode->node.resType, 1);
+  in.columnData = sclCreateColumnInfoData(&pValueNode->node.resType, 1);
   colDataAppend(in.columnData, 0, nodesGetValueFromNode(pValueNode), false);
 
-  colInfoDataEnsureCapacity(out->columnData, 0, 1);
+  colInfoDataEnsureCapacity(out->columnData, 1);
   int32_t code = vectorConvertImpl(&in, out);
   sclFreeParam(&in);
 
@@ -193,7 +193,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
       SValueNode *valueNode = (SValueNode *)node;
 
       param->numOfRows = 1;
-      param->columnData = createColumnInfoData(&valueNode->node.resType, 1);
+      param->columnData = sclCreateColumnInfoData(&valueNode->node.resType, 1);
       if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type) {
         colDataAppendNULL(param->columnData, 0);
       } else {
@@ -245,7 +245,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
       }
 
       SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, index);
-      if (NULL == block || ref->slotId >= block->info.numOfCols) {
+      if (NULL == block || ref->slotId >= taosArrayGetSize(block->pDataBlock)) {
         sclError("column slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(block->pDataBlock));
         SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
       }
@@ -431,7 +431,7 @@ int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outp
       SCL_ERR_JRET(code);
     }
   
-    output->columnData = createColumnInfoData(&node->node.resType, rowNum);
+    output->columnData = sclCreateColumnInfoData(&node->node.resType, rowNum);
     if (output->columnData == NULL) {
       sclError("calloc %d failed", (int32_t)(rowNum * output->columnData->info.bytes));
       SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
@@ -484,7 +484,7 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o
   output->numOfRows = rowNum;
 
   SDataType t = {.type = type, .bytes = tDataTypes[type].bytes};
-  output->columnData = createColumnInfoData(&t, rowNum);
+  output->columnData = sclCreateColumnInfoData(&t, rowNum);
   if (output->columnData == NULL) {
     sclError("calloc %d failed", (int32_t)(rowNum * sizeof(bool)));
     SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
@@ -539,7 +539,7 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp
   int32_t code = 0;
 
   SCL_ERR_RET(sclInitOperatorParams(&params, node, ctx, &rowNum));
-  output->columnData = createColumnInfoData(&node->node.resType, rowNum);
+  output->columnData = sclCreateColumnInfoData(&node->node.resType, rowNum);
   if (output->columnData == NULL) {
     sclError("calloc failed, size:%d", (int32_t)rowNum * node->node.resType.bytes);
     SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
@@ -901,7 +901,7 @@ EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) {
     return DEAL_RES_ERROR;
   }
 
-  colDataAssign(col, res->columnData, res->numOfRows);
+  colDataAssign(col, res->columnData, res->numOfRows, NULL);
   block->info.rows = res->numOfRows;
 
   sclFreeParam(res);
@@ -983,7 +983,8 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) {
       SCL_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
     }
 
-    colDataAssign(pDst->columnData, res->columnData, res->numOfRows);
+    colInfoDataEnsureCapacity(pDst->columnData, res->numOfRows);
+    colDataAssign(pDst->columnData, res->columnData, res->numOfRows, NULL);
     pDst->numOfRows = res->numOfRows;
     taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES);
   }

source/libs/scalar/src/sclfunc.c

@@ -120,7 +120,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
     }
 
     default: {
-      colDataAssign(pOutputData, pInputData, pInput->numOfRows);
+      colDataAssign(pOutputData, pInputData, pInput->numOfRows, NULL);
     }
   }
 
@@ -273,7 +273,7 @@ static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
     }
 
     default: {
-      colDataAssign(pOutputData, pInputData, pInput->numOfRows);
+      colDataAssign(pOutputData, pInputData, pInput->numOfRows, NULL);
     }
   }
 
@@ -856,10 +856,13 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
 int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
   int32_t type = GET_PARAM_TYPE(pInput);
 
+  bool tzPresent = (inputNum == 2) ? true : false;
   char* tz;
   int32_t tzLen;
-  tz = varDataVal(pInput[1].columnData->pData);
+  if (tzPresent) {
-  tzLen = varDataLen(pInput[1].columnData->pData);
+    tz = varDataVal(pInput[1].columnData->pData);
+    tzLen = varDataLen(pInput[1].columnData->pData);
+  }
 
   for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
     if (colDataIsNull_s(pInput[0].columnData, i)) {

source/libs/scalar/src/sclvector.c

@@ -852,7 +852,7 @@ int32_t vectorConvertScalarParam(SScalarParam *input, SScalarParam *output, int3
   SDataType t = {.type = type, .bytes = tDataTypes[type].bytes};
   output->numOfRows = input->numOfRows;
 
-  output->columnData = createColumnInfoData(&t, input->numOfRows);
+  output->columnData = sclCreateColumnInfoData(&t, input->numOfRows);
   if (output->columnData == NULL) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
@@ -923,7 +923,7 @@ static int32_t doConvertHelper(SScalarParam* pDest, int32_t* convert, const SSca
     pDest->numOfRows = pParam->numOfRows;
 
     SDataType t = {.type = type, .bytes = tDataTypes[type].bytes};
-    pDest->columnData = createColumnInfoData(&t, pParam->numOfRows);
+    pDest->columnData = sclCreateColumnInfoData(&t, pParam->numOfRows);
     if (pDest->columnData == NULL) {
       sclError("malloc %d failed", (int32_t)(pParam->numOfRows * sizeof(double)));
       return TSDB_CODE_OUT_OF_MEMORY;

source/libs/scalar/test/filter/filterTests.cpp

@@ -105,29 +105,14 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
   }
 
   if (NULL == *block) {
-    SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock));
+    SSDataBlock *res = createDataBlock();
-    res->info.numOfCols = 3;
-    res->info.rows = rowNum;
-    res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData));
     for (int32_t i = 0; i < 2; ++i) {
-      SColumnInfoData idata = {{0}};
+      SColumnInfoData idata = createColumnInfoData(TSDB_DATA_TYPE_NULL, 10, 1+i);
-      idata.info.type  = TSDB_DATA_TYPE_NULL;
+      blockDataAppendColInfo(res, &idata);
-      idata.info.bytes = 10;
-      idata.info.colId = i + 1;
-
-      int32_t size = idata.info.bytes * rowNum;
-      idata.pData = (char *)taosMemoryCalloc(1, size);
-      taosArrayPush(res->pDataBlock, &idata);
     }
 
-    SColumnInfoData idata = {{0}};
+    SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 3);
-    idata.info.type  = dataType;
+    blockDataAppendColInfo(res, &idata);
-    idata.info.bytes = dataBytes;
-    idata.info.colId = 3;
-    int32_t size = idata.info.bytes * rowNum;
-    idata.pData = (char *)taosMemoryCalloc(1, size);
-    taosArrayPush(res->pDataBlock, &idata);
-    
     blockDataEnsureCapacity(res, rowNum);
 
     SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
@@ -142,23 +127,18 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
 
     rnode->slotId = 2;
     rnode->colId = 3;
+    res->info.rows = rowNum;
 
     *block = res;
   } else {
     SSDataBlock *res = *block;
     
     int32_t idx = taosArrayGetSize(res->pDataBlock);
-    SColumnInfoData idata = {{0}};
+    SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 1+idx);
-    idata.info.type  = dataType;
+    blockDataAppendColInfo(res, &idata);
-    idata.info.bytes = dataBytes;
+    blockDataEnsureCapacity(res, rowNum);
-    idata.info.colId = 1 + idx;
-    int32_t size = idata.info.bytes * rowNum;
-    idata.pData = (char *)taosMemoryCalloc(1, size);
-    taosArrayPush(res->pDataBlock, &idata);
-    res->info.numOfCols++;
-    SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
 
-    colInfoDataEnsureCapacity(pColumn, 0, rowNum);
+    SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
 
     for (int32_t i = 0; i < rowNum; ++i) {
       colDataAppend(pColumn, i, (const char *)value, false);
@@ -399,7 +379,7 @@ TEST(columnTest, smallint_column_greater_double_value) {
   keep = filterRangeExecute(filter, &stat, 1, rowNum);
   ASSERT_EQ(keep, true);
 
-  SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
+  SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
   code = filterSetDataFromSlotId(filter, &param);
   ASSERT_EQ(code, 0);
 
@@ -407,7 +387,7 @@ TEST(columnTest, smallint_column_greater_double_value) {
   stat.min = 1;
   stat.numOfNull = 0;
   int8_t *rowRes = NULL;
-  keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
+  keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
   ASSERT_EQ(keep, false);
 
   for (int32_t i = 0; i < rowNum; ++i) {
@@ -454,7 +434,7 @@ TEST(columnTest, int_column_greater_smallint_value) {
   keep = filterRangeExecute(filter, &stat, 1, rowNum);
   ASSERT_EQ(keep, false);
 
-  SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
+  SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
   code = filterSetDataFromSlotId(filter, &param);
   ASSERT_EQ(code, 0);
 
@@ -462,7 +442,7 @@ TEST(columnTest, int_column_greater_smallint_value) {
   stat.min = 1;
   stat.numOfNull = 0;
   int8_t *rowRes = NULL;
-  keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
+  keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
   ASSERT_EQ(keep, false);
 
   for (int32_t i = 0; i < rowNum; ++i) {
@@ -498,7 +478,7 @@ TEST(columnTest, int_column_in_double_list) {
   ASSERT_EQ(code, 0);
 
   SColumnDataAgg stat = {0};
-  SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
+  SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
   code = filterSetDataFromSlotId(filter, &param);
   ASSERT_EQ(code, 0);
 
@@ -506,7 +486,7 @@ TEST(columnTest, int_column_in_double_list) {
   stat.min = 1;
   stat.numOfNull = 0;
   int8_t *rowRes = NULL;
-  bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
+  bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
   ASSERT_EQ(keep, false);
 
   for (int32_t i = 0; i < rowNum; ++i) {
@@ -562,7 +542,7 @@ TEST(columnTest, binary_column_in_binary_list) {
   ASSERT_EQ(code, 0);
 
   SColumnDataAgg stat = {0};
-  SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
+  SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
   code = filterSetDataFromSlotId(filter, &param);
   ASSERT_EQ(code, 0);
 
@@ -570,7 +550,7 @@ TEST(columnTest, binary_column_in_binary_list) {
   stat.min = 1;
   stat.numOfNull = 0;
   int8_t *rowRes = NULL;
-  bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
+  bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
   ASSERT_EQ(keep, false);
 
   for (int32_t i = 0; i < rowNum; ++i) {
@@ -610,7 +590,7 @@ TEST(columnTest, binary_column_like_binary) {
   ASSERT_EQ(code, 0);
 
   SColumnDataAgg stat = {0};
-  SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
+  SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
   code = filterSetDataFromSlotId(filter, &param);
   ASSERT_EQ(code, 0);
 
@@ -618,7 +598,7 @@ TEST(columnTest, binary_column_like_binary) {
   stat.min = 1;
   stat.numOfNull = 0;
   int8_t *rowRes = NULL;
-  bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
+  bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
   ASSERT_EQ(keep, false);
 
   for (int32_t i = 0; i < rowNum; ++i) {
@@ -657,7 +637,7 @@ TEST(columnTest, binary_column_is_null) {
   ASSERT_EQ(code, 0);
 
   SColumnDataAgg stat = {0};
-  SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
+  SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
   code = filterSetDataFromSlotId(filter, &param);
   ASSERT_EQ(code, 0);
 
@@ -665,7 +645,7 @@ TEST(columnTest, binary_column_is_null) {
   stat.min = 1;
   stat.numOfNull = 0;
   int8_t *rowRes = NULL;
-  bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
+  bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
   ASSERT_EQ(keep, false);
 
   for (int32_t i = 0; i < rowNum; ++i) {
@@ -704,7 +684,7 @@ TEST(columnTest, binary_column_is_not_null) {
   ASSERT_EQ(code, 0);
 
   SColumnDataAgg stat = {0};
-  SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
+  SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
   code = filterSetDataFromSlotId(filter, &param);
   ASSERT_EQ(code, 0);
 
@@ -712,7 +692,7 @@ TEST(columnTest, binary_column_is_not_null) {
   stat.min = 1;
   stat.numOfNull = 0;
   int8_t *rowRes = NULL;
-  bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
+  bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
   ASSERT_EQ(keep, false);
 
   for (int32_t i = 0; i < rowNum; ++i) {
@@ -741,7 +721,7 @@ TEST(opTest, smallint_column_greater_int_column) {
   ASSERT_EQ(code, 0);
 
   SColumnDataAgg stat = {0};
-  SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
+  SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
   code = filterSetDataFromSlotId(filter, &param);
   ASSERT_EQ(code, 0);
 
@@ -749,7 +729,7 @@ TEST(opTest, smallint_column_greater_int_column) {
   stat.min = 1;
   stat.numOfNull = 0;
   int8_t *rowRes = NULL;
-  bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
+  bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
   ASSERT_EQ(keep, false);
 
   for (int32_t i = 0; i < rowNum; ++i) {
@@ -779,7 +759,7 @@ TEST(opTest, smallint_value_add_int_column) {
   ASSERT_EQ(code, 0);
 
   SColumnDataAgg stat = {0};
-  SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
+  SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
   code = filterSetDataFromSlotId(filter, &param);
   ASSERT_EQ(code, 0);
 
@@ -787,7 +767,7 @@ TEST(opTest, smallint_value_add_int_column) {
   stat.min = 1;
   stat.numOfNull = 0;
   int8_t *rowRes = NULL;
-  bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
+  bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
   ASSERT_EQ(keep, false);
 
   for (int32_t i = 0; i < rowNum; ++i) {
@@ -822,7 +802,7 @@ TEST(opTest, bigint_column_multi_binary_column) {
   ASSERT_EQ(code, 0);
 
   SColumnDataAgg stat = {0};
-  SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
+  SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
   code = filterSetDataFromSlotId(filter, &param);
   ASSERT_EQ(code, 0);
 
@@ -830,7 +810,7 @@ TEST(opTest, bigint_column_multi_binary_column) {
   stat.min = 1;
   stat.numOfNull = 0;
   int8_t *rowRes = NULL;
-  bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
+  bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
   ASSERT_EQ(keep, false);
 
   for (int32_t i = 0; i < rowNum; ++i) {
@@ -865,7 +845,7 @@ TEST(opTest, smallint_column_and_binary_column) {
   ASSERT_EQ(code, 0);
 
   SColumnDataAgg stat = {0};
-  SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
+  SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
   code = filterSetDataFromSlotId(filter, &param);
   ASSERT_EQ(code, 0);
 
@@ -873,7 +853,7 @@ TEST(opTest, smallint_column_and_binary_column) {
   stat.min = 1;
   stat.numOfNull = 0;
   int8_t *rowRes = NULL;
-  bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
+  bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
   ASSERT_EQ(keep, false);
 
   for (int32_t i = 0; i < rowNum; ++i) {
@@ -903,7 +883,7 @@ TEST(opTest, smallint_column_or_float_column) {
   ASSERT_EQ(code, 0);
 
   SColumnDataAgg stat = {0};
-  SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
+  SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock };
   code = filterSetDataFromSlotId(filter, &param);
   ASSERT_EQ(code, 0);
 
@@ -911,7 +891,7 @@ TEST(opTest, smallint_column_or_float_column) {
   stat.min = 1;
   stat.numOfNull = 0;
   int8_t *rowRes = NULL;
-  bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
+  bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock));
   ASSERT_EQ(keep, false);
 
   for (int32_t i = 0; i < rowNum; ++i) {
@@ -941,7 +921,7 @@ TEST(opTest, smallint_column_or_double_value) {
   ASSERT_EQ(code, 0);
 
   SColumnDataAgg stat = {0};
-  SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
+  SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock };
   code = filterSetDataFromSlotId(filter, &param);
   ASSERT_EQ(code, 0);
 
@@ -949,7 +929,7 @@ TEST(opTest, smallint_column_or_double_value) {
   stat.min = 1;
   stat.numOfNull = 0;
   int8_t *rowRes = NULL;
-  bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
+  bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock));
   ASSERT_EQ(keep, true);
 
   for (int32_t i = 0; i < rowNum; ++i) {
@@ -985,7 +965,7 @@ TEST(opTest, binary_column_is_true) {
   ASSERT_EQ(code, 0);
 
   SColumnDataAgg stat = {0};
-  SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
+  SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock };
   code = filterSetDataFromSlotId(filter, &param);
   ASSERT_EQ(code, 0);
 
@@ -993,7 +973,7 @@ TEST(opTest, binary_column_is_true) {
   stat.min = 1;
   stat.numOfNull = 0;
   int8_t *rowRes = NULL;
-  bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
+  bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock));
   ASSERT_EQ(keep, false);
 
   for (int32_t i = 0; i < rowNum; ++i) {
@@ -1056,7 +1036,7 @@ TEST(filterModelogicTest, diff_columns_and_or_and) {
   ASSERT_EQ(code, 0);
 
   SColumnDataAgg stat = {0};
-  SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
+  SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock };
   code = filterSetDataFromSlotId(filter, &param);
   ASSERT_EQ(code, 0);
 
@@ -1064,7 +1044,7 @@ TEST(filterModelogicTest, diff_columns_and_or_and) {
   stat.min = 1;
   stat.numOfNull = 0;
   int8_t *rowRes = NULL;
-  bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
+  bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock));
   ASSERT_EQ(keep, false);
 
   for (int32_t i = 0; i < rowNum; ++i) {
@@ -1125,7 +1105,7 @@ TEST(filterModelogicTest, same_column_and_or_and) {
   ASSERT_EQ(code, 0);
 
   SColumnDataAgg stat = {0};
-  SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
+  SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock };
   code = filterSetDataFromSlotId(filter, &param);
   ASSERT_EQ(code, 0);
 
@@ -1133,7 +1113,7 @@ TEST(filterModelogicTest, same_column_and_or_and) {
   stat.min = 1;
   stat.numOfNull = 0;
   int8_t *rowRes = NULL;
-  bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
+  bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock));
   ASSERT_EQ(keep, false);
 
   for (int32_t i = 0; i < rowNum; ++i) {
@@ -1194,7 +1174,7 @@ TEST(filterModelogicTest, diff_columns_or_and_or) {
   ASSERT_EQ(code, 0);
 
   SColumnDataAgg stat = {0};
-  SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
+  SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
   code = filterSetDataFromSlotId(filter, &param);
   ASSERT_EQ(code, 0);
 
@@ -1202,7 +1182,7 @@ TEST(filterModelogicTest, diff_columns_or_and_or) {
   stat.min = 1;
   stat.numOfNull = 0;
   int8_t *rowRes = NULL;
-  bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
+  bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock));
   ASSERT_EQ(keep, false);
 
   for (int32_t i = 0; i < rowNum; ++i) {
@@ -1263,7 +1243,7 @@ TEST(filterModelogicTest, same_column_or_and_or) {
   ASSERT_EQ(code, 0);
 
   SColumnDataAgg stat = {0};
-  SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
+  SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock };
   code = filterSetDataFromSlotId(filter, &param);
   ASSERT_EQ(code, 0);
 
@@ -1271,7 +1251,7 @@ TEST(filterModelogicTest, same_column_or_and_or) {
   stat.min = 1;
   stat.numOfNull = 0;
   int8_t *rowRes = NULL;
-  bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
+  bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock));
   ASSERT_EQ(keep, false);
 
   for (int32_t i = 0; i < rowNum; ++i) {
@@ -1334,7 +1314,7 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) {
   ASSERT_EQ(code, 0);
 
   SColumnDataAgg stat = {0};
-  SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
+  SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock };
   code = filterSetDataFromSlotId(filter, &param);
   ASSERT_EQ(code, 0);
 
@@ -1342,7 +1322,7 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) {
   stat.min = 1;
   stat.numOfNull = 0;
   int8_t *rowRes = NULL;
-  bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
+  bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock));
   ASSERT_EQ(keep, false);
   
   for (int32_t i = 0; i < rowNum; ++i) {

source/libs/scalar/test/scalar/scalarTests.cpp

@@ -84,30 +84,29 @@ void scltInitLogFile() {
 
 void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *slotId, bool newBlock, int32_t rows, SColumnInfo *colInfo) {
  if (newBlock) {
-   SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock));
+   SSDataBlock *res = createDataBlock();
-   res->info.numOfCols = 1;
+
-   res->info.rows = rows;
-   res->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
    SColumnInfoData idata = {0};
-   idata.info  = *colInfo;
+   idata.info = *colInfo;
+   colInfoDataEnsureCapacity(&idata, rows);
+
+   blockDataAppendColInfo(res, &idata);
+
+   res->info.capacity = rows;
+   res->info.rows = rows;
+   SColumnInfoData* p = static_cast<SColumnInfoData *>(taosArrayGet(res->pDataBlock, 0));
+   ASSERT(p->pData != NULL && p->nullbitmap != NULL);
 
-   taosArrayPush(res->pDataBlock, &idata);
    taosArrayPush(pBlockList, &res);
-
-   blockDataEnsureCapacity(res, rows);
-
    *dataBlockId = taosArrayGetSize(pBlockList) - 1;
    res->info.blockId = *dataBlockId;
    *slotId = 0;
  } else {
    SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList);
-   res->info.numOfCols++;
    SColumnInfoData idata = {0};
    idata.info  = *colInfo;
-
+   colInfoDataEnsureCapacity(&idata, rows);
-   colInfoDataEnsureCapacity(&idata, 0, rows);
+   blockDataAppendColInfo(res, &idata);
-
-   taosArrayPush(res->pDataBlock, &idata);
 
    *dataBlockId = taosArrayGetSize(pBlockList) - 1;
    *slotId = taosArrayGetSize(res->pDataBlock) - 1;
@@ -144,29 +143,19 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
  }
 
  if (NULL == *block) {
-   SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock));
+   SSDataBlock *res = createDataBlock();
-   res->info.numOfCols = 3;
-   res->info.rows = rowNum;
-   res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData));
    for (int32_t i = 0; i < 2; ++i) {
-     SColumnInfoData idata = {{0}};
+     SColumnInfoData idata = createColumnInfoData(TSDB_DATA_TYPE_INT, 10, i + 1);
-     idata.info.type  = TSDB_DATA_TYPE_NULL;
+     colInfoDataEnsureCapacity(&idata, rowNum);
-     idata.info.bytes = 10;
+     blockDataAppendColInfo(res, &idata);
-     idata.info.colId = i + 1;
-
-     int32_t size = idata.info.bytes * rowNum;
-     idata.pData = (char *)taosMemoryCalloc(1, size);
-     taosArrayPush(res->pDataBlock, &idata);
    }
 
-   SColumnInfoData idata = {{0}};
+   SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 3);
-   idata.info.type  = dataType;
+   colInfoDataEnsureCapacity(&idata, rowNum);
-   idata.info.bytes = dataBytes;
+   blockDataAppendColInfo(res, &idata);
-   idata.info.colId = 3;
+   res->info.capacity = rowNum;
-   int32_t size = idata.info.bytes * rowNum;
+
-   idata.pData = (char *)taosMemoryCalloc(1, size);
+   res->info.rows = rowNum;
-   colInfoDataEnsureCapacity(&idata, 0, rowNum);
-   taosArrayPush(res->pDataBlock, &idata);
    SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
    for (int32_t i = 0; i < rowNum; ++i) {
      colDataAppend(pColumn, i, (const char *)value, false);
@@ -185,17 +174,13 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
    SSDataBlock *res = *block;
 
    int32_t idx = taosArrayGetSize(res->pDataBlock);
-   SColumnInfoData idata = {{0}};
+   SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 1 + idx);
-   idata.info.type  = dataType;
+   colInfoDataEnsureCapacity(&idata, rowNum);
-   idata.info.bytes = dataBytes;
-   idata.info.colId = 1 + idx;
-   int32_t size = idata.info.bytes * rowNum;
-   idata.pData = (char *)taosMemoryCalloc(1, size);
-   taosArrayPush(res->pDataBlock, &idata);
-   res->info.numOfCols++;
-   SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
 
-   colInfoDataEnsureCapacity(pColumn, 0, rowNum);
+   res->info.capacity = rowNum;
+   blockDataAppendColInfo(res, &idata);
+
+   SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
 
    for (int32_t i = 0; i < rowNum; ++i) {
      colDataAppend(pColumn, i, (const char *)value, false);
@@ -279,9 +264,6 @@ void scltMakeTargetNode(SNode **pNode, int16_t dataBlockId, int16_t slotId, SNod
 
  *pNode = (SNode *)onode;
 }
-
-
-
 }
 
 TEST(constantTest, bigint_add_bigint) {
@@ -1529,6 +1511,7 @@ TEST(columnTest, bigint_column_multi_binary_column) {
 
  SArray *blockList = taosArrayInit(1, POINTER_BYTES);
  taosArrayPush(blockList, &src);
+
  SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
  int16_t dataBlockId = 0, slotId = 0;
  scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo);
@@ -2033,7 +2016,7 @@ void scltMakeDataBlock(SScalarParam **pInput, int32_t type, void *pVal, int32_t
  input->numOfRows = num;
 
  input->columnData->info = createColumnInfo(0, type, bytes);
- colInfoDataEnsureCapacity(input->columnData, 0, num);
+ colInfoDataEnsureCapacity(input->columnData, num);
 
  if (setVal) {
    for (int32_t i = 0; i < num; ++i) {

source/libs/stream/src/streamDispatch.c

@@ -96,16 +96,17 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock)
   pRetrieve = taosMemoryCalloc(1, dataStrLen);
   if (pRetrieve == NULL) return -1;
 
+  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
   pRetrieve->useconds = 0;
   pRetrieve->precision = TSDB_DEFAULT_PRECISION;
   pRetrieve->compressed = 0;
   pRetrieve->completed = 1;
   pRetrieve->streamBlockType = pBlock->info.type;
   pRetrieve->numOfRows = htonl(pBlock->info.rows);
-  pRetrieve->numOfCols = htonl(pBlock->info.numOfCols);
+  pRetrieve->numOfCols = htonl(numOfCols);
 
   int32_t actualLen = 0;
-  blockCompressEncode(pBlock, pRetrieve->data, &actualLen, pBlock->info.numOfCols, false);
+  blockCompressEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false);
 
   SStreamRetrieveReq req = {
       .streamId = pTask->streamId,
@@ -170,10 +171,12 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis
   pRetrieve->completed = 1;
   pRetrieve->streamBlockType = pBlock->info.type;
   pRetrieve->numOfRows = htonl(pBlock->info.rows);
-  pRetrieve->numOfCols = htonl(pBlock->info.numOfCols);
+
+  int32_t numOfCols = (int32_t) taosArrayGetSize(pBlock->pDataBlock);
+  pRetrieve->numOfCols = htonl(numOfCols);
 
   int32_t actualLen = 0;
-  blockCompressEncode(pBlock, pRetrieve->data, &actualLen, pBlock->info.numOfCols, false);
+  blockCompressEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false);
   actualLen += sizeof(SRetrieveTableRsp);
   ASSERT(actualLen <= dataStrLen);
   taosArrayPush(pReq->dataLen, &actualLen);

source/libs/stream/src/streamExec.c

@@ -80,7 +80,6 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) {
       }
       qRes->type = STREAM_INPUT__DATA_BLOCK;
       qRes->blocks = pRes;
-      /*qRes->sourceVg = pTask->nodeId;*/
       if (streamTaskOutput(pTask, qRes) < 0) {
         streamQueueProcessFail(pTask->inputQueue);
         taosArrayDestroy(pRes);

source/libs/transport/src/transCli.c

@@ -983,17 +983,18 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
     if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
       if (pCtx->retryCount < pEpSet->numOfEps * 3) {
         pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps;
+        if (pThrd->quit == false) {
+          STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg));
+          arg->param1 = pMsg;
+          arg->param2 = pThrd;
+          transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL);
+          transPrintEpSet(pEpSet);
+          tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse,
+                 pCtx->retryCount + 1, pEpSet->numOfEps * 3);
 
-        STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg));
+          transUnrefCliHandle(pConn);
-        arg->param1 = pMsg;
+          return -1;
-        arg->param2 = pThrd;
+        }
-        transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL);
-        transPrintEpSet(pEpSet);
-        tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse,
-               pCtx->retryCount + 1, pEpSet->numOfEps * 3);
-
-        transUnrefCliHandle(pConn);
-        return -1;
       }
     } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) {
       if (pResp->contLen == 0) {
@@ -1010,15 +1011,16 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
         tTrace("%s use remote epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse,
                pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT);
       }
-      if (pConn->status != ConnInPool) {
+      if (pThrd->quit == false) {
-        addConnToPool(pThrd->pool, pConn);
+        if (pConn->status != ConnInPool) {
+          addConnToPool(pThrd->pool, pConn);
+        }
+        STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg));
+        arg->param1 = pMsg;
+        arg->param2 = pThrd;
+        transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL);
+        return -1;
       }
-
-      STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg));
-      arg->param1 = pMsg;
-      arg->param2 = pThrd;
-      transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL);
-      return -1;
     }
   }
 

source/util/src/terror.c

@@ -76,6 +76,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_NEED_RETRY,                   "Retry needed")
 TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE,      "Out of memory in rpc queue")
 TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TIMESTAMP,            "Invalid timestamp format")
 TAOS_DEFINE_ERROR(TSDB_CODE_MSG_DECODE_ERROR,             "Msg decode error")
+TAOS_DEFINE_ERROR(TSDB_CODE_NO_AVAIL_DISK,                "No available disk")
 
 TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY,                "Ref out of memory")
 TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL,                     "too many Ref Objs")
@@ -356,7 +357,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM,     "No table data in memo
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_ALREADY_EXISTS,      "File already exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECONFIGURE,        "Need to reconfigure table")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO,    "Invalid information to create table")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_AVAIL_DISK,            "No available disk")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_AVAIL_DISK,            "TSDB no available disk")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_MESSED_MSG,               "TSDB messed message")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVLD_TAG_VAL,             "TSDB invalid tag value")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_CACHE_LAST_ROW,        "TSDB no cache last row data")

source/util/src/tlog.c

@@ -743,7 +743,7 @@ cmp_end:
 }
 
 void taosSetAllDebugFlag(int32_t flag) {
-  if (!(flag & DEBUG_TRACE || flag & DEBUG_DEBUG || flag & DEBUG_DUMP)) return;
+  if (flag <= 0) return;
 
   dDebugFlag = flag;
   vDebugFlag = flag;

tests/script/general/parser/udf.sim

@@ -44,6 +44,7 @@ endi
 sql create function n02 as '/tmp/normal' outputtype bool;
 sql show functions;
 if $rows != 1 then
+  print expect 1, actual $rows
   return -1
 endi
 

tests/script/jenkins/basic.txt

@@ -109,6 +109,7 @@
 ./test.sh -f tsim/tmq/basic4Of2Cons.sim
 ./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim
 ./test.sh -f tsim/tmq/topic.sim
+./test.sh -f tsim/tmq/snapshot.sim
 
 # --- stable
 ./test.sh -f tsim/stable/disk.sim

tests/script/tsim/query/udf.sim

@@ -44,6 +44,7 @@ endi
 
 sql select udf2(f) from t;
 if $rows != 1 then
+  print expect 1, actual $rows
   return -1
 endi
 if $data00 != 2.236067977 then

tests/script/tsim/show/basic.sim

@@ -233,12 +233,12 @@ if $rows != 4 then
 endi
 
 sql show dnode 1 variables;
-if $rows != 114 then
+if $rows <= 0 then
   return -1
 endi
 
 sql show local variables;
-if $rows != 50 then
+if $rows <= 0 then
   return -1
 endi
 

tests/script/tsim/tmq/consume.sh

@@ -17,8 +17,9 @@ VALGRIND=0
 SIGNAL=SIGINT
 SHOW_MSG=0
 SHOW_ROW=0
+EXP_USE_SNAPSHOT=0
 
-while getopts "d:s:v:y:x:g:r:w:" arg
+while getopts "d:s:v:y:x:g:r:w:e:" arg
 do
   case $arg in
     d)
@@ -45,6 +46,9 @@ do
     w)
       CDB_NAME=$OPTARG
       ;;
+    e)
+      EXP_USE_SNAPSHOT=$OPTARG
+      ;;
     ?)
       echo "unkown argument"
       ;;
@@ -91,8 +95,8 @@ if [ "$EXEC_OPTON" = "start" ]; then
     echo nohup valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tmq_sim.log $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW > /dev/null 2>&1 &
     nohup valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tmq_sim.log $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW > /dev/null 2>&1 &
   else
-    echo  "nohup $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW -w $CDB_NAME > /dev/null 2>&1 &"
+    echo  "nohup $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW -w $CDB_NAME -e $EXP_USE_SNAPSHOT > /dev/null 2>&1 &"
-    nohup $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW -w $CDB_NAME > /dev/null 2>&1 &
+    nohup $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW -w $CDB_NAME -e $EXP_USE_SNAPSHOT > /dev/null 2>&1 &
   fi
 else
   PID=`ps -ef|grep tmq_sim | grep -v grep | awk '{print $2}'`

tests/script/tsim/tmq/snapshot.sim

@@ -0,0 +1,289 @@
+#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
+#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics
+#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics
+#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics
+#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics
+
+# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
+# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
+#
+# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
+#
+
+run tsim/tmq/prepareBasicEnv-1vgrp.sim
+
+#---- global parameters start ----#
+$dbName    = db
+$vgroups    = 1
+$stbPrefix  = stb
+$ctbPrefix  = ctb
+$ntbPrefix  = ntb
+$stbNum     = 1
+$ctbNum     = 10
+$ntbNum     = 10
+$rowsPerCtb = 10
+$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
+#---- global parameters end ----#
+
+$pullDelay    = 3
+$ifcheckdata  = 1
+$ifmanualcommit = 1
+$showMsg      = 1
+$showRow      = 0
+
+sql connect
+sql use $dbName
+
+print == create topics from super table
+sql create topic topic_stb_column as select ts, c3 from stb
+sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
+sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
+
+print == create topics from child table
+sql create topic topic_ctb_column as select ts, c3 from ctb0
+sql create topic topic_ctb_all as select * from ctb0
+sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
+
+print == create topics from normal table
+sql create topic topic_ntb_column as select ts, c3 from ntb0
+sql create topic topic_ntb_all as select * from ntb0
+sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
+
+#sql show topics
+#if $rows != 9 then 
+#  return -1
+#endi
+
+#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest'
+$keyList = ' . group.id:cgrp1
+$keyList = $keyList . ,
+$keyList = $keyList . enable.auto.commit:false
+#$keyList = $keyList . ,
+#$keyList = $keyList . auto.commit.interval.ms:6000
+#$keyList = $keyList . ,
+#$keyList = $keyList . auto.offset.reset:earliest
+$keyList = $keyList . '
+print ========== key list:  $keyList
+
+
+$cdb_index = 0
+#=============================== start consume =============================#
+
+print ================ test consume from stb
+$loop_cnt = 0
+loop_consume_diff_topic_from_stb:
+
+#######################################################################################
+# clear consume info and consume result 
+#run tsim/tmq/clearConsume.sim
+# because drop table function no stable, so by create new db for consume info and result. Modify it later
+$cdb_index = $cdb_index + 1
+$cdbName = cdb . $cdb_index
+sql create database $cdbName vgroups 1
+sleep 500
+sql use $cdbName
+
+print == create consume info table and consume result table
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+
+sql show tables
+if $rows != 2 then 
+  return -1
+endi
+#######################################################################################
+
+if $loop_cnt == 0 then
+  print == scenario 1: topic_stb_column
+  $topicList = ' . topic_stb_column
+  $topicList = $topicList . '
+elif $loop_cnt == 1 then
+  print == scenario 2: topic_stb_all
+  $topicList = ' . topic_stb_all
+  $topicList = $topicList . '
+elif $loop_cnt == 2 then
+  print == scenario 3: topic_stb_function
+  $topicList = ' . topic_stb_function
+  $topicList = $topicList . '
+else
+  goto loop_consume_diff_topic_from_stb_end
+endi
+
+$consumerId    = 0
+$totalMsgOfStb = $ctbNum * $rowsPerCtb
+$expectmsgcnt  = 1
+$expectrowcnt  = 100
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
+
+print == start consumer to pull msgs from stb
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -e 1 -s start
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -e 1 -s start
+
+print == check consume result
+wait_consumer_end_from_stb:
+sql select * from consumeresult
+print ==> rows: $rows 
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+if $rows != 1 then
+  sleep 1000
+  goto wait_consumer_end_from_stb
+endi
+if $data[0][1] != $consumerId then
+  return -1
+endi
+if $data[0][2] != $expectmsgcnt then
+  return -1
+endi
+if $data[0][3] != $expectrowcnt then
+  return -1
+endi
+$loop_cnt = $loop_cnt + 1
+goto loop_consume_diff_topic_from_stb
+loop_consume_diff_topic_from_stb_end:
+
+print ================ test consume from ctb
+$loop_cnt = 0
+loop_consume_diff_topic_from_ctb:
+
+#######################################################################################
+# clear consume info and consume result 
+#run tsim/tmq/clearConsume.sim
+# because drop table function no stable, so by create new db for consume info and result. Modify it later
+$cdb_index = $cdb_index + 1
+$cdbName = cdb . $cdb_index
+sql create database $cdbName vgroups 1
+sleep 500
+sql use $cdbName
+
+print == create consume info table and consume result table
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+
+sql show tables
+if $rows != 2 then 
+  return -1
+endi
+#######################################################################################
+
+if $loop_cnt == 0 then
+  print == scenario 1: topic_ctb_column
+  $topicList = ' . topic_ctb_column
+  $topicList = $topicList . '
+elif $loop_cnt == 1 then
+  print == scenario 2: topic_ctb_all
+  $topicList = ' . topic_ctb_all
+  $topicList = $topicList . '
+elif $loop_cnt == 2 then
+  print == scenario 3: topic_ctb_function
+  $topicList = ' . topic_ctb_function
+  $topicList = $topicList . '
+else
+  goto loop_consume_diff_topic_from_ctb_end
+endi
+
+$consumerId    = 0
+$totalMsgOfCtb = $rowsPerCtb
+$expectmsgcnt  = 1
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
+
+print == start consumer to pull msgs from ctb
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -e 1
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start -e 1
+
+print == check consume result
+wait_consumer_end_from_ctb:
+sql select * from consumeresult
+print ==> rows: $rows 
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+if $rows != 1 then
+  sleep 1000
+  goto wait_consumer_end_from_ctb
+endi
+if $data[0][1] != $consumerId then
+  return -1
+endi
+if $data[0][2] != 1 then
+  return -1
+endi
+if $data[0][3] != 10 then
+  return -1
+endi
+$loop_cnt = $loop_cnt + 1
+goto loop_consume_diff_topic_from_ctb
+loop_consume_diff_topic_from_ctb_end:
+
+print ================ test consume from ntb
+$loop_cnt = 0
+loop_consume_diff_topic_from_ntb:
+
+#######################################################################################
+# clear consume info and consume result 
+#run tsim/tmq/clearConsume.sim
+# because drop table function no stable, so by create new db for consume info and result. Modify it later
+$cdb_index = $cdb_index + 1
+$cdbName = cdb . $cdb_index
+sql create database $cdbName vgroups 1
+sleep 500
+sql use $cdbName
+
+print == create consume info table and consume result table
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+
+sql show tables
+if $rows != 2 then 
+  return -1
+endi
+#######################################################################################
+
+if $loop_cnt == 0 then
+  print == scenario 1: topic_ntb_column
+  $topicList = ' . topic_ntb_column
+  $topicList = $topicList . '
+elif $loop_cnt == 1 then
+  print == scenario 2: topic_ntb_all
+  $topicList = ' . topic_ntb_all
+  $topicList = $topicList . '
+elif $loop_cnt == 2 then
+  print == scenario 3: topic_ntb_function
+  $topicList = ' . topic_ntb_function
+  $topicList = $topicList . '
+else
+  goto loop_consume_diff_topic_from_ntb_end
+endi
+
+$consumerId    = 0
+$totalMsgOfNtb = $rowsPerCtb
+$expectmsgcnt  = $totalMsgOfNtb
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
+
+print == start consumer to pull msgs from ntb
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -e 1
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start -e 1
+
+print == check consume result from ntb
+wait_consumer_end_from_ntb:
+sql select * from consumeresult
+print ==> rows: $rows 
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+if $rows != 1 then
+  sleep 1000
+  goto wait_consumer_end_from_ntb
+endi
+if $data[0][1] != $consumerId then
+  return -1
+endi
+if $data[0][2] != 1 then
+  return -1
+endi
+if $data[0][3] != $totalMsgOfNtb then
+  return -1
+endi
+$loop_cnt = $loop_cnt + 1
+goto loop_consume_diff_topic_from_ntb
+loop_consume_diff_topic_from_ntb_end:
+
+#------ not need stop consumer, because it exit after pull msg overthan expect msg
+#system tsim/tmq/consume.sh -s stop -x SIGINT
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/test/c/tmqSim.c

@@ -22,9 +22,9 @@
 #include <time.h>
 
 #include "taos.h"
+#include "taosdef.h"
 #include "taoserror.h"
 #include "tlog.h"
-#include "taosdef.h"
 #include "types.h"
 
 #define GREEN     "\033[1;32m"
@@ -36,11 +36,7 @@
 #define MAX_CONSUMER_THREAD_CNT (16)
 #define MAX_VGROUP_CNT          (32)
 
-typedef enum {
+typedef enum { NOTIFY_CMD_START_CONSUM, NOTIFY_CMD_START_COMMIT, NOTIFY_CMD_ID_BUTT } NOTIFY_CMD_ID;
-    NOTIFY_CMD_START_CONSUM,
-	NOTIFY_CMD_START_COMMIT,
-	NOTIFY_CMD_ID_BUTT
-}NOTIFY_CMD_ID;
 
 typedef struct {
   TdThread thread;
@@ -52,8 +48,8 @@ typedef struct {
   // char     autoOffsetRest[16];    // none, earliest, latest
 
   TdFilePtr pConsumeRowsFile;
-  int32_t ifCheckData;
+  int32_t   ifCheckData;
-  int64_t expectMsgCnt;
+  int64_t   expectMsgCnt;
 
   int64_t consumeMsgCnt;
   int64_t consumeRowCnt;
@@ -89,6 +85,7 @@ typedef struct {
   int32_t     saveRowFlag;
   int32_t     consumeDelay;  // unit s
   int32_t     numOfThread;
+  int32_t     useSnapshot;
   SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT];
 } SConfInfo;
 
@@ -96,6 +93,8 @@ static SConfInfo g_stConfInfo;
 TdFilePtr        g_fp = NULL;
 static int       running = 1;
 
+int8_t useSnapshot = 0;
+
 // char* g_pRowValue = NULL;
 // TdFilePtr g_fp = NULL;
 
@@ -205,6 +204,8 @@ void parseArgument(int32_t argc, char* argv[]) {
       g_stConfInfo.saveRowFlag = atol(argv[++i]);
     } else if (strcmp(argv[i], "-y") == 0) {
       g_stConfInfo.consumeDelay = atol(argv[++i]);
+    } else if (strcmp(argv[i], "-e") == 0) {
+      useSnapshot = (int8_t)atol(argv[++i]);
     } else {
       pError("%s unknow para: %s %s", GREEN, argv[++i], NC);
       exit(-1);
@@ -298,11 +299,11 @@ int32_t saveConsumeContentToTbl(SThreadInfo* pInfo, char* buf) {
   return 0;
 }
 
-static char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) {
+static char* shellFormatTimestamp(char* buf, int64_t val, int32_t precision) {
-  //if (shell.args.is_raw_time) {
+  // if (shell.args.is_raw_time) {
-  //  sprintf(buf, "%" PRId64, val);
+  //   sprintf(buf, "%" PRId64, val);
-  //  return buf;
+  //   return buf;
-  //}
+  // }
 
   time_t  tt;
   int32_t ms = 0;
@@ -340,7 +341,7 @@ static char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) {
     }
   }
 
-  struct tm *ptm = taosLocalTime(&tt, NULL);
+  struct tm* ptm = taosLocalTime(&tt, NULL);
   size_t     pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm);
 
   if (precision == TSDB_TIME_PRECISION_NANO) {
@@ -354,7 +355,8 @@ static char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) {
   return buf;
 }
 
-static void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, int32_t length, int32_t precision) {
+static void shellDumpFieldToFile(TdFilePtr pFile, const char* val, TAOS_FIELD* field, int32_t length,
+                                 int32_t precision) {
   if (val == NULL) {
     taosFprintfFile(pFile, "%s", TSDB_DATA_NULL_STR);
     return;
@@ -364,31 +366,31 @@ static void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *f
   char buf[TSDB_MAX_BYTES_PER_ROW];
   switch (field->type) {
     case TSDB_DATA_TYPE_BOOL:
-      taosFprintfFile(pFile, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
+      taosFprintfFile(pFile, "%d", ((((int32_t)(*((char*)val))) == 1) ? 1 : 0));
       break;
     case TSDB_DATA_TYPE_TINYINT:
-      taosFprintfFile(pFile, "%d", *((int8_t *)val));
+      taosFprintfFile(pFile, "%d", *((int8_t*)val));
       break;
     case TSDB_DATA_TYPE_UTINYINT:
-      taosFprintfFile(pFile, "%u", *((uint8_t *)val));
+      taosFprintfFile(pFile, "%u", *((uint8_t*)val));
       break;
     case TSDB_DATA_TYPE_SMALLINT:
-      taosFprintfFile(pFile, "%d", *((int16_t *)val));
+      taosFprintfFile(pFile, "%d", *((int16_t*)val));
       break;
     case TSDB_DATA_TYPE_USMALLINT:
-      taosFprintfFile(pFile, "%u", *((uint16_t *)val));
+      taosFprintfFile(pFile, "%u", *((uint16_t*)val));
       break;
     case TSDB_DATA_TYPE_INT:
-      taosFprintfFile(pFile, "%d", *((int32_t *)val));
+      taosFprintfFile(pFile, "%d", *((int32_t*)val));
       break;
     case TSDB_DATA_TYPE_UINT:
-      taosFprintfFile(pFile, "%u", *((uint32_t *)val));
+      taosFprintfFile(pFile, "%u", *((uint32_t*)val));
       break;
     case TSDB_DATA_TYPE_BIGINT:
-      taosFprintfFile(pFile, "%" PRId64, *((int64_t *)val));
+      taosFprintfFile(pFile, "%" PRId64, *((int64_t*)val));
       break;
     case TSDB_DATA_TYPE_UBIGINT:
-      taosFprintfFile(pFile, "%" PRIu64, *((uint64_t *)val));
+      taosFprintfFile(pFile, "%" PRIu64, *((uint64_t*)val));
       break;
     case TSDB_DATA_TYPE_FLOAT:
       taosFprintfFile(pFile, "%.5f", GET_FLOAT_VAL(val));
@@ -409,7 +411,7 @@ static void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *f
       taosFprintfFile(pFile, "\'%s\'", buf);
       break;
     case TSDB_DATA_TYPE_TIMESTAMP:
-      shellFormatTimestamp(buf, *(int64_t *)val, precision);
+      shellFormatTimestamp(buf, *(int64_t*)val, precision);
       taosFprintfFile(pFile, "'%s'", buf);
       break;
     default:
@@ -417,12 +419,13 @@ static void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *f
   }
 }
 
-static void dumpToFileForCheck(TdFilePtr pFile, TAOS_ROW row, TAOS_FIELD* fields, int32_t* length, int32_t num_fields, int32_t precision) {
+static void dumpToFileForCheck(TdFilePtr pFile, TAOS_ROW row, TAOS_FIELD* fields, int32_t* length, int32_t num_fields,
+                               int32_t precision) {
   for (int32_t i = 0; i < num_fields; i++) {
     if (i > 0) {
       taosFprintfFile(pFile, "\n");
     }
-    shellDumpFieldToFile(pFile, (const char *)row[i], fields + i, length[i], precision);
+    shellDumpFieldToFile(pFile, (const char*)row[i], fields + i, length[i], precision);
   }
   taosFprintfFile(pFile, "\n");
 }
@@ -432,31 +435,32 @@ static int32_t msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex)
   int32_t totalRows = 0;
 
   // printf("topic: %s\n", tmq_get_topic_name(msg));
-  int32_t vgroupId = tmq_get_vgroup_id(msg);
+  int32_t     vgroupId = tmq_get_vgroup_id(msg);
-  const char*   dbName = tmq_get_db_name(msg);
+  const char* dbName = tmq_get_db_name(msg);
 
   taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex);
-  taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", tmq_get_topic_name(msg), vgroupId);
+  taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table",
+                  tmq_get_topic_name(msg), vgroupId);
 
   while (1) {
     TAOS_ROW row = taos_fetch_row(msg);
 
     if (row == NULL) break;
 
-    TAOS_FIELD* fields      = taos_fetch_fields(msg);
+    TAOS_FIELD* fields = taos_fetch_fields(msg);
     int32_t     numOfFields = taos_field_count(msg);
-	int32_t*    length      = taos_fetch_lengths(msg);
+    int32_t*    length = taos_fetch_lengths(msg);
-	int32_t     precision   = taos_result_precision(msg);
+    int32_t     precision = taos_result_precision(msg);
-    const char* tbName      = tmq_get_table_name(msg);
+    const char* tbName = tmq_get_table_name(msg);
 
-	dumpToFileForCheck(pInfo->pConsumeRowsFile, row, fields, length, numOfFields, precision);
+    dumpToFileForCheck(pInfo->pConsumeRowsFile, row, fields, length, numOfFields, precision);
     taos_print_row(buf, row, fields, numOfFields);
 
     if (0 != g_stConfInfo.showRowFlag) {
       taosFprintfFile(g_fp, "tbname:%s, rows[%d]: %s\n", (tbName != NULL ? tbName : "null table"), totalRows, buf);
-      //if (0 != g_stConfInfo.saveRowFlag) {
+      // if (0 != g_stConfInfo.saveRowFlag) {
-      //  saveConsumeContentToTbl(pInfo, buf);
+      //   saveConsumeContentToTbl(pInfo, buf);
-      //}
+      // }
     }
 
     totalRows++;
@@ -479,8 +483,7 @@ int queryDB(TAOS* taos, char* command) {
   return 0;
 }
 
-static void appNothing(void* param, TAOS_RES* res, int32_t numOfRows) {  
+static void appNothing(void* param, TAOS_RES* res, int32_t numOfRows) {}
-}
 
 int32_t notifyMainScript(SThreadInfo* pInfo, int32_t cmdId) {
   char sqlStr[1024] = {0};
@@ -488,11 +491,8 @@ int32_t notifyMainScript(SThreadInfo* pInfo, int32_t cmdId) {
   int64_t now = taosGetTimestampMs();
 
   // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int
-  sprintf(sqlStr, "insert into %s.notifyinfo values (%"PRId64", %d, %d)", 
+  sprintf(sqlStr, "insert into %s.notifyinfo values (%" PRId64 ", %d, %d)", g_stConfInfo.cdbName, now, cmdId,
-                   g_stConfInfo.cdbName, 
+          pInfo->consumerId);
-                   now,
-                   cmdId,
-                   pInfo->consumerId);
 
   taos_query_a(pInfo->taos, sqlStr, appNothing, NULL);
 
@@ -502,12 +502,12 @@ int32_t notifyMainScript(SThreadInfo* pInfo, int32_t cmdId) {
 }
 
 static int32_t g_once_commit_flag = 0;
-static void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) {
+static void    tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) {
   pError("tmq_commit_cb_print() commit %d\n", code);
 
   if (0 == g_once_commit_flag) {
     g_once_commit_flag = 1;
-	notifyMainScript((SThreadInfo*)param, (int32_t)NOTIFY_CMD_START_COMMIT);
+    notifyMainScript((SThreadInfo*)param, (int32_t)NOTIFY_CMD_START_COMMIT);
   }
   taosFprintfFile(g_fp, "tmq_commit_cb_print() be called\n");
 }
@@ -541,6 +541,10 @@ void build_consumer(SThreadInfo* pInfo) {
   // tmq_conf_set(conf, "auto.offset.reset", "none");
   // tmq_conf_set(conf, "auto.offset.reset", "earliest");
   // tmq_conf_set(conf, "auto.offset.reset", "latest");
+  //
+  if (useSnapshot) {
+    tmq_conf_set(conf, "experiment.use.snapshot", "true");
+  }
 
   pInfo->tmq = tmq_consumer_new(conf, NULL, 0);
 
@@ -597,10 +601,11 @@ void loop_consume(SThreadInfo* pInfo) {
   pInfo->ts = taosGetTimestampMs();
 
   if (pInfo->ifCheckData) {
-  	char filename[256] = {0};
+    char filename[256] = {0};
     char tmpString[128];
-	//sprintf(filename, "%s/../log/consumerid_%d_%s.txt", configDir, pInfo->consumerId, getCurrentTimeString(tmpString));
+    // sprintf(filename, "%s/../log/consumerid_%d_%s.txt", configDir, pInfo->consumerId,
-	sprintf(filename, "%s/../log/consumerid_%d.txt", configDir, pInfo->consumerId);
+    // getCurrentTimeString(tmpString));
+    sprintf(filename, "%s/../log/consumerid_%d.txt", configDir, pInfo->consumerId);
     pInfo->pConsumeRowsFile = taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM);
     if (pInfo->pConsumeRowsFile == NULL) {
       taosFprintfFile(g_fp, "%s create file fail for save rows content\n", getCurrentTimeString(tmpString));
@@ -619,10 +624,10 @@ void loop_consume(SThreadInfo* pInfo) {
 
       totalMsgs++;
 
-	  if (0 == once_flag) {
+      if (0 == once_flag) {
         once_flag = 1;
-		notifyMainScript(pInfo, NOTIFY_CMD_START_CONSUM);
+        notifyMainScript(pInfo, NOTIFY_CMD_START_CONSUM);
-	  }
+      }
 
       if (totalRows >= pInfo->expectMsgCnt) {
         char tmpString[128];
@@ -651,7 +656,7 @@ void* consumeThreadFunc(void* param) {
   pInfo->taos = taos_connect(NULL, "root", "taosdata", NULL, 0);
   if (pInfo->taos == NULL) {
     taosFprintfFile(g_fp, "taos_connect() fail, can not notify and save consume result to main scripte\n");
-	exit(-1);
+    exit(-1);
   }
 
   build_consumer(pInfo);