Merge branch '3.0' into feature/queryredirect

examples/c/stream_demo.c

@@ -98,9 +98,10 @@ int32_t create_stream() {
   /*const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";*/
   /*const char* sql = "select sum(k) from tu1 interval(10m)";*/
   /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/
-  pRes = taos_query(pConn,
+  pRes = taos_query(
-                    "create stream stream1 trigger at_once into outstb as select _wstartts, sum(k) from st1 partition "
+      pConn,
-                    "by tbname interval(10s) ");
+      "create stream stream1 trigger max_delay 10s into outstb as select _wstartts, sum(k) from st1 partition "
+      "by tbname session(ts, 10s) ");
   if (taos_errno(pRes) != 0) {
     printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes));
     return -1;

examples/c/tmq.c

@@ -137,8 +137,8 @@ int32_t create_topic() {
   }
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1");
+  /*pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1");*/
-  /*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");*/
+  pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");
   if (taos_errno(pRes) != 0) {
     printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes));
     return -1;
@@ -225,7 +225,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
   }
   int32_t cnt = 0;
   while (running) {
-    TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 0);
+    TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, -1);
     if (tmqmessage) {
       cnt++;
       msg_process(tmqmessage);

include/common/tcommon.h

@@ -25,10 +25,11 @@
 extern "C" {
 #endif
 
+// TODO remove it
 enum {
-  TMQ_CONF__RESET_OFFSET__LATEST = -1,
-  TMQ_CONF__RESET_OFFSET__EARLIEAST = -2,
   TMQ_CONF__RESET_OFFSET__NONE = -3,
+  TMQ_CONF__RESET_OFFSET__EARLIEAST = -2,
+  TMQ_CONF__RESET_OFFSET__LATEST = -1,
 };
 
 enum {
@@ -39,6 +40,16 @@ enum {
   TMQ_MSG_TYPE__END_RSP,
 };
 
+enum {
+  STREAM_INPUT__DATA_SUBMIT = 1,
+  STREAM_INPUT__DATA_BLOCK,
+  STREAM_INPUT__DATA_SCAN,
+  STREAM_INPUT__DATA_RETRIEVE,
+  STREAM_INPUT__TRIGGER,
+  STREAM_INPUT__CHECKPOINT,
+  STREAM_INPUT__DROP,
+};
+
 typedef enum EStreamType {
   STREAM_NORMAL = 1,
   STREAM_INVERT,
@@ -47,8 +58,8 @@ typedef enum EStreamType {
   STREAM_GET_ALL,
   STREAM_DELETE,
   STREAM_RETRIEVE,
-  STREAM_PUSH_DATA,
+  STREAM_PULL_DATA,
-  STREAM_PUSH_EMPTY,
+  STREAM_PULL_OVER,
 } EStreamType;
 
 typedef struct {

include/common/tdatablock.h

@@ -232,9 +232,8 @@ int32_t      blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColIn
 SColumnInfoData  createColumnInfoData(int16_t type, int32_t bytes, int16_t colId);
 SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index);
 
-void        blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols,
+void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress);
-                                int8_t needCompress);
+const char* blockDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData);
-const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData);
 
 void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag);
 // for debug

include/common/tmsg.h

@@ -623,6 +623,7 @@ typedef struct {
     col_id_t colId;
     int16_t  slotId;
   };
+  bool output;  // TODO remove it later
 
   int16_t type;
   int32_t bytes;
@@ -2467,21 +2468,36 @@ int32_t tDecodeSMqCMCommitOffsetReq(SDecoder* decoder, SMqCMCommitOffsetReq* pRe
 
 // tqOffset
 enum {
-  TMQ_OFFSET__SNAPSHOT = 1,
+  TMQ_OFFSET__RESET_NONE = -3,
-  TMQ_OFFSET__LOG,
+  TMQ_OFFSET__RESET_EARLIEAST = -2,
+  TMQ_OFFSET__RESET_LATEST = -1,
+  TMQ_OFFSET__LOG = 1,
+  TMQ_OFFSET__SNAPSHOT_DATA = 2,
+  TMQ_OFFSET__SNAPSHOT_META = 3,
 };
 
 typedef struct {
   int8_t type;
   union {
+    // snapshot data
     struct {
       int64_t uid;
       int64_t ts;
     };
+    // log
     struct {
       int64_t version;
     };
   };
+} STqOffsetVal;
+
+int32_t tEncodeSTqOffsetVal(SEncoder* pEncoder, const STqOffsetVal* pOffsetVal);
+int32_t tDecodeSTqOffsetVal(SDecoder* pDecoder, STqOffsetVal* pOffsetVal);
+int32_t tFormatOffset(char* buf, int32_t maxLen, const STqOffsetVal* pVal);
+bool    tOffsetEqual(const STqOffsetVal* pLeft, const STqOffsetVal* pRight);
+
+typedef struct {
+  STqOffsetVal val;
   char         subKey[TSDB_SUBSCRIBE_KEY_LEN];
 } STqOffset;
 
@@ -2715,7 +2731,8 @@ typedef struct {
   uint64_t reqId;
   int64_t  consumerId;
   int64_t  timeout;
-  int64_t  currentOffset;
+  // int64_t      currentOffset;
+  STqOffsetVal reqOffset;
 } SMqPollReq;
 
 typedef struct {
@@ -2787,6 +2804,8 @@ typedef struct {
   SMqRspHead   head;
   int64_t      reqOffset;
   int64_t      rspOffset;
+  STqOffsetVal reqOffsetNew;
+  STqOffsetVal rspOffsetNew;
   int16_t      resMsgType;
   int32_t      metaRspLen;
   void*        metaRsp;
@@ -2811,6 +2830,24 @@ static FORCE_INLINE void* tDecodeSMqMetaRsp(const void* buf, SMqMetaRsp* pRsp) {
   return (void*)buf;
 }
 
+typedef struct {
+  SMqRspHead   head;
+  STqOffsetVal reqOffset;
+  STqOffsetVal rspOffset;
+  int32_t      skipLogNum;
+  int32_t      blockNum;
+  int8_t       withTbName;
+  int8_t       withSchema;
+  SArray*      blockDataLen;
+  SArray*      blockData;
+  SArray*      blockTbName;
+  SArray*      blockSchema;
+} SMqDataRsp;
+
+int32_t tEncodeSMqDataRsp(SEncoder* pEncoder, const SMqDataRsp* pRsp);
+int32_t tDecodeSMqDataRsp(SDecoder* pDecoder, SMqDataRsp* pRsp);
+
+#if 0
 typedef struct {
   SMqRspHead head;
   int64_t    reqOffset;
@@ -2819,13 +2856,10 @@ typedef struct {
   int32_t    blockNum;
   int8_t     withTbName;
   int8_t     withSchema;
-  int8_t     withTag;
   SArray*    blockDataLen;  // SArray<int32_t>
   SArray*    blockData;     // SArray<SRetrieveTableRsp*>
   SArray*    blockTbName;   // SArray<char*>
   SArray*    blockSchema;   // SArray<SSchemaWrapper>
-  SArray*    blockTags;       // SArray<kvrow>
-  SArray*    blockTagSchema;  // SArray<kvrow>
 } SMqDataBlkRsp;
 
 static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp* pRsp) {
@@ -2837,7 +2871,6 @@ static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp
   if (pRsp->blockNum != 0) {
     tlen += taosEncodeFixedI8(buf, pRsp->withTbName);
     tlen += taosEncodeFixedI8(buf, pRsp->withSchema);
-    tlen += taosEncodeFixedI8(buf, pRsp->withTag);
 
     for (int32_t i = 0; i < pRsp->blockNum; i++) {
       int32_t bLen = *(int32_t*)taosArrayGet(pRsp->blockDataLen, i);
@@ -2867,7 +2900,6 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p
     pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(int32_t));
     buf = taosDecodeFixedI8(buf, &pRsp->withTbName);
     buf = taosDecodeFixedI8(buf, &pRsp->withSchema);
-    buf = taosDecodeFixedI8(buf, &pRsp->withTag);
     if (pRsp->withTbName) {
       pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void*));
     }
@@ -2896,6 +2928,7 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p
   }
   return (void*)buf;
 }
+#endif
 
 typedef struct {
   SMqRspHead head;

include/common/ttokendef.h

@@ -216,51 +216,55 @@
 #define TK_NOW            198
 #define TK_TODAY          199
 #define TK_TIMEZONE       200
-#define TK_COUNT         201
+#define TK_CLIENT_VERSION 201
-#define TK_LAST_ROW      202
+#define TK_SERVER_VERSION 202
-#define TK_BETWEEN       203
+#define TK_SERVER_STATUS  203
-#define TK_IS            204
+#define TK_CURRENT_USER   204
-#define TK_NK_LT         205
+#define TK_COUNT          205
-#define TK_NK_GT         206
+#define TK_LAST_ROW       206
-#define TK_NK_LE         207
+#define TK_BETWEEN        207
-#define TK_NK_GE         208
+#define TK_IS             208
-#define TK_NK_NE         209
+#define TK_NK_LT          209
-#define TK_MATCH         210
+#define TK_NK_GT          210
-#define TK_NMATCH        211
+#define TK_NK_LE          211
-#define TK_CONTAINS      212
+#define TK_NK_GE          212
-#define TK_JOIN          213
+#define TK_NK_NE          213
-#define TK_INNER         214
+#define TK_MATCH          214
-#define TK_SELECT        215
+#define TK_NMATCH         215
-#define TK_DISTINCT      216
+#define TK_CONTAINS       216
-#define TK_WHERE         217
+#define TK_JOIN           217
-#define TK_PARTITION     218
+#define TK_INNER          218
-#define TK_BY            219
+#define TK_SELECT         219
-#define TK_SESSION       220
+#define TK_DISTINCT       220
-#define TK_STATE_WINDOW  221
+#define TK_WHERE          221
-#define TK_SLIDING       222
+#define TK_PARTITION      222
-#define TK_FILL          223
+#define TK_BY             223
-#define TK_VALUE         224
+#define TK_SESSION        224
-#define TK_NONE          225
+#define TK_STATE_WINDOW   225
-#define TK_PREV          226
+#define TK_SLIDING        226
-#define TK_LINEAR        227
+#define TK_FILL           227
-#define TK_NEXT          228
+#define TK_VALUE          228
-#define TK_HAVING        229
+#define TK_NONE           229
-#define TK_RANGE         230
+#define TK_PREV           230
-#define TK_EVERY         231
+#define TK_LINEAR         231
-#define TK_ORDER         232
+#define TK_NEXT           232
-#define TK_SLIMIT        233
+#define TK_HAVING         233
-#define TK_SOFFSET       234
+#define TK_RANGE          234
-#define TK_LIMIT         235
+#define TK_EVERY          235
-#define TK_OFFSET        236
+#define TK_ORDER          236
-#define TK_ASC           237
+#define TK_SLIMIT         237
-#define TK_NULLS         238
+#define TK_SOFFSET        238
-#define TK_ID            239
+#define TK_LIMIT          239
-#define TK_NK_BITNOT     240
+#define TK_OFFSET         240
-#define TK_INSERT        241
+#define TK_ASC            241
-#define TK_VALUES        242
+#define TK_NULLS          242
-#define TK_IMPORT        243
+#define TK_ID             243
-#define TK_NK_SEMI       244
+#define TK_NK_BITNOT      244
-#define TK_FILE          245
+#define TK_INSERT         245
+#define TK_VALUES         246
+#define TK_IMPORT         247
+#define TK_NK_SEMI        248
+#define TK_FILE           249
 
 #define TK_NK_SPACE   300
 #define TK_NK_COMMENT 301

include/libs/executor/executor.h

@@ -36,15 +36,8 @@ typedef struct SReadHandle {
   void*   vnode;
   void*   mnd;
   SMsgCb* pMsgCb;
-//  int8_t  initTsdbReader;
 } SReadHandle;
 
-enum {
-  STREAM_DATA_TYPE_SUBMIT_BLOCK = 1,
-  STREAM_DATA_TYPE_SSDATA_BLOCK = 2,
-  STREAM_DATA_TYPE_FROM_SNAPSHOT = 3,
-};
-
 typedef enum {
   OPTR_EXEC_MODEL_BATCH = 0x1,
   OPTR_EXEC_MODEL_STREAM = 0x2,
@@ -140,12 +133,6 @@ int32_t qKillTask(qTaskInfo_t tinfo);
  */
 int32_t qAsyncKillTask(qTaskInfo_t tinfo);
 
-/**
- * return whether query is completed or not
- * @param tinfo
- * @return
- */
-int32_t qIsTaskCompleted(qTaskInfo_t tinfo);
 
 /**
  * destroy query info structure
@@ -176,6 +163,15 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len);
 
 int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len);
 
+/**
+ * return the scan info, in the form of tuple of two items, including table uid and current timestamp
+ * @param tinfo
+ * @param uid
+ * @param ts
+ * @return
+ */
+int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts);
+
 #ifdef __cplusplus
 }
 #endif

include/libs/function/functionMgt.h

@@ -105,7 +105,7 @@ typedef enum EFunctionType {
   // system function
   FUNCTION_TYPE_DATABASE = 3000,
   FUNCTION_TYPE_CLIENT_VERSION,
-  FUNCTION_TYPE_SERVER_SERSION,
+  FUNCTION_TYPE_SERVER_VERSION,
   FUNCTION_TYPE_SERVER_STATUS,
   FUNCTION_TYPE_CURRENT_USER,
   FUNCTION_TYPE_USER,
@@ -125,6 +125,7 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_BLOCK_DIST_INFO,  // block distribution pseudo column function
   FUNCTION_TYPE_TO_COLUMN,
   FUNCTION_TYPE_GROUP_KEY,
+  FUNCTION_TYPE_CACHE_LAST_ROW,
 
   // distributed splitting functions
   FUNCTION_TYPE_APERCENTILE_PARTIAL = 4000,
@@ -193,6 +194,7 @@ bool fmIsForbidGroupByFunc(int32_t funcId);
 bool fmIsIntervalInterpoFunc(int32_t funcId);
 bool fmIsInterpFunc(int32_t funcId);
 bool fmIsLastRowFunc(int32_t funcId);
+bool fmIsSystemInfoFunc(int32_t funcId);
 
 int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMergeFunc);
 

include/libs/nodes/plannodes.h

@@ -91,6 +91,7 @@ typedef struct SAggLogicNode {
   SLogicNode node;
   SNodeList* pGroupKeys;
   SNodeList* pAggFuncs;
+  bool       hasLastRow;
 } SAggLogicNode;
 
 typedef struct SProjectLogicNode {

include/libs/nodes/querynodes.h

@@ -89,6 +89,7 @@ typedef struct SValueNode {
   bool      isDuration;
   bool      translate;
   bool      notReserved;
+  bool      isNull;
   int16_t   placeholderNo;
   union {
     bool     b;

include/libs/scheduler/scheduler.h

@@ -69,18 +69,20 @@ typedef struct SSchdFetchParam {
   int32_t* code;
 } SSchdFetchParam;
 
-typedef void (*schedulerExecCallback)(SQueryResult* pResult, void* param, int32_t code);
+typedef void (*schedulerExecFp)(SQueryResult* pResult, void* param, int32_t code);
-typedef void (*schedulerFetchCallback)(void* pResult, void* param, int32_t code);
+typedef void (*schedulerFetchFp)(void* pResult, void* param, int32_t code);
+typedef bool (*schedulerChkKillFp)(void* param);
 
 typedef struct SSchedulerReq {
-  bool                 *reqKilled;
   SRequestConnInfo     *pConn;
   SArray               *pNodeList;
   SQueryPlan           *pDag;
   const char           *sql;
   int64_t               startTs;
-  schedulerExecCallback fp;
+  schedulerExecFp       execFp;
-  void*                 cbParam;
+  void*                 execParam;
+  schedulerChkKillFp    chkKillFp;
+  void*                 chkKillParam;
 } SSchedulerReq;
 
 
@@ -110,7 +112,7 @@ int32_t schedulerExecJob(SSchedulerReq *pReq, int64_t *pJob, SQueryResult *pRes)
  */
 int32_t schedulerFetchRows(int64_t job, void **data);
 
-void schedulerAsyncFetchRows(int64_t job, schedulerFetchCallback fp, void* param);
+void schedulerAsyncFetchRows(int64_t job, schedulerFetchFp fp, void* param);
 
 int32_t schedulerGetTasksStatus(int64_t job, SArray *pSub);
 

include/libs/stream/tstream.h

@@ -55,15 +55,6 @@ enum {
   TASK_OUTPUT_STATUS__BLOCKED,
 };
 
-enum {
-  STREAM_INPUT__DATA_SUBMIT = 1,
-  STREAM_INPUT__DATA_BLOCK,
-  STREAM_INPUT__DATA_RETRIEVE,
-  STREAM_INPUT__TRIGGER,
-  STREAM_INPUT__CHECKPOINT,
-  STREAM_INPUT__DROP,
-};
-
 typedef struct {
   int8_t type;
 } SStreamQueueItem;
@@ -152,10 +143,6 @@ typedef struct {
   void* executor;
 } STaskExec;
 
-typedef struct {
-  int32_t taskId;
-} STaskDispatcherInplace;
-
 typedef struct {
   int32_t taskId;
   int32_t nodeId;
@@ -208,7 +195,6 @@ enum {
 
 enum {
   TASK_DISPATCH__NONE = 1,
-  TASK_DISPATCH__INPLACE,
   TASK_DISPATCH__FIXED,
   TASK_DISPATCH__SHUFFLE,
 };
@@ -260,7 +246,7 @@ struct SStreamTask {
   // exec
   STaskExec exec;
 
-  // TODO: merge sink and dispatch
+  // TODO: unify sink and dispatch
 
   //  local sink
   union {
@@ -269,9 +255,8 @@ struct SStreamTask {
     STaskSinkFetch fetchSink;
   };
 
-  // dispatch
+  // remote dispatcher
   union {
-    STaskDispatcherInplace inplaceDispatcher;
     STaskDispatcherFixedEp fixedEpDispatcher;
     STaskDispatcherShuffle shuffleDispatcher;
   };
@@ -327,9 +312,8 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem
     taosWriteQitem(pTask->inputQueue->queue, pItem);
   }
 
-  if (pItem->type != STREAM_INPUT__TRIGGER && pItem->type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0 &&
+  if (pItem->type != STREAM_INPUT__TRIGGER && pItem->type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0) {
-      pTask->triggerStatus == TASK_TRIGGER_STATUS__IN_ACTIVE) {
+    atomic_val_compare_exchange_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__IN_ACTIVE, TASK_TRIGGER_STATUS__ACTIVE);
-    atomic_store_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__ACTIVE);
   }
 
   // TODO: back pressure

include/libs/sync/syncTools.h

@@ -324,6 +324,23 @@ void syncAppendEntriesPrint2(char* s, const SyncAppendEntries* pMsg);
 void syncAppendEntriesLog(const SyncAppendEntries* pMsg);
 void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg);
 
+// ---------------------------------------------
+typedef struct SyncAppendEntriesBatch {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  SyncTerm  term;
+  SyncIndex prevLogIndex;
+  SyncTerm  prevLogTerm;
+  SyncIndex commitIndex;
+  SyncTerm  privateTerm;
+  uint32_t  dataLen;
+  char      data[];
+} SyncAppendEntriesBatch;
+
 // ---------------------------------------------
 typedef struct SyncAppendEntriesReply {
   uint32_t bytes;

packaging/release.sh

@@ -111,9 +111,9 @@ else
 fi
 
 csudo=""
-if command -v sudo > /dev/null; then
+#if command -v sudo > /dev/null; then
- csudo="sudo "
+#  csudo="sudo "
-fi
+#fi
 
 function is_valid_version() {
   [ -z $1 ] && return 1 || :
@@ -182,14 +182,10 @@ cd "${curr_dir}"
 # 2. cmake executable file
 compile_dir="${top_dir}/debug"
 if [ -d ${compile_dir} ]; then
-  ${csudo}rm -rf ${compile_dir}
+  rm -rf ${compile_dir}
 fi
 
-if [ "$osType" != "Darwin" ]; then
+mkdir -p ${compile_dir}
-  ${csudo}mkdir -p ${compile_dir}
-else
-  mkdir -p ${compile_dir}
-fi
 cd ${compile_dir}
 
 if [[ "$allocator" == "jemalloc" ]]; then
@@ -255,18 +251,18 @@ if [ "$osType" != "Darwin" ]; then
       echo "====do deb package for the ubuntu system===="
       output_dir="${top_dir}/debs"
       if [ -d ${output_dir} ]; then
-        ${csudo}rm -rf ${output_dir}
+        rm -rf ${output_dir}
       fi
-      ${csudo}mkdir -p ${output_dir}
+      mkdir -p ${output_dir}
       cd ${script_dir}/deb
       ${csudo}./makedeb.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType}
 
       if [[ "$pagMode" == "full" ]]; then
         if [ -d ${top_dir}/tools/taos-tools/packaging/deb ]; then
           cd ${top_dir}/tools/taos-tools/packaging/deb
+          taos_tools_ver=$(git describe --tags | sed -e 's/ver-//g' | awk -F '-' '{print $1}')
           [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0"
 
-          taos_tools_ver=$(git describe --tags | sed -e 's/ver-//g' | awk -F '-' '{print $1}')
           ${csudo}./make-taos-tools-deb.sh ${top_dir} \
             ${compile_dir} ${output_dir} ${taos_tools_ver} ${cpuType} ${osType} ${verMode} ${verType}
         fi
@@ -280,18 +276,18 @@ if [ "$osType" != "Darwin" ]; then
       echo "====do rpm package for the centos system===="
       output_dir="${top_dir}/rpms"
       if [ -d ${output_dir} ]; then
-        ${csudo}rm -rf ${output_dir}
+        rm -rf ${output_dir}
       fi
-      ${csudo}mkdir -p ${output_dir}
+      mkdir -p ${output_dir}
       cd ${script_dir}/rpm
       ${csudo}./makerpm.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType}
 
       if [[ "$pagMode" == "full" ]]; then
         if [ -d ${top_dir}/tools/taos-tools/packaging/rpm ]; then
           cd ${top_dir}/tools/taos-tools/packaging/rpm
+          taos_tools_ver=$(git describe --tags | sed -e 's/ver-//g' | awk -F '-' '{print $1}' | sed -e 's/-/_/g')
           [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0"
 
-          taos_tools_ver=$(git describe --tags | sed -e 's/ver-//g' | awk -F '-' '{print $1}' | sed -e 's/-/_/g')
           ${csudo}./make-taos-tools-rpm.sh ${top_dir} \
             ${compile_dir} ${output_dir} ${taos_tools_ver} ${cpuType} ${osType} ${verMode} ${verType}
         fi
@@ -306,7 +302,7 @@ if [ "$osType" != "Darwin" ]; then
 
   ${csudo}./makepkg.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${verNumberComp} ${dbName}
   ${csudo}./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName}
-  # ${csudo}./makearbi.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode}
+  ${csudo}./makearbi.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode}
 
 else
   # only make client for Darwin

source/client/inc/clientInt.h

@@ -58,7 +58,6 @@ enum {
 #define SHOW_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE)
 #define SHOW_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE)
 
-
 #define TD_RES_QUERY(res)    (*(int8_t*)res == RES_TYPE__QUERY)
 #define TD_RES_TMQ(res)      (*(int8_t*)res == RES_TYPE__TMQ)
 #define TD_RES_TMQ_META(res) (*(int8_t*)res == RES_TYPE__TMQ_META)
@@ -195,7 +194,7 @@ typedef struct {
   int32_t        vgId;
   SSchemaWrapper schema;
   int32_t        resIter;
-  SMqDataBlkRsp  rsp;
+  SMqDataRsp     rsp;
   SReqResultInfo resInfo;
 } SMqRspObj;
 
@@ -249,8 +248,8 @@ int32_t      transferTableNameList(const char* tbList, int32_t acctId, char* dbN
 void    syncCatalogFn(SMetaData* pResult, void* param, int32_t code);
 
 SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool validateOnly);
-TAOS_RES *taosQueryImpl(TAOS *taos, const char *sql, bool validateOnly);
+TAOS_RES*    taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly);
-void      taosAsyncQueryImpl(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param, bool validateOnly);
+void         taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly);
 
 static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) {
   SMqRspObj* msg = (SMqRspObj*)res;

source/client/src/clientEnv.c

@@ -132,7 +132,7 @@ void closeAllRequests(SHashObj *pRequests) {
   }
 }
 
-void destroyAppInst(SAppInstInfo* pAppInfo) {
+void destroyAppInst(SAppInstInfo *pAppInfo) {
   tscDebug("destroy app inst mgr %p", pAppInfo);
 
   taosThreadMutexLock(&appInfo.mutex);

source/client/src/clientImpl.c

@@ -55,6 +55,18 @@ static char* getClusterKey(const char* user, const char* auth, const char* ip, i
   return strdup(key);
 }
 
+bool chkRequestKilled(void* param) {
+  bool killed = false;
+  SRequestObj* pRequest = acquireRequest((int64_t)param);
+  if (NULL == pRequest || pRequest->killed) {
+    killed = true;
+  }
+
+  releaseRequest((int64_t)param);
+  
+  return killed;
+}
+
 static STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param,
                                 SAppInstInfo* pAppInfo, int connType);
 
@@ -612,58 +624,6 @@ _return:
   return code;
 }
 
-int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) {
-  tsem_init(&schdRspSem, 0, 0);
-
-  SQueryResult     res = {.code = 0, .numOfRows = 0};
-  SRequestConnInfo conn = {.pTrans = pRequest->pTscObj->pAppInfo->pTransporter,
-                           .requestId = pRequest->requestId,
-                           .requestObjRefId = pRequest->self};
-  SSchedulerReq    req = {.pConn = &conn,
-                          .pNodeList = pNodeList,
-                          .pDag = pDag,
-                          .sql = pRequest->sqlstr,
-                          .startTs = pRequest->metric.start,
-                          .fp = schdExecCallback,
-                          .cbParam = &res};
-
-  int32_t code = schedulerAsyncExecJob(&req, &pRequest->body.queryJob);
-
-  pRequest->body.resInfo.execRes = res.res;
-
-  while (true) {
-    if (code != TSDB_CODE_SUCCESS) {
-      if (pRequest->body.queryJob != 0) {
-        schedulerFreeJob(pRequest->body.queryJob, 0);
-      }
-
-      pRequest->code = code;
-      terrno = code;
-      return pRequest->code;
-    } else {
-      tsem_wait(&schdRspSem);
-
-      if (res.code) {
-        code = res.code;
-      } else {
-        break;
-      }
-    }
-  }
-
-  if (TDMT_VND_SUBMIT == pRequest->type || TDMT_VND_CREATE_TABLE == pRequest->type) {
-    pRequest->body.resInfo.numOfRows = res.numOfRows;
-
-    if (pRequest->body.queryJob != 0) {
-      schedulerFreeJob(pRequest->body.queryJob, 0);
-    }
-  }
-
-  pRequest->code = res.code;
-  terrno = res.code;
-  return pRequest->code;
-}
-
 int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) {
   void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter;
 
@@ -676,9 +636,10 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
                        .pDag = pDag,
                        .sql = pRequest->sqlstr,
                        .startTs = pRequest->metric.start,
-                          .fp = NULL,
+                       .execFp = NULL,
-                          .cbParam = NULL,
+                       .execParam = NULL,
-                          .reqKilled = &pRequest->killed};
+                       .chkKillFp = chkRequestKilled,
+                       .chkKillParam = (void*)pRequest->self};
 
   int32_t code = schedulerExecJob(&req, &pRequest->body.queryJob, &res);
   pRequest->body.resInfo.execRes = res.res;
@@ -987,9 +948,10 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
                              .pDag = pDag,
                              .sql = pRequest->sqlstr,
                              .startTs = pRequest->metric.start,
-                             .fp = schedulerExecCb,
+                             .execFp = schedulerExecCb,
-                             .cbParam = pRequest,
+                             .execParam = pRequest,
-                             .reqKilled = &pRequest->killed};
+                             .chkKillFp = chkRequestKilled,
+                             .chkKillParam = (void*)pRequest->self};
         code = schedulerAsyncExecJob(&req, &pRequest->body.queryJob);
         taosArrayDestroy(pNodeList);
       } else {
@@ -1337,7 +1299,7 @@ TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, cons
 
   STscObj* pObj = taos_connect_internal(ip, user, NULL, auth, db, port, CONN_TYPE__QUERY);
   if (pObj) {
-    int64_t *rid = taosMemoryCalloc(1, sizeof(int64_t));
+    int64_t* rid = taosMemoryCalloc(1, sizeof(int64_t));
     *rid = pObj->id;
     return (TAOS*)rid;
   }

source/client/src/clientMain.c

@@ -81,16 +81,16 @@ void taos_cleanup(void) {
   taosCloseLog();
 }
 
-static setConfRet taos_set_config_imp(const char *config){
+static setConfRet taos_set_config_imp(const char *config) {
   setConfRet ret = {SET_CONF_RET_SUCC, {0}};
   // TODO: need re-implementation
   return ret;
 }
 
-setConfRet taos_set_config(const char *config){
+setConfRet taos_set_config(const char *config) {
-// TODO  pthread_mutex_lock(&setConfMutex);
+  // TODO  pthread_mutex_lock(&setConfMutex);
   setConfRet ret = taos_set_config_imp(config);
-//  pthread_mutex_unlock(&setConfMutex);
+  //  pthread_mutex_unlock(&setConfMutex);
   return ret;
 }
 
@@ -181,8 +181,6 @@ void taos_free_result(TAOS_RES *res) {
     SMqRspObj *pRsp = (SMqRspObj *)res;
     if (pRsp->rsp.blockData) taosArrayDestroyP(pRsp->rsp.blockData, taosMemoryFree);
     if (pRsp->rsp.blockDataLen) taosArrayDestroy(pRsp->rsp.blockDataLen);
-    if (pRsp->rsp.blockTags) taosArrayDestroy(pRsp->rsp.blockTags);
-    if (pRsp->rsp.blockTagSchema) taosArrayDestroy(pRsp->rsp.blockTagSchema);
     if (pRsp->rsp.withTbName) taosArrayDestroyP(pRsp->rsp.blockTbName, taosMemoryFree);
     if (pRsp->rsp.withSchema) taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper);
     pRsp->resInfo.pRspMsg = NULL;
@@ -740,6 +738,8 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) {
       .schemalessType = pTscObj->schemalessType,
       .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)),
       .async = true,
+      .svrVer = pTscObj->sVer,
+      .nodeOffline = (pTscObj->pAppInfo->onlineDnodes < pTscObj->pAppInfo->totalDnodes)
   };
   return TSDB_CODE_SUCCESS;
 }

source/client/src/clientMsgHandler.c

@@ -69,9 +69,9 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
     SEpSet* pOrig = &pTscObj->pAppInfo->mgmtEp.epSet;
     SEp*    pOrigEp = &pOrig->eps[pOrig->inUse];
     SEp*    pNewEp = &connectRsp.epSet.eps[connectRsp.epSet.inUse];
-    tscDebug("mnode epset updated from %d/%d=>%s:%d to %d/%d=>%s:%d in connRsp", 
+    tscDebug("mnode epset updated from %d/%d=>%s:%d to %d/%d=>%s:%d in connRsp", pOrig->inUse, pOrig->numOfEps,
-        pOrig->inUse, pOrig->numOfEps, pOrigEp->fqdn, pOrigEp->port, 
+             pOrigEp->fqdn, pOrigEp->port, connectRsp.epSet.inUse, connectRsp.epSet.numOfEps, pNewEp->fqdn,
-        connectRsp.epSet.inUse, connectRsp.epSet.numOfEps, pNewEp->fqdn, pNewEp->port);
+             pNewEp->port);
     updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &connectRsp.epSet);
   }
 
@@ -307,11 +307,11 @@ static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) {
   blockDataEnsureCapacity(pBlock, numOfCfg);
 
   for (int32_t i = 0, c = 0; i < numOfCfg; ++i, c = 0) {
-    SVariablesInfo *pInfo = taosArrayGet(pVars, i);
+    SVariablesInfo* pInfo = taosArrayGet(pVars, i);
 
     char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(name, pInfo->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE);
-    SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, c++);
+    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, c++);
     colDataAppend(pColInfo, i, name, false);
 
     char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0};
@@ -327,7 +327,6 @@ static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) {
   return TSDB_CODE_SUCCESS;
 }
 
-
 static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) {
   SSDataBlock* pBlock = NULL;
   int32_t      code = buildShowVariablesBlock(pVars, &pBlock);
@@ -350,7 +349,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) {
   (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS);
 
   int32_t len = 0;
-  blockCompressEncode(pBlock, (*pRsp)->data, &len, SHOW_VARIABLES_RESULT_COLS, false);
+  blockEncode(pBlock, (*pRsp)->data, &len, SHOW_VARIABLES_RESULT_COLS, false);
   ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
 
   blockDataDestroy(pBlock);
@@ -383,7 +382,6 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) {
   return code;
 }
 
-
 __async_send_cb_fn_t getMsgRspHandle(int32_t msgType) {
   switch (msgType) {
     case TDMT_MND_CONNECT:

source/client/src/tmq.c

@@ -126,8 +126,10 @@ typedef struct {
   // statistics
   int64_t pollCnt;
   // offset
-  int64_t committedOffset;
+  /*int64_t      committedOffset;*/
-  int64_t currentOffset;
+  /*int64_t      currentOffset;*/
+  STqOffsetVal committedOffsetNew;
+  STqOffsetVal currentOffsetNew;
   // connection info
   int32_t vgId;
   int32_t vgStatus;
@@ -152,7 +154,7 @@ typedef struct {
   SMqClientVg*    vgHandle;
   SMqClientTopic* topicHandle;
   union {
-    SMqDataBlkRsp dataRsp;
+    SMqDataRsp dataRsp;
     SMqMetaRsp metaRsp;
   };
 } SMqPollRspWrapper;
@@ -179,6 +181,7 @@ typedef struct {
   tsem_t          rspSem;
 } SMqPollCbParam;
 
+#if 0
 typedef struct {
   tmq_t*         tmq;
   int8_t         async;
@@ -190,12 +193,13 @@ typedef struct {
   SArray*        offsets;
   void*          userParam;
 } SMqCommitCbParam;
+#endif
 
 typedef struct {
   tmq_t* tmq;
   int8_t automatic;
   int8_t async;
-  int8_t         freeOffsets;
+  /*int8_t         freeOffsets;*/
   int32_t        waitingRspNum;
   int32_t        totalRspNum;
   int32_t        rspErr;
@@ -351,6 +355,7 @@ static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) {
   return sprintf(dst, "%s:%d", topicName, vg);
 }
 
+#if 0
 int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) {
   SMqCommitCbParam* pParam = (SMqCommitCbParam*)param;
   pParam->rspErr = code;
@@ -371,6 +376,7 @@ int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) {
   }
   return 0;
 }
+#endif
 
 int32_t tmqCommitCb2(void* param, SDataBuf* pBuf, int32_t code) {
   SMqCommitCbParam2*   pParam = (SMqCommitCbParam2*)param;
@@ -413,13 +419,76 @@ int32_t tmqCommitCb2(void* param, SDataBuf* pBuf, int32_t code) {
   return 0;
 }
 
-int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t async, tmq_commit_cb* userCb,
+static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pTopic, SMqCommitCbParamSet* pParamSet) {
-                        void* userParam) {
+  STqOffset* pOffset = taosMemoryCalloc(1, sizeof(STqOffset));
-  int32_t code = -1;
+  if (pOffset == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+  pOffset->val = pVg->currentOffsetNew;
 
-  if (msg != NULL) {
+  int32_t groupLen = strlen(tmq->groupId);
+  memcpy(pOffset->subKey, tmq->groupId, groupLen);
+  pOffset->subKey[groupLen] = TMQ_SEPARATOR;
+  strcpy(pOffset->subKey + groupLen + 1, pTopic->topicName);
+
+  int32_t len;
+  int32_t code;
+  tEncodeSize(tEncodeSTqOffset, pOffset, len, code);
+  if (code < 0) {
+    ASSERT(0);
+    return -1;
+  }
+  void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len);
+  if (buf == NULL) return -1;
+  ((SMsgHead*)buf)->vgId = htonl(pVg->vgId);
+
+  void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
+
+  SEncoder encoder;
+  tEncoderInit(&encoder, abuf, len);
+  tEncodeSTqOffset(&encoder, pOffset);
+
+  // build param
+  SMqCommitCbParam2* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam2));
+  pParam->params = pParamSet;
+  pParam->pOffset = pOffset;
+
+  // build send info
+  SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
+  if (pMsgSendInfo == NULL) {
+    return -1;
+  }
+  pMsgSendInfo->msgInfo = (SDataBuf){
+      .pData = buf,
+      .len = sizeof(SMsgHead) + len,
+      .handle = NULL,
+  };
+
+  tscDebug("consumer %ld commit offset of %s on vg %d, offset is %ld", tmq->consumerId, pOffset->subKey, pVg->vgId,
+           pOffset->val.version);
+
+  // TODO: put into cb
+  pVg->committedOffsetNew = pVg->currentOffsetNew;
+
+  pMsgSendInfo->requestId = generateRequestId();
+  pMsgSendInfo->requestObjRefId = 0;
+  pMsgSendInfo->param = pParam;
+  pMsgSendInfo->fp = tmqCommitCb2;
+  pMsgSendInfo->msgType = TDMT_VND_MQ_COMMIT_OFFSET;
+  // send msg
+
+  int64_t transporterId = 0;
+  asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, pMsgSendInfo);
+  pParamSet->waitingRspNum++;
+  pParamSet->totalRspNum++;
+  return 0;
+}
+
+int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_commit_cb* userCb, void* userParam) {
   char*   topic;
   int32_t vgId;
+  ASSERT(msg != NULL);
   if (TD_RES_TMQ(msg)) {
     SMqRspObj* pRspObj = (SMqRspObj*)msg;
     topic = pRspObj->topic;
@@ -438,92 +507,32 @@ int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_
     return -1;
   }
   pParamSet->tmq = tmq;
-    pParamSet->automatic = automatic;
+  pParamSet->automatic = 0;
   pParamSet->async = async;
-    pParamSet->freeOffsets = 1;
+  /*pParamSet->freeOffsets = 1;*/
   pParamSet->userCb = userCb;
   pParamSet->userParam = userParam;
   tsem_init(&pParamSet->rspSem, 0, 0);
 
+  int32_t code = -1;
+
   for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
     SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
-      if (strcmp(pTopic->topicName, topic) == 0) {
+    if (strcmp(pTopic->topicName, topic) != 0) continue;
     for (int32_t j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
       SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
-          if (pVg->vgId == vgId) {
+      if (pVg->vgId != vgId) continue;
-            if (pVg->currentOffset < 0 || pVg->committedOffset == pVg->currentOffset) {
-              tscDebug("consumer %ld skip commit for topic %s vg %d, current offset is %ld, committed offset is %ld",
-                       tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->currentOffset, pVg->committedOffset);
 
-              return 0;
+      if (pVg->currentOffsetNew.type > 0 && !tOffsetEqual(&pVg->currentOffsetNew, &pVg->committedOffsetNew)) {
-            }
+        if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) {
-
+          goto FAIL;
-            STqOffset* pOffset = taosMemoryCalloc(1, sizeof(STqOffset));
-            if (pOffset == NULL) {
-              terrno = TSDB_CODE_OUT_OF_MEMORY;
-              return -1;
-            }
-            pOffset->type = TMQ_OFFSET__LOG;
-            pOffset->version = pVg->currentOffset;
-
-            int32_t groupLen = strlen(tmq->groupId);
-            memcpy(pOffset->subKey, tmq->groupId, groupLen);
-            pOffset->subKey[groupLen] = TMQ_SEPARATOR;
-            strcpy(pOffset->subKey + groupLen + 1, pTopic->topicName);
-
-            int32_t len;
-            int32_t code;
-            tEncodeSize(tEncodeSTqOffset, pOffset, len, code);
-            if (code < 0) {
-              ASSERT(0);
-            }
-            void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len);
-            ((SMsgHead*)buf)->vgId = htonl(pVg->vgId);
-
-            void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
-
-            SEncoder encoder;
-            tEncoderInit(&encoder, abuf, len);
-            tEncodeSTqOffset(&encoder, pOffset);
-
-            // build param
-            SMqCommitCbParam2* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam2));
-            pParam->params = pParamSet;
-            pParam->pOffset = pOffset;
-
-            // build send info
-            SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
-            if (pMsgSendInfo == NULL) {
-              // TODO
-              continue;
-            }
-            pMsgSendInfo->msgInfo = (SDataBuf){
-                .pData = buf,
-                .len = sizeof(SMsgHead) + len,
-                .handle = NULL,
-            };
-
-            tscDebug("consumer %ld commit offset of %s on vg %d, offset is %ld", tmq->consumerId, pOffset->subKey,
-                     pVg->vgId, pOffset->version);
-
-            // TODO: put into cb
-            pVg->committedOffset = pVg->currentOffset;
-
-            pMsgSendInfo->requestId = generateRequestId();
-            pMsgSendInfo->requestObjRefId = 0;
-            pMsgSendInfo->param = pParam;
-            pMsgSendInfo->fp = tmqCommitCb2;
-            pMsgSendInfo->msgType = TDMT_VND_MQ_COMMIT_OFFSET;
-            // send msg
-
-            int64_t transporterId = 0;
-            asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, pMsgSendInfo);
-            pParamSet->waitingRspNum++;
-            pParamSet->totalRspNum++;
         }
+        goto HANDLE_RSP;
       }
     }
   }
+
+HANDLE_RSP:
   if (pParamSet->totalRspNum == 0) {
     tsem_destroy(&pParamSet->rspSem);
     taosMemoryFree(pParamSet);
@@ -534,18 +543,24 @@ int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_
     tsem_wait(&pParamSet->rspSem);
     code = pParamSet->rspErr;
     tsem_destroy(&pParamSet->rspSem);
+    return code;
   } else {
     code = 0;
   }
 
+FAIL:
   if (code != 0 && async) {
-      if (automatic) {
-        tmq->commitCb(tmq, code, tmq->commitCbUserParam);
-      } else {
     userCb(tmq, code, userParam);
   }
-    }
   return 0;
+}
+
+int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t async, tmq_commit_cb* userCb,
+                        void* userParam) {
+  int32_t code = -1;
+
+  if (msg != NULL) {
+    return tmqCommitMsgImpl(tmq, msg, async, userCb, userParam);
   }
 
   SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet));
@@ -556,7 +571,7 @@ int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_
   pParamSet->tmq = tmq;
   pParamSet->automatic = automatic;
   pParamSet->async = async;
-  pParamSet->freeOffsets = 1;
+  /*pParamSet->freeOffsets = 1;*/
   pParamSet->userCb = userCb;
   pParamSet->userParam = userParam;
   tsem_init(&pParamSet->rspSem, 0, 0);
@@ -572,75 +587,11 @@ int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_
 
       tscDebug("consumer %ld begin commit for topic %s, vgId %d", tmq->consumerId, pTopic->topicName, pVg->vgId);
 
-      /*if (pVg->currentOffset < 0) {*/
+      if (pVg->currentOffsetNew.type > 0 && !tOffsetEqual(&pVg->currentOffsetNew, &pVg->committedOffsetNew)) {
-      if (pVg->currentOffset < 0 || pVg->committedOffset == pVg->currentOffset) {
+        if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) {
-        tscDebug("consumer %ld skip commit for topic %s vg %d, current offset is %ld, committed offset is %ld",
-                 tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->currentOffset, pVg->committedOffset);
-
           continue;
         }
-      STqOffset* pOffset = taosMemoryCalloc(1, sizeof(STqOffset));
-      if (pOffset == NULL) {
-        terrno = TSDB_CODE_OUT_OF_MEMORY;
-        return -1;
       }
-      pOffset->type = TMQ_OFFSET__LOG;
-      pOffset->version = pVg->currentOffset;
-
-      int32_t groupLen = strlen(tmq->groupId);
-      memcpy(pOffset->subKey, tmq->groupId, groupLen);
-      pOffset->subKey[groupLen] = TMQ_SEPARATOR;
-      strcpy(pOffset->subKey + groupLen + 1, pTopic->topicName);
-
-      int32_t len;
-      int32_t code;
-      tEncodeSize(tEncodeSTqOffset, pOffset, len, code);
-      if (code < 0) {
-        ASSERT(0);
-      }
-      void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len);
-      ((SMsgHead*)buf)->vgId = htonl(pVg->vgId);
-
-      void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
-
-      SEncoder encoder;
-      tEncoderInit(&encoder, abuf, len);
-      tEncodeSTqOffset(&encoder, pOffset);
-
-      // build param
-      SMqCommitCbParam2* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam2));
-      pParam->params = pParamSet;
-      pParam->pOffset = pOffset;
-
-      // build send info
-      SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
-      if (pMsgSendInfo == NULL) {
-        // TODO
-        continue;
-      }
-      pMsgSendInfo->msgInfo = (SDataBuf){
-          .pData = buf,
-          .len = sizeof(SMsgHead) + len,
-          .handle = NULL,
-      };
-
-      tscDebug("consumer %ld commit offset of %s on vg %d, offset is %ld", tmq->consumerId, pOffset->subKey, pVg->vgId,
-               pOffset->version);
-
-      // TODO: put into cb
-      pVg->committedOffset = pVg->currentOffset;
-
-      pMsgSendInfo->requestId = generateRequestId();
-      pMsgSendInfo->requestObjRefId = 0;
-      pMsgSendInfo->param = pParam;
-      pMsgSendInfo->fp = tmqCommitCb2;
-      pMsgSendInfo->msgType = TDMT_VND_MQ_COMMIT_OFFSET;
-      // send msg
-
-      int64_t transporterId = 0;
-      asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, pMsgSendInfo);
-      pParamSet->waitingRspNum++;
-      pParamSet->totalRspNum++;
     }
   }
 
@@ -1173,8 +1124,11 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
   pRspWrapper->topicHandle = pTopic;
 
   if (rspType == TMQ_MSG_TYPE__POLL_RSP) {
+    SDecoder decoder;
+    tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead));
+    tDecodeSMqDataRsp(&decoder, &pRspWrapper->dataRsp);
     memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead));
-    tDecodeSMqDataBlkRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->dataRsp);
+    /*tDecodeSMqDataBlkRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->dataRsp);*/
   } else {
     ASSERT(rspType == TMQ_MSG_TYPE__POLL_META_RSP);
     memcpy(&pRspWrapper->metaRsp, pMsg->pData, sizeof(SMqRspHead));
@@ -1184,7 +1138,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
   taosMemoryFree(pMsg->pData);
 
   tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld, type %d", tmq->consumerId, pVg->vgId,
-           pRspWrapper->dataRsp.reqOffset, pRspWrapper->dataRsp.rspOffset, rspType);
+           pRspWrapper->dataRsp.reqOffset.version, pRspWrapper->dataRsp.rspOffset.version, rspType);
 
   taosWriteQitem(tmq->mqueue, pRspWrapper);
   tsem_post(&tmq->rspSem);
@@ -1226,9 +1180,11 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) {
       for (int32_t j = 0; j < vgNumCur; j++) {
         SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j);
         sprintf(vgKey, "%s:%d", pTopicCur->topicName, pVgCur->vgId);
-        tscDebug("consumer %ld epoch %d vg %d vgKey is %s, offset is %ld", tmq->consumerId, epoch, pVgCur->vgId, vgKey,
+        char buf[50];
-                 pVgCur->currentOffset);
+        tFormatOffset(buf, 50, &pVgCur->currentOffsetNew);
-        taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t));
+        tscDebug("consumer %ld epoch %d vg %d vgKey is %s, offset is %s", tmq->consumerId, epoch, pVgCur->vgId, vgKey,
+                 buf);
+        taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffsetNew, sizeof(STqOffsetVal));
       }
     }
   }
@@ -1247,17 +1203,17 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) {
     for (int32_t j = 0; j < vgNumGet; j++) {
       SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j);
       sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId);
-      int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey));
+      STqOffsetVal* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey));
-      int64_t  offset = tmq->resetOffsetCfg;
+      STqOffsetVal  offsetNew = {.type = tmq->resetOffsetCfg};
       if (pOffset != NULL) {
-        offset = *pOffset;
+        offsetNew = *pOffset;
       }
 
-      tscDebug("consumer %ld(epoch %d) offset of vg %d updated to %ld, vgKey is %s", tmq->consumerId, epoch,
+      /*tscDebug("consumer %ld(epoch %d) offset of vg %d updated to %ld, vgKey is %s", tmq->consumerId, epoch,*/
-               pVgEp->vgId, offset, vgKey);
+      /*pVgEp->vgId, offset, vgKey);*/
       SMqClientVg clientVg = {
           .pollCnt = 0,
-          .currentOffset = offset,
+          .currentOffsetNew = offsetNew,
           .vgId = pVgEp->vgId,
           .epSet = pVgEp->epSet,
           .vgStatus = TMQ_VG_STATUS__IDLE,
@@ -1281,7 +1237,7 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) {
   return set;
 }
 
-#if 1
+#if 0
 bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) {
   /*printf("call update ep %d\n", epoch);*/
   bool    set = false;
@@ -1516,16 +1472,16 @@ int32_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) {
 #endif
 
 SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) {
-  int64_t reqOffset;
+  /*int64_t reqOffset;*/
-  if (pVg->currentOffset >= 0) {
+  /*if (pVg->currentOffset >= 0) {*/
-    reqOffset = pVg->currentOffset;
+  /*reqOffset = pVg->currentOffset;*/
-  } else {
+  /*} else {*/
   /*if (tmq->resetOffsetCfg == TMQ_CONF__RESET_OFFSET__NONE) {*/
   /*tscError("unable to poll since no committed offset but reset offset is set to none");*/
   /*return NULL;*/
   /*}*/
-    reqOffset = tmq->resetOffsetCfg;
+  /*reqOffset = tmq->resetOffsetCfg;*/
-  }
+  /*}*/
 
   SMqPollReq* pReq = taosMemoryCalloc(1, sizeof(SMqPollReq));
   if (pReq == NULL) {
@@ -1544,7 +1500,8 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic*
   pReq->timeout = timeout;
   pReq->consumerId = tmq->consumerId;
   pReq->epoch = tmq->epoch;
-  pReq->currentOffset = reqOffset;
+  /*pReq->currentOffset = reqOffset;*/
+  pReq->reqOffset = pVg->currentOffsetNew;
   pReq->reqId = generateRequestId();
 
   pReq->useSnapshot = tmq->useSnapshot;
@@ -1572,7 +1529,7 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) {
   tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN);
   pRspObj->vgId = pWrapper->vgHandle->vgId;
   pRspObj->resIter = -1;
-  memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataBlkRsp));
+  memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp));
 
   pRspObj->resInfo.totalRows = 0;
   pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI;
@@ -1645,8 +1602,11 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
 
       int64_t transporterId = 0;
       /*printf("send poll\n");*/
-      tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %ld, reqId %lu", tmq->consumerId,
+
-               pTopic->topicName, pVg->vgId, tmq->epoch, pVg->currentOffset, pReq->reqId);
+      char offsetFormatBuf[50];
+      tFormatOffset(offsetFormatBuf, 50, &pVg->currentOffsetNew);
+      tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %s, reqId %lu", tmq->consumerId,
+               pTopic->topicName, pVg->vgId, tmq->epoch, offsetFormatBuf, pReq->reqId);
       /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/
       asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
       pVg->pollCnt++;
@@ -1695,7 +1655,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
       if (pollRspWrapper->dataRsp.head.epoch == consumerEpoch) {
         SMqClientVg* pVg = pollRspWrapper->vgHandle;
         /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/
-        pVg->currentOffset = pollRspWrapper->dataRsp.rspOffset;
+        pVg->currentOffsetNew = pollRspWrapper->dataRsp.rspOffset;
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
         if (pollRspWrapper->dataRsp.blockNum == 0) {
           taosFreeQitem(pollRspWrapper);
@@ -1717,7 +1677,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
       if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) {
         SMqClientVg* pVg = pollRspWrapper->vgHandle;
         /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/
-        pVg->currentOffset = pollRspWrapper->metaRsp.rspOffset;
+        pVg->currentOffsetNew = pollRspWrapper->metaRsp.rspOffsetNew;
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
         // build rsp
         SMqMetaRspObj* pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper);

source/common/src/tdatablock.c

@@ -1613,7 +1613,8 @@ void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) {
     size_t       numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
 
     int32_t rows = pDataBlock->info.rows;
-    printf("%s |block type %d |child id %d|group id %zX\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId);
+    printf("%s |block type %d |child id %d|group id %zX\n", flag, (int32_t)pDataBlock->info.type,
+           pDataBlock->info.childId, pDataBlock->info.groupId);
     for (int32_t j = 0; j < rows; j++) {
       printf("%s |", flag);
       for (int32_t k = 0; k < numOfCols; k++) {
@@ -1662,8 +1663,8 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf)
   int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock);
   int32_t rows = pDataBlock->info.rows;
   int32_t len = 0;
-  len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|\n", flag,
+  len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|group id %lu|\n", flag,
-                  (int32_t)pDataBlock->info.type, pDataBlock->info.childId);
+                  (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId);
   for (int32_t j = 0; j < rows; j++) {
     len += snprintf(dumpBuf + len, size - len, "%s |", flag);
     for (int32_t k = 0; k < colNum; k++) {
@@ -1902,8 +1903,7 @@ char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId) {
   return rname.childTableName;
 }
 
-void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols,
+void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress) {
-                         int8_t needCompress) {
   // todo extract method
   int32_t* actualLen = (int32_t*)data;
   data += sizeof(int32_t);
@@ -1913,6 +1913,7 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen
 
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
+
     *((int16_t*)data) = pColInfoData->info.type;
     data += sizeof(int16_t);
 
@@ -1960,7 +1961,7 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen
   *groupId = pBlock->info.groupId;
 }
 
-const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) {
+const char* blockDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) {
   const char* pStart = pData;
 
   int32_t dataLen = *(int32_t*)pStart;

source/common/src/tmsg.c

@@ -621,8 +621,8 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
     if (NULL == pReq->pFuncs) return -1;
   }
   for (int32_t i = 0; i < numOfFuncs; ++i) {
-    char *pFunc = NULL;
+    char pFunc[TSDB_FUNC_NAME_LEN] = {0};
-    if (tDecodeCStrAlloc(&decoder, &pFunc) < 0) return -1;
+    if (tDecodeCStrTo(&decoder, pFunc) < 0) return -1;
     if (taosArrayPush(pReq->pFuncs, pFunc) == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
@@ -2300,7 +2300,6 @@ int32_t tDeserializeSServerVerRsp(void *buf, int32_t bufLen, SServerVerRsp *pRsp
   return 0;
 }
 
-
 int32_t tSerializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp) {
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, bufLen);
@@ -2387,7 +2386,6 @@ int32_t tDeserializeSDnodeListRsp(void *buf, int32_t bufLen, SDnodeListRsp *pRsp
 
 void tFreeSDnodeListRsp(SDnodeListRsp *pRsp) { taosArrayDestroy(pRsp->dnodeList); }
 
-
 int32_t tSerializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq) {
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, bufLen);
@@ -2909,20 +2907,19 @@ int32_t tDeserializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesR
   return 0;
 }
 
-int32_t tEncodeSVariablesInfo(SEncoder* pEncoder, SVariablesInfo* pInfo) {
+int32_t tEncodeSVariablesInfo(SEncoder *pEncoder, SVariablesInfo *pInfo) {
   if (tEncodeCStr(pEncoder, pInfo->name) < 0) return -1;
   if (tEncodeCStr(pEncoder, pInfo->value) < 0) return -1;
   return 0;
 }
 
-int32_t tDecodeSVariablesInfo(SDecoder* pDecoder, SVariablesInfo* pInfo) {
+int32_t tDecodeSVariablesInfo(SDecoder *pDecoder, SVariablesInfo *pInfo) {
   if (tDecodeCStrTo(pDecoder, pInfo->name) < 0) return -1;
   if (tDecodeCStrTo(pDecoder, pInfo->value) < 0) return -1;
   return 0;
 }
 
-
+int32_t tSerializeSShowVariablesRsp(void *buf, int32_t bufLen, SShowVariablesRsp *pRsp) {
-int32_t tSerializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesRsp* pRsp) {
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, bufLen);
 
@@ -2930,7 +2927,7 @@ int32_t tSerializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesRsp
   int32_t varNum = taosArrayGetSize(pRsp->variables);
   if (tEncodeI32(&encoder, varNum) < 0) return -1;
   for (int32_t i = 0; i < varNum; ++i) {
-    SVariablesInfo* pInfo = taosArrayGet(pRsp->variables, i);
+    SVariablesInfo *pInfo = taosArrayGet(pRsp->variables, i);
     if (tEncodeSVariablesInfo(&encoder, pInfo) < 0) return -1;
   }
   tEndEncode(&encoder);
@@ -2940,7 +2937,7 @@ int32_t tSerializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesRsp
   return tlen;
 }
 
-int32_t tDeserializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesRsp* pRsp) {
+int32_t tDeserializeSShowVariablesRsp(void *buf, int32_t bufLen, SShowVariablesRsp *pRsp) {
   SDecoder decoder = {0};
   tDecoderInit(&decoder, buf, bufLen);
 
@@ -2962,7 +2959,7 @@ int32_t tDeserializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesR
   return 0;
 }
 
-void tFreeSShowVariablesRsp(SShowVariablesRsp* pRsp) {
+void tFreeSShowVariablesRsp(SShowVariablesRsp *pRsp) {
   if (NULL == pRsp) {
     return;
   }
@@ -5362,30 +5359,149 @@ void tFreeSMAlterStbRsp(SMAlterStbRsp *pRsp) {
   }
 }
 
-int32_t tEncodeSTqOffset(SEncoder *pEncoder, const STqOffset *pOffset) {
+int32_t tEncodeSTqOffsetVal(SEncoder *pEncoder, const STqOffsetVal *pOffsetVal) {
-  if (tEncodeI8(pEncoder, pOffset->type) < 0) return -1;
+  if (tEncodeI8(pEncoder, pOffsetVal->type) < 0) return -1;
-  if (pOffset->type == TMQ_OFFSET__SNAPSHOT) {
+  if (pOffsetVal->type == TMQ_OFFSET__SNAPSHOT_DATA) {
-    if (tEncodeI64(pEncoder, pOffset->uid) < 0) return -1;
+    if (tEncodeI64(pEncoder, pOffsetVal->uid) < 0) return -1;
-    if (tEncodeI64(pEncoder, pOffset->ts) < 0) return -1;
+    if (tEncodeI64(pEncoder, pOffsetVal->ts) < 0) return -1;
-  } else if (pOffset->type == TMQ_OFFSET__LOG) {
+  } else if (pOffsetVal->type == TMQ_OFFSET__LOG) {
-    if (tEncodeI64(pEncoder, pOffset->version) < 0) return -1;
+    if (tEncodeI64(pEncoder, pOffsetVal->version) < 0) return -1;
+  } else if (pOffsetVal->type < 0) {
+    // do nothing
   } else {
     ASSERT(0);
   }
+  return 0;
+}
+
+int32_t tDecodeSTqOffsetVal(SDecoder *pDecoder, STqOffsetVal *pOffsetVal) {
+  if (tDecodeI8(pDecoder, &pOffsetVal->type) < 0) return -1;
+  if (pOffsetVal->type == TMQ_OFFSET__SNAPSHOT_DATA) {
+    if (tDecodeI64(pDecoder, &pOffsetVal->uid) < 0) return -1;
+    if (tDecodeI64(pDecoder, &pOffsetVal->ts) < 0) return -1;
+  } else if (pOffsetVal->type == TMQ_OFFSET__LOG) {
+    if (tDecodeI64(pDecoder, &pOffsetVal->version) < 0) return -1;
+  } else if (pOffsetVal->type < 0) {
+    // do nothing
+  } else {
+    ASSERT(0);
+  }
+  return 0;
+}
+
+#if 1
+int32_t tFormatOffset(char *buf, int32_t maxLen, const STqOffsetVal *pVal) {
+  if (pVal->type == TMQ_OFFSET__RESET_NONE) {
+    snprintf(buf, maxLen, "offset(reset to none)");
+  } else if (pVal->type == TMQ_OFFSET__RESET_EARLIEAST) {
+    snprintf(buf, maxLen, "offset(reset to earlieast)");
+  } else if (pVal->type == TMQ_OFFSET__RESET_LATEST) {
+    snprintf(buf, maxLen, "offset(reset to latest)");
+  } else if (pVal->type == TMQ_OFFSET__LOG) {
+    snprintf(buf, maxLen, "offset(log) ver:%ld", pVal->version);
+  } else if (pVal->type == TMQ_OFFSET__SNAPSHOT_DATA) {
+    snprintf(buf, maxLen, "offset(snapshot data) uid:%ld, ts:%ld", pVal->uid, pVal->ts);
+  } else if (pVal->type == TMQ_OFFSET__SNAPSHOT_META) {
+    snprintf(buf, maxLen, "offset(snapshot meta) uid:%ld, ts:%ld", pVal->uid, pVal->ts);
+  } else {
+    ASSERT(0);
+  }
+  return 0;
+}
+#endif
+
+bool tOffsetEqual(const STqOffsetVal *pLeft, const STqOffsetVal *pRight) {
+  if (pLeft->type == pRight->type) {
+    if (pLeft->type == TMQ_OFFSET__LOG) {
+      return pLeft->version == pRight->version;
+    } else if (pLeft->type == TMQ_OFFSET__SNAPSHOT_DATA) {
+      return pLeft->uid == pRight->uid && pLeft->ts == pRight->ts;
+    } else if (pLeft->type == TMQ_OFFSET__SNAPSHOT_META) {
+      ASSERT(0);
+      // TODO
+      return pLeft->uid == pRight->uid && pLeft->ts == pRight->ts;
+    }
+  }
+  return false;
+}
+
+int32_t tEncodeSTqOffset(SEncoder *pEncoder, const STqOffset *pOffset) {
+  if (tEncodeSTqOffsetVal(pEncoder, &pOffset->val) < 0) return -1;
   if (tEncodeCStr(pEncoder, pOffset->subKey) < 0) return -1;
   return 0;
 }
 
 int32_t tDecodeSTqOffset(SDecoder *pDecoder, STqOffset *pOffset) {
-  if (tDecodeI8(pDecoder, &pOffset->type) < 0) return -1;
+  if (tDecodeSTqOffsetVal(pDecoder, &pOffset->val) < 0) return -1;
-  if (pOffset->type == TMQ_OFFSET__SNAPSHOT) {
-    if (tDecodeI64(pDecoder, &pOffset->uid) < 0) return -1;
-    if (tDecodeI64(pDecoder, &pOffset->ts) < 0) return -1;
-  } else if (pOffset->type == TMQ_OFFSET__LOG) {
-    if (tDecodeI64(pDecoder, &pOffset->version) < 0) return -1;
-  } else {
-    ASSERT(0);
-  }
   if (tDecodeCStrTo(pDecoder, pOffset->subKey) < 0) return -1;
   return 0;
 }
+
+int32_t tEncodeSMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
+  if (tEncodeSTqOffsetVal(pEncoder, &pRsp->reqOffset) < 0) return -1;
+  if (tEncodeSTqOffsetVal(pEncoder, &pRsp->rspOffset) < 0) return -1;
+  if (tEncodeI32(pEncoder, pRsp->skipLogNum) < 0) return -1;
+  if (tEncodeI32(pEncoder, pRsp->blockNum) < 0) return -1;
+  if (pRsp->blockNum != 0) {
+    if (tEncodeI8(pEncoder, pRsp->withTbName) < 0) return -1;
+    if (tEncodeI8(pEncoder, pRsp->withSchema) < 0) return -1;
+
+    for (int32_t i = 0; i < pRsp->blockNum; i++) {
+      int32_t bLen = *(int32_t *)taosArrayGet(pRsp->blockDataLen, i);
+      void   *data = taosArrayGetP(pRsp->blockData, i);
+      if (tEncodeBinary(pEncoder, (const uint8_t *)data, bLen) < 0) return -1;
+      if (pRsp->withSchema) {
+        SSchemaWrapper *pSW = (SSchemaWrapper *)taosArrayGetP(pRsp->blockSchema, i);
+        if (tEncodeSSchemaWrapper(pEncoder, pSW) < 0) return -1;
+      }
+      if (pRsp->withTbName) {
+        char *tbName = (char *)taosArrayGetP(pRsp->blockTbName, i);
+        if (tEncodeCStr(pEncoder, tbName) < 0) return -1;
+      }
+    }
+  }
+  return 0;
+}
+
+int32_t tDecodeSMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) {
+  if (tDecodeSTqOffsetVal(pDecoder, &pRsp->reqOffset) < 0) return -1;
+  if (tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pRsp->skipLogNum) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pRsp->blockNum) < 0) return -1;
+  if (pRsp->blockNum != 0) {
+    pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void *));
+    pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(int32_t));
+    if (tDecodeI8(pDecoder, &pRsp->withTbName) < 0) return -1;
+    if (tDecodeI8(pDecoder, &pRsp->withSchema) < 0) return -1;
+    if (pRsp->withTbName) {
+      pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void *));
+    }
+    if (pRsp->withSchema) {
+      pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void *));
+    }
+
+    for (int32_t i = 0; i < pRsp->blockNum; i++) {
+      void    *data;
+      uint64_t bLen;
+      if (tDecodeBinaryAlloc(pDecoder, &data, &bLen) < 0) return -1;
+      taosArrayPush(pRsp->blockData, &data);
+      int32_t len = bLen;
+      taosArrayPush(pRsp->blockDataLen, &len);
+
+      if (pRsp->withSchema) {
+        SSchemaWrapper *pSW = (SSchemaWrapper *)taosMemoryCalloc(1, sizeof(SSchemaWrapper));
+        if (pSW == NULL) return -1;
+        if (tDecodeSSchemaWrapper(pDecoder, pSW) < 0) return -1;
+        taosArrayPush(pRsp->blockSchema, &pSW);
+      }
+
+      if (pRsp->withTbName) {
+        char *tbName;
+        if (tDecodeCStrAlloc(pDecoder, &tbName) < 0) return -1;
+        taosArrayPush(pRsp->blockTbName, &tbName);
+      }
+    }
+  }
+  return 0;
+}
+

source/dnode/mgmt/mgmt_dnode/src/dmHandle.c

@@ -17,7 +17,6 @@
 #include "dmInt.h"
 #include "systable.h"
 
-
 extern SConfig *tsCfg;
 
 static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) {
@@ -186,10 +185,10 @@ int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   return 0;
 }
 
-SSDataBlock* dmBuildVariablesBlock(void) {
+SSDataBlock *dmBuildVariablesBlock(void) {
-  SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
+  SSDataBlock         *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
   size_t               size = 0;
-  const SSysTableMeta* pMeta = NULL;
+  const SSysTableMeta *pMeta = NULL;
   getInfosDbMeta(&pMeta, &size);
 
   int32_t index = 0;
@@ -215,7 +214,7 @@ SSDataBlock* dmBuildVariablesBlock(void) {
   return pBlock;
 }
 
-int32_t dmAppendVariablesToBlock(SSDataBlock* pBlock, int32_t dnodeId) {
+int32_t dmAppendVariablesToBlock(SSDataBlock *pBlock, int32_t dnodeId) {
   int32_t numOfCfg = taosArrayGetSize(tsCfg->array);
   int32_t numOfRows = 0;
   blockDataEnsureCapacity(pBlock, numOfCfg);
@@ -241,7 +240,6 @@ int32_t dmAppendVariablesToBlock(SSDataBlock* pBlock, int32_t dnodeId) {
     numOfRows++;
   }
 
-
   pBlock->info.rows = numOfRows;
 
   return TSDB_CODE_SUCCESS;
@@ -267,7 +265,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
     return -1;
   }
 
-  SSDataBlock* pBlock = dmBuildVariablesBlock();
+  SSDataBlock *pBlock = dmBuildVariablesBlock();
 
   dmAppendVariablesToBlock(pBlock, pMgmt->pData->dnodeId);
 
@@ -299,7 +297,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   }
 
   int32_t len = 0;
-  blockCompressEncode(pBlock, pStart, &len, numOfCols, false);
+  blockEncode(pBlock, pStart, &len, numOfCols, false);
 
   pRsp->numOfRows = htonl(pBlock->info.rows);
   pRsp->precision = TSDB_TIME_PRECISION_MILLI;  // millisecond time precision

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -107,13 +107,7 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf
     const STraceId *trace = &pMsg->info.traceId;
     dGTrace("vgId:%d, msg:%p get from vnode-sync queue", pVnode->vgId, pMsg);
 
-    int32_t code = vnodeProcessSyncReq(pVnode->pImpl, pMsg, NULL);
+    int32_t code = vnodeProcessSyncReq(pVnode->pImpl, pMsg, NULL); // no response here
-    if (code != 0) {
-      if (terrno != 0) code = terrno;
-      dGError("vgId:%d, msg:%p failed to sync since %s", pVnode->vgId, pMsg, terrstr());
-      vmSendRsp(pMsg, code);
-    }
-
     dGTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code);
     rpcFreeCont(pMsg->pCont);
     taosFreeQitem(pMsg);

source/dnode/mgmt/node_mgmt/src/dmTransport.c

@@ -265,7 +265,7 @@ int32_t dmInitClient(SDnode *pDnode) {
   SDnodeTrans *pTrans = &pDnode->trans;
 
   SRpcInit rpcInit = {0};
-  rpcInit.label = "DND";
+  rpcInit.label = "DND-C";
   rpcInit.numOfThreads = 1;
   rpcInit.cfp = (RpcCfp)dmProcessRpcMsg;
   rpcInit.sessions = 1024;
@@ -299,7 +299,7 @@ int32_t dmInitServer(SDnode *pDnode) {
   SRpcInit rpcInit = {0};
   strncpy(rpcInit.localFqdn, tsLocalFqdn, strlen(tsLocalFqdn));
   rpcInit.localPort = tsServerPort;
-  rpcInit.label = "DND";
+  rpcInit.label = "DND-S";
   rpcInit.numOfThreads = tsNumOfRpcThreads;
   rpcInit.cfp = (RpcCfp)dmProcessRpcMsg;
   rpcInit.sessions = tsMaxShellConns;

source/dnode/mnode/impl/src/mndShow.c

@@ -15,8 +15,8 @@
 
 #define _DEFAULT_SOURCE
 #include "mndShow.h"
-#include "systable.h"
 #include "mndPrivilege.h"
+#include "systable.h"
 
 #define SHOW_STEP_SIZE 100
 
@@ -307,7 +307,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
     }
 
     int32_t len = 0;
-    blockCompressEncode(pBlock, pStart, &len, pShow->pMeta->numOfColumns, false);
+    blockEncode(pBlock, pStart, &len, pShow->pMeta->numOfColumns, false);
   }
 
   pRsp->numOfRows = htonl(rowsRead);

source/dnode/vnode/inc/vnode.h

@@ -139,19 +139,19 @@ void    tsdbCleanupReadHandle(tsdbReaderT queryHandle);
 
 // tq
 
-typedef struct STqReadHandle STqReadHandle;
+typedef struct STqReadHandle SStreamReader;
 
-STqReadHandle *tqInitSubmitMsgScanner(SMeta *pMeta);
+SStreamReader *tqInitSubmitMsgScanner(SMeta *pMeta);
 
-void    tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SArray *pColIdList);
+void    tqReadHandleSetColIdList(SStreamReader *pReadHandle, SArray *pColIdList);
-int32_t tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList);
+int32_t tqReadHandleSetTbUidList(SStreamReader *pHandle, const SArray *tbUidList);
-int32_t tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList);
+int32_t tqReadHandleAddTbUidList(SStreamReader *pHandle, const SArray *tbUidList);
-int32_t tqReadHandleRemoveTbUidList(STqReadHandle *pHandle, const SArray *tbUidList);
+int32_t tqReadHandleRemoveTbUidList(SStreamReader *pHandle, const SArray *tbUidList);
 
-int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver);
+int32_t tqReadHandleSetMsg(SStreamReader *pHandle, SSubmitReq *pMsg, int64_t ver);
-bool    tqNextDataBlock(STqReadHandle *pHandle);
+bool    tqNextDataBlock(SStreamReader *pHandle);
-bool    tqNextDataBlockFilterOut(STqReadHandle *pHandle, SHashObj *filterOutUids);
+bool    tqNextDataBlockFilterOut(SStreamReader *pHandle, SHashObj *filterOutUids);
-int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReadHandle *pHandle);
+int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, SStreamReader *pHandle);
 
 // sma
 int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);

source/dnode/vnode/src/inc/sma.h

@@ -46,6 +46,10 @@ struct SSmaEnv {
   SSmaStat      *pStat;
 };
 
+typedef struct {
+  int32_t smaRef;
+} SSmaMgmt;
+
 #define SMA_ENV_LOCK(env) ((env)->lock)
 #define SMA_ENV_TYPE(env) ((env)->type)
 #define SMA_ENV_STAT(env) ((env)->pStat)
@@ -58,11 +62,12 @@ struct STSmaStat {
 
 struct SRSmaStat {
   SSma     *pSma;
-  void     *tmrHandle;
+  int64_t   refId;         // shared by persistence/fetch tasks
-  tmr_h     tmrId;
+  void     *tmrHandle;     // for persistence task
-  int32_t   tmrSeconds;
+  tmr_h     tmrId;         // for persistence task
-  int8_t    triggerStat;
+  int32_t   tmrSeconds;    // for persistence task
-  int8_t    runningStat;
+  int8_t    triggerStat;   // for persistence task
+  int8_t    runningStat;   // for persistence task
   SHashObj *rsmaInfoHash;  // key: stbUid, value: SRSmaInfo;
 };
 
@@ -73,6 +78,7 @@ struct SSmaStat {
   };
   T_REF_DECLARE()
 };
+
 #define SMA_TSMA_STAT(s)     (&(s)->tsmaStat)
 #define SMA_RSMA_STAT(s)     (&(s)->rsmaStat)
 #define RSMA_INFO_HASH(r)    ((r)->rsmaInfoHash)
@@ -80,6 +86,7 @@ struct SSmaStat {
 #define RSMA_TMR_HANDLE(r)   ((r)->tmrHandle)
 #define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat)
 #define RSMA_RUNNING_STAT(r) (&(r)->runningStat)
+#define RSMA_REF_ID(r)       ((r)->refId)
 
 enum {
   TASK_TRIGGER_STAT_INIT = 0,
@@ -192,10 +199,18 @@ typedef struct STFInfo STFInfo;
 typedef struct STFile  STFile;
 
 struct STFInfo {
+  // common fields
   uint32_t magic;
   uint32_t ftype;
   uint32_t fver;
   int64_t  fsize;
+
+  // specific fields
+  union {
+    struct {
+      int64_t applyVer[2];
+    } qTaskInfo;
+  };
 };
 
 struct STFile {
@@ -230,7 +245,7 @@ int32_t tdUpdateTFileHeader(STFile *pTFile);
 void    tdUpdateTFileMagic(STFile *pTFile, void *pCksm);
 void    tdCloseTFile(STFile *pTFile);
 
-void tdGetVndFileName(int32_t vid, const char *dname, const char *fname, char *outputName);
+void tdGetVndFileName(int32_t vgId, const char *dname, const char *fname, char *outputName);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/src/inc/tq.h

@@ -39,6 +39,16 @@ extern "C" {
 #define tqInfo(...)  do { if (tqDebugFlag & DEBUG_INFO)  { taosPrintLog("TQ  ", DEBUG_INFO, 255, __VA_ARGS__); }}            while(0)
 #define tqDebug(...) do { if (tqDebugFlag & DEBUG_DEBUG) { taosPrintLog("TQ  ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); }} while(0)
 #define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ  ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0)
+
+#define IS_META_MSG(x) ( \
+     x == TDMT_VND_CREATE_STB     \
+  || x == TDMT_VND_ALTER_STB      \
+  || x == TDMT_VND_DROP_STB       \
+  || x == TDMT_VND_CREATE_TABLE   \
+  || x == TDMT_VND_ALTER_TABLE    \
+  || x == TDMT_VND_DROP_TABLE     \
+  || x == TDMT_VND_DROP_TTL_TABLE \
+)
 // clang-format on
 
 typedef struct STqOffsetStore STqOffsetStore;
@@ -101,12 +111,13 @@ typedef struct {
 typedef struct {
   int8_t subType;
 
-  STqReadHandle* pExecReader[5];
+  SStreamReader* pExecReader[5];
   union {
     STqExecCol execCol;
     STqExecTb  execTb;
     STqExecDb  execDb;
   };
+
 } STqExecHandle;
 
 typedef struct {
@@ -149,9 +160,9 @@ static STqMgmt tqMgmt = {0};
 int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalHead** pHeadWithCkSum);
 
 // tqExec
-int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkRsp* pRsp, int32_t workerId);
+int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataRsp* pRsp, int32_t workerId);
-int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp, int32_t workerId);
+int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, int32_t workerId);
-int32_t tqSendPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataBlkRsp* pRsp);
+int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp);
 
 // tqMeta
 int32_t tqMetaOpen(STQ* pTq);

source/dnode/vnode/src/inc/vnodeInt.h

@@ -27,6 +27,7 @@
 #include "tdatablock.h"
 #include "tdb.h"
 #include "tencode.h"
+#include "tref.h"
 #include "tfs.h"
 #include "tglobal.h"
 #include "tjson.h"

source/dnode/vnode/src/sma/smaEnv.c

@@ -18,6 +18,9 @@
 typedef struct SSmaStat SSmaStat;
 
 #define RSMA_TASK_INFO_HASH_SLOT 8
+#define SMA_MGMT_REF_NUM         1024
+
+extern SSmaMgmt smaMgmt;
 
 // declaration of static functions
 
@@ -25,6 +28,7 @@ static int32_t  tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *p
 static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path);
 static int32_t  tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SSmaEnv **pEnv);
 static void    *tdFreeTSmaStat(STSmaStat *pStat);
+static void     tdDestroyRSmaStat(void *pRSmaStat);
 
 // implementation
 
@@ -128,6 +132,22 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS
     if (smaType == TSDB_SMA_TYPE_ROLLUP) {
       SRSmaStat *pRSmaStat = (SRSmaStat *)(*pSmaStat);
       pRSmaStat->pSma = (SSma *)pSma;
+
+      // init smaMgmt
+      smaMgmt.smaRef = taosOpenRef(SMA_MGMT_REF_NUM, tdDestroyRSmaStat);
+      if (smaMgmt.smaRef < 0) {
+        smaError("init smaRef failed, num:%d", SMA_MGMT_REF_NUM);
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        return TSDB_CODE_FAILED;
+      }
+
+      int64_t refId = taosAddRef(smaMgmt.smaRef, pRSmaStat);
+      if (refId < 0) {
+        smaError("taosAddRef smaRef failed, since:%s", tstrerror(terrno));
+        return TSDB_CODE_FAILED;
+      }
+      pRSmaStat->refId = refId;
+
       // init timer
       RSMA_TMR_HANDLE(pRSmaStat) = taosTmrInit(10000, 100, 10000, "RSMA");
       if (!RSMA_TMR_HANDLE(pRSmaStat)) {
@@ -169,9 +189,10 @@ static void *tdFreeTSmaStat(STSmaStat *pStat) {
   return NULL;
 }
 
-static void tdDestroyRSmaStat(SRSmaStat *pStat) {
+static void tdDestroyRSmaStat(void *pRSmaStat) {
-  if (pStat) {
+  if (pRSmaStat) {
-    smaDebug("vgId:%d destroy rsma stat", SMA_VID(pStat->pSma));
+    SRSmaStat *pStat = (SRSmaStat *)pRSmaStat;
+    smaDebug("vgId:%d %s:%d destroy rsma stat %p", SMA_VID(pStat->pSma), __func__, __LINE__, pRSmaStat);
     // step 1: set persistence task cancelled
     atomic_store_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_CANCELLED);
 
@@ -183,9 +204,11 @@ static void tdDestroyRSmaStat(SRSmaStat *pStat) {
     if (atomic_load_8(RSMA_RUNNING_STAT(pStat)) == 1) {
       while (1) {
         if (atomic_load_8(RSMA_TRIGGER_STAT(pStat)) == TASK_TRIGGER_STAT_FINISHED) {
+          smaDebug("rsma, persist task finished already");
           break;
         } else {
-          smaDebug("not destroyed since rsma stat in %" PRIi8, atomic_load_8(RSMA_TRIGGER_STAT(pStat)));
+          smaDebug("rsma, persist task not finished yet since rsma stat in %" PRIi8,
+                   atomic_load_8(RSMA_TRIGGER_STAT(pStat)));
         }
         ++nLoops;
         if (nLoops > 1000) {
@@ -209,7 +232,10 @@ static void tdDestroyRSmaStat(SRSmaStat *pStat) {
     nLoops = 0;
     while (1) {
       if (T_REF_VAL_GET((SSmaStat *)pStat) == 0) {
+        smaDebug("rsma, all fetch task finished already");
         break;
+      } else {
+        smaDebug("rsma, fetch tasks not all finished yet");
       }
       ++nLoops;
       if (nLoops > 1000) {
@@ -225,15 +251,13 @@ static void tdDestroyRSmaStat(SRSmaStat *pStat) {
   }
 }
 
-static void *tdFreeRSmaStat(SRSmaStat *pStat) {
-  tdDestroyRSmaStat(pStat);
-  taosMemoryFreeClear(pStat);
-  return NULL;
-}
-
 void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) {
   tdDestroySmaState(pSmaStat, smaType);
+  if (smaType == TSDB_SMA_TYPE_TIME_RANGE) {
     taosMemoryFreeClear(pSmaStat);
+  }
+  // tref used to free rsma stat
+
   return NULL;
 }
 
@@ -243,17 +267,21 @@ void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) {
  * @param pSmaStat
  * @return int32_t
  */
+
 int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) {
   if (pSmaStat) {
     if (smaType == TSDB_SMA_TYPE_TIME_RANGE) {
       tdDestroyTSmaStat(SMA_TSMA_STAT(pSmaStat));
     } else if (smaType == TSDB_SMA_TYPE_ROLLUP) {
-      tdDestroyRSmaStat(SMA_RSMA_STAT(pSmaStat));
+      SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSmaStat);
+      if (taosRemoveRef(smaMgmt.smaRef, RSMA_REF_ID(pRSmaStat)) < 0) {
+        smaError("remove refId from smaRef failed, refId:0x%" PRIx64, RSMA_REF_ID(pRSmaStat));
+      }
     } else {
       ASSERT(0);
     }
   }
-  return TSDB_CODE_SUCCESS;
+  return 0;
 }
 
 int32_t tdLockSma(SSma *pSma) {

source/dnode/vnode/src/sma/smaRollup.c

@@ -18,9 +18,13 @@
 #define RSMA_QTASKINFO_PERSIST_MS 7200000
 #define RSMA_QTASKINFO_BUFSIZE    32768
 #define RSMA_QTASKINFO_HEAD_LEN   (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t))  // len + type + suid
-typedef enum { TD_QTASK_TMP_FILE = 0, TD_QTASK_CUR_FILE } TD_QTASK_FILE_T;
-static const char *tdQTaskInfoFname[] = {"qtaskinfo.t", "qtaskinfo"};
 
+SSmaMgmt smaMgmt = {
+    .smaRef = -1,
+};
+
+typedef enum { TD_QTASK_TMP_F = 0, TD_QTASK_CUR_F } TD_QTASK_FILE_T;
+static const char                *tdQTaskInfoFname[] = {"qtaskinfo.t", "qtaskinfo"};
 typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem;
 typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter;
 
@@ -46,6 +50,7 @@ static int32_t tdRSmaRestoreTSDataReload(SSma *pSma);
 
 struct SRSmaInfoItem {
   SRSmaInfo *pRsmaInfo;
+  int64_t    refId;
   void      *taskInfo;  // qTaskInfo_t
   tmr_h      tmrId;
   int8_t     level;
@@ -56,11 +61,14 @@ struct SRSmaInfoItem {
 
 struct SRSmaInfo {
   STSchema     *pTSchema;
-  SSma         *pSma;
+  SRSmaStat    *pStat;
   int64_t       suid;
   SRSmaInfoItem items[TSDB_RETENTION_L2];
 };
 
+#define RSMA_INFO_SMA(r)  ((r)->pStat->pSma)
+#define RSMA_INFO_STAT(r) ((r)->pStat)
+
 struct SRSmaQTaskInfoItem {
   int32_t len;
   int8_t  type;
@@ -80,6 +88,10 @@ struct SRSmaQTaskInfoIter {
   int32_t nBufPos;
 };
 
+static void tdRSmaQTaskInfoGetFName(int32_t vgId, int8_t ftype, char *outputName) {
+  tdGetVndFileName(vgId, VNODE_RSMA_DIR, tdQTaskInfoFname[ftype], outputName);
+}
+
 static FORCE_INLINE int32_t tdRSmaQTaskInfoContLen(int32_t lenWithHead) {
   return lenWithHead - RSMA_QTASKINFO_HEAD_LEN;
 }
@@ -99,22 +111,21 @@ static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle, int32_t vgId,
 
 void *tdFreeRSmaInfo(SRSmaInfo *pInfo) {
   if (pInfo) {
+    SSma *pSma = RSMA_INFO_SMA(pInfo);
     for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
       SRSmaInfoItem *pItem = &pInfo->items[i];
       if (pItem->taskInfo) {
-        smaDebug("vgId:%d, stb %" PRIi64 " stop fetch-timer %p level %d", SMA_VID(pInfo->pSma), pInfo->suid,
+        smaDebug("vgId:%d, stb %" PRIi64 " stop fetch-timer %p level %d", SMA_VID(pSma), pInfo->suid, pItem->tmrId,
-                 pItem->tmrId, i + 1);
+                 i + 1);
         taosTmrStopA(&pItem->tmrId);
-        tdFreeTaskHandle(&pItem->taskInfo, SMA_VID(pInfo->pSma), i + 1);
+        tdFreeTaskHandle(&pItem->taskInfo, SMA_VID(pSma), i + 1);
       } else {
-        smaDebug("vgId:%d, stb %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo",
+        smaDebug("vgId:%d, stb %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", SMA_VID(pSma),
-                 SMA_VID(pInfo->pSma), pInfo->suid, i + 1);
+                 pInfo->suid, i + 1);
       }
     }
     taosMemoryFree(pInfo->pTSchema);
     taosMemoryFree(pInfo);
-  } else {
-    smaDebug("vgId:%d, stb %" PRIi64 " no need to destroy rsma info since empty", SMA_VID(pInfo->pSma), pInfo->suid);
   }
 
   return NULL;
@@ -247,6 +258,7 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo
 
   if (param->qmsg[idx]) {
     SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]);
+    pItem->refId = RSMA_REF_ID(pRSmaInfo->pStat);
     pItem->pRsmaInfo = pRSmaInfo;
     pItem->taskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], pReadHandle);
     if (!pItem->taskInfo) {
@@ -313,7 +325,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
     return TSDB_CODE_FAILED;
   }
 
-  STqReadHandle *pReadHandle = tqInitSubmitMsgScanner(pMeta);
+  SStreamReader *pReadHandle = tqInitSubmitMsgScanner(pMeta);
   if (!pReadHandle) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
@@ -332,7 +344,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
     goto _err;
   }
   pRSmaInfo->pTSchema = pTSchema;
-  pRSmaInfo->pSma = pSma;
+  pRSmaInfo->pStat = pStat;
   pRSmaInfo->suid = suid;
 
   if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 0) < 0) {
@@ -514,7 +526,7 @@ static void tdDestroySDataBlockArray(SArray *pArray) {
 static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) {
   SArray    *pResult = NULL;
   SRSmaInfo *pRSmaInfo = pItem->pRsmaInfo;
-  SSma      *pSma = pRSmaInfo->pSma;
+  SSma      *pSma = RSMA_INFO_SMA(pRSmaInfo);
 
   while (1) {
     SSDataBlock *output = NULL;
@@ -577,34 +589,44 @@ _err:
  */
 static void tdRSmaFetchTrigger(void *param, void *tmrId) {
   SRSmaInfoItem *pItem = param;
-  SSma          *pSma = pItem->pRsmaInfo->pSma;
+  SSma          *pSma = NULL;
-  SRSmaStat     *pStat = (SRSmaStat *)SMA_ENV_STAT((SSmaEnv *)pSma->pRSmaEnv);
+  SRSmaStat     *pStat = (SRSmaStat *)taosAcquireRef(smaMgmt.smaRef, pItem->refId);
+  if (!pStat) {
+    smaDebug("rsma fetch task not start since already destroyed");
+    return;
+  }
 
+  pSma = RSMA_INFO_SMA(pItem->pRsmaInfo);
+
+  // if rsma trigger stat in cancelled or finished, not start fetch task anymore
   int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat));
   if (rsmaTriggerStat == TASK_TRIGGER_STAT_CANCELLED || rsmaTriggerStat == TASK_TRIGGER_STAT_FINISHED) {
-    smaDebug("vgId:%d, level %" PRIi8 " not fetch since stat is cancelled for table suid:%" PRIi64, SMA_VID(pSma),
+    taosReleaseRef(smaMgmt.smaRef, pItem->refId);
-             pItem->level, pItem->pRsmaInfo->suid);
+    smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is cancelled",
+             SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
     return;
   }
 
   int8_t fetchTriggerStat =
       atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
   if (fetchTriggerStat == TASK_TRIGGER_STAT_ACTIVE) {
-    smaDebug("vgId:%d, level %" PRIi8 " stat is active for table suid:%" PRIi64, SMA_VID(pSma), pItem->level,
+    smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma),
-             pItem->pRsmaInfo->suid);
+             pItem->level, pItem->pRsmaInfo->suid);
 
     tdRefSmaStat(pSma, (SSmaStat *)pStat);
 
     SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL};
-    qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_DATA_TYPE_SSDATA_BLOCK, false);
+    qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false);
-    tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SSDATA_BLOCK);
+    tdFetchAndSubmitRSmaResult(pItem, STREAM_INPUT__DATA_BLOCK);
 
     tdUnRefSmaStat(pSma, (SSmaStat *)pStat);
 
   } else {
-    smaDebug("vgId:%d, level %" PRIi8 " stat is inactive for table suid:%" PRIi64, SMA_VID(pSma), pItem->level,
+    smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is inactive",
-             pItem->pRsmaInfo->suid);
+             SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
   }
+_end:
+  taosReleaseRef(smaMgmt.smaRef, pItem->refId);
 }
 
 static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem, tb_uid_t suid,
@@ -617,14 +639,13 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType
   smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level,
            pItem->taskInfo, suid);
 
-  if (qSetStreamInput(pItem->taskInfo, pMsg, inputType, true) < 0) {  // STREAM_DATA_TYPE_SUBMIT_BLOCK
+  if (qSetStreamInput(pItem->taskInfo, pMsg, inputType, true) < 0) {  // INPUT__DATA_SUBMIT
     smaError("vgId:%d, rsma % " PRIi8 " qSetStreamInput failed since %s", SMA_VID(pSma), level, tstrerror(terrno));
     return TSDB_CODE_FAILED;
   }
 
-  tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SUBMIT_BLOCK);
+  tdFetchAndSubmitRSmaResult(pItem, STREAM_INPUT__DATA_SUBMIT);
   atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE);
-  smaDebug("vgId:%d, process rsma insert", SMA_VID(pSma));
 
   SSmaEnv   *pEnv = SMA_RSMA_ENV(pSma);
   SRSmaStat *pStat = SMA_RSMA_STAT(pEnv->pStat);
@@ -660,7 +681,7 @@ static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb
     return TSDB_CODE_SUCCESS;
   }
 
-  if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
+  if (inputType == STREAM_INPUT__DATA_SUBMIT) {
     tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[0], suid, TSDB_RETENTION_L1);
     tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[1], suid, TSDB_RETENTION_L2);
   }
@@ -681,7 +702,7 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) {
     return TSDB_CODE_SUCCESS;
   }
 
-  if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
+  if (inputType == STREAM_INPUT__DATA_SUBMIT) {
     STbUidStore uidStore = {0};
     tdFetchSubmitReqSuids(pMsg, &uidStore);
 
@@ -761,7 +782,7 @@ static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma) {
   STFile  tFile = {0};
   char    qTaskInfoFName[TSDB_FILENAME_LEN];
 
-  tdRSmaQTaskInfoGetFName(TD_VID(pVnode), TD_QTASK_CUR_FILE, qTaskInfoFName);
+  tdRSmaQTaskInfoGetFName(TD_VID(pVnode), TD_QTASK_TMP_F, qTaskInfoFName);
   if (tdInitTFile(&tFile, pVnode->pTfs, qTaskInfoFName) < 0) {
     goto _err;
   }
@@ -861,7 +882,8 @@ static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *
              pItem->type, terrstr(terrno));
     return TSDB_CODE_FAILED;
   }
-  smaDebug("vgId:%d, restore rsma task success for table:%" PRIi64 " level %d", SMA_VID(pSma), pItem->suid, pItem->type);
+  smaDebug("vgId:%d, restore rsma task success for table:%" PRIi64 " level %d", SMA_VID(pSma), pItem->suid,
+           pItem->type);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -1003,10 +1025,6 @@ static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter) {
   return TSDB_CODE_SUCCESS;
 }
 
-static void tdRSmaQTaskInfoGetFName(int32_t vid, int8_t ftype, char *outputName) {
-  tdGetVndFileName(vid, VNODE_RSMA_DIR, tdQTaskInfoFname[ftype], outputName);
-}
-
 static void *tdRSmaPersistExec(void *param) {
   setThreadName("rsma-task-persist");
   SRSmaStat *pRSmaStat = param;
@@ -1031,7 +1049,7 @@ static void *tdRSmaPersistExec(void *param) {
     for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
       qTaskInfo_t taskInfo = pRSmaInfo->items[i].taskInfo;
       if (!taskInfo) {
-        smaDebug("vgId:%d, table %" PRIi64 " level %d qTaskInfo is NULL", vid, pRSmaInfo->suid, i + 1);
+        smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d qTaskInfo is NULL", vid, pRSmaInfo->suid, i + 1);
         continue;
       }
 
@@ -1039,31 +1057,24 @@ static void *tdRSmaPersistExec(void *param) {
       int32_t len = 0;
       int8_t  type = (int8_t)(i + 1);
       if (qSerializeTaskStatus(taskInfo, &pOutput, &len) < 0) {
-        smaError("vgId:%d, table %" PRIi64 " level %d serialize rsma task failed since %s", vid, pRSmaInfo->suid, i + 1,
+        smaError("vgId:%d, rsma, table %" PRIi64 " level %d serialize qTaskInfo failed since %s", vid, pRSmaInfo->suid,
-                 terrstr(terrno));
+                 i + 1, terrstr(terrno));
         goto _err;
       }
       if (!pOutput || len <= 0) {
-        smaDebug("vgId:%d, table %" PRIi64 " level %d serialize rsma task success but no output(len %d), not persist",
+        smaDebug("vgId:%d, rsma, table %" PRIi64
+                 " level %d serialize qTaskInfo success but no output(len %d), not persist",
                  vid, pRSmaInfo->suid, i + 1, len);
         taosMemoryFreeClear(pOutput);
         continue;
       }
 
-      smaDebug("vgId:%d, table %" PRIi64 " level %d serialize rsma task success with len %d, need persist", vid,
+      smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d serialize qTaskInfo success with len %d, need persist", vid,
                pRSmaInfo->suid, i + 1, len);
-#if 0
-      if (qDeserializeTaskStatus(taskInfo, pOutput, len) < 0) {
-        smaError("vgId:%d, table %" PRIi64 "level %d  deserialize rsma task failed since %s", vid, pRSmaInfo->suid,
-                 i + 1, terrstr(terrno));
-      } else {
-        smaDebug("vgId:%d, table %" PRIi64 " level %d deserialize rsma task success", vid, pRSmaInfo->suid, i + 1);
-      }
-#endif
 
       if (!isFileCreated) {
         char qTaskInfoFName[TSDB_FILENAME_LEN];
-        tdRSmaQTaskInfoGetFName(vid, TD_QTASK_TMP_FILE, qTaskInfoFName);
+        tdRSmaQTaskInfoGetFName(vid, TD_QTASK_TMP_F, qTaskInfoFName);
         tdInitTFile(&tFile, pTfs, qTaskInfoFName);
         tdCreateTFile(&tFile, pTfs, true, -1);
 
@@ -1079,11 +1090,11 @@ static void *tdRSmaPersistExec(void *param) {
 
       ASSERT(headLen <= RSMA_QTASKINFO_HEAD_LEN);
       tdAppendTFile(&tFile, (void *)&tmpBuf, headLen, &toffset);
-      smaDebug("vgId:%d, table %" PRIi64 " level %d head part len:%d appended to offset:%" PRIi64, vid, pRSmaInfo->suid,
+      smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d head part(len:%d) appended to offset:%" PRIi64, vid,
-               i + 1, headLen, toffset);
+               pRSmaInfo->suid, i + 1, headLen, toffset);
       tdAppendTFile(&tFile, pOutput, len, &toffset);
-      smaDebug("vgId:%d, table %" PRIi64 " level %d body part len:%d appended to offset:%" PRIi64, vid, pRSmaInfo->suid,
+      smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d body part len:%d appended to offset:%" PRIi64, vid,
-               i + 1, len, toffset);
+               pRSmaInfo->suid, i + 1, len, toffset);
 
       taosMemoryFree(pOutput);
     }
@@ -1093,26 +1104,26 @@ static void *tdRSmaPersistExec(void *param) {
 _normal:
   if (isFileCreated) {
     if (tdUpdateTFileHeader(&tFile) < 0) {
-      smaError("vgId:%d, failed to update tfile %s header since %s", vid, TD_TFILE_FULL_NAME(&tFile),
+      smaError("vgId:%d, rsma, failed to update tfile %s header since %s", vid, TD_TFILE_FULL_NAME(&tFile),
                tstrerror(terrno));
       tdCloseTFile(&tFile);
       tdRemoveTFile(&tFile);
       goto _err;
     } else {
-      smaDebug("vgId:%d, succeed to update tfile %s header", vid, TD_TFILE_FULL_NAME(&tFile));
+      smaDebug("vgId:%d, rsma, succeed to update tfile %s header", vid, TD_TFILE_FULL_NAME(&tFile));
     }
 
     tdCloseTFile(&tFile);
 
     char newFName[TSDB_FILENAME_LEN];
     strncpy(newFName, TD_TFILE_FULL_NAME(&tFile), TSDB_FILENAME_LEN);
-    char *pos = strstr(newFName, tdQTaskInfoFname[TD_QTASK_TMP_FILE]);
+    char *pos = strstr(newFName, tdQTaskInfoFname[TD_QTASK_TMP_F]);
-    strncpy(pos, tdQTaskInfoFname[TD_QTASK_CUR_FILE], TSDB_FILENAME_LEN - POINTER_DISTANCE(pos, newFName));
+    strncpy(pos, tdQTaskInfoFname[TD_QTASK_TMP_F], TSDB_FILENAME_LEN - POINTER_DISTANCE(pos, newFName));
     if (taosRenameFile(TD_TFILE_FULL_NAME(&tFile), newFName) != 0) {
-      smaError("vgId:%d, failed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName);
+      smaError("vgId:%d, rsma, failed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName);
       goto _err;
     } else {
-      smaDebug("vgId:%d, succeed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName);
+      smaDebug("vgId:%d, rsma, succeed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName);
     }
   }
   goto _end;
@@ -1124,16 +1135,17 @@ _end:
   if (TASK_TRIGGER_STAT_INACTIVE == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat),
                                                                   TASK_TRIGGER_STAT_INACTIVE,
                                                                   TASK_TRIGGER_STAT_ACTIVE)) {
-    smaDebug("vgId:%d, persist task is active again", vid);
+    smaDebug("vgId:%d, rsma persist task is active again", vid);
   } else if (TASK_TRIGGER_STAT_CANCELLED == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat),
                                                                           TASK_TRIGGER_STAT_CANCELLED,
                                                                           TASK_TRIGGER_STAT_FINISHED)) {
-    smaDebug("vgId:%d, persist task is cancelled", vid);
+    smaDebug("vgId:%d, rsma persist task is cancelled", vid);
   } else {
-    smaWarn("vgId:%d, persist task in abnormal stat %" PRIi8, vid, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)));
+    smaWarn("vgId:%d, rsma persist task in abnormal stat %" PRIi8, vid, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)));
     ASSERT(0);
   }
   atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
+  taosReleaseRef(smaMgmt.smaRef, pRSmaStat->refId);
   taosThreadExit(NULL);
   return NULL;
 }
@@ -1159,6 +1171,7 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) {
       ASSERT(0);
     }
     atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
+    taosReleaseRef(smaMgmt.smaRef, pRSmaStat->refId);
   }
 
   taosThreadAttrDestroy(&thAttr);
@@ -1171,7 +1184,13 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) {
  * @param tmrId
  */
 static void tdRSmaPersistTrigger(void *param, void *tmrId) {
-  SRSmaStat *pRSmaStat = param;
+  SRSmaStat *rsmaStat = param;
+  SRSmaStat *pRSmaStat = (SRSmaStat *)taosAcquireRef(smaMgmt.smaRef, rsmaStat->refId);
+
+  if (!pRSmaStat) {
+    smaDebug("rsma persistence task not start since already destroyed");
+    return;
+  }
 
   int8_t tmrStat =
       atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
@@ -1191,6 +1210,7 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) {
       } else {
         atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
       }
+      return;
     } break;
     case TASK_TRIGGER_STAT_CANCELLED: {
       atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_FINISHED);
@@ -1206,4 +1226,5 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) {
       smaWarn("rsma persistence not start since unknown stat %" PRIi8, tmrStat);
     } break;
   }
+  taosReleaseRef(smaMgmt.smaRef, rsmaStat->refId);
 }

source/dnode/vnode/src/sma/smaUtil.c

@@ -141,8 +141,8 @@ int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset)
   }
 
 #if 1
-  smaDebug("append to file %s, offset:%" PRIi64 " + nbyte:%" PRIi64 " =%" PRIi64, TD_TFILE_FULL_NAME(pTFile), toffset,
+  smaDebug("append to file %s, offset:%" PRIi64 " nbyte:%" PRIi64 " fsize:%" PRIi64, TD_TFILE_FULL_NAME(pTFile),
-           nbyte, toffset + nbyte);
+           toffset, nbyte, toffset + nbyte);
 #endif
 
   ASSERT(pTFile->info.fsize == toffset);
@@ -179,8 +179,8 @@ void tdCloseTFile(STFile *pTFile) {
   }
 }
 
-void tdGetVndFileName(int32_t vid, const char *dname, const char *fname, char *outputName) {
+void tdGetVndFileName(int32_t vgId, const char *dname, const char *fname, char *outputName) {
-  snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/%s", vid, dname, fname);
+  snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/%s", vgId, dname, fname);
 }
 
 int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname) {

source/dnode/vnode/src/tq/tq.c

@@ -113,8 +113,23 @@ int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq,
   return 0;
 }
 
-int32_t tqSendPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataBlkRsp* pRsp) {
+int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp) {
-  int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, pRsp);
+  ASSERT(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum);
+  ASSERT(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum);
+
+  if (pRsp->withSchema) {
+    ASSERT(taosArrayGetSize(pRsp->blockSchema) == pRsp->blockNum);
+  } else {
+    ASSERT(taosArrayGetSize(pRsp->blockSchema) == 0);
+  }
+
+  int32_t len;
+  int32_t code;
+  tEncodeSize(tEncodeSMqDataRsp, pRsp, len, code);
+  if (code < 0) {
+    return -1;
+  }
+  int32_t tlen = sizeof(SMqRspHead) + len;
   void*   buf = rpcMallocCont(tlen);
   if (buf == NULL) {
     return -1;
@@ -125,18 +140,26 @@ int32_t tqSendPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con
   ((SMqRspHead*)buf)->consumerId = pReq->consumerId;
 
   void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
-  tEncodeSMqDataBlkRsp(&abuf, pRsp);
 
-  SRpcMsg resp = {
+  SEncoder encoder;
+  tEncoderInit(&encoder, abuf, len);
+  tEncodeSMqDataRsp(&encoder, pRsp);
+  /*tEncodeSMqDataBlkRsp(&abuf, pRsp);*/
+
+  SRpcMsg rsp = {
       .info = pMsg->info,
       .pCont = buf,
       .contLen = tlen,
       .code = 0,
   };
-  tmsgSendRsp(&resp);
+  tmsgSendRsp(&rsp);
 
-  tqDebug("vg %d from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld",
+  char buf1[50];
-          TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, pRsp->reqOffset, pRsp->rspOffset);
+  char buf2[50];
+  tFormatOffset(buf1, 50, &pRsp->reqOffset);
+  tFormatOffset(buf2, 50, &pRsp->rspOffset);
+  tqDebug("vg %d from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %s, rspOffset: %s",
+          TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2);
 
   return 0;
 }
@@ -151,17 +174,17 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) {
   }
   tDecoderClear(&decoder);
 
-  if (offset.type == TMQ_OFFSET__SNAPSHOT) {
+  if (offset.val.type == TMQ_OFFSET__SNAPSHOT_DATA) {
     tqDebug("receive offset commit msg to %s on vg %d, offset(type:snapshot) uid: %ld, ts: %ld", offset.subKey,
-            TD_VID(pTq->pVnode), offset.uid, offset.ts);
+            TD_VID(pTq->pVnode), offset.val.uid, offset.val.ts);
-  } else if (offset.type == TMQ_OFFSET__LOG) {
+  } else if (offset.val.type == TMQ_OFFSET__LOG) {
     tqDebug("receive offset commit msg to %s on vg %d, offset(type:log) version: %ld", offset.subKey,
-            TD_VID(pTq->pVnode), offset.version);
+            TD_VID(pTq->pVnode), offset.val.version);
   } else {
     ASSERT(0);
   }
   STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, offset.subKey);
-  if (pOffset == NULL || pOffset->version < offset.version) {
+  if (pOffset == NULL || pOffset->val.version < offset.val.version) {
     if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) {
       ASSERT(0);
       return -1;
@@ -171,6 +194,238 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) {
   return 0;
 }
 
+static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t subType) {
+  pRsp->reqOffset = pReq->reqOffset;
+
+  pRsp->blockData = taosArrayInit(0, sizeof(void*));
+  pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t));
+
+  if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL) {
+    return -1;
+  }
+
+  pRsp->withTbName = pReq->withTbName;
+  if (pRsp->withTbName) {
+    pRsp->blockTbName = taosArrayInit(0, sizeof(void*));
+    if (pRsp->blockTbName == NULL) {
+      // TODO free
+      return -1;
+    }
+  }
+
+  if (subType == TOPIC_SUB_TYPE__COLUMN) {
+    pRsp->withSchema = false;
+  } else {
+    pRsp->withSchema = true;
+    pRsp->blockSchema = taosArrayInit(0, sizeof(void*));
+    if (pRsp->blockSchema == NULL) {
+      // TODO free
+      return -1;
+    }
+  }
+  return 0;
+}
+
+static int32_t tqInitMetaRsp(SMqMetaRsp* pRsp, const SMqPollReq* pReq) { return 0; }
+
+static FORCE_INLINE void tqOffsetResetToData(STqOffsetVal* pOffsetVal, int64_t uid, int64_t ts) {
+  pOffsetVal->type = TMQ_OFFSET__SNAPSHOT_DATA;
+  pOffsetVal->uid = uid;
+  pOffsetVal->ts = ts;
+}
+
+static FORCE_INLINE void tqOffsetResetToLog(STqOffsetVal* pOffsetVal, int64_t ver) {
+  pOffsetVal->type = TMQ_OFFSET__LOG;
+  pOffsetVal->version = ver;
+}
+
+int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
+  SMqPollReq*  pReq = pMsg->pCont;
+  int64_t      consumerId = pReq->consumerId;
+  int64_t      timeout = pReq->timeout;
+  int32_t      reqEpoch = pReq->epoch;
+  int32_t      code = 0;
+  STqOffsetVal reqOffset = pReq->reqOffset;
+  STqOffsetVal fetchOffsetNew;
+
+  // 1.find handle
+  char buf[50];
+  tFormatOffset(buf, 50, &reqOffset);
+  tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req offset %s", consumerId, pReq->epoch,
+          TD_VID(pTq->pVnode), buf);
+
+  STqHandle* pHandle = taosHashGet(pTq->handles, pReq->subKey, strlen(pReq->subKey));
+  /*ASSERT(pHandle);*/
+  if (pHandle == NULL) {
+    tqError("tmq poll: no consumer handle for consumer %ld in vg %d, subkey %s", consumerId, TD_VID(pTq->pVnode),
+            pReq->subKey);
+    return -1;
+  }
+
+  // check rebalance
+  if (pHandle->consumerId != consumerId) {
+    tqError("tmq poll: consumer handle mismatch for consumer %ld in vg %d, subkey %s, handle consumer id %ld",
+            consumerId, TD_VID(pTq->pVnode), pReq->subKey, pHandle->consumerId);
+    return -1;
+  }
+
+  // update epoch if need
+  int32_t consumerEpoch = atomic_load_32(&pHandle->epoch);
+  while (consumerEpoch < reqEpoch) {
+    consumerEpoch = atomic_val_compare_exchange_32(&pHandle->epoch, consumerEpoch, reqEpoch);
+  }
+
+  // 2.reset offset if needed
+  if (reqOffset.type > 0) {
+    fetchOffsetNew = reqOffset;
+  } else {
+    STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pReq->subKey);
+    if (pOffset != NULL) {
+      fetchOffsetNew = pOffset->val;
+      char formatBuf[50];
+      tFormatOffset(formatBuf, 50, &fetchOffsetNew);
+      tqDebug("tmq poll: consumer %ld, offset reset to %s", consumerId, formatBuf);
+    } else {
+      if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) {
+        if (pReq->useSnapshot) {
+          if (!pHandle->fetchMeta) {
+            tqOffsetResetToData(&fetchOffsetNew, 0, 0);
+          } else {
+            // reset to meta
+            ASSERT(0);
+          }
+        } else {
+          tqOffsetResetToLog(&fetchOffsetNew, walGetFirstVer(pTq->pVnode->pWal));
+        }
+      } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) {
+        tqOffsetResetToLog(&fetchOffsetNew, walGetLastVer(pTq->pVnode->pWal));
+      } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) {
+        tqError("tmq poll: no offset committed for consumer %ld in vg %d, subkey %s, reset none failed", consumerId,
+                TD_VID(pTq->pVnode), pReq->subKey);
+        terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET;
+        return -1;
+      }
+    }
+  }
+
+  // 3.query
+  SMqDataRsp dataRsp = {0};
+  tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType);
+
+  if (fetchOffsetNew.type == TMQ_OFFSET__LOG) {
+    int64_t   fetchVer = fetchOffsetNew.version + 1;
+    SWalHead* pHeadWithCkSum = taosMemoryMalloc(sizeof(SWalHead) + 2048);
+    if (pHeadWithCkSum == NULL) {
+      return -1;
+    }
+
+    walSetReaderCapacity(pHandle->pWalReader, 2048);
+
+    while (1) {
+      consumerEpoch = atomic_load_32(&pHandle->epoch);
+      if (consumerEpoch > reqEpoch) {
+        tqWarn("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d, discard req epoch %d",
+               consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchVer, consumerEpoch, reqEpoch);
+        break;
+      }
+
+      if (tqFetchLog(pTq, pHandle, &fetchVer, &pHeadWithCkSum) < 0) {
+        // TODO add push mgr
+
+        tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer);
+        ASSERT(dataRsp.rspOffset.version >= dataRsp.reqOffset.version);
+        if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
+          code = -1;
+        }
+        goto OVER;
+      }
+
+      SWalReadHead* pHead = &pHeadWithCkSum->head;
+
+      tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
+              TD_VID(pTq->pVnode), fetchVer, pHead->msgType);
+
+      if (pHead->msgType == TDMT_VND_SUBMIT) {
+        SSubmitReq* pCont = (SSubmitReq*)&pHead->body;
+
+        if (tqLogScanExec(pTq, &pHandle->execHandle, pCont, &dataRsp, workerId) < 0) {
+          /*ASSERT(0);*/
+        }
+        // TODO batch optimization:
+        // TODO continue scan until meeting batch requirement
+        if (dataRsp.blockNum > 0 /* threshold */) {
+          tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer);
+          ASSERT(dataRsp.rspOffset.version >= dataRsp.reqOffset.version);
+
+          if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
+            code = -1;
+          }
+          goto OVER;
+        } else {
+          fetchVer++;
+        }
+
+      } else {
+        ASSERT(pHandle->fetchMeta);
+        ASSERT(IS_META_MSG(pHead->msgType));
+        tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType);
+        SMqMetaRsp metaRsp = {0};
+        metaRsp.reqOffset = pReq->reqOffset.version;
+        /*tqOffsetResetToLog(&metaR)*/
+        metaRsp.rspOffset = fetchVer;
+        metaRsp.resMsgType = pHead->msgType;
+        metaRsp.metaRspLen = pHead->bodyLen;
+        metaRsp.metaRsp = pHead->body;
+        if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) {
+          code = -1;
+          goto OVER;
+        }
+        code = 0;
+        goto OVER;
+      }
+    }
+
+    taosMemoryFree(pHeadWithCkSum);
+  } else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_DATA) {
+    // 1. set uid and ts
+    // 2. get data (rebuild reader if needed)
+    // 3. get new uid and ts
+
+    char formatBuf[50];
+    tFormatOffset(formatBuf, 50, &dataRsp.reqOffset);
+    tqInfo("retrieve using snapshot req offset %s", formatBuf);
+    if (tqScanSnapshot(pTq, &pHandle->execHandle, &dataRsp, workerId) < 0) {
+      ASSERT(0);
+    }
+
+    // 4. send rsp
+    if (dataRsp.blockNum != 0) {
+      tqOffsetResetToData(&dataRsp.rspOffset, 0, 0);
+      if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
+        code = -1;
+      }
+    }
+  } else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_META) {
+    ASSERT(0);
+  }
+
+OVER:
+  // TODO wrap in destroy func
+  taosArrayDestroy(dataRsp.blockDataLen);
+  taosArrayDestroyP(dataRsp.blockData, (FDelete)taosMemoryFree);
+
+  if (dataRsp.withSchema) {
+    taosArrayDestroyP(dataRsp.blockSchema, (FDelete)tDeleteSSchemaWrapper);
+  }
+
+  if (dataRsp.withTbName) {
+    taosArrayDestroyP(dataRsp.blockTbName, (FDelete)taosMemoryFree);
+  }
+
+  return code;
+}
+
+#if 0
 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   SMqPollReq* pReq = pMsg->pCont;
   int64_t     consumerId = pReq->consumerId;
@@ -185,10 +440,10 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   } else {
     STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pReq->subKey);
     if (pOffset != NULL) {
-      ASSERT(pOffset->type == TMQ_OFFSET__LOG);
+      ASSERT(pOffset->val.type == TMQ_OFFSET__LOG);
       tqDebug("consumer %ld, restore offset of %s on vg %d, offset(type:log) version: %ld", consumerId, pReq->subKey,
-              TD_VID(pTq->pVnode), pOffset->version);
+              TD_VID(pTq->pVnode), pOffset->val.version);
-      fetchOffset = pOffset->version + 1;
+      fetchOffset = pOffset->val.version + 1;
     } else {
       if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) {
         fetchOffset = walGetFirstVer(pTq->pWal);
@@ -241,12 +496,11 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   if (rsp.withTbName) {
     rsp.blockTbName = taosArrayInit(0, sizeof(void*));
   }
+
   if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
     rsp.withSchema = false;
-    rsp.withTag = false;
   } else {
     rsp.withSchema = true;
-    rsp.withTag = false;
     rsp.blockSchema = taosArrayInit(0, sizeof(void*));
   }
 
@@ -302,10 +556,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
       }
     } else {
       ASSERT(pHandle->fetchMeta);
-      ASSERT(pHead->msgType == TDMT_VND_CREATE_STB || pHead->msgType == TDMT_VND_ALTER_STB ||
+      ASSERT(IS_META_MSG(pHead->msgType));
-             pHead->msgType == TDMT_VND_DROP_STB || pHead->msgType == TDMT_VND_CREATE_TABLE ||
-             pHead->msgType == TDMT_VND_ALTER_TABLE || pHead->msgType == TDMT_VND_DROP_TABLE ||
-             pHead->msgType == TDMT_VND_DROP_TTL_TABLE);
       tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType);
       SMqMetaRsp metaRsp = {0};
       metaRsp.reqOffset = pReq->currentOffset;
@@ -341,7 +592,7 @@ SEND_RSP:
 
   rsp.rspOffset = fetchOffset;
 
-  if (tqSendPollRsp(pTq, pMsg, pReq, &rsp) < 0) {
+  if (tqSendDataRsp(pTq, pMsg, pReq, &rsp) < 0) {
     code = -1;
   }
 OVER:
@@ -359,6 +610,7 @@ OVER:
 
   return code;
 }
+#endif
 
 int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen) {
   SMqVDeleteReq* pReq = (SMqVDeleteReq*)msg;
@@ -403,7 +655,6 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
             .reader = pHandle->execHandle.pExecReader[i],
             .meta = pTq->pVnode->pMeta,
             .vnode = pTq->pVnode,
-//            .initTsdbReader = 1,
         };
         pHandle->execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle);
         ASSERT(pHandle->execHandle.execCol.task[i]);
@@ -474,12 +725,11 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) {
   if (pTask->execType != TASK_EXEC__NONE) {
     // expand runners
     if (pTask->isDataScan) {
-      STqReadHandle* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
+      SStreamReader* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
       SReadHandle    handle = {
              .reader = pStreamReader,
              .meta = pTq->pVnode->pMeta,
              .vnode = pTq->pVnode,
-//             .initTsdbReader = 1,
       };
       /*pTask->exec.inputHandle = pStreamReader;*/
       pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle);

source/dnode/vnode/src/tq/tqExec.c

@@ -15,7 +15,7 @@
 
 #include "tq.h"
 
-static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataBlkRsp* pRsp) {
+static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp) {
   int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
   void*   buf = taosMemoryCalloc(1, dataStrLen);
   if (buf == NULL) return -1;
@@ -29,7 +29,7 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataBlkRsp* pRs
 
   // TODO enable compress
   int32_t actualLen = 0;
-  blockCompressEncode(pBlock, pRetrieve->data, &actualLen, taosArrayGetSize(pBlock->pDataBlock), false);
+  blockEncode(pBlock, pRetrieve->data, &actualLen, taosArrayGetSize(pBlock->pDataBlock), false);
   actualLen += sizeof(SRetrieveTableRsp);
   ASSERT(actualLen <= dataStrLen);
   taosArrayPush(pRsp->blockDataLen, &actualLen);
@@ -37,7 +37,7 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataBlkRsp* pRs
   return 0;
 }
 
-static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, int32_t workerId, SMqDataBlkRsp* pRsp) {
+static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, int32_t workerId, SMqDataRsp* pRsp) {
   SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pExecReader[workerId]->pSchemaWrapper);
   if (pSW == NULL) {
     return -1;
@@ -46,10 +46,9 @@ static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, int32_t workerI
   return 0;
 }
 
-static int32_t tqAddTbNameToRsp(const STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp, int32_t workerId) {
+static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, int32_t workerId) {
   SMetaReader mr = {0};
   metaReaderInit(&mr, pTq->pVnode->pMeta, 0);
-  int64_t uid = pExec->pExecReader[workerId]->msgIter.uid;
   if (metaGetTableEntryByUid(&mr, uid) < 0) {
     ASSERT(0);
     return -1;
@@ -60,13 +59,13 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, const STqExecHandle* pExec, SMqD
   return 0;
 }
 
-int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp, int32_t workerId) {
+int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, int32_t workerId) {
   ASSERT(pExec->subType == TOPIC_SUB_TYPE__COLUMN);
   qTaskInfo_t task = pExec->execCol.task[workerId];
+  // TODO set uid and ts
   if (qStreamScanSnapshot(task) < 0) {
     ASSERT(0);
   }
-  // set version
   while (1) {
     SSDataBlock* pDataBlock = NULL;
     uint64_t     ts = 0;
@@ -79,17 +78,24 @@ int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp
     ASSERT(taosArrayGetSize(pDataBlock->pDataBlock) != 0);
 
     tqAddBlockDataToRsp(pDataBlock, pRsp);
+
+    if (pRsp->withTbName) {
+      // TODO
+      pRsp->withTbName = 0;
+      /*int64_t uid = 0;*/
+      /*tqAddTbNameToRsp(pTq, uid, pRsp, workerId);*/
+    }
     pRsp->blockNum++;
   }
 
   return 0;
 }
 
-int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkRsp* pRsp, int32_t workerId) {
+int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataRsp* pRsp, int32_t workerId) {
   if (pExec->subType == TOPIC_SUB_TYPE__COLUMN) {
     qTaskInfo_t task = pExec->execCol.task[workerId];
     ASSERT(task);
-    qSetStreamInput(task, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK, false);
+    qSetStreamInput(task, pReq, STREAM_INPUT__DATA_SUBMIT, false);
     while (1) {
       SSDataBlock* pDataBlock = NULL;
       uint64_t     ts = 0;
@@ -102,13 +108,14 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR
 
       tqAddBlockDataToRsp(pDataBlock, pRsp);
       if (pRsp->withTbName) {
-        tqAddTbNameToRsp(pTq, pExec, pRsp, workerId);
+        int64_t uid = pExec->pExecReader[workerId]->msgIter.uid;
+        tqAddTbNameToRsp(pTq, uid, pRsp, workerId);
       }
       pRsp->blockNum++;
     }
   } else if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
     pRsp->withSchema = 1;
-    STqReadHandle* pReader = pExec->pExecReader[workerId];
+    SStreamReader* pReader = pExec->pExecReader[workerId];
     tqReadHandleSetMsg(pReader, pReq, 0);
     while (tqNextDataBlock(pReader)) {
       SSDataBlock block = {0};
@@ -118,14 +125,15 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR
       }
       tqAddBlockDataToRsp(&block, pRsp);
       if (pRsp->withTbName) {
-        tqAddTbNameToRsp(pTq, pExec, pRsp, workerId);
+        int64_t uid = pExec->pExecReader[workerId]->msgIter.uid;
+        tqAddTbNameToRsp(pTq, uid, pRsp, workerId);
       }
       tqAddBlockSchemaToRsp(pExec, workerId, pRsp);
       pRsp->blockNum++;
     }
   } else if (pExec->subType == TOPIC_SUB_TYPE__DB) {
     pRsp->withSchema = 1;
-    STqReadHandle* pReader = pExec->pExecReader[workerId];
+    SStreamReader* pReader = pExec->pExecReader[workerId];
     tqReadHandleSetMsg(pReader, pReq, 0);
     while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) {
       SSDataBlock block = {0};
@@ -135,7 +143,8 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR
       }
       tqAddBlockDataToRsp(&block, pRsp);
       if (pRsp->withTbName) {
-        tqAddTbNameToRsp(pTq, pExec, pRsp, workerId);
+        int64_t uid = pExec->pExecReader[workerId]->msgIter.uid;
+        tqAddTbNameToRsp(pTq, uid, pRsp, workerId);
       }
       tqAddBlockSchemaToRsp(pExec, workerId, pRsp);
       pRsp->blockNum++;

source/dnode/vnode/src/tq/tqOffset.c

@@ -92,8 +92,8 @@ STqOffset* tqOffsetRead(STqOffsetStore* pStore, const char* subscribeKey) {
 }
 
 int32_t tqOffsetWrite(STqOffsetStore* pStore, const STqOffset* pOffset) {
-  ASSERT(pOffset->type == TMQ_OFFSET__LOG);
+  /*ASSERT(pOffset->val.type == TMQ_OFFSET__LOG);*/
-  ASSERT(pOffset->version >= 0);
+  /*ASSERT(pOffset->val.version >= 0);*/
   return taosHashPut(pStore->pHash, pOffset->subKey, strlen(pOffset->subKey), pOffset, sizeof(STqOffset));
 }
 

source/dnode/vnode/src/tq/tqPush.c

@@ -15,16 +15,17 @@
 
 #include "tq.h"
 
+#if 0
 void tqTmrRspFunc(void* param, void* tmrId) {
   STqHandle* pHandle = (STqHandle*)param;
   atomic_store_8(&pHandle->pushHandle.tmrStopped, 1);
 }
 
-static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubmit** ppSubmit, SMqDataBlkRsp* pRsp) {
+static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubmit** ppSubmit, SMqDataRsp* pRsp) {
   SStreamDataSubmit* pSubmit = *ppSubmit;
   while (pSubmit != NULL) {
     ASSERT(pSubmit->ver == pHandle->pushHandle.processedVer + 1);
-    if (tqDataExec(pTq, &pHandle->execHandle, pSubmit->data, pRsp, 0) < 0) {
+    if (tqLogScanExec(pTq, &pHandle->execHandle, pSubmit->data, pRsp, 0) < 0) {
       /*ASSERT(0);*/
     }
     // update processed
@@ -43,7 +44,7 @@ static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubm
 }
 
 int32_t tqExecFromInputQ(STQ* pTq, STqHandle* pHandle) {
-  SMqDataBlkRsp rsp = {0};
+  SMqDataRsp rsp = {0};
   // 1. guard and set status executing
   int8_t execStatus = atomic_val_compare_exchange_8(&pHandle->pushHandle.execStatus, TASK_EXEC_STATUS__IDLE,
                                                     TASK_EXEC_STATUS__EXECUTING);
@@ -175,13 +176,13 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_
 
     taosWLockLatch(&pHandle->pushHandle.lock);
 
-    SMqDataBlkRsp rsp = {0};
+    SMqDataRsp rsp = {0};
     rsp.reqOffset = pHandle->pushHandle.reqOffset;
     rsp.blockData = taosArrayInit(0, sizeof(void*));
     rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
 
     if (msgType == TDMT_VND_SUBMIT) {
-      tqDataExec(pTq, &pHandle->execHandle, pReq, &rsp, workerId);
+      tqLogScanExec(pTq, &pHandle->execHandle, pReq, &rsp, workerId);
     } else {
       // TODO
       ASSERT(0);
@@ -233,6 +234,7 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_
 
   return 0;
 }
+#endif
 
 int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) {
   if (msgType == TDMT_VND_SUBMIT) {

source/dnode/vnode/src/tq/tqRead.c

@@ -44,10 +44,7 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalHead*
     } else {
       if (pHandle->fetchMeta) {
         SWalReadHead* pHead = &((*ppHeadWithCkSum)->head);
-        if (pHead->msgType == TDMT_VND_CREATE_STB || pHead->msgType == TDMT_VND_ALTER_STB ||
+        if (IS_META_MSG(pHead->msgType)) {
-            pHead->msgType == TDMT_VND_DROP_STB || pHead->msgType == TDMT_VND_CREATE_TABLE ||
-            pHead->msgType == TDMT_VND_ALTER_TABLE || pHead->msgType == TDMT_VND_DROP_TABLE ||
-            pHead->msgType == TDMT_VND_DROP_TTL_TABLE) {
           code = walFetchBody(pHandle->pWalReader, ppHeadWithCkSum);
 
           if (code < 0) {
@@ -76,8 +73,8 @@ END:
   return code;
 }
 
-STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) {
+SStreamReader* tqInitSubmitMsgScanner(SMeta* pMeta) {
-  STqReadHandle* pReadHandle = taosMemoryMalloc(sizeof(STqReadHandle));
+  SStreamReader* pReadHandle = taosMemoryMalloc(sizeof(SStreamReader));
   if (pReadHandle == NULL) {
     return NULL;
   }
@@ -85,15 +82,15 @@ STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) {
   pReadHandle->pMsg = NULL;
   pReadHandle->ver = -1;
   pReadHandle->pColIdList = NULL;
-  pReadHandle->cachedSchemaVer = -1;
+  pReadHandle->cachedSchemaVer = 0;
-  pReadHandle->cachedSchemaSuid = -1;
+  pReadHandle->cachedSchemaSuid = 0;
   pReadHandle->pSchema = NULL;
   pReadHandle->pSchemaWrapper = NULL;
   pReadHandle->tbIdHash = NULL;
   return pReadHandle;
 }
 
-int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t ver) {
+int32_t tqReadHandleSetMsg(SStreamReader* pReadHandle, SSubmitReq* pMsg, int64_t ver) {
   pReadHandle->pMsg = pMsg;
 
   if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1;
@@ -108,7 +105,7 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t
   return 0;
 }
 
-bool tqNextDataBlock(STqReadHandle* pHandle) {
+bool tqNextDataBlock(SStreamReader* pHandle) {
   if (pHandle->pMsg == NULL) return false;
   while (1) {
     if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) {
@@ -130,7 +127,7 @@ bool tqNextDataBlock(STqReadHandle* pHandle) {
   return false;
 }
 
-bool tqNextDataBlockFilterOut(STqReadHandle* pHandle, SHashObj* filterOutUids) {
+bool tqNextDataBlockFilterOut(SStreamReader* pHandle, SHashObj* filterOutUids) {
   while (1) {
     if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) {
       return false;
@@ -146,7 +143,7 @@ bool tqNextDataBlockFilterOut(STqReadHandle* pHandle, SHashObj* filterOutUids) {
   return false;
 }
 
-int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle) {
+int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, SStreamReader* pHandle) {
   // TODO: cache multiple schema
   int32_t sversion = htonl(pHandle->pBlock->sversion);
   if (pHandle->cachedSchemaSuid == 0 || pHandle->cachedSchemaVer != sversion ||
@@ -231,7 +228,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle) {
   tInitSubmitBlkIter(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter);
 
   pBlock->info.groupId = 0;
-  pBlock->info.uid = pHandle->msgIter.uid;  // set the uid of table for submit block
+  pBlock->info.uid = pHandle->msgIter.uid;
   pBlock->info.rows = pHandle->msgIter.numOfRows;
 
   while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) {
@@ -251,14 +248,14 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle) {
   }
   return 0;
 
-FAIL:  // todo refactor here
+FAIL:
-       //  if (*ppCols) taosArrayDestroy(*ppCols);
+  tDeleteSSDataBlock(pBlock);
   return -1;
 }
 
-void tqReadHandleSetColIdList(STqReadHandle* pReadHandle, SArray* pColIdList) { pReadHandle->pColIdList = pColIdList; }
+void tqReadHandleSetColIdList(SStreamReader* pReadHandle, SArray* pColIdList) { pReadHandle->pColIdList = pColIdList; }
 
-int tqReadHandleSetTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) {
+int tqReadHandleSetTbUidList(SStreamReader* pHandle, const SArray* tbUidList) {
   if (pHandle->tbIdHash) {
     taosHashClear(pHandle->tbIdHash);
   }
@@ -277,7 +274,7 @@ int tqReadHandleSetTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) {
   return 0;
 }
 
-int tqReadHandleAddTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) {
+int tqReadHandleAddTbUidList(SStreamReader* pHandle, const SArray* tbUidList) {
   if (pHandle->tbIdHash == NULL) {
     pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
     if (pHandle->tbIdHash == NULL) {
@@ -294,7 +291,7 @@ int tqReadHandleAddTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) {
   return 0;
 }
 
-int tqReadHandleRemoveTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) {
+int tqReadHandleRemoveTbUidList(SStreamReader* pHandle, const SArray* tbUidList) {
   ASSERT(pHandle->tbIdHash != NULL);
 
   for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) {

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -136,7 +136,7 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
       if (vnodeProcessDropTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
       break;
     case TDMT_VND_DROP_TTL_TABLE:
-      if (vnodeProcessDropTtlTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
+      //if (vnodeProcessDropTtlTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
       break;
     case TDMT_VND_CREATE_SMA: {
       if (vnodeProcessCreateTSmaReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
@@ -781,7 +781,7 @@ _exit:
   // TODO: the partial success scenario and the error case
   // TODO: refactor
   if ((terrno == TSDB_CODE_SUCCESS) && (pRsp->code == TSDB_CODE_SUCCESS)) {
-    tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK);
+    tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_INPUT__DATA_SUBMIT);
   }
 
   return 0;

source/dnode/vnode/src/vnd/vnodeSync.c

@@ -73,7 +73,7 @@ static int32_t vnodeSetStandBy(SVnode *pVnode) {
     vInfo("vgId:%d, set standby success", TD_VID(pVnode));
     return 0;
   } else {
-    vError("vgId:%d, failed to set standby since %s", TD_VID(pVnode), terrstr());
+    vError("vgId:%d, failed to set standby after leader transfer since %s", TD_VID(pVnode), terrstr());
     return -1;
   }
 }

source/libs/command/src/command.c

@@ -18,7 +18,31 @@
 #include "tdatablock.h"
 #include "tglobal.h"
 
-extern SConfig *tsCfg;
+extern SConfig* tsCfg;
+
+static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRetrieveTableRsp** pRsp) {
+  size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
+  *pRsp = taosMemoryCalloc(1, rspSize);
+  if (NULL == *pRsp) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  (*pRsp)->useconds = 0;
+  (*pRsp)->completed = 1;
+  (*pRsp)->precision = 0;
+  (*pRsp)->compressed = 0;
+  (*pRsp)->compLen = 0;
+  (*pRsp)->numOfRows = htonl(pBlock->info.rows);
+  (*pRsp)->numOfCols = htonl(numOfCols);
+
+  int32_t len = 0;
+  blockEncode(pBlock, (*pRsp)->data, &len, numOfCols, false);
+  ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
+
+  blockDataDestroy(pBlock);
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t getSchemaBytes(const SSchema* pSchema) {
   switch (pSchema->type) {
     case TSDB_DATA_TYPE_BINARY:
@@ -89,31 +113,11 @@ static int32_t execDescribe(SNode* pStmt, SRetrieveTableRsp** pRsp) {
   SSDataBlock* pBlock = buildDescResultDataBlock();
   setDescResultIntoDataBlock(pBlock, numOfRows, pDesc->pMeta);
 
-  size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
+  return buildRetrieveTableRsp(pBlock, DESCRIBE_RESULT_COLS, pRsp);
-  *pRsp = taosMemoryCalloc(1, rspSize);
-  if (NULL == *pRsp) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-
-  (*pRsp)->useconds = 0;
-  (*pRsp)->completed = 1;
-  (*pRsp)->precision = 0;
-  (*pRsp)->compressed = 0;
-  (*pRsp)->compLen = 0;
-  (*pRsp)->numOfRows = htonl(numOfRows);
-  (*pRsp)->numOfCols = htonl(DESCRIBE_RESULT_COLS);
-
-  int32_t len = 0;
-  blockCompressEncode(pBlock, (*pRsp)->data, &len, DESCRIBE_RESULT_COLS, false);
-  ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
-
-  blockDataDestroy(pBlock);
-  return TSDB_CODE_SUCCESS;
 }
 
 static int32_t execResetQueryCache() { return catalogClearCache(); }
 
-
 static SSDataBlock* buildCreateDBResultDataBlock() {
   SSDataBlock*    pBlock = createDataBlock();
   SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_COLS, 1);
@@ -149,14 +153,14 @@ int64_t getValOfDiffPrecision(int8_t unit, int64_t val) {
   return v;
 }
 
-char *buildRetension(SArray *pRetension) {
+char* buildRetension(SArray* pRetension) {
   size_t size = taosArrayGetSize(pRetension);
   if (size == 0) {
     return NULL;
   }
 
-  char       *p1 = taosMemoryCalloc(1, 100);
+  char*       p1 = taosMemoryCalloc(1, 100);
-  SRetention *p = taosArrayGet(pRetension, 0);
+  SRetention* p = taosArrayGet(pRetension, 0);
 
   int32_t len = 0;
 
@@ -185,8 +189,7 @@ char *buildRetension(SArray *pRetension) {
   return p1;
 }
 
-
+static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbFName, SDbCfgInfo* pCfg) {
-static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char *dbFName, SDbCfgInfo* pCfg) {
   blockDataEnsureCapacity(pBlock, 1);
   pBlock->info.rows = 1;
 
@@ -198,7 +201,7 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char *dbFName, S
   SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1);
   char             buf2[SHOW_CREATE_DB_RESULT_FIELD2_LEN] = {0};
   int32_t          len = 0;
-  char *prec = NULL;
+  char*            prec = NULL;
   switch (pCfg->precision) {
     case TSDB_TIME_PRECISION_MILLI:
       prec = TSDB_TIME_PRECISION_MILLI_STR;
@@ -214,14 +217,15 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char *dbFName, S
       break;
   }
 
-  char *retentions = buildRetension(pCfg->pRetensions);
+  char* retentions = buildRetension(pCfg->pRetensions);
 
-  len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE DATABASE `%s` BUFFER %d CACHELAST %d COMP %d DURATION %dm "
+  len += sprintf(buf2 + VARSTR_HEADER_SIZE,
+                 "CREATE DATABASE `%s` BUFFER %d CACHELAST %d COMP %d DURATION %dm "
                  "FSYNC %d MAXROWS %d MINROWS %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d "
                  "STRICT %d WAL %d VGROUPS %d SINGLE_STABLE %d",
-                    dbFName, pCfg->buffer, pCfg->cacheLastRow, pCfg->compression, pCfg->daysPerFile,
+                 dbFName, pCfg->buffer, pCfg->cacheLastRow, pCfg->compression, pCfg->daysPerFile, pCfg->fsyncPeriod,
-                    pCfg->fsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2,
+                 pCfg->maxRows, pCfg->minRows, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2, pCfg->pages,
-                    pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->strict, pCfg->walLevel, pCfg->numOfVgroups,
+                 pCfg->pageSize, prec, pCfg->replications, pCfg->strict, pCfg->walLevel, pCfg->numOfVgroups,
                  1 == pCfg->numOfStables);
 
   if (retentions) {
@@ -234,31 +238,10 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char *dbFName, S
   colDataAppend(pCol2, 0, buf2, false);
 }
 
-
 static int32_t execShowCreateDatabase(SShowCreateDatabaseStmt* pStmt, SRetrieveTableRsp** pRsp) {
   SSDataBlock* pBlock = buildCreateDBResultDataBlock();
   setCreateDBResultIntoDataBlock(pBlock, pStmt->dbName, pStmt->pCfg);
-
+  return buildRetrieveTableRsp(pBlock, SHOW_CREATE_DB_RESULT_COLS, pRsp);
-  size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
-  *pRsp = taosMemoryCalloc(1, rspSize);
-  if (NULL == *pRsp) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-
-  (*pRsp)->useconds = 0;
-  (*pRsp)->completed = 1;
-  (*pRsp)->precision = 0;
-  (*pRsp)->compressed = 0;
-  (*pRsp)->compLen = 0;
-  (*pRsp)->numOfRows = htonl(1);
-  (*pRsp)->numOfCols = htonl(SHOW_CREATE_DB_RESULT_COLS);
-
-  int32_t len = 0;
-  blockCompressEncode(pBlock, (*pRsp)->data, &len, SHOW_CREATE_DB_RESULT_COLS, false);
-  ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
-
-  blockDataDestroy(pBlock);
-  return TSDB_CODE_SUCCESS;
 }
 
 static SSDataBlock* buildCreateTbResultDataBlock() {
@@ -281,7 +264,7 @@ void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) {
     if (TSDB_DATA_TYPE_VARCHAR == pSchema->type) {
       sprintf(type + strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE));
     } else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) {
-      sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE));    
+      sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE));
     }
 
     *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type);
@@ -296,14 +279,13 @@ void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) {
     if (TSDB_DATA_TYPE_VARCHAR == pSchema->type) {
       sprintf(type + strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE));
     } else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) {
-      sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE));    
+      sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE));
     }
 
     *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type);
   }
 }
 
-
 void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) {
   for (int32_t i = 0; i < pCfg->numOfTags; ++i) {
     SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i;
@@ -311,13 +293,12 @@ void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) {
   }
 }
 
-
 int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
-  SArray *pTagVals = NULL;
+  SArray* pTagVals = NULL;
-  STag *pTag = (STag*)pCfg->pTags;
+  STag*   pTag = (STag*)pCfg->pTags;
 
   if (pCfg->pTags && pTag->flags & TD_TAG_JSON) {
-    char *pJson = parseTagDatatoJson(pTag);
+    char* pJson = parseTagDatatoJson(pTag);
     if (pJson) {
       *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s", pJson);
       taosMemoryFree(pJson);
@@ -326,7 +307,7 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
     return TSDB_CODE_SUCCESS;
   }
 
-  int32_t code = tTagToValArray((const STag *)pCfg->pTags, &pTagVals);
+  int32_t code = tTagToValArray((const STag*)pCfg->pTags, &pTagVals);
   if (code) {
     return code;
   }
@@ -345,7 +326,7 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
       continue;
     }
 
-    STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, j);
+    STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, j);
     if (pSchema->colId > pTagVal->cid) {
       qError("tag value and column mismatch, schemaId:%d, valId:%d", pSchema->colId, pTagVal->cid);
       taosArrayDestroy(pTagVals);
@@ -365,7 +346,6 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
       *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "NULL");
     }
 
-
     /*
     if (type == TSDB_DATA_TYPE_BINARY) {
       if (pTagVal->nData > 0) {
@@ -436,7 +416,7 @@ void appendTableOptions(char* buf, int32_t* len, STableCfg* pCfg) {
   }
 }
 
-static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, char *tbName, STableCfg* pCfg) {
+static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, char* tbName, STableCfg* pCfg) {
   int32_t code = 0;
   blockDataEnsureCapacity(pBlock, 1);
   pBlock->info.rows = 1;
@@ -480,34 +460,13 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, char *tbName,
   return TSDB_CODE_SUCCESS;
 }
 
-
 static int32_t execShowCreateTable(SShowCreateTableStmt* pStmt, SRetrieveTableRsp** pRsp) {
   SSDataBlock* pBlock = buildCreateTbResultDataBlock();
   int32_t      code = setCreateTBResultIntoDataBlock(pBlock, pStmt->tableName, pStmt->pCfg);
   if (code) {
     return code;
   }
-
+  return buildRetrieveTableRsp(pBlock, SHOW_CREATE_TB_RESULT_COLS, pRsp);
-  size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
-  *pRsp = taosMemoryCalloc(1, rspSize);
-  if (NULL == *pRsp) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-
-  (*pRsp)->useconds = 0;
-  (*pRsp)->completed = 1;
-  (*pRsp)->precision = 0;
-  (*pRsp)->compressed = 0;
-  (*pRsp)->compLen = 0;
-  (*pRsp)->numOfRows = htonl(1);
-  (*pRsp)->numOfCols = htonl(SHOW_CREATE_TB_RESULT_COLS);
-
-  int32_t len = 0;
-  blockCompressEncode(pBlock, (*pRsp)->data, &len, SHOW_CREATE_TB_RESULT_COLS, false);
-  ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
-
-  blockDataDestroy(pBlock);
-  return TSDB_CODE_SUCCESS;
 }
 
 static int32_t execShowCreateSTable(SShowCreateTableStmt* pStmt, SRetrieveTableRsp** pRsp) {
@@ -551,18 +510,17 @@ static SSDataBlock* buildLocalVariablesResultDataBlock() {
   return pBlock;
 }
 
-
 int32_t setLocalVariablesResultIntoDataBlock(SSDataBlock* pBlock) {
   int32_t numOfCfg = taosArrayGetSize(tsCfg->array);
   int32_t numOfRows = 0;
   blockDataEnsureCapacity(pBlock, numOfCfg);
 
   for (int32_t i = 0, c = 0; i < numOfCfg; ++i, c = 0) {
-    SConfigItem *pItem = taosArrayGet(tsCfg->array, i);
+    SConfigItem* pItem = taosArrayGet(tsCfg->array, i);
 
     char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE);
-    SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, c++);
+    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, c++);
     colDataAppend(pColInfo, i, name, false);
 
     char    value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0};
@@ -575,42 +533,70 @@ int32_t setLocalVariablesResultIntoDataBlock(SSDataBlock* pBlock) {
     numOfRows++;
   }
 
-
   pBlock->info.rows = numOfRows;
 
   return TSDB_CODE_SUCCESS;
 }
 
-
 static int32_t execShowLocalVariables(SRetrieveTableRsp** pRsp) {
   SSDataBlock* pBlock = buildLocalVariablesResultDataBlock();
   int32_t      code = setLocalVariablesResultIntoDataBlock(pBlock);
   if (code) {
     return code;
   }
+  return buildRetrieveTableRsp(pBlock, SHOW_LOCAL_VARIABLES_RESULT_COLS, pRsp);
+}
 
-  size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
+static int32_t createSelectResultDataBlock(SNodeList* pProjects, SSDataBlock** pOutput) {
-  *pRsp = taosMemoryCalloc(1, rspSize);
+  SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
-  if (NULL == *pRsp) {
+  if (NULL == pBlock) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
-  (*pRsp)->useconds = 0;
+  pBlock->pDataBlock = taosArrayInit(LIST_LENGTH(pProjects), sizeof(SColumnInfoData));
-  (*pRsp)->completed = 1;
-  (*pRsp)->precision = 0;
-  (*pRsp)->compressed = 0;
-  (*pRsp)->compLen = 0;
-  (*pRsp)->numOfRows = htonl(pBlock->info.rows);
-  (*pRsp)->numOfCols = htonl(SHOW_LOCAL_VARIABLES_RESULT_COLS);
 
-  int32_t len = 0;
+  SNode* pProj = NULL;
-  blockCompressEncode(pBlock, (*pRsp)->data, &len, SHOW_LOCAL_VARIABLES_RESULT_COLS, false);
+  FOREACH(pProj, pProjects) {
-  ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
+    SColumnInfoData infoData = {0};
-
+    infoData.info.type = ((SExprNode*)pProj)->resType.type;
-  blockDataDestroy(pBlock);
+    infoData.info.bytes = ((SExprNode*)pProj)->resType.bytes;
+    taosArrayPush(pBlock->pDataBlock, &infoData);
+  }
+  *pOutput = pBlock;
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t buildSelectResultDataBlock(SNodeList* pProjects, SSDataBlock* pBlock) {
+  int32_t numOfCols = LIST_LENGTH(pProjects);
+  blockDataEnsureCapacity(pBlock, 1);
+
+  int32_t index = 0;
+  SNode*  pProj = NULL;
+  FOREACH(pProj, pProjects) {
+    if (((SValueNode*)pProj)->isNull) {
+      colDataAppend(taosArrayGet(pBlock->pDataBlock, index++), 0, NULL, true);
+    } else {
+      colDataAppend(taosArrayGet(pBlock->pDataBlock, index++), 0, nodesGetValueFromNode((SValueNode*)pProj), false);
+    }
+  }
+
+  pBlock->info.rows = 1;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t execSelectWithoutFrom(SSelectStmt* pSelect, SRetrieveTableRsp** pRsp) {
+  SSDataBlock* pBlock = NULL;
+  int32_t      code = createSelectResultDataBlock(pSelect->pProjectionList, &pBlock);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = buildSelectResultDataBlock(pSelect->pProjectionList, pBlock);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = buildRetrieveTableRsp(pBlock, LIST_LENGTH(pSelect->pProjectionList), pRsp);
+  }
+  return code;
+}
+
 int32_t qExecCommand(SNode* pStmt, SRetrieveTableRsp** pRsp) {
   switch (nodeType(pStmt)) {
     case QUERY_NODE_DESCRIBE_STMT:
@@ -627,6 +613,8 @@ int32_t qExecCommand(SNode* pStmt, SRetrieveTableRsp** pRsp) {
       return execAlterLocal((SAlterLocalStmt*)pStmt);
     case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT:
       return execShowLocalVariables(pRsp);
+    case QUERY_NODE_SELECT_STMT:
+      return execSelectWithoutFrom((SSelectStmt*)pStmt, pRsp);
     default:
       break;
   }

source/libs/command/src/explain.c

@@ -13,11 +13,11 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tdatablock.h"
 #include "commandInt.h"
 #include "plannodes.h"
 #include "query.h"
 #include "tcommon.h"
+#include "tdatablock.h"
 
 int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pRes);
 int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level);
@@ -216,7 +216,7 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo
   SExplainResNode *pResNode = NULL;
   FOREACH(node, pPhysiChildren) {
     QRY_ERR_RET(qExplainGenerateResNode((SPhysiNode *)node, group, &pResNode));
-    QRY_ERR_RET(nodesListAppend(*pChildren, (SNode*)pResNode));
+    QRY_ERR_RET(nodesListAppend(*pChildren, (SNode *)pResNode));
   }
 
   return TSDB_CODE_SUCCESS;
@@ -232,14 +232,14 @@ int32_t qExplainGenerateResNodeExecInfo(SArray **pExecInfo, SExplainGroup *group
   SExplainRsp *rsp = NULL;
   for (int32_t i = 0; i < group->nodeNum; ++i) {
     rsp = taosArrayGet(group->nodeExecInfo, i);
-/*
+    /*
         if (group->physiPlanExecIdx >= rsp->numOfPlans) {
           qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans);
           return TSDB_CODE_QRY_APP_ERROR;
         }
 
         taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx);
-*/
+    */
     taosArrayPush(*pExecInfo, rsp->subplanInfo);
   }
 
@@ -431,7 +431,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
 
         int32_t maxIndex = 0;
         int32_t totalRows = 0;
-        for(int32_t i = 0; i < nodeNum; ++i) {
+        for (int32_t i = 0; i < nodeNum; ++i) {
           SExplainExecInfo      *execInfo = taosArrayGet(pResNode->pExecInfo, i);
           STableScanAnalyzeInfo *pScanInfo = (STableScanAnalyzeInfo *)execInfo->verboseInfo;
 
@@ -468,13 +468,14 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
 
         QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
 
-        //Rows out: Avg 4166.7 rows x 24 workers. Max 4187 rows (seg7) with 0.220 ms to first row, 1.738 ms to end, start offset by 1.470 ms.
+        // Rows out: Avg 4166.7 rows x 24 workers. Max 4187 rows (seg7) with 0.220 ms to first row, 1.738 ms to end,
+        // start offset by 1.470 ms.
         SExplainExecInfo      *execInfo = taosArrayGet(pResNode->pExecInfo, maxIndex);
         STableScanAnalyzeInfo *p1 = (STableScanAnalyzeInfo *)execInfo->verboseInfo;
 
         EXPLAIN_ROW_NEW(level + 1, " ");
-        EXPLAIN_ROW_APPEND("max_row_task=%d, total_rows:%" PRId64 ", ep:%s (cost=%.3f..%.3f)", maxIndex, p1->totalRows, "tbd",
+        EXPLAIN_ROW_APPEND("max_row_task=%d, total_rows:%" PRId64 ", ep:%s (cost=%.3f..%.3f)", maxIndex, p1->totalRows,
-                           execInfo->startupCost, execInfo->totalCost);
+                           "tbd", execInfo->startupCost, execInfo->totalCost);
         EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
         EXPLAIN_ROW_END();
 
@@ -752,7 +753,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
         EXPLAIN_ROW_NEW(level + 1, "Sort Key: ");
         if (pResNode->pExecInfo) {
           for (int32_t i = 0; i < LIST_LENGTH(pSortNode->pSortKeys); ++i) {
-            SOrderByExprNode *ptn = (SOrderByExprNode*)nodesListGetNode(pSortNode->pSortKeys, i);
+            SOrderByExprNode *ptn = (SOrderByExprNode *)nodesListGetNode(pSortNode->pSortKeys, i);
             EXPLAIN_ROW_APPEND("%s ", nodesGetNameFromColumnNode(ptn->pExpr));
           }
         }
@@ -907,15 +908,15 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
         EXPLAIN_ROW_END();
         QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
         if (pFillNode->pValues) {
-          SNodeListNode *pValues = (SNodeListNode*)pFillNode->pValues;
+          SNodeListNode *pValues = (SNodeListNode *)pFillNode->pValues;
           EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILL_VALUE_FORMAT);
-          SNode* tNode = NULL;
+          SNode  *tNode = NULL;
           int32_t i = 0;
           FOREACH(tNode, pValues->pNodeList) {
             if (i) {
               EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
             }
-            SValueNode* tValue = (SValueNode*)tNode;
+            SValueNode *tValue = (SValueNode *)tNode;
             char       *value = nodesGetStrValueFromNode(tValue);
             EXPLAIN_ROW_APPEND(EXPLAIN_STRING_TYPE_FORMAT, value);
             taosMemoryFree(value);
@@ -926,8 +927,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
           QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
         }
 
-        EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pFillNode->timeRange.skey,
+        EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pFillNode->timeRange.skey, pFillNode->timeRange.ekey);
-                        pFillNode->timeRange.ekey);
         EXPLAIN_ROW_END();
         QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
 
@@ -1076,7 +1076,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
         EXPLAIN_ROW_NEW(level + 1, "Merge Key: ");
         if (pResNode->pExecInfo) {
           for (int32_t i = 0; i < LIST_LENGTH(pMergeNode->pMergeKeys); ++i) {
-            SOrderByExprNode *ptn = (SOrderByExprNode*)nodesListGetNode(pMergeNode->pMergeKeys, i);
+            SOrderByExprNode *ptn = (SOrderByExprNode *)nodesListGetNode(pMergeNode->pMergeKeys, i);
             EXPLAIN_ROW_APPEND("%s ", nodesGetNameFromColumnNode(ptn->pExpr));
           }
         }
@@ -1115,7 +1115,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
 
         EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGE_KEYS_FORMAT);
         for (int32_t i = 0; i < LIST_LENGTH(pMergeNode->pMergeKeys); ++i) {
-          SOrderByExprNode *ptn = (SOrderByExprNode*)nodesListGetNode(pMergeNode->pMergeKeys, i);
+          SOrderByExprNode *ptn = (SOrderByExprNode *)nodesListGetNode(pMergeNode->pMergeKeys, i);
           EXPLAIN_ROW_APPEND("%s ", nodesGetNameFromColumnNode(ptn->pExpr));
         }
         EXPLAIN_ROW_END();
@@ -1195,7 +1195,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
   blockDataAppendColInfo(pBlock, &infoData);
   blockDataEnsureCapacity(pBlock, rowNum);
 
-  SColumnInfoData* pInfoData = taosArrayGet(pBlock->pDataBlock, 0);
+  SColumnInfoData *pInfoData = taosArrayGet(pBlock->pDataBlock, 0);
 
   char buf[1024] = {0};
   for (int32_t i = 0; i < rowNum; ++i) {
@@ -1219,7 +1219,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
   rsp->numOfRows = htonl(rowNum);
 
   int32_t len = 0;
-  blockCompressEncode(pBlock, rsp->data, &len, taosArrayGetSize(pBlock->pDataBlock), 0);
+  blockEncode(pBlock, rsp->data, &len, taosArrayGetSize(pBlock->pDataBlock), 0);
   ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
 
   rsp->compLen = htonl(len);

source/libs/executor/inc/executil.h

@@ -25,19 +25,18 @@
 #define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid)     \
   do {                                               \
     assert(sizeof(_uid) == sizeof(uint64_t));        \
-    *(uint64_t *)(_k) = (_uid);                      \
+    *(uint64_t*)(_k) = (_uid);                       \
     memcpy((_k) + sizeof(uint64_t), (_ori), (_len)); \
   } while (0)
 
 #define SET_RES_EXT_WINDOW_KEY(_k, _ori, _len, _uid, _buf)           \
   do {                                                               \
     assert(sizeof(_uid) == sizeof(uint64_t));                        \
-    *(void **)(_k) = (_buf);                                             \
+    *(void**)(_k) = (_buf);                                          \
-    *(uint64_t *)((_k) + POINTER_BYTES) = (_uid);                      \
+    *(uint64_t*)((_k) + POINTER_BYTES) = (_uid);                     \
     memcpy((_k) + POINTER_BYTES + sizeof(uint64_t), (_ori), (_len)); \
   } while (0)
 
-
 #define GET_RES_WINDOW_KEY_LEN(_l)     ((_l) + sizeof(uint64_t))
 #define GET_RES_EXT_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t) + POINTER_BYTES)
 
@@ -50,7 +49,7 @@ typedef struct SGroupResInfo {
 
 typedef struct SResultRow {
   int32_t                    pageId;  // pageId & rowId is the position of current result in disk-based output buffer
-  int32_t       offset:29;   // row index in buffer page
+  int32_t                    offset : 29;  // row index in buffer page
   bool                       startInterp;  // the time window start timestamp has done the interpolation already.
   bool                       endInterp;    // the time window end timestamp has done the interpolation already.
   bool                       closed;       // this result status: closed or opened
@@ -84,14 +83,14 @@ void    cleanupResultRowInfo(SResultRowInfo* pResultRowInfo);
 
 void closeAllResultRows(SResultRowInfo* pResultRowInfo);
 
-void    initResultRow(SResultRow *pResultRow);
+void initResultRow(SResultRow* pResultRow);
 void closeResultRow(SResultRow* pResultRow);
 bool isResultRowClosed(SResultRow* pResultRow);
 
 struct SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t index, const int32_t* offset);
 
-static FORCE_INLINE SResultRow *getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos) {
+static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos) {
-  SFilePage*  bufPage = (SFilePage*) getBufPage(pBuf, pos->pageId);
+  SFilePage*  bufPage = (SFilePage*)getBufPage(pBuf, pos->pageId);
   SResultRow* pRow = (SResultRow*)((char*)bufPage + pos->offset);
   return pRow;
 }
@@ -110,7 +109,8 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext);
 int32_t  getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo* pListInfo);
 SArray*  createSortInfo(SNodeList* pNodeList);
 SArray*  extractPartitionColInfo(SNodeList* pNodeList);
-SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, int32_t type);
+SArray*  extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols,
+                             int32_t type);
 
 SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs);
 

source/libs/executor/inc/executorimpl.h

@@ -253,18 +253,15 @@ typedef struct STableScanInfo {
   SReadHandle     readHandle;
 
   SFileBlockLoadRecorder readRecorder;
-  int64_t         numOfRows;
   SScanInfo       scanInfo;
   int32_t         scanTimes;
   SNode*          pFilterNode;  // filter info, which is push down by optimizer
-  SqlFunctionCtx* pCtx;         // which belongs to the direct upstream operator operator query context
+  SqlFunctionCtx* pCtx;         // which belongs to the direct upstream operator operator query context,todo: remove this by using SExprSup
-  SResultRowInfo* pResultRowInfo;
+  int32_t*        rowEntryInfoOffset;  // todo: remove this by using SExprSup
-  int32_t*        rowEntryInfoOffset;
+  SExprInfo*      pExpr;// todo: remove this by using SExprSup
-  SExprInfo*      pExpr;
+
   SSDataBlock*    pResBlock;
   SArray*         pColMatchInfo;
-  int32_t         numOfOutput;
-
   SExprSupp       pseudoSup;
   SQueryTableDataCond cond;
   int32_t         scanFlag;     // table scan flag to denote if it is a repeat/reverse/main scan
@@ -275,8 +272,13 @@ typedef struct STableScanInfo {
   int32_t         curTWinIdx;
 
   int32_t         currentGroupId;
-  uint64_t        queryId;
+  uint64_t        queryId;   // todo remove it
-  uint64_t        taskId;
+  uint64_t        taskId;    // todo remove it
+
+  struct {
+    uint64_t uid;
+    int64_t t;
+  } scanStatus;
 } STableScanInfo;
 
 typedef struct STagScanInfo {
@@ -321,31 +323,31 @@ typedef struct SessionWindowSupporter {
 } SessionWindowSupporter;
 
 typedef struct SStreamBlockScanInfo {
+  uint64_t        tableUid;         // queried super table uid
+  SExprInfo*      pPseudoExpr;
+  int32_t         numOfPseudoExpr;
+  int32_t         primaryTsIndex;    // primary time stamp slot id
+  SReadHandle     readHandle;
+  SInterval       interval;     // if the upstream is an interval operator, the interval info is also kept here.
+  SArray*         pColMatchInfo;    //
+  SNode*          pCondition;
+
   SArray*         pBlockLists;      // multiple SSDatablock.
   SSDataBlock*    pRes;             // result SSDataBlock
   SSDataBlock*    pUpdateRes;       // update SSDataBlock
   int32_t         updateResIndex;
   int32_t         blockType;        // current block type
   int32_t         validBlockIndex;  // Is current data has returned?
-  SColumnInfo*    pCols;            // the output column info
   uint64_t        numOfExec;        // execution times
   void*           streamBlockReader;// stream block reader handle
-  SArray*         pColMatchInfo;    //
+
-  SNode*          pCondition;
   int32_t         tsArrayIndex;
   SArray*         tsArray;
   uint64_t        groupId;
   SUpdateInfo*    pUpdateInfo;
 
-  SExprInfo*      pPseudoExpr;
-  int32_t         numOfPseudoExpr;
-
-  int32_t         primaryTsIndex;    // primary time stamp slot id
-  SReadHandle     readHandle;
-  uint64_t        tableUid;         // queried super table uid
   EStreamScanMode scanMode;
   SOperatorInfo* pSnapshotReadOp;
-  SInterval      interval;     // if the upstream is an interval operator, the interval info is also kept here.
   SArray*        childIds;
   SessionWindowSupporter sessionSup;
   bool            assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA.
@@ -416,6 +418,7 @@ typedef struct SIntervalAggOperatorInfo {
   STimeWindowAggSupp twAggSup;
   bool               invertible;
   SArray*            pPrevValues;        //  SArray<SGroupKeys> used to keep the previous not null value for interpolation.
+  bool               ignoreCloseWindow;
 } SIntervalAggOperatorInfo;
 
 typedef struct SStreamFinalIntervalOperatorInfo {
@@ -437,6 +440,7 @@ typedef struct SStreamFinalIntervalOperatorInfo {
   SArray*            pPullWins;          // SPullWindowInfo
   int32_t            pullIndex;
   SSDataBlock*       pPullDataRes;
+  bool               ignoreCloseWindow;
 } SStreamFinalIntervalOperatorInfo;
 
 typedef struct SAggOperatorInfo {
@@ -574,6 +578,7 @@ typedef struct SStreamSessionAggOperatorInfo {
   SArray*              pChildren;       // cache for children's result; final stream operator
   SPhysiNode*          pPhyNode;           // create new child
   bool                 isFinal;
+  bool                 ignoreCloseWindow;
 } SStreamSessionAggOperatorInfo;
 
 typedef struct STimeSliceOperatorInfo {
@@ -617,6 +622,7 @@ typedef struct SStreamStateAggOperatorInfo {
   void*                pDelIterator;
   SArray*              pScanWindow;
   SArray*              pChildren;       // cache for children's result;
+  bool                 ignoreCloseWindow;
 } SStreamStateAggOperatorInfo;
 
 typedef struct SSortedMergeOperatorInfo {
@@ -683,7 +689,7 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t
 void    initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock);
 void    cleanupBasicInfo(SOptrBasicInfo* pInfo);
 int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr);
-void    cleanupExprSup(SExprSupp* pSup);
+void    cleanupExprSupp(SExprSupp* pSup);
 int32_t initAggInfo(SExprSupp *pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
                     const char* pkey);
 void    initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows);
@@ -707,7 +713,7 @@ void    destroyBasicOperatorInfo(void* param, int32_t numOfOutput);
 void    appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle);
 void    setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId);
 
-void    cleanupExecSupp(SExprSupp* pSupp);
+int32_t doGetScanStatus(SOperatorInfo* pOperator, uint64_t* uid, int64_t* ts);
 
 SSDataBlock* loadNextDataBlock(void* param);
 

source/libs/executor/src/dataDispatcher.c

@@ -69,8 +69,10 @@ static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) {
 
 // data format:
 // +----------------+--------------+----------+--------------------------------------------+--------------------------------------+-------------+-----------+-------------+-----------+
-// |SDataCacheEntry | total length | group id | col1_schema | col2_schema | col3_schema ...| column#1 length, column#2 length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | ....
+// |SDataCacheEntry | total length | group id | col1_schema | col2_schema | col3_schema ...| column#1 length, column#2
-// |                |  (4 bytes)   |(8 bytes) |(sizeof(int16_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols          | actual size |           | actual size |           |
+// length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | .... |                |  (4 bytes)   |(8 bytes)
+// |(sizeof(int16_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols          | actual size |           |
+// actual size |           |
 // +----------------+--------------+----------+--------------------------------------------+--------------------------------------+-------------+-----------+-------------+-----------+
 // The length of bitmap is decided by number of rows of this data block, and the length of each column data is
 // recorded in the first segment, next to the struct header
@@ -90,7 +92,7 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn
   pEntry->dataLen = 0;
 
   pBuf->useSize = sizeof(SDataCacheEntry);
-  blockCompressEncode(pInput->pData, pEntry->data, &pEntry->dataLen, numOfCols, pEntry->compressed);
+  blockEncode(pInput->pData, pEntry->data, &pEntry->dataLen, numOfCols, pEntry->compressed);
 
   pBuf->useSize += pEntry->dataLen;
 
@@ -198,7 +200,6 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
   pOutput->precision = pDispatcher->pSchema->precision;
   taosThreadMutexUnlock(&pDispatcher->mutex);
 
-  
   return TSDB_CODE_SUCCESS;
 }
 

source/libs/executor/src/executil.c

@@ -13,10 +13,10 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "os.h"
-#include "index.h"
 #include "function.h"
 #include "functionMgt.h"
+#include "index.h"
+#include "os.h"
 #include "tdatablock.h"
 #include "thash.h"
 #include "tmsg.h"
@@ -25,45 +25,41 @@
 #include "executorimpl.h"
 #include "tcompression.h"
 
-void initResultRowInfo(SResultRowInfo *pResultRowInfo) {
+void initResultRowInfo(SResultRowInfo* pResultRowInfo) {
   pResultRowInfo->size = 0;
   pResultRowInfo->cur.pageId = -1;
 }
 
-void cleanupResultRowInfo(SResultRowInfo *pResultRowInfo) {
+void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo) {
   if (pResultRowInfo == NULL) {
     return;
   }
 
-  for(int32_t i = 0; i < pResultRowInfo->size; ++i) {
+  for (int32_t i = 0; i < pResultRowInfo->size; ++i) {
-//    if (pResultRowInfo->pResult[i]) {
+    //    if (pResultRowInfo->pResult[i]) {
-//      taosMemoryFreeClear(pResultRowInfo->pResult[i]->key);
+    //      taosMemoryFreeClear(pResultRowInfo->pResult[i]->key);
-//    }
+    //    }
   }
 }
 
-void closeAllResultRows(SResultRowInfo *pResultRowInfo) {
+void closeAllResultRows(SResultRowInfo* pResultRowInfo) {
-// do nothing
+  // do nothing
 }
 
-bool isResultRowClosed(SResultRow* pRow) {
+bool isResultRowClosed(SResultRow* pRow) { return (pRow->closed == true); }
-  return (pRow->closed == true);
-}
 
-void closeResultRow(SResultRow* pResultRow) {
+void closeResultRow(SResultRow* pResultRow) { pResultRow->closed = true; }
-  pResultRow->closed = true;
-}
 
 // TODO refactor: use macro
 SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t index, const int32_t* offset) {
   assert(index >= 0 && offset != NULL);
-  return (SResultRowEntryInfo*)((char*) pRow->pEntryInfo + offset[index]);
+  return (SResultRowEntryInfo*)((char*)pRow->pEntryInfo + offset[index]);
 }
 
 size_t getResultRowSize(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
   int32_t rowSize = (numOfOutput * sizeof(SResultRowEntryInfo)) + sizeof(SResultRow);
 
-  for(int32_t i = 0; i < numOfOutput; ++i) {
+  for (int32_t i = 0; i < numOfOutput; ++i) {
     rowSize += pCtx[i].resDataInfo.interBufSize;
   }
 
@@ -79,26 +75,24 @@ void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) {
 }
 
 static int32_t resultrowComparAsc(const void* p1, const void* p2) {
-  SResKeyPos* pp1 = *(SResKeyPos**) p1;
+  SResKeyPos* pp1 = *(SResKeyPos**)p1;
-  SResKeyPos* pp2 = *(SResKeyPos**) p2;
+  SResKeyPos* pp2 = *(SResKeyPos**)p2;
 
   if (pp1->groupId == pp2->groupId) {
-    int64_t pts1 = *(int64_t*) pp1->key;
+    int64_t pts1 = *(int64_t*)pp1->key;
-    int64_t pts2 = *(int64_t*) pp2->key;
+    int64_t pts2 = *(int64_t*)pp2->key;
 
     if (pts1 == pts2) {
       return 0;
     } else {
-      return pts1 < pts2? -1:1;
+      return pts1 < pts2 ? -1 : 1;
     }
   } else {
-    return pp1->groupId < pp2->groupId? -1:1;
+    return pp1->groupId < pp2->groupId ? -1 : 1;
   }
 }
 
-static int32_t resultrowComparDesc(const void* p1, const void* p2) {
+static int32_t resultrowComparDesc(const void* p1, const void* p2) { return resultrowComparAsc(p2, p1); }
-  return resultrowComparAsc(p2, p1);
-}
 
 void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int32_t order) {
   if (pGroupResInfo->pRows != NULL) {
@@ -110,20 +104,20 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int
   pGroupResInfo->pRows = taosArrayInit(10, POINTER_BYTES);
 
   size_t keyLen = 0;
-  while((pData = taosHashIterate(pHashmap, pData)) != NULL) {
+  while ((pData = taosHashIterate(pHashmap, pData)) != NULL) {
     void* key = taosHashGetKey(pData, &keyLen);
 
     SResKeyPos* p = taosMemoryMalloc(keyLen + sizeof(SResultRowPosition));
 
-    p->groupId = *(uint64_t*) key;
+    p->groupId = *(uint64_t*)key;
-    p->pos = *(SResultRowPosition*) pData;
+    p->pos = *(SResultRowPosition*)pData;
     memcpy(p->key, (char*)key + sizeof(uint64_t), keyLen - sizeof(uint64_t));
 
     taosArrayPush(pGroupResInfo->pRows, &p);
   }
 
   if (order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC) {
-    __compar_fn_t fn = (order == TSDB_ORDER_ASC)? resultrowComparAsc:resultrowComparDesc;
+    __compar_fn_t fn = (order == TSDB_ORDER_ASC) ? resultrowComparAsc : resultrowComparDesc;
     qsort(pGroupResInfo->pRows->pData, taosArrayGetSize(pGroupResInfo->pRows), POINTER_BYTES, fn);
   }
 
@@ -155,7 +149,7 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo) {
     return 0;
   }
 
-  return (int32_t) taosArrayGetSize(pGroupResInfo->pRows);
+  return (int32_t)taosArrayGetSize(pGroupResInfo->pRows);
 }
 
 SArray* createSortInfo(SNodeList* pNodeList) {
@@ -195,11 +189,12 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) {
 
   for (int32_t i = 0; i < numOfCols; ++i) {
     SSlotDescNode* pDescNode = (SSlotDescNode*)nodesListGetNode(pNode->pSlots, i);
-//    if (!pDescNode->output) {  // todo disable it temporarily
+    /*if (!pDescNode->output) {  // todo disable it temporarily*/
-//      continue;
+    /*continue;*/
-//    }
+    /*}*/
 
-    SColumnInfoData idata = createColumnInfoData(pDescNode->dataType.type, pDescNode->dataType.bytes, pDescNode->slotId);
+    SColumnInfoData idata =
+        createColumnInfoData(pDescNode->dataType.type, pDescNode->dataType.bytes, pDescNode->slotId);
     idata.info.scale = pDescNode->dataType.scale;
     idata.info.precision = pDescNode->dataType.precision;
 
@@ -211,10 +206,10 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) {
 
 EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) {
   SMetaReader* mr = (SMetaReader*)pContext;
-  if(nodeType(*pNode) == QUERY_NODE_COLUMN){
+  if (nodeType(*pNode) == QUERY_NODE_COLUMN) {
     SColumnNode* pSColumnNode = *(SColumnNode**)pNode;
 
-    SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE);
+    SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
     if (NULL == res) {
       return DEAL_RES_ERROR;
     }
@@ -227,8 +222,8 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) {
     const char* p = metaGetTableTagVal(&mr->me, pSColumnNode->node.resType.type, &tagVal);
     if (p == NULL) {
       res->node.resType.type = TSDB_DATA_TYPE_NULL;
-    }else if (pSColumnNode->node.resType.type == TSDB_DATA_TYPE_JSON) {
+    } else if (pSColumnNode->node.resType.type == TSDB_DATA_TYPE_JSON) {
-      int32_t len = ((const STag*)p) -> len;
+      int32_t len = ((const STag*)p)->len;
       res->datum.p = taosMemoryCalloc(len + 1, 1);
       memcpy(res->datum.p, p, len);
     } else if (IS_VAR_DATA_TYPE(pSColumnNode->node.resType.type)) {
@@ -240,10 +235,10 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) {
     }
     nodesDestroyNode(*pNode);
     *pNode = (SNode*)res;
-  }else if (nodeType(*pNode) == QUERY_NODE_FUNCTION){
+  } else if (nodeType(*pNode) == QUERY_NODE_FUNCTION) {
-    SFunctionNode * pFuncNode = *(SFunctionNode**)pNode;
+    SFunctionNode* pFuncNode = *(SFunctionNode**)pNode;
-    if(pFuncNode->funcType == FUNCTION_TYPE_TBNAME){
+    if (pFuncNode->funcType == FUNCTION_TYPE_TBNAME) {
-      SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE);
+      SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
       if (NULL == res) {
         return DEAL_RES_ERROR;
       }
@@ -263,12 +258,12 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) {
   return DEAL_RES_CONTINUE;
 }
 
-static bool isTableOk(STableKeyInfo* info, SNode *pTagCond, SMeta *metaHandle){
+static bool isTableOk(STableKeyInfo* info, SNode* pTagCond, SMeta* metaHandle) {
   SMetaReader mr = {0};
   metaReaderInit(&mr, metaHandle, 0);
   metaGetTableEntryByUid(&mr, info->uid);
 
-  SNode *pTagCondTmp = nodesCloneNode(pTagCond);
+  SNode* pTagCondTmp = nodesCloneNode(pTagCond);
 
   nodesRewriteExprPostOrder(&pTagCondTmp, doTranslateTagExpr, &mr);
   metaReaderClear(&mr);
@@ -281,7 +276,7 @@ static bool isTableOk(STableKeyInfo* info, SNode *pTagCond, SMeta *metaHandle){
   }
 
   ASSERT(nodeType(pNew) == QUERY_NODE_VALUE);
-  SValueNode *pValue = (SValueNode *)pNew;
+  SValueNode* pValue = (SValueNode*)pNew;
 
   ASSERT(pValue->node.resType.type == TSDB_DATA_TYPE_BOOL);
   bool result = pValue->datum.b;
@@ -292,7 +287,7 @@ static bool isTableOk(STableKeyInfo* info, SNode *pTagCond, SMeta *metaHandle){
 int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo* pListInfo) {
   int32_t code = TSDB_CODE_SUCCESS;
   pListInfo->pTableList = taosArrayInit(8, sizeof(STableKeyInfo));
-  if(pListInfo->pTableList == NULL) return TSDB_CODE_OUT_OF_MEMORY;
+  if (pListInfo->pTableList == NULL) return TSDB_CODE_OUT_OF_MEMORY;
 
   uint64_t tableUid = pScanNode->uid;
 
@@ -306,7 +301,7 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo
           .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = tsdbGetIvtIdx(metaHandle), .suid = tableUid};
 
       SArray* res = taosArrayInit(8, sizeof(uint64_t));
-      //code = doFilterTag(pTagIndexCond, &metaArg, res);
+      // code = doFilterTag(pTagIndexCond, &metaArg, res);
       code = TSDB_CODE_INDEX_REBUILDING;
       if (code == TSDB_CODE_INDEX_REBUILDING) {
         code = tsdbGetAllTableList(metaHandle, tableUid, pListInfo->pTableList);
@@ -328,33 +323,33 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo
       code = tsdbGetAllTableList(metaHandle, tableUid, pListInfo->pTableList);
     }
 
-    if(pTagCond){
+    if (pTagCond) {
       int32_t i = 0;
-      while(i < taosArrayGetSize(pListInfo->pTableList)) {
+      while (i < taosArrayGetSize(pListInfo->pTableList)) {
         STableKeyInfo* info = taosArrayGet(pListInfo->pTableList, i);
         bool           isOk = isTableOk(info, pTagCond, metaHandle);
-        if(!isOk){
+        if (!isOk) {
           taosArrayRemove(pListInfo->pTableList, i);
           continue;
         }
         i++;
       }
     }
-  }else {  // Create one table group.
+  } else {  // Create one table group.
     STableKeyInfo info = {.lastKey = 0, .uid = tableUid, .groupId = 0};
     taosArrayPush(pListInfo->pTableList, &info);
   }
   pListInfo->pGroupList = taosArrayInit(4, POINTER_BYTES);
-  if(pListInfo->pGroupList == NULL) return TSDB_CODE_OUT_OF_MEMORY;
+  if (pListInfo->pGroupList == NULL) return TSDB_CODE_OUT_OF_MEMORY;
 
-  //put into list as default group, remove it if grouping sorting is required later
+  // put into list as default group, remove it if grouping sorting is required later
   taosArrayPush(pListInfo->pGroupList, &pListInfo->pTableList);
 
   return code;
 }
 
 SArray* extractPartitionColInfo(SNodeList* pNodeList) {
-  if(!pNodeList) {
+  if (!pNodeList) {
     return NULL;
   }
 
@@ -383,7 +378,6 @@ SArray* extractPartitionColInfo(SNodeList* pNodeList) {
   return pList;
 }
 
-
 SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols,
                             int32_t type) {
   size_t  numOfCols = LIST_LENGTH(pNodeList);
@@ -675,8 +669,8 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
   }
 
   for (int32_t i = 1; i < numOfOutput; ++i) {
-    (*rowEntryInfoOffset)[i] =
+    (*rowEntryInfoOffset)[i] = (int32_t)((*rowEntryInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) +
-        (int32_t)((*rowEntryInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pFuncCtx[i - 1].resDataInfo.interBufSize);
+                                         pFuncCtx[i - 1].resDataInfo.interBufSize);
   }
 
   setSelectValueColumnInfo(pFuncCtx, numOfOutput);

source/libs/executor/src/executor.c

@@ -44,12 +44,12 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
     // prevent setting a different type of block
     pInfo->blockType = type;
 
-    if (type == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
+    if (type == STREAM_INPUT__DATA_SUBMIT) {
       if (tqReadHandleSetMsg(pInfo->streamBlockReader, input, 0) < 0) {
         qError("submit msg messed up when initing stream block, %s" PRIx64, id);
         return TSDB_CODE_QRY_APP_ERROR;
       }
-    } else if (type == STREAM_DATA_TYPE_SSDATA_BLOCK) {
+    } else if (type == STREAM_INPUT__DATA_BLOCK) {
       for (int32_t i = 0; i < numOfBlocks; ++i) {
         SSDataBlock* pDataBlock = &((SSDataBlock*)input)[i];
 
@@ -60,9 +60,9 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
         taosArrayAddAll(p->pDataBlock, pDataBlock->pDataBlock);
         taosArrayPush(pInfo->pBlockLists, &p);
       }
-    } else if (type == STREAM_DATA_TYPE_FROM_SNAPSHOT) {
+    } else if (type == STREAM_INPUT__DATA_SCAN) {
       // do nothing
-      ASSERT(pInfo->blockType == STREAM_DATA_TYPE_FROM_SNAPSHOT);
+      ASSERT(pInfo->blockType == STREAM_INPUT__DATA_SCAN);
     } else {
       ASSERT(0);
     }
@@ -76,7 +76,7 @@ int32_t qStreamScanSnapshot(qTaskInfo_t tinfo) {
     return TSDB_CODE_QRY_APP_ERROR;
   }
   SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
-  return doSetStreamBlock(pTaskInfo->pRoot, NULL, 0, STREAM_DATA_TYPE_FROM_SNAPSHOT, 0, NULL);
+  return doSetStreamBlock(pTaskInfo->pRoot, NULL, 0, STREAM_INPUT__DATA_SCAN, 0, NULL);
 }
 
 int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type, bool assignUid) {

source/libs/executor/src/executorMain.c

@@ -191,16 +191,6 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t qIsTaskCompleted(qTaskInfo_t qinfo) {
-  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo;
-
-  if (pTaskInfo == NULL) {
-    return TSDB_CODE_QRY_INVALID_QHANDLE;
-  }
-
-  return isTaskKilled(pTaskInfo);
-}
-
 void qDestroyTask(qTaskInfo_t qTaskHandle) {
   SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qTaskHandle;
   qDebug("%s execTask completed, numOfRows:%" PRId64, GET_TASKID(pTaskInfo), pTaskInfo->pRoot->resultInfo.totalRows);
@@ -242,3 +232,10 @@ int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t le
 }
 
 
+int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts) {
+  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*) tinfo;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+

source/libs/executor/src/executorimpl.c

@@ -1033,7 +1033,7 @@ static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSData
   SqlFunctionCtx* pCtx = pTableScanInfo->pCtx;
   uint32_t        status = BLK_DATA_NOT_LOAD;
 
-  int32_t numOfOutput = pTableScanInfo->numOfOutput;
+  int32_t numOfOutput = 0;//pTableScanInfo->numOfOutput;
   for (int32_t i = 0; i < numOfOutput; ++i) {
     int32_t functionId = pCtx[i].functionId;
     int32_t colId = pTableScanInfo->pExpr[i].base.pParam[0].pCol->colId;
@@ -2049,7 +2049,7 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo
   if (pColList == NULL) {  // data from other sources
     blockDataCleanup(pRes);
     //    blockDataEnsureCapacity(pRes, numOfRows);
-    blockCompressDecode(pRes, numOfOutput, numOfRows, pData);
+    blockDecode(pRes, numOfOutput, numOfRows, pData);
   } else {  // extract data according to pColList
     ASSERT(numOfOutput == taosArrayGetSize(pColList));
     char* pStart = pData;
@@ -2073,7 +2073,7 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo
       blockDataAppendColInfo(pBlock, &idata);
     }
 
-    blockCompressDecode(pBlock, numOfCols, numOfRows, pStart);
+    blockDecode(pBlock, numOfCols, numOfRows, pStart);
     blockDataEnsureCapacity(pRes, numOfRows);
 
     // data from mnode
@@ -2822,6 +2822,24 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan
   }
 }
 
+int32_t doGetScanStatus(SOperatorInfo* pOperator, uint64_t* uid, int64_t* ts) {
+  int32_t type = pOperator->operatorType;
+  if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
+    SStreamBlockScanInfo* pScanInfo = pOperator->info;
+    STableScanInfo* pSnapShotScanInfo = pScanInfo->pSnapshotReadOp->info;
+    *uid = pSnapShotScanInfo->scanStatus.uid;
+    *ts = pSnapShotScanInfo->scanStatus.t;
+  } else {
+    if (pOperator->pDownstream[0] == NULL) {
+      return TSDB_CODE_INVALID_PARA;
+    } else {
+      doGetScanStatus(pOperator->pDownstream[0], uid, ts);
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
 // this is a blocking operator
 static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
   if (OPTR_IS_OPENED(pOperator)) {
@@ -3544,7 +3562,7 @@ static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) {
   taosArrayDestroy(pInfo->pPseudoColInfo);
 }
 
-void cleanupExecSupp(SExprSupp* pSupp) {
+void cleanupExprSupp(SExprSupp* pSupp) {
   destroySqlFunctionCtx(pSupp->pCtx, pSupp->numOfExprs);
   destroyExprInfo(pSupp->pExprInfo, pSupp->numOfExprs);
 
@@ -3557,7 +3575,7 @@ static void destroyIndefinitOperatorInfo(void* param, int32_t numOfOutput) {
 
   taosArrayDestroy(pInfo->pPseudoColInfo);
   cleanupAggSup(&pInfo->aggSup);
-  cleanupExecSupp(&pInfo->scalarSup);
+  cleanupExprSupp(&pInfo->scalarSup);
 }
 
 void destroyExchangeOperatorInfo(void* param, int32_t numOfOutput) {
@@ -3899,60 +3917,59 @@ int32_t extractTableSchemaVersion(SReadHandle* pHandle, uint64_t uid, SExecTaskI
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t sortTableGroup(STableListInfo* pTableListInfo, int32_t groupNum){
+static int32_t sortTableGroup(STableListInfo* pTableListInfo, int32_t groupNum) {
   taosArrayClear(pTableListInfo->pGroupList);
-  SArray *sortSupport = taosArrayInit(groupNum, sizeof(uint64_t));
+  SArray* sortSupport = taosArrayInit(groupNum, sizeof(uint64_t));
-  if(sortSupport == NULL) return TSDB_CODE_OUT_OF_MEMORY;
+  if (sortSupport == NULL) return TSDB_CODE_OUT_OF_MEMORY;
   for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); i++) {
     STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i);
     uint64_t*      groupId = taosHashGet(pTableListInfo->map, &info->uid, sizeof(uint64_t));
 
     int32_t index = taosArraySearchIdx(sortSupport, groupId, compareUint64Val, TD_EQ);
-    if (index == -1){
+    if (index == -1) {
-      void *p = taosArraySearch(sortSupport, groupId, compareUint64Val, TD_GT);
+      void*   p = taosArraySearch(sortSupport, groupId, compareUint64Val, TD_GT);
-      SArray *tGroup = taosArrayInit(8, sizeof(STableKeyInfo));
+      SArray* tGroup = taosArrayInit(8, sizeof(STableKeyInfo));
-      if(tGroup == NULL) {
+      if (tGroup == NULL) {
         taosArrayDestroy(sortSupport);
         return TSDB_CODE_OUT_OF_MEMORY;
       }
-      if(taosArrayPush(tGroup, info) == NULL){
+      if (taosArrayPush(tGroup, info) == NULL) {
         qError("taos push info array error");
         taosArrayDestroy(sortSupport);
         return TSDB_CODE_QRY_APP_ERROR;
       }
-      if(p == NULL){
+      if (p == NULL) {
-        if(taosArrayPush(sortSupport, groupId) != NULL){
+        if (taosArrayPush(sortSupport, groupId) != NULL) {
           qError("taos push support array error");
           taosArrayDestroy(sortSupport);
           return TSDB_CODE_QRY_APP_ERROR;
         }
-        if(taosArrayPush(pTableListInfo->pGroupList, &tGroup) != NULL){
+        if (taosArrayPush(pTableListInfo->pGroupList, &tGroup) != NULL) {
           qError("taos push group array error");
           taosArrayDestroy(sortSupport);
           return TSDB_CODE_QRY_APP_ERROR;
         }
-      }else{
+      } else {
         int32_t pos = TARRAY_ELEM_IDX(sortSupport, p);
-        if(taosArrayInsert(sortSupport, pos, groupId) == NULL){
+        if (taosArrayInsert(sortSupport, pos, groupId) == NULL) {
           qError("taos insert support array error");
           taosArrayDestroy(sortSupport);
           return TSDB_CODE_QRY_APP_ERROR;
         }
-        if(taosArrayInsert(pTableListInfo->pGroupList, pos, &tGroup) == NULL){
+        if (taosArrayInsert(pTableListInfo->pGroupList, pos, &tGroup) == NULL) {
           qError("taos insert group array error");
           taosArrayDestroy(sortSupport);
           return TSDB_CODE_QRY_APP_ERROR;
         }
       }
-    }else{
+    } else {
       SArray* tGroup = (SArray*)taosArrayGetP(pTableListInfo->pGroupList, index);
-      if(taosArrayPush(tGroup, info) == NULL){
+      if (taosArrayPush(tGroup, info) == NULL) {
         qError("taos push uid array error");
         taosArrayDestroy(sortSupport);
         return TSDB_CODE_QRY_APP_ERROR;
       }
     }
-
   }
   taosArrayDestroy(sortSupport);
   return TDB_CODE_SUCCESS;
@@ -3972,7 +3989,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
 
   SNode* node;
   FOREACH(node, group) {
-    SExprNode *pExpr =  (SExprNode *)node;
+    SExprNode* pExpr = (SExprNode*)node;
     keyLen += pExpr->resType.bytes;
   }
 
@@ -3991,7 +4008,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
     metaReaderInit(&mr, pHandle->meta, 0);
     metaGetTableEntryByUid(&mr, info->uid);
 
-    SNodeList *groupNew = nodesCloneList(group);
+    SNodeList* groupNew = nodesCloneList(group);
 
     nodesRewriteExprsPostOrder(groupNew, doTranslateTagExpr, &mr);
     char* isNull = (char*)keyBuf;
@@ -3999,7 +4016,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
 
     SNode*  pNode;
     int32_t index = 0;
-    FOREACH(pNode, groupNew){
+    FOREACH(pNode, groupNew) {
       SNode*  pNew = NULL;
       int32_t code = scalarCalculateConstants(pNode, &pNew);
       if (TSDB_CODE_SUCCESS == code) {
@@ -4011,7 +4028,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
       }
 
       ASSERT(nodeType(pNew) == QUERY_NODE_VALUE);
-      SValueNode *pValue = (SValueNode *)pNew;
+      SValueNode* pValue = (SValueNode*)pNew;
 
       if (pValue->node.resType.type == TSDB_DATA_TYPE_NULL) {
         isNull[index++] = 1;
@@ -4019,7 +4036,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
       } else {
         isNull[index++] = 0;
         char* data = nodesGetValueFromNode(pValue);
-        if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON){
+        if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON) {
           int32_t len = getJsonValueLen(data);
           memcpy(pStart, data, len);
           pStart += len;
@@ -4043,7 +4060,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
   }
   taosMemoryFree(keyBuf);
 
-  if(pTableListInfo->needSortTableByGroupId){
+  if (pTableListInfo->needSortTableByGroupId) {
     return sortTableGroup(pTableListInfo, groupNum);
   }
 
@@ -4051,7 +4068,8 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
 }
 
 SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle,
-                                  uint64_t queryId, uint64_t taskId, STableListInfo* pTableListInfo, const char* pUser) {
+                                  uint64_t queryId, uint64_t taskId, STableListInfo* pTableListInfo,
+                                  const char* pUser) {
   int32_t type = nodeType(pPhyNode);
 
   if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) {
@@ -4059,7 +4077,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
 
       int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
-      if(code){
+      if (code) {
         pTaskInfo->code = code;
         return NULL;
       }
@@ -4077,7 +4095,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) {
       STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode;
       int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
-      if(code){
+      if (code) {
         return NULL;
       }
       code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo);
@@ -4086,7 +4104,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
         return NULL;
       }
 
-      SOperatorInfo*  pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId);
+      SOperatorInfo* pOperator =
+          createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId);
 
       STableScanInfo* pScanInfo = pOperator->info;
       pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder;
@@ -4106,7 +4125,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
         createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
       }
 
-      SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTaskInfo, &twSup, queryId, taskId);
+      SOperatorInfo* pOperator =
+          createStreamScanOperatorInfo(pHandle, pTableScanNode, pTaskInfo, &twSup, queryId, taskId);
       return pOperator;
 
     } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) {
@@ -4604,8 +4624,8 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead
   (*pTaskInfo)->sql = sql;
   (*pTaskInfo)->tableqinfoList.pTagCond = pPlan->pTagCond;
   (*pTaskInfo)->tableqinfoList.pTagIndexCond = pPlan->pTagIndexCond;
-  (*pTaskInfo)->pRoot =
+  (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId,
-      createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoList, pPlan->user);
+                                           &(*pTaskInfo)->tableqinfoList, pPlan->user);
 
   if (NULL == (*pTaskInfo)->pRoot) {
     code = (*pTaskInfo)->code;
@@ -4623,8 +4643,8 @@ _complete:
 static void doDestroyTableList(STableListInfo* pTableqinfoList) {
   taosArrayDestroy(pTableqinfoList->pTableList);
   taosHashCleanup(pTableqinfoList->map);
-  if(pTableqinfoList->needSortTableByGroupId){
+  if (pTableqinfoList->needSortTableByGroupId) {
-    for(int32_t i = 0; i < taosArrayGetSize(pTableqinfoList->pGroupList); i++){
+    for (int32_t i = 0; i < taosArrayGetSize(pTableqinfoList->pGroupList); i++) {
       SArray* tmp = taosArrayGetP(pTableqinfoList->pGroupList, i);
       taosArrayDestroy(tmp);
     }

source/libs/executor/src/groupoperator.c

@@ -37,7 +37,7 @@ static void destroyGroupOperatorInfo(void* param, int32_t numOfOutput) {
   taosMemoryFreeClear(pInfo->keyBuf);
   taosArrayDestroy(pInfo->pGroupCols);
   taosArrayDestroy(pInfo->pGroupColVals);
-  cleanupExecSupp(&pInfo->scalarSup);
+  cleanupExprSupp(&pInfo->scalarSup);
 }
 
 static int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) {
@@ -701,7 +701,7 @@ static void destroyPartitionOperatorInfo(void* param, int32_t numOfOutput) {
   taosHashCleanup(pInfo->pGroupSet);
   taosMemoryFree(pInfo->columnOffset);
 
-  cleanupExecSupp(&pInfo->scalarSup);
+  cleanupExprSupp(&pInfo->scalarSup);
 }
 
 SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo) {

source/libs/executor/src/scanoperator.c

@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <executorimpl.h>
 #include <vnode.h>
 #include "filter.h"
 #include "function.h"
@@ -413,6 +414,11 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
     pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
 
     pOperator->cost.totalCost = pTableScanInfo->readRecorder.elapsedTime;
+
+    // todo refactor
+    pTableScanInfo->scanStatus.uid = pBlock->info.uid;
+    pTableScanInfo->scanStatus.t = pBlock->info.window.ekey;
+
     return pBlock;
   }
   return NULL;
@@ -459,7 +465,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) {
   int32_t total = pTableScanInfo->scanInfo.numOfAsc + pTableScanInfo->scanInfo.numOfDesc;
   if (pTableScanInfo->scanTimes < total) {
     if (pTableScanInfo->cond.order == TSDB_ORDER_ASC) {
-      prepareForDescendingScan(pTableScanInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput);
+      prepareForDescendingScan(pTableScanInfo, pTableScanInfo->pCtx, 0);
       tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
       pTableScanInfo->curTWinIdx = 0;
     }
@@ -962,7 +968,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
 
   size_t total = taosArrayGetSize(pInfo->pBlockLists);
   // TODO: refactor
-  if (pInfo->blockType == STREAM_DATA_TYPE_SSDATA_BLOCK) {
+  if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) {
     if (pInfo->validBlockIndex >= total) {
       /*doClearBufferedBlocks(pInfo);*/
       pOperator->status = OP_EXEC_DONE;
@@ -973,7 +979,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
     SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current);
     blockDataUpdateTsWindow(pBlock, 0);
     if (pBlock->info.type == STREAM_RETRIEVE) {
-      pInfo->blockType = STREAM_DATA_TYPE_SUBMIT_BLOCK;
+      pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
       pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RETRIEVE;
       copyDataBlock(pInfo->pPullDataRes, pBlock);
       pInfo->pullDataResIndex = 0;
@@ -981,7 +987,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
       updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo);
     }
     return pBlock;
-  } else if (pInfo->blockType == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
+  } else if (pInfo->blockType == STREAM_INPUT__DATA_SUBMIT) {
     if (pInfo->scanMode == STREAM_SCAN_FROM_RES) {
       blockDataDestroy(pInfo->pUpdateRes);
       pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
@@ -996,7 +1002,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
       SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pPullDataRes, 0, &pInfo->pullDataResIndex);
       if (pSDB != NULL) {
         getUpdateDataBlock(pInfo, true, pSDB, NULL);
-        pSDB->info.type = STREAM_PUSH_DATA;
+        pSDB->info.type = STREAM_PULL_DATA;
         return pSDB;
       }
       pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
@@ -1133,7 +1139,9 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
 
     return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes;
 
-  } else if (pInfo->blockType == STREAM_DATA_TYPE_FROM_SNAPSHOT) {
+  } else if (pInfo->blockType == STREAM_INPUT__DATA_SCAN) {
+    // check reader last status
+    // if not match, reset status
     SSDataBlock* pResult = doTableScan(pInfo->pSnapshotReadOp);
     return pResult && pResult->info.rows > 0 ? pResult : NULL;
 
@@ -1213,7 +1221,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
     pInfo->tableUid = pScanPhyNode->uid;
 
     // set the extract column id to streamHandle
-    tqReadHandleSetColIdList((STqReadHandle*)pHandle->reader, pColIds);
+    tqReadHandleSetColIdList((SStreamReader*)pHandle->reader, pColIds);
     SArray* tableIdList = extractTableIdList(&pTaskInfo->tableqinfoList);
     int32_t code = tqReadHandleSetTbUidList(pHandle->reader, tableIdList);
     if (code != 0) {

source/libs/executor/src/timewindowoperator.c

@@ -695,11 +695,11 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num
 }
 
 void printDataBlock(SSDataBlock* pBlock, const char* flag) {
-  if (pBlock == NULL){
+  if (pBlock == NULL) {
     qDebug("======printDataBlock Block is Null");
     return;
   }
-  char *pBuf = NULL;
+  char* pBuf = NULL;
   qDebug("%s", dumpBlockData(pBlock, flag, &pBuf));
   taosMemoryFree(pBuf);
 }
@@ -813,6 +813,13 @@ static void removeResults(SArray* pWins, SArray* pUpdated) {
   }
 }
 
+bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup) {
+  ASSERT(pSup->maxTs == INT64_MIN || pSup->maxTs > 0);
+  return pSup->maxTs != INT64_MIN && ts < pSup->maxTs - pSup->waterMark;
+}
+
+bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup) { return isOverdue(pWin->ekey, pSup); }
+
 static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
                             int32_t scanFlag, SArray* pUpdated) {
   SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
@@ -830,15 +837,15 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
 
   STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval,
                                         pInfo->interval.precision, &pInfo->win);
-
+  int32_t     ret = TSDB_CODE_SUCCESS;
-  int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
+  if (!pInfo->ignoreCloseWindow || !isCloseWindow(&win, &pInfo->twAggSup)) {
-                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
+    ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
+                                 numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
     if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
       longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
     }
 
-  if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
+    if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM && pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
-    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
       saveResultRow(pResult, tableGroupId, pUpdated);
     }
   }
@@ -864,9 +871,11 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
     doWindowBorderInterpolation(pInfo, pBlock, pResult, &win, startPos, forwardRows, pSup);
   }
 
+  if (!pInfo->ignoreCloseWindow || !isCloseWindow(&win, &pInfo->twAggSup)) {
     updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
     doApplyFunctions(pTaskInfo, pSup->pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols,
                      pBlock->info.rows, numOfOutput, pInfo->order);
+  }
 
   doCloseWindow(pResultRowInfo, pInfo, pResult);
 
@@ -877,6 +886,12 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
     if (startPos < 0) {
       break;
     }
+    if (pInfo->ignoreCloseWindow && isCloseWindow(&nextWin, &pInfo->twAggSup)) {
+      ekey = ascScan ? nextWin.ekey : nextWin.skey;
+      forwardRows =
+          getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order);
+      continue;
+    }
 
     // null data, failed to allocate more memory buffer
     int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
@@ -885,11 +900,9 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
       longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
     }
 
-    if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
+    if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM && pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
-      if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
       saveResultRow(pResult, tableGroupId, pUpdated);
     }
-    }
 
     ekey = ascScan ? nextWin.ekey : nextWin.skey;
     forwardRows =
@@ -970,9 +983,8 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
     getTableScanInfo(pOperator, &pInfo->order, &scanFlag);
 
     if (pInfo->scalarSupp.pExprInfo != NULL) {
-      SExprSupp* pExprSup =& pInfo->scalarSupp;
+      SExprSupp* pExprSup = &pInfo->scalarSupp;
-      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx,
+      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
-          pExprSup->numOfExprs, NULL);
     }
 
     // the pDataBlock are always the same one, no need to call this again
@@ -1292,13 +1304,8 @@ static int32_t getAllIntervalWindow(SHashObj* pHashMap, SArray* resWins) {
   return TSDB_CODE_SUCCESS;
 }
 
-bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup) {
+static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval,
-  ASSERT(pSup->maxTs == INT64_MIN || pSup->maxTs > 0);
+                                   SHashObj* pPullDataMap, SArray* closeWins) {
-  return pSup->maxTs != INT64_MIN && pWin->ekey < pSup->maxTs - pSup->waterMark;
-}
-
-static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup,
-    SInterval* pInterval, SHashObj* pPullDataMap, SArray* closeWins) {
   void*  pIte = NULL;
   size_t keyLen = 0;
   while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
@@ -1309,15 +1316,18 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup,
     SResultRowInfo dumyInfo;
     dumyInfo.cur.pageId = -1;
     STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, ts, pInterval, pInterval->precision, NULL);
-    SWinRes winRe = {.ts = win.skey, .groupId = groupId,};
+    SWinRes     winRe = {
+            .ts = win.skey,
+            .groupId = groupId,
+    };
     void* chIds = taosHashGet(pPullDataMap, &winRe, sizeof(SWinRes));
     if (isCloseWindow(&win, pSup)) {
       if (chIds && pPullDataMap) {
-        SArray* chAy = *(SArray**) chIds;
+        SArray* chAy = *(SArray**)chIds;
         int32_t size = taosArrayGetSize(chAy);
-        qInfo("======window %ld wait child size:%d", win.skey ,size);
+        qInfo("======window %ld wait child size:%d", win.skey, size);
         for (int32_t i = 0; i < size; i++) {
-          qInfo("======window %ld wait chid id:%d", win.skey ,*(int32_t*)taosArrayGet(chAy, i));
+          qInfo("======window %ld wait chid id:%d", win.skey, *(int32_t*)taosArrayGet(chAy, i));
         }
         continue;
       } else if (pPullDataMap) {
@@ -1411,7 +1421,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
   doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
 
   pOperator->status = OP_RES_TO_RETURN;
-
+  printDataBlock(pInfo->binfo.pRes, "single interval");
   return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
 }
 
@@ -1431,7 +1441,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) {
   SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)param;
   cleanupBasicInfo(&pInfo->binfo);
   cleanupAggSup(&pInfo->aggSup);
-  //it should be empty.
+  // it should be empty.
   taosHashCleanup(pInfo->pPullDataMap);
   taosArrayDestroy(pInfo->pPullWins);
   blockDataDestroy(pInfo->pPullDataRes);
@@ -1509,7 +1519,8 @@ void increaseTs(SqlFunctionCtx* pCtx) {
 
 SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
                                           SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
-                                          STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, bool isStream) {
+                                          STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode,
+                                          SExecTaskInfo* pTaskInfo, bool isStream) {
   SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
   SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {
@@ -1521,6 +1532,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo*
   pInfo->interval = *pInterval;
   pInfo->execModel = pTaskInfo->execModel;
   pInfo->twAggSup = *pTwAggSupp;
+  pInfo->ignoreCloseWindow = false;
 
   if (pPhyNode->window.pExprs != NULL) {
     int32_t    numOfScalar = 0;
@@ -2236,10 +2248,10 @@ bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) {
   return p1 == NULL;
 }
 
-int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols,
+int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols, int32_t startPos, TSKEY eKey,
-    int32_t startPos, TSKEY eKey, STimeWindow* pNextWin) {
+                        STimeWindow* pNextWin) {
-  int32_t forwardRows = getNumOfRowsInTimeWindow(pBlockInfo, tsCols, startPos,
+  int32_t forwardRows =
-      eKey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
+      getNumOfRowsInTimeWindow(pBlockInfo, tsCols, startPos, eKey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
   int32_t prevEndPos = forwardRows - 1 + startPos;
   return getNextQualifiedWindow(pInterval, pNextWin, pBlockInfo, tsCols, prevEndPos, TSDB_ORDER_ASC);
 }
@@ -2276,9 +2288,20 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc
   STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval,
                                             pInfo->interval.precision, NULL);
   while (1) {
-    if (IS_FINAL_OP(pInfo) && isCloseWindow(&nextWin, &pInfo->twAggSup) && pInfo->pChildren) {
+    bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);
+    if (pInfo->ignoreCloseWindow && isClosed) {
+      startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
+      if (startPos < 0) {
+        break;
+      }
+      continue;
+    }
+    if (IS_FINAL_OP(pInfo) && isClosed && pInfo->pChildren) {
       bool    ignore = true;
-      SWinRes winRes = {.ts = nextWin.skey, .groupId = tableGroupId,};
+      SWinRes winRes = {
+          .ts = nextWin.skey,
+          .groupId = tableGroupId,
+      };
       void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinRes));
       if (isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup) && !chIds) {
         SPullWindowInfo pull = {.window = nextWin, .groupId = tableGroupId};
@@ -2289,18 +2312,18 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc
         int32_t index = -1;
         SArray* chArray = NULL;
         if (chIds) {
-          chArray = *(void**) chIds;
+          chArray = *(void**)chIds;
           int32_t chId = getChildIndex(pSDataBlock);
           index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
         }
-        if (index != -1 && pSDataBlock->info.type == STREAM_PUSH_DATA) {
+        if (index != -1 && pSDataBlock->info.type == STREAM_PULL_DATA) {
           taosArrayRemove(chArray, index);
           if (taosArrayGetSize(chArray) == 0) {
             // pull data is over
             taosHashRemove(pInfo->pPullDataMap, &winRes, sizeof(SWinRes));
           }
         }
-        if ( index == -1 || pSDataBlock->info.type == STREAM_PUSH_DATA) {
+        if (index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA) {
           ignore = false;
         }
       }
@@ -2385,17 +2408,17 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB
   if (size - (*pIndex) == 0) {
     return;
   }
-  blockDataEnsureCapacity(pBlock, size - (*pIndex) );
+  blockDataEnsureCapacity(pBlock, size - (*pIndex));
   ASSERT(3 <= taosArrayGetSize(pBlock->pDataBlock));
   for (; (*pIndex) < size; (*pIndex)++) {
-    SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex) );
+    SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex));
-    SColumnInfoData* pStartTs = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 0);
+    SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, 0);
     colDataAppend(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false);
 
-    SColumnInfoData* pEndTs = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 1);
+    SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, 1);
     colDataAppend(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false);
 
-    SColumnInfoData* pGroupId = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 2);
+    SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, 2);
     colDataAppend(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false);
     pBlock->info.rows++;
   }
@@ -2406,7 +2429,7 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB
   blockDataUpdateTsWindow(pBlock, 0);
 }
 
-void processPushEmpty(SSDataBlock* pBlock, SHashObj* pMap) {
+void processPullOver(SSDataBlock* pBlock, SHashObj* pMap) {
   SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, 0);
   TSKEY*           tsData = (TSKEY*)pStartCol->pData;
   SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, 2);
@@ -2416,7 +2439,7 @@ void processPushEmpty(SSDataBlock* pBlock, SHashObj* pMap) {
     SWinRes winRes = {.ts = tsData[i], .groupId = groupIdData[i]};
     void*   chIds = taosHashGet(pMap, &winRes, sizeof(SWinRes));
     if (chIds) {
-      SArray* chArray = *(SArray**) chIds;
+      SArray* chArray = *(SArray**)chIds;
       int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
       if (index != -1) {
         taosArrayRemove(chArray, index);
@@ -2484,7 +2507,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
     printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval Final recv" : "interval  Semi recv");
     maxTs = TMAX(maxTs, pBlock->info.window.ekey);
 
-    if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PUSH_DATA || pBlock->info.type == STREAM_INVALID) {
+    if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA ||
+        pBlock->info.type == STREAM_INVALID) {
       pInfo->binfo.pRes->info.type = pBlock->info.type;
     } else if (pBlock->info.type == STREAM_CLEAR) {
       SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow));
@@ -2513,16 +2537,15 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
       continue;
     } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) {
       SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow));
-      doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, 0, pOperator->exprSupp.numOfExprs,
+      doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, 0, pOperator->exprSupp.numOfExprs, pBlock, pUpWins);
-          pBlock, pUpWins);
       removeResults(pUpWins, pUpdated);
       taosArrayDestroy(pUpWins);
       if (taosArrayGetSize(pUpdated) > 0) {
         break;
       }
       continue;
-    } else if (pBlock->info.type == STREAM_PUSH_EMPTY && IS_FINAL_OP(pInfo)) {
+    } else if (pBlock->info.type == STREAM_PULL_OVER && IS_FINAL_OP(pInfo)) {
-      processPushEmpty(pBlock, pInfo->pPullDataMap);
+      processPullOver(pBlock, pInfo->pPullDataMap);
       continue;
     }
 
@@ -2552,8 +2575,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
 
   pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
   if (IS_FINAL_OP(pInfo)) {
-    closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup,
+    closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pInfo->pPullDataMap,
-        &pInfo->interval, pInfo->pPullDataMap, pUpdated);
+                        pUpdated);
     closeChildIntervalWindow(pInfo->pChildren, pInfo->twAggSup.maxTs);
   }
 
@@ -2684,6 +2707,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
   _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
   pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK);
   pInfo->pPullDataRes = createPullDataBlock();
+  pInfo->ignoreCloseWindow = false;
 
   pOperator->operatorType = pPhyNode->type;
   pOperator->blocking = true;
@@ -2830,6 +2854,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
   pInfo->pChildren = NULL;
   pInfo->isFinal = false;
   pInfo->pPhyNode = pPhyNode;
+  pInfo->ignoreCloseWindow = false;
 
   pOperator->name = "StreamSessionWindowAggOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
@@ -3007,6 +3032,9 @@ static int32_t doOneWindowAggImpl(int32_t tsColId, SOptrBasicInfo* pBinfo, SStre
   updateTimeWindowInfo(pTimeWindowData, &pCurWin->win, false);
   doApplyFunctions(pTaskInfo, pSup->pCtx, &pCurWin->win, pTimeWindowData, startIndex, winRows, tsCols,
                    pSDataBlock->info.rows, numOutput, TSDB_ORDER_ASC);
+  SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, pCurWin->pos.pageId);
+  setBufPageDirty(bufPage, true);
+  releaseBufPage(pAggSup->pResultBuf, bufPage);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -3063,7 +3091,13 @@ void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex,
       pWinInfo->isOutput = false;
     }
     taosArrayRemove(pInfo->streamAggSup.pCurWins, i);
+    SFilePage* tmpPage = getBufPage(pInfo->streamAggSup.pResultBuf, pWinInfo->pos.pageId);
+    releaseBufPage(pInfo->streamAggSup.pResultBuf, tmpPage);
   }
+  SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pCurWin->pos.pageId);
+  ASSERT(num > 0);
+  setBufPageDirty(bufPage, true);
+  releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage);
 }
 
 static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pStUpdated,
@@ -3083,7 +3117,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData
   SResultRow* pResult = NULL;
   int32_t     winRows = 0;
 
-  if (pSDataBlock->pDataBlock != NULL) {
+  ASSERT(pSDataBlock->pDataBlock);
   SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
   startTsCols = (int64_t*)pStartTsCol->pData;
   SColumnInfoData* pEndTsCol = NULL;
@@ -3093,12 +3127,13 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData
     pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
   }
   endTsCols = (int64_t*)pEndTsCol->pData;
-  } else {
-    return;
-  }
 
   SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
   for (int32_t i = 0; i < pSDataBlock->info.rows;) {
+    if (pInfo->ignoreCloseWindow && isOverdue(endTsCols[i], &pInfo->twAggSup)) {
+      i++;
+      continue;
+    }
     int32_t            winIndex = 0;
     SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, startTsCols[i], endTsCols[i], groupId, gap, &winIndex);
     winRows =
@@ -3205,17 +3240,24 @@ static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWin
         index = 0;
       }
       for (int32_t k = index; k < chWinSize; k++) {
-        SResultWindowInfo* pcw = taosArrayGet(pChWins, k);
+        SResultWindowInfo* pChWin = taosArrayGet(pChWins, k);
-        if (pParentWin->win.skey <= pcw->win.skey && pcw->win.ekey <= pParentWin->win.ekey) {
+        if (pParentWin->win.skey <= pChWin->win.skey && pChWin->win.ekey <= pParentWin->win.ekey) {
           SResultRow* pChResult = NULL;
-          setWindowOutputBuf(pcw, &pChResult, pChild->exprSupp.pCtx, groupId, numOfOutput,
+          setWindowOutputBuf(pChWin, &pChResult, pChild->exprSupp.pCtx, groupId, numOfOutput,
                              pChild->exprSupp.rowEntryInfoOffset, &pChInfo->streamAggSup, pTaskInfo);
           compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo);
+          SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pChWin->pos.pageId);
+          setBufPageDirty(bufPage, true);
+          releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage);
           continue;
         }
         break;
       }
     }
+    SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pParentWin->pos.pageId);
+    ASSERT(size > 0);
+    setBufPageDirty(bufPage, true);
+    releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage);
   }
 }
 
@@ -3234,7 +3276,7 @@ int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArra
     for (int32_t i = 0; i < size; i++) {
       void*              pWin = taosArrayGet(pWins, i);
       SResultWindowInfo* pSeWin = fn(pWin);
-      if (pSeWin->win.ekey < pTwSup->maxTs - pTwSup->waterMark) {
+      if (isCloseWindow(&pSeWin->win, pTwSup)) {
         if (!pSeWin->isClosed) {
           pSeWin->isClosed = true;
           if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
@@ -3277,14 +3319,14 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
   } else if (pOperator->status == OP_RES_TO_RETURN) {
     doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
     if (pInfo->pDelRes->info.rows > 0) {
-      printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo)? "Final  Session" : "Single Session");
+      printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "Final  Session" : "Single Session");
       return pInfo->pDelRes;
     }
     doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
     if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
       doSetOperatorCompleted(pOperator);
     }
-    printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo)? "Final  Session" : "Single Session");
+    printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "Final  Session" : "Single Session");
     return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
   }
 
@@ -3352,11 +3394,11 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
   blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
   doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
   if (pInfo->pDelRes->info.rows > 0) {
-    printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo)? "Final  Session" : "Single Session");
+    printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "Final  Session" : "Single Session");
     return pInfo->pDelRes;
   }
   doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
-  printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo)? "Final  Session" : "Single Session");
+  printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "Final  Session" : "Single Session");
   return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
 }
 
@@ -3745,6 +3787,10 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
   SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
   SColumnInfoData*     pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
   for (int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) {
+    if (pInfo->ignoreCloseWindow && isOverdue(tsCols[i], &pInfo->twAggSup)) {
+      i++;
+      continue;
+    }
     char*             pKeyData = colDataGetData(pKeyColInfo, i);
     int32_t           winIndex = 0;
     bool              allEqual = true;
@@ -3895,6 +3941,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
   pInfo->pDelRes->info.type = STREAM_DELETE;
   blockDataEnsureCapacity(pInfo->pDelRes, 64);
   pInfo->pChildren = NULL;
+  pInfo->ignoreCloseWindow = false;
 
   pOperator->name = "StreamStateAggOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
@@ -4169,7 +4216,8 @@ void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) {
   destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo, numOfOutput);
 }
 
-static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, STimeWindow* win, SSDataBlock* pResultBlock) {
+static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, STimeWindow* win,
+                                    SSDataBlock* pResultBlock) {
   SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
   SIntervalAggOperatorInfo*      iaInfo = &miaInfo->intervalAggOperatorInfo;
   SExecTaskInfo*                 pTaskInfo = pOperatorInfo->pTaskInfo;
@@ -4202,7 +4250,7 @@ static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t t
   SListNode* listNode = NULL;
   while ((listNode = tdListNext(&iter)) != NULL) {
     SGroupTimeWindow* prevGrpWin = (SGroupTimeWindow*)listNode->data;
-    if (prevGrpWin->groupId != tableGroupId ) {
+    if (prevGrpWin->groupId != tableGroupId) {
       continue;
     }
     STimeWindow* prevWin = &prevGrpWin->window;

source/libs/function/inc/functionMgtInt.h

@@ -46,6 +46,7 @@ extern "C" {
 #define FUNC_MGT_FORBID_STREAM_FUNC     FUNC_MGT_FUNC_CLASSIFICATION_MASK(17)
 #define FUNC_MGT_FORBID_WINDOW_FUNC     FUNC_MGT_FUNC_CLASSIFICATION_MASK(18)
 #define FUNC_MGT_FORBID_GROUP_BY_FUNC   FUNC_MGT_FUNC_CLASSIFICATION_MASK(19)
+#define FUNC_MGT_SYSTEM_INFO_FUNC       FUNC_MGT_FUNC_CLASSIFICATION_MASK(20)
 
 #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0)
 

source/libs/function/src/builtins.c

@@ -39,6 +39,20 @@ static int32_t invaildFuncParaValueErrMsg(char* pErrBuf, int32_t len, const char
   return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_PARA_VALUE, "Invalid parameter value : %s", pFuncName);
 }
 
+void static addDbPrecisonParam(SNodeList** pList, uint8_t precision) {
+  SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
+  pVal->literal = NULL;
+  pVal->isDuration = false;
+  pVal->translate = true;
+  pVal->node.resType.type = TSDB_DATA_TYPE_TINYINT;
+  pVal->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TINYINT].bytes;
+  pVal->node.resType.precision = precision;
+  pVal->datum.i = (int64_t)precision;
+  pVal->typeData = (int64_t)precision;
+
+  nodesListMakeAppend(pList, (SNode*)pVal);
+}
+
 // There is only one parameter of numeric type, and the return type is parameter type
 static int32_t translateInOutNum(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   if (1 != LIST_LENGTH(pFunc->pParameterList)) {
@@ -220,8 +234,20 @@ static int32_t translateWduration(SFunctionNode* pFunc, char* pErrBuf, int32_t l
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t translateNowToday(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  // pseudo column do not need to check parameters
+
+  //add database precision as param
+  uint8_t dbPrec = pFunc->node.resType.precision;
+  addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
+
+  pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_TIMESTAMP};
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t translateTimePseudoColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   // pseudo column do not need to check parameters
+
   pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_TIMESTAMP};
   return TSDB_CODE_SUCCESS;
 }
@@ -1446,6 +1472,10 @@ static int32_t translateToUnixtimestamp(SFunctionNode* pFunc, char* pErrBuf, int
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
+  //add database precision as param
+  uint8_t dbPrec = pFunc->node.resType.precision;
+  addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
+
   pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
   return TSDB_CODE_SUCCESS;
 }
@@ -1462,6 +1492,10 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
+  //add database precision as param
+  uint8_t dbPrec = pFunc->node.resType.precision;
+  addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
+
   pFunc->node.resType =
       (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP};
   return TSDB_CODE_SUCCESS;
@@ -1486,6 +1520,10 @@ static int32_t translateTimeDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t le
     }
   }
 
+  //add database precision as param
+  uint8_t dbPrec = pFunc->node.resType.precision;
+  addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
+
   pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
   return TSDB_CODE_SUCCESS;
 }
@@ -1539,6 +1577,36 @@ static int32_t translateGroupKey(SFunctionNode* pFunc, char* pErrBuf, int32_t le
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t translateDatabaseFunc(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  pFunc->node.resType = (SDataType){.bytes = TSDB_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR};
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t translateClientVersionFunc(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  pFunc->node.resType = (SDataType){.bytes = TSDB_VERSION_LEN, .type = TSDB_DATA_TYPE_VARCHAR};
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t translateServerVersionFunc(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  pFunc->node.resType = (SDataType){.bytes = TSDB_VERSION_LEN, .type = TSDB_DATA_TYPE_VARCHAR};
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t translateServerStatusFunc(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes, .type = TSDB_DATA_TYPE_INT};
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t translateCurrentUserFunc(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  pFunc->node.resType = (SDataType){.bytes = TSDB_USER_LEN, .type = TSDB_DATA_TYPE_VARCHAR};
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t translateUserFunc(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  pFunc->node.resType = (SDataType){.bytes = TSDB_USER_LEN, .type = TSDB_DATA_TYPE_VARCHAR};
+  return TSDB_CODE_SUCCESS;
+}
+
 // clang-format off
 const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
@@ -1640,7 +1708,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "leastsquares",
     .type = FUNCTION_TYPE_LEASTSQUARES,
-    .classification = FUNC_MGT_AGG_FUNC,
+    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_FORBID_STREAM_FUNC,
     .translateFunc = translateLeastSQR,
     .getEnvFunc   = getLeastSQRFuncEnv,
     .initFunc     = leastSQRFunctionSetup,
@@ -1882,6 +1950,19 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .name = "last_row",
     .type = FUNCTION_TYPE_LAST_ROW,
     .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
+    .translateFunc = translateFirstLast,
+    .getEnvFunc   = getFirstLastFuncEnv,
+    .initFunc     = functionSetup,
+    .processFunc  = lastFunction,
+    .finalizeFunc = firstLastFinalize,
+    .pPartialFunc = "_last_partial",
+    .pMergeFunc   = "_last_merge",
+    .combineFunc  = lastCombine,
+  },
+  {
+    .name = "_cache_last_row",
+    .type = FUNCTION_TYPE_CACHE_LAST_ROW,
+    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
     .translateFunc = translateLastRow,
     .getEnvFunc   = getMinmaxFuncEnv,
     .initFunc     = minmaxFunctionSetup,
@@ -1904,7 +1985,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "_first_partial",
     .type = FUNCTION_TYPE_FIRST_PARTIAL,
-    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
+    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
     .translateFunc = translateFirstLastPartial,
     .getEnvFunc   = getFirstLastFuncEnv,
     .initFunc     = functionSetup,
@@ -1915,7 +1996,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "_first_merge",
     .type = FUNCTION_TYPE_FIRST_MERGE,
-    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
+    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
     .translateFunc = translateFirstLastMerge,
     .getEnvFunc   = getFirstLastFuncEnv,
     .initFunc     = functionSetup,
@@ -1939,7 +2020,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "_last_partial",
     .type = FUNCTION_TYPE_LAST_PARTIAL,
-    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
+    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
     .translateFunc = translateFirstLastPartial,
     .getEnvFunc   = getFirstLastFuncEnv,
     .initFunc     = functionSetup,
@@ -1950,7 +2031,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "_last_merge",
     .type = FUNCTION_TYPE_LAST_MERGE,
-    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
+    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
     .translateFunc = translateFirstLastMerge,
     .getEnvFunc   = getFirstLastFuncEnv,
     .initFunc     = functionSetup,
@@ -2410,7 +2491,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .name = "now",
     .type = FUNCTION_TYPE_NOW,
     .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_DATETIME_FUNC,
-    .translateFunc = translateTimePseudoColumn,
+    .translateFunc = translateNowToday,
     .getEnvFunc   = NULL,
     .initFunc     = NULL,
     .sprocessFunc = nowFunction,
@@ -2420,7 +2501,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .name = "today",
     .type = FUNCTION_TYPE_TODAY,
     .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_DATETIME_FUNC,
-    .translateFunc = translateTimePseudoColumn,
+    .translateFunc = translateNowToday,
     .getEnvFunc   = NULL,
     .initFunc     = NULL,
     .sprocessFunc = todayFunction,
@@ -2546,6 +2627,42 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .pPartialFunc = "_group_key",
     .pMergeFunc   = "_group_key"
   },
+  {
+    .name = "database",
+    .type = FUNCTION_TYPE_DATABASE,
+    .classification = FUNC_MGT_SYSTEM_INFO_FUNC | FUNC_MGT_SCALAR_FUNC,
+    .translateFunc = translateDatabaseFunc,
+  },
+  {
+    .name = "client_version",
+    .type = FUNCTION_TYPE_CLIENT_VERSION,
+    .classification = FUNC_MGT_SYSTEM_INFO_FUNC | FUNC_MGT_SCALAR_FUNC,
+    .translateFunc = translateClientVersionFunc,
+  },
+  {
+    .name = "server_version",
+    .type = FUNCTION_TYPE_SERVER_VERSION,
+    .classification = FUNC_MGT_SYSTEM_INFO_FUNC | FUNC_MGT_SCALAR_FUNC,
+    .translateFunc = translateServerVersionFunc,
+  },
+  {
+    .name = "server_status",
+    .type = FUNCTION_TYPE_SERVER_STATUS,
+    .classification = FUNC_MGT_SYSTEM_INFO_FUNC | FUNC_MGT_SCALAR_FUNC,
+    .translateFunc = translateServerStatusFunc,
+  },
+  {
+    .name = "current_user",
+    .type = FUNCTION_TYPE_CURRENT_USER,
+    .classification = FUNC_MGT_SYSTEM_INFO_FUNC | FUNC_MGT_SCALAR_FUNC,
+    .translateFunc = translateCurrentUserFunc,
+  },
+  {
+    .name = "user",
+    .type = FUNCTION_TYPE_USER,
+    .classification = FUNC_MGT_SYSTEM_INFO_FUNC | FUNC_MGT_SCALAR_FUNC,
+    .translateFunc = translateUserFunc,
+  },
 };
 // clang-format on
 

source/libs/function/src/builtinsimpl.c

@@ -2409,11 +2409,11 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx)
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t getFirstLastInfoSize(int32_t resBytes) { return sizeof(SFirstLastRes) + resBytes + sizeof(int64_t); }
+int32_t getFirstLastInfoSize(int32_t resBytes) { return sizeof(SFirstLastRes) + resBytes + sizeof(int64_t) + sizeof(STuplePos); }
 
 bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
   SColumnNode* pNode = (SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0);
-  pEnv->calcMemSize = sizeof(SFirstLastRes) + pNode->node.resType.bytes + sizeof(int64_t);
+  pEnv->calcMemSize = getFirstLastInfoSize(pNode->node.resType.bytes);
   return true;
 }
 
@@ -2491,9 +2491,17 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) {
         }
         memcpy(pInfo->buf, data, bytes);
         *(TSKEY*)(pInfo->buf + bytes) = cts;
+        //handle selectivity
+        if (pCtx->subsidiaries.num > 0) {
+          STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY));
+          if (!pInfo->hasResult) {
+            saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
+          } else {
+            copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
+          }
+        }
         pInfo->hasResult = true;
-        //        DO_UPDATE_TAG_COLUMNS(pCtx, ts);
+        //DO_UPDATE_TAG_COLUMNS(pCtx, ts);
-
         pResInfo->numOfRes = 1;
         break;
       }
@@ -2525,8 +2533,17 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) {
         }
         memcpy(pInfo->buf, data, bytes);
         *(TSKEY*)(pInfo->buf + bytes) = cts;
+        //handle selectivity
+        if (pCtx->subsidiaries.num > 0) {
+          STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY));
+          if (!pInfo->hasResult) {
+            saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
+          } else {
+            copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
+          }
+        }
         pInfo->hasResult = true;
-        //        DO_UPDATE_TAG_COLUMNS(pCtx, ts);
+        //DO_UPDATE_TAG_COLUMNS(pCtx, ts);
         pResInfo->numOfRes = 1;
         break;
       }
@@ -2580,8 +2597,17 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
         }
         memcpy(pInfo->buf, data, bytes);
         *(TSKEY*)(pInfo->buf + bytes) = cts;
-        //        DO_UPDATE_TAG_COLUMNS(pCtx, ts);
+        //handle selectivity
+        if (pCtx->subsidiaries.num > 0) {
+          STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY));
+          if (!pInfo->hasResult) {
+            saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
+          } else {
+            copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
+          }
+        }
         pInfo->hasResult = true;
+        //DO_UPDATE_TAG_COLUMNS(pCtx, ts);
         pResInfo->numOfRes = 1;
       }
       break;
@@ -2603,9 +2629,18 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
         }
         memcpy(pInfo->buf, data, bytes);
         *(TSKEY*)(pInfo->buf + bytes) = cts;
+        //handle selectivity
+        if (pCtx->subsidiaries.num > 0) {
+          STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY));
+          if (!pInfo->hasResult) {
+            saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
+          } else {
+            copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
+          }
+        }
         pInfo->hasResult = true;
         pResInfo->numOfRes = 1;
-        //        DO_UPDATE_TAG_COLUMNS(pCtx, ts);
+        //DO_UPDATE_TAG_COLUMNS(pCtx, ts);
       }
       break;
     }
@@ -2615,7 +2650,10 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
   return TSDB_CODE_SUCCESS;
 }
 
-static void firstLastTransferInfo(SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst) {
+static void firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst) {
+  SInputColumnInfoData* pColInfo = &pCtx->input;
+  int32_t start = pColInfo->startRowIndex;
+
   pOutput->bytes = pInput->bytes;
   TSKEY* tsIn = (TSKEY*)(pInput->buf + pInput->bytes);
   TSKEY* tsOut = (TSKEY*)(pOutput->buf + pInput->bytes);
@@ -2632,7 +2670,17 @@ static void firstLastTransferInfo(SFirstLastRes* pInput, SFirstLastRes* pOutput,
   }
   *tsOut = *tsIn;
   memcpy(pOutput->buf, pInput->buf, pOutput->bytes);
+  //handle selectivity
+  STuplePos* pTuplePos = (STuplePos*)(pOutput->buf + pOutput->bytes + sizeof(TSKEY));
+  if (pCtx->subsidiaries.num > 0) {
+    if (!pOutput->hasResult) {
+      saveTupleData(pCtx, start, pCtx->pSrcBlock, pTuplePos);
+    } else {
+      copyTupleData(pCtx, start, pCtx->pSrcBlock, pTuplePos);
+    }
+  }
   pOutput->hasResult = true;
+
   return;
 }
 
@@ -2647,7 +2695,7 @@ static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx* pCtx, bool isFirstQuer
   char*          data = colDataGetData(pCol, start);
   SFirstLastRes* pInputInfo = (SFirstLastRes*)varDataVal(data);
 
-  firstLastTransferInfo(pInputInfo, pInfo, isFirstQuery);
+  firstLastTransferInfo(pCtx, pInputInfo, pInfo, isFirstQuery);
 
   int32_t numOfElems = pInputInfo->hasResult ? 1 : 0;
 
@@ -2669,6 +2717,9 @@ int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
 
   SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pResInfo);
   colDataAppend(pCol, pBlock->info.rows, pRes->buf, pResInfo->isNullRes);
+  //handle selectivity
+  STuplePos* pTuplePos = (STuplePos*)(pRes->buf + pRes->bytes + sizeof(TSKEY));
+  setSelectivityValue(pCtx, pBlock, pTuplePos, pBlock->info.rows);
 
   return pResInfo->numOfRes;
 }
@@ -2687,6 +2738,9 @@ int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
 
   colDataAppend(pCol, pBlock->info.rows, res, false);
+  //handle selectivity
+  STuplePos* pTuplePos = (STuplePos*)(pRes->buf + pRes->bytes + sizeof(TSKEY));
+  setSelectivityValue(pCtx, pBlock, pTuplePos, pBlock->info.rows);
 
   taosMemoryFree(res);
   return 1;
@@ -3043,7 +3097,9 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData
     pItem->uid = uid;
 
     // save the data of this tuple
+    if (pCtx->subsidiaries.num > 0) {
       saveTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos);
+    }
 
     // allocate the buffer and keep the data of this row into the new allocated buffer
     pEntryInfo->numOfRes++;
@@ -3062,7 +3118,10 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData
       pItem->uid = uid;
 
       // save the data of this tuple by over writing the old data
+      if (pCtx->subsidiaries.num > 0) {
         copyTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos);
+      }
+
       taosheapadjust((void*)pItems, sizeof(STopBotResItem), 0, pEntryInfo->numOfRes - 1, (const void*)&type,
                      topBotResComparFn, NULL, !isTopQuery);
     }
@@ -4224,7 +4283,7 @@ int32_t stateDurationFunction(SqlFunctionCtx* pCtx) {
   SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput;
 
   // TODO: process timeUnit for different db precisions
-  int32_t timeUnit = 1000;
+  int32_t timeUnit = 1;
   if (pCtx->numOfParams == 5) {  // TODO: param number incorrect
     timeUnit = pCtx->param[3].param.i;
   }
@@ -5466,7 +5525,7 @@ int32_t groupKeyFunction(SqlFunctionCtx* pCtx) {
 
   int32_t startIndex = pInput->startRowIndex;
 
-  //escape rest of data blocks to avoid first entry be overwritten.
+  //escape rest of data blocks to avoid first entry to be overwritten.
   if (pInfo->hasResult) {
     goto _group_key_over;
   }

source/libs/function/src/functionMgt.c

@@ -179,6 +179,8 @@ bool fmIsForbidWindowFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId
 
 bool fmIsForbidGroupByFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_FORBID_GROUP_BY_FUNC); }
 
+bool fmIsSystemInfoFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SYSTEM_INFO_FUNC); }
+
 bool fmIsInterpFunc(int32_t funcId) {
   if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
     return false;

source/libs/parser/inc/sql.y

@@ -685,6 +685,12 @@ literal_func(A) ::= NOW(B).
 noarg_func(A) ::= NOW(B).                                                         { A = B; }
 noarg_func(A) ::= TODAY(B).                                                       { A = B; }
 noarg_func(A) ::= TIMEZONE(B).                                                    { A = B; }
+noarg_func(A) ::= DATABASE(B).                                                    { A = B; }
+noarg_func(A) ::= CLIENT_VERSION(B).                                              { A = B; }
+noarg_func(A) ::= SERVER_VERSION(B).                                              { A = B; }
+noarg_func(A) ::= SERVER_STATUS(B).                                               { A = B; }
+noarg_func(A) ::= CURRENT_USER(B).                                                { A = B; }
+noarg_func(A) ::= USER(B).                                                        { A = B; }
 
 %type star_func                                                                   { SToken }
 %destructor star_func                                                             { }

source/libs/parser/src/parInsert.c

@@ -67,6 +67,13 @@ typedef struct SInsertParseContext {
   SParseMetaCache*   pMetaCache;
 } SInsertParseContext;
 
+typedef struct SInsertParseSyntaxCxt {
+  SParseContext*   pComCxt;
+  char*            pSql;
+  SMsgBuf          msg;
+  SParseMetaCache* pMetaCache;
+} SInsertParseSyntaxCxt;
+
 typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param);
 
 static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE;
@@ -1098,11 +1105,24 @@ static int32_t storeTableMeta(SInsertParseContext* pCxt, SHashObj* pHash, SName*
   return taosHashPut(pHash, pName, len, &pBackup, POINTER_BYTES);
 }
 
+static int32_t skipUsingClause(SInsertParseSyntaxCxt* pCxt);
+
+// pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)
+static int32_t ignoreAutoCreateTableClause(SInsertParseContext* pCxt) {
+  SToken sToken;
+  NEXT_TOKEN(pCxt->pSql, sToken);
+  SInsertParseSyntaxCxt cxt = {.pComCxt = pCxt->pComCxt, .pSql = pCxt->pSql, .msg = pCxt->msg, .pMetaCache = NULL};
+  int32_t               code = skipUsingClause(&cxt);
+  pCxt->pSql = cxt.pSql;
+  return code;
+}
+
 // pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)
 static int32_t parseUsingClause(SInsertParseContext* pCxt, SName* name, char* tbFName) {
   int32_t      len = strlen(tbFName);
   STableMeta** pMeta = taosHashGet(pCxt->pSubTableHashObj, tbFName, len);
   if (NULL != pMeta) {
+    CHECK_CODE(ignoreAutoCreateTableClause(pCxt));
     return cloneTableMeta(*pMeta, &pCxt->pTableMeta);
   }
 
@@ -1522,13 +1542,6 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery, SParseMetaCache
   return code;
 }
 
-typedef struct SInsertParseSyntaxCxt {
-  SParseContext*   pComCxt;
-  char*            pSql;
-  SMsgBuf          msg;
-  SParseMetaCache* pMetaCache;
-} SInsertParseSyntaxCxt;
-
 static int32_t skipParentheses(SInsertParseSyntaxCxt* pCxt) {
   SToken  sToken;
   int32_t expectRightParenthesis = 1;

source/libs/parser/src/parTokenizer.c

@@ -54,6 +54,7 @@ static SKeyword keywordTable[] = {
     {"CACHE",          TK_CACHE},
     {"CACHELAST",      TK_CACHELAST},
     {"CAST",           TK_CAST},
+    {"CLIENT_VERSION", TK_CLIENT_VERSION},
     {"CLUSTER",        TK_CLUSTER},
     {"COLUMN",         TK_COLUMN},
     {"COMMENT",        TK_COMMENT},
@@ -64,9 +65,10 @@ static SKeyword keywordTable[] = {
     {"CONNECTIONS",    TK_CONNECTIONS},
     {"CONSUMER",       TK_CONSUMER},
     {"CONSUMERS",      TK_CONSUMERS},
+    {"CONTAINS",       TK_CONTAINS},
     {"COUNT",          TK_COUNT},
     {"CREATE",         TK_CREATE},
-    {"CONTAINS",      TK_CONTAINS}, 
+    {"CURRENT_USER",   TK_CURRENT_USER},
     {"DATABASE",       TK_DATABASE},
     {"DATABASES",      TK_DATABASES},
     {"DBS",            TK_DBS},
@@ -169,6 +171,8 @@ static SKeyword keywordTable[] = {
     {"SCHEMALESS",     TK_SCHEMALESS},
     {"SCORES",         TK_SCORES},
     {"SELECT",         TK_SELECT},
+    {"SERVER_STATUS",  TK_SERVER_STATUS},
+    {"SERVER_VERSION", TK_SERVER_VERSION},
     {"SESSION",        TK_SESSION},
     {"SET",            TK_SET},
     {"SHOW",           TK_SHOW},

source/libs/parser/src/parTranslater.c

@@ -35,8 +35,7 @@ typedef struct STranslateContext {
   SArray*          pNsLevel;  // element is SArray*, the element of this subarray is STableNode*
   int32_t          currLevel;
   ESqlClause       currClause;
-  SSelectStmt*     pCurrSelectStmt;
+  SNode*           pCurrStmt;
-  SSetOperator*    pCurrSetOperator;
   SCmdMsgInfo*     pCmdMsg;
   SHashObj*        pDbs;
   SHashObj*        pTables;
@@ -397,6 +396,34 @@ static void destroyTranslateContext(STranslateContext* pCxt) {
   taosHashCleanup(pCxt->pTables);
 }
 
+static bool isSelectStmt(SNode* pCurrStmt) {
+  return NULL != pCurrStmt && QUERY_NODE_SELECT_STMT == nodeType(pCurrStmt);
+}
+
+static bool isSetOperator(SNode* pCurrStmt) {
+  return NULL != pCurrStmt && QUERY_NODE_SET_OPERATOR == nodeType(pCurrStmt);
+}
+
+static SNodeList* getProjectListFromCurrStmt(SNode* pCurrStmt) {
+  if (isSelectStmt(pCurrStmt)) {
+    return ((SSelectStmt*)pCurrStmt)->pProjectionList;
+  }
+  if (isSetOperator(pCurrStmt)) {
+    return ((SSetOperator*)pCurrStmt)->pProjectionList;
+  }
+  return NULL;
+}
+
+static uint8_t getPrecisionFromCurrStmt(SNode* pCurrStmt, uint8_t defaultVal) {
+  if (isSelectStmt(pCurrStmt)) {
+    return ((SSelectStmt*)pCurrStmt)->precision;
+  }
+  if (isSetOperator(pCurrStmt)) {
+    return ((SSetOperator*)pCurrStmt)->precision;
+  }
+  return defaultVal;
+}
+
 static bool isAliasColumn(const SNode* pNode) {
   return (QUERY_NODE_COLUMN == nodeType(pNode) && ('\0' == ((SColumnNode*)pNode)->tableAlias[0]));
 }
@@ -426,7 +453,8 @@ static bool isVectorFunc(const SNode* pNode) {
 }
 
 static bool isDistinctOrderBy(STranslateContext* pCxt) {
-  return (SQL_CLAUSE_ORDER_BY == pCxt->currClause && pCxt->pCurrSelectStmt->isDistinct);
+  return (SQL_CLAUSE_ORDER_BY == pCxt->currClause && isSelectStmt(pCxt->pCurrStmt) &&
+          ((SSelectStmt*)pCxt->pCurrStmt)->isDistinct);
 }
 
 static bool belongTable(const char* currentDb, const SColumnNode* pCol, const STableNode* pTable) {
@@ -646,7 +674,7 @@ static EDealRes translateColumnWithoutPrefix(STranslateContext* pCxt, SColumnNod
   }
   if (!found) {
     if (isInternalPk) {
-      if (NULL != pCxt->pCurrSelectStmt && NULL != pCxt->pCurrSelectStmt->pWindow) {
+      if (isSelectStmt(pCxt->pCurrStmt) && NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pWindow) {
         return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_NOT_ALLOWED_WIN_QUERY);
       }
       return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_INTERNAL_PK);
@@ -657,18 +685,8 @@ static EDealRes translateColumnWithoutPrefix(STranslateContext* pCxt, SColumnNod
   return DEAL_RES_CONTINUE;
 }
 
-static SNodeList* getProjectListFromCxt(STranslateContext* pCxt) {
-  if (NULL != pCxt->pCurrSelectStmt) {
-    return pCxt->pCurrSelectStmt->pProjectionList;
-  } else if (NULL != pCxt->pCurrSetOperator) {
-    return pCxt->pCurrSetOperator->pProjectionList;
-  } else {
-    return NULL;
-  }
-}
-
 static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** pCol, bool* pFound) {
-  SNodeList* pProjectionList = getProjectListFromCxt(pCxt);
+  SNodeList* pProjectionList = getProjectListFromCurrStmt(pCxt->pCurrStmt);
   SNode*     pNode;
   FOREACH(pNode, pProjectionList) {
     SExprNode* pExpr = (SExprNode*)pNode;
@@ -690,7 +708,7 @@ static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** p
 }
 
 static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) {
-  if (NULL != pCxt->pCurrSelectStmt && NULL == pCxt->pCurrSelectStmt->pFromTable) {
+  if (isSelectStmt(pCxt->pCurrStmt) && NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) {
     return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_COLUMN, (*pCol)->colName);
   }
 
@@ -708,7 +726,7 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) {
       res = translateColumnUseAlias(pCxt, pCol, &found);
     }
     if (DEAL_RES_ERROR != res && !found) {
-      if (NULL != pCxt->pCurrSetOperator) {
+      if (isSetOperator(pCxt->pCurrStmt)) {
         res = generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_COLUMN, (*pCol)->colName);
       } else {
         res = translateColumnWithoutPrefix(pCxt, pCol);
@@ -760,9 +778,9 @@ static int32_t parseBoolFromValueNode(STranslateContext* pCxt, SValueNode* pVal)
 }
 
 static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SDataType targetDt) {
-  uint8_t precision = (NULL != pCxt->pCurrSelectStmt ? pCxt->pCurrSelectStmt->precision : targetDt.precision);
+  uint8_t precision = getPrecisionFromCurrStmt(pCxt->pCurrStmt, targetDt.precision);
   pVal->node.resType.precision = precision;
-  if (pVal->placeholderNo > 0) {
+  if (pVal->placeholderNo > 0 || pVal->isNull) {
     return DEAL_RES_CONTINUE;
   }
   if (pVal->isDuration) {
@@ -1102,6 +1120,8 @@ static bool hasInvalidFuncNesting(SNodeList* pParameterList) {
 }
 
 static int32_t getFuncInfo(STranslateContext* pCxt, SFunctionNode* pFunc) {
+  // the time precision of the function execution environment
+  pFunc->node.resType.precision = getPrecisionFromCurrStmt(pCxt->pCurrStmt, TSDB_TIME_PRECISION_MILLI);
   int32_t code = fmGetFuncInfo(pFunc, pCxt->msgBuf.buf, pCxt->msgBuf.len);
   if (TSDB_CODE_FUNC_NOT_BUILTIN_FUNTION == code) {
     code = getUdfInfo(pCxt, pFunc);
@@ -1119,7 +1139,7 @@ static int32_t translateAggFunc(STranslateContext* pCxt, SFunctionNode* pFunc) {
   if (hasInvalidFuncNesting(pFunc->pParameterList)) {
     return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AGG_FUNC_NESTING);
   }
-  if (NULL != pCxt->pCurrSelectStmt && pCxt->pCurrSelectStmt->hasIndefiniteRowsFunc) {
+  if (isSelectStmt(pCxt->pCurrStmt) && ((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc) {
     return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC);
   }
 
@@ -1134,7 +1154,8 @@ static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SFunctionN
     return TSDB_CODE_SUCCESS;
   }
   if (0 == LIST_LENGTH(pFunc->pParameterList)) {
-    if (QUERY_NODE_REAL_TABLE != nodeType(pCxt->pCurrSelectStmt->pFromTable)) {
+    if (!isSelectStmt(pCxt->pCurrStmt) ||
+        QUERY_NODE_REAL_TABLE != nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) {
       return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME);
     }
   } else {
@@ -1152,8 +1173,8 @@ static int32_t translateIndefiniteRowsFunc(STranslateContext* pCxt, SFunctionNod
   if (!fmIsIndefiniteRowsFunc(pFunc->funcId)) {
     return TSDB_CODE_SUCCESS;
   }
-  if (SQL_CLAUSE_SELECT != pCxt->currClause || pCxt->pCurrSelectStmt->hasIndefiniteRowsFunc ||
+  if (!isSelectStmt(pCxt->pCurrStmt) || SQL_CLAUSE_SELECT != pCxt->currClause ||
-      pCxt->pCurrSelectStmt->hasAggFuncs) {
+      ((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc || ((SSelectStmt*)pCxt->pCurrStmt)->hasAggFuncs) {
     return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC);
   }
   if (hasInvalidFuncNesting(pFunc->pParameterList)) {
@@ -1162,13 +1183,20 @@ static int32_t translateIndefiniteRowsFunc(STranslateContext* pCxt, SFunctionNod
   return TSDB_CODE_SUCCESS;
 }
 
+static bool hasFillClause(SNode* pCurrStmt) {
+  if (!isSelectStmt(pCurrStmt)) {
+    return false;
+  }
+  SSelectStmt* pSelect = (SSelectStmt*)pCurrStmt;
+  return NULL != pSelect->pWindow && QUERY_NODE_INTERVAL_WINDOW == nodeType(pSelect->pWindow) &&
+         NULL != ((SIntervalWindowNode*)pSelect->pWindow)->pFill;
+}
+
 static int32_t translateForbidFillFunc(STranslateContext* pCxt, SFunctionNode* pFunc) {
   if (!fmIsForbidFillFunc(pFunc->funcId)) {
     return TSDB_CODE_SUCCESS;
   }
-  if (NULL != pCxt->pCurrSelectStmt->pWindow &&
+  if (hasFillClause(pCxt->pCurrStmt)) {
-      QUERY_NODE_INTERVAL_WINDOW == nodeType(pCxt->pCurrSelectStmt->pWindow) &&
-      NULL != ((SIntervalWindowNode*)pCxt->pCurrSelectStmt->pWindow)->pFill) {
     return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_FILL_NOT_ALLOWED_FUNC, pFunc->functionName);
   }
   return TSDB_CODE_SUCCESS;
@@ -1178,7 +1206,7 @@ static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SFunctio
   if (!fmIsWindowPseudoColumnFunc(pFunc->funcId)) {
     return TSDB_CODE_SUCCESS;
   }
-  if (NULL == pCxt->pCurrSelectStmt->pWindow) {
+  if (!isSelectStmt(pCxt->pCurrStmt) || NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pWindow) {
     return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_WINDOW_PC);
   }
   return TSDB_CODE_SUCCESS;
@@ -1188,7 +1216,7 @@ static int32_t translateForbidWindowFunc(STranslateContext* pCxt, SFunctionNode*
   if (!fmIsForbidWindowFunc(pFunc->funcId)) {
     return TSDB_CODE_SUCCESS;
   }
-  if (NULL != pCxt->pCurrSelectStmt->pWindow) {
+  if (isSelectStmt(pCxt->pCurrStmt) && NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pWindow) {
     return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC, pFunc->functionName);
   }
   return TSDB_CODE_SUCCESS;
@@ -1208,14 +1236,15 @@ static int32_t translateForbidGroupByFunc(STranslateContext* pCxt, SFunctionNode
   if (!fmIsForbidGroupByFunc(pFunc->funcId)) {
     return TSDB_CODE_SUCCESS;
   }
-  if (NULL != pCxt->pCurrSelectStmt->pGroupByList) {
+  if (isSelectStmt(pCxt->pCurrStmt) && NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pGroupByList) {
     return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GROUP_BY_NOT_ALLOWED_FUNC, pFunc->functionName);
   }
   return TSDB_CODE_SUCCESS;
 }
 
-static void setFuncClassification(SSelectStmt* pSelect, SFunctionNode* pFunc) {
+static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) {
-  if (NULL != pSelect) {
+  if (NULL != pCurrStmt && QUERY_NODE_SELECT_STMT == nodeType(pCurrStmt)) {
+    SSelectStmt* pSelect = (SSelectStmt*)pCurrStmt;
     pSelect->hasAggFuncs = pSelect->hasAggFuncs ? true : fmIsAggFunc(pFunc->funcId);
     pSelect->hasRepeatScanFuncs = pSelect->hasRepeatScanFuncs ? true : fmIsRepeatScanFunc(pFunc->funcId);
     pSelect->hasIndefiniteRowsFunc = pSelect->hasIndefiniteRowsFunc ? true : fmIsIndefiniteRowsFunc(pFunc->funcId);
@@ -1226,41 +1255,138 @@ static void setFuncClassification(SSelectStmt* pSelect, SFunctionNode* pFunc) {
   }
 }
 
-static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) {
+static int32_t rewriteSystemInfoFuncImpl(STranslateContext* pCxt, char* pLiteral, SNode** pNode) {
+  SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
+  if (NULL == pVal) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  strcpy(pVal->node.aliasName, ((SExprNode*)*pNode)->aliasName);
+  pVal->node.resType = ((SExprNode*)*pNode)->resType;
+  if (NULL == pLiteral) {
+    pVal->isNull = true;
+  } else {
+    pVal->literal = pLiteral;
+  }
+  if (DEAL_RES_ERROR != translateValue(pCxt, pVal)) {
+    *pNode = (SNode*)pVal;
+  } else {
+    nodesDestroyNode((SNode*)pVal);
+  }
+  return pCxt->errCode;
+}
+
+static int32_t rewriteDatabaseFunc(STranslateContext* pCxt, SNode** pNode) {
+  char* pCurrDb = NULL;
+  if (NULL != pCxt->pParseCxt->db) {
+    pCurrDb = taosMemoryStrDup((void*)pCxt->pParseCxt->db);
+    if (NULL == pCurrDb) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
+  return rewriteSystemInfoFuncImpl(pCxt, pCurrDb, pNode);
+}
+
+static int32_t rewriteClentVersionFunc(STranslateContext* pCxt, SNode** pNode) {
+  char* pVer = taosMemoryStrDup((void*)version);
+  if (NULL == pVer) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  return rewriteSystemInfoFuncImpl(pCxt, pVer, pNode);
+}
+
+static int32_t rewriteServerVersionFunc(STranslateContext* pCxt, SNode** pNode) {
+  char* pVer = taosMemoryStrDup((void*)pCxt->pParseCxt->svrVer);
+  if (NULL == pVer) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  return rewriteSystemInfoFuncImpl(pCxt, pVer, pNode);
+}
+
+static int32_t rewriteServerStatusFunc(STranslateContext* pCxt, SNode** pNode) {
+  if (pCxt->pParseCxt->nodeOffline) {
+    return TSDB_CODE_RPC_NETWORK_UNAVAIL;
+  }
+  char* pStatus = taosMemoryStrDup((void*)"1");
+  return rewriteSystemInfoFuncImpl(pCxt, pStatus, pNode);
+}
+
+static int32_t rewriteUserFunc(STranslateContext* pCxt, SNode** pNode) {
+  char    userConn[TSDB_USER_LEN + 1 + TSDB_FQDN_LEN] = {0};  // format 'user@host'
+  int32_t len = snprintf(userConn, sizeof(userConn), "%s@", pCxt->pParseCxt->pUser);
+  taosGetFqdn(userConn + len);
+  char* pUserConn = taosMemoryStrDup((void*)userConn);
+  if (NULL == pUserConn) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  return rewriteSystemInfoFuncImpl(pCxt, pUserConn, pNode);
+}
+
+static int32_t rewriteSystemInfoFunc(STranslateContext* pCxt, SNode** pNode) {
+  switch (((SFunctionNode*)*pNode)->funcType) {
+    case FUNCTION_TYPE_DATABASE:
+      return rewriteDatabaseFunc(pCxt, pNode);
+    case FUNCTION_TYPE_CLIENT_VERSION:
+      return rewriteClentVersionFunc(pCxt, pNode);
+    case FUNCTION_TYPE_SERVER_VERSION:
+      return rewriteServerVersionFunc(pCxt, pNode);
+    case FUNCTION_TYPE_SERVER_STATUS:
+      return rewriteServerStatusFunc(pCxt, pNode);
+    case FUNCTION_TYPE_CURRENT_USER:
+    case FUNCTION_TYPE_USER:
+      return rewriteUserFunc(pCxt, pNode);
+    default:
+      break;
+  }
+  return TSDB_CODE_PAR_INTERNAL_ERROR;
+}
+
+static int32_t translateNoramlFunction(STranslateContext* pCxt, SFunctionNode* pFunc) {
+  int32_t code = translateAggFunc(pCxt, pFunc);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = translateScanPseudoColumnFunc(pCxt, pFunc);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = translateIndefiniteRowsFunc(pCxt, pFunc);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = translateForbidFillFunc(pCxt, pFunc);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = translateWindowPseudoColumnFunc(pCxt, pFunc);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = translateForbidWindowFunc(pCxt, pFunc);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = translateForbidStreamFunc(pCxt, pFunc);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = translateForbidGroupByFunc(pCxt, pFunc);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    setFuncClassification(pCxt->pCurrStmt, pFunc);
+  }
+  return code;
+}
+
+static int32_t translateFunctionImpl(STranslateContext* pCxt, SFunctionNode** pFunc) {
+  if (fmIsSystemInfoFunc((*pFunc)->funcId)) {
+    return rewriteSystemInfoFunc(pCxt, (SNode**)pFunc);
+  }
+  return translateNoramlFunction(pCxt, *pFunc);
+}
+
+static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode** pFunc) {
   SNode* pParam = NULL;
-  FOREACH(pParam, pFunc->pParameterList) {
+  FOREACH(pParam, (*pFunc)->pParameterList) {
     if (isMultiResFunc(pParam)) {
       return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)pParam)->aliasName);
     }
   }
 
-  pCxt->errCode = getFuncInfo(pCxt, pFunc);
+  pCxt->errCode = getFuncInfo(pCxt, *pFunc);
   if (TSDB_CODE_SUCCESS == pCxt->errCode) {
-    pCxt->errCode = translateAggFunc(pCxt, pFunc);
+    pCxt->errCode = translateFunctionImpl(pCxt, pFunc);
-  }
-  if (TSDB_CODE_SUCCESS == pCxt->errCode) {
-    pCxt->errCode = translateScanPseudoColumnFunc(pCxt, pFunc);
-  }
-  if (TSDB_CODE_SUCCESS == pCxt->errCode) {
-    pCxt->errCode = translateIndefiniteRowsFunc(pCxt, pFunc);
-  }
-  if (TSDB_CODE_SUCCESS == pCxt->errCode) {
-    pCxt->errCode = translateForbidFillFunc(pCxt, pFunc);
-  }
-  if (TSDB_CODE_SUCCESS == pCxt->errCode) {
-    pCxt->errCode = translateWindowPseudoColumnFunc(pCxt, pFunc);
-  }
-  if (TSDB_CODE_SUCCESS == pCxt->errCode) {
-    pCxt->errCode = translateForbidWindowFunc(pCxt, pFunc);
-  }
-  if (TSDB_CODE_SUCCESS == pCxt->errCode) {
-    pCxt->errCode = translateForbidStreamFunc(pCxt, pFunc);
-  }
-  if (TSDB_CODE_SUCCESS == pCxt->errCode) {
-    pCxt->errCode = translateForbidGroupByFunc(pCxt, pFunc);
-  }
-  if (TSDB_CODE_SUCCESS == pCxt->errCode) {
-    setFuncClassification(pCxt->pCurrSelectStmt, pFunc);
   }
   return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR;
 }
@@ -1285,7 +1411,7 @@ static EDealRes doTranslateExpr(SNode** pNode, void* pContext) {
     case QUERY_NODE_OPERATOR:
       return translateOperator(pCxt, (SOperatorNode**)pNode);
     case QUERY_NODE_FUNCTION:
-      return translateFunction(pCxt, (SFunctionNode*)*pNode);
+      return translateFunction(pCxt, (SFunctionNode**)pNode);
     case QUERY_NODE_LOGIC_CONDITION:
       return translateLogicCond(pCxt, (SLogicConditionNode*)*pNode);
     case QUERY_NODE_TEMP_TABLE:
@@ -1308,9 +1434,9 @@ static int32_t translateExprList(STranslateContext* pCxt, SNodeList* pList) {
 
 static SNodeList* getGroupByList(STranslateContext* pCxt) {
   if (isDistinctOrderBy(pCxt)) {
-    return pCxt->pCurrSelectStmt->pProjectionList;
+    return ((SSelectStmt*)pCxt->pCurrStmt)->pProjectionList;
   }
-  return pCxt->pCurrSelectStmt->pGroupByList;
+  return ((SSelectStmt*)pCxt->pCurrStmt)->pGroupByList;
 }
 
 static SNode* getGroupByNode(SNode* pNode) {
@@ -1348,7 +1474,7 @@ static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode
   }
   if (TSDB_CODE_SUCCESS == pCxt->errCode) {
     *pNode = (SNode*)pFunc;
-    pCxt->pCurrSelectStmt->hasSelectValFunc = true;
+    ((SSelectStmt*)pCxt->pCurrStmt)->hasSelectValFunc = true;
   } else {
     nodesDestroyNode((SNode*)pFunc);
   }
@@ -1378,7 +1504,7 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) {
     }
   }
   SNode* pPartKey = NULL;
-  FOREACH(pPartKey, pCxt->pTranslateCxt->pCurrSelectStmt->pPartitionByList) {
+  FOREACH(pPartKey, ((SSelectStmt*)pCxt->pTranslateCxt->pCurrStmt)->pPartitionByList) {
     if (nodesEqualNode(pPartKey, *pNode)) {
       return DEAL_RES_IGNORE_CHILD;
     }
@@ -1463,7 +1589,7 @@ static EDealRes doCheckAggColCoexist(SNode* pNode, void* pContext) {
     return DEAL_RES_IGNORE_CHILD;
   }
   SNode* pPartKey = NULL;
-  FOREACH(pPartKey, pCxt->pTranslateCxt->pCurrSelectStmt->pPartitionByList) {
+  FOREACH(pPartKey, ((SSelectStmt*)pCxt->pTranslateCxt->pCurrStmt)->pPartitionByList) {
     if (nodesEqualNode(pPartKey, pNode)) {
       return DEAL_RES_IGNORE_CHILD;
     }
@@ -1627,7 +1753,8 @@ static uint8_t getStmtPrecision(SNode* pStmt) {
 
 static bool stmtIsSingleTable(SNode* pStmt) {
   if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) {
-    return ((STableNode*)((SSelectStmt*)pStmt)->pFromTable)->singleTable;
+    SSelectStmt* pSelect = (SSelectStmt*)pStmt;
+    return NULL == pSelect->pFromTable || ((STableNode*)pSelect->pFromTable)->singleTable;
   }
   return false;
 }
@@ -1655,8 +1782,8 @@ static int32_t setTableIndex(STranslateContext* pCxt, SName* pName, SRealTableNo
   if (pCxt->createStream || QUERY_SMA_OPTIMIZE_DISABLE == tsQuerySmaOptimize) {
     return TSDB_CODE_SUCCESS;
   }
-  if (NULL != pCxt->pCurrSelectStmt && NULL != pCxt->pCurrSelectStmt->pWindow &&
+  if (isSelectStmt(pCxt->pCurrStmt) && NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pWindow &&
-      QUERY_NODE_INTERVAL_WINDOW == nodeType(pCxt->pCurrSelectStmt->pWindow)) {
+      QUERY_NODE_INTERVAL_WINDOW == nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pWindow)) {
     return getTableIndex(pCxt, pName, &pRealTable->pSmaIndexes);
   }
   return TSDB_CODE_SUCCESS;
@@ -2369,7 +2496,7 @@ static EDealRes rewriteSeletcValueFunc(STranslateContext* pCxt, SNode** pNode) {
   nodesDestroyNode(*pNode);
   *pNode = (SNode*)pFirst;
   pCxt->errCode = fmGetFuncInfo(pFirst, pCxt->msgBuf.buf, pCxt->msgBuf.len);
-  pCxt->pCurrSelectStmt->hasAggFuncs = true;
+  ((SSelectStmt*)pCxt->pCurrStmt)->hasAggFuncs = true;
   return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR;
 }
 
@@ -2408,13 +2535,13 @@ static int32_t replaceOrderByAlias(STranslateContext* pCxt, SNodeList* pProjecti
 }
 
 static int32_t translateSelectWithoutFrom(STranslateContext* pCxt, SSelectStmt* pSelect) {
-  pCxt->pCurrSelectStmt = pSelect;
+  pCxt->pCurrStmt = (SNode*)pSelect;
   pCxt->currClause = SQL_CLAUSE_SELECT;
   return translateExprList(pCxt, pSelect->pProjectionList);
 }
 
 static int32_t translateSelectFrom(STranslateContext* pCxt, SSelectStmt* pSelect) {
-  pCxt->pCurrSelectStmt = pSelect;
+  pCxt->pCurrStmt = (SNode*)pSelect;
   int32_t code = translateFrom(pCxt, pSelect->pFromTable);
   if (TSDB_CODE_SUCCESS == code) {
     pSelect->precision = ((STableNode*)pSelect->pFromTable)->precision;
@@ -2538,8 +2665,7 @@ static int32_t translateSetOperOrderBy(STranslateContext* pCxt, SSetOperator* pS
   if (TSDB_CODE_SUCCESS == code) {
     if (other) {
       pCxt->currClause = SQL_CLAUSE_ORDER_BY;
-      pCxt->pCurrSelectStmt = NULL;
+      pCxt->pCurrStmt = (SNode*)pSetOperator;
-      pCxt->pCurrSetOperator = pSetOperator;
       code = translateExprList(pCxt, pSetOperator->pOrderByList);
     }
   }
@@ -4420,11 +4546,11 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
 static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode) {
   ++(pCxt->currLevel);
   ESqlClause currClause = pCxt->currClause;
-  SSelectStmt* pCurrStmt = pCxt->pCurrSelectStmt;
+  SNode*     pCurrStmt = pCxt->pCurrStmt;
   int32_t    code = translateQuery(pCxt, pNode);
   --(pCxt->currLevel);
   pCxt->currClause = currClause;
-  pCxt->pCurrSelectStmt = pCurrStmt;
+  pCxt->pCurrStmt = pCurrStmt;
   return code;
 }
 
@@ -5876,6 +6002,11 @@ static int32_t setRefreshMate(STranslateContext* pCxt, SQuery* pQuery) {
 static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) {
   switch (nodeType(pQuery->pRoot)) {
     case QUERY_NODE_SELECT_STMT:
+      if (NULL == ((SSelectStmt*)pQuery->pRoot)->pFromTable) {
+        pQuery->execMode = QUERY_EXEC_MODE_LOCAL;
+        pQuery->haveResultSet = true;
+        break;
+      }
     case QUERY_NODE_SET_OPERATOR:
     case QUERY_NODE_EXPLAIN_STMT:
       pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;

source/libs/parser/test/parInsertTest.cpp

@@ -265,6 +265,11 @@ TEST_F(InsertTest, autoCreateTableTest) {
       "insert into st1s1 using st1 (tag1, tag2) tags(1, 'wxy') values (now, 1, \"beijing\")"
       "(now+1s, 2, \"shanghai\")(now+2s, 3, \"guangzhou\")");
   ASSERT_EQ(runAsync(), TSDB_CODE_SUCCESS);
+
+  bind(
+      "insert into st1s1 using st1 tags(1, 'wxy', now) values (now, 1, \"beijing\")"
+      "st1s1 using st1 tags(1, 'wxy', now) values (now+1s, 2, \"shanghai\")");
+  ASSERT_EQ(run(), TSDB_CODE_SUCCESS);
 }
 
 TEST_F(InsertTest, toleranceTest) {

source/libs/parser/test/parSelectTest.cpp

@@ -423,6 +423,18 @@ TEST_F(ParserSelectTest, withoutFrom) {
   useDb("root", "test");
 
   run("SELECT 1");
+
+  run("SELECT DATABASE()");
+
+  run("SELECT CLIENT_VERSION()");
+
+  run("SELECT SERVER_VERSION()");
+
+  run("SELECT SERVER_STATUS()");
+
+  run("SELECT CURRENT_USER()");
+
+  run("SELECT USER()");
 }
 
 }  // namespace ParserTest

source/libs/parser/test/parTestUtil.cpp

@@ -203,6 +203,7 @@ class ParserTestBaseImpl {
     pCxt->pMsg = stmtEnv_.msgBuf_.data();
     pCxt->msgLen = stmtEnv_.msgBuf_.max_size();
     pCxt->async = async;
+    pCxt->svrVer = "3.0.0.0";
   }
 
   void doParse(SParseContext* pCxt, SQuery** pQuery) {

source/libs/planner/src/planLogicCreater.c

@@ -154,16 +154,12 @@ static int32_t createSelectRootLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p
   return createRootLogicNode(pCxt, pSelect, pSelect->precision, (FCreateLogicNode)func, pRoot);
 }
 
-static EScanType getScanType(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNodeList* pScanPseudoCols,
+static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols, SNodeList* pScanCols,
-                             SNodeList* pScanCols, int8_t tableType) {
+                             int8_t tableType) {
   if (pCxt->pPlanCxt->topicQuery || pCxt->pPlanCxt->streamQuery) {
     return SCAN_TYPE_STREAM;
   }
 
-  if (pSelect->hasLastRowFunc) {
-    return SCAN_TYPE_LAST_ROW;
-  }
-
   if (NULL == pScanCols) {
     // select count(*) from t
     return NULL == pScanPseudoCols
@@ -279,7 +275,7 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
     code = rewriteExprsForSelect(pScan->pScanPseudoCols, pSelect, SQL_CLAUSE_FROM);
   }
 
-  pScan->scanType = getScanType(pCxt, pSelect, pScan->pScanPseudoCols, pScan->pScanCols, pScan->tableType);
+  pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->tableType);
 
   if (TSDB_CODE_SUCCESS == code) {
     code = addPrimaryKeyCol(pScan->tableId, &pScan->pScanCols);
@@ -474,6 +470,8 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect,
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
+  pAgg->hasLastRow = pSelect->hasLastRowFunc;
+
   int32_t code = TSDB_CODE_SUCCESS;
 
   // set grouyp keys, agg funcs and having conditions

source/libs/planner/src/planOptimizer.c

@@ -276,7 +276,8 @@ static int32_t pushDownCondOptAppendCond(SNode** pCond, SNode** pAdditionalCond)
   }
 
   int32_t code = TSDB_CODE_SUCCESS;
-  if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pCond)) {
+  if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pCond) &&
+      LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)*pCond)->condType) {
     code = nodesListAppend(((SLogicConditionNode*)*pCond)->pParameterList, *pAdditionalCond);
     if (TSDB_CODE_SUCCESS == code) {
       *pAdditionalCond = NULL;
@@ -1083,6 +1084,11 @@ static int32_t partTagsOptRebuildTbanme(SNodeList* pPartKeys) {
   return code;
 }
 
+// todo refact: just to mask compilation warnings
+static void partTagsSetAlias(char* pAlias, int32_t len, const char* pTableAlias, const char* pColName) {
+  snprintf(pAlias, len, "%s.%s", pTableAlias, pColName);
+}
+
 static SNode* partTagsCreateWrapperFunc(const char* pFuncName, SNode* pNode) {
   SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION);
   if (NULL == pFunc) {
@@ -1092,7 +1098,7 @@ static SNode* partTagsCreateWrapperFunc(const char* pFuncName, SNode* pNode) {
   strcpy(pFunc->functionName, pFuncName);
   if (QUERY_NODE_COLUMN == nodeType(pNode)) {
     SColumnNode* pCol = (SColumnNode*)pNode;
-    sprintf(pFunc->node.aliasName, "%s.%s", pCol->tableAlias, pCol->colName);
+    partTagsSetAlias(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), pCol->tableAlias, pCol->colName);
   } else {
     strcpy(pFunc->node.aliasName, ((SExprNode*)pNode)->aliasName);
   }
@@ -1464,9 +1470,9 @@ static SNode* rewriteUniqueOptCreateFirstFunc(SFunctionNode* pSelectValue, SNode
 
   strcpy(pFunc->functionName, "first");
   if (NULL != pSelectValue) {
-    sprintf(pFunc->node.aliasName, "%s", pSelectValue->node.aliasName);
+    strcpy(pFunc->node.aliasName, pSelectValue->node.aliasName);
   } else {
-    sprintf(pFunc->node.aliasName, "%s.%p", pFunc->functionName, pFunc);
+    snprintf(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), "%s.%p", pFunc->functionName, pFunc);
   }
   int32_t code = nodesListMakeStrictAppend(&pFunc->pParameterList, nodesCloneNode(pCol));
   if (TSDB_CODE_SUCCESS == code) {
@@ -1616,6 +1622,46 @@ static int32_t rewriteUniqueOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLog
   return rewriteUniqueOptimizeImpl(pCxt, pLogicSubplan, pIndef);
 }
 
+static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) {
+  if (QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode) || !(((SAggLogicNode*)pNode)->hasLastRow) ||
+      NULL != ((SAggLogicNode*)pNode)->pGroupKeys || 1 != LIST_LENGTH(pNode->pChildren) ||
+      QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(nodesListGetNode(pNode->pChildren, 0)) ||
+      NULL != ((SScanLogicNode*)nodesListGetNode(pNode->pChildren, 0))->node.pConditions) {
+    return false;
+  }
+
+  SNode* pFunc = NULL;
+  FOREACH(pFunc, ((SAggLogicNode*)pNode)->pAggFuncs) {
+    if (FUNCTION_TYPE_LAST_ROW != ((SFunctionNode*)pFunc)->funcType) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
+  SAggLogicNode* pAgg = (SAggLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, lastRowScanOptMayBeOptimized);
+
+  if (NULL == pAgg) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SNode* pNode = NULL;
+  FOREACH(pNode, pAgg->pAggFuncs) {
+    SFunctionNode* pFunc = (SFunctionNode*)pNode;
+    int32_t        len = snprintf(pFunc->functionName, sizeof(pFunc->functionName), "_cache_last_row");
+    pFunc->functionName[len] = '\0';
+    fmGetFuncInfo(pFunc, NULL, 0);
+  }
+  pAgg->hasLastRow = false;
+
+  ((SScanLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0))->scanType = SCAN_TYPE_LAST_ROW;
+
+  pCxt->optimized = true;
+  return TSDB_CODE_SUCCESS;
+}
+
 // merge projects
 static bool mergeProjectsMayBeOptimized(SLogicNode* pNode) {
   if (QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pNode) || 1 != LIST_LENGTH(pNode->pChildren)) {
@@ -1704,7 +1750,8 @@ static const SOptimizeRule optimizeRuleSet[] = {
   {.pName = "EliminateProject",           .optimizeFunc = eliminateProjOptimize},
   {.pName = "EliminateSetOperator",       .optimizeFunc = eliminateSetOpOptimize},
   {.pName = "RewriteTail",                .optimizeFunc = rewriteTailOptimize},
-  {.pName = "RewriteUnique",              .optimizeFunc = rewriteUniqueOptimize}
+  {.pName = "RewriteUnique",              .optimizeFunc = rewriteUniqueOptimize},
+  {.pName = "LastRowScan",                .optimizeFunc = lastRowScanOptimize}
 };
 // clang-format on
 

source/libs/planner/src/planPhysiCreater.c

@@ -1344,7 +1344,7 @@ static int32_t createMergePhysiNode(SPhysiPlanContext* pCxt, SMergeLogicNode* pM
     }
   }
 
-  if (TSDB_CODE_SUCCESS == code) {
+  if (TSDB_CODE_SUCCESS == code && NULL != pMergeLogicNode->pMergeKeys) {
     code = setListSlotId(pCxt, pMerge->node.pOutputDataBlockDesc->dataBlockId, -1, pMergeLogicNode->pMergeKeys,
                          &pMerge->pMergeKeys);
   }

source/libs/planner/src/planSpliter.c

@@ -197,6 +197,8 @@ static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) {
       return stbSplIsMultiTbScan(streamQuery, (SScanLogicNode*)pNode);
     case QUERY_NODE_LOGIC_PLAN_JOIN:
       return !(((SJoinLogicNode*)pNode)->isSingleTableJoin);
+    case QUERY_NODE_LOGIC_PLAN_PARTITION:
+      return stbSplHasMultiTbScan(streamQuery, pNode);
     case QUERY_NODE_LOGIC_PLAN_AGG:
       return !stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) && stbSplHasMultiTbScan(streamQuery, pNode);
     case QUERY_NODE_LOGIC_PLAN_WINDOW:
@@ -431,7 +433,7 @@ static int32_t stbSplSplitIntervalForBatch(SSplitContext* pCxt, SStableSplitInfo
     SNodeList* pMergeKeys = NULL;
     code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pInfo->pSplitNode)->pTspk, &pMergeKeys);
     if (TSDB_CODE_SUCCESS == code) {
-      code = stbSplCreateMergeNode(pCxt, NULL, pInfo->pSplitNode, pMergeKeys, pPartWindow, false);
+      code = stbSplCreateMergeNode(pCxt, NULL, pInfo->pSplitNode, pMergeKeys, pPartWindow, true);
     }
     if (TSDB_CODE_SUCCESS != code) {
       nodesDestroyList(pMergeKeys);
@@ -887,6 +889,16 @@ static int32_t stbSplSplitJoinNode(SSplitContext* pCxt, SStableSplitInfo* pInfo)
   return code;
 }
 
+static int32_t stbSplSplitPartitionNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
+  int32_t code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, NULL, pInfo->pSplitNode, true);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren,
+                                     (SNode*)splCreateScanSubplan(pCxt, pInfo->pSplitNode, SPLIT_FLAG_STABLE_SPLIT));
+  }
+  ++(pCxt->groupId);
+  return code;
+}
+
 static int32_t stableSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) {
   if (pCxt->pPlanCxt->rSmaQuery) {
     return TSDB_CODE_SUCCESS;
@@ -905,6 +917,9 @@ static int32_t stableSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) {
     case QUERY_NODE_LOGIC_PLAN_JOIN:
       code = stbSplSplitJoinNode(pCxt, &info);
       break;
+    case QUERY_NODE_LOGIC_PLAN_PARTITION:
+      code = stbSplSplitPartitionNode(pCxt, &info);
+      break;
     case QUERY_NODE_LOGIC_PLAN_AGG:
       code = stbSplSplitAggNode(pCxt, &info);
       break;

source/libs/planner/src/planner.c

@@ -102,7 +102,6 @@ static void clearSubplanExecutionNode(SPhysiNode* pNode) {
 
 void qClearSubplanExecutionNode(SSubplan* pSubplan) { clearSubplanExecutionNode(pSubplan->pNode); }
 
-
 int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen) {
   if (SUBPLAN_TYPE_MODIFY == pSubplan->subplanType && NULL == pSubplan->pNode) {
     SDataInserterNode* insert = (SDataInserterNode*)pSubplan->pDataSink;

source/libs/planner/test/planBasicTest.cpp

@@ -99,6 +99,8 @@ TEST_F(PlanBasicTest, lastRowFunc) {
   run("SELECT LAST_ROW(c1, c2) FROM t1");
 
   run("SELECT LAST_ROW(c1) FROM st1");
+
+  run("SELECT LAST_ROW(c1), SUM(c3) FROM t1");
 }
 
 TEST_F(PlanBasicTest, sampleFunc) {

source/libs/planner/test/planTestUtil.cpp

@@ -310,6 +310,7 @@ class PlannerTestBaseImpl {
     cxt.sqlLen = stmtEnv_.sql_.length();
     cxt.pMsg = stmtEnv_.msgBuf_.data();
     cxt.msgLen = stmtEnv_.msgBuf_.max_size();
+    cxt.svrVer = "3.0.0.0";
 
     DO_WITH_THROW(qParseSql, &cxt, pQuery);
     if (prepare) {

source/libs/scalar/src/sclfunc.c

@@ -1116,7 +1116,8 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *
 
 int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
   int32_t type = GET_PARAM_TYPE(pInput);
-  int32_t timePrec = GET_PARAM_PRECISON(pInput);
+  int64_t timePrec;
+  GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData);
 
   for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
     if (colDataIsNull_s(pInput[0].columnData, i)) {
@@ -1192,10 +1193,10 @@ int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
 
 int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
   int32_t type = GET_PARAM_TYPE(&pInput[0]);
-  int32_t timePrec = GET_PARAM_PRECISON(&pInput[0]);
 
-  int64_t timeUnit, timeVal = 0;
+  int64_t timeUnit, timePrec, timeVal = 0;
   GET_TYPED_DATA(timeUnit, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData);
+  GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData);
 
   int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 :
                    (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000);
@@ -1380,10 +1381,12 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara
 }
 
 int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
-  int32_t timePrec = GET_PARAM_PRECISON(&pInput[0]);
+  int64_t timeUnit = -1, timePrec, timeVal[2] = {0};
-  int64_t timeUnit = -1, timeVal[2] = {0};
+  if (inputNum == 4) {
-  if (inputNum == 3) {
     GET_TYPED_DATA(timeUnit, int64_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData);
+    GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[3]), pInput[3].columnData->pData);
+  } else {
+    GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData);
   }
 
   int32_t numOfRows = 0;
@@ -1512,7 +1515,10 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
 }
 
 int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
-  int64_t ts = taosGetTimestamp(TSDB_TIME_PRECISION_MILLI);
+  int64_t timePrec;
+  GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[0]), pInput[0].columnData->pData);
+
+  int64_t ts = taosGetTimestamp(timePrec);
   for (int32_t i = 0; i < pInput->numOfRows; ++i) {
     colDataAppendInt64(pOutput->columnData, i, &ts);
   }
@@ -1521,7 +1527,10 @@ int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
 }
 
 int32_t todayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
-  int64_t ts = taosGetTimestampToday(TSDB_TIME_PRECISION_MILLI);
+  int64_t timePrec;
+  GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[0]), pInput[0].columnData->pData);
+
+  int64_t ts = taosGetTimestampToday(timePrec);
   for (int32_t i = 0; i < pInput->numOfRows; ++i) {
     colDataAppendInt64(pOutput->columnData, i, &ts);
   }

source/libs/scheduler/inc/schedulerInt.h

@@ -99,8 +99,8 @@ typedef struct SSchStat {
 typedef struct SSchResInfo {
   SQueryResult*          queryRes;
   void**                 fetchRes;
-  schedulerExecCallback  execFp; 
+  schedulerExecFp        execFp; 
-  schedulerFetchCallback fetchFp; 
+  schedulerFetchFp       fetchFp; 
   void*                  userParam;
 } SSchResInfo;
 
@@ -226,7 +226,8 @@ typedef struct SSchJob {
   SQueryNodeAddr     resNode;
   tsem_t             rspSem;
   SSchOpStatus       opStatus;
-  bool            *reqKilled;
+  schedulerChkKillFp chkKillFp;
+  void*              chkKillParam;
   SSchTask          *fetchTask;
   int32_t            errCode;
   SRWLatch           resLock;

source/libs/scheduler/src/schJob.c

@@ -55,9 +55,10 @@ int32_t schInitJob(SSchedulerReq *pReq, SSchJob **pSchJob) {
   pJob->conn = *pReq->pConn;
   pJob->sql = pReq->sql;
   pJob->pDag = pReq->pDag;
-  pJob->reqKilled = pReq->reqKilled;
+  pJob->chkKillFp = pReq->chkKillFp;
-  pJob->userRes.execFp = pReq->fp;
+  pJob->chkKillParam = pReq->chkKillParam;
-  pJob->userRes.userParam = pReq->cbParam;
+  pJob->userRes.execFp = pReq->execFp;
+  pJob->userRes.userParam = pReq->execParam;
 
   if (pReq->pNodeList == NULL || taosArrayGetSize(pReq->pNodeList) <= 0) {
     qDebug("QID:0x%" PRIx64 " input exec nodeList is empty", pReq->pDag->queryId);
@@ -182,7 +183,7 @@ FORCE_INLINE bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus) {
     *pStatus = status;
   }
 
-  if (*pJob->reqKilled) {
+  if ((*pJob->chkKillFp)(pJob->chkKillParam)) {
     schUpdateJobStatus(pJob, JOB_TASK_STATUS_DROPPING);
     schUpdateJobErrCode(pJob, TSDB_CODE_TSC_QUERY_KILLED);
 
@@ -1611,7 +1612,7 @@ _return:
 
   schEndOperation(pJob);
   if (!sync) {
-    pReq->fp(NULL, pReq->cbParam, code);
+    pReq->execFp(NULL, pReq->execParam, code);
   }
   
   schFreeJobImpl(pJob);
@@ -1658,7 +1659,7 @@ int32_t schExecJobImpl(SSchedulerReq *pReq, SSchJob *pJob, bool sync) {
   SCH_ERR_JRET(schBeginOperation(pJob, SCH_OP_EXEC, sync));
 
   if (EXPLAIN_MODE_STATIC == pReq->pDag->explainInfo.mode) {
-    code = schLaunchStaticExplainJob(pReq, pJob, true);
+    code = schLaunchStaticExplainJob(pReq, pJob, sync);
   } else {
     code = schLaunchJob(pJob);
     if (sync) {
@@ -1678,7 +1679,7 @@ int32_t schExecJobImpl(SSchedulerReq *pReq, SSchJob *pJob, bool sync) {
 _return:
 
   if (!sync) {
-    pReq->fp(NULL, pReq->cbParam, code);
+    pReq->execFp(NULL, pReq->execParam, code);
   }
   
   SCH_RET(code);

source/libs/scheduler/src/scheduler.c

@@ -113,10 +113,6 @@ _return:
     schReleaseJob(pJob->refId);
   }
 
-  if (code != TSDB_CODE_SUCCESS) {
-    pReq->fp(NULL, pReq->cbParam, code);
-  }
-
   return code;
 }
 
@@ -144,7 +140,7 @@ int32_t schedulerFetchRows(int64_t job, void **pData) {
   SCH_RET(code);
 }
 
-void schedulerAsyncFetchRows(int64_t job, schedulerFetchCallback fp, void* param) {
+void schedulerAsyncFetchRows(int64_t job, schedulerFetchFp fp, void* param) {
   qDebug("scheduler async fetch rows start");
 
   int32_t code = 0;

source/libs/scheduler/test/schedulerTests.cpp

@@ -511,8 +511,8 @@ void* schtRunJobThread(void *aa) {
     req.pNodeList = qnodeList;
     req.pDag = &dag;
     req.sql = "select * from tb";
-    req.fp = schtQueryCb;
+    req.execFp = schtQueryCb;
-    req.cbParam = &queryDone;
+    req.execParam = &queryDone;
     
     code = schedulerAsyncExecJob(&req, &queryJobRefId);      
     assert(code == 0);
@@ -663,8 +663,8 @@ TEST(queryTest, normalCase) {
   req.pNodeList = qnodeList;
   req.pDag = &dag;
   req.sql = "select * from tb";
-  req.fp = schtQueryCb;
+  req.execFp = schtQueryCb;
-  req.cbParam = &queryDone;
+  req.execParam = &queryDone;
     
   code = schedulerAsyncExecJob(&req, &job);  
   ASSERT_EQ(code, 0);
@@ -767,8 +767,8 @@ TEST(queryTest, readyFirstCase) {
   req.pNodeList = qnodeList;
   req.pDag = &dag;
   req.sql = "select * from tb";
-  req.fp = schtQueryCb;
+  req.execFp = schtQueryCb;
-  req.cbParam = &queryDone;
+  req.execParam = &queryDone;
   code = schedulerAsyncExecJob(&req, &job);
   ASSERT_EQ(code, 0);
 
@@ -874,8 +874,8 @@ TEST(queryTest, flowCtrlCase) {
   req.pNodeList = qnodeList;
   req.pDag = &dag;
   req.sql = "select * from tb";
-  req.fp = schtQueryCb;
+  req.execFp = schtQueryCb;
-  req.cbParam = &queryDone;
+  req.execParam = &queryDone;
 
   code = schedulerAsyncExecJob(&req, &job);
   ASSERT_EQ(code, 0);
@@ -987,8 +987,8 @@ TEST(insertTest, normalCase) {
   req.pNodeList = qnodeList;
   req.pDag = &dag;
   req.sql = "insert into tb values(now,1)";
-  req.fp = schtQueryCb;
+  req.execFp = schtQueryCb;
-  req.cbParam = NULL;
+  req.execParam = NULL;
   
   code = schedulerExecJob(&req, &insertJobRefId, &res);
   ASSERT_EQ(code, 0);

source/libs/stream/src/streamData.c

@@ -30,7 +30,7 @@ int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock
     /*int32_t            len = *(int32_t*)taosArrayGet(pReq->dataLen, i);*/
     SRetrieveTableRsp* pRetrieve = taosArrayGetP(pReq->data, i);
     SSDataBlock*       pDataBlock = taosArrayGet(pArray, i);
-    blockCompressDecode(pDataBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data);
+    blockDecode(pDataBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data);
     // TODO: refactor
     pDataBlock->info.window.skey = be64toh(pRetrieve->skey);
     pDataBlock->info.window.ekey = be64toh(pRetrieve->ekey);
@@ -50,7 +50,7 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock
   taosArraySetSize(pArray, 1);
   SRetrieveTableRsp* pRetrieve = pReq->pRetrieve;
   SSDataBlock*       pDataBlock = taosArrayGet(pArray, 0);
-  blockCompressDecode(pDataBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data);
+  blockDecode(pDataBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data);
   // TODO: refactor
   pDataBlock->info.window.skey = be64toh(pRetrieve->skey);
   pDataBlock->info.window.ekey = be64toh(pRetrieve->ekey);

source/libs/stream/src/streamDispatch.c

@@ -108,7 +108,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock)
   pRetrieve->ekey = htobe64(pBlock->info.window.ekey);
 
   int32_t actualLen = 0;
-  blockCompressEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false);
+  blockEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false);
 
   SStreamRetrieveReq req = {
       .streamId = pTask->streamId,
@@ -181,7 +181,7 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis
   pRetrieve->numOfCols = htonl(numOfCols);
 
   int32_t actualLen = 0;
-  blockCompressEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false);
+  blockEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false);
   actualLen += sizeof(SRetrieveTableRsp);
   ASSERT(actualLen <= dataStrLen);
   taosArrayPush(pReq->dataLen, &actualLen);

source/libs/stream/src/streamExec.c

@@ -17,21 +17,20 @@
 
 static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) {
   void* exec = pTask->exec.executor;
-  bool  hasData = false;
 
   // set input
   SStreamQueueItem* pItem = (SStreamQueueItem*)data;
   if (pItem->type == STREAM_INPUT__TRIGGER) {
     SStreamTrigger* pTrigger = (SStreamTrigger*)data;
-    qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_DATA_TYPE_SSDATA_BLOCK, false);
+    qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK, false);
   } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) {
     ASSERT(pTask->isDataScan);
     SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data;
-    qSetStreamInput(exec, pSubmit->data, STREAM_DATA_TYPE_SUBMIT_BLOCK, false);
+    qSetStreamInput(exec, pSubmit->data, STREAM_INPUT__DATA_SUBMIT, false);
   } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) {
     SStreamDataBlock* pBlock = (SStreamDataBlock*)data;
     SArray*           blocks = pBlock->blocks;
-    qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK, false);
+    qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__DATA_BLOCK, false);
   } else if (pItem->type == STREAM_INPUT__DROP) {
     // TODO exec drop
     return 0;
@@ -46,19 +45,16 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes)
     }
     if (output == NULL) {
       if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) {
-        //SSDataBlock block = {0};
+        SSDataBlock       block = {0};
-        //block.info.type = STREAM_PUSH_EMPTY;
-        //block.info.childId = pTask->selfChildId;
         SStreamDataBlock* pRetrieveBlock = (SStreamDataBlock*)data;
         ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1);
-        SSDataBlock* pBlock = createOneDataBlock(taosArrayGet(pRetrieveBlock->blocks, 0), true);
+        assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0));
-        pBlock->info.type = STREAM_PUSH_EMPTY;
+        block.info.type = STREAM_PULL_OVER;
-        pBlock->info.childId = pTask->selfChildId;
+        block.info.childId = pTask->selfChildId;
-        taosArrayPush(pRes, pBlock);
+        taosArrayPush(pRes, &block);
       }
       break;
     }
-    hasData = true;
 
     if (output->info.type == STREAM_RETRIEVE) {
       if (streamBroadcastToChildren(pTask, output) < 0) {
@@ -72,9 +68,6 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes)
     assignOneDataBlock(&block, output);
     block.info.childId = pTask->selfChildId;
     taosArrayPush(pRes, &block);
-    /*SSDataBlock* outputCopy = createOneDataBlock(output, true);*/
-    /*outputCopy->info.childId = pTask->selfChildId;*/
-    /*taosArrayPush(pRes, outputCopy);*/
   }
   return 0;
 }
@@ -163,4 +156,3 @@ FAIL:
   atomic_store_8(&pTask->execStatus, TASK_EXEC_STATUS__IDLE);
   return -1;
 }
-

source/libs/stream/src/streamTask.c

@@ -86,9 +86,7 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
     ASSERT(pTask->sinkType == TASK_SINK__NONE);
   }
 
-  if (pTask->dispatchType == TASK_DISPATCH__INPLACE) {
+  if (pTask->dispatchType == TASK_DISPATCH__FIXED) {
-    if (tEncodeI32(pEncoder, pTask->inplaceDispatcher.taskId) < 0) return -1;
-  } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) {
     if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.taskId) < 0) return -1;
     if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.nodeId) < 0) return -1;
     if (tEncodeSEpSet(pEncoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1;
@@ -147,9 +145,7 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
     ASSERT(pTask->sinkType == TASK_SINK__NONE);
   }
 
-  if (pTask->dispatchType == TASK_DISPATCH__INPLACE) {
+  if (pTask->dispatchType == TASK_DISPATCH__FIXED) {
-    if (tDecodeI32(pDecoder, &pTask->inplaceDispatcher.taskId) < 0) return -1;
-  } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) {
     if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.taskId) < 0) return -1;
     if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.nodeId) < 0) return -1;
     if (tDecodeSEpSet(pDecoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1;

source/libs/sync/inc/syncInt.h

@@ -221,7 +221,6 @@ void syncNodeVoteForSelf(SSyncNode* pSyncNode);
 
 // snapshot --------------
 bool syncNodeHasSnapshot(SSyncNode* pSyncNode);
-bool syncNodeIsIndexInSnapshot(SSyncNode* pSyncNode, SyncIndex index);
 
 SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode);
 SyncTerm  syncNodeGetLastTerm(SSyncNode* pSyncNode);

source/libs/sync/src/syncAppendEntries.c

@@ -860,7 +860,9 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
         }
 
         code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry);
-        ASSERT(code == 0);
+        if (code != 0) {
+          return -1;
+        }
 
         // pre commit
         code = syncNodePreCommit(ths, pAppendEntry);
@@ -971,7 +973,9 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
         ASSERT(pAppendEntry != NULL);
 
         code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry);
-        ASSERT(code == 0);
+        if (code != 0) {
+          return -1;
+        }
 
         // pre commit
         code = syncNodePreCommit(ths, pAppendEntry);

source/libs/sync/src/syncElection.c

@@ -75,7 +75,11 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) {
   if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) {
     syncNodeFollower2Candidate(pSyncNode);
   }
-  ASSERT(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE);
+
+  if (pSyncNode->state != TAOS_SYNC_STATE_CANDIDATE) {
+    syncNodeErrorLog(pSyncNode, "not candidate, can not elect");
+    return -1;
+  }
 
   // start election
   raftStoreNextTerm(pSyncNode->pRaftStore);

source/libs/sync/src/syncMain.c

@@ -1923,6 +1923,8 @@ void syncNodeVoteForSelf(SSyncNode* pSyncNode) {
 }
 
 // snapshot --------------
+
+// return if has a snapshot
 bool syncNodeHasSnapshot(SSyncNode* pSyncNode) {
   bool      ret = false;
   SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1};
@@ -1935,21 +1937,10 @@ bool syncNodeHasSnapshot(SSyncNode* pSyncNode) {
   return ret;
 }
 
-#if 0
+// return max(logLastIndex, snapshotLastIndex)
-bool syncNodeIsIndexInSnapshot(SSyncNode* pSyncNode, SyncIndex index) {
+// if no snapshot and log, return -1
-  ASSERT(syncNodeHasSnapshot(pSyncNode));
-  ASSERT(pSyncNode->pFsm->FpGetSnapshotInfo != NULL);
-  ASSERT(index >= SYNC_INDEX_BEGIN);
-
-  SSnapshot snapshot;
-  pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
-  bool b = (index <= snapshot.lastApplyIndex);
-  return b;
-}
-#endif
-
 SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode) {
-  SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0};
+  SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1};
   if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) {
     pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
   }
@@ -1959,6 +1950,8 @@ SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode) {
   return lastIndex;
 }
 
+// return the last term of snapshot and log
+// if error, return SYNC_TERM_INVALID (by syncLogLastTerm)
 SyncTerm syncNodeGetLastTerm(SSyncNode* pSyncNode) {
   SyncTerm lastTerm = 0;
   if (syncNodeHasSnapshot(pSyncNode)) {
@@ -1990,11 +1983,14 @@ int32_t syncNodeGetLastIndexTerm(SSyncNode* pSyncNode, SyncIndex* pLastIndex, Sy
   return 0;
 }
 
+// return append-entries first try index
 SyncIndex syncNodeSyncStartIndex(SSyncNode* pSyncNode) {
   SyncIndex syncStartIndex = syncNodeGetLastIndex(pSyncNode) + 1;
   return syncStartIndex;
 }
 
+// if index > 0, return index - 1
+// else, return -1
 SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index) {
   SyncIndex preIndex = index - 1;
   if (preIndex < SYNC_INDEX_INVALID) {
@@ -2004,21 +2000,10 @@ SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index) {
   return preIndex;
 }
 
-/*
+// if index < 0, return SYNC_TERM_INVALID
-SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index) {
+// if index == 0, return 0
-  ASSERT(index >= SYNC_INDEX_BEGIN);
+// if index > 0, return preTerm
-
+// if error, return SYNC_TERM_INVALID
-  SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode);
-  if (index > syncStartIndex) {
-    syncNodeLog3("syncNodeGetPreIndex", pSyncNode);
-    ASSERT(0);
-  }
-
-  SyncIndex preIndex = index - 1;
-  return preIndex;
-}
-*/
-
 SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) {
   if (index < SYNC_INDEX_BEGIN) {
     return SYNC_TERM_INVALID;
@@ -2056,112 +2041,6 @@ SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) {
   return SYNC_TERM_INVALID;
 }
 
-#if 0
-SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) {
-  ASSERT(index >= SYNC_INDEX_BEGIN);
-
-  SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode);
-  if (index > syncStartIndex) {
-    syncNodeLog3("syncNodeGetPreTerm", pSyncNode);
-    ASSERT(0);
-  }
-
-  if (index == SYNC_INDEX_BEGIN) {
-    return 0;
-  }
-
-  SyncTerm        preTerm = 0;
-  SyncIndex       preIndex = index - 1;
-  SSyncRaftEntry* pPreEntry = NULL;
-  int32_t         code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, preIndex, &pPreEntry);
-  if (code == 0) {
-    ASSERT(pPreEntry != NULL);
-    preTerm = pPreEntry->term;
-    taosMemoryFree(pPreEntry);
-    return preTerm;
-  } else {
-    if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) {
-      SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1};
-      if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) {
-        pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
-        if (snapshot.lastApplyIndex == preIndex) {
-          return snapshot.lastApplyTerm;
-        }
-      }
-    }
-  }
-
-  ASSERT(0);
-  return -1;
-}
-#endif
-
-#if 0
-SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) {
-  ASSERT(index >= SYNC_INDEX_BEGIN);
-
-  SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode);
-  if (index > syncStartIndex) {
-    syncNodeLog3("syncNodeGetPreTerm", pSyncNode);
-    ASSERT(0);
-  }
-
-  if (index == SYNC_INDEX_BEGIN) {
-    return 0;
-  }
-
-  SyncTerm preTerm = 0;
-  if (syncNodeHasSnapshot(pSyncNode)) {
-    // has snapshot
-    SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1};
-    if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) {
-      pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
-    }
-
-    if (index > snapshot.lastApplyIndex + 1) {
-      // should be log preTerm
-      SSyncRaftEntry* pPreEntry = NULL;
-      int32_t         code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, index - 1, &pPreEntry);
-      ASSERT(code == 0);
-      ASSERT(pPreEntry != NULL);
-
-      preTerm = pPreEntry->term;
-      taosMemoryFree(pPreEntry);
-
-    } else if (index == snapshot.lastApplyIndex + 1) {
-      preTerm = snapshot.lastApplyTerm;
-
-    } else {
-      // maybe snapshot change
-      sError("sync get pre term, bad scene. index:%ld", index);
-      logStoreLog2("sync get pre term, bad scene", pSyncNode->pLogStore);
-
-      SSyncRaftEntry* pPreEntry = NULL;
-      int32_t         code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, index - 1, &pPreEntry);
-      ASSERT(code == 0);
-      ASSERT(pPreEntry != NULL);
-
-      preTerm = pPreEntry->term;
-      taosMemoryFree(pPreEntry);
-    }
-
-  } else {
-    // no snapshot
-    ASSERT(index > SYNC_INDEX_BEGIN);
-
-    SSyncRaftEntry* pPreEntry = NULL;
-    int32_t         code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, index - 1, &pPreEntry);
-    ASSERT(code == 0);
-    ASSERT(pPreEntry != NULL);
-
-    preTerm = pPreEntry->term;
-    taosMemoryFree(pPreEntry);
-  }
-
-  return preTerm;
-}
-#endif
-
 // get pre index and term of "index"
 int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex* pPreIndex, SyncTerm* pPreTerm) {
   *pPreIndex = syncNodeGetPreIndex(pSyncNode, index);
@@ -2351,8 +2230,8 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) {
   ASSERT(pEntry != NULL);
 
   if (ths->state == TAOS_SYNC_STATE_LEADER) {
-    // ths->pLogStore->appendEntry(ths->pLogStore, pEntry);
+    int32_t code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry);
-    ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry);
+    ASSERT(code == 0);
     syncNodeReplicate(ths);
   }
 
@@ -2406,6 +2285,7 @@ int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) {
 //
 int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex) {
   int32_t ret = 0;
+  int32_t code = 0;
   syncClientRequestLog2("==syncNodeOnClientRequestCb==", pMsg);
 
   SyncIndex       index = ths->pLogStore->syncLogWriteIndex(ths->pLogStore);
@@ -2414,18 +2294,24 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncI
   ASSERT(pEntry != NULL);
 
   if (ths->state == TAOS_SYNC_STATE_LEADER) {
-    // ths->pLogStore->appendEntry(ths->pLogStore, pEntry);
+    // append entry
-    ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry);
+    code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry);
+    if (code != 0) {
+      // del resp mgr, call FpCommitCb
+      ASSERT(0);
+      return -1;
+    }
 
-    // start replicate right now!
+    // if mulit replica, start replicate right now
+    if (ths->replicaNum > 1) {
       syncNodeReplicate(ths);
+    }
 
     // pre commit
     SRpcMsg rpcMsg;
     syncEntry2OriginalRpc(pEntry, &rpcMsg);
 
     if (ths->pFsm != NULL) {
-      // if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_SYNC_NOOP) {
       if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) {
         SFsmCbMeta cbMeta = {0};
         cbMeta.index = pEntry->index;
@@ -2439,8 +2325,10 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncI
     }
     rpcFreeCont(rpcMsg.pCont);
 
-    // only myself, maybe commit
+    // if only myself, maybe commit right now
+    if (ths->replicaNum == 1) {
       syncMaybeAdvanceCommitIndex(ths);
+    }
 
   } else {
     // pre commit
@@ -2448,7 +2336,6 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncI
     syncEntry2OriginalRpc(pEntry, &rpcMsg);
 
     if (ths->pFsm != NULL) {
-      // if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_SYNC_NOOP) {
       if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) {
         SFsmCbMeta cbMeta = {0};
         cbMeta.index = pEntry->index;

source/libs/sync/src/syncRaftCfg.c

@@ -101,7 +101,7 @@ cJSON *syncCfg2Json(SSyncCfg *pSyncCfg) {
 
 char *syncCfg2Str(SSyncCfg *pSyncCfg) {
   cJSON *pJson = syncCfg2Json(pSyncCfg);
-  char * serialized = cJSON_Print(pJson);
+  char  *serialized = cJSON_Print(pJson);
   cJSON_Delete(pJson);
   return serialized;
 }
@@ -109,7 +109,7 @@ char *syncCfg2Str(SSyncCfg *pSyncCfg) {
 char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg) {
   if (pSyncCfg != NULL) {
     int32_t len = 512;
-    char *  s = taosMemoryMalloc(len);
+    char   *s = taosMemoryMalloc(len);
     memset(s, 0, len);
 
     snprintf(s, len, "{replica-num:%d, my-index:%d, ", pSyncCfg->replicaNum, pSyncCfg->myIndex);
@@ -205,7 +205,7 @@ cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) {
 
 char *raftCfg2Str(SRaftCfg *pRaftCfg) {
   cJSON *pJson = raftCfg2Json(pRaftCfg);
-  char * serialized = cJSON_Print(pJson);
+  char  *serialized = cJSON_Print(pJson);
   cJSON_Delete(pJson);
   return serialized;
 }
@@ -214,7 +214,16 @@ int32_t raftCfgCreateFile(SSyncCfg *pCfg, SRaftCfgMeta meta, const char *path) {
   ASSERT(pCfg != NULL);
 
   TdFilePtr pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE);
-  ASSERT(pFile != NULL);
+  if (pFile == NULL) {
+    int32_t     err = terrno;
+    const char *errStr = tstrerror(err);
+    int32_t     sysErr = errno;
+    const char *sysErrStr = strerror(errno);
+    sError("create raft cfg file error, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", err, err, errStr, sysErr, sysErrStr);
+    ASSERT(0);
+
+    return -1;
+  }
 
   SRaftCfg raftCfg;
   raftCfg.cfg = *pCfg;
@@ -271,7 +280,7 @@ int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg) {
     (pRaftCfg->configIndexArr)[i] = atoll(pIndex->valuestring);
   }
 
-  cJSON * pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg");
+  cJSON  *pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg");
   int32_t code = syncCfgFromJson(pJsonSyncCfg, &(pRaftCfg->cfg));
   ASSERT(code == 0);
 

source/libs/sync/src/syncRaftLog.c

@@ -17,24 +17,27 @@
 #include "syncRaftCfg.h"
 #include "syncRaftStore.h"
 
-// refactor, log[0 .. n] ==> log[m .. n]
+//-------------------------------
+// log[m .. n]
+
+// public function
 static int32_t   raftLogRestoreFromSnapshot(struct SSyncLogStore* pLogStore, SyncIndex snapshotIndex);
-// static int32_t   raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex beginIndex);
 static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore);
 static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore);
 static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore);
 static bool      raftLogIsEmpty(struct SSyncLogStore* pLogStore);
 static int32_t   raftLogEntryCount(struct SSyncLogStore* pLogStore);
-
 static SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore);
 static SyncTerm  raftLogLastTerm(struct SSyncLogStore* pLogStore);
 static int32_t   raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry);
 static int32_t   raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry);
 static int32_t   raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIndex);
 
+// private function
 static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** ppLastEntry);
 
 //-------------------------------
+// log[0 .. n]
 static SSyncRaftEntry* logStoreGetLastEntry(SSyncLogStore* pLogStore);
 static SyncIndex       logStoreLastIndex(SSyncLogStore* pLogStore);
 static SyncTerm        logStoreLastTerm(SSyncLogStore* pLogStore);
@@ -44,28 +47,86 @@ static int32_t         logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex from
 static int32_t         logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index);
 static SyncIndex       logStoreGetCommitIndex(SSyncLogStore* pLogStore);
 
-// refactor, log[0 .. n] ==> log[m .. n]
+//-------------------------------
-/*
+SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) {
-static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex beginIndex) {
+  SSyncLogStore* pLogStore = taosMemoryMalloc(sizeof(SSyncLogStore));
-  // if beginIndex == 0, donot need call this funciton
+  ASSERT(pLogStore != NULL);
-  ASSERT(beginIndex > 0);
+
+  pLogStore->data = taosMemoryMalloc(sizeof(SSyncLogStoreData));
+  ASSERT(pLogStore->data != NULL);
 
   SSyncLogStoreData* pData = pLogStore->data;
-  SWal*              pWal = pData->pWal;
+  pData->pSyncNode = pSyncNode;
-  sTrace("vgId:%d, reset wal begin index:%ld", pData->pSyncNode->vgId, beginIndex);
+  pData->pWal = pSyncNode->pWal;
+  ASSERT(pData->pWal != NULL);
 
-  pData->beginIndex = beginIndex;
+  taosThreadMutexInit(&(pData->mutex), NULL);
-  walRestoreFromSnapshot(pWal, beginIndex - 1);
+  pData->pWalHandle = walOpenReadHandle(pData->pWal);
-  return 0;
+  ASSERT(pData->pWalHandle != NULL);
+
+  pLogStore->appendEntry = logStoreAppendEntry;
+  pLogStore->getEntry = logStoreGetEntry;
+  pLogStore->truncate = logStoreTruncate;
+  pLogStore->getLastIndex = logStoreLastIndex;
+  pLogStore->getLastTerm = logStoreLastTerm;
+  pLogStore->updateCommitIndex = logStoreUpdateCommitIndex;
+  pLogStore->getCommitIndex = logStoreGetCommitIndex;
+
+  pLogStore->syncLogRestoreFromSnapshot = raftLogRestoreFromSnapshot;
+  pLogStore->syncLogBeginIndex = raftLogBeginIndex;
+  pLogStore->syncLogEndIndex = raftLogEndIndex;
+  pLogStore->syncLogIsEmpty = raftLogIsEmpty;
+  pLogStore->syncLogEntryCount = raftLogEntryCount;
+  pLogStore->syncLogLastIndex = raftLogLastIndex;
+  pLogStore->syncLogLastTerm = raftLogLastTerm;
+  pLogStore->syncLogAppendEntry = raftLogAppendEntry;
+  pLogStore->syncLogGetEntry = raftLogGetEntry;
+  pLogStore->syncLogTruncate = raftLogTruncate;
+  pLogStore->syncLogWriteIndex = raftLogWriteIndex;
+
+  return pLogStore;
 }
-*/
 
+void logStoreDestory(SSyncLogStore* pLogStore) {
+  if (pLogStore != NULL) {
+    SSyncLogStoreData* pData = pLogStore->data;
+
+    taosThreadMutexLock(&(pData->mutex));
+    if (pData->pWalHandle != NULL) {
+      walCloseReadHandle(pData->pWalHandle);
+      pData->pWalHandle = NULL;
+    }
+    taosThreadMutexUnlock(&(pData->mutex));
+    taosThreadMutexDestroy(&(pData->mutex));
+
+    taosMemoryFree(pLogStore->data);
+    taosMemoryFree(pLogStore);
+  }
+}
+
+//-------------------------------
+// log[m .. n]
 static int32_t raftLogRestoreFromSnapshot(struct SSyncLogStore* pLogStore, SyncIndex snapshotIndex) {
   ASSERT(snapshotIndex >= 0);
 
   SSyncLogStoreData* pData = pLogStore->data;
   SWal*              pWal = pData->pWal;
-  walRestoreFromSnapshot(pWal, snapshotIndex);
+  int32_t            code = walRestoreFromSnapshot(pWal, snapshotIndex);
+  if (code != 0) {
+    int32_t     err = terrno;
+    const char* errStr = tstrerror(err);
+    int32_t     sysErr = errno;
+    const char* sysErrStr = strerror(errno);
+
+    char logBuf[128];
+    snprintf(logBuf, sizeof(logBuf),
+             "wal restore from snapshot error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", snapshotIndex, err,
+             err, errStr, sysErr, sysErrStr);
+    syncNodeErrorLog(pData->pSyncNode, logBuf);
+
+    return -1;
+  }
+
   return 0;
 }
 
@@ -91,18 +152,6 @@ static int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore) {
   return count > 0 ? count : 0;
 }
 
-#if 0
-static bool raftLogInRange(struct SSyncLogStore* pLogStore, SyncIndex index) {
-  SyncIndex beginIndex = raftLogBeginIndex(pLogStore);
-  SyncIndex endIndex = raftLogEndIndex(pLogStore);
-  if (index >= beginIndex && index <= endIndex) {
-    return true;
-  } else {
-    return false;
-  }
-}
-#endif
-
 static SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore) {
   SyncIndex          lastIndex;
   SSyncLogStoreData* pData = pLogStore->data;
@@ -119,6 +168,9 @@ static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore) {
   return lastVer + 1;
 }
 
+// if success, return last term
+// if not log, return 0
+// if error, return SYNC_TERM_INVALID
 static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) {
   SSyncLogStoreData* pData = pLogStore->data;
   SWal*              pWal = pData->pWal;
@@ -127,34 +179,18 @@ static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) {
   } else {
     SSyncRaftEntry* pLastEntry;
     int32_t         code = raftLogGetLastEntry(pLogStore, &pLastEntry);
-    ASSERT(code == 0);
+    if (code == 0 && pLastEntry != NULL) {
-    ASSERT(pLastEntry != NULL);
-
       SyncTerm lastTerm = pLastEntry->term;
       taosMemoryFree(pLastEntry);
       return lastTerm;
-  }
-
-  return 0;
-}
-
-/*
-static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) {
-  SyncTerm lastTerm = 0;
-  if (raftLogEntryCount(pLogStore) == 0) {
-    lastTerm = 0;
     } else {
-    SSyncRaftEntry* pLastEntry;
+      return SYNC_TERM_INVALID;
-    int32_t         code = raftLogGetLastEntry(pLogStore, &pLastEntry);
-    ASSERT(code == 0);
-    if (pLastEntry != NULL) {
-      lastTerm = pLastEntry->term;
-      taosMemoryFree(pLastEntry);
     }
   }
-  return lastTerm;
+
+  // can not be here!
+  return SYNC_TERM_INVALID;
 }
-*/
 
 static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
   SSyncLogStoreData* pData = pLogStore->data;
@@ -187,63 +223,21 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr
     ASSERT(0);
   }
 
-  walFsync(pWal, true);
+  // walFsync(pWal, true);
 
+  do {
     char eventLog[128];
     snprintf(eventLog, sizeof(eventLog), "write index:%ld, type:%s,%d, type2:%s,%d", pEntry->index,
              TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType);
     syncNodeEventLog(pData->pSyncNode, eventLog);
+  } while (0);
 
   return code;
 }
 
-#if 0
+// entry found, return 0
-static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry) {
+// entry not found, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST
-  SSyncLogStoreData* pData = pLogStore->data;
+// other error, return -1
-  SWal*              pWal = pData->pWal;
-  int32_t            code;
-
-  *ppEntry = NULL;
-  if (raftLogInRange(pLogStore, index)) {
-    SWalReadHandle* pWalHandle = walOpenReadHandle(pWal);
-    ASSERT(pWalHandle != NULL);
-
-    code = walReadWithHandle(pWalHandle, index);
-    if (code != 0) {
-      int32_t     err = terrno;
-      const char* errStr = tstrerror(err);
-      int32_t     linuxErr = errno;
-      const char* linuxErrMsg = strerror(errno);
-      sError("raftLogGetEntry error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr,
-             linuxErrMsg);
-      ASSERT(0);
-      walCloseReadHandle(pWalHandle);
-      return code;
-    }
-
-    *ppEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen);
-    ASSERT(*ppEntry != NULL);
-    (*ppEntry)->msgType = TDMT_SYNC_CLIENT_REQUEST;
-    (*ppEntry)->originalRpcType = pWalHandle->pHead->head.msgType;
-    (*ppEntry)->seqNum = pWalHandle->pHead->head.syncMeta.seqNum;
-    (*ppEntry)->isWeak = pWalHandle->pHead->head.syncMeta.isWeek;
-    (*ppEntry)->term = pWalHandle->pHead->head.syncMeta.term;
-    (*ppEntry)->index = index;
-    ASSERT((*ppEntry)->dataLen == pWalHandle->pHead->head.bodyLen);
-    memcpy((*ppEntry)->data, pWalHandle->pHead->head.body, pWalHandle->pHead->head.bodyLen);
-
-    // need to hold, do not new every time!!
-    walCloseReadHandle(pWalHandle);
-
-  } else {
-    // index not in range
-    code = 0;
-  }
-
-  return code;
-}
-#endif
-
 static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry) {
   SSyncLogStoreData* pData = pLogStore->data;
   SWal*              pWal = pData->pWal;
@@ -254,6 +248,7 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index,
   // SWalReadHandle* pWalHandle = walOpenReadHandle(pWal);
   SWalReadHandle* pWalHandle = pData->pWalHandle;
   if (pWalHandle == NULL) {
+    terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
     return -1;
   }
 
@@ -325,6 +320,9 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn
   return code;
 }
 
+// entry found, return 0
+// entry not found, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST
+// other error, return -1
 static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** ppLastEntry) {
   SSyncLogStoreData* pData = pLogStore->data;
   SWal*              pWal = pData->pWal;
@@ -336,6 +334,7 @@ static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** pp
     return -1;
   } else {
     SyncIndex lastIndex = raftLogLastIndex(pLogStore);
+    ASSERT(lastIndex >= SYNC_INDEX_BEGIN);
     int32_t code = raftLogGetEntry(pLogStore, lastIndex, ppLastEntry);
     return code;
   }
@@ -343,91 +342,8 @@ static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** pp
   return -1;
 }
 
-/*
-static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** ppLastEntry) {
-  *ppLastEntry = NULL;
-  if (raftLogEntryCount(pLogStore) == 0) {
-    return 0;
-  }
-  SyncIndex lastIndex = raftLogLastIndex(pLogStore);
-  int32_t   code = raftLogGetEntry(pLogStore, lastIndex, ppLastEntry);
-  return code;
-}
-*/
-
-//-------------------------------
-SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) {
-  SSyncLogStore* pLogStore = taosMemoryMalloc(sizeof(SSyncLogStore));
-  ASSERT(pLogStore != NULL);
-
-  pLogStore->data = taosMemoryMalloc(sizeof(SSyncLogStoreData));
-  ASSERT(pLogStore->data != NULL);
-
-  SSyncLogStoreData* pData = pLogStore->data;
-  pData->pSyncNode = pSyncNode;
-  pData->pWal = pSyncNode->pWal;
-  ASSERT(pData->pWal != NULL);
-
-  taosThreadMutexInit(&(pData->mutex), NULL);
-  pData->pWalHandle = walOpenReadHandle(pData->pWal);
-  ASSERT(pData->pWalHandle != NULL);
-
-  /*
-    SyncIndex firstVer = walGetFirstVer(pData->pWal);
-    SyncIndex lastVer = walGetLastVer(pData->pWal);
-    if (firstVer >= 0) {
-      pData->beginIndex = firstVer;
-    } else if (firstVer == -1) {
-      pData->beginIndex = lastVer + 1;
-    } else {
-      ASSERT(0);
-    }
-  */
-
-  pLogStore->appendEntry = logStoreAppendEntry;
-  pLogStore->getEntry = logStoreGetEntry;
-  pLogStore->truncate = logStoreTruncate;
-  pLogStore->getLastIndex = logStoreLastIndex;
-  pLogStore->getLastTerm = logStoreLastTerm;
-  pLogStore->updateCommitIndex = logStoreUpdateCommitIndex;
-  pLogStore->getCommitIndex = logStoreGetCommitIndex;
-
-  // pLogStore->syncLogSetBeginIndex = raftLogSetBeginIndex;
-  pLogStore->syncLogRestoreFromSnapshot = raftLogRestoreFromSnapshot;
-  pLogStore->syncLogBeginIndex = raftLogBeginIndex;
-  pLogStore->syncLogEndIndex = raftLogEndIndex;
-  pLogStore->syncLogIsEmpty = raftLogIsEmpty;
-  pLogStore->syncLogEntryCount = raftLogEntryCount;
-  pLogStore->syncLogLastIndex = raftLogLastIndex;
-  pLogStore->syncLogLastTerm = raftLogLastTerm;
-  pLogStore->syncLogAppendEntry = raftLogAppendEntry;
-  pLogStore->syncLogGetEntry = raftLogGetEntry;
-  pLogStore->syncLogTruncate = raftLogTruncate;
-  pLogStore->syncLogWriteIndex = raftLogWriteIndex;
-
-  // pLogStore->syncLogInRange = raftLogInRange;
-
-  return pLogStore;
-}
-
-void logStoreDestory(SSyncLogStore* pLogStore) {
-  if (pLogStore != NULL) {
-    SSyncLogStoreData* pData = pLogStore->data;
-
-    taosThreadMutexLock(&(pData->mutex));
-    if (pData->pWalHandle != NULL) {
-      walCloseReadHandle(pData->pWalHandle);
-      pData->pWalHandle = NULL;
-    }
-    taosThreadMutexUnlock(&(pData->mutex));
-    taosThreadMutexDestroy(&(pData->mutex));
-
-    taosMemoryFree(pLogStore->data);
-    taosMemoryFree(pLogStore);
-  }
-}
-
 //-------------------------------
+// log[0 .. n]
 int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
   SSyncLogStoreData* pData = pLogStore->data;
   SWal*              pWal = pData->pWal;
@@ -455,7 +371,7 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
     ASSERT(0);
   }
 
-  walFsync(pWal, true);
+  // walFsync(pWal, true);
 
   char eventLog[128];
   snprintf(eventLog, sizeof(eventLog), "old write index:%ld, type:%s,%d, type2:%s,%d", pEntry->index,
@@ -593,55 +509,6 @@ SSyncRaftEntry* logStoreGetLastEntry(SSyncLogStore* pLogStore) {
   return pEntry;
 }
 
-/*
-cJSON* logStore2Json(SSyncLogStore* pLogStore) {
-  char               u64buf[128] = {0};
-  SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data;
-  cJSON*             pRoot = cJSON_CreateObject();
-
-  if (pData != NULL && pData->pWal != NULL) {
-    snprintf(u64buf, sizeof(u64buf), "%p", pData->pSyncNode);
-    cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf);
-    snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal);
-    cJSON_AddStringToObject(pRoot, "pWal", u64buf);
-
-    snprintf(u64buf, sizeof(u64buf), "%ld", pData->beginIndex);
-    cJSON_AddStringToObject(pRoot, "beginIndex", u64buf);
-
-    SyncIndex endIndex = raftLogEndIndex(pLogStore);
-    snprintf(u64buf, sizeof(u64buf), "%ld", endIndex);
-    cJSON_AddStringToObject(pRoot, "endIndex", u64buf);
-
-    int32_t count = raftLogEntryCount(pLogStore);
-    cJSON_AddNumberToObject(pRoot, "entryCount", count);
-
-    snprintf(u64buf, sizeof(u64buf), "%ld", raftLogWriteIndex(pLogStore));
-    cJSON_AddStringToObject(pRoot, "WriteIndex", u64buf);
-
-    snprintf(u64buf, sizeof(u64buf), "%d", raftLogIsEmpty(pLogStore));
-    cJSON_AddStringToObject(pRoot, "IsEmpty", u64buf);
-
-    snprintf(u64buf, sizeof(u64buf), "%ld", raftLogLastIndex(pLogStore));
-    cJSON_AddStringToObject(pRoot, "LastIndex", u64buf);
-    snprintf(u64buf, sizeof(u64buf), "%lu", raftLogLastTerm(pLogStore));
-    cJSON_AddStringToObject(pRoot, "LastTerm", u64buf);
-
-    cJSON* pEntries = cJSON_CreateArray();
-    cJSON_AddItemToObject(pRoot, "pEntries", pEntries);
-
-    for (SyncIndex i = pData->beginIndex; i <= endIndex; ++i) {
-      SSyncRaftEntry* pEntry = logStoreGetEntry(pLogStore, i);
-      cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry));
-      syncEntryDestory(pEntry);
-    }
-  }
-
-  cJSON* pJson = cJSON_CreateObject();
-  cJSON_AddItemToObject(pJson, "SSyncLogStore", pRoot);
-  return pJson;
-}
-*/
-
 cJSON* logStore2Json(SSyncLogStore* pLogStore) {
   char               u64buf[128] = {0};
   SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data;

source/libs/sync/src/syncRequestVote.c

@@ -156,6 +156,10 @@ static bool syncNodeOnRequestVoteLogOK(SSyncNode* pSyncNode, SyncRequestVote* pM
   SyncTerm  myLastTerm = syncNodeGetLastTerm(pSyncNode);
   SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode);
 
+  if (myLastTerm == SYNC_TERM_INVALID) {
+    return false;
+  }
+
   if (pMsg->lastLogTerm > myLastTerm) {
     return true;
   }

source/libs/sync/src/syncSnapshot.c

@@ -508,31 +508,50 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) {
   } while (0);
 }
 
-static void snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) {
+static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) {
   ASSERT(pMsg->seq == SYNC_SNAPSHOT_SEQ_END);
 
-  if (pReceiver->pWriter != NULL) {
   int32_t code = 0;
+  if (pReceiver->pWriter != NULL) {
+    // write data
     if (pMsg->dataLen > 0) {
       code = pReceiver->pSyncNode->pFsm->FpSnapshotDoWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, pMsg->data,
                                                            pMsg->dataLen);
-      ASSERT(code == 0);
+      if (code != 0) {
+        syncNodeErrorLog(pReceiver->pSyncNode, "snapshot write error");
+        return -1;
+      }
     }
 
-    code = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, true);
+    // reset wal
-    ASSERT(code == 0);
+    code =
-    pReceiver->pWriter = NULL;
+        pReceiver->pSyncNode->pLogStore->syncLogRestoreFromSnapshot(pReceiver->pSyncNode->pLogStore, pMsg->lastIndex);
+    if (code != 0) {
+      syncNodeErrorLog(pReceiver->pSyncNode, "wal restore from snapshot error");
+      return -1;
     }
 
-  pReceiver->ack = SYNC_SNAPSHOT_SEQ_END;
-
     // update commit index
     if (pReceiver->snapshot.lastApplyIndex > pReceiver->pSyncNode->commitIndex) {
       pReceiver->pSyncNode->commitIndex = pReceiver->snapshot.lastApplyIndex;
     }
 
-  // reset wal
+    // stop writer
-  pReceiver->pSyncNode->pLogStore->syncLogRestoreFromSnapshot(pReceiver->pSyncNode->pLogStore, pMsg->lastIndex);
+    code = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, true);
+    if (code != 0) {
+      syncNodeErrorLog(pReceiver->pSyncNode, "snapshot stop writer true error");
+      ASSERT(0);
+      return -1;
+    }
+    pReceiver->pWriter = NULL;
+
+    // update progress
+    pReceiver->ack = SYNC_SNAPSHOT_SEQ_END;
+
+  } else {
+    syncNodeErrorLog(pReceiver->pSyncNode, "snapshot stop writer true error");
+    return -1;
+  }
 
   // event log
   do {
@@ -542,6 +561,8 @@ static void snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnapsho
     syncNodeEventLog(pReceiver->pSyncNode, eventLog);
     taosMemoryFree(eventLog);
   } while (0);
+
+  return 0;
 }
 
 static void snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) {
@@ -642,6 +663,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
   // get receiver
   SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver;
   bool                   needRsp = false;
+  int32_t                code = 0;
 
   // state, term, seq/ack
   if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) {
@@ -653,8 +675,10 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
 
       } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) {
         // end, finish FSM
-        snapshotReceiverFinish(pReceiver, pMsg);
+        code = snapshotReceiverFinish(pReceiver, pMsg);
+        if (code == 0) {
           snapshotReceiverStop(pReceiver);
+        }
         needRsp = true;
 
         // maybe update lastconfig
@@ -772,6 +796,7 @@ int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) {
         snapshotReSend(pSender);
 
       } else {
+        // error log
         do {
           char logBuf[96];
           snprintf(logBuf, sizeof(logBuf), "snapshot sender recv error ack:%d, my seq:%d", pMsg->ack, pSender->seq);

source/libs/transport/inc/transComm.h

@@ -296,7 +296,6 @@ void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STra
 void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp);
 void transSendResponse(const STransMsg* msg);
 void transRegisterMsg(const STransMsg* msg);
-int  transGetConnInfo(void* thandle, STransHandleInfo* pInfo);
 void transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn);
 
 void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle);

source/libs/transport/src/trans.c

@@ -83,6 +83,7 @@ void rpcClose(void* arg) {
   SRpcInfo* pRpc = (SRpcInfo*)arg;
   (*taosCloseHandle[pRpc->connType])(pRpc->tcphandle);
   taosMemoryFree(pRpc);
+  tInfo("finish to close rpc");
 
   return;
 }
@@ -140,7 +141,9 @@ void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) {
 }
 
 void    rpcSendResponse(const SRpcMsg* pMsg) { transSendResponse(pMsg); }
-int32_t rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return transGetConnInfo((void*)thandle, pInfo); }
+
+int32_t rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return 0; }
+
 
 void rpcRefHandle(void* handle, int8_t type) {
   assert(type == TAOS_CONN_SERVER || type == TAOS_CONN_CLIENT);

source/libs/transport/src/transCli.c

@@ -317,8 +317,9 @@ void cliHandleResp(SCliConn* conn) {
 
   if (CONN_NO_PERSIST_BY_APP(conn)) {
     pMsg = transQueuePop(&conn->cliMsgs);
-    pCtx = pMsg->ctx;
+
-    transMsg.info.ahandle = pCtx->ahandle;
+    pCtx = pMsg ? pMsg->ctx : NULL;
+    transMsg.info.ahandle = pCtx ? pCtx->ahandle : NULL;
     tDebug("%s conn %p get ahandle %p, persist: 0", CONN_GET_INST_LABEL(conn), conn, transMsg.info.ahandle);
   } else {
     uint64_t ahandle = (uint64_t)pHead->ahandle;
@@ -465,7 +466,6 @@ void* destroyConnPool(void* pool) {
   while (connList != NULL) {
     while (!QUEUE_IS_EMPTY(&connList->conn)) {
       queue*    h = QUEUE_HEAD(&connList->conn);
-      // QUEUE_REMOVE(h);
       SCliConn* c = QUEUE_DATA(h, SCliConn, conn);
       cliDestroyConn(c, true);
     }
@@ -502,6 +502,7 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) {
 static void allocConnRef(SCliConn* conn, bool update) {
   if (update) {
     transRemoveExHandle(conn->refId);
+    conn->refId = -1;
   }
   SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle));
   exh->handle = conn;
@@ -601,8 +602,10 @@ static void cliDestroyConn(SCliConn* conn, bool clear) {
   QUEUE_REMOVE(&conn->conn);
   QUEUE_INIT(&conn->conn);
   transRemoveExHandle(conn->refId);
+  conn->refId = -1;
+
   if (clear) {
-    if (uv_is_active((uv_handle_t*)conn->stream)) {
+    if (!uv_is_closing((uv_handle_t*)conn->stream)) {
       uv_close((uv_handle_t*)conn->stream, cliDestroy);
     } else {
       cliDestroy((uv_handle_t*)conn->stream);
@@ -610,8 +613,14 @@ static void cliDestroyConn(SCliConn* conn, bool clear) {
   }
 }
 static void cliDestroy(uv_handle_t* handle) {
+  if (uv_handle_get_type(handle) != UV_TCP || handle->data == NULL) {
+    return;
+  }
+
   SCliConn* conn = handle->data;
+  transRemoveExHandle(conn->refId);
   taosMemoryFree(conn->ip);
+  conn->stream->data = NULL;
   taosMemoryFree(conn->stream);
   transCtxCleanup(&conn->ctx);
   transQueueDestroy(&conn->cliMsgs);
@@ -969,7 +978,7 @@ void cliSendQuit(SCliThrd* thrd) {
 }
 void cliWalkCb(uv_handle_t* handle, void* arg) {
   if (!uv_is_closing(handle)) {
-    uv_close(handle, NULL);
+    uv_close(handle, cliDestroy);
   }
 }
 

source/libs/transport/src/transSvr.c

@@ -139,7 +139,6 @@ static void uvHandleResp(SSvrMsg* msg, SWorkThrd* thrd);
 static void uvHandleRegister(SSvrMsg* msg, SWorkThrd* thrd);
 static void (*transAsyncHandle[])(SSvrMsg* msg, SWorkThrd* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease,
                                                                     uvHandleRegister, NULL};
-
 static void uvDestroyConn(uv_handle_t* handle);
 
 // server and worker thread
@@ -332,7 +331,6 @@ void uvOnTimeoutCb(uv_timer_t* handle) {
 
 void uvOnSendCb(uv_write_t* req, int status) {
   SSvrConn* conn = req->data;
-  // transClearBuffer(&conn->readBuf);
   if (status == 0) {
     tTrace("conn %p data already was written on stream", conn);
     if (!transQueueEmpty(&conn->srvMsgs)) {
@@ -362,6 +360,7 @@ void uvOnSendCb(uv_write_t* req, int status) {
         }
       }
     }
+    transUnrefSrvHandle(conn);
   } else {
     tError("conn %p failed to write data, %s", conn, uv_err_name(status));
     conn->broken = true;
@@ -424,11 +423,15 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) {
 }
 
 static void uvStartSendRespInternal(SSvrMsg* smsg) {
+  SSvrConn* pConn = smsg->pConn;
+  if (pConn->broken) {
+    return;
+  }
+
   uv_buf_t wb;
   uvPrepareSendData(smsg, &wb);
 
-  SSvrConn* pConn = smsg->pConn;
+  transRefSrvHandle(pConn);
-  // uv_timer_stop(&pConn->pTimer);
   uv_write(&pConn->pWriter, (uv_stream_t*)pConn->pTcp, &wb, 1, uvOnSendCb);
 }
 static void uvStartSendResp(SSvrMsg* smsg) {
@@ -769,12 +772,9 @@ static void destroyConn(SSvrConn* conn, bool clear) {
 
   transDestroyBuffer(&conn->readBuf);
   if (clear) {
-    if (uv_is_active((uv_handle_t*)conn->pTcp)) {
+    if (!uv_is_closing((uv_handle_t*)conn->pTcp)) {
       tTrace("conn %p to be destroyed", conn);
-      // uv_shutdown_t* req = taosMemoryMalloc(sizeof(uv_shutdown_t));
       uv_close((uv_handle_t*)conn->pTcp, uvDestroyConn);
-      // uv_close(conn->pTcp)
-      // uv_shutdown(req, (uv_stream_t*)conn->pTcp, uvShutDownCb);
     } else {
       uvDestroyConn((uv_handle_t*)conn->pTcp);
     }

source/os/src/osSocket.c

@@ -925,10 +925,24 @@ uint32_t taosGetIpv4FromFqdn(const char *fqdn) {
 }
 
 int32_t taosGetFqdn(char *fqdn) {
+#ifdef WINDOWS
+  // Initialize Winsock
+  WSADATA wsaData;
+  int     iResult;
+  iResult = WSAStartup(MAKEWORD(2, 2), &wsaData);
+  if (iResult != 0) {
+    // printf("WSAStartup failed: %d\n", iResult);
+    return 1;
+  }
+#endif
   char hostname[1024];
   hostname[1023] = '\0';
   if (gethostname(hostname, 1023) == -1) {
-    // printf("failed to get hostname, reason:%s", strerror(errno));
+#ifdef WINDOWS
+    printf("failed to get hostname, reason:%s", strerror(WSAGetLastError()));
+#else
+    printf("failed to get hostname, reason:%s", strerror(errno));
+#endif
     assert(0);
     return -1;
   }

tests/pytest/a.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+for i in {1..100}
+do 
+	echo $i
+        python3 ./test.py -f query/nestedQuery/nestedQuery_datacheck.py >>log 2>&1
+        if [ $? -eq 0 ] 
+	then
+		echo success
+	else
+		echo failed
+		break
+	fi
+done

tests/script/sh/exec.bat

@@ -6,6 +6,7 @@ if %1 == -n set NODE_NAME=%2
 if %1 == -s set EXEC_OPTON=%2
 if %3 == -n set NODE_NAME=%4
 if %3 == -s set EXEC_OPTON=%4
+if "%5" == "-x" set EXEC_SIGNAL=%6
 
 rem echo NODE_NAME:  %NODE_NAME%
 rem echo NODE:       %EXEC_OPTON%
@@ -39,7 +40,7 @@ rem echo TAOS_LOG:   %TAOS_LOG%
 if %EXEC_OPTON% == start (
   rm -rf %TAOS_LOG%
   echo start %TAOSD% -c %CFG_DIR%
-  start %TAOSD% -c %CFG_DIR%
+  mintty -h never %TAOSD% -c %CFG_DIR%
   set /a check_num=0
 :check_online
   sleep 1
@@ -58,13 +59,25 @@ if %EXEC_OPTON% == stop (
   rem echo wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" list INSTANCE
   rem wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" call terminate > NUL 2>&1
   
-  for /f "tokens=1 skip=1" %%A in (
+  for /f "tokens=2" %%A in ('wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" get processId ^| xargs echo') do (
-    'wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" get processId '
+    for /f "tokens=1" %%B in ('ps ^| grep %%A') do (
-  ) do (
+      if "%EXEC_SIGNAL%" == "SIGKILL" (
-    rem echo taskkill /IM %%A 
+        kill -9 %%B
-    taskkill /IM %%A > NUL 2>&1
+      ) else (
+        kill -INT %%B
+        call :check_offline
+      )
+    )
     goto :finish
   ) 
 )
-
 :finish
+goto :eof
+
+:check_offline
+sleep 1
+for /f "tokens=2" %%C in ('wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" get processId ^| xargs echo') do (
+  echo check taosd offline
+  goto :check_offline
+)
+goto :eof

tests/script/test-all.bat

@@ -62,7 +62,5 @@ goto :eof
 
 :CheckSkipCase
 set skipCase=false
-if "%*" == "./test.sh -f tsim/query/scalarFunction.sim" ( set skipCase=true )
+@REM if "%*" == "./test.sh -f tsim/query/scalarFunction.sim" ( set skipCase=true )
-if "%*" == "./test.sh -f tsim/stream/distributeInterval0.sim" ( set skipCase=true )
-if "%*" == "./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim" ( set skipCase=true )
 :goto eof

tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim

@@ -124,7 +124,7 @@ if $data(5)[3] != 3 then
   return -1
 endi
 
-#sql reset query cache
+sql reset query cache
 
 print =============== step4: select data
 sql show d1.tables

tests/script/tsim/tmq/basic2Of2ConsOverlap.sim

@@ -134,6 +134,8 @@ endi
 $totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb
 $sumOfMsgCnt = $data[0][2] + $data[1][2]
 if $sumOfMsgCnt != $totalMsgCons then
+  print total: $totalMsgCons
+  print sum: $sumOfMsgCnt
   return -1
 endi
 

tests/system-test/2-query/json_tag.py

@@ -199,7 +199,7 @@ class TDTestCase:
         tdSql.checkData(0, 0, "true")
         # test select json tag->'key', value is null
         tdSql.query("select jtag->'tag1' from jsons1_4")
-        tdSql.checkData(0, 0, "null")
+        tdSql.checkData(0, 0, None)
         # test select json tag->'key', value is double
         tdSql.query("select jtag->'tag1' from jsons1_5")
         tdSql.checkData(0, 0, "1.232000000")