Merge branch '3.0' into feature/3.0_liaohj

.gitignore

@@ -24,7 +24,6 @@ mac/
 *.orig
 src/connector/nodejs/node_modules/
 src/connector/nodejs/out/
-tests/test/
 tests/taoshebei/
 tests/taoscsv/
 tests/taosdalipu/

.gitmodules

@@ -10,10 +10,9 @@
 [submodule "deps/TSZ"]
 	path = deps/TSZ
 	url = https://github.com/taosdata/TSZ.git
-[submodule "tests"]
-	path = tests
-	url = https://github.com/taosdata/tests
-	branch = 3.0
 [submodule "examples/rust"]
 	path = examples/rust
 	url = https://github.com/songtianyi/tdengine-rust-bindings.git
+[submodule "tools/taos-tools"]
+	path = tools/taos-tools
+	url = https://github.com/taosdata/taos-tools

CMakeLists.txt

@@ -6,9 +6,36 @@ project(
     DESCRIPTION "An open-source big data platform designed and optimized for the Internet of Things(IOT)"
 )
 
+IF ("${BUILD_TOOLS}" STREQUAL "")
+  IF (TD_LINUX)
+    IF (TD_ARM_32)
+      SET(BUILD_TOOLS "false")
+    ELSEIF (TD_ARM_64)
+      SET(BUILD_TOOLS "false")
+    ELSE ()
+      SET(BUILD_TOOLS "false")
+    ENDIF ()
+  ELSEIF (TD_DARWIN)
+    SET(BUILD_TOOLS "false")
+  ELSE ()
+    SET(BUILD_TOOLS "false")
+  ENDIF ()
+ENDIF ()
+
+IF ("${BUILD_TOOLS}" MATCHES "false")
+    MESSAGE("${Yellow} Will _not_ build taos_tools! ${ColourReset}")
+    SET(TD_TAOS_TOOLS FALSE)
+ELSE ()
+    MESSAGE("")
+    MESSAGE("${Green} Will build taos_tools! ${ColourReset}")
+    MESSAGE("")
+    SET(TD_TAOS_TOOLS TRUE)
+ENDIF ()
+
 set(CMAKE_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/cmake")
 set(CMAKE_CONTRIB_DIR "${CMAKE_SOURCE_DIR}/contrib")
 include(${CMAKE_SUPPORT_DIR}/cmake.options)
+include(${CMAKE_SUPPORT_DIR}/cmake.version)
 
 SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -fPIC -gdwarf-2 -msse4.2 -mfma -g3")
 SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -fPIC -gdwarf-2 -msse4.2 -mfma -g3")

Jenkinsfile2

@@ -74,37 +74,9 @@ def pre_test(){
     git pull >/dev/null
     git fetch origin +refs/pull/${CHANGE_ID}/merge
     git checkout -qf FETCH_HEAD
-    git submodule update --init --recursive --remote
+    git submodule update --init --recursive
     '''
-    script {
-      if (env.CHANGE_TARGET == 'master') {
-        sh '''
-        cd ${WKCT}
-        git checkout master
-        '''
-        }
-      else if(env.CHANGE_TARGET == '2.0'){
-        sh '''
-        cd ${WKCT}
-        git checkout 2.0
-        '''
-      }
-      else if(env.CHANGE_TARGET == '3.0'){
-        sh '''
-        cd ${WKCT}
-        git checkout 3.0
-        '''
-      } 
-      else{
-        sh '''
-        cd ${WKCT}
-        git checkout develop
-        '''
-      }
-    }
     sh'''
-    cd ${WKCT}
-    git pull >/dev/null
     cd ${WKC}
     export TZ=Asia/Harbin
     date
@@ -123,7 +95,6 @@ pipeline {
   environment{
       WK = '/var/lib/jenkins/workspace/TDinternal'
       WKC= '/var/lib/jenkins/workspace/TDengine'
-      WKCT= '/var/lib/jenkins/workspace/TDengine/tests'
   }
   stages {
       stage('pre_build'){

cmake/cmake.version

@@ -0,0 +1,77 @@
+
+IF (DEFINED VERNUMBER)
+  SET(TD_VER_NUMBER ${VERNUMBER})
+ELSE ()
+  SET(TD_VER_NUMBER "3.0.0.0")
+ENDIF ()
+
+IF (DEFINED VERCOMPATIBLE)
+  SET(TD_VER_COMPATIBLE ${VERCOMPATIBLE})
+ELSE ()
+  SET(TD_VER_COMPATIBLE "3.0.0.0")
+ENDIF ()
+
+find_program(HAVE_GIT NAMES git)
+
+IF (DEFINED GITINFO)
+  SET(TD_VER_GIT ${GITINFO})
+ELSEIF (HAVE_GIT)
+  execute_process(COMMAND git log -1 --format=%H WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE GIT_COMMITID)
+  #message(STATUS "git log result:${GIT_COMMITID}")
+  IF (GIT_COMMITID)
+    string (REGEX REPLACE "[\n\t\r]" "" GIT_COMMITID ${GIT_COMMITID})
+    SET(TD_VER_GIT ${GIT_COMMITID})
+  ELSE ()
+    message(STATUS "not a git repository")
+    SET(TD_VER_GIT "no git commit id")
+  ENDIF ()
+ELSE ()
+  message(STATUS "no git cmd")
+  SET(TD_VER_GIT "no git commit id")
+ENDIF ()
+
+IF (DEFINED VERDATE)
+  SET(TD_VER_DATE ${VERDATE})
+ELSE ()
+  STRING(TIMESTAMP TD_VER_DATE "%Y-%m-%d %H:%M:%S") 
+ENDIF ()
+
+IF (DEFINED VERTYPE)
+  SET(TD_VER_VERTYPE ${VERTYPE})
+ELSE ()
+  SET(TD_VER_VERTYPE "stable")
+ENDIF ()
+
+IF (DEFINED CPUTYPE)
+  SET(TD_VER_CPUTYPE ${CPUTYPE})
+ELSE ()
+  IF (TD_WINDOWS_32)
+    SET(TD_VER_CPUTYPE "x86")
+  ELSEIF (TD_LINUX_32)
+    SET(TD_VER_CPUTYPE "x86")  
+  ELSEIF (TD_ARM_32)
+    SET(TD_VER_CPUTYPE "x86")  
+  ELSEIF (TD_MIPS_32)
+    SET(TD_VER_CPUTYPE "x86")  
+  ELSE ()
+    SET(TD_VER_CPUTYPE "x64")
+  ENDIF ()
+ENDIF ()
+
+IF (DEFINED OSTYPE)
+  SET(TD_VER_OSTYPE ${OSTYPE})
+ELSE ()
+  SET(TD_VER_OSTYPE "Linux")
+ENDIF ()
+
+MESSAGE(STATUS "============= compile version parameter information start ============= ")
+MESSAGE(STATUS "ver number:" ${TD_VER_NUMBER})
+MESSAGE(STATUS "compatible ver number:" ${TD_VER_COMPATIBLE})
+MESSAGE(STATUS "communit commit id:" ${TD_VER_GIT})
+MESSAGE(STATUS "build date:" ${TD_VER_DATE})
+MESSAGE(STATUS "ver type:" ${TD_VER_VERTYPE})
+MESSAGE(STATUS "ver cpu:" ${TD_VER_CPUTYPE})
+MESSAGE(STATUS "os type:" ${TD_VER_OSTYPE})
+MESSAGE(STATUS "============= compile version parameter information end  ============= ")
+
+STRING(REPLACE "." "_" TD_LIB_VER_NUMBER ${TD_VER_NUMBER})

contrib/test/craft/raftMain.c

@@ -377,7 +377,7 @@ void printConf(SRaftServerConfig *pConf) {
 
 
 int main(int argc, char **argv) { 
-	srand(time(NULL));
+	taosSeedRand(time(NULL));
 	int32_t ret;
 
 	exe_name = argv[0];

contrib/test/craft/simulate_vnode.c

@@ -132,7 +132,7 @@ static void proposeValue(struct raft *r) {
     buf.base = raft_malloc(buf.len);
 
     // mock ts value
-    int vid = rand() % VNODE_COUNT;
+    int vid = taosRand() % VNODE_COUNT;
     snprintf(buf.base, buf.len, "%d:value_%ld", vid, time(NULL));
 
 	printf("propose value: %s \n", (char*)buf.base);
@@ -174,7 +174,7 @@ void usage() {
 }
 
 int main(int argc, char **argv) {
-	srand(time(NULL));
+	taosSeedRand(time(NULL));
 
 	exe_name = argv[0];
 	if (argc < 2) {

examples/c/schemaless.c

@@ -19,7 +19,7 @@ void shuffle(char**lines, size_t n)
     size_t i;
     for (i = 0; i < n - 1; i++)
     {
-      size_t j = i + rand() / (RAND_MAX / (n - i) + 1);
+      size_t j = i + taosRand() / (RAND_MAX / (n - i) + 1);
       char* t = lines[j];
       lines[j] = lines[i];
       lines[i] = t;

include/client/taos.h

@@ -256,6 +256,10 @@ DLL_EXPORT void           tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_co
 void    tmqShowMsg(tmq_message_t *tmq_message);
 int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message);
 
+typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code);
+
+DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT* stmt);
+
 #ifdef __cplusplus
 }
 #endif

include/common/taosdef.h

@@ -56,8 +56,15 @@ typedef enum {
   TSDB_STATIS_NONE = 1,  // statis part not exist
 } ETsdbStatisStatus;
 
+typedef enum {
+  TSDB_SMA_STAT_OK = 0,       // ready to provide service
+  TSDB_SMA_STAT_EXPIRED = 1,  // not ready or expired
+} ETsdbSmaStat;
+
 extern char *qtypeStr[];
 
+#define TSDB_PORT_HTTP      11
+
 #ifdef __cplusplus
 }
 #endif

include/common/tcommon.h

@@ -32,7 +32,8 @@ enum {
 };
 
 enum {
-  TMQ_MSG_TYPE__POLL_RSP = 0,
+  TMQ_MSG_TYPE__DUMMY = 0,
+  TMQ_MSG_TYPE__POLL_RSP,
   TMQ_MSG_TYPE__EP_RSP,
 };
 
@@ -134,7 +135,7 @@ static FORCE_INLINE void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock)
   return (void*)buf;
 }
 
-static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp* pRsp) {
+static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp) {
   int32_t tlen = 0;
   int32_t sz = 0;
   tlen += taosEncodeFixedI64(buf, pRsp->consumerId);
@@ -155,7 +156,7 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp
   return tlen;
 }
 
-static FORCE_INLINE void* tDecodeSMqConsumeRsp(void* buf, SMqConsumeRsp* pRsp) {
+static FORCE_INLINE void* tDecodeSMqPollRsp(void* buf, SMqPollRsp* pRsp) {
   int32_t sz;
   buf = taosDecodeFixedI64(buf, &pRsp->consumerId);
   buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
@@ -193,7 +194,7 @@ static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) {
   // tfree(pBlock);
 }
 
-static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqConsumeRsp* pRsp) {
+static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqPollRsp* pRsp) {
   if (pRsp->schemas) {
     if (pRsp->schemas->nCols) {
       tfree(pRsp->schemas->pSchema);

include/common/tglobal.h

@@ -58,6 +58,7 @@ extern int32_t  tsMonitorInterval;
 extern char     tsMonitorFqdn[];
 extern uint16_t tsMonitorPort;
 extern int32_t  tsMonitorMaxLogs;
+extern bool     tsMonitorComp;
 
 // query buffer management
 extern int32_t tsQueryBufferSize;  // maximum allowed usage buffer size in MB for each data node during query processing
@@ -83,7 +84,6 @@ extern int64_t tsMaxRetentWindow;
 extern char version[];
 extern char compatible_version[];
 extern char gitinfo[];
-extern char gitinfoOfInternal[];
 extern char buildinfo[];
 
 // lossy

include/common/tmsg.h

@@ -70,7 +70,7 @@ typedef uint16_t tmsg_t;
 
 typedef enum {
   HEARTBEAT_TYPE_MQ = 0,
-  HEARTBEAT_TYPE_QUERY = 1,
+  HEARTBEAT_TYPE_QUERY,
   // types can be added here
   //
   HEARTBEAT_TYPE_MAX
@@ -671,6 +671,11 @@ typedef struct {
   int64_t totalStorage;
   int64_t compStorage;
   int64_t pointsWritten;
+  int64_t numOfSelectReqs;
+  int64_t numOfInsertReqs;
+  int64_t numOfInsertSuccessReqs;
+  int64_t numOfBatchInsertReqs;
+  int64_t numOfBatchInsertSuccessReqs;
 } SVnodeLoad;
 
 typedef struct {
@@ -790,7 +795,7 @@ typedef struct SVgroupInfo {
   int32_t  vgId;
   uint32_t hashBegin;
   uint32_t hashEnd;
-  SEpSet   epset;
+  SEpSet   epSet;
 } SVgroupInfo;
 
 typedef struct {
@@ -1103,18 +1108,34 @@ typedef struct {
   char*  sql;
   char*  physicalPlan;
   char*  logicalPlan;
-} SMCreateTopicReq;
+} SCMCreateStreamReq;
 
-int32_t tSerializeMCreateTopicReq(void* buf, int32_t bufLen, const SMCreateTopicReq* pReq);
+typedef struct {
-int32_t tDeserializeSMCreateTopicReq(void* buf, int32_t bufLen, SMCreateTopicReq* pReq);
+  int64_t streamId;
-void    tFreeSMCreateTopicReq(SMCreateTopicReq* pReq);
+} SCMCreateStreamRsp;
+
+int32_t tSerializeSCMCreateStreamReq(void* buf, int32_t bufLen, const SCMCreateStreamReq* pReq);
+int32_t tDeserializeSCMCreateStreamReq(void* buf, int32_t bufLen, SCMCreateStreamReq* pReq);
+void    tFreeSCMCreateStreamReq(SCMCreateStreamReq* pReq);
+
+typedef struct {
+  char   name[TSDB_TOPIC_FNAME_LEN];
+  int8_t igExists;
+  char*  sql;
+  char*  physicalPlan;
+  char*  logicalPlan;
+} SCMCreateTopicReq;
+
+int32_t tSerializeSCMCreateTopicReq(void* buf, int32_t bufLen, const SCMCreateTopicReq* pReq);
+int32_t tDeserializeSCMCreateTopicReq(void* buf, int32_t bufLen, SCMCreateTopicReq* pReq);
+void    tFreeSCMCreateTopicReq(SCMCreateTopicReq* pReq);
 
 typedef struct {
   int64_t topicId;
-} SMCreateTopicRsp;
+} SCMCreateTopicRsp;
 
-int32_t tSerializeSMCreateTopicRsp(void* buf, int32_t bufLen, const SMCreateTopicRsp* pRsp);
+int32_t tSerializeSCMCreateTopicRsp(void* buf, int32_t bufLen, const SCMCreateTopicRsp* pRsp);
-int32_t tDeserializeSMCreateTopicRsp(void* buf, int32_t bufLen, SMCreateTopicRsp* pRsp);
+int32_t tDeserializeSCMCreateTopicRsp(void* buf, int32_t bufLen, SCMCreateTopicRsp* pRsp);
 
 typedef struct {
   int32_t topicNum;
@@ -1383,7 +1404,7 @@ typedef struct {
 typedef struct SMqCMGetSubEpReq {
   int64_t consumerId;
   int32_t epoch;
-  char    cgroup[TSDB_CONSUMER_GROUP_LEN];
+  char    cgroup[TSDB_CGROUP_LEN];
 } SMqCMGetSubEpReq;
 
 static FORCE_INLINE int32_t tEncodeSMsgHead(void** buf, const SMsgHead* pMsg) {
@@ -1683,7 +1704,7 @@ typedef struct {
   int32_t vgId;
   int64_t consumerId;
   char    topicName[TSDB_TOPIC_FNAME_LEN];
-  char    cgroup[TSDB_CONSUMER_GROUP_LEN];
+  char    cgroup[TSDB_CGROUP_LEN];
   char*   sql;
   char*   logicalPlan;
   char*   physicalPlan;
@@ -1746,7 +1767,7 @@ typedef struct {
   int32_t  vgId;
   int64_t  consumerId;
   char     topicName[TSDB_TOPIC_FNAME_LEN];
-  char     cgroup[TSDB_CONSUMER_GROUP_LEN];
+  char     cgroup[TSDB_CGROUP_LEN];
 } SMqSetCVgRsp;
 
 typedef struct {
@@ -1754,14 +1775,14 @@ typedef struct {
   int32_t  vgId;
   int64_t  consumerId;
   char     topicName[TSDB_TOPIC_FNAME_LEN];
-  char     cgroup[TSDB_CONSUMER_GROUP_LEN];
+  char     cgroup[TSDB_CGROUP_LEN];
 } SMqMVRebRsp;
 
 typedef struct {
   int32_t vgId;
   int64_t offset;
   char    topicName[TSDB_TOPIC_FNAME_LEN];
-  char    cgroup[TSDB_CONSUMER_GROUP_LEN];
+  char    cgroup[TSDB_CGROUP_LEN];
 } SMqOffset;
 
 typedef struct {
@@ -1837,6 +1858,243 @@ static FORCE_INLINE void* tDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW)
   }
   return buf;
 }
+typedef enum {
+  TD_TIME_UNIT_UNKNOWN = -1,
+  TD_TIME_UNIT_YEAR = 0,
+  TD_TIME_UNIT_SEASON = 1,
+  TD_TIME_UNIT_MONTH = 2,
+  TD_TIME_UNIT_WEEK = 3,
+  TD_TIME_UNIT_DAY = 4,
+  TD_TIME_UNIT_HOUR = 5,
+  TD_TIME_UNIT_MINUTE = 6,
+  TD_TIME_UNIT_SEC = 7,
+  TD_TIME_UNIT_MILLISEC = 8,
+  TD_TIME_UNIT_MICROSEC = 9,
+  TD_TIME_UNIT_NANOSEC = 10
+} ETDTimeUnit;
+
+typedef struct {
+  uint16_t  funcId;
+  uint16_t  nColIds;
+  col_id_t* colIds;  // sorted colIds
+} SFuncColIds;
+
+typedef struct {
+  uint8_t      version;  // for compatibility
+  uint8_t      intervalUnit;
+  uint8_t      slidingUnit;
+  char         indexName[TSDB_INDEX_NAME_LEN];
+  char         timezone[TD_TIMEZONE_LEN];
+  uint16_t     nFuncColIds;
+  uint16_t     tagsFilterLen;
+  tb_uid_t     tableUid;  // super/common table uid
+  int64_t      interval;
+  int64_t      sliding;
+  SFuncColIds* funcColIds;  // sorted funcIds
+  char*        tagsFilter;
+} STSma;  // Time-range-wise SMA
+
+typedef struct {
+  int64_t ver;  // use a general definition
+  STSma   tSma;
+} SVCreateTSmaReq;
+
+typedef struct {
+  int8_t      type;                                // 0 status report, 1 update data
+  char        indexName[TSDB_INDEX_NAME_LEN];  //
+  STimeWindow windows;
+} STSmaMsg;
+
+typedef struct {
+  int64_t ver;  // use a general definition
+  char    indexName[TSDB_INDEX_NAME_LEN];
+} SVDropTSmaReq;
+typedef struct {
+} SVCreateTSmaRsp, SVDropTSmaRsp;
+
+int32_t tSerializeSVCreateTSmaReq(void** buf, SVCreateTSmaReq* pReq);
+void*   tDeserializeSVCreateTSmaReq(void* buf, SVCreateTSmaReq* pReq);
+int32_t tSerializeSVDropTSmaReq(void** buf, SVDropTSmaReq* pReq);
+void*   tDeserializeSVDropTSmaReq(void* buf, SVDropTSmaReq* pReq);
+
+typedef struct {
+  STimeWindow tsWindow;     // [skey, ekey]
+  uint64_t    tableUid;     // sub/common table uid
+  int32_t     numOfBlocks;  // number of sma blocks for each column, total number is numOfBlocks*numOfColId
+  int32_t     dataLen;      // total data length
+  col_id_t*   colIds;       // e.g. 2,4,9,10
+  col_id_t    numOfColIds;  // e.g. 4
+  char        data[];       // the sma blocks
+} STSmaData;
+
+// TODO: move to the final location afte schema of STSma/STSmaData defined
+static FORCE_INLINE void tdDestroySmaData(STSmaData* pSmaData) {
+  if (pSmaData) {
+    if (pSmaData->colIds) {
+      tfree(pSmaData->colIds);
+    }
+    tfree(pSmaData);
+  }
+}
+
+// RSma: Rollup SMA
+typedef struct {
+  int64_t  interval;
+  int32_t  retention;  // unit: day
+  uint16_t days;       // unit: day
+  int8_t   intervalUnit;
+} SSmaParams;
+
+typedef struct {
+  STSma   tsma;
+  float   xFilesFactor;
+  SArray* smaParams;  // SSmaParams
+} SRSma;
+
+typedef struct {
+  uint32_t number;
+  STSma*   tSma;
+} STSmaWrapper;
+
+static FORCE_INLINE void tdDestroyTSma(STSma* pSma) {
+  if (pSma) {
+    if (pSma->funcColIds != NULL) {
+      for (uint16_t i = 0; i < pSma->nFuncColIds; ++i) {
+        tfree((pSma->funcColIds + i)->colIds);
+      }
+      tfree(pSma->funcColIds);
+    }
+
+    tfree(pSma->tagsFilter);
+  }
+}
+
+static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW) {
+  if (pSW) {
+    if (pSW->tSma) {
+      for (uint32_t i = 0; i < pSW->number; ++i) {
+        tdDestroyTSma(pSW->tSma + i);
+      }
+      tfree(pSW->tSma);
+    }
+  }
+}
+
+static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) {
+  int32_t tlen = 0;
+
+  tlen += taosEncodeFixedU8(buf, pSma->version);
+  tlen += taosEncodeFixedU8(buf, pSma->intervalUnit);
+  tlen += taosEncodeFixedU8(buf, pSma->slidingUnit);
+  tlen += taosEncodeString(buf, pSma->indexName);
+  tlen += taosEncodeString(buf, pSma->timezone);
+  tlen += taosEncodeFixedU16(buf, pSma->nFuncColIds);
+  tlen += taosEncodeFixedU16(buf, pSma->tagsFilterLen);
+  tlen += taosEncodeFixedI64(buf, pSma->tableUid);
+  tlen += taosEncodeFixedI64(buf, pSma->interval);
+  tlen += taosEncodeFixedI64(buf, pSma->sliding);
+
+  for (uint16_t i = 0; i < pSma->nFuncColIds; ++i) {
+    SFuncColIds* funcColIds = pSma->funcColIds + i;
+    tlen += taosEncodeFixedU16(buf, funcColIds->funcId);
+    tlen += taosEncodeFixedU16(buf, funcColIds->nColIds);
+    for (uint16_t j = 0; j < funcColIds->nColIds; ++j) {
+      tlen += taosEncodeFixedU16(buf, *(funcColIds->colIds + j));
+    }
+  }
+
+  if (pSma->tagsFilterLen > 0) {
+    tlen += taosEncodeString(buf, pSma->tagsFilter);
+  }
+
+  return tlen;
+}
+
+static FORCE_INLINE int32_t tEncodeTSmaWrapper(void** buf, const STSmaWrapper* pSW) {
+  int32_t tlen = 0;
+
+  tlen += taosEncodeFixedU32(buf, pSW->number);
+  for (uint32_t i = 0; i < pSW->number; ++i) {
+    tlen += tEncodeTSma(buf, pSW->tSma + i);
+  }
+  return tlen;
+}
+
+static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) {
+  buf = taosDecodeFixedU8(buf, &pSma->version);
+  buf = taosDecodeFixedU8(buf, &pSma->intervalUnit);
+  buf = taosDecodeFixedU8(buf, &pSma->slidingUnit);
+  buf = taosDecodeStringTo(buf, pSma->indexName);
+  buf = taosDecodeStringTo(buf, pSma->timezone);
+  buf = taosDecodeFixedU16(buf, &pSma->nFuncColIds);
+  buf = taosDecodeFixedU16(buf, &pSma->tagsFilterLen);
+  buf = taosDecodeFixedI64(buf, &pSma->tableUid);
+  buf = taosDecodeFixedI64(buf, &pSma->interval);
+  buf = taosDecodeFixedI64(buf, &pSma->sliding);
+
+  if (pSma->nFuncColIds > 0) {
+    pSma->funcColIds = (SFuncColIds*)calloc(pSma->nFuncColIds, sizeof(SFuncColIds));
+    if (pSma->funcColIds == NULL) {
+      tdDestroyTSma(pSma);
+      return NULL;
+    }
+    for (uint16_t i = 0; i < pSma->nFuncColIds; ++i) {
+      SFuncColIds* funcColIds = pSma->funcColIds + i;
+      buf = taosDecodeFixedU16(buf, &funcColIds->funcId);
+      buf = taosDecodeFixedU16(buf, &funcColIds->nColIds);
+      if (funcColIds->nColIds > 0) {
+        funcColIds->colIds = (col_id_t*)calloc(funcColIds->nColIds, sizeof(col_id_t));
+        if (funcColIds->colIds != NULL) {
+          for (uint16_t j = 0; j < funcColIds->nColIds; ++j) {
+            buf = taosDecodeFixedU16(buf, funcColIds->colIds + j);
+          }
+        } else {
+          tdDestroyTSma(pSma);
+          return NULL;
+        }
+      } else {
+        funcColIds->colIds = NULL;
+      }
+    }
+  } else {
+    pSma->funcColIds = NULL;
+  }
+
+  if (pSma->tagsFilterLen > 0) {
+    pSma->tagsFilter = (char*)calloc(pSma->tagsFilterLen, 1);
+    if (pSma->tagsFilter != NULL) {
+      buf = taosDecodeStringTo(buf, pSma->tagsFilter);
+    } else {
+      tdDestroyTSma(pSma);
+      return NULL;
+    }
+
+  } else {
+    pSma->tagsFilter = NULL;
+  }
+
+  return buf;
+}
+
+static FORCE_INLINE void* tDecodeTSmaWrapper(void* buf, STSmaWrapper* pSW) {
+  buf = taosDecodeFixedU32(buf, &pSW->number);
+
+  pSW->tSma = (STSma*)calloc(pSW->number, sizeof(STSma));
+  if (pSW->tSma == NULL) {
+    return NULL;
+  }
+
+  for (uint32_t i = 0; i < pSW->number; ++i) {
+    if ((buf = tDecodeTSma(buf, pSW->tSma + i)) == NULL) {
+      for (uint32_t j = i; j >= 0; --i) {
+        tdDestroyTSma(pSW->tSma + j);
+      }
+      free(pSW->tSma);
+      return NULL;
+    }
+  }
+  return buf;
+}
 
 typedef struct {
   int64_t uid;
@@ -1869,7 +2127,7 @@ typedef struct {
   int32_t         skipLogNum;
   int32_t         numOfTopics;
   SArray*         pBlockData;  // SArray<SSDataBlock>
-} SMqConsumeRsp;
+} SMqPollRsp;
 
 // one req for one vg+topic
 typedef struct {
@@ -1878,11 +2136,11 @@ typedef struct {
   int64_t consumerId;
   int64_t blockingTime;
   int32_t epoch;
-  char    cgroup[TSDB_CONSUMER_GROUP_LEN];
+  char    cgroup[TSDB_CGROUP_LEN];
 
   int64_t currentOffset;
   char    topic[TSDB_TOPIC_FNAME_LEN];
-} SMqConsumeReq;
+} SMqPollReq;
 
 typedef struct {
   int32_t vgId;
@@ -1897,14 +2155,14 @@ typedef struct {
 
 typedef struct {
   int64_t consumerId;
-  char    cgroup[TSDB_CONSUMER_GROUP_LEN];
+  char    cgroup[TSDB_CGROUP_LEN];
   SArray* topics;  // SArray<SMqSubTopicEp>
 } SMqCMGetSubEpRsp;
 
 struct tmq_message_t {
   SMqRspHead head;
   union {
-    SMqConsumeRsp    consumeRsp;
+    SMqPollRsp       consumeRsp;
     SMqCMGetSubEpRsp getEpRsp;
   };
   void* extra;

include/common/tmsgdef.h

@@ -148,6 +148,9 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp)
+  TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "mnode-create-stream", SCMCreateStreamReq, SCMCreateStreamRsp)
+  TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STREAM, "mnode-alter-stream", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_DROP_STREAM, "mnode-drop-stream", NULL, NULL)
 
   // Requests handled by VNODE
   TD_NEW_MSG_SEG(TDMT_VND_MSG)
@@ -184,6 +187,9 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqCVConsumeReq, SMqCVConsumeRsp)
 
+  TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL)
 
   // Requests handled by QNODE
   TD_NEW_MSG_SEG(TDMT_QND_MSG)

include/common/trow.h

@@ -118,6 +118,8 @@ typedef struct {
 } SKvRow;
 
 typedef struct {
+  /// timestamp
+  TSKEY ts;
   union {
     /// union field for encode and decode
     uint32_t info;
@@ -138,8 +140,6 @@ typedef struct {
   uint32_t len;
   /// row version
   uint64_t ver;
-  /// timestamp
-  TSKEY ts;
   /// the inline data, maybe a tuple or a k-v tuple
   char data[];
 } STSRow;
@@ -173,7 +173,7 @@ typedef struct {
 #define TD_ROW_DATA(r)     ((r)->data)
 #define TD_ROW_LEN(r)      ((r)->len)
 #define TD_ROW_KEY(r)      ((r)->ts)
-#define TD_ROW_KEY_ADDR(r) POINTER_SHIFT((r), 16)
+#define TD_ROW_KEY_ADDR(r) (r)
 
 // N.B. If without STSchema, getExtendedRowSize() is used to get the rowMaxBytes and
 // (int32_t)ceil((double)nCols/TD_VTYPE_PARTS) should be added if TD_SUPPORT_BITMAP defined.

include/dnode/mnode/sdb/sdb.h

@@ -113,15 +113,16 @@ typedef enum {
   SDB_USER = 7,
   SDB_AUTH = 8,
   SDB_ACCT = 9,
-  SDB_OFFSET = 10,
+  SDB_STREAM = 10,
-  SDB_SUBSCRIBE = 11,
+  SDB_OFFSET = 11,
-  SDB_CONSUMER = 12,
+  SDB_SUBSCRIBE = 12,
-  SDB_TOPIC = 13,
+  SDB_CONSUMER = 13,
-  SDB_VGROUP = 14,
+  SDB_TOPIC = 14,
-  SDB_STB = 15,
+  SDB_VGROUP = 15,
-  SDB_DB = 16,
+  SDB_STB = 16,
-  SDB_FUNC = 17,
+  SDB_DB = 17,
-  SDB_MAX = 18
+  SDB_FUNC = 18,
+  SDB_MAX = 19
 } ESdbType;
 
 typedef struct SSdb SSdb;

include/dnode/snode/snode.h

@@ -16,6 +16,9 @@
 #ifndef _TD_SNODE_H_
 #define _TD_SNODE_H_
 
+#include "tmsg.h"
+#include "trpc.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif

include/libs/monitor/monitor.h

@@ -18,6 +18,7 @@
 
 #include "tarray.h"
 #include "tdef.h"
+#include "tlog.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -31,6 +32,8 @@ extern "C" {
 typedef struct {
   int32_t dnode_id;
   char    dnode_ep[TSDB_EP_LEN];
+  int64_t cluster_id;
+  int32_t protocol;
 } SMonBasicInfo;
 
 typedef struct {
@@ -85,29 +88,26 @@ typedef struct {
 
 typedef struct {
   float   uptime;  // day
-  float   cpu_engine;
+  double  cpu_engine;
-  float   cpu_system;
+  double  cpu_system;
   float   cpu_cores;
   int64_t mem_engine;   // KB
   int64_t mem_system;   // KB
   int64_t mem_total;    // KB
-  float   disk_engine;    // GB
+  int64_t disk_engine;  // Byte
-  float   disk_used;      // GB
+  int64_t disk_used;    // Byte
-  float   disk_total;     // GB
+  int64_t disk_total;   // Byte
-  int64_t net_in;
+  int64_t net_in;       // bytes
-  int64_t net_out;
+  int64_t net_out;      // bytes
-  float   io_read;
+  int64_t io_read;      // bytes
-  float   io_write;
+  int64_t io_write;     // bytes
-  float   io_read_disk;
+  int64_t io_read_disk;   // bytes
-  float   io_write_disk;
+  int64_t io_write_disk;  // bytes
-  int32_t req_select;
+  int64_t req_select;
-  float   req_select_rate;
+  int64_t req_insert;
-  int32_t req_insert;
+  int64_t req_insert_success;
-  int32_t req_insert_success;
+  int64_t req_insert_batch;
-  float   req_insert_rate;
+  int64_t req_insert_batch_success;
-  int32_t req_insert_batch;
-  int32_t req_insert_batch_success;
-  float   req_insert_batch_rate;
   int32_t errors;
   int32_t vnodes_num;
   int32_t masters;
@@ -126,30 +126,18 @@ typedef struct {
   SMonDiskDesc tempdir;
 } SMonDiskInfo;
 
-typedef enum {
-  MON_LEVEL_ERROR = 0,
-  MON_LEVEL_INFO = 1,
-  MON_LEVEL_DEBUG = 2,
-  MON_LEVEL_TRACE = 3,
-} EMonLogLevel;
-
-typedef struct {
-  int64_t      ts;
-  EMonLogLevel level;
-  char         content[MON_LOG_LEN];
-} SMonLogItem;
-
 typedef struct SMonInfo SMonInfo;
 
 typedef struct {
   const char *server;
   uint16_t    port;
   int32_t     maxLogs;
+  bool        comp;
 } SMonCfg;
 
 int32_t monInit(const SMonCfg *pCfg);
 void    monCleanup();
-void    monAddLogItem(SMonLogItem *pItem);
+void    monRecordLog(int64_t ts, ELogLevel level, const char *content);
 
 SMonInfo *monCreateMonitorInfo();
 void      monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo);

include/libs/qcom/query.h

@@ -56,24 +56,26 @@ typedef struct STableComInfo {
  * The cached child table meta info. For each child table, 24 bytes are required to keep the essential table info.
  */
 typedef struct SCTableMeta {
-  int32_t  vgId:24;
+  int32_t  vgId : 24;
   int8_t   tableType;
   uint64_t uid;
   uint64_t suid;
 } SCTableMeta;
 
 /*
- * Note that the first 24 bytes of STableMeta are identical to SCTableMeta, it is safe to cast a STableMeta to be a SCTableMeta.
+ * Note that the first 24 bytes of STableMeta are identical to SCTableMeta, it is safe to cast a STableMeta to be a
+ * SCTableMeta.
  */
 typedef struct STableMeta {
-  //BEGIN: KEEP THIS PART SAME WITH SCTableMeta
+  // BEGIN: KEEP THIS PART SAME WITH SCTableMeta
-  int32_t        vgId:24;
+  int32_t  vgId : 24;
   int8_t   tableType;
   uint64_t uid;
   uint64_t suid;
-  //END: KEEP THIS PART SAME WITH SCTableMeta
+  // END: KEEP THIS PART SAME WITH SCTableMeta
 
-  // if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta info
+  // if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta
+  // info
   int16_t       sversion;
   int16_t       tversion;
   STableComInfo tableInfo;
@@ -83,22 +85,16 @@ typedef struct STableMeta {
 typedef struct SDBVgInfo {
   int32_t   vgVersion;
   int8_t    hashMethod;
-  SHashObj *vgHash;  //key:vgId, value:SVgroupInfo
+  SHashObj* vgHash;  // key:vgId, value:SVgroupInfo
 } SDBVgInfo;
 
 typedef struct SUseDbOutput {
   char       db[TSDB_DB_FNAME_LEN];
   uint64_t   dbId;
-  SDBVgInfo     *dbVgroup;
+  SDBVgInfo* dbVgroup;
 } SUseDbOutput;
 
-enum {
+enum { META_TYPE_NULL_TABLE = 1, META_TYPE_CTABLE, META_TYPE_TABLE, META_TYPE_BOTH_TABLE };
-  META_TYPE_NULL_TABLE = 1,
-  META_TYPE_CTABLE,
-  META_TYPE_TABLE,
-  META_TYPE_BOTH_TABLE
-};
-
 
 typedef struct STableMetaOutput {
   int32_t     metaType;
@@ -107,21 +103,21 @@ typedef struct STableMetaOutput {
   char        ctbName[TSDB_TABLE_NAME_LEN];
   char        tbName[TSDB_TABLE_NAME_LEN];
   SCTableMeta ctbMeta;
-  STableMeta *tbMeta;
+  STableMeta* tbMeta;
 } STableMetaOutput;
 
 typedef struct SDataBuf {
-  void     *pData;
+  void*    pData;
   uint32_t len;
-  void     *handle;
+  void*    handle;
 } SDataBuf;
 
 typedef int32_t (*__async_send_cb_fn_t)(void* param, const SDataBuf* pMsg, int32_t code);
 typedef int32_t (*__async_exec_fn_t)(void* param);
 
 typedef struct SMsgSendInfo {
-  __async_send_cb_fn_t fp;        //async callback function
+  __async_send_cb_fn_t fp;  // async callback function
-  void     *param;
+  void*                param;
   uint64_t             requestId;
   uint64_t             requestObjRefId;
   int32_t              msgType;
@@ -130,7 +126,7 @@ typedef struct SMsgSendInfo {
 
 typedef struct SQueryNodeAddr {
   int32_t nodeId;  // vgId or qnodeId
-  SEpSet  epset;
+  SEpSet  epSet;
 } SQueryNodeAddr;
 
 int32_t initTaskQueue();
@@ -154,34 +150,67 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code);
  * @param pInfo
  * @return
  */
-int32_t asyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo);
+int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo);
 
-int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp);
+int32_t queryBuildUseDbOutput(SUseDbOutput* pOut, SUseDbRsp* usedbRsp);
 
 void initQueryModuleMsgHandle();
 
 const SSchema* tGetTbnameColumnSchema();
 bool           tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags);
 
-int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta **pMeta);
+int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta);
-
-extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen);
-extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char *msg, int32_t msgSize);
 
+extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen);
+extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t msgSize);
 
 #define SET_META_TYPE_NULL(t)       (t) = META_TYPE_NULL_TABLE
 #define SET_META_TYPE_CTABLE(t)     (t) = META_TYPE_CTABLE
 #define SET_META_TYPE_TABLE(t)      (t) = META_TYPE_TABLE
 #define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE
 
-#define qFatal(...)  do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", qDebugFlag, __VA_ARGS__); }} while(0)
+#define qFatal(...)                                                     \
-#define qError(...)  do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", qDebugFlag, __VA_ARGS__); }} while(0)
+  do {                                                                  \
-#define qWarn(...)   do { if (qDebugFlag & DEBUG_WARN)  { taosPrintLog("QRY WARN ", qDebugFlag, __VA_ARGS__); }}  while(0)
+    if (qDebugFlag & DEBUG_FATAL) {                                     \
-#define qInfo(...)   do { if (qDebugFlag & DEBUG_INFO)  { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
+      taosPrintLog("QRY FATAL ", DEBUG_FATAL, qDebugFlag, __VA_ARGS__); \
-#define qDebug(...)  do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
+    }                                                                   \
-#define qTrace(...)  do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
+  } while (0)
-#define qDebugL(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLongString("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
+#define qError(...)                                                     \
-
+  do {                                                                  \
+    if (qDebugFlag & DEBUG_ERROR) {                                     \
+      taosPrintLog("QRY ERROR ", DEBUG_ERROR, qDebugFlag, __VA_ARGS__); \
+    }                                                                   \
+  } while (0)
+#define qWarn(...)                                                    \
+  do {                                                                \
+    if (qDebugFlag & DEBUG_WARN) {                                    \
+      taosPrintLog("QRY WARN ", DEBUG_WARN, qDebugFlag, __VA_ARGS__); \
+    }                                                                 \
+  } while (0)
+#define qInfo(...)                                               \
+  do {                                                           \
+    if (qDebugFlag & DEBUG_INFO) {                               \
+      taosPrintLog("QRY ", DEBUG_INFO, qDebugFlag, __VA_ARGS__); \
+    }                                                            \
+  } while (0)
+#define qDebug(...)                                               \
+  do {                                                            \
+    if (qDebugFlag & DEBUG_DEBUG) {                               \
+      taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \
+    }                                                             \
+  } while (0)
+#define qTrace(...)                                               \
+  do {                                                            \
+    if (qDebugFlag & DEBUG_TRACE) {                               \
+      taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); \
+    }                                                             \
+  } while (0)
+#define qDebugL(...)                                                     \
+  do {                                                                   \
+    if (qDebugFlag & DEBUG_DEBUG) {                                      \
+      taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \
+    }                                                                    \
+  } while (0)
 
 #ifdef __cplusplus
 }

include/libs/sync/sync.h

@@ -24,6 +24,7 @@ extern "C" {
 #include <tdatablock.h>
 #include "taosdef.h"
 #include "trpc.h"
+#include "wal.h"
 
 typedef uint64_t SyncNodeId;
 typedef int32_t  SyncGroupId;
@@ -31,9 +32,9 @@ typedef int64_t  SyncIndex;
 typedef uint64_t SyncTerm;
 
 typedef enum {
-  TAOS_SYNC_STATE_FOLLOWER = 0,
+  TAOS_SYNC_STATE_FOLLOWER = 100,
-  TAOS_SYNC_STATE_CANDIDATE = 1,
+  TAOS_SYNC_STATE_CANDIDATE = 101,
-  TAOS_SYNC_STATE_LEADER = 2,
+  TAOS_SYNC_STATE_LEADER = 102,
 } ESyncState;
 
 typedef struct SSyncBuffer {
@@ -87,25 +88,22 @@ typedef struct SSyncFSM {
 
 } SSyncFSM;
 
+struct SSyncRaftEntry;
+typedef struct SSyncRaftEntry SSyncRaftEntry;
+
 // abstract definition of log store in raft
 // SWal implements it
 typedef struct SSyncLogStore {
   void* data;
 
   // append one log entry
-  int32_t (*appendEntry)(struct SSyncLogStore* pLogStore, SRpcMsg* pBuf);
+  int32_t (*appendEntry)(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry);
 
-  // get one log entry, user need to free pBuf->data
+  // get one log entry, user need to free pEntry->pCont
-  int32_t (*getEntry)(struct SSyncLogStore* pLogStore, SyncIndex index, SRpcMsg* pBuf);
+  SSyncRaftEntry* (*getEntry)(struct SSyncLogStore* pLogStore, SyncIndex index);
 
-  // update log store commit index with "index"
+  // truncate log with index, entries after the given index (>=index) will be deleted
-  int32_t (*updateCommitIndex)(struct SSyncLogStore* pLogStore, SyncIndex index);
+  int32_t (*truncate)(struct SSyncLogStore* pLogStore, SyncIndex fromIndex);
-
-  // truncate log with index, entries after the given index (>index) will be deleted
-  int32_t (*truncate)(struct SSyncLogStore* pLogStore, SyncIndex index);
-
-  // return commit index of log
-  SyncIndex (*getCommitIndex)(struct SSyncLogStore* pLogStore);
 
   // return index of last entry
   SyncIndex (*getLastIndex)(struct SSyncLogStore* pLogStore);
@@ -113,6 +111,12 @@ typedef struct SSyncLogStore {
   // return term of last entry
   SyncTerm (*getLastTerm)(struct SSyncLogStore* pLogStore);
 
+  // update log store commit index with "index"
+  int32_t (*updateCommitIndex)(struct SSyncLogStore* pLogStore, SyncIndex index);
+
+  // return commit index of log
+  SyncIndex (*getCommitIndex)(struct SSyncLogStore* pLogStore);
+
 } SSyncLogStore;
 
 // raft need to persist two variables in storage: currentTerm, voteFor
@@ -134,10 +138,13 @@ typedef struct SSyncInfo {
   SyncGroupId vgId;
   SSyncCfg    syncCfg;
   char        path[TSDB_FILENAME_LEN];
+  SWal*       pWal;
   SSyncFSM*   pFsm;
 
   void* rpcClient;
   int32_t (*FpSendMsg)(void* rpcClient, const SEpSet* pEpSet, SRpcMsg* pMsg);
+  void* queue;
+  int32_t (*FpEqMsg)(void* queue, SRpcMsg* pMsg);
 
 } SSyncInfo;
 
@@ -150,10 +157,7 @@ void    syncCleanUp();
 int64_t    syncStart(const SSyncInfo* pSyncInfo);
 void       syncStop(int64_t rid);
 int32_t    syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
-
+int32_t    syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
-int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak);
-// int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak);
-
 ESyncState syncGetMyRole(int64_t rid);
 void       syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole);
 

include/libs/transport/thttp.h

@@ -22,7 +22,9 @@
 extern "C" {
 #endif
 
-int32_t taosSendHttpReport(const char* server, uint16_t port, const char* pCont, int32_t contLen);
+typedef enum { HTTP_GZIP, HTTP_FLAT } EHttpCompFlag;
+
+int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag);
 
 #ifdef __cplusplus
 }

include/libs/wal/wal.h

@@ -24,42 +24,40 @@
 extern "C" {
 #endif
 
-extern int32_t wDebugFlag;
-
 #define wFatal(...)                                              \
   {                                                              \
     if (wDebugFlag & DEBUG_FATAL) {                              \
-      taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); \
+      taosPrintLog("WAL FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \
     }                                                            \
   }
 #define wError(...)                                              \
   {                                                              \
     if (wDebugFlag & DEBUG_ERROR) {                              \
-      taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); \
+      taosPrintLog("WAL ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \
     }                                                            \
   }
 #define wWarn(...)                                             \
   {                                                            \
     if (wDebugFlag & DEBUG_WARN) {                             \
-      taosPrintLog("WAL WARN ", 255, __VA_ARGS__); \
+      taosPrintLog("WAL WARN ", DEBUG_WARN, 255, __VA_ARGS__); \
     }                                                          \
   }
 #define wInfo(...)                                        \
   {                                                       \
     if (wDebugFlag & DEBUG_INFO) {                        \
-      taosPrintLog("WAL ", 255, __VA_ARGS__); \
+      taosPrintLog("WAL ", DEBUG_INFO, 255, __VA_ARGS__); \
     }                                                     \
   }
 #define wDebug(...)                                               \
   {                                                               \
     if (wDebugFlag & DEBUG_DEBUG) {                               \
-      taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \
+      taosPrintLog("WAL ", DEBUG_DEBUG, wDebugFlag, __VA_ARGS__); \
     }                                                             \
   }
 #define wTrace(...)                                               \
   {                                                               \
     if (wDebugFlag & DEBUG_TRACE) {                               \
-      taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \
+      taosPrintLog("WAL ", DEBUG_TRACE, wDebugFlag, __VA_ARGS__); \
     }                                                             \
   }
 

include/os/osDir.h

@@ -16,10 +16,24 @@
 #ifndef _TD_OS_DIR_H_
 #define _TD_OS_DIR_H_
 
+// If the error is in a third-party library, place this header file under the third-party library header file.
+#ifndef ALLOW_FORBID_FUNC
+    #define opendir OPENDIR_FUNC_TAOS_FORBID
+    #define readdir READDIR_FUNC_TAOS_FORBID
+    #define closedir CLOSEDIR_FUNC_TAOS_FORBID
+    #define dirname DIRNAME_FUNC_TAOS_FORBID
+    #undef basename
+    #define basename BASENAME_FUNC_TAOS_FORBID
+#endif
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+typedef struct TdDir *TdDirPtr;
+typedef struct TdDirEntry *TdDirEntryPtr;
+
+
 void    taosRemoveDir(const char *dirname);
 bool    taosDirExist(char *dirname);
 int32_t taosMkDir(const char *dirname);
@@ -27,6 +41,14 @@ void    taosRemoveOldFiles(const char *dirname, int32_t keepDays);
 int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen);
 int32_t taosRealPath(char *dirname, int32_t maxlen);
 bool    taosIsDir(const char *dirname);
+char*   taosDirName(char *dirname);
+char*   taosDirEntryBaseName(char *dirname);
+
+TdDirPtr      taosOpenDir(const char *dirname);
+TdDirEntryPtr taosReadDir(TdDirPtr pDir);
+bool          taosDirEntryIsDir(TdDirEntryPtr pDirEntry);
+char*         taosGetDirEntryName(TdDirEntryPtr pDirEntry);
+int32_t       taosCloseDir(TdDirPtr pDir);
 
 #ifdef __cplusplus
 }

include/os/osFile.h

@@ -22,6 +22,7 @@ extern "C" {
 
 #include "osSocket.h"
 
+// If the error is in a third-party library, place this header file under the third-party library header file.
 #ifndef ALLOW_FORBID_FUNC
     #define open OPEN_FUNC_TAOS_FORBID
     #define fopen FOPEN_FUNC_TAOS_FORBID
@@ -31,6 +32,8 @@ extern "C" {
     #define fstat FSTAT_FUNC_TAOS_FORBID
     #define close CLOSE_FUNC_TAOS_FORBID
     #define fclose FCLOSE_FUNC_TAOS_FORBID
+    #define fsync FSYNC_FUNC_TAOS_FORBID
+    // #define fflush FFLUSH_FUNC_TAOS_FORBID
 #endif
 
 #ifndef PATH_MAX
@@ -47,7 +50,7 @@ typedef struct TdFile *TdFilePtr;
 #define TD_FILE_TEXT      0x0020
 #define TD_FILE_AUTO_DEL  0x0040
 #define TD_FILE_EXCL      0x0080
-#define TD_FILE_STREAM    0x0100   // Only support taosFprintfFile, taosGetLineFile, taosGetLineFile, taosEOFFile
+#define TD_FILE_STREAM    0x0100   // Only support taosFprintfFile, taosGetLineFile, taosEOFFile
 TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions);
 
 #define TD_FILE_ACCESS_EXIST_OK 0x1
@@ -80,10 +83,10 @@ int64_t taosCloseFile(TdFilePtr *ppFile);
  
 int32_t taosRenameFile(const char *oldName, const char *newName);
 int64_t taosCopyFile(const char *from, const char *to);
+int32_t taosRemoveFile(const char *path);
  
 void    taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath);
  
-int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size);
 int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size);
 
 void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length);

include/os/osRand.h

@@ -20,7 +20,16 @@
 extern "C" {
 #endif
 
+// If the error is in a third-party library, place this header file under the third-party library header file.
+#ifndef ALLOW_FORBID_FUNC
+    #define rand RAND_FUNC_TAOS_FORBID
+    #define srand SRAND_FUNC_TAOS_FORBID
+    #define rand_r RANDR_FUNC_TAOS_FORBID
+#endif
+
+void taosSeedRand(uint32_t seed);
 uint32_t taosRand(void);
+uint32_t taosRandR(uint32_t *pSeed);
 void     taosRandStr(char* str, int32_t size);
 uint32_t taosSafeRand(void);
 

include/os/osSleep.h

@@ -20,7 +20,17 @@
 extern "C" {
 #endif
 
+// If the error is in a third-party library, place this header file under the third-party library header file.
+#ifndef ALLOW_FORBID_FUNC
+    #define Sleep SLEEP_FUNC_TAOS_FORBID
+    #define sleep SLEEP_FUNC_TAOS_FORBID
+    #define usleep USLEEP_FUNC_TAOS_FORBID
+    #define nanosleep NANOSLEEP_FUNC_TAOS_FORBID
+#endif
+
+void taosSsleep(int32_t s);
 void taosMsleep(int32_t ms);
+void taosUsleep(int32_t us);
 
 #ifdef __cplusplus
 }

include/os/osSocket.h

@@ -16,6 +16,17 @@
 #ifndef _TD_OS_SOCKET_H_
 #define _TD_OS_SOCKET_H_
 
+// If the error is in a third-party library, place this header file under the third-party library header file.
+#ifndef ALLOW_FORBID_FUNC
+    #define socket SOCKET_FUNC_TAOS_FORBID
+    #define bind   BIND_FUNC_TAOS_FORBID
+    #define listen LISTEN_FUNC_TAOS_FORBID
+    #define accept ACCEPT_FUNC_TAOS_FORBID
+    #define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID
+    #define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID
+    #define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID
+#endif
+
 #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
   #include "winsock2.h"
   #include <WS2tcpip.h>
@@ -30,34 +41,6 @@
 extern "C" {
 #endif
 
-#define TAOS_EPOLL_WAIT_TIME 500
-typedef int32_t SOCKET;
-typedef SOCKET  EpollFd;
-#define EpollClose(pollFd) taosCloseSocket(pollFd)
-
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
-typedef SOCKET SocketFd;
-#else
-typedef int32_t SocketFd;
-#endif
-
-int32_t taosSendto(SocketFd fd, void * msg, int len, unsigned int flags, const struct sockaddr * to, int tolen);
-int32_t taosWriteSocket(SocketFd fd, void *msg, int len);
-int32_t taosReadSocket(SocketFd fd, void *msg, int len);
-int32_t taosCloseSocketNoCheck(SocketFd fd);
-int32_t taosCloseSocket(SocketFd fd);
-void    taosShutDownSocketRD(SOCKET fd);
-void    taosShutDownSocketWR(SOCKET fd);
-int32_t taosSetNonblocking(SOCKET sock, int32_t on);
-void    taosIgnSIGPIPE();
-void    taosBlockSIGPIPE();
-void    taosSetMaskSIGPIPE();
-int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen);
-int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t *optlen);
-
-uint32_t    taosInetAddr(const char *ipAddr);
-const char *taosInetNtoa(struct in_addr ipInt);
-
 #if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) 
   #define htobe64 htonll
   #if defined(_TD_GO_DLL_)
@@ -69,22 +52,54 @@ const char *taosInetNtoa(struct in_addr ipInt);
   #define htobe64 htonll
 #endif
 
-int32_t taosReadn(SOCKET sock, char *buffer, int32_t len);
+#define TAOS_EPOLL_WAIT_TIME 500
-int32_t taosWriteMsg(SOCKET fd, void *ptr, int32_t nbytes);
-int32_t taosReadMsg(SOCKET fd, void *ptr, int32_t nbytes);
-int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes);
-int64_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len);
-int32_t taosSetNonblocking(SOCKET sock, int32_t on);
 
-SOCKET  taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
+typedef struct TdSocketServer *TdSocketServerPtr;
-SOCKET  taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
+typedef struct TdSocket *TdSocketPtr;
-SOCKET  taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
+typedef struct TdEpoll *TdEpollPtr;
-int32_t taosKeepTcpAlive(SOCKET sockFd);
 
-int32_t  taosGetFqdn(char *);
+int32_t taosSendto(TdSocketPtr pSocket, void * msg, int len, unsigned int flags, const struct sockaddr * to, int tolen);
+int32_t taosWriteSocket(TdSocketPtr pSocket, void *msg, int len);
+int32_t taosReadSocket(TdSocketPtr pSocket, void *msg, int len);
+int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, socklen_t *addrLen);
+int32_t taosCloseSocket(TdSocketPtr *ppSocket);
+int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer);
+int32_t taosShutDownSocketRD(TdSocketPtr pSocket);
+int32_t taosShutDownSocketServerRD(TdSocketServerPtr pSocketServer);
+int32_t taosShutDownSocketWR(TdSocketPtr pSocket);
+int32_t taosShutDownSocketServerWR(TdSocketServerPtr pSocketServer);
+int32_t taosShutDownSocketRDWR(TdSocketPtr pSocket);
+int32_t taosShutDownSocketServerRDWR(TdSocketServerPtr pSocketServer);
+int32_t taosSetNonblocking(TdSocketPtr pSocket, int32_t on);
+int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t optlen);
+int32_t taosGetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t *optlen);
+int32_t taosWriteMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
+int32_t taosReadMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
+int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes);
+int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len);
+
+TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
+TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
+TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
+int32_t taosKeepTcpAlive(TdSocketPtr pSocket);
+TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, socklen_t *addrLen);
+
+int32_t taosGetSocketName(TdSocketPtr pSocket,struct sockaddr *destAddr, socklen_t *addrLen);
+
+void     taosBlockSIGPIPE();
 uint32_t taosGetIpv4FromFqdn(const char *);
+int32_t  taosGetFqdn(char *);
 void     tinet_ntoa(char *ipstr, uint32_t ip);
 uint32_t ip2uint(const char *const ip_addr);
+void     taosIgnSIGPIPE();
+void     taosSetMaskSIGPIPE();
+uint32_t taosInetAddr(const char *ipAddr);
+const char *taosInetNtoa(struct in_addr ipInt);
+
+TdEpollPtr taosCreateEpoll(int32_t size);
+int32_t taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocket, struct epoll_event *event);
+int32_t taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxEvents, int32_t timeout);
+int32_t taosCloseEpoll(TdEpollPtr *ppEpoll);
 
 #ifdef __cplusplus
 }

include/os/osSysinfo.h

@@ -33,22 +33,20 @@ typedef struct {
   SDiskSize size;
 } SDiskSpace;
 
+bool    taosCheckSystemIsSmallEnd();
 void    taosGetSystemInfo();
 int32_t taosGetEmail(char *email, int32_t maxLen);
 int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen);
 int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores);
 int32_t taosGetCpuCores(float *numOfCores);
-int32_t taosGetCpuUsage(float *cpu_system, float *cpu_engine);
+int32_t taosGetCpuUsage(double *cpu_system, double *cpu_engine);
 int32_t taosGetTotalMemory(int64_t *totalKB);
 int32_t taosGetProcMemory(int64_t *usedKB);
 int32_t taosGetSysMemory(int64_t *usedKB);
 int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize);
-int32_t taosReadProcIO(int64_t *rchars, int64_t *wchars);
+int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes);
-int32_t taosGetProcIO(float *readKB, float *writeKB);
+int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes);
-int32_t taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes);
-int32_t taosGetBandSpeed(float *bandSpeedKb);
 
-int32_t taosSystem(const char *cmd);
 void    taosKillSystem();
 int32_t taosGetSystemUUID(char *uid, int32_t uidlen);
 char   *taosGetCmdlineByPID(int32_t pid);

include/os/osSystem.h

@@ -20,11 +20,23 @@
 extern "C" {
 #endif
 
+// If the error is in a third-party library, place this header file under the third-party library header file.
+#ifndef ALLOW_FORBID_FUNC
+    #define popen POPEN_FUNC_TAOS_FORBID
+    #define pclose PCLOSE_FUNC_TAOS_FORBID
+    #define tcsetattr TCSETATTR_FUNC_TAOS_FORBID
+    #define tcgetattr TCGETATTR_FUNC_TAOS_FORBID
+#endif
+
+int32_t taosSystem(const char *cmd, char *buf, int32_t bufSize);
 void* taosLoadDll(const char* filename);
 void* taosLoadSym(void* handle, char* name);
 void  taosCloseDll(void* handle);
 
 int32_t taosSetConsoleEcho(bool on);
+void setTerminalMode();
+int32_t getOldTerminalMode();
+void resetTerminalMode();
 
 #ifdef __cplusplus
 }

include/util/taoserror.h

@@ -84,6 +84,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_INVALID_VERSION_NUMBER        TAOS_DEF_ERROR_CODE(0, 0x0120)
 #define TSDB_CODE_INVALID_VERSION_STRING        TAOS_DEF_ERROR_CODE(0, 0x0121)
 #define TSDB_CODE_VERSION_NOT_COMPATIBLE        TAOS_DEF_ERROR_CODE(0, 0x0122)
+#define TSDB_CODE_COMPRESS_ERROR                TAOS_DEF_ERROR_CODE(0, 0x0123)
 
 //client
 #define TSDB_CODE_TSC_INVALID_OPERATION         TAOS_DEF_ERROR_CODE(0, 0x0200)
@@ -266,9 +267,11 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_UNSUPPORTED_TOPIC         TAOS_DEF_ERROR_CODE(0, 0x03E8)
 #define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03E9)
 #define TSDB_CODE_MND_OFFSET_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03EA)
-#define TSDB_CODE_MND_MQ_PLACEHOLDER            TAOS_DEF_ERROR_CODE(0, 0x03F0)
-
 
+// mnode-stream
+#define TSDB_CODE_MND_STREAM_ALREADY_EXIST      TAOS_DEF_ERROR_CODE(0, 0x03F0)
+#define TSDB_CODE_MND_STREAM_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03F1)
+#define TSDB_CODE_MND_INVALID_STREAM_OPTION     TAOS_DEF_ERROR_CODE(0, 0x03F2)
 
 // dnode
 #define TSDB_CODE_DND_ACTION_IN_PROGRESS        TAOS_DEF_ERROR_CODE(0, 0x0400)

include/util/tdef.h

@@ -195,6 +195,7 @@ typedef enum ELogicConditionType {
 #define TSDB_NODE_NAME_LEN  64
 #define TSDB_TABLE_NAME_LEN 193  // it is a null-terminated string
 #define TSDB_TOPIC_NAME_LEN 193  // it is a null-terminated string
+#define TSDB_CGROUP_LEN     193  // it is a null-terminated string
 #define TSDB_DB_NAME_LEN    65
 #define TSDB_DB_FNAME_LEN   (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
 
@@ -206,12 +207,13 @@ typedef enum ELogicConditionType {
 #define TSDB_FUNC_TYPE_AGGREGATE 2
 #define TSDB_FUNC_MAX_RETRIEVE   1024
 
+#define TSDB_INDEX_NAME_LEN      33  // 32 + 1 '\0'
 #define TSDB_TYPE_STR_MAX_LEN    32
 #define TSDB_TABLE_FNAME_LEN     (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
 #define TSDB_TOPIC_FNAME_LEN     TSDB_TABLE_FNAME_LEN
-#define TSDB_CONSUMER_GROUP_LEN  192
+#define TSDB_STREAM_FNAME_LEN    TSDB_TABLE_FNAME_LEN
-#define TSDB_SUBSCRIBE_KEY_LEN   (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2)
+#define TSDB_SUBSCRIBE_KEY_LEN   (TSDB_CGROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2)
-#define TSDB_PARTITION_KEY_LEN   (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2)
+#define TSDB_PARTITION_KEY_LEN   (TSDB_SUBSCRIBE_KEY_LEN + 20)
 #define TSDB_COL_NAME_LEN        65
 #define TSDB_MAX_SAVED_SQL_LEN   TSDB_MAX_COLUMNS * 64
 #define TSDB_MAX_SQL_LEN         TSDB_PAYLOAD_SIZE
@@ -427,6 +429,8 @@ typedef struct {
   int32_t primary;
 } SDiskCfg;
 
+#define TMQ_SEPARATOR ':'
+
 #ifdef __cplusplus
 }
 #endif

include/util/tlog.h

@@ -22,9 +22,29 @@
 extern "C" {
 #endif
 
+typedef enum {
+  DEBUG_FATAL = 1,
+  DEBUG_ERROR = 1,
+  DEBUG_WARN = 2,
+  DEBUG_INFO = 2,
+  DEBUG_DEBUG = 4,
+  DEBUG_TRACE = 8,
+  DEBUG_DUMP = 16,
+  DEBUG_SCREEN = 64,
+  DEBUG_FILE = 128
+} ELogLevel;
+
+typedef void (*LogFp)(int64_t ts, ELogLevel level, const char *content);
+
+extern bool    tsLogEmbedded;
 extern bool    tsAsyncLog;
 extern int32_t tsNumOfLogLines;
 extern int32_t tsLogKeepDays;
+extern LogFp   tsLogFp;
+extern int64_t tsNumOfErrorLogs;
+extern int64_t tsNumOfInfoLogs;
+extern int64_t tsNumOfDebugLogs;
+extern int64_t tsNumOfTraceLogs;
 extern int32_t dDebugFlag;
 extern int32_t vDebugFlag;
 extern int32_t mDebugFlag;
@@ -40,45 +60,33 @@ extern int32_t tsdbDebugFlag;
 extern int32_t tqDebugFlag;
 extern int32_t fsDebugFlag;
 
-#define DEBUG_FATAL  1U
-#define DEBUG_ERROR  DEBUG_FATAL
-#define DEBUG_WARN   2U
-#define DEBUG_INFO   DEBUG_WARN
-#define DEBUG_DEBUG  4U
-#define DEBUG_TRACE  8U
-#define DEBUG_DUMP   16U
-#define DEBUG_SCREEN 64U
-#define DEBUG_FILE   128U
-
 int32_t taosInitLog(const char *logName, int32_t maxFiles);
 void    taosCloseLog();
 void    taosResetLog();
 void    taosSetAllDebugFlag(int32_t flag);
 void    taosDumpData(uint8_t *msg, int32_t len);
 
-void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...)
+void taosPrintLog(const char *flags, ELogLevel level, int32_t dflag, const char *format, ...)
 #ifdef __GNUC__
-    __attribute__((format(printf, 3, 4)))
+    __attribute__((format(printf, 4, 5)))
 #endif
     ;
 
-void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...)
+void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, const char *format, ...)
 #ifdef __GNUC__
-    __attribute__((format(printf, 3, 4)))
+    __attribute__((format(printf, 4, 5)))
 #endif
     ;
 
-extern int8_t tscEmbeddedInUtil;
+#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", DEBUG_FATAL, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
+#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", DEBUG_ERROR, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
+#define uWarn(...)  { if (uDebugFlag & DEBUG_WARN)  { taosPrintLog("UTL WARN ", DEBUG_WARN, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
+#define uInfo(...)  { if (uDebugFlag & DEBUG_INFO)  { taosPrintLog("UTL ", DEBUG_INFO, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
+#define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", DEBUG_DEBUG, uDebugFlag, __VA_ARGS__); }}
+#define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", DEBUG_TRACE, uDebugFlag, __VA_ARGS__); }}
 
-#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }}
+#define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }
-#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }}
+#define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); }
-#define uWarn(...)  { if (uDebugFlag & DEBUG_WARN)  { taosPrintLog("UTL WARN ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }}
-#define uInfo(...)  { if (uDebugFlag & DEBUG_INFO)  { taosPrintLog("UTL ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }}
-#define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }}
-#define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }}
-
-#define pError(...) { taosPrintLog("APP ERROR ", 255, __VA_ARGS__); }
-#define pPrint(...) { taosPrintLog("APP ", 255, __VA_ARGS__); }
 
 #ifdef __cplusplus
 }

include/util/version.h

@@ -23,7 +23,6 @@ extern "C" {
 extern char version[];
 extern char compatible_version[];
 extern char gitinfo[];
-extern char gitinfoOfInternal[];
 extern char buildinfo[];
 
 #ifdef __cplusplus

source/client/inc/clientInt.h

@@ -47,12 +47,12 @@ extern "C" {
 
 typedef struct SAppInstInfo SAppInstInfo;
 
-typedef struct SHbConnInfo {
+typedef struct {
   void*         param;
   SClientHbReq* req;
 } SHbConnInfo;
 
-typedef struct SAppHbMgr {
+typedef struct {
   char* key;
   // statistics
   int32_t reportCnt;
@@ -68,11 +68,11 @@ typedef struct SAppHbMgr {
   SHashObj* connInfo;    // hash<SClientHbKey, SHbConnInfo>
 } SAppHbMgr;
 
-typedef int32_t (*FHbRspHandle)(struct SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp);
+typedef int32_t (*FHbRspHandle)(SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp);
 
 typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq* req);
 
-typedef struct SClientHbMgr {
+typedef struct {
   int8_t inited;
   // ctl
   int8_t          threadStop;
@@ -114,7 +114,7 @@ struct SAppInstInfo {
   SList*           pConnList;  // STscObj linked list
   int64_t          clusterId;
   void*            pTransporter;
-  struct SAppHbMgr* pAppHbMgr;
+  SAppHbMgr*       pAppHbMgr;
 };
 
 typedef struct SAppInfo {
@@ -141,10 +141,6 @@ typedef struct STscObj {
   SAppInstInfo*   pAppInfo;
 } STscObj;
 
-typedef struct SMqConsumer {
-  STscObj* pTscObj;
-} SMqConsumer;
-
 typedef struct SReqResultInfo {
   const char* pRspMsg;
   const char* pData;

source/client/inc/clientLog.h

@@ -22,13 +22,13 @@ extern "C" {
 
 #include "tlog.h"
 
-#define tscFatal(...)  do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", cDebugFlag, __VA_ARGS__); }} while(0)
+#define tscFatal(...)  do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", DEBUG_FATAL, cDebugFlag, __VA_ARGS__); }} while(0)
-#define tscError(...)  do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", cDebugFlag, __VA_ARGS__); }} while(0)
+#define tscError(...)  do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", DEBUG_ERROR, cDebugFlag, __VA_ARGS__); }} while(0)
-#define tscWarn(...)   do { if (cDebugFlag & DEBUG_WARN)  { taosPrintLog("TSC WARN ", cDebugFlag, __VA_ARGS__); }}  while(0)
+#define tscWarn(...)   do { if (cDebugFlag & DEBUG_WARN)  { taosPrintLog("TSC WARN ", DEBUG_WARN, cDebugFlag, __VA_ARGS__); }}  while(0)
-#define tscInfo(...)   do { if (cDebugFlag & DEBUG_INFO)  { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0)
+#define tscInfo(...)   do { if (cDebugFlag & DEBUG_INFO)  { taosPrintLog("TSC ", DEBUG_INFO, cDebugFlag, __VA_ARGS__); }} while(0)
-#define tscDebug(...)  do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0)
+#define tscDebug(...)  do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0)
-#define tscTrace(...)  do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0)
+#define tscTrace(...)  do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", DEBUG_TRACE, cDebugFlag, __VA_ARGS__); }} while(0)
-#define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }} while(0)
+#define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0)
 
 #ifdef __cplusplus
 }

source/client/src/clientEnv.c

@@ -208,7 +208,7 @@ void taos_init_imp(void) {
   atexit(taos_cleanup);
 
   errno = TSDB_CODE_SUCCESS;
-  srand(taosGetTimestampSec());
+  taosSeedRand(taosGetTimestampSec());
 
   deltaToUtcInitOnce();
 

source/client/src/clientHb.c

@@ -23,7 +23,7 @@ static SClientHbMgr clientHbMgr = {0};
 static int32_t hbCreateThread();
 static void    hbStopThread();
 
-static int32_t hbMqHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; }
+static int32_t hbMqHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; }
 
 static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) {
   int32_t code = 0;
@@ -104,7 +104,7 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
+static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
   SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &pRsp->connKey, sizeof(SClientHbKey));
   if (NULL == info) {
     tscWarn("fail to get connInfo, may be dropped, connId:%d, type:%d", pRsp->connKey.connId, pRsp->connKey.hbType);
@@ -163,7 +163,7 @@ static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp *pRs
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t hbMqAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code) {
+static int32_t hbAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code) {
   static int32_t emptyRspNum = 0;
   if (code != 0) {
     tfree(param);
@@ -226,7 +226,11 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl
     db->vgVersion = htonl(db->vgVersion);
   }
 
-  SKv kv = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = sizeof(SDbVgVersion) * dbNum, .value = dbs};
+  SKv kv = {
+      .key = HEARTBEAT_KEY_DBINFO,
+      .valueLen = sizeof(SDbVgVersion) * dbNum,
+      .value = dbs,
+  };
 
   tscDebug("hb got %d expired db, valueLen:%d", dbNum, kv.valueLen);
 
@@ -256,7 +260,11 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC
     stb->tversion = htons(stb->tversion);
   }
 
-  SKv kv = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = sizeof(SSTableMetaVersion) * stbNum, .value = stbs};
+  SKv kv = {
+      .key = HEARTBEAT_KEY_STBINFO,
+      .valueLen = sizeof(SSTableMetaVersion) * stbNum,
+      .value = stbs,
+  };
 
   tscDebug("hb got %d expired stb, valueLen:%d", stbNum, kv.valueLen);
 
@@ -288,7 +296,7 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) {}
+int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { return 0; }
 
 void hbMgrInitMqHbHandle() {
   clientHbMgr.reqHandle[HEARTBEAT_TYPE_QUERY] = hbQueryHbReqHandle;
@@ -396,7 +404,7 @@ static void *hbThreadFunc(void *param) {
         free(buf);
         break;
       }
-      pInfo->fp = hbMqAsyncCallBack;
+      pInfo->fp = hbAsyncCallBack;
       pInfo->msgInfo.pData = buf;
       pInfo->msgInfo.len = tlen;
       pInfo->msgType = TDMT_MND_HEARTBEAT;
@@ -441,14 +449,13 @@ static void hbStopThread() {
   }
 
   while (2 != atomic_load_8(&clientHbMgr.threadStop)) {
-    usleep(10);
+    taosUsleep(10);
   }
 
   tscDebug("hb thread stopped");
 }
 
 SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) {
-  /*return NULL;*/
   hbMgrInit();
   SAppHbMgr *pAppHbMgr = malloc(sizeof(SAppHbMgr));
   if (pAppHbMgr == NULL) {
@@ -507,7 +514,6 @@ void appHbMgrCleanup(void) {
 }
 
 int hbMgrInit() {
-  /*return 0;*/
   // init once
   int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 0, 1);
   if (old == 1) return 0;
@@ -525,7 +531,7 @@ int hbMgrInit() {
 }
 
 void hbMgrCleanUp() {
-  return;
+#if 0
   hbStopThread();
 
   // destroy all appHbMgr
@@ -538,6 +544,7 @@ void hbMgrCleanUp() {
   pthread_mutex_unlock(&clientHbMgr.lock);
 
   clientHbMgr.appHbMgrs = NULL;
+#endif
 }
 
 int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo *info) {
@@ -564,8 +571,10 @@ int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo *
 }
 
 int hbRegisterConn(SAppHbMgr *pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType) {
-  /*return 0;*/
+  SClientHbKey connKey = {
-  SClientHbKey connKey = {.connId = connId, .hbType = HEARTBEAT_TYPE_QUERY};
+      .connId = connId,
+      .hbType = HEARTBEAT_TYPE_QUERY,
+  };
   SHbConnInfo info = {0};
 
   switch (hbType) {
@@ -587,7 +596,6 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int32_t connId, int64_t clusterId, int3
 }
 
 void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) {
-  /*return;*/
   int32_t code = 0;
   code = taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
   code = taosHashRemove(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey));
@@ -599,7 +607,6 @@ void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) {
 
 int hbAddConnInfo(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void *key, void *value, int32_t keyLen,
                   int32_t valueLen) {
-  return 0;
   // find req by connection id
   SClientHbReq *pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
   ASSERT(pReq != NULL);

source/client/src/clientImpl.c

@@ -517,7 +517,7 @@ void* doFetchRow(SRequestObj* pRequest) {
       SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo;
       SVgroupInfo*  pVgroupInfo = taosArrayGet(pShowReqInfo->pArray, pShowReqInfo->currentIndex);
 
-      epSet = pVgroupInfo->epset;
+      epSet = pVgroupInfo->epSet;
     } else if (pRequest->type == TDMT_VND_SHOW_TABLES_FETCH) {
       pRequest->type = TDMT_VND_SHOW_TABLES;
       SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo;
@@ -534,7 +534,7 @@ void* doFetchRow(SRequestObj* pRequest) {
       pRequest->body.requestMsg.pData = pShowReq;
 
       SMsgSendInfo* body = buildMsgInfoImpl(pRequest);
-      epSet = pVgroupInfo->epset;
+      epSet = pVgroupInfo->epSet;
 
       int64_t  transporterId = 0;
       STscObj* pTscObj = pRequest->pTscObj;

source/client/src/clientMain.c

@@ -343,3 +343,77 @@ void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param
 void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
   // TODO
 }
+
+TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval) {
+    // TODO
+    return NULL;
+}
+
+TAOS_RES *taos_consume(TAOS_SUB *tsub) {
+    // TODO
+    return NULL;
+}
+
+void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) {
+    // TODO
+}
+
+TAOS_STMT* taos_stmt_init(TAOS* taos) {
+    // TODO
+    return NULL;
+}
+
+int taos_stmt_close(TAOS_STMT* stmt) {
+    // TODO
+    return -1;
+}
+
+int taos_stmt_execute(TAOS_STMT* stmt) {
+    // TODO
+    return -1;
+}
+
+char *taos_stmt_errstr(TAOS_STMT *stmt) {
+    // TODO
+    return NULL;
+}
+
+int taos_stmt_affected_rows(TAOS_STMT* stmt) {
+    // TODO
+    return -1;
+}
+
+TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) {
+    // TODO
+    return NULL;
+}
+
+int taos_stmt_bind_param(TAOS_STMT* stmt, TAOS_BIND* bind) {
+    // TODO
+    return -1;
+}
+
+int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) {
+    // TODO
+    return -1;
+}
+
+int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags) {
+    // TODO
+    return -1;
+}
+
+int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) {
+    // TODO
+    return -1;
+}
+
+int taos_stmt_add_batch(TAOS_STMT* stmt) {
+    // TODO
+    return -1;
+}
+
+int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind) {
+    // TODO
+    return -1;
+}

source/client/src/tmq.c

@@ -13,8 +13,6 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#define _DEFAULT_SOURCE
-
 #include "clientInt.h"
 #include "clientLog.h"
 #include "parser.h"
@@ -46,7 +44,7 @@ struct tmq_topic_vgroup_list_t {
 
 struct tmq_conf_t {
   char           clientId[256];
-  char           groupId[256];
+  char           groupId[TSDB_CGROUP_LEN];
   int8_t         auto_commit;
   int8_t         resetOffset;
   tmq_commit_cb* commit_cb;
@@ -56,9 +54,10 @@ struct tmq_conf_t {
 
 struct tmq_t {
   // conf
-  char           groupId[256];
+  char           groupId[TSDB_CGROUP_LEN];
   char           clientId[256];
   int8_t         autoCommit;
+  int8_t         inWaiting;
   int64_t        consumerId;
   int32_t        epoch;
   int32_t        resetOffsetCfg;
@@ -66,9 +65,12 @@ struct tmq_t {
   STscObj*       pTscObj;
   tmq_commit_cb* commit_cb;
   int32_t        nextTopicIdx;
+  int32_t        waitingRequest;
+  int32_t        readyRequest;
   SArray*        clientTopics;  // SArray<SMqClientTopic>
   STaosQueue*    mqueue;        // queue of tmq_message_t
   STaosQall*     qall;
+  tsem_t         rspSem;
   // stat
   int64_t pollCnt;
 };
@@ -121,6 +123,8 @@ typedef struct {
   SMqClientVg*    pVg;
   int32_t         epoch;
   tsem_t          rspSem;
+  tmq_message_t** msg;
+  int32_t         sync;
 } SMqPollCbParam;
 
 typedef struct {
@@ -196,6 +200,26 @@ int32_t tmq_list_append(tmq_list_t* ptr, const char* src) {
   return 0;
 }
 
+void tmqClearUnhandleMsg(tmq_t* tmq) {
+  tmq_message_t* msg;
+  while (1) {
+    taosGetQitem(tmq->qall, (void**)&msg);
+    if (msg)
+      taosFreeQitem(msg);
+    else
+      break;
+  }
+
+  taosReadAllQitems(tmq->mqueue, tmq->qall);
+  while (1) {
+    taosGetQitem(tmq->qall, (void**)&msg);
+    if (msg)
+      taosFreeQitem(msg);
+    else
+      break;
+  }
+}
+
 int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) {
   SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param;
   pParam->rspErr = code;
@@ -205,16 +229,11 @@ int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) {
 
 int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) {
   SMqCommitCbParam* pParam = (SMqCommitCbParam*)param;
-  tmq_resp_err_t    rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL;
+  pParam->rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL;
   if (pParam->tmq->commit_cb) {
-    pParam->tmq->commit_cb(pParam->tmq, rspErr, NULL, NULL);
+    pParam->tmq->commit_cb(pParam->tmq, pParam->rspErr, NULL, NULL);
-  }
-  if (!pParam->async)
-    tsem_post(&pParam->rspSem);
-  else {
-    tsem_destroy(&pParam->rspSem);
-    free(param);
   }
+  if (!pParam->async) tsem_post(&pParam->rspSem);
   return 0;
 }
 
@@ -240,9 +259,12 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
     return NULL;
   }
   pTmq->pTscObj = (STscObj*)conn;
+  pTmq->inWaiting = 0;
   pTmq->status = 0;
   pTmq->pollCnt = 0;
   pTmq->epoch = 0;
+  pTmq->waitingRequest = 0;
+  pTmq->readyRequest = 0;
   // set conf
   strcpy(pTmq->clientId, conf->clientId);
   strcpy(pTmq->groupId, conf->groupId);
@@ -250,6 +272,8 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
   pTmq->commit_cb = conf->commit_cb;
   pTmq->resetOffsetCfg = conf->resetOffset;
 
+  tsem_init(&pTmq->rspSem, 0, 0);
+
   pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1);
   pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic));
 
@@ -315,6 +339,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in
   }
   pParam->tmq = tmq;
   tsem_init(&pParam->rspSem, 0, 0);
+  pParam->async = async;
 
   pRequest->body.requestMsg = (SDataBuf){
       .pData = buf,
@@ -335,6 +360,9 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in
     resp = pParam->rspErr;
   }
 
+  tsem_destroy(&pParam->rspSem);
+  free(pParam);
+
   if (pArray) {
     taosArrayDestroy(pArray);
   }
@@ -483,7 +511,7 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i
   tNameFromString(&name, dbName, T_NAME_ACCT | T_NAME_DB);
   tNameFromString(&name, topicName, T_NAME_TABLE);
 
-  SMCreateTopicReq req = {
+  SCMCreateTopicReq req = {
       .igExists = 1,
       .physicalPlan = (char*)pStr,
       .sql = (char*)sql,
@@ -491,13 +519,13 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i
   };
   tNameExtractFullName(&name, req.name);
 
-  int   tlen = tSerializeMCreateTopicReq(NULL, 0, &req);
+  int   tlen = tSerializeSCMCreateTopicReq(NULL, 0, &req);
   void* buf = malloc(tlen);
   if (buf == NULL) {
     goto _return;
   }
 
-  tSerializeMCreateTopicReq(buf, tlen, &req);
+  tSerializeSCMCreateTopicReq(buf, tlen, &req);
   /*printf("formatted: %s\n", dagStr);*/
 
   pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen, .handle = NULL};
@@ -576,7 +604,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
 
 int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) {
   if (tmq_message == NULL) return 0;
-  SMqConsumeRsp* pRsp = (SMqConsumeRsp*)tmq_message;
+  SMqPollRsp* pRsp = &tmq_message->consumeRsp;
   return pRsp->skipLogNum;
 }
 
@@ -585,7 +613,7 @@ void tmqShowMsg(tmq_message_t* tmq_message) {
 
   static bool noPrintSchema;
   char        pBuf[128];
-  SMqConsumeRsp* pRsp = &tmq_message->consumeRsp;
+  SMqPollRsp* pRsp = &tmq_message->consumeRsp;
   int32_t     colNum = pRsp->schemas->nCols;
   if (!noPrintSchema) {
     printf("|");
@@ -625,56 +653,74 @@ void tmqShowMsg(tmq_message_t* tmq_message) {
 }
 
 int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
-  printf("recv poll\n");
+  /*printf("recv poll\n");*/
   SMqPollCbParam* pParam = (SMqPollCbParam*)param;
   SMqClientVg*    pVg = pParam->pVg;
   tmq_t*          tmq = pParam->tmq;
   if (code != 0) {
     printf("msg discard\n");
-    if (pParam->epoch == tmq->epoch) {
+    goto WRITE_QUEUE_FAIL;
-      atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-    }
-    return 0;
   }
 
   int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch;
   int32_t tmqEpoch = atomic_load_32(&tmq->epoch);
   if (msgEpoch < tmqEpoch) {
+    tsem_post(&tmq->rspSem);
     printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);
     return 0;
   }
 
   if (msgEpoch != tmqEpoch) {
     printf("mismatch rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);
+  } else {
+    atomic_sub_fetch_32(&tmq->waitingRequest, 1);
   }
 
+#if 0
+  if (pParam->sync == 1) {
+    /**pParam->msg = malloc(sizeof(tmq_message_t));*/
+    *pParam->msg = taosAllocateQitem(sizeof(tmq_message_t));
+    if (*pParam->msg) {
+      memcpy(*pParam->msg, pMsg->pData, sizeof(SMqRspHead));
+      tDecodeSMqConsumeRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &((*pParam->msg)->consumeRsp));
+      if ((*pParam->msg)->consumeRsp.numOfTopics != 0) {
+        pVg->currentOffset = (*pParam->msg)->consumeRsp.rspOffset;
+      }
+      taosWriteQitem(tmq->mqueue, *pParam->msg);
+      tsem_post(&pParam->rspSem);
+      return 0;
+    }
+    tsem_post(&pParam->rspSem);
+    return -1;
+  }
+#endif
+
   /*SMqConsumeRsp* pRsp = calloc(1, sizeof(SMqConsumeRsp));*/
   tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t));
   if (pRsp == NULL) {
-    printf("fail\n");
+    goto WRITE_QUEUE_FAIL;
-    return -1;
   }
   memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));
-  tDecodeSMqConsumeRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->consumeRsp);
+  tDecodeSMqPollRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->consumeRsp);
   /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/
   if (pRsp->consumeRsp.numOfTopics == 0) {
-    printf("no data\n");
+    /*printf("no data\n");*/
+    taosFreeQitem(pRsp);
+    goto WRITE_QUEUE_FAIL;
+  }
+
+  pRsp->extra = pParam->pVg;
+  taosWriteQitem(tmq->mqueue, pRsp);
+  atomic_add_fetch_32(&tmq->readyRequest, 1);
+  tsem_post(&tmq->rspSem);
+  return 0;
+
+WRITE_QUEUE_FAIL:
   if (pParam->epoch == tmq->epoch) {
     atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
   }
-    taosFreeQitem(pRsp);
+  tsem_post(&tmq->rspSem);
-    return 0;
+  return code;
-  }
-  pRsp->extra = pParam->pVg;
-  taosWriteQitem(tmq->mqueue, pRsp);
-  printf("poll in queue\n");
-  /*pParam->rspMsg = (tmq_message_t*)pRsp;*/
-  /*pVg->currentOffset = pRsp->consumeRsp.rspOffset;*/
-
-  /*printf("rsp offset: %ld\n", rsp.rspOffset);*/
-  /*printf("-----msg begin----\n");*/
-  /*printf("\n-----msg end------\n");*/
-  return 0;
 }
 
 bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
@@ -711,81 +757,94 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
   tmq_t*           tmq = pParam->tmq;
   if (code != 0) {
     printf("get topic endpoint error, not ready, wait:%d\n", pParam->sync);
-    if (pParam->sync) {
+    goto END;
-      tsem_post(&pParam->rspSem);
   }
-    return 0;
+
-  }
+  // tmq's epoch is monotomically increase,
-  tscDebug("tmq ask ep cb called");
+  // so it's safe to discard any old epoch msg.
-  if (pParam->sync) {
+  // epoch will only increase when received newer epoch ep msg
   SMqRspHead* head = pMsg->pData;
+  int32_t     epoch = atomic_load_32(&tmq->epoch);
+  if (head->epoch <= epoch) {
+    goto END;
+  }
+
+  if (pParam->sync) {
     SMqCMGetSubEpRsp rsp;
     tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp);
     /*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/
     /*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/
-    int32_t epoch = atomic_load_32(&tmq->epoch);
+    if (tmqUpdateEp(tmq, head->epoch, &rsp)) {
-    if (head->epoch > epoch && tmqUpdateEp(tmq, head->epoch, &rsp)) {
       atomic_store_64(&tmq->status, TMQ_CONSUMER_STATUS__READY);
     }
-    tsem_post(&pParam->rspSem);
     tDeleteSMqCMGetSubEpRsp(&rsp);
   } else {
     tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t));
     if (pRsp == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
-      return -1;
+      code = -1;
+      goto END;
     }
     memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));
     tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->getEpRsp);
+
     taosWriteQitem(tmq->mqueue, pRsp);
+    tsem_post(&tmq->rspSem);
   }
-  return 0;
+
+END:
+  if (pParam->sync) {
+    tsem_post(&pParam->rspSem);
+  }
+  return code;
 }
 
 int32_t tmqAskEp(tmq_t* tmq, bool sync) {
-  printf("ask ep sync %d\n", sync);
   int32_t           tlen = sizeof(SMqCMGetSubEpReq);
-  SMqCMGetSubEpReq* buf = malloc(tlen);
+  SMqCMGetSubEpReq* req = malloc(tlen);
-  if (buf == NULL) {
+  if (req == NULL) {
     tscError("failed to malloc get subscribe ep buf");
-    goto END;
+    return -1;
   }
-  buf->consumerId = htobe64(tmq->consumerId);
+  req->consumerId = htobe64(tmq->consumerId);
-  buf->epoch = htonl(tmq->epoch);
+  req->epoch = htonl(tmq->epoch);
-  strcpy(buf->cgroup, tmq->groupId);
+  strcpy(req->cgroup, tmq->groupId);
-
-  SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_GET_SUB_EP);
-  if (pRequest == NULL) {
-    tscError("failed to malloc subscribe ep request");
-    goto END;
-  }
-
-  pRequest->body.requestMsg = (SDataBuf){
-      .pData = buf,
-      .len = tlen,
-      .handle = NULL,
-  };
 
   SMqAskEpCbParam* pParam = malloc(sizeof(SMqAskEpCbParam));
   if (pParam == NULL) {
     tscError("failed to malloc subscribe param");
-    goto END;
+    free(req);
+    return -1;
   }
   pParam->tmq = tmq;
   pParam->sync = sync;
   tsem_init(&pParam->rspSem, 0, 0);
 
-  SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
+  SMsgSendInfo* sendInfo = malloc(sizeof(SMsgSendInfo));
+  if (sendInfo == NULL) {
+    tsem_destroy(&pParam->rspSem);
+    free(pParam);
+    free(req);
+    return -1;
+  }
+
+  sendInfo->msgInfo = (SDataBuf){
+      .pData = req,
+      .len = tlen,
+      .handle = NULL,
+  };
+
+  sendInfo->requestId = generateRequestId();
   sendInfo->requestObjRefId = 0;
   sendInfo->param = pParam;
   sendInfo->fp = tmqAskEpCb;
+  sendInfo->msgType = TDMT_MND_GET_SUB_EP;
 
   SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
 
   int64_t transporterId = 0;
   asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
 
-END:
   if (sync) tsem_wait(&pParam->rspSem);
   return 0;
 }
@@ -804,6 +863,7 @@ tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) {
         SMqClientVg* pVg = taosArrayGet(clientTopic->vgs, j);
         if (pVg->vgId == pOffset->vgId) {
           pVg->currentOffset = pOffset->offset;
+          tmqClearUnhandleMsg(tmq);
           return TMQ_RESP_ERR__SUCCESS;
         }
       }
@@ -812,7 +872,7 @@ tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) {
   return TMQ_RESP_ERR__FAIL;
 }
 
-SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClientTopic* pTopic, SMqClientVg* pVg) {
+SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) {
   int64_t reqOffset;
   if (pVg->currentOffset >= 0) {
     reqOffset = pVg->currentOffset;
@@ -824,7 +884,7 @@ SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClie
     reqOffset = tmq->resetOffsetCfg;
   }
 
-  SMqConsumeReq* pReq = malloc(sizeof(SMqConsumeReq));
+  SMqPollReq* pReq = malloc(sizeof(SMqPollReq));
   if (pReq == NULL) {
     return NULL;
   }
@@ -832,38 +892,86 @@ SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClie
   strcpy(pReq->topic, pTopic->topicName);
   strcpy(pReq->cgroup, tmq->groupId);
 
-  pReq->blockingTime = blocking_time;
+  pReq->blockingTime = blockingTime;
   pReq->consumerId = tmq->consumerId;
   pReq->epoch = tmq->epoch;
   pReq->currentOffset = reqOffset;
 
   pReq->head.vgId = htonl(pVg->vgId);
-  pReq->head.contLen = htonl(sizeof(SMqConsumeReq));
+  pReq->head.contLen = htonl(sizeof(SMqPollReq));
   return pReq;
 }
 
-void tmqClearUnhandleMsg(tmq_t* tmq) {
+tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) {
-  tmq_message_t* msg;
+  tmq_message_t* msg = NULL;
-  while (1) {
+  for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
-    taosGetQitem(tmq->qall, (void**)&msg);
+    SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
-    if (msg)
+    for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
-      taosFreeQitem(msg);
+      SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
-    else
+      int32_t      vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
-      break;
+      /*if (vgStatus != TMQ_VG_STATUS__IDLE) {*/
+      /*continue;*/
+      /*}*/
+      SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
+      if (pReq == NULL) {
+        atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
+        // TODO: out of mem
+        return NULL;
       }
 
-  taosReadAllQitems(tmq->mqueue, tmq->qall);
+      SMqPollCbParam* pParam = malloc(sizeof(SMqPollCbParam));
-  while (1) {
+      if (pParam == NULL) {
-    taosGetQitem(tmq->qall, (void**)&msg);
+        atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-    if (msg)
+        // TODO: out of mem
-      taosFreeQitem(msg);
+        return NULL;
-    else
-      break;
       }
+      pParam->tmq = tmq;
+      pParam->pVg = pVg;
+      pParam->epoch = tmq->epoch;
+      pParam->sync = 1;
+      pParam->msg = &msg;
+      tsem_init(&pParam->rspSem, 0, 0);
+
+      SMsgSendInfo* sendInfo = malloc(sizeof(SMsgSendInfo));
+      if (sendInfo == NULL) {
+        return NULL;
+      }
+
+      sendInfo->msgInfo = (SDataBuf){
+          .pData = pReq,
+          .len = sizeof(SMqPollReq),
+          .handle = NULL,
+      };
+      sendInfo->requestId = generateRequestId();
+      sendInfo->requestObjRefId = 0;
+      sendInfo->param = pParam;
+      sendInfo->fp = tmqPollCb;
+      sendInfo->msgType = TDMT_VND_CONSUME;
+
+      int64_t transporterId = 0;
+      /*printf("send poll\n");*/
+      atomic_add_fetch_32(&tmq->waitingRequest, 1);
+      asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
+      pVg->pollCnt++;
+      tmq->pollCnt++;
+
+      tsem_wait(&pParam->rspSem);
+      tmq_message_t* nmsg = NULL;
+      while (1) {
+        taosReadQitem(tmq->mqueue, (void**)&nmsg);
+        if (nmsg == NULL) continue;
+        while (nmsg->head.mqMsgType != TMQ_MSG_TYPE__POLL_RSP) {
+          taosReadQitem(tmq->mqueue, (void**)&nmsg);
+        }
+        return nmsg;
+      }
+    }
+  }
+  return NULL;
 }
 
 int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
-  printf("call poll\n");
+  /*printf("call poll\n");*/
   for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
     SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
     for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
@@ -872,35 +980,47 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
       if (vgStatus != TMQ_VG_STATUS__IDLE) {
         continue;
       }
-      SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
+      SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
       if (pReq == NULL) {
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        // TODO: out of mem
+        tsem_post(&tmq->rspSem);
         return -1;
       }
-      SMqPollCbParam* param = malloc(sizeof(SMqPollCbParam));
+      SMqPollCbParam* pParam = malloc(sizeof(SMqPollCbParam));
-      if (param == NULL) {
+      if (pParam == NULL) {
+        free(pReq);
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        // TODO: out of mem
+        tsem_post(&tmq->rspSem);
         return -1;
       }
-      param->tmq = tmq;
+      pParam->tmq = tmq;
-      param->pVg = pVg;
+      pParam->pVg = pVg;
-      param->epoch = tmq->epoch;
+      pParam->epoch = tmq->epoch;
-      SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME);
+      pParam->sync = 0;
-      pRequest->body.requestMsg = (SDataBuf){
+
+      SMsgSendInfo* sendInfo = malloc(sizeof(SMsgSendInfo));
+      if (sendInfo == NULL) {
+        free(pReq);
+        free(pParam);
+        atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
+        tsem_post(&tmq->rspSem);
+        return -1;
+      }
+
+      sendInfo->msgInfo = (SDataBuf){
           .pData = pReq,
-          .len = sizeof(SMqConsumeReq),
+          .len = sizeof(SMqPollReq),
           .handle = NULL,
       };
-
+      sendInfo->requestId = generateRequestId();
-      SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
       sendInfo->requestObjRefId = 0;
-      sendInfo->param = param;
+      sendInfo->param = pParam;
       sendInfo->fp = tmqPollCb;
+      sendInfo->msgType = TDMT_VND_CONSUME;
 
       int64_t transporterId = 0;
-      printf("send poll\n");
+      /*printf("send poll\n");*/
+      atomic_add_fetch_32(&tmq->waitingRequest, 1);
       asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
       pVg->pollCnt++;
       tmq->pollCnt++;
@@ -912,7 +1032,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
 // return
 int32_t tmqHandleRes(tmq_t* tmq, tmq_message_t* rspMsg, bool* pReset) {
   if (rspMsg->head.mqMsgType == TMQ_MSG_TYPE__EP_RSP) {
-    printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch);
+    /*printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch);*/
     if (rspMsg->head.epoch > atomic_load_32(&tmq->epoch)) {
       tmqUpdateEp(tmq, rspMsg->head.epoch, &rspMsg->getEpRsp);
       tmqClearUnhandleMsg(tmq);
@@ -931,23 +1051,26 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese
     tmq_message_t* rspMsg = NULL;
     taosGetQitem(tmq->qall, (void**)&rspMsg);
     if (rspMsg == NULL) {
-      break;
+      taosReadAllQitems(tmq->mqueue, tmq->qall);
+      taosGetQitem(tmq->qall, (void**)&rspMsg);
+      if (rspMsg == NULL) return NULL;
     }
 
     if (rspMsg->head.mqMsgType == TMQ_MSG_TYPE__POLL_RSP) {
-      printf("handle poll rsp %d\n", rspMsg->head.mqMsgType);
+      atomic_sub_fetch_32(&tmq->readyRequest, 1);
+      /*printf("handle poll rsp %d\n", rspMsg->head.mqMsgType);*/
       if (rspMsg->head.epoch == atomic_load_32(&tmq->epoch)) {
-        printf("epoch match\n");
+        /*printf("epoch match\n");*/
         SMqClientVg* pVg = rspMsg->extra;
         pVg->currentOffset = rspMsg->consumeRsp.rspOffset;
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
         return rspMsg;
       } else {
-        printf("epoch mismatch\n");
+        /*printf("epoch mismatch\n");*/
         taosFreeQitem(rspMsg);
       }
     } else {
-      printf("handle ep rsp %d\n", rspMsg->head.mqMsgType);
+      /*printf("handle ep rsp %d\n", rspMsg->head.mqMsgType);*/
       bool reset = false;
       tmqHandleRes(tmq, rspMsg, &reset);
       taosFreeQitem(rspMsg);
@@ -957,36 +1080,57 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese
       }
     }
   }
+}
+
+tmq_message_t* tmq_consumer_poll_v1(tmq_t* tmq, int64_t blocking_time) {
+  tmq_message_t* rspMsg = NULL;
+  int64_t        startTime = taosGetTimestampMs();
+
+  int64_t status = atomic_load_64(&tmq->status);
+  tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT);
+
+  while (1) {
+    rspMsg = tmqSyncPollImpl(tmq, blocking_time);
+    if (rspMsg && rspMsg->consumeRsp.numOfTopics) {
+      return rspMsg;
+    }
+
+    if (blocking_time != 0) {
+      int64_t endTime = taosGetTimestampMs();
+      if (endTime - startTime > blocking_time) {
         return NULL;
+      }
+    } else
+      return NULL;
+  }
 }
 
 tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
-  tmq_message_t* rspMsg = NULL;
+  tmq_message_t* rspMsg;
   int64_t        startTime = taosGetTimestampMs();
 
   // TODO: put into another thread or delayed queue
   int64_t status = atomic_load_64(&tmq->status);
   tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT);
 
-  taosGetQitem(tmq->qall, (void**)&rspMsg);
+  rspMsg = tmqHandleAllRsp(tmq, blocking_time, false);
-  if (rspMsg == NULL) {
+  if (rspMsg) {
-    taosReadAllQitems(tmq->mqueue, tmq->qall);
+    return rspMsg;
   }
-  tmqHandleAllRsp(tmq, blocking_time, false);
-
-  tmqPollImpl(tmq, blocking_time);
 
   while (1) {
     /*printf("cycle\n");*/
-    taosReadAllQitems(tmq->mqueue, tmq->qall);
+    tmqPollImpl(tmq, blocking_time);
-    rspMsg = tmqHandleAllRsp(tmq, blocking_time, true);
+
+    tsem_wait(&tmq->rspSem);
+
+    rspMsg = tmqHandleAllRsp(tmq, blocking_time, false);
     if (rspMsg) {
       return rspMsg;
     }
     if (blocking_time != 0) {
       int64_t endTime = taosGetTimestampMs();
       if (endTime - startTime > blocking_time) {
-        printf("normal exit\n");
         return NULL;
       }
     }
@@ -1002,13 +1146,13 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
   if (taosArrayGetSize(tmq->clientTopics) == 0) {
     tscDebug("consumer:%ld poll but not assigned", tmq->consumerId);
     /*printf("over1\n");*/
-    usleep(blocking_time * 1000);
+    taosMsleep(blocking_time);
     return NULL;
   }
   SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx);
   if (taosArrayGetSize(pTopic->vgs) == 0) {
     /*printf("over2\n");*/
-    usleep(blocking_time * 1000);
+    taosMsleep(blocking_time);
     return NULL;
   }
 
@@ -1021,14 +1165,14 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
     SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, pTopic, pVg);
     if (pReq == NULL) {
       ASSERT(false);
-      usleep(blocking_time * 1000);
+      taosMsleep(blocking_time);
       return NULL;
     }
 
     SMqPollCbParam* param = malloc(sizeof(SMqPollCbParam));
     if (param == NULL) {
       ASSERT(false);
-      usleep(blocking_time * 1000);
+      taosMsleep(blocking_time);
       return NULL;
     }
     param->tmq = tmq;
@@ -1060,7 +1204,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
 
     if (tmq_message == NULL) {
       if (beginVgIdx == pTopic->nextVgIdx) {
-        usleep(blocking_time * 1000);
+        taosMsleep(blocking_time);
       } else {
         continue;
       }
@@ -1125,8 +1269,9 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v
 
 void tmq_message_destroy(tmq_message_t* tmq_message) {
   if (tmq_message == NULL) return;
-  SMqConsumeRsp* pRsp = &tmq_message->consumeRsp;
+  SMqPollRsp* pRsp = &tmq_message->consumeRsp;
   tDeleteSMqConsumeRsp(pRsp);
+  /*free(tmq_message);*/
   taosFreeQitem(tmq_message);
 }
 
@@ -1138,6 +1283,7 @@ const char* tmq_err2str(tmq_resp_err_t err) {
   }
   return "fail";
 }
+
 #if 0
 tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) {
   tmq_t* pTmq = malloc(sizeof(tmq_t));

source/common/src/tglobal.c

@@ -48,10 +48,11 @@ bool    tsPrintAuth = 0;
 
 // monitor
 bool     tsEnableMonitor = 1;
-int32_t  tsMonitorInterval = 5;
+int32_t  tsMonitorInterval = 30;
 char     tsMonitorFqdn[TSDB_FQDN_LEN] = {0};
 uint16_t tsMonitorPort = 6043;
 int32_t  tsMonitorMaxLogs = 100;
+bool     tsMonitorComp = false;
 
 /*
  * denote if the server needs to compress response message at the application layer to client, including query rsp,
@@ -313,7 +314,6 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) {
   if (cfgAddString(pCfg, "version", version, 1) != 0) return -1;
   if (cfgAddString(pCfg, "compatible_version", compatible_version, 1) != 0) return -1;
   if (cfgAddString(pCfg, "gitinfo", gitinfo, 1) != 0) return -1;
-  if (cfgAddString(pCfg, "gitinfoOfInternal", gitinfoOfInternal, 1) != 0) return -1;
   if (cfgAddString(pCfg, "buildinfo", buildinfo, 1) != 0) return -1;
   return 0;
 }
@@ -346,6 +346,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, 0) != 0) return -1;
+  if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, 0) != 0) return -1;
 
   return 0;
 }
@@ -462,6 +463,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tstrncpy(tsMonitorFqdn, cfgGetItem(pCfg, "monitorFqdn")->str, TSDB_FQDN_LEN);
   tsMonitorPort = (uint16_t)cfgGetItem(pCfg, "monitorPort")->i32;
   tsMonitorMaxLogs = cfgGetItem(pCfg, "monitorMaxLogs")->i32;
+  tsMonitorComp = cfgGetItem(pCfg, "monitorComp")->bval;
 
   if (tsQueryBufferSize >= 0) {
     tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL;
@@ -478,10 +480,10 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
   if (pCfg == NULL) return -1;
 
   if (tsc) {
-    tscEmbeddedInUtil = 0;
+    tsLogEmbedded = 0;
     if (taosAddClientLogCfg(pCfg) != 0) return -1;
   } else {
-    tscEmbeddedInUtil = 1;
+    tsLogEmbedded = 1;
     if (taosAddClientLogCfg(pCfg) != 0) return -1;
     if (taosAddServerLogCfg(pCfg) != 0) return -1;
   }

source/common/src/tmsg.c

@@ -1481,7 +1481,7 @@ static int32_t tSerializeSUseDbRspImp(SCoder *pEncoder, SUseDbRsp *pRsp) {
     if (tEncodeI32(pEncoder, pVgInfo->vgId) < 0) return -1;
     if (tEncodeU32(pEncoder, pVgInfo->hashBegin) < 0) return -1;
     if (tEncodeU32(pEncoder, pVgInfo->hashEnd) < 0) return -1;
-    if (tEncodeSEpSet(pEncoder, &pVgInfo->epset) < 0) return -1;
+    if (tEncodeSEpSet(pEncoder, &pVgInfo->epSet) < 0) return -1;
   }
 
   return 0;
@@ -1541,7 +1541,7 @@ int32_t tDeserializeSUseDbRspImp(SCoder *pDecoder, SUseDbRsp *pRsp) {
     if (tDecodeI32(pDecoder, &vgInfo.vgId) < 0) return -1;
     if (tDecodeU32(pDecoder, &vgInfo.hashBegin) < 0) return -1;
     if (tDecodeU32(pDecoder, &vgInfo.hashEnd) < 0) return -1;
-    if (tDecodeSEpSet(pDecoder, &vgInfo.epset) < 0) return -1;
+    if (tDecodeSEpSet(pDecoder, &vgInfo.epSet) < 0) return -1;
     taosArrayPush(pRsp->pVgroupInfos, &vgInfo);
   }
 
@@ -1899,7 +1899,7 @@ int32_t tDeserializeSMDropTopicReq(void *buf, int32_t bufLen, SMDropTopicReq *pR
   return 0;
 }
 
-int32_t tSerializeMCreateTopicReq(void *buf, int32_t bufLen, const SMCreateTopicReq *pReq) {
+int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTopicReq *pReq) {
   int32_t sqlLen = 0;
   int32_t physicalPlanLen = 0;
   int32_t logicalPlanLen = 0;
@@ -1927,7 +1927,7 @@ int32_t tSerializeMCreateTopicReq(void *buf, int32_t bufLen, const SMCreateTopic
   return tlen;
 }
 
-int32_t tDeserializeSMCreateTopicReq(void *buf, int32_t bufLen, SMCreateTopicReq *pReq) {
+int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicReq *pReq) {
   int32_t sqlLen = 0;
   int32_t physicalPlanLen = 0;
   int32_t logicalPlanLen = 0;
@@ -1956,13 +1956,13 @@ int32_t tDeserializeSMCreateTopicReq(void *buf, int32_t bufLen, SMCreateTopicReq
   return 0;
 }
 
-void tFreeSMCreateTopicReq(SMCreateTopicReq *pReq) {
+void tFreeSCMCreateTopicReq(SCMCreateTopicReq *pReq) {
   tfree(pReq->sql);
   tfree(pReq->physicalPlan);
   tfree(pReq->logicalPlan);
 }
 
-int32_t tSerializeSMCreateTopicRsp(void *buf, int32_t bufLen, const SMCreateTopicRsp *pRsp) {
+int32_t tSerializeSCMCreateTopicRsp(void *buf, int32_t bufLen, const SCMCreateTopicRsp *pRsp) {
   SCoder encoder = {0};
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
 
@@ -1975,7 +1975,7 @@ int32_t tSerializeSMCreateTopicRsp(void *buf, int32_t bufLen, const SMCreateTopi
   return tlen;
 }
 
-int32_t tDeserializeSMCreateTopicRsp(void *buf, int32_t bufLen, SMCreateTopicRsp *pRsp) {
+int32_t tDeserializeSCMCreateTopicRsp(void *buf, int32_t bufLen, SCMCreateTopicRsp *pRsp) {
   SCoder decoder = {0};
   tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
 
@@ -2390,3 +2390,76 @@ int32_t tDecodeSMqCMCommitOffsetReq(SCoder *decoder, SMqCMCommitOffsetReq *pReq)
   tEndDecode(decoder);
   return 0;
 }
+
+int32_t tSerializeSVCreateTSmaReq(void **buf, SVCreateTSmaReq *pReq) {
+  int32_t tlen = 0;
+
+  tlen += taosEncodeFixedI64(buf, pReq->ver);
+  tlen += tEncodeTSma(buf, &pReq->tSma);
+
+  return tlen;
+}
+
+void *tDeserializeSVCreateTSmaReq(void *buf, SVCreateTSmaReq *pReq) {
+  buf = taosDecodeFixedI64(buf, &(pReq->ver));
+
+  if ((buf = tDecodeTSma(buf, &pReq->tSma)) == NULL) {
+    tdDestroyTSma(&pReq->tSma);
+  }
+  return buf;
+}
+
+int32_t tSerializeSVDropTSmaReq(void **buf, SVDropTSmaReq *pReq) {
+  int32_t tlen = 0;
+
+  tlen += taosEncodeFixedI64(buf, pReq->ver);
+  tlen += taosEncodeString(buf, pReq->indexName);
+
+  return tlen;
+}
+void *tDeserializeSVDropTSmaReq(void *buf, SVDropTSmaReq *pReq) {
+  buf = taosDecodeFixedI64(buf, &(pReq->ver));
+  buf = taosDecodeStringTo(buf, pReq->indexName);
+
+  return buf;
+}
+
+int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateStreamReq *pReq) {
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->physicalPlan) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->logicalPlan) < 0) return -1;
+
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStreamReq *pReq) {
+  SCoder decoder = {0};
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1;
+  if (tDecodeCStr(&decoder, (const char **)&pReq->sql) < 0) return -1;
+  if (tDecodeCStr(&decoder, (const char **)&pReq->physicalPlan) < 0) return -1;
+  if (tDecodeCStr(&decoder, (const char **)&pReq->logicalPlan) < 0) return -1;
+  tEndDecode(&decoder);
+
+  tCoderClear(&decoder);
+  return 0;
+}
+
+void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) {
+  tfree(pReq->sql);
+  tfree(pReq->physicalPlan);
+  tfree(pReq->logicalPlan);
+}

source/common/src/ttszip.c

@@ -46,7 +46,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) {
   }
 
   if (!autoDelete) {
-    remove(pTSBuf->path);
+    taosRemoveFile(pTSBuf->path);
   }
 
   if (NULL == allocResForTSBuf(pTSBuf)) {
@@ -178,7 +178,7 @@ void* tsBufDestroy(STSBuf* pTSBuf) {
 
   if (pTSBuf->autoDelete) {
     //    ("tsBuf %p destroyed, delete tmp file:%s", pTSBuf, pTSBuf->path);
-    remove(pTSBuf->path);
+    taosRemoveFile(pTSBuf->path);
   } else {
     //    tscDebug("tsBuf %p destroyed, tmp file:%s, remains", pTSBuf, pTSBuf->path);
   }

source/dnode/mgmt/daemon/src/dmnMain.c

@@ -97,6 +97,11 @@ int32_t dmnRunDnode() {
 }
 
 int main(int argc, char const *argv[]) {
+  if (!taosCheckSystemIsSmallEnd()) {
+    uError("TDengine does not run on non-small-end machines.");
+    return -1;
+  }
+
   if (dmnParseOption(argc, argv) != 0) {
     return -1;
   }

source/dnode/mgmt/daemon/src/dmnUtil.c

@@ -30,6 +30,5 @@ void dmnPrintVersion() {
 #endif
   printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version);
   printf("gitinfo: %s\n", gitinfo);
-  printf("gitinfoI: %s\n", gitinfoOfInternal);
   printf("builuInfo: %s\n", buildinfo);
 }

source/dnode/mgmt/impl/inc/dndEnv.h

@@ -105,9 +105,19 @@ typedef struct {
 } SBnodeMgmt;
 
 typedef struct {
-  SHashObj    *hash;
   int32_t openVnodes;
   int32_t totalVnodes;
+  int32_t masterNum;
+  int64_t numOfSelectReqs;
+  int64_t numOfInsertReqs;
+  int64_t numOfInsertSuccessReqs;
+  int64_t numOfBatchInsertReqs;
+  int64_t numOfBatchInsertSuccessReqs;
+} SVnodesStat;
+
+typedef struct {
+  SVnodesStat  stat;
+  SHashObj    *hash;
   SRWLatch     latch;
   SQWorkerPool queryPool;
   SFWorkerPool fetchPool;

source/dnode/mgmt/impl/inc/dndInt.h

@@ -47,14 +47,12 @@ extern "C" {
 #include "vnode.h"
 #include "tfs.h"
 
-extern int32_t dDebugFlag;
+#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
-
+#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
-#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", 255, __VA_ARGS__); }}
+#define dWarn(...)  { if (dDebugFlag & DEBUG_WARN)  { taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
-#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", 255, __VA_ARGS__); }}
+#define dInfo(...)  { if (dDebugFlag & DEBUG_INFO)  { taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); }}
-#define dWarn(...)  { if (dDebugFlag & DEBUG_WARN)  { taosPrintLog("DND WARN ", 255, __VA_ARGS__); }}
+#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }}
-#define dInfo(...)  { if (dDebugFlag & DEBUG_INFO)  { taosPrintLog("DND ", 255, __VA_ARGS__); }}
+#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }}
-#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }}
-#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }}
 
 typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EStat;
 typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EWorkerType;

source/dnode/mgmt/impl/inc/dndMnode.h

@@ -34,7 +34,6 @@ int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
 
 int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
                                SMonGrantInfo *pGrantInfo);
-int8_t  dndIsMnode(SDnode *pDnode);
 
 #ifdef __cplusplus
 }

source/dnode/mgmt/impl/src/dndEnv.c

@@ -298,7 +298,7 @@ int32_t dndInit() {
     return -1;
   }
 
-  SMonCfg monCfg = {.maxLogs = tsMonitorMaxLogs, .port = tsMonitorPort, .server = tsMonitorFqdn};
+  SMonCfg monCfg = {.maxLogs = tsMonitorMaxLogs, .port = tsMonitorPort, .server = tsMonitorFqdn, .comp = tsMonitorComp};
   if (monInit(&monCfg) != 0) {
     dError("failed to init monitor since %s", terrstr());
     dndCleanup();

source/dnode/mgmt/impl/src/dndMgmt.c

@@ -477,6 +477,8 @@ void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) {
 static void dndGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) {
   pInfo->dnode_id = dndGetDnodeId(pDnode);
   tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN);
+  pInfo->cluster_id = dndGetClusterId(pDnode);
+  pInfo->protocol = 1;
 }
 
 static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) {
@@ -487,24 +489,21 @@ static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) {
   taosGetSysMemory(&pInfo->mem_system);
   pInfo->mem_total = tsTotalMemoryKB;
   pInfo->disk_engine = 0;
-  pInfo->disk_used = tsDataSpace.size.used / (1024 * 1024 * 1024.0);
+  pInfo->disk_used = tsDataSpace.size.used;
-  pInfo->disk_total = tsDataSpace.size.avail / (1024 * 1024 * 1024.0);
+  pInfo->disk_total = tsDataSpace.size.total;
-  taosGetCardInfo(NULL, &pInfo->net_in, &pInfo->net_out);
+  taosGetCardInfo(&pInfo->net_in, &pInfo->net_out);
-  taosGetProcIO(&pInfo->io_read, &pInfo->io_write);
+  taosGetProcIO(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk);
-  pInfo->io_read_disk = 0;
+
-  pInfo->io_write_disk = 0;
+  SVnodesStat *pStat = &pDnode->vmgmt.stat;
-  pInfo->req_select = 0;
+  pInfo->req_select = pStat->numOfSelectReqs;
-  pInfo->req_select_rate = 0;
+  pInfo->req_insert = pStat->numOfInsertReqs;
-  pInfo->req_insert = 0;
+  pInfo->req_insert_success = pStat->numOfInsertSuccessReqs;
-  pInfo->req_insert_success = 0;
+  pInfo->req_insert_batch = pStat->numOfBatchInsertReqs;
-  pInfo->req_insert_rate = 0;
+  pInfo->req_insert_batch_success = pStat->numOfBatchInsertSuccessReqs;
-  pInfo->req_insert_batch = 0;
+  pInfo->errors = tsNumOfErrorLogs;
-  pInfo->req_insert_batch_success = 0;
+  pInfo->vnodes_num = pStat->totalVnodes;
-  pInfo->req_insert_batch_rate = 0;
+  pInfo->masters = pStat->masterNum;
-  pInfo->errors = 0;
+  pInfo->has_mnode = pDnode->mmgmt.deployed;
-  pInfo->vnodes_num = 0;
-  pInfo->masters = 0;
-  pInfo->has_mnode = dndIsMnode(pDnode);
 }
 
 static void dndSendMonitorReport(SDnode *pDnode) {

source/dnode/mgmt/impl/src/dndMnode.c

@@ -640,10 +640,3 @@ int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SM
   dndReleaseMnode(pDnode, pMnode);
   return code;
 }
-
-int8_t dndIsMnode(SDnode *pDnode) {
-  SMnode *pMnode = dndAcquireMnode(pDnode);
-  if (pMnode == NULL) return 0;
-  dndReleaseMnode(pDnode, pMnode);
-  return 1;
-}

source/dnode/mgmt/impl/src/dndVnodes.c

@@ -17,6 +17,7 @@
 #include "dndVnodes.h"
 #include "dndMgmt.h"
 #include "dndTransport.h"
+#include "sync.h"
 
 typedef struct {
   int32_t  vgId;
@@ -381,7 +382,7 @@ static void *dnodeOpenVnodeFunc(void *param) {
 
     char stepDesc[TSDB_STEP_DESC_LEN] = {0};
     snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pCfg->vgId,
-             pMgmt->openVnodes, pMgmt->totalVnodes);
+             pMgmt->stat.openVnodes, pMgmt->stat.totalVnodes);
     dndReportStartup(pDnode, "open-vnodes", stepDesc);
 
     SVnodeCfg cfg = {.pDnode = pDnode, .pTfs = pDnode->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid};
@@ -395,7 +396,7 @@ static void *dnodeOpenVnodeFunc(void *param) {
       pThread->opened++;
     }
 
-    atomic_add_fetch_32(&pMgmt->openVnodes, 1);
+    atomic_add_fetch_32(&pMgmt->stat.openVnodes, 1);
   }
 
   dDebug("thread:%d, total vnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened,
@@ -421,7 +422,7 @@ static int32_t dndOpenVnodes(SDnode *pDnode) {
     return -1;
   }
 
-  pMgmt->totalVnodes = numOfVnodes;
+  pMgmt->stat.totalVnodes = numOfVnodes;
 
   int32_t threadNum = tsNumOfCores;
 #if 1
@@ -469,11 +470,11 @@ static int32_t dndOpenVnodes(SDnode *pDnode) {
   free(threads);
   free(pCfgs);
 
-  if (pMgmt->openVnodes != pMgmt->totalVnodes) {
+  if (pMgmt->stat.openVnodes != pMgmt->stat.totalVnodes) {
-    dError("there are total vnodes:%d, opened:%d", pMgmt->totalVnodes, pMgmt->openVnodes);
+    dError("there are total vnodes:%d, opened:%d", pMgmt->stat.totalVnodes, pMgmt->stat.openVnodes);
     return -1;
   } else {
-    dInfo("total vnodes:%d open successfully", pMgmt->totalVnodes);
+    dInfo("total vnodes:%d open successfully", pMgmt->stat.totalVnodes);
     return 0;
   }
 }
@@ -979,10 +980,17 @@ void dndCleanupVnodes(SDnode *pDnode) {
 
 void dndGetVnodeLoads(SDnode *pDnode, SArray *pLoads) {
   SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+  SVnodesStat *pStat = &pMgmt->stat;
+  int32_t      totalVnodes = 0;
+  int32_t      masterNum = 0;
+  int64_t      numOfSelectReqs = 0;
+  int64_t      numOfInsertReqs = 0;
+  int64_t      numOfInsertSuccessReqs = 0;
+  int64_t      numOfBatchInsertReqs = 0;
+  int64_t      numOfBatchInsertSuccessReqs = 0;
 
   taosRLockLatch(&pMgmt->latch);
 
-  int32_t v = 0;
   void *pIter = taosHashIterate(pMgmt->hash, NULL);
   while (pIter) {
     SVnodeObj **ppVnode = pIter;
@@ -993,8 +1001,24 @@ void dndGetVnodeLoads(SDnode *pDnode, SArray *pLoads) {
     vnodeGetLoad(pVnode->pImpl, &vload);
     taosArrayPush(pLoads, &vload);
 
+    numOfSelectReqs += vload.numOfSelectReqs;
+    numOfInsertReqs += vload.numOfInsertReqs;
+    numOfInsertSuccessReqs += vload.numOfInsertSuccessReqs;
+    numOfBatchInsertReqs += vload.numOfBatchInsertReqs;
+    numOfBatchInsertSuccessReqs += vload.numOfBatchInsertSuccessReqs;
+    totalVnodes++;
+    if (vload.role == TAOS_SYNC_STATE_LEADER) masterNum++;
+
     pIter = taosHashIterate(pMgmt->hash, pIter);
   }
 
   taosRUnLockLatch(&pMgmt->latch);
+
+  pStat->totalVnodes = totalVnodes;
+  pStat->masterNum = masterNum;
+  pStat->numOfSelectReqs = numOfSelectReqs;
+  pStat->numOfInsertReqs = numOfInsertReqs;
+  pStat->numOfInsertSuccessReqs = numOfInsertSuccessReqs;
+  pStat->numOfBatchInsertReqs = numOfBatchInsertReqs;
+  pStat->numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs;
 }

source/dnode/mgmt/impl/test/sut/src/sut.cpp

@@ -28,7 +28,7 @@ void Testbase::InitLog(const char* path) {
   wDebugFlag = 0;
   sDebugFlag = 0;
   tsdbDebugFlag = 0;
-  tscEmbeddedInUtil = 1;
+  tsLogEmbedded = 1;
   tsAsyncLog = 0;
 
   taosRemoveDir(path);

source/dnode/mnode/impl/inc/mndDef.h

@@ -34,46 +34,6 @@
 extern "C" {
 #endif
 
-extern int32_t mDebugFlag;
-
-// mnode log function
-#define mFatal(...)                                 \
-  {                                                 \
-    if (mDebugFlag & DEBUG_FATAL) {                 \
-      taosPrintLog("MND FATAL ", 255, __VA_ARGS__); \
-    }                                               \
-  }
-#define mError(...)                                 \
-  {                                                 \
-    if (mDebugFlag & DEBUG_ERROR) {                 \
-      taosPrintLog("MND ERROR ", 255, __VA_ARGS__); \
-    }                                               \
-  }
-#define mWarn(...)                                 \
-  {                                                \
-    if (mDebugFlag & DEBUG_WARN) {                 \
-      taosPrintLog("MND WARN ", 255, __VA_ARGS__); \
-    }                                              \
-  }
-#define mInfo(...)                            \
-  {                                           \
-    if (mDebugFlag & DEBUG_INFO) {            \
-      taosPrintLog("MND ", 255, __VA_ARGS__); \
-    }                                         \
-  }
-#define mDebug(...)                                  \
-  {                                                  \
-    if (mDebugFlag & DEBUG_DEBUG) {                  \
-      taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); \
-    }                                                \
-  }
-#define mTrace(...)                                  \
-  {                                                  \
-    if (mDebugFlag & DEBUG_TRACE) {                  \
-      taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); \
-    }                                                \
-  }
-
 typedef enum {
   MND_AUTH_ACCT_START = 0,
   MND_AUTH_ACCT_USER,
@@ -124,6 +84,7 @@ typedef enum {
   TRN_TYPE_SUBSCRIBE = 1016,
   TRN_TYPE_REBALANCE = 1017,
   TRN_TYPE_COMMIT_OFFSET = 1018,
+  TRN_TYPE_CREATE_STREAM = 1019,
   TRN_TYPE_BASIC_SCOPE_END,
   TRN_TYPE_GLOBAL_SCOPE = 2000,
   TRN_TYPE_CREATE_DNODE = 2001,
@@ -634,7 +595,7 @@ typedef struct {
   int64_t  consumerId;
   int64_t  connId;
   SRWLatch lock;
-  char     cgroup[TSDB_CONSUMER_GROUP_LEN];
+  char     cgroup[TSDB_CGROUP_LEN];
   SArray*  currentTopics;        // SArray<char*>
   SArray*  recentRemovedTopics;  // SArray<char*>
   int32_t  epoch;
@@ -701,6 +662,25 @@ static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pCons
   return buf;
 }
 
+typedef struct {
+  char     name[TSDB_TOPIC_FNAME_LEN];
+  char     db[TSDB_DB_FNAME_LEN];
+  int64_t  createTime;
+  int64_t  updateTime;
+  int64_t  uid;
+  int64_t  dbUid;
+  int32_t  version;
+  SRWLatch lock;
+  int8_t   status;
+  // int32_t  sqlLen;
+  char* sql;
+  char* logicalPlan;
+  char* physicalPlan;
+} SStreamObj;
+
+int32_t tEncodeSStreamObj(SCoder* pEncoder, const SStreamObj* pObj);
+int32_t tDecodeSStreamObj(SCoder* pDecoder, SStreamObj* pObj);
+
 typedef struct SMnodeMsg {
   char    user[TSDB_USER_LEN];
   char    db[TSDB_DB_FNAME_LEN];

source/dnode/mnode/impl/inc/mndInt.h

@@ -31,6 +31,13 @@
 extern "C" {
 #endif
 
+#define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
+#define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
+#define mWarn(...)  { if (mDebugFlag & DEBUG_WARN)  { taosPrintLog("MND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
+#define mInfo(...)  { if (mDebugFlag & DEBUG_INFO)  { taosPrintLog("MND ", DEBUG_INFO, 255, __VA_ARGS__); }}
+#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }}
+#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }}
+
 typedef int32_t (*MndMsgFp)(SMnodeMsg *pMsg);
 typedef int32_t (*MndInitFp)(SMnode *pMnode);
 typedef void (*MndCleanupFp)(SMnode *pMnode);

source/dnode/mnode/impl/inc/mndScheduler.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_MND_SCHEDULER_H_
+#define _TD_MND_SCHEDULER_H_
+
+#include "mndInt.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t mndInitScheduler(SMnode* pMnode);
+void    mndCleanupScheduler(SMnode* pMnode);
+
+int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_MND_SCHEDULER_H_ */

source/dnode/mnode/impl/inc/mndStream.h

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_MND_STREAM_H_
+#define _TD_MND_STREAM_H_
+
+#include "mndInt.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t mndInitStream(SMnode *pMnode);
+void    mndCleanupStream(SMnode *pMnode);
+
+SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName);
+void        mndReleaseStream(SMnode *pMnode, SStreamObj *pStream);
+
+SSdbRaw *mndStreamActionEncode(SStreamObj *pStream);
+SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_MND_STREAM_H_*/

source/dnode/mnode/impl/src/mndDb.c

@@ -900,10 +900,10 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) {
       vgInfo.vgId = pVgroup->vgId;
       vgInfo.hashBegin = pVgroup->hashBegin;
       vgInfo.hashEnd = pVgroup->hashEnd;
-      vgInfo.epset.numOfEps = pVgroup->replica;
+      vgInfo.epSet.numOfEps = pVgroup->replica;
       for (int32_t gid = 0; gid < pVgroup->replica; ++gid) {
         SVnodeGid *pVgid = &pVgroup->vnodeGid[gid];
-        SEp       *pEp = &vgInfo.epset.eps[gid];
+        SEp       *pEp = &vgInfo.epSet.eps[gid];
         SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
         if (pDnode != NULL) {
           memcpy(pEp->fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
@@ -911,7 +911,7 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) {
         }
         mndReleaseDnode(pMnode, pDnode);
         if (pVgid->role == TAOS_SYNC_STATE_LEADER) {
-          vgInfo.epset.inUse = gid;
+          vgInfo.epSet.inUse = gid;
         }
       }
       vindex++;

source/dnode/mnode/impl/src/mndDef.c

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mndDef.h"
+
+int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) {
+  if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1;
+  if (tEncodeCStr(pEncoder, pObj->db) < 0) return -1;
+  if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1;
+  if (tEncodeI64(pEncoder, pObj->updateTime) < 0) return -1;
+  if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1;
+  if (tEncodeI64(pEncoder, pObj->dbUid) < 0) return -1;
+  if (tEncodeI32(pEncoder, pObj->version) < 0) return -1;
+  if (tEncodeI8(pEncoder, pObj->status) < 0) return -1;
+  if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1;
+  if (tEncodeCStr(pEncoder, pObj->logicalPlan) < 0) return -1;
+  if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1;
+  return pEncoder->pos;
+}
+
+int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) {
+  if (tDecodeCStrTo(pDecoder, pObj->name) < 0) return -1;
+  if (tDecodeCStrTo(pDecoder, pObj->db) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pObj->createTime) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pObj->updateTime) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pObj->dbUid) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1;
+  if (tDecodeCStr(pDecoder, (const char **)&pObj->sql) < 0) return -1;
+  if (tDecodeCStr(pDecoder, (const char **)&pObj->logicalPlan) < 0) return -1;
+  if (tDecodeCStr(pDecoder, (const char **)&pObj->physicalPlan) < 0) return -1;
+  return 0;
+}

source/dnode/mnode/impl/src/mndDnode.c

@@ -340,9 +340,16 @@ static int32_t mndProcessStatusReq(SMnodeMsg *pReq) {
         pVgroup->compStorage = pVload->compStorage;
         pVgroup->pointsWritten = pVload->pointsWritten;
       }
+      bool roleChanged = false;
       for (int32_t vg = 0; vg < pVgroup->replica; ++vg) {
+        if (pVgroup->vnodeGid[vg].role != pVload->role) {
+          roleChanged = true;
+        }
         pVgroup->vnodeGid[vg].role = pVload->role;
       }
+      if (roleChanged) {
+        // notify scheduler role has changed
+      }
     }
 
     mndReleaseVgroup(pMnode, pVgroup);

source/dnode/mnode/impl/src/mndScheduler.c

@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mndScheduler.h"
+#include "mndConsumer.h"
+#include "mndDb.h"
+#include "mndDnode.h"
+#include "mndMnode.h"
+#include "mndOffset.h"
+#include "mndShow.h"
+#include "mndStb.h"
+#include "mndSubscribe.h"
+#include "mndTopic.h"
+#include "mndTrans.h"
+#include "mndUser.h"
+#include "mndVgroup.h"
+#include "tcompare.h"
+#include "tname.h"
+
+int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub) {
+  SSdb*      pSdb = pMnode->pSdb;
+  SVgObj*    pVgroup = NULL;
+  SQueryDag* pDag = qStringToDag(pTopic->physicalPlan);
+  if (pDag == NULL) {
+    terrno = TSDB_CODE_QRY_INVALID_INPUT;
+    return -1;
+  }
+
+  ASSERT(pSub->vgNum == 0);
+
+  int32_t levelNum = taosArrayGetSize(pDag->pSubplans);
+  if (levelNum != 1) {
+    qDestroyQueryDag(pDag);
+    terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC;
+    return -1;
+  }
+
+  SArray* plans = taosArrayGetP(pDag->pSubplans, 0);
+
+  int32_t opNum = taosArrayGetSize(plans);
+  if (opNum != 1) {
+    qDestroyQueryDag(pDag);
+    terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC;
+    return -1;
+  }
+  SSubplan* plan = taosArrayGetP(plans, 0);
+
+  void* pIter = NULL;
+  while (1) {
+    pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup);
+    if (pIter == NULL) break;
+    if (pVgroup->dbUid != pTopic->dbUid) {
+      sdbRelease(pSdb, pVgroup);
+      continue;
+    }
+
+    pSub->vgNum++;
+    plan->execNode.nodeId = pVgroup->vgId;
+    plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup);
+
+    SMqConsumerEp consumerEp = {0};
+    consumerEp.status = 0;
+    consumerEp.consumerId = -1;
+    consumerEp.epSet = plan->execNode.epSet;
+    consumerEp.vgId = plan->execNode.nodeId;
+    int32_t msgLen;
+    if (qSubPlanToString(plan, &consumerEp.qmsg, &msgLen) < 0) {
+      sdbRelease(pSdb, pVgroup);
+      qDestroyQueryDag(pDag);
+      terrno = TSDB_CODE_QRY_INVALID_INPUT;
+      return -1;
+    }
+    taosArrayPush(pSub->unassignedVg, &consumerEp);
+  }
+
+  qDestroyQueryDag(pDag);
+
+  return 0;
+}

source/dnode/mnode/impl/src/mndStream.c

@@ -0,0 +1,444 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mndStream.h"
+#include "mndAuth.h"
+#include "mndDb.h"
+#include "mndDnode.h"
+#include "mndMnode.h"
+#include "mndShow.h"
+#include "mndStb.h"
+#include "mndTrans.h"
+#include "mndUser.h"
+#include "mndVgroup.h"
+#include "tname.h"
+
+#define MND_STREAM_VER_NUMBER   1
+#define MND_STREAM_RESERVE_SIZE 64
+
+static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream);
+static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream);
+static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream);
+static int32_t mndProcessCreateStreamReq(SMnodeMsg *pReq);
+/*static int32_t mndProcessDropStreamReq(SMnodeMsg *pReq);*/
+/*static int32_t mndProcessDropStreamInRsp(SMnodeMsg *pRsp);*/
+static int32_t mndProcessStreamMetaReq(SMnodeMsg *pReq);
+static int32_t mndGetStreamMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta);
+static int32_t mndRetrieveStream(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows);
+static void    mndCancelGetNextStream(SMnode *pMnode, void *pIter);
+
+int32_t mndInitStream(SMnode *pMnode) {
+  SSdbTable table = {.sdbType = SDB_STREAM,
+                     .keyType = SDB_KEY_BINARY,
+                     .encodeFp = (SdbEncodeFp)mndStreamActionEncode,
+                     .decodeFp = (SdbDecodeFp)mndStreamActionDecode,
+                     .insertFp = (SdbInsertFp)mndStreamActionInsert,
+                     .updateFp = (SdbUpdateFp)mndStreamActionUpdate,
+                     .deleteFp = (SdbDeleteFp)mndStreamActionDelete};
+
+  mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STREAM, mndProcessCreateStreamReq);
+  /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM, mndProcessDropStreamReq);*/
+  /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM_RSP, mndProcessDropStreamInRsp);*/
+
+  mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_TP, mndGetStreamMeta);
+  mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TP, mndRetrieveStream);
+  mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TP, mndCancelGetNextStream);
+
+  return sdbSetTable(pMnode->pSdb, table);
+}
+
+void mndCleanupStream(SMnode *pMnode) {}
+
+SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) {
+  terrno = TSDB_CODE_OUT_OF_MEMORY;
+  void *buf = NULL;
+
+  SCoder encoder;
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER);
+  if (tEncodeSStreamObj(NULL, pStream) < 0) {
+    tCoderClear(&encoder);
+    goto STREAM_ENCODE_OVER;
+  }
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+
+  int32_t  size = sizeof(int32_t) + tlen + MND_STREAM_RESERVE_SIZE;
+  SSdbRaw *pRaw = sdbAllocRaw(SDB_STREAM, MND_STREAM_VER_NUMBER, size);
+  if (pRaw == NULL) goto STREAM_ENCODE_OVER;
+
+  buf = malloc(tlen);
+  if (buf == NULL) goto STREAM_ENCODE_OVER;
+
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, tlen, TD_ENCODER);
+  if (tEncodeSStreamObj(NULL, pStream) < 0) {
+    tCoderClear(&encoder);
+    goto STREAM_ENCODE_OVER;
+  }
+  tCoderClear(&encoder);
+
+  int32_t dataPos = 0;
+  SDB_SET_INT32(pRaw, dataPos, tlen, STREAM_ENCODE_OVER);
+  SDB_SET_BINARY(pRaw, dataPos, buf, tlen, STREAM_ENCODE_OVER);
+  SDB_SET_DATALEN(pRaw, dataPos, STREAM_ENCODE_OVER);
+
+  terrno = TSDB_CODE_SUCCESS;
+
+STREAM_ENCODE_OVER:
+  tfree(buf);
+  if (terrno != TSDB_CODE_SUCCESS) {
+    mError("stream:%s, failed to encode to raw:%p since %s", pStream->name, pRaw, terrstr());
+    sdbFreeRaw(pRaw);
+    return NULL;
+  }
+
+  mTrace("stream:%s, encode to raw:%p, row:%p", pStream->name, pRaw, pStream);
+  return pRaw;
+}
+
+SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) {
+  terrno = TSDB_CODE_OUT_OF_MEMORY;
+  void *buf = NULL;
+
+  int8_t sver = 0;
+  if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto STREAM_DECODE_OVER;
+
+  if (sver != MND_STREAM_VER_NUMBER) {
+    terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
+    goto STREAM_DECODE_OVER;
+  }
+
+  int32_t  size = sizeof(SStreamObj);
+  SSdbRow *pRow = sdbAllocRow(size);
+  if (pRow == NULL) goto STREAM_DECODE_OVER;
+
+  SStreamObj *pStream = sdbGetRowObj(pRow);
+  if (pStream == NULL) goto STREAM_DECODE_OVER;
+
+  int32_t tlen;
+  int32_t dataPos = 0;
+  SDB_GET_INT32(pRaw, dataPos, &tlen, STREAM_DECODE_OVER);
+  buf = malloc(tlen + 1);
+  if (buf == NULL) goto STREAM_DECODE_OVER;
+  SDB_GET_BINARY(pRaw, dataPos, buf, tlen, STREAM_DECODE_OVER);
+
+  SCoder decoder;
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, NULL, 0, TD_DECODER);
+  if (tDecodeSStreamObj(&decoder, pStream) < 0) {
+    goto STREAM_DECODE_OVER;
+  }
+
+  terrno = TSDB_CODE_SUCCESS;
+
+STREAM_DECODE_OVER:
+  tfree(buf);
+  if (terrno != TSDB_CODE_SUCCESS) {
+    mError("stream:%s, failed to decode from raw:%p since %s", pStream->name, pRaw, terrstr());
+    tfree(pRow);
+    return NULL;
+  }
+
+  mTrace("stream:%s, decode from raw:%p, row:%p", pStream->name, pRaw, pStream);
+  return pRow;
+}
+
+static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream) {
+  mTrace("stream:%s, perform insert action", pStream->name);
+  return 0;
+}
+
+static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream) {
+  mTrace("stream:%s, perform delete action", pStream->name);
+  return 0;
+}
+
+static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStreamObj *pNewStream) {
+  mTrace("stream:%s, perform update action", pOldStream->name);
+  atomic_exchange_32(&pOldStream->updateTime, pNewStream->updateTime);
+  atomic_exchange_32(&pOldStream->version, pNewStream->version);
+
+  taosWLockLatch(&pOldStream->lock);
+
+  // TODO handle update
+
+  taosWUnLockLatch(&pOldStream->lock);
+  return 0;
+}
+
+SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName) {
+  SSdb       *pSdb = pMnode->pSdb;
+  SStreamObj *pStream = sdbAcquire(pSdb, SDB_STREAM, streamName);
+  if (pStream == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
+    terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
+  }
+  return pStream;
+}
+
+void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream) {
+  SSdb *pSdb = pMnode->pSdb;
+  sdbRelease(pSdb, pStream);
+}
+
+static SDbObj *mndAcquireDbByStream(SMnode *pMnode, char *streamName) {
+  SName name = {0};
+  tNameFromString(&name, streamName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
+
+  char db[TSDB_STREAM_FNAME_LEN] = {0};
+  tNameGetFullDbName(&name, db);
+
+  return mndAcquireDb(pMnode, db);
+}
+
+static int32_t mndCheckCreateStreamReq(SCMCreateStreamReq *pCreate) {
+  if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0) {
+    terrno = TSDB_CODE_MND_INVALID_STREAM_OPTION;
+    return -1;
+  }
+  return 0;
+}
+
+static int32_t mndCreateStream(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateStreamReq *pCreate, SDbObj *pDb) {
+  mDebug("stream:%s to create", pCreate->name);
+  SStreamObj streamObj = {0};
+  tstrncpy(streamObj.name, pCreate->name, TSDB_STREAM_FNAME_LEN);
+  tstrncpy(streamObj.db, pDb->name, TSDB_DB_FNAME_LEN);
+  streamObj.createTime = taosGetTimestampMs();
+  streamObj.updateTime = streamObj.createTime;
+  streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name));
+  streamObj.dbUid = pDb->uid;
+  streamObj.version = 1;
+  streamObj.sql = pCreate->sql;
+  streamObj.physicalPlan = pCreate->physicalPlan;
+  streamObj.logicalPlan = pCreate->logicalPlan;
+
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg);
+  if (pTrans == NULL) {
+    mError("stream:%s, failed to create since %s", pCreate->name, terrstr());
+    return -1;
+  }
+  mDebug("trans:%d, used to create stream:%s", pTrans->id, pCreate->name);
+
+  SSdbRaw *pRedoRaw = mndStreamActionEncode(&streamObj);
+  if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
+    mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
+    mndTransDrop(pTrans);
+    return -1;
+  }
+  sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
+
+  if (mndTransPrepare(pMnode, pTrans) != 0) {
+    mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
+    mndTransDrop(pTrans);
+    return -1;
+  }
+
+  mndTransDrop(pTrans);
+  return 0;
+}
+
+static int32_t mndProcessCreateStreamReq(SMnodeMsg *pReq) {
+  SMnode            *pMnode = pReq->pMnode;
+  int32_t            code = -1;
+  SStreamObj        *pStream = NULL;
+  SDbObj            *pDb = NULL;
+  SUserObj          *pUser = NULL;
+  SCMCreateStreamReq createStreamReq = {0};
+
+  if (tDeserializeSCMCreateStreamReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createStreamReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    goto CREATE_STREAM_OVER;
+  }
+
+  mDebug("stream:%s, start to create, sql:%s", createStreamReq.name, createStreamReq.sql);
+
+  if (mndCheckCreateStreamReq(&createStreamReq) != 0) {
+    mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr());
+    goto CREATE_STREAM_OVER;
+  }
+
+  pStream = mndAcquireStream(pMnode, createStreamReq.name);
+  if (pStream != NULL) {
+    if (createStreamReq.igExists) {
+      mDebug("stream:%s, already exist, ignore exist is set", createStreamReq.name);
+      code = 0;
+      goto CREATE_STREAM_OVER;
+    } else {
+      terrno = TSDB_CODE_MND_STREAM_ALREADY_EXIST;
+      goto CREATE_STREAM_OVER;
+    }
+  } else if (terrno != TSDB_CODE_MND_STREAM_NOT_EXIST) {
+    goto CREATE_STREAM_OVER;
+  }
+
+  pDb = mndAcquireDbByStream(pMnode, createStreamReq.name);
+  if (pDb == NULL) {
+    terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
+    goto CREATE_STREAM_OVER;
+  }
+
+  pUser = mndAcquireUser(pMnode, pReq->user);
+  if (pUser == NULL) {
+    goto CREATE_STREAM_OVER;
+  }
+
+  if (mndCheckWriteAuth(pUser, pDb) != 0) {
+    goto CREATE_STREAM_OVER;
+  }
+
+  code = mndCreateStream(pMnode, pReq, &createStreamReq, pDb);
+  if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
+
+CREATE_STREAM_OVER:
+  if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
+    mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr());
+  }
+
+  mndReleaseStream(pMnode, pStream);
+  mndReleaseDb(pMnode, pDb);
+  mndReleaseUser(pMnode, pUser);
+
+  tFreeSCMCreateStreamReq(&createStreamReq);
+  return code;
+}
+
+static int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams) {
+  SSdb   *pSdb = pMnode->pSdb;
+  SDbObj *pDb = mndAcquireDb(pMnode, dbName);
+  if (pDb == NULL) {
+    terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
+    return -1;
+  }
+
+  int32_t numOfStreams = 0;
+  void   *pIter = NULL;
+  while (1) {
+    SStreamObj *pStream = NULL;
+    pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
+    if (pIter == NULL) break;
+
+    if (pStream->dbUid == pDb->uid) {
+      numOfStreams++;
+    }
+
+    sdbRelease(pSdb, pStream);
+  }
+
+  *pNumOfStreams = numOfStreams;
+  mndReleaseDb(pMnode, pDb);
+  return 0;
+}
+
+static int32_t mndGetStreamMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) {
+  SMnode *pMnode = pReq->pMnode;
+  SSdb   *pSdb = pMnode->pSdb;
+
+  if (mndGetNumOfStreams(pMnode, pShow->db, &pShow->numOfRows) != 0) {
+    return -1;
+  }
+
+  int32_t  cols = 0;
+  SSchema *pSchema = pMeta->pSchemas;
+
+  pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
+  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
+  strcpy(pSchema[cols].name, "name");
+  pSchema[cols].bytes = pShow->bytes[cols];
+  cols++;
+
+  pShow->bytes[cols] = 8;
+  pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
+  strcpy(pSchema[cols].name, "create_time");
+  pSchema[cols].bytes = pShow->bytes[cols];
+  cols++;
+
+  pShow->bytes[cols] = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE;
+  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
+  strcpy(pSchema[cols].name, "sql");
+  pSchema[cols].bytes = pShow->bytes[cols];
+  cols++;
+
+  pMeta->numOfColumns = cols;
+  pShow->numOfColumns = cols;
+
+  pShow->offset[0] = 0;
+  for (int32_t i = 1; i < cols; ++i) {
+    pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
+  }
+
+  pShow->numOfRows = sdbGetSize(pSdb, SDB_STREAM);
+  pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
+  strcpy(pMeta->tbName, mndShowStr(pShow->type));
+
+  return 0;
+}
+
+static int32_t mndRetrieveStream(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) {
+  SMnode     *pMnode = pReq->pMnode;
+  SSdb       *pSdb = pMnode->pSdb;
+  int32_t     numOfRows = 0;
+  SStreamObj *pStream = NULL;
+  int32_t     cols = 0;
+  char       *pWrite;
+  char        prefix[TSDB_DB_FNAME_LEN] = {0};
+
+  SDbObj *pDb = mndAcquireDb(pMnode, pShow->db);
+  if (pDb == NULL) return 0;
+
+  tstrncpy(prefix, pShow->db, TSDB_DB_FNAME_LEN);
+  strcat(prefix, TS_PATH_DELIMITER);
+  int32_t prefixLen = (int32_t)strlen(prefix);
+
+  while (numOfRows < rows) {
+    pShow->pIter = sdbFetch(pSdb, SDB_STREAM, pShow->pIter, (void **)&pStream);
+    if (pShow->pIter == NULL) break;
+
+    if (pStream->dbUid != pDb->uid) {
+      if (strncmp(pStream->name, prefix, prefixLen) != 0) {
+        mError("Inconsistent stream data, name:%s, db:%s, dbUid:%" PRIu64, pStream->name, pDb->name, pDb->uid);
+      }
+
+      sdbRelease(pSdb, pStream);
+      continue;
+    }
+
+    cols = 0;
+
+    char streamName[TSDB_TABLE_NAME_LEN] = {0};
+    tstrncpy(streamName, pStream->name + prefixLen, TSDB_TABLE_NAME_LEN);
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    STR_TO_VARSTR(pWrite, streamName);
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int64_t *)pWrite = pStream->createTime;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pStream->sql, pShow->bytes[cols]);
+    cols++;
+
+    numOfRows++;
+    sdbRelease(pSdb, pStream);
+  }
+
+  mndReleaseDb(pMnode, pDb);
+  pShow->numOfReads += numOfRows;
+  mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
+  return numOfRows;
+}
+
+static void mndCancelGetNextStream(SMnode *pMnode, void *pIter) {
+  SSdb *pSdb = pMnode->pSdb;
+  sdbCancelFetch(pSdb, pIter);
+}

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -20,6 +20,7 @@
 #include "mndDnode.h"
 #include "mndMnode.h"
 #include "mndOffset.h"
+#include "mndScheduler.h"
 #include "mndShow.h"
 #include "mndStb.h"
 #include "mndTopic.h"
@@ -39,7 +40,7 @@ enum {
   MQ_SUBSCRIBE_STATUS__DELETED,
 };
 
-static char *mndMakeSubscribeKey(const char *cgroup, const char *topicName);
+static int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName);
 
 static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *);
 static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw);
@@ -87,22 +88,24 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
-  char *key = mndMakeSubscribeKey(cgroup, pTopic->name);
+  char key[TSDB_SUBSCRIBE_KEY_LEN];
-  if (key == NULL) {
+  mndMakeSubscribeKey(key, cgroup, pTopic->name);
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
+  strcpy(pSub->key, key);
+
+  if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) {
     tDeleteSMqSubscribeObj(pSub);
     free(pSub);
     return NULL;
   }
-  strcpy(pSub->key, key);
-  free(key);
 
+#if 0
   if (mndInitUnassignedVg(pMnode, pTopic, pSub) < 0) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     tDeleteSMqSubscribeObj(pSub);
     free(pSub);
     return NULL;
   }
+#endif
   // TODO: disable alter subscribed table
   return pSub;
 }
@@ -291,7 +294,11 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) {
           for (int32_t k = 0; k < vgsz; k++) {
             char           offsetKey[TSDB_PARTITION_KEY_LEN];
             SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, k);
-            SMqSubVgEp     vgEp = {.epSet = pConsumerEp->epSet, .vgId = pConsumerEp->vgId, .offset = -1};
+            SMqSubVgEp     vgEp = {
+                    .epSet = pConsumerEp->epSet,
+                    .vgId = pConsumerEp->vgId,
+                    .offset = -1,
+            };
             mndMakePartitionKey(offsetKey, pConsumer->cgroup, topicName, pConsumerEp->vgId);
             SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, offsetKey);
             if (pOffsetObj != NULL) {
@@ -325,15 +332,14 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) {
   return 0;
 }
 
-static int32_t mndSplitSubscribeKey(char *key, char **topic, char **cgroup) {
+static int32_t mndSplitSubscribeKey(const char *key, char *topic, char *cgroup) {
   int32_t i = 0;
-  while (key[i] != ':') {
+  while (key[i] != TMQ_SEPARATOR) {
     i++;
   }
-  key[i] = 0;
+  memcpy(topic, key, i - 1);
-  *cgroup = strdup(key);
+  topic[i] = 0;
-  key[i] = ':';
+  strcpy(cgroup, &key[i + 1]);
-  *topic = strdup(&key[i + 1]);
   return 0;
 }
 
@@ -342,7 +348,7 @@ static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) {
   if (pRebSub == NULL) {
     pRebSub = tNewSMqRebSubscribe(key);
     if (pRebSub == NULL) {
-      // TODO
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
       return NULL;
     }
     taosHashPut(pHash, key, strlen(key), pRebSub, sizeof(SMqRebSubscribe));
@@ -370,7 +376,8 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) {
         int32_t sz = taosArrayGetSize(pConsumer->currentTopics);
         for (int32_t i = 0; i < sz; i++) {
           char *topic = taosArrayGetP(pConsumer->currentTopics, i);
-          char            *key = mndMakeSubscribeKey(pConsumer->cgroup, topic);
+          char  key[TSDB_SUBSCRIBE_KEY_LEN];
+          mndMakeSubscribeKey(key, pConsumer->cgroup, topic);
           SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key);
           taosArrayPush(pRebSub->lostConsumers, &pConsumer->consumerId);
         }
@@ -387,7 +394,8 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) {
       int32_t sz = taosArrayGetSize(rebSubs);
       for (int32_t i = 0; i < sz; i++) {
         char *topic = taosArrayGetP(rebSubs, i);
-        char            *key = mndMakeSubscribeKey(pConsumer->cgroup, topic);
+        char  key[TSDB_SUBSCRIBE_KEY_LEN];
+        mndMakeSubscribeKey(key, pConsumer->cgroup, topic);
         SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key);
         if (status == MQ_CONSUMER_STATUS__INIT) {
           taosArrayPush(pRebSub->newConsumers, &pConsumer->consumerId);
@@ -407,7 +415,11 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) {
   }
   if (taosHashGetSize(pRebMsg->rebSubHash) != 0) {
     mInfo("mq rebalance will be triggered");
-    SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_DO_REBALANCE, .pCont = pRebMsg, .contLen = sizeof(SMqDoRebalanceMsg)};
+    SRpcMsg rpcMsg = {
+        .msgType = TDMT_MND_MQ_DO_REBALANCE,
+        .pCont = pRebMsg,
+        .contLen = sizeof(SMqDoRebalanceMsg),
+    };
     pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg);
   } else {
     taosHashCleanup(pRebMsg->rebSubHash);
@@ -520,9 +532,9 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) {
             taosArrayPush(pSubConsumer->vgInfo, pConsumerEp);
 
             if (pConsumerEp->oldConsumerId == -1) {
-              char *topic;
+              char topic[TSDB_TOPIC_FNAME_LEN];
-              char *cgroup;
+              char cgroup[TSDB_CGROUP_LEN];
-              mndSplitSubscribeKey(pSub->key, &topic, &cgroup);
+              mndSplitSubscribeKey(pSub->key, topic, cgroup);
               SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
 
               mInfo("mq set conn: assign vgroup %d of topic %s to consumer %ld", pConsumerEp->vgId, topic,
@@ -530,8 +542,6 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) {
 
               mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp);
               mndReleaseTopic(pMnode, pTopic);
-              free(topic);
-              free(cgroup);
             } else {
               mInfo("mq rebalance: assign vgroup %d, from consumer %ld to consumer %ld", pConsumerEp->vgId,
                     pConsumerEp->oldConsumerId, pConsumerEp->consumerId);
@@ -759,6 +769,7 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) {
 }
 #endif
 
+#if 0
 static int32_t mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSubscribeObj *pSub) {
   SSdb      *pSdb = pMnode->pSdb;
   SVgObj    *pVgroup = NULL;
@@ -804,6 +815,7 @@ static int32_t mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SM
   /*qDestroyQueryDag(pDag);*/
   return 0;
 }
+#endif
 
 static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup,
                                       const SMqConsumerEp *pConsumerEp) {
@@ -949,23 +961,19 @@ static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubsc
   return 0;
 }
 
-static char *mndMakeSubscribeKey(const char *cgroup, const char *topicName) {
+static int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName) {
-  char *key = malloc(TSDB_SHOW_SUBQUERY_LEN);
-  if (key == NULL) {
-    return NULL;
-  }
   int32_t tlen = strlen(cgroup);
   memcpy(key, cgroup, tlen);
-  key[tlen] = ':';
+  key[tlen] = TMQ_SEPARATOR;
   strcpy(key + tlen + 1, topicName);
-  return key;
+  return 0;
 }
 
 SMqSubscribeObj *mndAcquireSubscribe(SMnode *pMnode, const char *cgroup, const char *topicName) {
   SSdb *pSdb = pMnode->pSdb;
-  char            *key = mndMakeSubscribeKey(cgroup, topicName);
+  char  key[TSDB_SUBSCRIBE_KEY_LEN];
+  mndMakeSubscribeKey(key, cgroup, topicName);
   SMqSubscribeObj *pSub = sdbAcquire(pSdb, SDB_SUBSCRIBE, key);
-  free(key);
   if (pSub == NULL) {
     terrno = TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST;
   }

source/dnode/mnode/impl/src/mndTelem.c

@@ -87,7 +87,7 @@ static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) {
   taosWLockLatch(&pMgmt->lock);
   char* pCont = mndBuildTelemetryReport(pMnode);
   if (pCont != NULL) {
-    taosSendHttpReport(TELEMETRY_SERVER, TELEMETRY_PORT, pCont, strlen(pCont));
+    taosSendHttpReport(TELEMETRY_SERVER, TELEMETRY_PORT, pCont, strlen(pCont), HTTP_FLAT);
     free(pCont);
   }
   taosWUnLockLatch(&pMgmt->lock);

source/dnode/mnode/impl/src/mndTopic.c

@@ -13,7 +13,6 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#define _DEFAULT_SOURCE
 #include "mndTopic.h"
 #include "mndAuth.h"
 #include "mndDb.h"
@@ -229,7 +228,7 @@ static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMq
   return pDrop;
 }
 
-static int32_t mndCheckCreateTopicReq(SMCreateTopicReq *pCreate) {
+static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) {
   if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0) {
     terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION;
     return -1;
@@ -237,7 +236,7 @@ static int32_t mndCheckCreateTopicReq(SMCreateTopicReq *pCreate) {
   return 0;
 }
 
-static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SMCreateTopicReq *pCreate, SDbObj *pDb) {
+static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb) {
   mDebug("topic:%s to create", pCreate->name);
   SMqTopicObj topicObj = {0};
   tstrncpy(topicObj.name, pCreate->name, TSDB_TOPIC_FNAME_LEN);
@@ -283,9 +282,9 @@ static int32_t mndProcessCreateTopicReq(SMnodeMsg *pReq) {
   SMqTopicObj      *pTopic = NULL;
   SDbObj           *pDb = NULL;
   SUserObj         *pUser = NULL;
-  SMCreateTopicReq createTopicReq = {0};
+  SCMCreateTopicReq createTopicReq = {0};
 
-  if (tDeserializeSMCreateTopicReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createTopicReq) != 0) {
+  if (tDeserializeSCMCreateTopicReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createTopicReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
     goto CREATE_TOPIC_OVER;
   }
@@ -338,7 +337,7 @@ CREATE_TOPIC_OVER:
   mndReleaseDb(pMnode, pDb);
   mndReleaseUser(pMnode, pUser);
 
-  tFreeSMCreateTopicReq(&createTopicReq);
+  tFreeSCMCreateTopicReq(&createTopicReq);
   return code;
 }
 
@@ -409,35 +408,6 @@ static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pRsp) {
   return 0;
 }
 
-#if 0
-static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) {
-  SSdb   *pSdb = pMnode->pSdb;
-  SDbObj *pDb = mndAcquireDb(pMnode, dbName);
-  if (pDb == NULL) {
-    terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
-    return -1;
-  }
-
-  int32_t numOfTopics = 0;
-  void   *pIter = NULL;
-  while (1) {
-    SMqTopicObj *pTopic = NULL;
-    pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic);
-    if (pIter == NULL) break;
-
-    if (pTopic->dbUid == pDb->uid) {
-      numOfTopics++;
-    }
-
-    sdbRelease(pSdb, pTopic);
-  }
-
-  *pNumOfTopics = numOfTopics;
-  mndReleaseDb(pMnode, pDb);
-  return 0;
-}
-#endif
-
 static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) {
   SSdb   *pSdb = pMnode->pSdb;
   SDbObj *pDb = mndAcquireDb(pMnode, dbName);

source/dnode/mnode/impl/src/mnode.c

@@ -96,7 +96,11 @@ static void mndCalMqRebalance(void *param, void *tmrId) {
   if (mndIsMaster(pMnode)) {
     int32_t contLen = 0;
     void   *pReq = mndBuildTimerMsg(&contLen);
-    SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_TIMER, .pCont = pReq, .contLen = contLen};
+    SRpcMsg rpcMsg = {
+        .msgType = TDMT_MND_MQ_TIMER,
+        .pCont = pReq,
+        .contLen = contLen,
+    };
     pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg);
   }
 

source/dnode/mnode/impl/test/db/db.cpp

@@ -292,9 +292,9 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) {
       EXPECT_GT(pInfo->vgId, 0);
       EXPECT_EQ(pInfo->hashBegin, 0);
       EXPECT_EQ(pInfo->hashEnd, UINT32_MAX / 2 - 1);
-      EXPECT_EQ(pInfo->epset.inUse, 0);
+      EXPECT_EQ(pInfo->epSet.inUse, 0);
-      EXPECT_EQ(pInfo->epset.numOfEps, 1);
+      EXPECT_EQ(pInfo->epSet.numOfEps, 1);
-      SEp* pAddr = &pInfo->epset.eps[0];
+      SEp* pAddr = &pInfo->epSet.eps[0];
       EXPECT_EQ(pAddr->port, 9030);
       EXPECT_STREQ(pAddr->fqdn, "localhost");
     }
@@ -307,9 +307,9 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) {
       EXPECT_GT(pInfo->vgId, 0);
       EXPECT_EQ(pInfo->hashBegin, UINT32_MAX / 2);
       EXPECT_EQ(pInfo->hashEnd, UINT32_MAX);
-      EXPECT_EQ(pInfo->epset.inUse, 0);
+      EXPECT_EQ(pInfo->epSet.inUse, 0);
-      EXPECT_EQ(pInfo->epset.numOfEps, 1);
+      EXPECT_EQ(pInfo->epSet.numOfEps, 1);
-      SEp* pAddr = &pInfo->epset.eps[0];
+      SEp* pAddr = &pInfo->epSet.eps[0];
       EXPECT_EQ(pAddr->port, 9030);
       EXPECT_STREQ(pAddr->fqdn, "localhost");
     }

source/dnode/mnode/impl/test/topic/topic.cpp

@@ -61,16 +61,16 @@ void* MndTestTopic::BuildCreateDbReq(const char* dbname, int32_t* pContLen) {
 }
 
 void* MndTestTopic::BuildCreateTopicReq(const char* topicName, const char* sql, int32_t* pContLen) {
-  SMCreateTopicReq createReq = {0};
+  SCMCreateTopicReq createReq = {0};
   strcpy(createReq.name, topicName);
   createReq.igExists = 0;
   createReq.sql = (char*)sql;
   createReq.physicalPlan = (char*)"physicalPlan";
   createReq.logicalPlan = (char*)"logicalPlan";
 
-  int32_t contLen = tSerializeMCreateTopicReq(NULL, 0, &createReq);
+  int32_t contLen = tSerializeSCMCreateTopicReq(NULL, 0, &createReq);
   void*   pReq = rpcMallocCont(contLen);
-  tSerializeMCreateTopicReq(pReq, contLen, &createReq);
+  tSerializeSCMCreateTopicReq(pReq, contLen, &createReq);
 
   *pContLen = contLen;
   return pReq;
@@ -100,9 +100,7 @@ TEST_F(MndTestTopic, 01_Create_Topic) {
     ASSERT_EQ(pRsp->code, 0);
   }
 
-  {
+  { test.SendShowMetaReq(TSDB_MGMT_TABLE_TP, ""); }
-    test.SendShowMetaReq(TSDB_MGMT_TABLE_TP, "");
-  }
 
   {
     int32_t  contLen = 0;

source/dnode/mnode/sdb/inc/sdbInt.h

@@ -28,12 +28,12 @@
 extern "C" {
 #endif
 
-#define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", 255, __VA_ARGS__); }}
+#define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
-#define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", 255, __VA_ARGS__); }}
+#define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
-#define mWarn(...)  { if (mDebugFlag & DEBUG_WARN)  { taosPrintLog("MND WARN ", 255, __VA_ARGS__); }}
+#define mWarn(...)  { if (mDebugFlag & DEBUG_WARN)  { taosPrintLog("MND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
-#define mInfo(...)  { if (mDebugFlag & DEBUG_INFO)  { taosPrintLog("MND ", 255, __VA_ARGS__); }}
+#define mInfo(...)  { if (mDebugFlag & DEBUG_INFO)  { taosPrintLog("MND ", DEBUG_INFO, 255, __VA_ARGS__); }}
-#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }}
+#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }}
-#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }}
+#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }}
 
 #define SDB_MAX_SIZE (32 * 1024)
 

source/dnode/snode/inc/sndInt.h

@@ -20,6 +20,7 @@
 
 #include "tlog.h"
 #include "tmsg.h"
+#include "tqueue.h"
 #include "trpc.h"
 
 #include "snode.h"
@@ -28,10 +29,49 @@
 extern "C" {
 #endif
 
+enum {
+  STREAM_STATUS__READY = 1,
+  STREAM_STATUS__STOPPED,
+  STREAM_STATUS__CREATING,
+  STREAM_STATUS__STOPING,
+  STREAM_STATUS__RESUMING,
+  STREAM_STATUS__DELETING,
+};
+
+enum {
+  STREAM_RUNNER__RUNNING = 1,
+  STREAM_RUNNER__STOP,
+};
+
 typedef struct SSnode {
   SSnodeOpt cfg;
 } SSnode;
 
+typedef struct {
+  int64_t streamId;
+  int32_t IdxInLevel;
+  int32_t level;
+} SStreamInfo;
+
+typedef struct {
+  SStreamInfo meta;
+  int8_t      status;
+  void*       executor;
+  STaosQueue* queue;
+  void*       stateStore;
+  // storage handle
+} SStreamRunner;
+
+typedef struct {
+  SHashObj* pHash;
+} SStreamMeta;
+
+int32_t sndCreateStream();
+int32_t sndDropStream();
+
+int32_t sndStopStream();
+int32_t sndResumeStream();
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/snode/src/snode.c

@@ -17,6 +17,7 @@
 
 SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
   SSnode *pSnode = calloc(1, sizeof(SSnode));
+  memcpy(&pSnode->cfg, pOption, sizeof(SSnodeOpt));
   return pSnode;
 }
 

source/dnode/vnode/CMakeLists.txt

@@ -54,5 +54,5 @@ elseif(${META_DB_IMPL} STREQUAL "TDB")
 endif()
 
 if(${BUILD_TEST})
-    # add_subdirectory(test)
+    add_subdirectory(test)
 endif(${BUILD_TEST})

source/dnode/vnode/inc/meta.h

@@ -38,22 +38,29 @@ typedef struct SMetaCfg {
 
 typedef struct SMTbCursor  SMTbCursor;
 typedef struct SMCtbCursor SMCtbCursor;
+typedef struct SMSmaCursor SMSmaCursor;
 
 typedef SVCreateTbReq   STbCfg;
+typedef SVCreateTSmaReq SSmaCfg;
 
 // SMeta operations
-SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF);
+SMeta * metaOpen(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF);
 void    metaClose(SMeta *pMeta);
 void    metaRemove(const char *path);
 int     metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg);
 int     metaDropTable(SMeta *pMeta, tb_uid_t uid);
 int     metaCommit(SMeta *pMeta);
+int32_t metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg);
+int32_t metaDropTSma(SMeta *pMeta, char *indexName);
 
 // For Query
 STbCfg *        metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid);
 STbCfg *        metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid);
 SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline);
 STSchema *      metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver);
+STSma *         metaGetSmaInfoByName(SMeta *pMeta, const char *indexName);
+STSmaWrapper *  metaGetSmaInfoByUid(SMeta *pMeta, tb_uid_t uid);
+SArray *        metaGetSmaTbUids(SMeta *pMeta, bool isDup);
 
 SMTbCursor *metaOpenTbCursor(SMeta *pMeta);
 void        metaCloseTbCursor(SMTbCursor *pTbCur);
@@ -63,6 +70,10 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid);
 void         metaCloseCtbCurosr(SMCtbCursor *pCtbCur);
 tb_uid_t     metaCtbCursorNext(SMCtbCursor *pCtbCur);
 
+SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid);
+void         metaCloseSmaCurosr(SMSmaCursor *pSmaCur);
+const char * metaSmaCursorNext(SMSmaCursor *pSmaCur);
+
 // Options
 void metaOptionsInit(SMetaCfg *pMetaCfg);
 void metaOptionsClear(SMetaCfg *pMetaCfg);

source/dnode/vnode/inc/tq.h

@@ -13,16 +13,14 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_TQ_H_
+#ifndef _TQ_H_
-#define _TD_TQ_H_
+#define _TQ_H_
 
-#include "tcommon.h"
 #include "executor.h"
-#include "tmallocator.h"
 #include "meta.h"
-#include "scheduler.h"
 #include "taoserror.h"
-#include "tlist.h"
+#include "tcommon.h"
+#include "tmallocator.h"
 #include "tmsg.h"
 #include "trpc.h"
 #include "ttimer.h"
@@ -54,7 +52,7 @@ void tqClose(STQ*);
 int tqPushMsg(STQ*, void* msg, tmsg_t msgType, int64_t version);
 int tqCommit(STQ*);
 
-int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg);
+int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessSetConnReq(STQ* pTq, char* msg);
 int32_t tqProcessRebReq(STQ* pTq, char* msg);
 
@@ -62,4 +60,4 @@ int32_t tqProcessRebReq(STQ* pTq, char* msg);
 }
 #endif
 
-#endif /*_TD_TQ_H_*/
+#endif /*_TQ_H_*/

source/dnode/vnode/inc/tsdb.h

@@ -87,6 +87,27 @@ int    tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp);
 int    tsdbPrepareCommit(STsdb *pTsdb);
 int    tsdbCommit(STsdb *pTsdb);
 
+/**
+ * @brief Insert tSma(Time-range-wise SMA) data from stream computing engine
+ * 
+ * @param pTsdb 
+ * @param param 
+ * @param pData 
+ * @return int32_t 
+ */
+int32_t tsdbInsertTSmaData(STsdb *pTsdb, STSma *param, STSmaData *pData);
+
+/**
+ * @brief Insert RSma(Time-range-wise Rollup SMA) data.
+ * 
+ * @param pTsdb 
+ * @param param 
+ * @param pData 
+ * @return int32_t 
+ */
+int32_t tsdbInsertRSmaData(STsdb *pTsdb, SRSma *param, STSmaData *pData);
+
+
 // STsdbCfg
 int  tsdbOptionsInit(STsdbCfg *);
 void tsdbOptionsClear(STsdbCfg *);

source/dnode/vnode/inc/vnode.h

@@ -202,6 +202,22 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
 
 /* ------------------------- TQ READ --------------------------- */
 
+enum {
+  TQ_STREAM_TOKEN__DATA = 1,
+  TQ_STREAM_TOKEN__WATERMARK,
+  TQ_STREAM_TOKEN__CHECKPOINT,
+};
+
+typedef struct {
+  int8_t type;
+  int8_t reserved[7];
+  union {
+    void   *data;
+    int64_t wmTs;
+    int64_t checkpointId;
+  };
+} STqStreamToken;
+
 STqReadHandle *tqInitSubmitMsgScanner(SMeta *pMeta);
 
 static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SArray *pColIdList) {
@@ -213,6 +229,10 @@ static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SA
 //}
 
 static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList) {
+  if (pHandle->tbIdHash) {
+    taosHashClear(pHandle->tbIdHash);
+  }
+
   pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
   if (pHandle->tbIdHash == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -227,6 +247,23 @@ static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const S
   return 0;
 }
 
+static FORCE_INLINE int tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList) {
+  if (pHandle->tbIdHash == NULL) {
+    pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
+    if (pHandle->tbIdHash == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return -1;
+    }
+  }
+
+  for (int i = 0; i < taosArrayGetSize(tbUidList); i++) {
+    int64_t *pKey = (int64_t *)taosArrayGet(tbUidList, i);
+    taosHashPut(pHandle->tbIdHash, pKey, sizeof(int64_t), NULL, 0);
+  }
+
+  return 0;
+}
+
 int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver);
 bool    tqNextDataBlock(STqReadHandle *pHandle);
 int     tqRetrieveDataBlockInfo(STqReadHandle *pHandle, SDataBlockInfo *pBlockInfo);

source/dnode/vnode/src/inc/metaDef.h

@@ -33,6 +33,8 @@ int  metaOpenDB(SMeta* pMeta);
 void metaCloseDB(SMeta* pMeta);
 int  metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg);
 int  metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid);
+int  metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg);
+int  metaRemoveSmaFromDb(SMeta* pMeta, const char* indexName);
 
 // SMetaCache
 int  metaOpenCache(SMeta* pMeta);

source/dnode/vnode/src/inc/tqInt.h

@@ -19,48 +19,51 @@
 #include "meta.h"
 #include "tlog.h"
 #include "tq.h"
-#include "trpc.h"
+#include "tqPush.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-extern int32_t tqDebugFlag;
-
 #define tqFatal(...)                                             \
   {                                                              \
     if (tqDebugFlag & DEBUG_FATAL) {                             \
-      taosPrintLog("TQ  FATAL ", 255, __VA_ARGS__); \
+      taosPrintLog("TQ  FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \
     }                                                            \
   }
+
 #define tqError(...)                                             \
   {                                                              \
     if (tqDebugFlag & DEBUG_ERROR) {                             \
-      taosPrintLog("TQ  ERROR ", 255, __VA_ARGS__); \
+      taosPrintLog("TQ  ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \
     }                                                            \
   }
+
 #define tqWarn(...)                                            \
   {                                                            \
     if (tqDebugFlag & DEBUG_WARN) {                            \
-      taosPrintLog("TQ  WARN ", 255, __VA_ARGS__); \
+      taosPrintLog("TQ  WARN ", DEBUG_WARN, 255, __VA_ARGS__); \
     }                                                          \
   }
+
 #define tqInfo(...)                                       \
   {                                                       \
     if (tqDebugFlag & DEBUG_INFO) {                       \
-      taosPrintLog("TQ  ", 255, __VA_ARGS__); \
+      taosPrintLog("TQ  ", DEBUG_INFO, 255, __VA_ARGS__); \
     }                                                     \
   }
+
 #define tqDebug(...)                                               \
   {                                                                \
     if (tqDebugFlag & DEBUG_DEBUG) {                               \
-      taosPrintLog("TQ  ", tqDebugFlag, __VA_ARGS__); \
+      taosPrintLog("TQ  ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); \
     }                                                              \
   }
+
 #define tqTrace(...)                                               \
   {                                                                \
     if (tqDebugFlag & DEBUG_TRACE) {                               \
-      taosPrintLog("TQ  ", tqDebugFlag, __VA_ARGS__); \
+      taosPrintLog("TQ  ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); \
     }                                                              \
   }
 
@@ -140,9 +143,7 @@ typedef struct {
   // topics that are not connectted
   STqMetaList* unconnectTopic;
 
-  // TODO:temporaral use, to be replaced by unified tfile
   TdFilePtr pFile;
-  // TODO:temporaral use, to be replaced by unified tfile
   TdFilePtr pIdxFile;
 
   char*          dirPath;
@@ -159,6 +160,7 @@ struct STQ {
   STqCfg*       tqConfig;
   STqMemRef     tqMemRef;
   STqMetaStore* tqMeta;
+  STqPushMgr*   tqPushMgr;
   SWal*         pWal;
   SMeta*        pVnodeMeta;
 };

source/dnode/vnode/src/inc/tqPush.h

@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TQ_PUSH_H_
+#define _TQ_PUSH_H_
+
+#include "executor.h"
+#include "thash.h"
+#include "trpc.h"
+#include "ttimer.h"
+#include "vnode.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum {
+  TQ_PUSHER_TYPE__CLIENT = 1,
+  TQ_PUSHER_TYPE__STREAM,
+};
+
+typedef struct {
+  int8_t   type;
+  int8_t   reserved[3];
+  int32_t  ttl;
+  int64_t  consumerId;
+  SRpcMsg* pMsg;
+  // SMqPollRsp* rsp;
+} STqClientPusher;
+
+typedef struct {
+  int8_t      type;
+  int8_t      nodeType;
+  int8_t      reserved[6];
+  int64_t     streamId;
+  qTaskInfo_t task;
+  // TODO sync function
+} STqStreamPusher;
+
+typedef struct {
+  int8_t type;  // mq or stream
+} STqPusher;
+
+typedef struct {
+  SHashObj* pHash;  // <id, STqPush*>
+} STqPushMgr;
+
+typedef struct {
+  int8_t inited;
+  tmr_h  timer;
+} STqPushMgmt;
+
+static STqPushMgmt tqPushMgmt;
+
+int32_t tqPushMgrInit();
+void    tqPushMgrCleanUp();
+
+STqPushMgr* tqPushMgrOpen();
+void        tqPushMgrClose(STqPushMgr* pushMgr);
+
+STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl);
+STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TQ_PUSH_H_*/

source/dnode/vnode/src/inc/tsdbCommit.h

@@ -16,6 +16,10 @@
 #ifndef _TD_TSDB_COMMIT_H_
 #define _TD_TSDB_COMMIT_H_
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 typedef struct {
   int   minFid;
   int   midFid;
@@ -66,4 +70,8 @@ int   tsdbApplyRtn(STsdbRepo *pRepo);
 
 #endif
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif /* _TD_TSDB_COMMIT_H_ */

source/dnode/vnode/src/inc/tsdbDef.h

@@ -35,6 +35,7 @@
 #include "tsdbMemory.h"
 #include "tsdbOptions.h"
 #include "tsdbReadImpl.h"
+#include "tsdbSma.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -51,6 +52,7 @@ struct STsdb {
   STsdbFS *             fs;
   SMeta *               pMeta;
   STfs *                pTfs;
+  SSmaStat *            pSmaStat;
 };
 
 #define REPO_ID(r) ((r)->vgId)

source/dnode/vnode/src/inc/tsdbFS.h

@@ -18,6 +18,10 @@
 
 #include "tsdbFile.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 // ================== TSDB global config
 extern bool tsdbForceKeepFile;
 
@@ -38,9 +42,31 @@ typedef struct {
 typedef struct {
   STsdbFSMeta meta;  // FS meta
   SArray *    df;    // data file array
+
+  // SArray *    v2t100.index_name
+  
+  SArray *    smaf;  // sma data file array    v2t1900.index_name
 } SFSStatus;
 
-typedef struct {
+/**
+ * @brief Directory structure of .tsma data files.
+ * 
+ *  root@cary /vnode2/tsdb $ tree .tsma/
+ *   .tsma/
+ *   ├── v2t100.index_name_1
+ *   ├── v2t101.index_name_1
+ *   ├── v2t102.index_name_1
+ *   ├── v2t1900.index_name_3
+ *   ├── v2t1901.index_name_3
+ *   ├── v2t1902.index_name_3
+ *   ├── v2t200.index_name_2
+ *   ├── v2t201.index_name_2
+ *   └── v2t202.index_name_2
+ *
+ *   0 directories, 9 files
+ */
+
+    typedef struct {
   pthread_rwlock_t lock;
 
   SFSStatus *cstatus;        // current status
@@ -111,4 +137,8 @@ static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) {
   return 0;
 }
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif /* _TD_TSDB_FS_H_ */

source/dnode/vnode/src/inc/tsdbFile.h

@@ -19,6 +19,10 @@
 #include "tchecksum.h"
 #include "tfs.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 #define TSDB_FILE_HEAD_SIZE 512
 #define TSDB_FILE_DELIMITER 0xF00AFA0F
 #define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF
@@ -52,10 +56,11 @@ typedef enum {
   TSDB_FILE_SMAL,      // .smal(Block-wise SMA)
   TSDB_FILE_MAX,       //
   TSDB_FILE_META,      // meta
-  TSDB_FILE_TSMA,      // .tsma.${sma_index_name}, Time-range-wise SMA
+  TSDB_FILE_TSMA,      // v2t100.${sma_index_name}, Time-range-wise SMA
-  TSDB_FILE_RSMA,      // .rsma.${sma_index_name}, Time-range-wise Rollup SMA
+  TSDB_FILE_RSMA,      // v2r100.${sma_index_name}, Time-range-wise Rollup SMA
-} TSDB_FILE_T;
+} E_TSDB_FILE_T;
 
+typedef int32_t TSDB_FILE_T;
 typedef enum {
   TSDB_FS_VER_0 = 0,
   TSDB_FS_VER_MAX,
@@ -410,4 +415,8 @@ static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet* pSet) {
   return true;
 }
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif /* _TS_TSDB_FILE_H_ */

source/dnode/vnode/src/inc/tsdbLog.h

@@ -18,13 +18,21 @@
 
 #include "tlog.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 extern int32_t tsdbDebugFlag;
 
-#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }}     while(0)
+#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}     while(0)
-#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }}     while(0)
+#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}     while(0)
-#define tsdbWarn(...)  do { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }}      while(0)
+#define tsdbWarn(...)  do { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TDB WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}      while(0)
-#define tsdbInfo(...)  do { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TDB ", 255, __VA_ARGS__); }}           while(0)
+#define tsdbInfo(...)  do { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TDB ", DEBUG_INFO, 255, __VA_ARGS__); }}           while(0)
-#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
+#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0)
-#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
+#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
+
+#ifdef __cplusplus
+}
+#endif
 
 #endif /* _TD_TSDB_LOG_H_ */

source/dnode/vnode/src/inc/tsdbMemory.h

@@ -16,6 +16,10 @@
 #ifndef _TD_TSDB_MEMORY_H_
 #define _TD_TSDB_MEMORY_H_
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 static void * taosTMalloc(size_t size);
 static void * taosTCalloc(size_t nmemb, size_t size);
 static void * taosTRealloc(void *ptr, size_t size);
@@ -70,5 +74,8 @@ static FORCE_INLINE void* taosTZfree(void* ptr) {
   return NULL;
 }
 
+#ifdef __cplusplus
+}
+#endif
 
 #endif /* _TD_TSDB_MEMORY_H_ */

source/dnode/vnode/src/inc/tsdbReadImpl.h

@@ -24,6 +24,10 @@
 #include "tsdbMemory.h"
 #include "tcommon.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 typedef struct SReadH SReadH;
 
 typedef struct {
@@ -244,4 +248,8 @@ static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) {
   return 0;
 }
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif /*_TD_TSDB_READ_IMPL_H_*/

source/dnode/vnode/src/inc/tsdbSma.h

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_SMA_H_
+#define _TD_TSDB_SMA_H_
+
+typedef struct SSmaStat SSmaStat;
+
+// insert/update interface
+int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData);
+int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, SRSma *param, STSmaData *pData);
+
+
+// query interface
+// TODO: This is the basic params, and should wrap the params to a queryHandle.
+int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData, STimeWindow *queryWin, int32_t nMaxResult);
+
+// management interface
+int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg);
+int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result);
+int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin);
+int32_t tsdbDestroySmaState(SSmaStat *pSmaStat);
+
+// internal func
+
+
+static FORCE_INLINE int32_t tsdbEncodeTSmaKey(uint64_t tableUid, col_id_t colId, TSKEY tsKey, void **pData) {
+  int32_t len = 0;
+  len += taosEncodeFixedU64(pData, tableUid);
+  len += taosEncodeFixedU16(pData, colId);
+  len += taosEncodeFixedI64(pData, tsKey);
+  return len;
+}
+
+#endif /* _TD_TSDB_SMA_H_ */

source/dnode/vnode/src/inc/vnd.h

@@ -88,43 +88,40 @@ int vnodeScheduleTask(SVnodeTask* task);
 int32_t vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq);
 void    vnodeSendReqToDnode(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq);
 
-// For Log
-extern int32_t vDebugFlag;
-
 #define vFatal(...)                                              \
   do {                                                           \
     if (vDebugFlag & DEBUG_FATAL) {                              \
-      taosPrintLog("VND FATAL ", 255, __VA_ARGS__); \
+      taosPrintLog("VND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \
     }                                                            \
   } while (0)
 #define vError(...)                                              \
   do {                                                           \
     if (vDebugFlag & DEBUG_ERROR) {                              \
-      taosPrintLog("VND ERROR ", 255, __VA_ARGS__); \
+      taosPrintLog("VND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \
     }                                                            \
   } while (0)
 #define vWarn(...)                                             \
   do {                                                         \
     if (vDebugFlag & DEBUG_WARN) {                             \
-      taosPrintLog("VND WARN ", 255, __VA_ARGS__); \
+      taosPrintLog("VND WARN ", DEBUG_WARN, 255, __VA_ARGS__); \
     }                                                          \
   } while (0)
 #define vInfo(...)                                        \
   do {                                                    \
     if (vDebugFlag & DEBUG_INFO) {                        \
-      taosPrintLog("VND ", 255, __VA_ARGS__); \
+      taosPrintLog("VND ", DEBUG_INFO, 255, __VA_ARGS__); \
     }                                                     \
   } while (0)
 #define vDebug(...)                                                  \
   do {                                                               \
     if (vDebugFlag & DEBUG_DEBUG) {                                  \
-      taosPrintLog("VND ", tsdbDebugFlag, __VA_ARGS__); \
+      taosPrintLog("VND ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); \
     }                                                                \
   } while (0)
 #define vTrace(...)                                                  \
   do {                                                               \
     if (vDebugFlag & DEBUG_TRACE) {                                  \
-      taosPrintLog("VND ", tsdbDebugFlag, __VA_ARGS__); \
+      taosPrintLog("VND ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); \
     }                                                                \
   } while (0)
 

source/dnode/vnode/src/meta/metaBDBImpl.c

@@ -38,11 +38,14 @@ struct SMetaDB {
   // DB
   DB *pTbDB;
   DB *pSchemaDB;
+  DB *pSmaDB;
+
   // IDX
   DB *pNameIdx;
   DB *pStbIdx;
   DB *pNtbIdx;
   DB *pCtbIdx;
+  DB *pSmaIdx;
   // ENV
   DB_ENV *pEvn;
 };
@@ -61,11 +64,12 @@ static int      metaNameIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT
 static int      metaStbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey);
 static int      metaNtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey);
 static int      metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey);
+static int      metaSmaIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey);
 static int      metaEncodeTbInfo(void **buf, STbCfg *pTbCfg);
-static void *   metaDecodeTbInfo(void *buf, STbCfg *pTbCfg);
+static void    *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg);
 static void     metaClearTbCfg(STbCfg *pTbCfg);
 static int      metaEncodeSchema(void **buf, SSchemaWrapper *pSW);
-static void *   metaDecodeSchema(void *buf, SSchemaWrapper *pSW);
+static void    *metaDecodeSchema(void *buf, SSchemaWrapper *pSW);
 static void     metaDBWLock(SMetaDB *pDB);
 static void     metaDBRLock(SMetaDB *pDB);
 static void     metaDBULock(SMetaDB *pDB);
@@ -100,6 +104,11 @@ int metaOpenDB(SMeta *pMeta) {
     return -1;
   }
 
+  if (metaOpenBDBDb(&(pDB->pSmaDB), pDB->pEvn, "sma.db", false) < 0) {
+    metaCloseDB(pMeta);
+    return -1;
+  }
+
   // Open Indices
   if (metaOpenBDBIdx(&(pDB->pNameIdx), pDB->pEvn, "name.index", pDB->pTbDB, &metaNameIdxCb, false) < 0) {
     metaCloseDB(pMeta);
@@ -121,15 +130,22 @@ int metaOpenDB(SMeta *pMeta) {
     return -1;
   }
 
+  if (metaOpenBDBIdx(&(pDB->pSmaIdx), pDB->pEvn, "sma.index", pDB->pSmaDB, &metaSmaIdxCb, true) < 0) {
+    metaCloseDB(pMeta);
+    return -1;
+  }
+
   return 0;
 }
 
 void metaCloseDB(SMeta *pMeta) {
   if (pMeta->pDB) {
+    metaCloseBDBIdx(pMeta->pDB->pSmaIdx);
     metaCloseBDBIdx(pMeta->pDB->pCtbIdx);
     metaCloseBDBIdx(pMeta->pDB->pNtbIdx);
     metaCloseBDBIdx(pMeta->pDB->pStbIdx);
     metaCloseBDBIdx(pMeta->pDB->pNameIdx);
+    metaCloseBDBDb(pMeta->pDB->pSmaDB);
     metaCloseBDBDb(pMeta->pDB->pSchemaDB);
     metaCloseBDBDb(pMeta->pDB->pTbDB);
     metaCloseBDBEnv(pMeta->pDB->pEvn);
@@ -142,7 +158,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
   tb_uid_t uid;
   char     buf[512];
   char     buf1[512];
-  void *   pBuf;
+  void    *pBuf;
   DBT      key1, value1;
   DBT      key2, value2;
   SSchema *pSchema = NULL;
@@ -210,6 +226,49 @@ int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) {
   return 0;
 }
 
+int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) {
+  char  buf[512] = {0};  // TODO: may overflow
+  void *pBuf = NULL;
+  DBT   key1 = {0}, value1 = {0};
+
+  {
+    // save sma info
+    pBuf = buf;
+
+    key1.data = pSmaCfg->indexName;
+    key1.size = strlen(key1.data);
+
+    tEncodeTSma(&pBuf, pSmaCfg);
+
+    value1.data = buf;
+    value1.size = POINTER_DISTANCE(pBuf, buf);
+    value1.app_data = pSmaCfg;
+  }
+
+  metaDBWLock(pMeta->pDB);
+  pMeta->pDB->pSmaDB->put(pMeta->pDB->pSmaDB, NULL, &key1, &value1, 0);
+  metaDBULock(pMeta->pDB);
+
+  return 0;
+}
+
+int metaRemoveSmaFromDb(SMeta *pMeta, const char *indexName) {
+  // TODO
+#if 0
+  DBT key = {0};
+
+  key.data = (void *)indexName;
+  key.size = strlen(indexName);
+
+  metaDBWLock(pMeta->pDB);
+  // TODO: No guarantee of consistence.
+  // Use transaction or DB->sync() for some guarantee.
+  pMeta->pDB->pSmaDB->del(pMeta->pDB->pSmaDB, NULL, &key, 0);
+  metaDBULock(pMeta->pDB);
+#endif
+  return 0;
+}
+
 /* ------------------------ STATIC METHODS ------------------------ */
 static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW) {
   int      tlen = 0;
@@ -394,7 +453,7 @@ static int metaNtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey
 
 static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey) {
   STbCfg *pTbCfg = (STbCfg *)(pValue->app_data);
-  DBT *   pDbt;
+  DBT    *pDbt;
 
   if (pTbCfg->type == META_CHILD_TABLE) {
     // pDbt = calloc(2, sizeof(DBT));
@@ -425,6 +484,16 @@ static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey
   }
 }
 
+static int metaSmaIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey) {
+  STSma *pSmaCfg = (STSma *)(pValue->app_data);
+
+  memset(pSKey, 0, sizeof(*pSKey));
+  pSKey->data = &(pSmaCfg->tableUid);
+  pSKey->size = sizeof(pSmaCfg->tableUid);
+
+  return 0;
+}
+
 static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) {
   int tsize = 0;
 
@@ -479,7 +548,7 @@ static void metaClearTbCfg(STbCfg *pTbCfg) {
 
 /* ------------------------ FOR QUERY ------------------------ */
 STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) {
-  STbCfg * pTbCfg = NULL;
+  STbCfg  *pTbCfg = NULL;
   SMetaDB *pDB = pMeta->pDB;
   DBT      key = {0};
   DBT      value = {0};
@@ -509,7 +578,7 @@ STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) {
 }
 
 STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) {
-  STbCfg * pTbCfg = NULL;
+  STbCfg  *pTbCfg = NULL;
   SMetaDB *pDB = pMeta->pDB;
   DBT      key = {0};
   DBT      pkey = {0};
@@ -540,13 +609,43 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) {
   return pTbCfg;
 }
 
+STSma *metaGetSmaInfoByName(SMeta *pMeta, const char *indexName) {
+  STSma *  pCfg = NULL;
+  SMetaDB *pDB = pMeta->pDB;
+  DBT      key = {0};
+  DBT      value = {0};
+  int      ret;
+
+  // Set key/value
+  key.data = (void *)indexName;
+  key.size = strlen(indexName);
+
+  // Query
+  metaDBRLock(pDB);
+  ret = pDB->pTbDB->get(pDB->pSmaDB, NULL, &key, &value, 0);
+  metaDBULock(pDB);
+  if (ret != 0) {
+    return NULL;
+  }
+
+  // Decode
+  pCfg = (STSma *)malloc(sizeof(STSma));
+  if (pCfg == NULL) {
+    return NULL;
+  }
+
+  tDecodeTSma(value.data, pCfg);
+
+  return pCfg;
+}
+
 SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) {
   uint32_t        nCols;
   SSchemaWrapper *pSW = NULL;
-  SMetaDB *       pDB = pMeta->pDB;
+  SMetaDB        *pDB = pMeta->pDB;
   int             ret;
-  void *          pBuf;
+  void           *pBuf;
-  SSchema *       pSchema;
+  SSchema        *pSchema;
   SSchemaKey      schemaKey = {uid, sver, 0};
   DBT             key = {0};
   DBT             value = {0};
@@ -578,7 +677,7 @@ struct SMTbCursor {
 
 SMTbCursor *metaOpenTbCursor(SMeta *pMeta) {
   SMTbCursor *pTbCur = NULL;
-  SMetaDB *   pDB = pMeta->pDB;
+  SMetaDB    *pDB = pMeta->pDB;
 
   pTbCur = (SMTbCursor *)calloc(1, sizeof(*pTbCur));
   if (pTbCur == NULL) {
@@ -609,7 +708,7 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) {
   DBT    key = {0};
   DBT    value = {0};
   STbCfg tbCfg;
-  void * pBuf;
+  void  *pBuf;
 
   for (;;) {
     if (pTbCur->pCur->get(pTbCur->pCur, &key, &value, DB_NEXT) == 0) {
@@ -631,10 +730,10 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) {
 
 STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
   STSchemaBuilder sb;
-  STSchema *      pTSchema = NULL;
+  STSchema       *pTSchema = NULL;
-  SSchema *       pSchema;
+  SSchema        *pSchema;
   SSchemaWrapper *pSW;
-  STbCfg *        pTbCfg;
+  STbCfg         *pTbCfg;
   tb_uid_t        quid;
 
   pTbCfg = metaGetTbInfoByUid(pMeta, uid);
@@ -662,13 +761,13 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
 }
 
 struct SMCtbCursor {
-  DBC *    pCur;
+  DBC     *pCur;
   tb_uid_t suid;
 };
 
 SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) {
   SMCtbCursor *pCtbCur = NULL;
-  SMetaDB *    pDB = pMeta->pDB;
+  SMetaDB     *pDB = pMeta->pDB;
   int          ret;
 
   pCtbCur = (SMCtbCursor *)calloc(1, sizeof(*pCtbCur));
@@ -700,7 +799,7 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) {
   DBT    skey = {0};
   DBT    pkey = {0};
   DBT    pval = {0};
-  void * pBuf;
+  void  *pBuf;
   STbCfg tbCfg;
 
   // Set key
@@ -718,6 +817,142 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) {
   }
 }
 
+struct SMSmaCursor {
+  DBC     *pCur;
+  tb_uid_t uid;
+};
+
+SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) {
+  SMSmaCursor *pCur = NULL;
+  SMetaDB     *pDB = pMeta->pDB;
+  int          ret;
+
+  pCur = (SMSmaCursor *)calloc(1, sizeof(*pCur));
+  if (pCur == NULL) {
+    return NULL;
+  }
+
+  pCur->uid = uid;
+  // TODO: lock?
+  ret = pDB->pCtbIdx->cursor(pDB->pSmaIdx, NULL, &(pCur->pCur), 0);
+  if (ret != 0) {
+    free(pCur);
+    return NULL;
+  }
+
+  return pCur;
+}
+
+void metaCloseSmaCurosr(SMSmaCursor *pCur) {
+  if (pCur) {
+    if (pCur->pCur) {
+      pCur->pCur->close(pCur->pCur);
+    }
+
+    free(pCur);
+  }
+}
+
+const char *metaSmaCursorNext(SMSmaCursor *pCur) {
+  DBT skey = {0};
+  DBT pkey = {0};
+  DBT pval = {0};
+
+  // Set key
+  skey.data = &(pCur->uid);
+  skey.size = sizeof(pCur->uid);
+  // TODO: lock?
+  if (pCur->pCur->pget(pCur->pCur, &skey, &pkey, &pval, DB_NEXT) == 0) {
+    const char *indexName = (const char *)pkey.data;
+    assert(indexName != NULL);
+    return indexName;
+  } else {
+    return NULL;
+  }
+}
+
+STSmaWrapper *metaGetSmaInfoByUid(SMeta *pMeta, tb_uid_t uid) {
+  STSmaWrapper *pSW = NULL;
+
+  pSW = calloc(sizeof(*pSW), 1);
+  if (pSW == NULL) {
+    return NULL;
+  }
+
+  SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid);
+  if (pCur == NULL) {
+    free(pSW);
+    return NULL;
+  }
+
+  DBT   skey = {.data = &(pCur->uid), .size = sizeof(pCur->uid)};
+  DBT   pval = {0};
+  void *pBuf = NULL;
+
+  while (true) {
+    // TODO: lock?
+    if (pCur->pCur->pget(pCur->pCur, &skey, NULL, &pval, DB_NEXT) == 0) {
+      ++pSW->number;
+      STSma *tptr = (STSma *)realloc(pSW->tSma, pSW->number * sizeof(STSma));
+      if (tptr == NULL) {
+        metaCloseSmaCurosr(pCur);
+        tdDestroyTSmaWrapper(pSW);
+        tfree(pSW);
+        return NULL;
+      }
+      pSW->tSma = tptr;
+      pBuf = pval.data;
+      if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) {
+        metaCloseSmaCurosr(pCur);
+        tdDestroyTSmaWrapper(pSW);
+        tfree(pSW);
+        return NULL;
+      }
+      continue;
+    }
+    break;
+  }
+
+  metaCloseSmaCurosr(pCur);
+  
+  return pSW;
+}
+
+SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) {
+  SArray * pUids = NULL;
+  SMetaDB *pDB = pMeta->pDB;
+  DBC *    pCur = NULL;
+  DBT      pkey = {0}, pval = {0};
+  uint32_t mode = isDup ? DB_NEXT_DUP : DB_NEXT_NODUP;
+  int      ret;
+
+  pUids = taosArrayInit(16, sizeof(tb_uid_t));
+
+  if (!pUids) {
+    return NULL;
+  }
+
+  // TODO: lock?
+  ret = pDB->pCtbIdx->cursor(pDB->pSmaIdx, NULL, &pCur, 0);
+  if (ret != 0) {
+    taosArrayDestroy(pUids);
+    return NULL;
+  }
+
+  void *pBuf = NULL;
+
+  // TODO: lock?
+  while ((ret = pCur->get(pCur, &pkey, &pval, mode)) == 0) {
+      taosArrayPush(pUids, pkey.data);
+  }
+
+  if (pCur) {
+    pCur->close(pCur);
+  }
+
+  return pUids;
+}
+
 static void metaDBWLock(SMetaDB *pDB) {
 #if IMPL_WITH_LOCK
   pthread_rwlock_wrlock(&(pDB->rwlock));

source/dnode/vnode/src/meta/metaIdx.c

@@ -106,3 +106,28 @@ int metaRemoveTableFromIdx(SMeta *pMeta, tb_uid_t uid) {
   // TODO
   return 0;
 }
+
+int32_t metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg) {
+  // TODO: Validate the cfg
+  // The table uid should exists and be super table or common table.
+  // Check other cfg value
+
+  // TODO: add atomicity
+
+  if (metaSaveSmaToDB(pMeta, &pCfg->tSma) < 0) {
+    // TODO: handle error
+    return -1;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t metaDropTSma(SMeta *pMeta, char* indexName) {
+  // TODO: Validate the cfg
+  // TODO: add atomicity
+
+  if (metaRemoveSmaFromDb(pMeta, indexName) < 0) {
+    // TODO: handle error
+    return -1;
+  }
+  return TSDB_CODE_SUCCESS;
+}

source/dnode/vnode/src/meta/metaTable.c

@@ -50,3 +50,4 @@ int metaDropTable(SMeta *pMeta, tb_uid_t uid) {
 
   return 0;
 }
+

source/dnode/vnode/src/tq/tq.c

@@ -12,28 +12,16 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#define _DEFAULT_SOURCE
 
 #include "tcompare.h"
 #include "tqInt.h"
 #include "tqMetaStore.h"
 
-int tqInit() {
+int32_t tqInit() { return tqPushMgrInit(); }
-  int8_t old = atomic_val_compare_exchange_8(&tqMgmt.inited, 0, 1);
-  if (old == 1) return 0;
 
-  tqMgmt.timer = taosTmrInit(0, 0, 0, "TQ");
+void tqCleanUp() { tqPushMgrCleanUp(); }
-  return 0;
-}
 
-void tqCleanUp() {
+STQ* tqOpen(const char* path, SWal* pWal, SMeta* pVnodeMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) {
-  int8_t old = atomic_val_compare_exchange_8(&tqMgmt.inited, 1, 0);
-  if (old == 0) return;
-  taosTmrStop(tqMgmt.timer);
-  taosTmrCleanUp(tqMgmt.timer);
-}
-
-STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) {
   STQ* pTq = malloc(sizeof(STQ));
   if (pTq == NULL) {
     terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
@@ -42,7 +30,7 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl
   pTq->path = strdup(path);
   pTq->tqConfig = tqConfig;
   pTq->pWal = pWal;
-  pTq->pVnodeMeta = pMeta;
+  pTq->pVnodeMeta = pVnodeMeta;
 #if 0
   pTq->tqMemRef.pAllocatorFactory = allocFac;
   pTq->tqMemRef.pAllocator = allocFac->create(allocFac);
@@ -60,6 +48,13 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl
     return NULL;
   }
 
+  pTq->tqPushMgr = tqPushMgrOpen();
+  if (pTq->tqPushMgr == NULL) {
+    // free store
+    free(pTq);
+    return NULL;
+  }
+
   return pTq;
 }
 
@@ -72,6 +67,30 @@ void tqClose(STQ* pTq) {
 }
 
 int tqPushMsg(STQ* pTq, void* msg, tmsg_t msgType, int64_t version) {
+  if (msgType != TDMT_VND_SUBMIT) return 0;
+  void* pIter = taosHashIterate(pTq->tqPushMgr->pHash, NULL);
+  while (pIter != NULL) {
+    STqPusher* pusher = *(STqPusher**)pIter;
+    if (pusher->type == TQ_PUSHER_TYPE__STREAM) {
+      STqStreamPusher* streamPusher = (STqStreamPusher*)pusher;
+      // repack
+      STqStreamToken* token = malloc(sizeof(STqStreamToken));
+      if (token == NULL) {
+        taosHashCancelIterate(pTq->tqPushMgr->pHash, pIter);
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        return -1;
+      }
+      token->type = TQ_STREAM_TOKEN__DATA;
+      token->data = msg;
+      // set input
+      // exec
+    }
+    // send msg to ep
+  }
+  // iterate hash
+  // process all msg
+  // if waiting
+  // memcpy and send msg to fetch thread
   // TODO: add reference
   // if handle waiting, launch query and response to consumer
   //
@@ -197,7 +216,10 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu
     for (int j = 0; j < TQ_BUFFER_SIZE; j++) {
       pTopic->buffer.output[j].status = 0;
       STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta);
-      SReadHandle    handle = {.reader = pReadHandle, .meta = pTq->pVnodeMeta};
+      SReadHandle    handle = {
+             .reader = pReadHandle,
+             .meta = pTq->pVnodeMeta,
+      };
       pTopic->buffer.output[j].pReadHandle = pReadHandle;
       pTopic->buffer.output[j].task = qCreateStreamExecTaskInfo(pTopic->qmsg, &handle);
     }
@@ -206,11 +228,11 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu
   return 0;
 }
 
-int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
+int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
-  SMqConsumeReq* pReq = pMsg->pCont;
+  SMqPollReq* pReq = pMsg->pCont;
   int64_t     consumerId = pReq->consumerId;
   int64_t     fetchOffset;
-  /*int64_t        blockingTime = pReq->blockingTime;*/
+  int64_t     blockingTime = pReq->blockingTime;
 
   if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) {
     fetchOffset = 0;
@@ -220,7 +242,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
     fetchOffset = pReq->currentOffset + 1;
   }
 
-  SMqConsumeRsp rsp = {
+  SMqPollRsp rsp = {
       .consumerId = consumerId,
       .numOfTopics = 0,
       .pBlockData = NULL,
@@ -234,6 +256,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
     rpcSendResponse(pMsg);
     return 0;
   }
+
   int sz = taosArrayGetSize(pConsumer->topics);
   ASSERT(sz == 1);
   STqTopic* pTopic = taosArrayGet(pConsumer->topics, 0);
@@ -245,13 +268,14 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
 
   SWalHead* pHead;
   while (1) {
-    int8_t pos = fetchOffset % TQ_BUFFER_SIZE;
+    /*if (fetchOffset > walGetLastVer(pTq->pWal) || walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {*/
     if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {
       // TODO: no more log, set timer to wait blocking time
       // if data inserted during waiting, launch query and
       // response to user
       break;
     }
+    int8_t pos = fetchOffset % TQ_BUFFER_SIZE;
     pHead = pTopic->pReadhandle->pHead;
     if (pHead->head.msgType == TDMT_VND_SUBMIT) {
       SSubmitReq* pCont = (SSubmitReq*)&pHead->head.body;
@@ -278,7 +302,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
         rsp.numOfTopics = 1;
         rsp.pBlockData = pRes;
 
-        int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqConsumeRsp(NULL, &rsp);
+        int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRsp(NULL, &rsp);
         void*   buf = rpcMallocCont(tlen);
         if (buf == NULL) {
           pMsg->code = -1;
@@ -288,7 +312,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
         ((SMqRspHead*)buf)->epoch = pReq->epoch;
 
         void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
-        tEncodeSMqConsumeRsp(&abuf, &rsp);
+        tEncodeSMqPollRsp(&abuf, &rsp);
         taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock);
         pMsg->pCont = buf;
         pMsg->contLen = tlen;
@@ -302,7 +326,10 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
     }
   }
 
-  int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqConsumeRsp(NULL, &rsp);
+  /*if (blockingTime != 0) {*/
+  /*tqAddClientPusher(pTq->tqPushMgr, pMsg, consumerId, blockingTime);*/
+  /*} else {*/
+  int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRsp(NULL, &rsp);
   void*   buf = rpcMallocCont(tlen);
   if (buf == NULL) {
     pMsg->code = -1;
@@ -312,12 +339,14 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
   ((SMqRspHead*)buf)->epoch = pReq->epoch;
 
   void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
-  tEncodeSMqConsumeRsp(&abuf, &rsp);
+  tEncodeSMqPollRsp(&abuf, &rsp);
   rsp.pBlockData = NULL;
   pMsg->pCont = buf;
   pMsg->contLen = tlen;
   pMsg->code = 0;
   rpcSendResponse(pMsg);
+  /*}*/
+
   return 0;
 }
 
@@ -374,7 +403,10 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
   for (int i = 0; i < TQ_BUFFER_SIZE; i++) {
     pTopic->buffer.output[i].status = 0;
     STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta);
-    SReadHandle    handle = {.reader = pReadHandle, .meta = pTq->pVnodeMeta};
+    SReadHandle    handle = {
+           .reader = pReadHandle,
+           .meta = pTq->pVnodeMeta,
+    };
     pTopic->buffer.output[i].pReadHandle = pReadHandle;
     pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, &handle);
   }

source/dnode/vnode/src/tq/tqPush.c

@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tqPush.h"
+
+int32_t tqPushMgrInit() {
+  //
+  int8_t old = atomic_val_compare_exchange_8(&tqPushMgmt.inited, 0, 1);
+  if (old == 1) return 0;
+
+  tqPushMgmt.timer = taosTmrInit(0, 0, 0, "TQ");
+  return 0;
+}
+
+void tqPushMgrCleanUp() {
+  int8_t old = atomic_val_compare_exchange_8(&tqPushMgmt.inited, 1, 0);
+  if (old == 0) return;
+  taosTmrStop(tqPushMgmt.timer);
+  taosTmrCleanUp(tqPushMgmt.timer);
+}
+
+STqPushMgr* tqPushMgrOpen() {
+  STqPushMgr* mgr = malloc(sizeof(STqPushMgr));
+  if (mgr == NULL) {
+    return NULL;
+  }
+  mgr->pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
+  return mgr;
+}
+
+void tqPushMgrClose(STqPushMgr* pushMgr) {
+  taosHashCleanup(pushMgr->pHash);
+  free(pushMgr);
+}
+
+STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl) {
+  STqClientPusher* clientPusher = malloc(sizeof(STqClientPusher));
+  if (clientPusher == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+  clientPusher->type = TQ_PUSHER_TYPE__CLIENT;
+  clientPusher->pMsg = pMsg;
+  clientPusher->consumerId = consumerId;
+  clientPusher->ttl = ttl;
+  if (taosHashPut(pushMgr->pHash, &consumerId, sizeof(int64_t), &clientPusher, sizeof(void*)) < 0) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    free(clientPusher);
+    // TODO send rsp back
+    return NULL;
+  }
+  return clientPusher;
+}
+
+STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet) {
+  STqStreamPusher* streamPusher = malloc(sizeof(STqStreamPusher));
+  if (streamPusher == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+  streamPusher->type = TQ_PUSHER_TYPE__STREAM;
+  streamPusher->nodeType = 0;
+  streamPusher->streamId = streamId;
+  /*memcpy(&streamPusher->epSet, pEpSet, sizeof(SEpSet));*/
+
+  if (taosHashPut(pushMgr->pHash, &streamId, sizeof(int64_t), &streamPusher, sizeof(void*)) < 0) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    free(streamPusher);
+    return NULL;
+  }
+  return streamPusher;
+}

source/dnode/vnode/src/tq/tqRead.c

@@ -12,7 +12,6 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#define _DEFAULT_SOURCE
 
 #include "vnode.h"
 
@@ -28,6 +27,7 @@ STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) {
   pReadHandle->sver = -1;
   pReadHandle->pSchema = NULL;
   pReadHandle->pSchemaWrapper = NULL;
+  pReadHandle->tbIdHash = NULL;
   return pReadHandle;
 }
 
@@ -36,6 +36,7 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t
   pMsg->length = htonl(pMsg->length);
   pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
 
+  // iterate and convert
   if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1;
   while (true) {
     if (tGetSubmitMsgNext(&pReadHandle->msgIter, &pReadHandle->pBlock) < 0) return -1;

source/dnode/vnode/src/tsdb/tsdbBDBImpl.c

@@ -0,0 +1,14 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -1054,7 +1054,7 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) {
 //     pfs->metaCacheComp = NULL;
 //   } else {
 //     // remove meta.tmp file
-//     remove(mf.f.aname);
+//     taosRemoveFile(mf.f.aname);
 //     taosHashCleanup(pfs->metaCacheComp);
 //     pfs->metaCacheComp = NULL;
 //   }

source/dnode/vnode/src/tsdb/tsdbFS.c

@@ -439,7 +439,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
   if (taosWriteFile(pFile, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     taosCloseFile(&pFile);
-    remove(tfname);
+    taosRemoveFile(tfname);
     return -1;
   }
 
@@ -447,7 +447,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
   if (fsheader.len > 0) {
     if (tsdbMakeRoom(&(pBuf), fsheader.len) < 0) {
       taosCloseFile(&pFile);
-      remove(tfname);
+      taosRemoveFile(tfname);
       return -1;
     }
 
@@ -458,7 +458,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
     if (taosWriteFile(pFile, pBuf, fsheader.len) < fsheader.len) {
       terrno = TAOS_SYSTEM_ERROR(errno);
       taosCloseFile(&pFile);
-      (void)remove(tfname);
+      (void)taosRemoveFile(tfname);
       taosTZfree(pBuf);
       return -1;
     }
@@ -468,7 +468,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
   if (taosFsyncFile(pFile) < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     taosCloseFile(&pFile);
-    remove(tfname);
+    taosRemoveFile(tfname);
     taosTZfree(pBuf);
     return -1;
   }

source/dnode/vnode/src/tsdb/tsdbFile.c

@@ -23,8 +23,8 @@ static const char *TSDB_FNAME_SUFFIX[] = {
     "smal",  // TSDB_FILE_SMAL
     "",      // TSDB_FILE_MAX
     "meta",  // TSDB_FILE_META
-    "tsma",  // TSDB_FILE_TSMA
+    "sma",   // TSDB_FILE_TSMA(directory name)
-    "rsma",  // TSDB_FILE_RSMA
+    "sma",   // TSDB_FILE_RSMA(directory name)
 };
 
 static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname);
@@ -365,7 +365,7 @@ int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader, TSDB_FILE_T
     if (errno == ENOENT) {
       // Try to create directory recursively
       char *s = strdup(TSDB_FILE_REL_NAME(pDFile));
-      if (tfsMkdirRecurAt(pRepo->pTfs, dirname(s), TSDB_FILE_DID(pDFile)) < 0) {
+      if (tfsMkdirRecurAt(pRepo->pTfs, taosDirName(s), TSDB_FILE_DID(pDFile)) < 0) {
         tfree(s);
         return -1;
       }

source/dnode/vnode/src/tsdb/tsdbMain.c

@@ -89,6 +89,7 @@ static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg,
 static void tsdbFree(STsdb *pTsdb) {
   if (pTsdb) {
     tsdbFreeFS(pTsdb->fs);
+    tsdbDestroySmaState(pTsdb->pSmaStat);
     tfree(pTsdb->path);
     free(pTsdb);
   }

source/dnode/vnode/src/tsdb/tsdbSma.c

@@ -0,0 +1,729 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tsdbDef.h"
+
+#define SMA_STORAGE_TSDB_DAYS   30
+#define SMA_STORAGE_SPLIT_HOURS 24
+#define SMA_KEY_LEN             18  // tableUid_colId_TSKEY 8+2+8
+
+#define SMA_STORE_SINGLE_BLOCKS  // store SMA data by single block or multiple blocks
+
+#define SMA_STATE_HASH_SLOT      4
+#define SMA_STATE_ITEM_HASH_SLOT 32
+
+#define SMA_TEST_INDEX_NAME "smaTestIndexName"  // TODO: just for test
+typedef enum {
+  SMA_STORAGE_LEVEL_TSDB = 0,     // store TSma in dir  e.g. vnode${N}/tsdb/.tsma
+  SMA_STORAGE_LEVEL_DFILESET = 1  // store TSma in file e.g. vnode${N}/tsdb/v2f1900.tsma.${sma_index_name}
+} ESmaStorageLevel;
+
+typedef struct {
+  STsdb * pTsdb;
+  char *  pDFile;     // TODO: use the real DFile type, not char*
+  int32_t interval;   // interval with the precision of DB
+  int32_t blockSize;  // size of SMA block item
+  // TODO
+} STSmaWriteH;
+
+typedef struct {
+  int32_t iter;
+} SmaFsIter;
+typedef struct {
+  STsdb *   pTsdb;
+  char *    pDFile;     // TODO: use the real DFile type, not char*
+  int32_t   interval;   // interval with the precision of DB
+  int32_t   blockSize;  // size of SMA block item
+  int8_t    storageLevel;
+  int8_t    days;
+  SmaFsIter smaFsIter;
+  // TODO
+} STSmaReadH;
+
+typedef struct {
+  /**
+   * @brief The field 'state' is here to demonstrate if one smaIndex is ready to provide service.
+   *    - TSDB_SMA_STAT_EXPIRED: 1) If sma calculation of history TS data is not finished; 2) Or if the TSDB is open,
+   * without information about its previous state.
+   *    - TSDB_SMA_STAT_OK: 1) The sma calculation of history data is finished; 2) Or recevied information from
+   * Streaming Module or TSDB local persistence.
+   */
+  int8_t    state;           // ETsdbSmaStat
+  SHashObj *expiredWindows;  // key: skey of time window, value: N/A
+} SSmaStatItem;
+
+struct SSmaStat {
+  SHashObj *smaStatItems;  // key: indexName, value: SSmaStatItem
+};
+
+// declaration of static functions
+static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData);
+static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData);
+static int32_t tsdbJudgeStorageLevel(int64_t interval, int8_t intervalUnit);
+static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaData *pData, int32_t sectionDataLen, int32_t nBlocks);
+static int32_t tsdbInsertTSmaBlocks(void *bTree, const char *smaKey, const char *pData, int32_t dataLen);
+static int32_t tsdbTSmaDataSplit(STSmaWriteH *pSmaH, STSma *param, STSmaData *pData, int32_t days, int32_t nOffset,
+                                 int32_t fid, int32_t *nSmaBlocks);
+static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision);
+static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSma *param, STSmaData *pData, int32_t storageLevel,
+                                   int32_t fid);
+
+static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData);
+static int32_t tsdbInitTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWindow *queryWin);
+static bool    tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWindow *queryWin);
+
+static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) {
+  ASSERT(pSmaStat != NULL);
+
+  if (*pSmaStat != NULL) {  // no lock
+    return TSDB_CODE_SUCCESS;
+  }
+
+  // TODO: lock. lazy mode when update expired window, or hungry mode during tsdbNew.
+  if (*pSmaStat == NULL) {
+    *pSmaStat = (SSmaStat *)calloc(1, sizeof(SSmaStat));
+    if (*pSmaStat == NULL) {
+      // TODO: unlock
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return TSDB_CODE_FAILED;
+    }
+
+    (*pSmaStat)->smaStatItems =
+        taosHashInit(SMA_STATE_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+
+    if ((*pSmaStat)->smaStatItems == NULL) {
+      tfree(*pSmaStat);
+      // TODO: unlock
+      return TSDB_CODE_FAILED;
+    }
+  }
+  // TODO: unlock
+  return TSDB_CODE_SUCCESS;
+}
+
+static SSmaStatItem *tsdbNewSmaStatItem(int8_t state) {
+  SSmaStatItem *pItem = NULL;
+
+  pItem = (SSmaStatItem *)calloc(1, sizeof(SSmaStatItem));
+  if (pItem) {
+    pItem->state = state;
+    pItem->expiredWindows = taosHashInit(SMA_STATE_ITEM_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP),
+                                         true, HASH_ENTRY_LOCK);
+    if (!pItem->expiredWindows) {
+      tfree(pItem);
+    }
+  }
+  return pItem;
+}
+
+int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) {
+  if (pSmaStat) {
+    // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready.
+    SSmaStatItem *item = taosHashIterate(pSmaStat->smaStatItems, NULL);
+    while (item != NULL) {
+      taosHashCleanup(item->expiredWindows);
+      item = taosHashIterate(pSmaStat->smaStatItems, item);
+    }
+
+    taosHashCleanup(pSmaStat->smaStatItems);
+    free(pSmaStat);
+  }
+}
+
+/**
+ * @brief Update expired window according to msg from stream computing module.
+ *
+ * @param pTsdb
+ * @param msg
+ * @return int32_t
+ */
+int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg) {
+  if (msg == NULL) {
+    return TSDB_CODE_FAILED;
+  }
+
+  tsdbInitSmaStat(&pTsdb->pSmaStat);  // lazy mode
+
+  // TODO: decode the msg => start
+  const char *  indexName = SMA_TEST_INDEX_NAME;
+  const int32_t SMA_TEST_EXPIRED_WINDOW_SIZE = 10;
+  TSKEY         expiredWindows[SMA_TEST_EXPIRED_WINDOW_SIZE];
+  int64_t       now = taosGetTimestampMs();
+  for (int32_t i = 0; i < SMA_TEST_EXPIRED_WINDOW_SIZE; ++i) {
+    expiredWindows[i] = now + i;
+  }
+
+  // TODO: decode the msg <= end
+  SHashObj *pItemsHash = pTsdb->pSmaStat->smaStatItems;
+
+  SSmaStatItem *pItem = (SSmaStatItem *)taosHashGet(pItemsHash, indexName, strlen(indexName));
+  if (!pItem) {
+    pItem = tsdbNewSmaStatItem(TSDB_SMA_STAT_EXPIRED);  // TODO use the real state
+    if (!pItem) {
+      // Response to stream computing: OOM
+      // For query, if the indexName not found, the TSDB should tell query module to query raw TS data.
+      return TSDB_CODE_FAILED;
+    }
+
+    if (taosHashPut(pItemsHash, indexName, strnlen(indexName, TSDB_INDEX_NAME_LEN), &pItem, sizeof(pItem)) != 0) {
+      // If error occurs during put smaStatItem, free the resources of pItem
+      taosHashCleanup(pItem->expiredWindows);
+      free(pItem);
+      return TSDB_CODE_FAILED;
+    }
+  }
+
+  int8_t state = TSDB_SMA_STAT_EXPIRED;
+  for (int32_t i = 0; i < SMA_TEST_EXPIRED_WINDOW_SIZE; ++i) {
+    if (taosHashPut(pItem->expiredWindows, &expiredWindows[i], sizeof(TSKEY), &state, sizeof(state)) != 0) {
+      // If error occurs during taosHashPut expired windows, remove the smaIndex from pTsdb->pSmaStat, thus TSDB would
+      // tell query module to query raw TS data.
+      // N.B.
+      //  1) It is assumed to be extemely little probability event of fail to taosHashPut.
+      //  2) This would solve the inconsistency to some extent, but not completely, unless we record all expired
+      // windows failed to put into hash table.
+      taosHashCleanup(pItem->expiredWindows);
+      taosHashRemove(pItemsHash, indexName, sizeof(indexName));
+      return TSDB_CODE_FAILED;
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, const char *indexName, void *timeWindow) {
+  SSmaStatItem *pItem = NULL;
+
+  if (pTsdb->pSmaStat && pTsdb->pSmaStat->smaStatItems) {
+    pItem = (SSmaStatItem *)taosHashGet(pTsdb->pSmaStat->smaStatItems, indexName, strlen(indexName));
+  }
+
+  if (pItem != NULL) {
+    // TODO: reset time windows for the sma data blocks
+    while (true) {
+      TSKEY thisWindow = 0;
+      taosHashRemove(pItem->expiredWindows, &thisWindow, sizeof(thisWindow));
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief Judge the tSma storage level
+ *
+ * @param interval
+ * @param intervalUnit
+ * @return int32_t
+ */
+static int32_t tsdbJudgeStorageLevel(int64_t interval, int8_t intervalUnit) {
+  // TODO: configurable for SMA_STORAGE_SPLIT_HOURS?
+  switch (intervalUnit) {
+    case TD_TIME_UNIT_HOUR:
+      if (interval < SMA_STORAGE_SPLIT_HOURS) {
+        return SMA_STORAGE_LEVEL_DFILESET;
+      }
+      break;
+    case TD_TIME_UNIT_MINUTE:
+      if (interval < 60 * SMA_STORAGE_SPLIT_HOURS) {
+        return SMA_STORAGE_LEVEL_DFILESET;
+      }
+      break;
+    case TD_TIME_UNIT_SEC:
+      if (interval < 3600 * SMA_STORAGE_SPLIT_HOURS) {
+        return SMA_STORAGE_LEVEL_DFILESET;
+      }
+      break;
+    case TD_TIME_UNIT_MILLISEC:
+      if (interval < 3600 * 1e3 * SMA_STORAGE_SPLIT_HOURS) {
+        return SMA_STORAGE_LEVEL_DFILESET;
+      }
+      break;
+    case TD_TIME_UNIT_MICROSEC:
+      if (interval < 3600 * 1e6 * SMA_STORAGE_SPLIT_HOURS) {
+        return SMA_STORAGE_LEVEL_DFILESET;
+      }
+      break;
+    case TD_TIME_UNIT_NANOSEC:
+      if (interval < 3600 * 1e9 * SMA_STORAGE_SPLIT_HOURS) {
+        return SMA_STORAGE_LEVEL_DFILESET;
+      }
+      break;
+    default:
+      break;
+  }
+  return SMA_STORAGE_LEVEL_TSDB;
+}
+
+/**
+ * @brief Insert TSma data blocks to B+Tree
+ *
+ * @param bTree
+ * @param smaKey
+ * @param pData
+ * @param dataLen
+ * @return int32_t
+ */
+static int32_t tsdbInsertTSmaBlocks(void *bTree, const char *smaKey, const char *pData, int32_t dataLen) {
+  // TODO: insert sma data blocks into B+Tree
+  printf("insert sma data blocks into B+Tree: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", dataLen %d\n",
+         *(uint64_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8), *(int64_t *)POINTER_SHIFT(smaKey, 10), dataLen);
+  return TSDB_CODE_SUCCESS;
+}
+
+static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision) {
+  if (intervalUnit < TD_TIME_UNIT_MILLISEC) {
+    switch (intervalUnit) {
+      case TD_TIME_UNIT_YEAR:
+      case TD_TIME_UNIT_SEASON:
+      case TD_TIME_UNIT_MONTH:
+      case TD_TIME_UNIT_WEEK:
+        // illegal time unit
+        tsdbError("invalid interval unit: %d\n", intervalUnit);
+        TASSERT(0);
+        break;
+      case TD_TIME_UNIT_DAY:  // the interval for tSma calculation must <= day
+        interval *= 86400 * 1e3;
+        break;
+      case TD_TIME_UNIT_HOUR:
+        interval *= 3600 * 1e3;
+        break;
+      case TD_TIME_UNIT_MINUTE:
+        interval *= 60 * 1e3;
+        break;
+      case TD_TIME_UNIT_SEC:
+        interval *= 1e3;
+        break;
+      default:
+        break;
+    }
+  }
+
+  switch (intervalUnit) {
+    case TD_TIME_UNIT_MILLISEC:
+      if (TSDB_TIME_PRECISION_MILLI == precision) {
+        return interval;
+      } else if (TSDB_TIME_PRECISION_MICRO == precision) {
+        return interval * 1e3;
+      } else {  //  nano second
+        return interval * 1e6;
+      }
+      break;
+    case TD_TIME_UNIT_MICROSEC:
+      if (TSDB_TIME_PRECISION_MILLI == precision) {
+        return interval / 1e3;
+      } else if (TSDB_TIME_PRECISION_MICRO == precision) {
+        return interval;
+      } else {  //  nano second
+        return interval * 1e3;
+      }
+      break;
+    case TD_TIME_UNIT_NANOSEC:
+      if (TSDB_TIME_PRECISION_MILLI == precision) {
+        return interval / 1e6;
+      } else if (TSDB_TIME_PRECISION_MICRO == precision) {
+        return interval / 1e3;
+      } else {  // nano second
+        return interval;
+      }
+      break;
+    default:
+      if (TSDB_TIME_PRECISION_MILLI == precision) {
+        return interval * 1e3;
+      } else if (TSDB_TIME_PRECISION_MICRO == precision) {
+        return interval * 1e6;
+      } else {  // nano second
+        return interval * 1e9;
+      }
+      break;
+  }
+  return interval;
+}
+
+/**
+ * @brief Split the TSma data blocks into expected size and insert into B+Tree.
+ *
+ * @param pSmaH
+ * @param pData
+ * @param nOffset The nOffset of blocks since fid changes.
+ * @param nBlocks The nBlocks with the same fid since nOffset.
+ * @return int32_t
+ */
+static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaData *pData, int32_t nOffset, int32_t nBlocks) {
+  STsdb *pTsdb = pSmaH->pTsdb;
+
+  TASSERT(pData->colIds != NULL);
+
+  tsdbDebug("tsdbInsertTSmaDataSection: nOffset %d, nBlocks %d", nOffset, nBlocks);
+  printf("tsdbInsertTSmaDataSection: nOffset %d, nBlocks %d\n", nOffset, nBlocks);
+
+  int32_t colDataLen = pData->dataLen / pData->numOfColIds;
+  int32_t sectionDataLen = pSmaH->blockSize * nBlocks;
+
+  for (col_id_t i = 0; i < pData->numOfColIds; ++i) {
+    // param: pointer of B+Tree, key, value, dataLen
+    void *bTree = pSmaH->pDFile;
+#ifndef SMA_STORE_SINGLE_BLOCKS
+    // save tSma data blocks as a whole
+    char  smaKey[SMA_KEY_LEN] = {0};
+    void *pSmaKey = &smaKey;
+    tsdbEncodeTSmaKey(pData->tableUid, *(pData->colIds + i), pData->tsWindow.skey + nOffset * pSmaH->interval,
+                      (void **)&pSmaKey);
+    if (tsdbInsertTSmaBlocks(bTree, smaKey, pData->data + i * colDataLen + nOffset * pSmaH->blockSize, sectionDataLen) <
+        0) {
+      tsdbWarn("vgId:%d insert tSma blocks failed since %s", REPO_ID(pTsdb), tstrerror(terrno));
+    }
+#else
+    // save tSma data blocks separately
+    for (int32_t n = 0; n < nBlocks; ++n) {
+      char  smaKey[SMA_KEY_LEN] = {0};
+      void *pSmaKey = &smaKey;
+      tsdbEncodeTSmaKey(pData->tableUid, *(pData->colIds + i), pData->tsWindow.skey + (nOffset + n) * pSmaH->interval,
+                        (void **)&pSmaKey);
+      if (tsdbInsertTSmaBlocks(bTree, smaKey, pData->data + i * colDataLen + (nOffset + n) * pSmaH->blockSize,
+                               pSmaH->blockSize) < 0) {
+        tsdbWarn("vgId:%d insert tSma blocks failed since %s", REPO_ID(pTsdb), tstrerror(terrno));
+      }
+    }
+#endif
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData) {
+  pSmaH->pTsdb = pTsdb;
+  pSmaH->interval = tsdbGetIntervalByPrecision(param->interval, param->intervalUnit, REPO_CFG(pTsdb)->precision);
+  // pSmaH->blockSize = param->numOfFuncIds * sizeof(int64_t);
+}
+
+static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSma *param, STSmaData *pData, int32_t storageLevel,
+                                   int32_t fid) {
+  // TODO
+  pSmaH->pDFile = "tSma_interval_file_name";
+
+  return TSDB_CODE_SUCCESS;
+} /**
+   * @brief Split the sma data blocks by fid.
+   *
+   * @param pSmaH
+   * @param param
+   * @param pData
+   * @param nOffset
+   * @param fid
+   * @param nSmaBlocks
+   * @return int32_t
+   */
+static int32_t tsdbTSmaDataSplit(STSmaWriteH *pSmaH, STSma *param, STSmaData *pData, int32_t days, int32_t nOffset,
+                                 int32_t fid, int32_t *nSmaBlocks) {
+  STsdbCfg *pCfg = REPO_CFG(pSmaH->pTsdb);
+
+  // TODO: use binary search
+  for (int32_t n = nOffset + 1; n < pData->numOfBlocks; ++n) {
+    // TODO: The tsWindow.skey should use the precision of DB.
+    int32_t tFid = (int32_t)(TSDB_KEY_FID(pData->tsWindow.skey + pSmaH->interval * n, days, pCfg->precision));
+    if (tFid > fid) {
+      *nSmaBlocks = n - nOffset;
+      break;
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief Insert/Update Time-range-wise SMA data.
+ *  - If interval < SMA_STORAGE_SPLIT_HOURS(e.g. 24), save the SMA data as a part of DFileSet to e.g.
+ * v3f1900.tsma.${sma_index_name}. The days is the same with that for TS data files.
+ *  - If interval >= SMA_STORAGE_SPLIT_HOURS, save the SMA data to e.g. vnode3/tsma/v3f632.tsma.${sma_index_name}. The
+ * days is 30 times of the interval, and the minimum days is SMA_STORAGE_TSDB_DAYS(30d).
+ *  - The destination file of one data block for some interval is determined by its start TS key.
+ *
+ * @param pTsdb
+ * @param param
+ * @param pData
+ * @return int32_t
+ */
+int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData) {
+  STsdbCfg *  pCfg = REPO_CFG(pTsdb);
+  STSmaData * curData = pData;
+  STSmaWriteH tSmaH = {0};
+
+  tsdbInitTSmaWriteH(&tSmaH, pTsdb, param, pData);
+
+  if (pData->numOfBlocks <= 0 || pData->numOfColIds <= 0 || pData->dataLen <= 0) {
+    TASSERT(0);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+
+  // Step 1: Judge the storage level
+  int32_t storageLevel = tsdbJudgeStorageLevel(param->interval, param->intervalUnit);
+  int32_t daysPerFile = storageLevel == SMA_STORAGE_LEVEL_TSDB ? SMA_STORAGE_TSDB_DAYS : pCfg->daysPerFile;
+
+  // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file
+  //         - Set and open the DFile or the B+Tree file
+
+  int32_t minFid = (int32_t)(TSDB_KEY_FID(pData->tsWindow.skey, daysPerFile, pCfg->precision));
+  int32_t maxFid = (int32_t)(TSDB_KEY_FID(pData->tsWindow.ekey, daysPerFile, pCfg->precision));
+
+  if (minFid == maxFid) {
+    // Save all the TSma data to one file
+    // TODO: tsdbStartTSmaCommit();
+    tsdbSetTSmaDataFile(&tSmaH, param, pData, storageLevel, minFid);
+    tsdbInsertTSmaDataSection(&tSmaH, pData, 0, pData->numOfBlocks);
+    // TODO:tsdbEndTSmaCommit();
+  } else if (minFid < maxFid) {
+    // Split the TSma data and save to multiple files. As there is limit for the span, it can't span more than 2 files
+    // actually.
+    // TODO: tsdbStartTSmaCommit();
+    int32_t tFid = minFid;
+    int32_t nOffset = 0;
+    int32_t nSmaBlocks = 0;
+    do {
+      tsdbTSmaDataSplit(&tSmaH, param, pData, daysPerFile, nOffset, tFid, &nSmaBlocks);
+      tsdbSetTSmaDataFile(&tSmaH, param, pData, storageLevel, tFid);
+      if (tsdbInsertTSmaDataSection(&tSmaH, pData, nOffset, nSmaBlocks) < 0) {
+        return terrno;
+      }
+
+      ++tFid;
+      nOffset += nSmaBlocks;
+
+      if (tFid == maxFid) {
+        tsdbSetTSmaDataFile(&tSmaH, param, pData, storageLevel, tFid);
+        tsdbInsertTSmaDataSection(&tSmaH, pData, nOffset, pData->numOfBlocks - nOffset);
+        break;
+      }
+    } while (true);
+
+    // TODO:tsdbEndTSmaCommit();
+  } else {
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+
+  // reset the SSmaStat
+  tsdbResetExpiredWindow(pTsdb, param->indexName, &pData->tsWindow);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, SRSma *param, STSmaData *pData, int32_t fid) {
+  // TODO
+  pSmaH->pDFile = "rSma_interval_file_name";
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, SRSma *param, STSmaData *pData) {
+  STsdbCfg *  pCfg = REPO_CFG(pTsdb);
+  STSma *     tParam = &param->tsma;
+  STSmaData * curData = pData;
+  STSmaWriteH tSmaH = {0};
+
+  tsdbInitTSmaWriteH(&tSmaH, pTsdb, tParam, pData);
+
+  int32_t nSmaBlocks = pData->numOfBlocks;
+  int32_t colDataLen = pData->dataLen / nSmaBlocks;
+
+  // Step 2.2: Storage of SMA_STORAGE_LEVEL_DFILESET
+  // TODO: Use the daysPerFile for rSma data, not for TS data.
+  // TODO: The lifecycle of rSma data should be processed like the TS data files.
+  int32_t minFid = (int32_t)(TSDB_KEY_FID(pData->tsWindow.skey, pCfg->daysPerFile, pCfg->precision));
+  int32_t maxFid = (int32_t)(TSDB_KEY_FID(pData->tsWindow.ekey, pCfg->daysPerFile, pCfg->precision));
+
+  if (minFid == maxFid) {
+    // Save all the TSma data to one file
+    tsdbSetRSmaDataFile(&tSmaH, param, pData, minFid);
+    // TODO: tsdbStartTSmaCommit();
+    tsdbInsertTSmaDataSection(&tSmaH, pData, colDataLen, nSmaBlocks);
+    // TODO:tsdbEndTSmaCommit();
+  } else if (minFid < maxFid) {
+    // Split the TSma data and save to multiple files. As there is limit for the span, it can't span more than 2 files
+    // actually.
+    // TODO: tsdbStartTSmaCommit();
+    int32_t tmpFid = 0;
+    int32_t step = 0;
+    for (int32_t n = 0; n < pData->numOfBlocks; ++n) {
+    }
+    tsdbInsertTSmaDataSection(&tSmaH, pData, colDataLen, nSmaBlocks);
+    // TODO:tsdbEndTSmaCommit();
+  } else {
+    TASSERT(0);
+    return TSDB_CODE_INVALID_PARA;
+  }
+
+  // reset the SSmaStat
+  tsdbResetExpiredWindow(pTsdb, param->tsma.indexName, &pData->tsWindow);
+
+  // Step 4: finish
+  return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief Init of tSma ReadH
+ *
+ * @param pSmaH
+ * @param pTsdb
+ * @param param
+ * @param pData
+ * @return int32_t
+ */
+static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData) {
+  pSmaH->pTsdb = pTsdb;
+  pSmaH->interval = tsdbGetIntervalByPrecision(param->interval, param->intervalUnit, REPO_CFG(pTsdb)->precision);
+  // pSmaH->blockSize = param->numOfFuncIds * sizeof(int64_t);
+}
+
+/**
+ * @brief Init of tSma FS
+ *
+ * @param pReadH
+ * @param param
+ * @param queryWin
+ * @return int32_t
+ */
+static int32_t tsdbInitTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWindow *queryWin) {
+  int32_t storageLevel = tsdbJudgeStorageLevel(param->interval, param->intervalUnit);
+  int32_t daysPerFile =
+      storageLevel == SMA_STORAGE_LEVEL_TSDB ? SMA_STORAGE_TSDB_DAYS : REPO_CFG(pReadH->pTsdb)->daysPerFile;
+  pReadH->storageLevel = storageLevel;
+  pReadH->days = daysPerFile;
+  pReadH->smaFsIter.iter = 0;
+}
+
+/**
+ * @brief Set and open tSma file if it has key locates in queryWin.
+ *
+ * @param pReadH
+ * @param param
+ * @param queryWin
+ * @return true
+ * @return false
+ */
+static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWindow *queryWin) {
+  SArray *smaFs = pReadH->pTsdb->fs->cstatus->smaf;
+  int32_t nSmaFs = taosArrayGetSize(smaFs);
+
+  pReadH->pDFile = NULL;
+
+  while (pReadH->smaFsIter.iter < nSmaFs) {
+    void *pSmaFile = taosArrayGet(smaFs, pReadH->smaFsIter.iter);
+    if (pSmaFile) {  // match(indexName, queryWindow)
+      // TODO: select the file by index_name ...
+      pReadH->pDFile = pSmaFile;
+      ++pReadH->smaFsIter.iter;
+      break;
+    }
+    ++pReadH->smaFsIter.iter;
+  }
+
+  if (pReadH->pDFile != NULL) {
+    tsdbDebug("vg%d: smaFile %s matched", REPO_ID(pReadH->pTsdb), "[pSmaFile dir]");
+    return true;
+  }
+
+  return false;
+}
+
+/**
+ * @brief Return the data between queryWin and fill the pData.
+ *
+ * @param pTsdb
+ * @param param
+ * @param pData
+ * @param queryWin
+ * @param nMaxResult The query invoker should control the nMaxResult need to return to avoid OOM.
+ * @return int32_t
+ */
+int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData, STimeWindow *queryWin, int32_t nMaxResult) {
+  const char *indexName = param->indexName;
+
+  SSmaStatItem *pItem = (SSmaStatItem *)taosHashGet(pTsdb->pSmaStat->smaStatItems, indexName, strlen(indexName));
+  if (pItem == NULL) {
+    // mark all window as expired and notify query module to query raw TS data.
+    return TSDB_CODE_SUCCESS;
+  }
+
+  int32_t nQueryWin = 0;
+  for (int32_t n = 0; n < nQueryWin; ++n) {
+    TSKEY thisWindow = n;
+    if (taosHashGet(pItem->expiredWindows, &thisWindow, sizeof(thisWindow)) != NULL) {
+      // TODO: mark this window as expired.
+    }
+  }
+
+  STSmaReadH tReadH = {0};
+  tsdbInitTSmaReadH(&tReadH, pTsdb, param, pData);
+
+  tsdbInitTSmaFile(&tReadH, param, queryWin);
+
+  int32_t nResult = 0;
+  int64_t lastKey = 0;
+
+  while (true) {
+    if (nResult >= nMaxResult) {
+      break;
+    }
+
+    // set and open the file according to the STSma param
+    if (tsdbSetAndOpenTSmaFile(&tReadH, param, queryWin)) {
+      char bTree[100] = "\0";
+      while (strncmp(bTree, "has more nodes", 100) == 0) {
+        if (nResult >= nMaxResult) {
+          break;
+        }
+        // tsdbGetDataFromBTree(bTree, queryWin, lastKey)
+        // fill the pData
+        ++nResult;
+      }
+    }
+  }
+
+  // read data from file and fill the result
+  return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief Get the start TS key of the last data block of one interval/sliding.
+ *
+ * @param pTsdb
+ * @param param
+ * @param result
+ * @return int32_t
+ *         1) Return 0 and fill the result if the check procedure is normal;
+ *         2) Return -1 if error occurs during the check procedure.
+ */
+int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result) {
+  const char *procedure = "";
+  if (strncmp(procedure, "get the start TS key of the last data block", 100) != 0) {
+    return -1;
+  }
+  // fill the result
+  return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief Remove the tSma data files related to param between pWin.
+ *
+ * @param pTsdb
+ * @param param
+ * @param pWin
+ * @return int32_t
+ */
+int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin) {
+  // for ("tSmaFiles of param-interval-sliding between pWin") {
+  //   // remove the tSmaFile
+  // }
+  return TSDB_CODE_SUCCESS;
+}