Merge branch '3.0' of github.com:taosdata/TDengine into feature/udf

cmake/cmake.define

@@ -14,25 +14,6 @@ MESSAGE(STATUS "Project binary files output path: " ${PROJECT_BINARY_DIR})
 MESSAGE(STATUS "Project executable files output path: " ${EXECUTABLE_OUTPUT_PATH})
 MESSAGE(STATUS "Project library files output path: " ${LIBRARY_OUTPUT_PATH})
 
-find_package(Git QUIET)
-if(GIT_FOUND AND EXISTS "${TD_SOURCE_DIR}/.git")
-# Update submodules as needed
-    option(GIT_SUBMODULE "Check submodules during build" ON)
-    if(GIT_SUBMODULE)
-        message(STATUS "Submodule update")
-        execute_process(COMMAND cd ${TD_SOURCE_DIR} && ${GIT_EXECUTABLE} submodule update --init --recursive
-                        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-                        RESULT_VARIABLE GIT_SUBMOD_RESULT)
-        if(NOT GIT_SUBMOD_RESULT EQUAL "0")
-            message(WARNING "git submodule update --init --recursive failed with ${GIT_SUBMOD_RESULT}, please checkout submodules")
-        endif()
-    endif()
-endif()
-
-if(NOT EXISTS "${TD_SOURCE_DIR}/tools/taos-tools/CMakeLists.txt")
-    message(WARNING "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules manually if you need build them.")
-endif()
-
 if (NOT DEFINED TD_GRANT)
   SET(TD_GRANT FALSE)
 endif()

cmake/cmake.options

@@ -78,6 +78,12 @@ option(
     OFF
 )
 
+option(
+    BUILD_WITH_BDB
+    "If build with BDB" 
+    OFF
+)
+
 option(
     BUILD_WITH_LUCENE 
     "If build with lucene" 

contrib/CMakeLists.txt

@@ -78,9 +78,9 @@ if(${BUILD_WITH_UV})
 endif(${BUILD_WITH_UV})
 
 # bdb
-#if(${BUILD_WITH_BDB})
+if(${BUILD_WITH_BDB})
-    #cat("${TD_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    cat("${TD_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
-#endif(${BUILD_WITH_BDB})
+endif(${BUILD_WITH_BDB})
 
 # sqlite
 if(${BUILD_WITH_SQLITE})

contrib/test/CMakeLists.txt

@@ -7,9 +7,9 @@ if(${BUILD_WITH_LUCENE})
     add_subdirectory(lucene)
 endif(${BUILD_WITH_LUCENE})
 
-#if(${BUILD_WITH_BDB})
+if(${BUILD_WITH_BDB})
-    #add_subdirectory(bdb)
+    add_subdirectory(bdb)
-#endif(${BUILD_WITH_BDB})
+endif(${BUILD_WITH_BDB})
 
 if(${BUILD_WITH_SQLITE})
     add_subdirectory(sqlite)

include/common/taosdef.h

@@ -37,7 +37,8 @@ typedef enum {
   TSDB_STREAM_TABLE = 4,  // table created from stream computing
   TSDB_TEMP_TABLE = 5,    // temp table created by nest query
   TSDB_SYSTEM_TABLE = 6,
-  TSDB_TABLE_MAX = 7
+  TSDB_TSMA_TABLE = 7,  // time-range-wise sma
+  TSDB_TABLE_MAX = 8
 } ETableType;
 
 typedef enum {

include/common/tmsg.h

@@ -1670,6 +1670,7 @@ typedef struct SVDropStbReq {
 int32_t tEncodeSVDropStbReq(SEncoder* pCoder, const SVDropStbReq* pReq);
 int32_t tDecodeSVDropStbReq(SDecoder* pCoder, SVDropStbReq* pReq);
 
+// TDMT_VND_CREATE_TABLE ==============
 #define TD_CREATE_IF_NOT_EXISTS 0x1
 typedef struct SVCreateTbReq {
   int32_t     flags;
@@ -1759,6 +1760,43 @@ typedef struct {
 int32_t tEncodeSVDropTbBatchRsp(SEncoder* pCoder, const SVDropTbBatchRsp* pRsp);
 int32_t tDecodeSVDropTbBatchRsp(SDecoder* pCoder, SVDropTbBatchRsp* pRsp);
 
+// TDMT_VND_ALTER_TABLE =====================
+typedef struct {
+  const char* tbName;
+  int8_t      action;
+  const char* colName;
+  // TSDB_ALTER_TABLE_ADD_COLUMN
+  int8_t  type;
+  int8_t  flags;
+  int32_t bytes;
+  // TSDB_ALTER_TABLE_DROP_COLUMN
+  // TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES
+  int32_t colModBytes;
+  // TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME
+  const char* colNewName;
+  // TSDB_ALTER_TABLE_UPDATE_TAG_VAL
+  const char*    tagName;
+  int8_t         isNull;
+  uint32_t       nTagVal;
+  const uint8_t* pTagVal;
+  // TSDB_ALTER_TABLE_UPDATE_OPTIONS
+  int8_t      updateTTL;
+  int32_t     newTTL;
+  int8_t      updateComment;
+  const char* newComment;
+} SVAlterTbReq;
+
+int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq);
+int32_t tDecodeSVAlterTbReq(SDecoder* pDecoder, SVAlterTbReq* pReq);
+
+typedef struct {
+  int32_t code;
+} SVAlterTbRsp;
+
+int32_t tEncodeSVAlterTbRsp(SEncoder* pEncoder, const SVAlterTbRsp* pRsp);
+int32_t tDecodeSVAlterTbRsp(SDecoder* pDecoder, SVAlterTbRsp* pRsp);
+// ======================
+
 typedef struct {
   SMsgHead head;
   int64_t  uid;
@@ -2160,26 +2198,23 @@ int32_t tSerializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq);
 int32_t tDeserializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq);
 
 typedef struct {
-  int8_t   version;       // for compatibility(default 0)
+  int8_t      version;       // for compatibility(default 0)
-  int8_t   intervalUnit;  // MACRO: TIME_UNIT_XXX
+  int8_t      intervalUnit;  // MACRO: TIME_UNIT_XXX
-  int8_t   slidingUnit;   // MACRO: TIME_UNIT_XXX
+  int8_t      slidingUnit;   // MACRO: TIME_UNIT_XXX
-  int8_t   timezoneInt;   // sma data expired if timezone changes.
+  int8_t      timezoneInt;   // sma data expired if timezone changes.
-  char     indexName[TSDB_INDEX_NAME_LEN];
+  char        indexName[TSDB_INDEX_NAME_LEN];
-  int32_t  exprLen;
+  int32_t     exprLen;
-  int32_t  tagsFilterLen;
+  int32_t     tagsFilterLen;
-  int64_t  indexUid;
+  int64_t     indexUid;
-  tb_uid_t tableUid;  // super/child/common table uid
+  tb_uid_t    tableUid;  // super/child/common table uid
-  int64_t  interval;
+  int64_t     interval;
-  int64_t  offset;  // use unit by precision of DB
+  int64_t     offset;  // use unit by precision of DB
-  int64_t  sliding;
+  int64_t     sliding;
-  char*    expr;  // sma expression
+  const char* expr;  // sma expression
-  char*    tagsFilter;
+  const char* tagsFilter;
 } STSma;  // Time-range-wise SMA
 
-typedef struct {
+typedef STSma SVCreateTSmaReq;
-  int64_t ver;  // use a general definition
-  STSma   tSma;
-} SVCreateTSmaReq;
 
 typedef struct {
   int8_t  type;  // 0 status report, 1 update data
@@ -2188,7 +2223,6 @@ typedef struct {
 } STSmaMsg;
 
 typedef struct {
-  int64_t ver;  // use a general definition
   int64_t indexUid;
   char    indexName[TSDB_INDEX_NAME_LEN];
 } SVDropTSmaReq;
@@ -2197,28 +2231,21 @@ typedef struct {
   int tmp;  // TODO: to avoid compile error
 } SVCreateTSmaRsp, SVDropTSmaRsp;
 
+#if 0
 int32_t tSerializeSVCreateTSmaReq(void** buf, SVCreateTSmaReq* pReq);
 void*   tDeserializeSVCreateTSmaReq(void* buf, SVCreateTSmaReq* pReq);
 int32_t tSerializeSVDropTSmaReq(void** buf, SVDropTSmaReq* pReq);
 void*   tDeserializeSVDropTSmaReq(void* buf, SVDropTSmaReq* pReq);
+#endif
 
-// RSma: Rollup SMA
+int32_t tEncodeSVCreateTSmaReq(SEncoder* pCoder, const SVCreateTSmaReq* pReq);
-typedef struct {
+int32_t tDecodeSVCreateTSmaReq(SDecoder* pCoder, SVCreateTSmaReq* pReq);
-  int64_t  interval;
+int32_t tEncodeSVDropTSmaReq(SEncoder* pCoder, const SVDropTSmaReq* pReq);
-  int32_t  retention;  // unit: day
+int32_t tDecodeSVDropTSmaReq(SDecoder* pCoder, SVDropTSmaReq* pReq);
-  uint16_t days;       // unit: day
-  int8_t   intervalUnit;
-} SSmaParams;
 
 typedef struct {
-  STSma   tsma;
+  int32_t number;
-  float   xFilesFactor;
+  STSma*  tSma;
-  SArray* smaParams;  // SSmaParams
-} SRSma;
-
-typedef struct {
-  uint32_t number;
-  STSma*   tSma;
 } STSmaWrapper;
 
 static FORCE_INLINE void tdDestroyTSma(STSma* pSma) {
@@ -2245,96 +2272,26 @@ static FORCE_INLINE void* tdFreeTSmaWrapper(STSmaWrapper* pSW) {
   return NULL;
 }
 
-static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) {
+int32_t tEncodeSVCreateTSmaReq(SEncoder* pCoder, const SVCreateTSmaReq* pReq);
-  int32_t tlen = 0;
+int32_t tDecodeSVCreateTSmaReq(SDecoder* pCoder, SVCreateTSmaReq* pReq);
 
-  tlen += taosEncodeFixedI8(buf, pSma->version);
+int32_t tEncodeTSma(SEncoder* pCoder, const STSma* pSma);
-  tlen += taosEncodeFixedI8(buf, pSma->intervalUnit);
+int32_t tDecodeTSma(SDecoder* pCoder, STSma* pSma);
-  tlen += taosEncodeFixedI8(buf, pSma->slidingUnit);
-  tlen += taosEncodeFixedI8(buf, pSma->timezoneInt);
-  tlen += taosEncodeString(buf, pSma->indexName);
-  tlen += taosEncodeFixedI32(buf, pSma->exprLen);
-  tlen += taosEncodeFixedI32(buf, pSma->tagsFilterLen);
-  tlen += taosEncodeFixedI64(buf, pSma->indexUid);
-  tlen += taosEncodeFixedI64(buf, pSma->tableUid);
-  tlen += taosEncodeFixedI64(buf, pSma->interval);
-  tlen += taosEncodeFixedI64(buf, pSma->offset);
-  tlen += taosEncodeFixedI64(buf, pSma->sliding);
 
-  if (pSma->exprLen > 0) {
+static int32_t tEncodeTSmaWrapper(SEncoder* pEncoder, const STSmaWrapper* pReq) {
-    tlen += taosEncodeString(buf, pSma->expr);
+  if (tEncodeI32(pEncoder, pReq->number) < 0) return -1;
+  for (int32_t i = 0; i < pReq->number; ++i) {
+    tEncodeTSma(pEncoder, pReq->tSma + i);
   }
-
+  return 0;
-  if (pSma->tagsFilterLen > 0) {
-    tlen += taosEncodeString(buf, pSma->tagsFilter);
-  }
-
-  return tlen;
 }
 
-static FORCE_INLINE int32_t tEncodeTSmaWrapper(void** buf, const STSmaWrapper* pSW) {
+static int32_t tDecodeTSmaWrapper(SDecoder* pDecoder, STSmaWrapper* pReq) {
-  int32_t tlen = 0;
+  if (tDecodeI32(pDecoder, &pReq->number) < 0) return -1;
-
+  for (int32_t i = 0; i < pReq->number; ++i) {
-  tlen += taosEncodeFixedU32(buf, pSW->number);
+    tDecodeTSma(pDecoder, pReq->tSma + i);
-  for (uint32_t i = 0; i < pSW->number; ++i) {
-    tlen += tEncodeTSma(buf, pSW->tSma + i);
   }
-  return tlen;
+  return 0;
-}
-
-static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) {
-  buf = taosDecodeFixedI8(buf, &pSma->version);
-  buf = taosDecodeFixedI8(buf, &pSma->intervalUnit);
-  buf = taosDecodeFixedI8(buf, &pSma->slidingUnit);
-  buf = taosDecodeFixedI8(buf, &pSma->timezoneInt);
-  buf = taosDecodeStringTo(buf, pSma->indexName);
-  buf = taosDecodeFixedI32(buf, &pSma->exprLen);
-  buf = taosDecodeFixedI32(buf, &pSma->tagsFilterLen);
-  buf = taosDecodeFixedI64(buf, &pSma->indexUid);
-  buf = taosDecodeFixedI64(buf, &pSma->tableUid);
-  buf = taosDecodeFixedI64(buf, &pSma->interval);
-  buf = taosDecodeFixedI64(buf, &pSma->offset);
-  buf = taosDecodeFixedI64(buf, &pSma->sliding);
-
-  if (pSma->exprLen > 0) {
-    if ((buf = taosDecodeString(buf, &pSma->expr)) == NULL) {
-      tdDestroyTSma(pSma);
-      return NULL;
-    }
-  } else {
-    pSma->expr = NULL;
-  }
-
-  if (pSma->tagsFilterLen > 0) {
-    if ((buf = taosDecodeString(buf, &pSma->tagsFilter)) == NULL) {
-      tdDestroyTSma(pSma);
-      return NULL;
-    }
-  } else {
-    pSma->tagsFilter = NULL;
-  }
-
-  return buf;
-}
-
-static FORCE_INLINE void* tDecodeTSmaWrapper(void* buf, STSmaWrapper* pSW) {
-  buf = taosDecodeFixedU32(buf, &pSW->number);
-
-  pSW->tSma = (STSma*)taosMemoryCalloc(pSW->number, sizeof(STSma));
-  if (pSW->tSma == NULL) {
-    return NULL;
-  }
-
-  for (uint32_t i = 0; i < pSW->number; ++i) {
-    if ((buf = tDecodeTSma(buf, pSW->tSma + i)) == NULL) {
-      for (uint32_t j = i; j >= 0; --i) {
-        tdDestroyTSma(pSW->tSma + j);
-      }
-      taosMemoryFree(pSW->tSma);
-      return NULL;
-    }
-  }
-  return buf;
 }
 
 typedef struct {

include/libs/stream/tstream.h

@@ -40,6 +40,7 @@ enum {
   TASK_INPUT_STATUS__BLOCKED,
   TASK_INPUT_STATUS__RECOVER,
   TASK_INPUT_STATUS__STOP,
+  TASK_INPUT_STATUS__FAILED,
 };
 
 enum {
@@ -234,7 +235,9 @@ struct SStreamTask {
   int8_t outputStatus;
 
   STaosQueue* inputQ;
+  STaosQall*  inputQAll;
   STaosQueue* outputQ;
+  STaosQall*  outputQAll;
 
   // application storage
   void* ahandle;
@@ -282,7 +285,7 @@ int32_t streamDequeueOutput(SStreamTask* pTask, void** output);
 
 int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId);
 
-int32_t streamTaskExecNew(SStreamTask* pTask);
+int32_t streamTaskRun(SStreamTask* pTask);
 
 int32_t streamTaskHandleInput(SStreamTask* pTask, void* data);
 

include/libs/sync/sync.h

@@ -144,6 +144,7 @@ int32_t     syncInit();
 void        syncCleanUp();
 int64_t     syncOpen(const SSyncInfo* pSyncInfo);
 void        syncStart(int64_t rid);
+void        syncStartStandBy(int64_t rid);
 void        syncStop(int64_t rid);
 int32_t     syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
 ESyncState  syncGetMyRole(int64_t rid);

include/util/taoserror.h

@@ -323,6 +323,9 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_VND_SMA_NOT_EXIST             TAOS_DEF_ERROR_CODE(0, 0x0516)
 #define TSDB_CODE_VND_HASH_MISMATCH             TAOS_DEF_ERROR_CODE(0, 0x0517)
 #define TSDB_CODE_VND_TABLE_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x0518)
+#define TSDB_CODE_VND_INVALID_TABLE_ACTION      TAOS_DEF_ERROR_CODE(0, 0x0519)
+#define TSDB_CODE_VND_COL_ALREADY_EXISTS        TAOS_DEF_ERROR_CODE(0, 0x051a)
+#define TSDB_CODE_VND_TABLE_COL_NOT_EXISTS      TAOS_DEF_ERROR_CODE(0, 0x051b)
 
 // tsdb
 #define TSDB_CODE_TDB_INVALID_TABLE_ID          TAOS_DEF_ERROR_CODE(0, 0x0600)
@@ -354,7 +357,8 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_TDB_TABLE_RECREATED           TAOS_DEF_ERROR_CODE(0, 0x061A)
 #define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR        TAOS_DEF_ERROR_CODE(0, 0x061B)
 #define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META      TAOS_DEF_ERROR_CODE(0, 0x061C)
-#define TSDB_CODE_TDB_INVALID_SMA_STAT          TAOS_DEF_ERROR_CODE(0, 0x062D)
+#define TSDB_CODE_TDB_INVALID_SMA_STAT          TAOS_DEF_ERROR_CODE(0, 0x061D)
+#define TSDB_CODE_TDB_TSMA_ALREADY_EXIST        TAOS_DEF_ERROR_CODE(0, 0x061E)
 
 // query
 #define TSDB_CODE_QRY_INVALID_QHANDLE           TAOS_DEF_ERROR_CODE(0, 0x0700)

include/util/tencode.h

@@ -18,7 +18,6 @@
 
 #include "tcoding.h"
 #include "tlist.h"
-// #include "tfreelist.h"
 
 #ifdef __cplusplus
 extern "C" {

include/util/tfreelist.h

@@ -1,60 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_UTIL_FREELIST_H_
-#define _TD_UTIL_FREELIST_H_
-
-#include "tlist.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-struct SFreeListNode {
-  TD_SLIST_NODE(SFreeListNode);
-  char payload[];
-};
-
-typedef TD_SLIST(SFreeListNode) SFreeList;
-
-#define TFL_MALLOC(PTR, TYPE, SIZE, LIST)                      \
-  do {                                                         \
-    void *ptr = taosMemoryMalloc((SIZE) + sizeof(struct SFreeListNode)); \
-    if (ptr) {                                                 \
-      TD_SLIST_PUSH((LIST), (struct SFreeListNode *)ptr);      \
-      ptr = ((struct SFreeListNode *)ptr)->payload;            \
-      (PTR) = (TYPE)(ptr);                                     \
-    }else{                                                     \
-      (PTR) = NULL;                                            \
-    }                                                          \
-  }while(0);
-
-#define tFreeListInit(pFL) TD_SLIST_INIT(pFL)
-
-static FORCE_INLINE void tFreeListClear(SFreeList *pFL) {
-  struct SFreeListNode *pNode;
-  for (;;) {
-    pNode = TD_SLIST_HEAD(pFL);
-    if (pNode == NULL) break;
-    TD_SLIST_POP(pFL);
-    taosMemoryFree(pNode);
-  }
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_UTIL_FREELIST_H_*/

include/util/tlog.h

@@ -61,6 +61,7 @@ extern int32_t tqDebugFlag;
 extern int32_t fsDebugFlag;
 extern int32_t metaDebugFlag;
 extern int32_t fnDebugFlag;
+extern int32_t smaDebugFlag;
 
 int32_t taosInitLog(const char *logName, int32_t maxFiles);
 void    taosCloseLog();

source/client/src/clientImpl.c

@@ -518,8 +518,9 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t
   if (pRequest->code != TSDB_CODE_SUCCESS) {
     const char* errorMsg =
         (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(pRequest->code);
-    printf("failed to connect to server, reason: %s\n\n", errorMsg);
+    fprintf(stderr,"failed to connect to server, reason: %s\n\n", errorMsg);
 
+    terrno = pRequest->code;
     destroyRequest(pRequest);
     taos_close(pTscObj);
     pTscObj = NULL;

source/client/src/clientSml.c

@@ -1007,8 +1007,6 @@ static int32_t smlParseTelnetTags(const char* data, SArray *cols, SHashObj *dump
     while(*sql != '\0') {
       // parse value
       if (*sql == SPACE) {
-        valueLen = sql - value;
-        sql++;
         break;
       }
       if (*sql == EQUAL) {
@@ -1017,9 +1015,7 @@ static int32_t smlParseTelnetTags(const char* data, SArray *cols, SHashObj *dump
       }
       sql++;
     }
-    if(valueLen == 0){
+    valueLen = sql - value;
-      valueLen = sql - value;
-    }
 
     if(valueLen == 0){
       smlBuildInvalidDataMsg(msg, "invalid value", value);
@@ -1365,7 +1361,7 @@ static void smlDestroySTableMeta(SSmlSTableMeta *meta){
 static void smlDestroyCols(SArray *cols) {
   if (!cols) return;
   for (int i = 0; i < taosArrayGetSize(cols); ++i) {
-    void *kv = taosArrayGet(cols, i);
+    void *kv = taosArrayGetP(cols, i);
     taosMemoryFree(kv);
   }
 }
@@ -2077,12 +2073,16 @@ static int32_t smlParseTelnetLine(SSmlHandle* info, void *data) {
   if(ret != TSDB_CODE_SUCCESS){
     uError("SML:0x%"PRIx64" smlParseTelnetLine failed", info->id);
     smlDestroyTableInfo(info, tinfo);
+    smlDestroyCols(cols);
     taosArrayDestroy(cols);
     return ret;
   }
 
   if(taosArrayGetSize(tinfo->tags) <= 0 || taosArrayGetSize(tinfo->tags) > TSDB_MAX_TAGS){
     smlBuildInvalidDataMsg(&info->msgBuf, "invalidate tags length:[1,128]", NULL);
+    smlDestroyTableInfo(info, tinfo);
+    smlDestroyCols(cols);
+    taosArrayDestroy(cols);
     return TSDB_CODE_SML_INVALID_DATA;
   }
   taosHashClear(info->dumplicateKey);

source/client/test/smlTest.cpp

@@ -516,8 +516,8 @@ TEST(testCase, smlProcess_influx_Test) {
   int ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0]));
   ASSERT_EQ(ret, 0);
 
-  TAOS_RES *res = taos_query(taos, "select * from t_6885c584b98481584ee13dac399e173d");
+//  TAOS_RES *res = taos_query(taos, "select * from t_6885c584b98481584ee13dac399e173d");
-  ASSERT_NE(res, nullptr);
+//  ASSERT_NE(res, nullptr);
 //  int fieldNum = taos_field_count(res);
 //  ASSERT_EQ(fieldNum, 5);
 //  int rowNum = taos_affected_rows(res);
@@ -525,7 +525,7 @@ TEST(testCase, smlProcess_influx_Test) {
 //  for (int i = 0; i < rowNum; ++i) {
 //    TAOS_ROW rows = taos_fetch_row(res);
 //  }
-  taos_free_result(res);
+//  taos_free_result(res);
   destroyRequest(request);
   smlDestroyInfo(info);
 }
@@ -605,8 +605,8 @@ TEST(testCase, smlProcess_telnet_Test) {
   int ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0]));
   ASSERT_EQ(ret, 0);
 
-  TAOS_RES *res = taos_query(taos, "select * from t_8c30283b3c4131a071d1e16cf6d7094a");
+//  TAOS_RES *res = taos_query(taos, "select * from t_8c30283b3c4131a071d1e16cf6d7094a");
-  ASSERT_NE(res, nullptr);
+//  ASSERT_NE(res, nullptr);
 //  int fieldNum = taos_field_count(res);
 //  ASSERT_EQ(fieldNum, 2);
 //  int rowNum = taos_affected_rows(res);
@@ -614,7 +614,7 @@ TEST(testCase, smlProcess_telnet_Test) {
 //  for (int i = 0; i < rowNum; ++i) {
 //   TAOS_ROW rows = taos_fetch_row(res);
 //  }
-  taos_free_result(res);
+//  taos_free_result(res);
 
 //  res = taos_query(taos, "select * from t_6931529054e5637ca92c78a1ad441961");
 //  ASSERT_NE(res, nullptr);
@@ -670,16 +670,16 @@ TEST(testCase, smlProcess_json1_Test) {
   int ret = smlProcess(info, (char **)(&sql), -1);
   ASSERT_EQ(ret, 0);
 
-  TAOS_RES *res = taos_query(taos, "select * from t_cb27a7198d637b4f1c6464bd73f756a7");
+//  TAOS_RES *res = taos_query(taos, "select * from t_cb27a7198d637b4f1c6464bd73f756a7");
-  ASSERT_NE(res, nullptr);
+//  ASSERT_NE(res, nullptr);
-  int fieldNum = taos_field_count(res);
+//  int fieldNum = taos_field_count(res);
-  ASSERT_EQ(fieldNum, 2);
+//  ASSERT_EQ(fieldNum, 2);
   //   int rowNum = taos_affected_rows(res);
   //   ASSERT_EQ(rowNum, 1);
   //   for (int i = 0; i < rowNum; ++i) {
   //     TAOS_ROW rows = taos_fetch_row(res);
   //   }
-  taos_free_result(res);
+//  taos_free_result(res);
   destroyRequest(request);
   smlDestroyInfo(info);
 }

source/common/src/tglobal.c

@@ -299,6 +299,7 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "fnDebugFlag", fnDebugFlag, 0, 255, 0) != 0) return -1;
+  if (cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, 0) != 0) return -1;
   return 0;
 }
 
@@ -480,6 +481,7 @@ static void taosSetServerLogCfg(SConfig *pCfg) {
   tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32;
   fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32;
   fnDebugFlag = cfgGetItem(pCfg, "fnDebugFlag")->i32;
+  smaDebugFlag = cfgGetItem(pCfg, "smaDebugFlag")->i32;
 }
 
 static int32_t taosSetClientCfg(SConfig *pCfg) {

source/common/src/tmsg.c

@@ -3562,39 +3562,92 @@ int tDecodeSVCreateTbBatchRsp(SDecoder *pCoder, SVCreateTbBatchRsp *pRsp) {
   return 0;
 }
 
-int32_t tSerializeSVCreateTSmaReq(void **buf, SVCreateTSmaReq *pReq) {
+int32_t tEncodeTSma(SEncoder *pCoder, const STSma *pSma) {
-  int32_t tlen = 0;
+  if (tEncodeI8(pCoder, pSma->version) < 0) return -1;
-
+  if (tEncodeI8(pCoder, pSma->intervalUnit) < 0) return -1;
-  tlen += taosEncodeFixedI64(buf, pReq->ver);
+  if (tEncodeI8(pCoder, pSma->slidingUnit) < 0) return -1;
-  tlen += tEncodeTSma(buf, &pReq->tSma);
+  if (tEncodeI8(pCoder, pSma->timezoneInt) < 0) return -1;
-
+  if (tEncodeCStr(pCoder, pSma->indexName) < 0) return -1;
-  return tlen;
+  if (tEncodeI32(pCoder, pSma->exprLen) < 0) return -1;
-}
+  if (tEncodeI32(pCoder, pSma->tagsFilterLen) < 0) return -1;
-
+  if (tEncodeI64(pCoder, pSma->indexUid) < 0) return -1;
-void *tDeserializeSVCreateTSmaReq(void *buf, SVCreateTSmaReq *pReq) {
+  if (tEncodeI64(pCoder, pSma->tableUid) < 0) return -1;
-  buf = taosDecodeFixedI64(buf, &(pReq->ver));
+  if (tEncodeI64(pCoder, pSma->interval) < 0) return -1;
-
+  if (tEncodeI64(pCoder, pSma->offset) < 0) return -1;
-  if ((buf = tDecodeTSma(buf, &pReq->tSma)) == NULL) {
+  if (tEncodeI64(pCoder, pSma->sliding) < 0) return -1;
-    tdDestroyTSma(&pReq->tSma);
+  if (pSma->exprLen > 0) {
+    if (tEncodeCStr(pCoder, pSma->expr) < 0) return -1;
   }
-  return buf;
+  if (pSma->tagsFilterLen > 0) {
+    if (tEncodeCStr(pCoder, pSma->tagsFilter) < 0) return -1;
+  }
+
+  return 0;
 }
 
-int32_t tSerializeSVDropTSmaReq(void **buf, SVDropTSmaReq *pReq) {
+int32_t tDecodeTSma(SDecoder *pCoder, STSma *pSma) {
-  int32_t tlen = 0;
+  if (tDecodeI8(pCoder, &pSma->version) < 0) return -1;
+  if (tDecodeI8(pCoder, &pSma->intervalUnit) < 0) return -1;
+  if (tDecodeI8(pCoder, &pSma->slidingUnit) < 0) return -1;
+  if (tDecodeI8(pCoder, &pSma->timezoneInt) < 0) return -1;
+  if (tDecodeCStrTo(pCoder, pSma->indexName) < 0) return -1;
+  if (tDecodeI32(pCoder, &pSma->exprLen) < 0) return -1;
+  if (tDecodeI32(pCoder, &pSma->tagsFilterLen) < 0) return -1;
+  if (tDecodeI64(pCoder, &pSma->indexUid) < 0) return -1;
+  if (tDecodeI64(pCoder, &pSma->tableUid) < 0) return -1;
+  if (tDecodeI64(pCoder, &pSma->interval) < 0) return -1;
+  if (tDecodeI64(pCoder, &pSma->offset) < 0) return -1;
+  if (tDecodeI64(pCoder, &pSma->sliding) < 0) return -1;
+  if (pSma->exprLen > 0) {
+    if (tDecodeCStr(pCoder, &pSma->expr) < 0) return -1;
+  } else {
+    pSma->expr = NULL;
+  }
+  if (pSma->tagsFilterLen > 0) {
+    if (tDecodeCStr(pCoder, &pSma->tagsFilter) < 0) return -1;
+  } else {
+    pSma->tagsFilter = NULL;
+  }
 
-  tlen += taosEncodeFixedI64(buf, pReq->ver);
+  return 0;
-  tlen += taosEncodeFixedI64(buf, pReq->indexUid);
-  tlen += taosEncodeString(buf, pReq->indexName);
-
-  return tlen;
 }
-void *tDeserializeSVDropTSmaReq(void *buf, SVDropTSmaReq *pReq) {
-  buf = taosDecodeFixedI64(buf, &(pReq->ver));
-  buf = taosDecodeFixedI64(buf, &(pReq->indexUid));
-  buf = taosDecodeStringTo(buf, pReq->indexName);
 
-  return buf;
+int32_t tEncodeSVCreateTSmaReq(SEncoder *pCoder, const SVCreateTSmaReq *pReq) {
+  if (tStartEncode(pCoder) < 0) return -1;
+
+  tEncodeTSma(pCoder, pReq);
+
+  tEndEncode(pCoder);
+  return 0;
+}
+
+int32_t tDecodeSVCreateTSmaReq(SDecoder *pCoder, SVCreateTSmaReq *pReq) {
+  if (tStartDecode(pCoder) < 0) return -1;
+
+  tDecodeTSma(pCoder, pReq);
+
+  tEndDecode(pCoder);
+  return 0;
+}
+
+int32_t tEncodeSVDropTSmaReq(SEncoder *pCoder, const SVDropTSmaReq *pReq) {
+  if (tStartEncode(pCoder) < 0) return -1;
+
+  if (tEncodeI64(pCoder, pReq->indexUid) < 0) return -1;
+  if (tEncodeCStr(pCoder, pReq->indexName) < 0) return -1;
+
+  tEndEncode(pCoder);
+  return 0;
+}
+
+int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) {
+  if (tStartDecode(pCoder) < 0) return -1;
+
+  if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1;
+  if (tDecodeCStrTo(pCoder, pReq->indexName) < 0) return -1;
+
+  tEndDecode(pCoder);
+  return 0;
 }
 
 int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateStreamReq *pReq) {
@@ -4110,3 +4163,114 @@ void tFreeSSubmitRsp(SSubmitRsp *pRsp) {
 
   taosMemoryFree(pRsp);
 }
+
+int32_t tEncodeSVAlterTbReq(SEncoder *pEncoder, const SVAlterTbReq *pReq) {
+  if (tStartEncode(pEncoder) < 0) return -1;
+
+  if (tEncodeCStr(pEncoder, pReq->tbName) < 0) return -1;
+  if (tEncodeI8(pEncoder, pReq->action) < 0) return -1;
+  switch (pReq->action) {
+    case TSDB_ALTER_TABLE_ADD_COLUMN:
+      if (tEncodeCStr(pEncoder, pReq->colName) < 0) return -1;
+      if (tEncodeI8(pEncoder, pReq->type) < 0) return -1;
+      if (tEncodeI8(pEncoder, pReq->flags) < 0) return -1;
+      if (tEncodeI32v(pEncoder, pReq->bytes) < 0) return -1;
+      break;
+    case TSDB_ALTER_TABLE_DROP_COLUMN:
+      if (tEncodeCStr(pEncoder, pReq->colName) < 0) return -1;
+      break;
+    case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:
+      if (tEncodeCStr(pEncoder, pReq->colName) < 0) return -1;
+      if (tEncodeI32v(pEncoder, pReq->colModBytes) < 0) return -1;
+      break;
+    case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME:
+      if (tEncodeCStr(pEncoder, pReq->colName) < 0) return -1;
+      if (tEncodeCStr(pEncoder, pReq->colNewName) < 0) return -1;
+      break;
+    case TSDB_ALTER_TABLE_UPDATE_TAG_VAL:
+      if (tEncodeCStr(pEncoder, pReq->tagName) < 0) return -1;
+      if (tEncodeI8(pEncoder, pReq->isNull) < 0) return -1;
+      if (!pReq->isNull) {
+        if (tEncodeBinary(pEncoder, pReq->pTagVal, pReq->nTagVal) < 0) return -1;
+      }
+      break;
+    case TSDB_ALTER_TABLE_UPDATE_OPTIONS:
+      if (tEncodeI8(pEncoder, pReq->updateTTL) < 0) return -1;
+      if (pReq->updateTTL) {
+        if (tEncodeI32v(pEncoder, pReq->newTTL) < 0) return -1;
+      }
+      if (tEncodeI8(pEncoder, pReq->updateComment) < 0) return -1;
+      if (pReq->updateComment) {
+        if (tEncodeCStr(pEncoder, pReq->newComment) < 0) return -1;
+      }
+      break;
+    default:
+      break;
+  }
+
+  tEndEncode(pEncoder);
+  return 0;
+}
+
+int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) {
+  if (tStartDecode(pDecoder) < 0) return -1;
+
+  if (tDecodeCStr(pDecoder, &pReq->tbName) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pReq->action) < 0) return -1;
+  if (tDecodeCStr(pDecoder, &pReq->colName) < 0) return -1;
+  switch (pReq->action) {
+    case TSDB_ALTER_TABLE_ADD_COLUMN:
+      if (tDecodeCStr(pDecoder, &pReq->colName) < 0) return -1;
+      if (tDecodeI8(pDecoder, &pReq->type) < 0) return -1;
+      if (tDecodeI8(pDecoder, &pReq->flags) < 0) return -1;
+      if (tDecodeI32v(pDecoder, &pReq->bytes) < 0) return -1;
+      break;
+    case TSDB_ALTER_TABLE_DROP_COLUMN:
+      if (tDecodeCStr(pDecoder, &pReq->colName) < 0) return -1;
+      break;
+    case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:
+      if (tDecodeCStr(pDecoder, &pReq->colName) < 0) return -1;
+      if (tDecodeI32v(pDecoder, &pReq->colModBytes) < 0) return -1;
+      break;
+    case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME:
+      if (tDecodeCStr(pDecoder, &pReq->colName) < 0) return -1;
+      if (tDecodeCStr(pDecoder, &pReq->colNewName) < 0) return -1;
+      break;
+    case TSDB_ALTER_TABLE_UPDATE_TAG_VAL:
+      if (tDecodeCStr(pDecoder, &pReq->tagName) < 0) return -1;
+      if (tDecodeI8(pDecoder, &pReq->isNull) < 0) return -1;
+      if (!pReq->isNull) {
+        if (tDecodeBinary(pDecoder, &pReq->pTagVal, &pReq->nTagVal) < 0) return -1;
+      }
+      break;
+    case TSDB_ALTER_TABLE_UPDATE_OPTIONS:
+      if (tDecodeI8(pDecoder, &pReq->updateTTL) < 0) return -1;
+      if (pReq->updateTTL) {
+        if (tDecodeI32v(pDecoder, &pReq->newTTL) < 0) return -1;
+      }
+      if (tDecodeI8(pDecoder, &pReq->updateComment) < 0) return -1;
+      if (pReq->updateComment) {
+        if (tDecodeCStr(pDecoder, &pReq->newComment) < 0) return -1;
+      }
+      break;
+    default:
+      break;
+  }
+
+  tEndDecode(pDecoder);
+  return 0;
+}
+
+int32_t tEncodeSVAlterTbRsp(SEncoder *pEncoder, const SVAlterTbRsp *pRsp) {
+  if (tStartEncode(pEncoder) < 0) return -1;
+  if (tEncodeI32(pEncoder, pRsp->code) < 0) return -1;
+  tEndEncode(pEncoder);
+  return 0;
+}
+
+int32_t tDecodeSVAlterTbRsp(SDecoder *pDecoder, SVAlterTbRsp *pRsp) {
+  if (tStartDecode(pDecoder) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pRsp->code) < 0) return -1;
+  tEndDecode(pDecoder);
+  return 0;
+}

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -192,7 +192,8 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
     rpcFreeCont(originalRpcMsg.pCont);
 
     // if leader, send response
-    if (pMsg->rpcMsg.handle != NULL && pMsg->rpcMsg.ahandle != NULL) {
+    //if (pMsg->rpcMsg.handle != NULL && pMsg->rpcMsg.ahandle != NULL) {
+    if (pMsg->rpcMsg.handle != NULL) {
       rsp.ahandle = pMsg->rpcMsg.ahandle;
       rsp.handle = pMsg->rpcMsg.handle;
       rsp.refId = pMsg->rpcMsg.refId;
@@ -213,7 +214,23 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf
 
     // todo
     SRpcMsg *pRsp = NULL;
-    (void)vnodeProcessSyncReq(pVnode->pImpl, &pMsg->rpcMsg, &pRsp);
+    int32_t ret = vnodeProcessSyncReq(pVnode->pImpl, &pMsg->rpcMsg, &pRsp);
+    if (ret != 0) {
+      // if leader, send response
+      if (pMsg->rpcMsg.handle != NULL) {
+        SRpcMsg    rsp;
+        rsp.pCont = NULL;
+        rsp.contLen = 0;
+
+        rsp.code = terrno;
+        dTrace("vmProcessSyncQueue error, code:%d", terrno);
+        
+        rsp.ahandle = pMsg->rpcMsg.ahandle;
+        rsp.handle = pMsg->rpcMsg.handle;
+        rsp.refId = pMsg->rpcMsg.refId;
+        tmsgSendRsp(&rsp);
+      }
+    }
 
     rpcFreeCont(pMsg->rpcMsg.pCont);
     taosFreeQitem(pMsg);

source/dnode/mnode/impl/src/mndScheduler.c

@@ -194,6 +194,7 @@ int32_t mndAddShuffledSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* p
 
     // source
     pTask->sourceType = TASK_SOURCE__MERGE;
+    pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK;
 
     // exec
     pTask->execType = TASK_EXEC__NONE;
@@ -235,6 +236,7 @@ int32_t mndAddFixedSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStr
   pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup);
   // source
   pTask->sourceType = TASK_SOURCE__MERGE;
+  pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK;
 
   // exec
   pTask->execType = TASK_EXEC__NONE;
@@ -309,6 +311,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
         SStreamTask* pTask = tNewSStreamTask(pStream->uid);
         // source part
         pTask->sourceType = TASK_SOURCE__SCAN;
+        pTask->inputType = TASK_INPUT_TYPE__SUMBIT_BLOCK;
 
         // sink part
         if (level == 0) {
@@ -372,6 +375,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
 
       // source part, currently only support multi source
       pTask->sourceType = TASK_SOURCE__PIPE;
+      pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK;
 
       // sink part
       pTask->sinkType = TASK_SINK__NONE;
@@ -459,6 +463,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
 
       // source part
       pTask->sourceType = TASK_SOURCE__MERGE;
+      pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK;
 
       // sink part
       pTask->sinkType = TASK_SINK__NONE;

source/dnode/mnode/impl/src/mndSma.c

@@ -242,26 +242,35 @@ SDbObj *mndAcquireDbBySma(SMnode *pMnode, const char *smaName) {
 }
 
 static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, int32_t *pContLen) {
-  SName name = {0};
+  SEncoder encoder = {0};
+  int32_t  contLen = 0;
+  SName    name = {0};
   tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
 
   SVCreateTSmaReq req = {0};
-  req.tSma.version = 0;
+  req.version = 0;
-  req.tSma.intervalUnit = pSma->intervalUnit;
+  req.intervalUnit = pSma->intervalUnit;
-  req.tSma.slidingUnit = pSma->slidingUnit;
+  req.slidingUnit = pSma->slidingUnit;
-  req.tSma.timezoneInt = pSma->timezone;
+  req.timezoneInt = pSma->timezone;
-  tstrncpy(req.tSma.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN);
+  tstrncpy(req.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN);
-  req.tSma.exprLen = pSma->exprLen;
+  req.exprLen = pSma->exprLen;
-  req.tSma.tagsFilterLen = pSma->tagsFilterLen;
+  req.tagsFilterLen = pSma->tagsFilterLen;
-  req.tSma.indexUid = pSma->uid;
+  req.indexUid = pSma->uid;
-  req.tSma.tableUid = pSma->stbUid;
+  req.tableUid = pSma->stbUid;
-  req.tSma.interval = pSma->interval;
+  req.interval = pSma->interval;
-  req.tSma.offset = pSma->offset;
+  req.offset = pSma->offset;
-  req.tSma.sliding = pSma->sliding;
+  req.sliding = pSma->sliding;
-  req.tSma.expr = pSma->expr;
+  req.expr = pSma->expr;
-  req.tSma.tagsFilter = pSma->tagsFilter;
+  req.tagsFilter = pSma->tagsFilter;
+
+  // get length
+  int32_t ret = 0;
+  tEncodeSize(tEncodeSVCreateTSmaReq, &req, contLen, ret);
+  if (ret < 0) {
+    return NULL;
+  }
+  contLen += sizeof(SMsgHead);
 
-  int32_t   contLen = tSerializeSVCreateTSmaReq(NULL, &req) + sizeof(SMsgHead);
   SMsgHead *pHead = taosMemoryMalloc(contLen);
   if (pHead == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -272,22 +281,38 @@ static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSm
   pHead->vgId = htonl(pVgroup->vgId);
 
   void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead));
-  tSerializeSVCreateTSmaReq(&pBuf, &req);
+  tEncoderInit(&encoder, pBuf, contLen - sizeof(SMsgHead));
+  if (tEncodeSVCreateTSmaReq(&encoder, &req) < 0) {
+    taosMemoryFreeClear(pHead);
+    tEncoderClear(&encoder);
+    return NULL;
+  }
+
+  tEncoderClear(&encoder);
 
   *pContLen = contLen;
   return pHead;
 }
 
 static void *mndBuildVDropSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, int32_t *pContLen) {
+  SEncoder       encoder = {0};
+  int32_t        contLen;
   SName name = {0};
   tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
 
   SVDropTSmaReq req = {0};
-  req.ver = 0;
   req.indexUid = pSma->uid;
   tstrncpy(req.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN);
 
-  int32_t   contLen = tSerializeSVDropTSmaReq(NULL, &req) + sizeof(SMsgHead);
+  // get length
+  int32_t ret = 0;
+  tEncodeSize(tEncodeSVDropTSmaReq, &req, contLen, ret);
+  if (ret < 0) {
+    return NULL;
+  }
+
+  contLen += sizeof(SMsgHead);
+
   SMsgHead *pHead = taosMemoryMalloc(contLen);
   if (pHead == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -298,7 +323,14 @@ static void *mndBuildVDropSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma,
   pHead->vgId = htonl(pVgroup->vgId);
 
   void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead));
-  tDeserializeSVDropTSmaReq(&pBuf, &req);
+  tEncoderInit(&encoder, pBuf, contLen - sizeof(SMsgHead));
+
+  if (tEncodeSVDropTSmaReq(&encoder, &req) < 0) {
+    taosMemoryFreeClear(pHead);
+    tEncoderClear(&encoder);
+    return NULL;
+  }
+  tEncoderClear(&encoder);
 
   *pContLen = contLen;
   return pHead;

source/dnode/mnode/impl/src/mndStb.c

@@ -383,9 +383,10 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
   req.suid = pStb->uid;
   req.rollup = pStb->ast1Len > 0 ? 1 : 0;
   req.schema.nCols = pStb->numOfColumns;
-  req.schema.sver = 0;
+  req.schema.sver = pStb->version;
   req.schema.pSchema = pStb->pColumns;
   req.schemaTag.nCols = pStb->numOfTags;
+  req.schemaTag.nCols = 0;
   req.schemaTag.pSchema = pStb->pTags;
 
   if (req.rollup) {
@@ -425,6 +426,10 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
   void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead));
   tEncoderInit(&encoder, pBuf, contLen - sizeof(SMsgHead));
   if (tEncodeSVCreateStbReq(&encoder, &req) < 0) {
+    taosMemoryFreeClear(pHead);
+    taosMemoryFreeClear(req.pRSmaParam.qmsg1);
+    taosMemoryFreeClear(req.pRSmaParam.qmsg2);
+    tEncoderClear(&encoder);
     return NULL;
   }
   tEncoderClear(&encoder);

source/dnode/vnode/CMakeLists.txt

@@ -18,12 +18,21 @@ target_sources(
     "src/meta/metaOpen.c"
     "src/meta/metaIdx.c"
     "src/meta/metaTable.c"
+    "src/meta/metaSma.c"
     "src/meta/metaQuery.c"
     "src/meta/metaCommit.c"
     "src/meta/metaEntry.c"
 
+    # sma
+    "src/sma/sma.c"
+    "src/sma/smaTDBImpl.c"
+    "src/sma/smaEnv.c"
+    "src/sma/smaOpen.c"
+    "src/sma/smaRollup.c"
+    "src/sma/smaTimeRange.c"
+
     # tsdb
-    "src/tsdb/tsdbTDBImpl.c"
+    # "src/tsdb/tsdbTDBImpl.c"
     "src/tsdb/tsdbCommit.c"
     "src/tsdb/tsdbCommit2.c"
     "src/tsdb/tsdbFile.c"
@@ -33,7 +42,7 @@ target_sources(
     "src/tsdb/tsdbMemTable2.c"
     "src/tsdb/tsdbRead.c"
     "src/tsdb/tsdbReadImpl.c"
-    "src/tsdb/tsdbSma.c"
+    # "src/tsdb/tsdbSma.c"
     "src/tsdb/tsdbWrite.c"
 
     # tq

source/dnode/vnode/inc/vnode.h

@@ -189,8 +189,12 @@ struct SMetaEntry {
     struct {
       int64_t        ctime;
       int32_t        ttlDays;
+      int32_t        ncid;  // next column id
       SSchemaWrapper schema;
     } ntbEntry;
+    struct {
+      STSma *tsma;
+    } smaEntry;
   };
 };
 

source/dnode/vnode/src/inc/meta.h

@@ -73,6 +73,7 @@ struct SMeta {
   TDB*      pCtbIdx;
   TDB*      pTagIdx;
   TDB*      pTtlIdx;
+  TDB*      pSmaIdx;
   SMetaIdx* pIdx;
 };
 
@@ -108,6 +109,11 @@ typedef struct {
   tb_uid_t uid;
 } STtlIdxKey;
 
+typedef struct {
+  tb_uid_t uid;
+  int64_t  smaUid;
+} SSmaIdxKey;
+
 #if 1
 
 SMSmaCursor* metaOpenSmaCursor(SMeta* pMeta, tb_uid_t uid);
@@ -118,7 +124,7 @@ int64_t      metaSmaCursorNext(SMSmaCursor* pSmaCur);
 // SMetaDB
 int  metaOpenDB(SMeta* pMeta);
 void metaCloseDB(SMeta* pMeta);
-// int  metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
+int  metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
 int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid);
 int metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg);
 int metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid);

source/dnode/vnode/src/inc/sma.h

@@ -0,0 +1,225 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_VNODE_SMA_H_
+#define _TD_VNODE_SMA_H_
+
+#include "vnodeInt.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// smaDebug ================
+// clang-format off
+#define smaFatal(...) do { if (smaDebugFlag & DEBUG_FATAL) { taosPrintLog("SMA FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}     while(0)
+#define smaError(...) do { if (smaDebugFlag & DEBUG_ERROR) { taosPrintLog("SMA ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}     while(0)
+#define smaWarn(...)  do { if (smaDebugFlag & DEBUG_WARN)  { taosPrintLog("SMA WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}       while(0)
+#define smaInfo(...)  do { if (smaDebugFlag & DEBUG_INFO)  { taosPrintLog("SMA ", DEBUG_INFO, 255, __VA_ARGS__); }}            while(0)
+#define smaDebug(...) do { if (smaDebugFlag & DEBUG_DEBUG) { taosPrintLog("SMA ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0)
+#define smaTrace(...) do { if (smaDebugFlag & DEBUG_TRACE) { taosPrintLog("SMA ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
+// clang-format on
+
+typedef struct SSmaEnv      SSmaEnv;
+typedef struct SSmaStat     SSmaStat;
+typedef struct SSmaStatItem SSmaStatItem;
+typedef struct SSmaKey      SSmaKey;
+typedef struct SRSmaInfo    SRSmaInfo;
+
+#define SMA_IVLD_FID INT_MIN
+
+struct SSmaEnv {
+  TdThreadRwlock lock;
+  int8_t         type;
+  TXN            txn;
+  void          *pPool;  // SPoolMem
+  SDiskID        did;
+  TENV          *dbEnv;  // TODO: If it's better to put it in smaIndex level?
+  char          *path;   // relative path
+  SSmaStat      *pStat;
+};
+
+#define SMA_ENV_LOCK(env)       ((env)->lock)
+#define SMA_ENV_TYPE(env)       ((env)->type)
+#define SMA_ENV_DID(env)        ((env)->did)
+#define SMA_ENV_ENV(env)        ((env)->dbEnv)
+#define SMA_ENV_PATH(env)       ((env)->path)
+#define SMA_ENV_STAT(env)       ((env)->pStat)
+#define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems)
+
+struct SSmaStatItem {
+  /**
+   * @brief The field 'state' is here to demonstrate if one smaIndex is ready to provide service.
+   *    - TSDB_SMA_STAT_OK: 1) The sma calculation of history data is finished; 2) Or recevied information from
+   * Streaming Module or TSDB local persistence.
+   *    - TSDB_SMA_STAT_EXPIRED: 1) If sma calculation of history TS data is not finished; 2) Or if the TSDB is open,
+   * without information about its previous state.
+   *    - TSDB_SMA_STAT_DROPPED: 1)sma dropped
+   * N.B. only applicable to tsma
+   */
+  int8_t    state;           // ETsdbSmaStat
+  SHashObj *expiredWindows;  // key: skey of time window, value: N/A
+  STSma    *pTSma;           // cache schema
+};
+
+struct SSmaStat {
+  union {
+    SHashObj *smaStatItems;  // key: indexUid, value: SSmaStatItem for tsma
+    SHashObj *rsmaInfoHash;  // key: stbUid, value: SRSmaInfo;
+  };
+  T_REF_DECLARE()
+};
+#define SMA_STAT_ITEMS(s)     ((s)->smaStatItems)
+#define SMA_STAT_INFO_HASH(s) ((s)->rsmaInfoHash)
+
+struct SSmaKey {
+  TSKEY   skey;
+  int64_t groupId;
+};
+
+typedef struct SDBFile SDBFile;
+
+struct SDBFile {
+  int32_t fid;
+  TDB    *pDB;
+  char   *path;
+};
+
+int32_t tdSmaBeginCommit(SSmaEnv *pEnv);
+int32_t tdSmaEndCommit(SSmaEnv *pEnv);
+
+int32_t smaOpenDBEnv(TENV **ppEnv, const char *path);
+int32_t smaCloseDBEnv(TENV *pEnv);
+int32_t smaOpenDBF(TENV *pEnv, SDBFile *pDBF);
+int32_t smaCloseDBF(SDBFile *pDBF);
+int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn);
+void   *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen);
+
+void  tdDestroySmaEnv(SSmaEnv *pSmaEnv);
+void *tdFreeSmaEnv(SSmaEnv *pSmaEnv);
+#if 0
+int32_t tbGetTSmaStatus(SSma *pSma, STSma *param, void *result);
+int32_t tbRemoveTSmaData(SSma *pSma, STSma *param, STimeWindow *pWin);
+#endif
+
+static FORCE_INLINE int32_t tdEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) {
+  int32_t len = 0;
+  len += taosEncodeFixedI64(pData, tsKey);
+  len += taosEncodeFixedI64(pData, groupId);
+  return len;
+}
+
+int32_t tdInitSma(SSma *pSma);
+int32_t tdDropTSma(SSma *pSma, char *pMsg);
+int32_t tdDropTSmaData(SSma *pSma, int64_t indexUid);
+int32_t tdInsertRSmaData(SSma *pSma, char *msg);
+
+int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat);
+int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat);
+int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType);
+
+int32_t tdLockSma(SSma *pSma);
+int32_t tdUnLockSma(SSma *pSma);
+
+int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg);
+
+static FORCE_INLINE int16_t tdTSmaAdd(SSma *pSma, int16_t n) { return atomic_add_fetch_16(&SMA_TSMA_NUM(pSma), n); }
+static FORCE_INLINE int16_t tdTSmaSub(SSma *pSma, int16_t n) { return atomic_sub_fetch_16(&SMA_TSMA_NUM(pSma), n); }
+
+static FORCE_INLINE int32_t tdRLockSmaEnv(SSmaEnv *pEnv) {
+  int code = taosThreadRwlockRdlock(&(pEnv->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+static FORCE_INLINE int32_t tdWLockSmaEnv(SSmaEnv *pEnv) {
+  int code = taosThreadRwlockWrlock(&(pEnv->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+static FORCE_INLINE int32_t tdUnLockSmaEnv(SSmaEnv *pEnv) {
+  int code = taosThreadRwlockUnlock(&(pEnv->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+static FORCE_INLINE int8_t tdSmaStat(SSmaStatItem *pStatItem) {
+  if (pStatItem) {
+    return atomic_load_8(&pStatItem->state);
+  }
+  return TSDB_SMA_STAT_UNKNOWN;
+}
+
+static FORCE_INLINE bool tdSmaStatIsOK(SSmaStatItem *pStatItem, int8_t *state) {
+  if (!pStatItem) {
+    return false;
+  }
+
+  if (state) {
+    *state = atomic_load_8(&pStatItem->state);
+    return *state == TSDB_SMA_STAT_OK;
+  }
+  return atomic_load_8(&pStatItem->state) == TSDB_SMA_STAT_OK;
+}
+
+static FORCE_INLINE bool tdSmaStatIsExpired(SSmaStatItem *pStatItem) {
+  return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_EXPIRED) : true;
+}
+
+static FORCE_INLINE bool tdSmaStatIsDropped(SSmaStatItem *pStatItem) {
+  return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_DROPPED) : true;
+}
+
+static FORCE_INLINE void tdSmaStatSetOK(SSmaStatItem *pStatItem) {
+  if (pStatItem) {
+    atomic_store_8(&pStatItem->state, TSDB_SMA_STAT_OK);
+  }
+}
+
+static FORCE_INLINE void tdSmaStatSetExpired(SSmaStatItem *pStatItem) {
+  if (pStatItem) {
+    atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_EXPIRED);
+  }
+}
+
+static FORCE_INLINE void tdSmaStatSetDropped(SSmaStatItem *pStatItem) {
+  if (pStatItem) {
+    atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_DROPPED);
+  }
+}
+
+static int32_t  tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType);
+void           *tdFreeSmaStatItem(SSmaStatItem *pSmaStatItem);
+static int32_t  tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType);
+static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path, SDiskID did);
+static int32_t  tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SDiskID did, SSmaEnv **pEnv);
+
+void *tdFreeRSmaInfo(SRSmaInfo *pInfo);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_VNODE_SMA_H_*/

source/dnode/vnode/src/inc/tsdb.h

@@ -60,31 +60,22 @@ typedef struct {
   TSKEY minKey;
 } SRtn;
 
-struct SSmaEnvs {
+#define TSDB_DATA_DIR_LEN 6
-  int16_t  nTSma;
-  int16_t  nRSma;
-  SSmaEnv *pTSmaEnv;
-  SSmaEnv *pRSmaEnv;
-};
-
 struct STsdb {
   char          *path;
   SVnode        *pVnode;
   TdThreadMutex  mutex;
+  char           dir[TSDB_DATA_DIR_LEN];
   bool           repoLocked;
-  int8_t         level;  // retention level
   STsdbKeepCfg   keepCfg;
   STsdbMemTable *mem;
   STsdbMemTable *imem;
   SRtn           rtn;
   STsdbFS       *fs;
-  SSmaEnvs       smaEnvs;
 };
 
 #if 1  // ======================================
 
-typedef struct SSmaStat SSmaStat;
-
 struct STable {
   uint64_t  tid;
   uint64_t  uid;
@@ -95,10 +86,6 @@ struct STable {
 #define TABLE_UID(t) (t)->uid
 
 int     tsdbPrepareCommit(STsdb *pTsdb);
-int32_t tsdbInitSma(STsdb *pTsdb);
-int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg);
-int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid);
-int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg);
 typedef enum {
   TSDB_FILE_HEAD = 0,  // .head
   TSDB_FILE_DATA,      // .data
@@ -107,8 +94,6 @@ typedef enum {
   TSDB_FILE_SMAL,      // .smal(Block-wise SMA)
   TSDB_FILE_MAX,       //
   TSDB_FILE_META,      // meta
-  TSDB_FILE_TSMA,      // v2t100.${sma_index_name}, Time-range-wise SMA
-  TSDB_FILE_RSMA,      // v2r100.${sma_index_name}, Time-range-wise Rollup SMA
 } E_TSDB_FILE_T;
 
 typedef struct {
@@ -186,15 +171,10 @@ struct STsdbFS {
 #define REPO_ID(r)        TD_VID((r)->pVnode)
 #define REPO_CFG(r)       (&(r)->pVnode->config.tsdbCfg)
 #define REPO_KEEP_CFG(r)  (&(r)->keepCfg)
-#define REPO_LEVEL(r)     ((r)->level)
 #define REPO_FS(r)        ((r)->fs)
 #define REPO_META(r)      ((r)->pVnode->pMeta)
 #define REPO_TFS(r)       ((r)->pVnode->pTfs)
 #define IS_REPO_LOCKED(r) ((r)->repoLocked)
-#define REPO_TSMA_NUM(r)  ((r)->smaEnvs.nTSma)
-#define REPO_RSMA_NUM(r)  ((r)->smaEnvs.nRSma)
-#define REPO_TSMA_ENV(r)  ((r)->smaEnvs.pTSmaEnv)
-#define REPO_RSMA_ENV(r)  ((r)->smaEnvs.pRSmaEnv)
 
 int tsdbLockRepo(STsdb *pTsdb);
 int tsdbUnlockRepo(STsdb *pTsdb);
@@ -794,25 +774,6 @@ typedef struct {
 } SFSHeader;
 
 // ================== TSDB File System Meta
-
-/**
- * @brief Directory structure of .tsma data files.
- *
- *  /vnode2/tsdb $ tree tsma/
- *   tsma/
- *   ├── v2f100.index_name_1
- *   ├── v2f101.index_name_1
- *   ├── v2f102.index_name_1
- *   ├── v2f1900.index_name_3
- *   ├── v2f1901.index_name_3
- *   ├── v2f1902.index_name_3
- *   ├── v2f200.index_name_2
- *   ├── v2f201.index_name_2
- *   └── v2f202.index_name_2
- *
- *   0 directories, 9 files
- */
-
 #define FS_CURRENT_STATUS(pfs) ((pfs)->cstatus)
 #define FS_NEW_STATUS(pfs)     ((pfs)->nstatus)
 #define FS_IN_TXN(pfs)         (pfs)->intxn
@@ -874,43 +835,6 @@ static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) {
   return 0;
 }
 
-typedef struct SSmaKey SSmaKey;
-
-struct SSmaKey {
-  TSKEY   skey;
-  int64_t groupId;
-};
-
-typedef struct SDBFile SDBFile;
-
-struct SDBFile {
-  int32_t fid;
-  TDB    *pDB;
-  char   *path;
-};
-
-int32_t tsdbOpenDBEnv(TENV **ppEnv, const char *path);
-int32_t tsdbCloseDBEnv(TENV *pEnv);
-int32_t tsdbOpenDBF(TENV *pEnv, SDBFile *pDBF);
-int32_t tsdbCloseDBF(SDBFile *pDBF);
-int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn);
-void   *tsdbGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen);
-
-void  tsdbDestroySmaEnv(SSmaEnv *pSmaEnv);
-void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv);
-#if 0
-int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result);
-int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin);
-#endif
-
-// internal func
-static FORCE_INLINE int32_t tsdbEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) {
-  int32_t len = 0;
-  len += taosEncodeFixedI64(pData, tsKey);
-  len += taosEncodeFixedI64(pData, groupId);
-  return len;
-}
-
 #endif
 
 #ifdef __cplusplus

source/dnode/vnode/src/inc/vnodeInt.h

@@ -47,13 +47,15 @@
 extern "C" {
 #endif
 
-typedef struct SVnodeInfo SVnodeInfo;
+typedef struct SVnodeInfo   SVnodeInfo;
-typedef struct SMeta      SMeta;
+typedef struct SMeta        SMeta;
-typedef struct STsdb      STsdb;
+typedef struct SSma         SSma;
-typedef struct STQ        STQ;
+typedef struct STsdb        STsdb;
-typedef struct SVState    SVState;
+typedef struct STQ          STQ;
-typedef struct SVBufPool  SVBufPool;
+typedef struct SVState      SVState;
-typedef struct SQWorker   SQHandle;
+typedef struct SVBufPool    SVBufPool;
+typedef struct SQWorker     SQHandle;
+typedef struct STsdbKeepCfg STsdbKeepCfg;
 
 #define VNODE_META_DIR  "meta"
 #define VNODE_TSDB_DIR  "tsdb"
@@ -77,9 +79,11 @@ int             metaClose(SMeta* pMeta);
 int             metaBegin(SMeta* pMeta);
 int             metaCommit(SMeta* pMeta);
 int             metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq);
+int             metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq);
 int             metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq);
 int             metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq);
 int             metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq);
+int             metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq);
 SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline);
 STSchema*       metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver);
 int             metaGetTableEntryByName(SMetaReader* pReader, const char* name);
@@ -90,17 +94,14 @@ tb_uid_t        metaCtbCursorNext(SMCtbCursor* pCtbCur);
 SArray*         metaGetSmaTbUids(SMeta* pMeta, bool isDup);
 void*           metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid, bool isDecode);
 STSmaWrapper*   metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid);
-int32_t         metaCreateTSma(SMeta* pMeta, SSmaCfg* pCfg);
+int32_t         metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg);
 int32_t         metaDropTSma(SMeta* pMeta, int64_t indexUid);
 
 // tsdb
-int          tsdbOpen(SVnode* pVnode, int8_t type);
+int          tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg);
-int          tsdbClose(STsdb* pTsdb);
+int          tsdbClose(STsdb** pTsdb);
 int          tsdbBegin(STsdb* pTsdb);
 int          tsdbCommit(STsdb* pTsdb);
-int32_t      tsdbUpdateSmaWindow(STsdb* pTsdb, SSubmitReq* pMsg, int64_t version);
-int32_t      tsdbCreateTSma(STsdb* pTsdb, char* pMsg);
-int32_t      tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg);
 int          tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp);
 int          tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkRsp* pRsp);
 tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId,
@@ -121,13 +122,31 @@ int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t wo
 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId);
 
 // sma
+int32_t smaOpen(SVnode* pVnode);
+int32_t smaClose(SSma* pSma);
 
+int32_t tdUpdateExpireWindow(SSma* pSma, SSubmitReq* pMsg, int64_t version);
+int32_t tdProcessTSmaCreate(SSma* pSma, char* pMsg);
+int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg);
+
+int32_t tdProcessRSmaCreate(SSma* pSma, SMeta* pMeta, SVCreateStbReq* pReq, SMsgCb* pMsgCb);
+int32_t tdProcessRSmaSubmit(SSma* pSma, void* pMsg, int32_t inputType);
+int32_t tdFetchTbUidList(SSma* pSma, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid);
+int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore);
+void    tdUidStoreDestory(STbUidStore* pStore);
+void*   tdUidStoreFree(STbUidStore* pStore);
+
+#if 0
+int32_t tsdbUpdateSmaWindow(STsdb* pTsdb, SSubmitReq* pMsg, int64_t version);
+int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg);
+int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg);
 int32_t tsdbRegisterRSma(STsdb* pTsdb, SMeta* pMeta, SVCreateStbReq* pReq, SMsgCb* pMsgCb);
 int32_t tsdbFetchTbUidList(STsdb* pTsdb, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid);
 int32_t tsdbUpdateTbUidList(STsdb* pTsdb, STbUidStore* pUidStore);
 void    tsdbUidStoreDestory(STbUidStore* pStore);
 void*   tsdbUidStoreFree(STbUidStore* pStore);
 int32_t tsdbTriggerRSma(STsdb* pTsdb, void* pMsg, int32_t inputType);
+#endif
 
 typedef struct {
   int8_t  streamType;  // sma or other
@@ -164,13 +183,13 @@ typedef enum {
   TSDB_TYPE_RSMA_L2 = 4,  // RSMA Level 2
 } ETsdbType;
 
-typedef struct {
+struct STsdbKeepCfg {
   int8_t  precision;  // precision always be used with below keep cfgs
   int32_t days;
   int32_t keep0;
   int32_t keep1;
   int32_t keep2;
-} STsdbKeepCfg;
+};
 
 struct SVnode {
   char*      path;
@@ -183,9 +202,8 @@ struct SVnode {
   SVBufPool* onCommit;
   SVBufPool* onRecycle;
   SMeta*     pMeta;
+  SSma*      pSma;
   STsdb*     pTsdb;
-  STsdb*     pRSma1;
-  STsdb*     pRSma2;
   SWal*      pWal;
   STQ*       pTq;
   SSink*     pSink;
@@ -194,10 +212,12 @@ struct SVnode {
   SQHandle*  pQuery;
 };
 
+#define TD_VID(PVNODE) (PVNODE)->config.vgId
+
 #define VND_TSDB(vnd)       ((vnd)->pTsdb)
 #define VND_RSMA0(vnd)      ((vnd)->pTsdb)
-#define VND_RSMA1(vnd)      ((vnd)->pRSma1)
+#define VND_RSMA1(vnd)      ((vnd)->pSma->pRSmaTsdb1)
-#define VND_RSMA2(vnd)      ((vnd)->pRSma2)
+#define VND_RSMA2(vnd)      ((vnd)->pSma->pRSmaTsdb2)
 #define VND_RETENTIONS(vnd) (&(vnd)->config.tsdbCfg.retentions)
 
 struct STbUidStore {
@@ -207,7 +227,29 @@ struct STbUidStore {
   SHashObj* uidHash;
 };
 
-#define TD_VID(PVNODE) (PVNODE)->config.vgId
+struct SSma {
+  int16_t       nTSma;
+  bool          locked;
+  TdThreadMutex mutex;
+  SVnode*       pVnode;
+  STsdb*        pRSmaTsdb1;
+  STsdb*        pRSmaTsdb2;
+  void*         pTSmaEnv;
+  void*         pRSmaEnv;
+};
+
+#define SMA_CFG(s)        (&(s)->pVnode->config)
+#define SMA_TSDB_CFG(s)   (&(s)->pVnode->config.tsdbCfg)
+#define SMA_LOCKED(s)     ((s)->locked)
+#define SMA_META(s)       ((s)->pVnode->pMeta)
+#define SMA_VID(s)        TD_VID((s)->pVnode)
+#define SMA_TFS(s)        ((s)->pVnode->pTfs)
+#define SMA_TSMA_NUM(s)   ((s)->nTSma)
+#define SMA_TSMA_ENV(s)   ((s)->pTSmaEnv)
+#define SMA_RSMA_ENV(s)   ((s)->pRSmaEnv)
+#define SMA_RSMA_TSDB0(s) ((s)->pVnode->pTsdb)
+#define SMA_RSMA_TSDB1(s) ((s)->pRSmaTsdb1)
+#define SMA_RSMA_TSDB2(s) ((s)->pRSmaTsdb2)
 
 static FORCE_INLINE bool vnodeIsRollup(SVnode* pVnode) {
   SRetention* pRetention = &(pVnode->config.tsdbCfg.retentions[0]);

source/dnode/vnode/src/meta/metaEntry.c

@@ -34,7 +34,10 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) {
   } else if (pME->type == TSDB_NORMAL_TABLE) {
     if (tEncodeI64(pCoder, pME->ntbEntry.ctime) < 0) return -1;
     if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1;
+    if (tEncodeI32v(pCoder, pME->ntbEntry.ncid) < 0) return -1;
     if (tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1;
+  } else if (pME->type == TSDB_TSMA_TABLE) {
+    if (tEncodeTSma(pCoder, pME->smaEntry.tsma) < 0) return -1;
   } else {
     ASSERT(0);
   }
@@ -63,7 +66,10 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
   } else if (pME->type == TSDB_NORMAL_TABLE) {
     if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1;
     if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1;
+    if (tDecodeI32v(pCoder, &pME->ntbEntry.ncid) < 0) return -1;
     if (tDecodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1;
+  } else if (pME->type == TSDB_TSMA_TABLE) {
+    if (tDecodeTSma(pCoder, pME->smaEntry.tsma) < 0) return -1;
   } else {
     ASSERT(0);
   }

source/dnode/vnode/src/meta/metaIdx.c

@@ -112,35 +112,4 @@ int metaRemoveTableFromIdx(SMeta *pMeta, tb_uid_t uid) {
 #endif
   // TODO
   return 0;
-}
-
-int32_t metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg) {
-  // TODO: Validate the cfg
-  // The table uid should exists and be super table or common table.
-  // Check other cfg value
-
-  // TODO: add atomicity
-
-#ifdef META_REFACT
-#else
-  if (metaSaveSmaToDB(pMeta, &pCfg->tSma) < 0) {
-    // TODO: handle error
-    return -1;
-  }
-#endif
-  return TSDB_CODE_SUCCESS;
-}
-
-int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid) {
-  // TODO: Validate the cfg
-  // TODO: add atomicity
-
-#ifdef META_REFACT
-#else
-  if (metaRemoveSmaFromDb(pMeta, indexUid) < 0) {
-    // TODO: handle error
-    return -1;
-  }
-#endif
-  return TSDB_CODE_SUCCESS;
 }

source/dnode/vnode/src/meta/metaOpen.c

@@ -21,6 +21,7 @@ static int ctbIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL
 static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
 static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
 static int uidIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
+static int smaIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
 
 static int32_t metaInitLock(SMeta *pMeta) { return taosThreadRwlockInit(&pMeta->lock, NULL); }
 static int32_t metaDestroyLock(SMeta *pMeta) { return taosThreadRwlockDestroy(&pMeta->lock); }
@@ -104,6 +105,13 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
     goto _err;
   }
 
+  // open pSmaIdx
+  ret = tdbDbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx);
+  if (ret < 0) {
+    metaError("vgId:%d failed to open meta sma index since %s", TD_VID(pVnode), tstrerror(terrno));
+    goto _err;
+  }
+
   // open index
   if (metaOpenIdx(pMeta) < 0) {
     metaError("vgId:%d failed to open meta index since %s", TD_VID(pVnode), tstrerror(terrno));
@@ -117,11 +125,12 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
 
 _err:
   if (pMeta->pIdx) metaCloseIdx(pMeta);
+  if (pMeta->pSmaIdx) tdbDbClose(pMeta->pSmaIdx);
   if (pMeta->pTtlIdx) tdbDbClose(pMeta->pTtlIdx);
   if (pMeta->pTagIdx) tdbDbClose(pMeta->pTagIdx);
   if (pMeta->pCtbIdx) tdbDbClose(pMeta->pCtbIdx);
   if (pMeta->pNameIdx) tdbDbClose(pMeta->pNameIdx);
-  if (pMeta->pNameIdx) tdbDbClose(pMeta->pUidIdx);
+  if (pMeta->pUidIdx) tdbDbClose(pMeta->pUidIdx);
   if (pMeta->pSkmDb) tdbDbClose(pMeta->pSkmDb);
   if (pMeta->pTbDb) tdbDbClose(pMeta->pTbDb);
   if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv);
@@ -133,11 +142,12 @@ _err:
 int metaClose(SMeta *pMeta) {
   if (pMeta) {
     if (pMeta->pIdx) metaCloseIdx(pMeta);
+    if (pMeta->pSmaIdx) tdbDbClose(pMeta->pSmaIdx);
     if (pMeta->pTtlIdx) tdbDbClose(pMeta->pTtlIdx);
     if (pMeta->pTagIdx) tdbDbClose(pMeta->pTagIdx);
     if (pMeta->pCtbIdx) tdbDbClose(pMeta->pCtbIdx);
     if (pMeta->pNameIdx) tdbDbClose(pMeta->pNameIdx);
-    if (pMeta->pNameIdx) tdbDbClose(pMeta->pUidIdx);
+    if (pMeta->pUidIdx) tdbDbClose(pMeta->pUidIdx);
     if (pMeta->pSkmDb) tdbDbClose(pMeta->pSkmDb);
     if (pMeta->pTbDb) tdbDbClose(pMeta->pTbDb);
     if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv);
@@ -295,3 +305,22 @@ static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL
 
   return 0;
 }
+
+static int smaIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
+  SSmaIdxKey *pSmaIdxKey1 = (SSmaIdxKey *)pKey1;
+  SSmaIdxKey *pSmaIdxKey2 = (SSmaIdxKey *)pKey2;
+
+  if (pSmaIdxKey1->uid > pSmaIdxKey2->uid) {
+    return 1;
+  } else if (pSmaIdxKey1->uid < pSmaIdxKey2->uid) {
+    return -1;
+  }
+
+  if (pSmaIdxKey1->smaUid > pSmaIdxKey2->smaUid) {
+    return 1;
+  } else if (pSmaIdxKey1->smaUid < pSmaIdxKey2->smaUid) {
+    return -1;
+  }
+
+  return 0;
+}

source/dnode/vnode/src/meta/metaSma.c

@@ -0,0 +1,208 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "meta.h"
+
+static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME);
+static int metaSaveSmaToDB(SMeta *pMeta,  const SMetaEntry *pME);
+
+int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) {
+  // TODO: Validate the cfg
+  // The table uid should exists and be super table or normal table.
+  // Check other cfg value
+
+  SMetaEntry  me = {0};
+  int         kLen = 0;
+  int         vLen = 0;
+  const void *pKey = NULL;
+  const void *pVal = NULL;
+  void       *pBuf = NULL;
+  int32_t     szBuf = 0;
+  void       *p = NULL;
+  SMetaReader mr = {0};
+
+  // validate req
+  metaReaderInit(&mr, pMeta, 0);
+  if (metaGetTableEntryByUid(&mr, pCfg->indexUid) == 0) {
+// TODO: just for pass case
+#if 1
+    terrno = TSDB_CODE_TDB_TSMA_ALREADY_EXIST;
+    metaReaderClear(&mr);
+    return -1;
+#else
+    metaReaderClear(&mr);
+    return 0;
+#endif
+  }
+  metaReaderClear(&mr);
+
+  // set structs
+  me.version = version;
+  me.type = TSDB_TSMA_TABLE;
+  me.uid = pCfg->indexUid;
+  me.name = pCfg->indexName;
+  me.smaEntry.tsma = pCfg;
+
+  if (metaHandleSmaEntry(pMeta, &me) < 0) goto _err;
+
+  metaDebug("vgId:%d tsma is created, name:%s uid: %" PRId64, TD_VID(pMeta->pVnode), pCfg->indexName, pCfg->indexUid);
+
+  return 0;
+
+_err:
+  metaError("vgId:%d failed to create tsma: %s uid: %" PRId64 " since %s", TD_VID(pMeta->pVnode), pCfg->indexName,
+            pCfg->indexUid, tstrerror(terrno));
+  return -1;
+}
+
+int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid) {
+  // TODO: Validate the cfg
+  // TODO: add atomicity
+
+#ifdef META_REFACT
+#else
+  if (metaRemoveSmaFromDb(pMeta, indexUid) < 0) {
+    // TODO: handle error
+    return -1;
+  }
+#endif
+  return TSDB_CODE_SUCCESS;
+}
+
+// static int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) {
+//   int32_t ret = 0;
+//   void   *pBuf = NULL, *qBuf = NULL;
+//   void   *key = {0}, *val = {0};
+
+//   // save sma info
+//   int32_t len = tEncodeTSma(NULL, pSmaCfg);
+//   pBuf = taosMemoryCalloc(1, len);
+//   if (pBuf == NULL) {
+//     terrno = TSDB_CODE_OUT_OF_MEMORY;
+//     return -1;
+//   }
+
+//   key = (void *)&pSmaCfg->indexUid;
+//   qBuf = pBuf;
+//   tEncodeTSma(&qBuf, pSmaCfg);
+//   val = pBuf;
+
+//   int32_t kLen = sizeof(pSmaCfg->indexUid);
+//   int32_t vLen = POINTER_DISTANCE(qBuf, pBuf);
+
+//   ret = tdbDbInsert(pMeta->pTbDb, key, kLen, val, vLen, &pMeta->txn);
+//   if (ret < 0) {
+//     taosMemoryFreeClear(pBuf);
+//     return -1;
+//   }
+
+//   // add sma idx
+//   SSmaIdxKey smaIdxKey;
+//   smaIdxKey.uid = pSmaCfg->tableUid;
+//   smaIdxKey.smaUid = pSmaCfg->indexUid;
+//   key = &smaIdxKey;
+//   kLen = sizeof(smaIdxKey);
+//   val = NULL;
+//   vLen = 0;
+
+//   ret = tdbDbInsert(pMeta->pSmaIdx, key, kLen, val, vLen, &pMeta->txn);
+//   if (ret < 0) {
+//     taosMemoryFreeClear(pBuf);
+//     return -1;
+//   }
+
+//   // release
+//   taosMemoryFreeClear(pBuf);
+
+//   return 0;
+// }
+
+
+static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME) {
+  STbDbKey tbDbKey;
+  void    *pKey = NULL;
+  void    *pVal = NULL;
+  int      kLen = 0;
+  int      vLen = 0;
+  SEncoder coder = {0};
+
+  // set key and value
+  tbDbKey.version = pME->version;
+  tbDbKey.uid = pME->uid;
+
+  pKey = &tbDbKey;
+  kLen = sizeof(tbDbKey);
+
+  int32_t ret = 0;
+  tEncodeSize(metaEncodeEntry, pME, vLen, ret);
+  if (ret < 0) {
+    goto _err;
+  }
+
+  pVal = taosMemoryMalloc(vLen);
+  if (pVal == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+  tEncoderInit(&coder, pVal, vLen);
+
+  if (metaEncodeEntry(&coder, pME) < 0) {
+    goto _err;
+  }
+
+  tEncoderClear(&coder);
+
+  // write to table.db
+  if (tdbDbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) {
+    goto _err;
+  }
+
+  taosMemoryFree(pVal);
+  return 0;
+
+_err:
+  taosMemoryFree(pVal);
+  return -1;
+}
+
+static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) {
+  return tdbDbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn);
+}
+
+static int metaUpdateSmaIdx(SMeta *pMeta, const SMetaEntry *pME) {
+  SSmaIdxKey smaIdxKey = {.uid = pME->smaEntry.tsma->tableUid, .smaUid = pME->smaEntry.tsma->indexUid};
+
+  return tdbDbInsert(pMeta->pSmaIdx, &smaIdxKey, sizeof(smaIdxKey), NULL, 0, &pMeta->txn);
+}
+
+static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME) {
+  metaWLock(pMeta);
+
+  // save to table.db
+  if (metaSaveSmaToDB(pMeta, pME) < 0) goto _err;
+
+  // // update uid.idx
+  if (metaUpdateUidIdx(pMeta, pME) < 0) goto _err;
+
+  if (metaUpdateSmaIdx(pMeta, pME) < 0) goto _err;
+
+  metaULock(pMeta);
+  return 0;
+
+_err:
+  metaULock(pMeta);
+  return -1;
+}

source/dnode/vnode/src/meta/metaTable.c

@@ -131,6 +131,75 @@ _err:
   return -1;
 }
 
+int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
+  SMetaEntry  oStbEntry = {0};
+  SMetaEntry  nStbEntry = {0};
+  TDBC       *pUidIdxc = NULL;
+  TDBC       *pTbDbc = NULL;
+  const void *pData;
+  int         nData;
+  int64_t     oversion;
+  SDecoder    dc = {0};
+  int32_t     ret;
+  int32_t     c;
+
+  tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn);
+  ret = tdbDbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c);
+  if (ret < 0 || c) {
+    ASSERT(0);
+    return -1;
+  }
+
+  ret = tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData);
+  if (ret < 0) {
+    ASSERT(0);
+    return -1;
+  }
+
+  oversion = *(int64_t *)pData;
+
+  tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn);
+  ret = tdbDbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c);
+  ASSERT(ret == 0 && c == 0);
+
+  ret = tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData);
+  ASSERT(ret == 0);
+
+  tDecoderInit(&dc, pData, nData);
+  metaDecodeEntry(&dc, &oStbEntry);
+
+  nStbEntry.version = version;
+  nStbEntry.type = TSDB_SUPER_TABLE;
+  nStbEntry.uid = pReq->suid;
+  nStbEntry.name = pReq->name;
+  nStbEntry.stbEntry.schema = pReq->schema;
+  nStbEntry.stbEntry.schemaTag = pReq->schemaTag;
+
+  metaWLock(pMeta);
+  // compare two entry
+  if (oStbEntry.stbEntry.schema.sver != pReq->schema.sver) {
+    if (oStbEntry.stbEntry.schema.nCols != pReq->schema.nCols) {
+      metaSaveToSkmDb(pMeta, &nStbEntry);
+    }
+  }
+
+  // if (oStbEntry.stbEntry.schemaTag.sver != pReq->schemaTag.sver) {
+  //   // change tag schema
+  // }
+
+  // update table.db
+  metaSaveToTbDb(pMeta, &nStbEntry);
+
+  // update uid index
+  tdbDbcUpsert(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &version, sizeof(version), 0);
+
+  metaULock(pMeta);
+  tDecoderClear(&dc);
+  tdbDbcClose(pTbDbc);
+  tdbDbcClose(pUidIdxc);
+  return 0;
+}
+
 int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) {
   SMetaEntry  me = {0};
   SMetaReader mr = {0};
@@ -171,6 +240,7 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) {
     me.ntbEntry.ctime = pReq->ctime;
     me.ntbEntry.ttlDays = pReq->ttl;
     me.ntbEntry.schema = pReq->ntb.schema;
+    me.ntbEntry.ncid = me.ntbEntry.schema.pSchema[me.ntbEntry.schema.nCols - 1].colId + 1;
   }
 
   if (metaHandleEntry(pMeta, &me) < 0) goto _err;
@@ -305,6 +375,170 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) {
   return 0;
 }
 
+static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
+  void           *pVal = NULL;
+  int             nVal = 0;
+  const void     *pData = NULL;
+  int             nData = 0;
+  int             ret = 0;
+  tb_uid_t        uid;
+  int64_t         oversion;
+  SSchema        *pColumn = NULL;
+  SMetaEntry      entry = {0};
+  SSchemaWrapper *pSchema;
+  int             c;
+
+  // search name index
+  ret = tdbDbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal);
+  if (ret < 0) {
+    terrno = TSDB_CODE_VND_TABLE_NOT_EXIST;
+    return -1;
+  }
+
+  uid = *(tb_uid_t *)pVal;
+  tdbFree(pVal);
+  pVal = NULL;
+
+  // search uid index
+  TDBC *pUidIdxc = NULL;
+
+  tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn);
+  tdbDbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c);
+  ASSERT(c == 0);
+
+  tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData);
+  oversion = *(int64_t *)pData;
+
+  // search table.db
+  TDBC *pTbDbc = NULL;
+
+  tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn);
+  tdbDbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c);
+  ASSERT(c == 0);
+  tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData);
+
+  // get table entry
+  SDecoder dc = {0};
+  tDecoderInit(&dc, pData, nData);
+  metaDecodeEntry(&dc, &entry);
+
+  if (entry.type != TSDB_NORMAL_TABLE) {
+    terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION;
+    goto _err;
+  }
+
+  // search the column to add/drop/update
+  pSchema = &entry.ntbEntry.schema;
+  int32_t iCol = 0;
+  for (;;) {
+    pColumn = NULL;
+
+    if (iCol >= pSchema->nCols) break;
+    pColumn = &pSchema->pSchema[iCol];
+
+    if (strcmp(pColumn->name, pAlterTbReq->colName) == 0) break;
+    iCol++;
+  }
+
+  entry.version = version;
+  int tlen;
+  switch (pAlterTbReq->action) {
+    case TSDB_ALTER_TABLE_ADD_COLUMN:
+      if (pColumn) {
+        terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS;
+        goto _err;
+      }
+      pSchema->sver++;
+      pSchema->nCols++;
+      pSchema->pSchema =
+          taosMemoryRealloc(entry.ntbEntry.schema.pSchema, sizeof(SSchema) * entry.ntbEntry.schema.nCols);
+      pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].bytes = pAlterTbReq->bytes;
+      pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].type = pAlterTbReq->type;
+      pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].flags = pAlterTbReq->flags;
+      pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].colId = entry.ntbEntry.ncid++;
+      strcpy(pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].name, pAlterTbReq->colName);
+      break;
+    case TSDB_ALTER_TABLE_DROP_COLUMN:
+      if (pColumn == NULL) {
+        terrno = TSDB_CODE_VND_TABLE_COL_NOT_EXISTS;
+        goto _err;
+      }
+      if (pColumn->colId == 0) {
+        terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION;
+        goto _err;
+      }
+      pSchema->sver++;
+      pSchema->nCols--;
+      tlen = (pSchema->nCols - iCol - 1) * sizeof(SSchema);
+      if (tlen) {
+        memmove(pColumn, pColumn + 1, tlen);
+      }
+      break;
+    case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:
+      if (pColumn == NULL) {
+        terrno = TSDB_CODE_VND_TABLE_COL_NOT_EXISTS;
+        goto _err;
+      }
+      if (!IS_VAR_DATA_TYPE(pColumn->type) || pColumn->bytes <= pAlterTbReq->bytes) {
+        terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION;
+        goto _err;
+      }
+      pSchema->sver++;
+      pColumn->bytes = pAlterTbReq->bytes;
+      break;
+    case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME:
+      if (pColumn == NULL) {
+        terrno = TSDB_CODE_VND_TABLE_COL_NOT_EXISTS;
+        goto _err;
+      }
+      pSchema->sver++;
+      strcpy(pColumn->name, pAlterTbReq->colNewName);
+      break;
+  }
+
+  entry.version = version;
+
+  tDecoderClear(&dc);
+  tdbDbcClose(pTbDbc);
+  tdbDbcClose(pUidIdxc);
+  return 0;
+
+_err:
+  tDecoderClear(&dc);
+  tdbDbcClose(pTbDbc);
+  tdbDbcClose(pUidIdxc);
+  return -1;
+}
+
+static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
+  // TODO
+  return 0;
+}
+
+static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
+  // TODO
+  ASSERT(0);
+  return 0;
+}
+
+int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) {
+  switch (pReq->action) {
+    case TSDB_ALTER_TABLE_ADD_COLUMN:
+    case TSDB_ALTER_TABLE_DROP_COLUMN:
+    case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:
+    case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME:
+      return metaAlterTableColumn(pMeta, version, pReq);
+    case TSDB_ALTER_TABLE_UPDATE_TAG_VAL:
+      return metaUpdateTableTagVal(pMeta, version, pReq);
+    case TSDB_ALTER_TABLE_UPDATE_OPTIONS:
+      return metaUpdateTableOptions(pMeta, version, pReq);
+    default:
+      terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION;
+      return -1;
+      break;
+  }
+}
+
 static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) {
   STbDbKey tbDbKey;
   void    *pKey = NULL;

source/dnode/vnode/src/sma/sma.c

@@ -13,35 +13,18 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_VNODE_TSDB_SMA_H_
+#include "sma.h"
-#define _TD_VNODE_TSDB_SMA_H_
 
-#include "tsdb.h"
 
-#ifdef __cplusplus
+// TODO: Who is responsible for resource allocate and release?
-extern "C" {
+int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg) {
-#endif
+  int32_t code = TSDB_CODE_SUCCESS;
 
-// typedef int32_t (*__tb_ddl_fn_t)(void *ahandle, void **result, void *p1, void *p2);
+  if ((code = tdProcessTSmaInsertImpl(pSma, indexUid, msg)) < 0) {
-
+    smaWarn("vgId:%d insert tsma data failed since %s", SMA_VID(pSma), tstrerror(terrno));
-// struct STbDdlH {
-//   void         *ahandle;
-//   void         *result;
-//   __tb_ddl_fn_t fp;
-// };
-
-static FORCE_INLINE int32_t tsdbUidStoreInit(STbUidStore **pStore) {
-  ASSERT(*pStore == NULL);
-  *pStore = taosMemoryCalloc(1, sizeof(STbUidStore));
-  if (*pStore == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return TSDB_CODE_FAILED;
   }
-  return TSDB_CODE_SUCCESS;
+  // TODO: destroy SSDataBlocks(msg)
+  return code;
 }
 
-#ifdef __cplusplus
-}
-#endif
 
-#endif /*_TD_VNODE_TSDB_SMA_H_*/

source/dnode/vnode/src/sma/smaEnv.c

@@ -0,0 +1,463 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "sma.h"
+
+typedef struct SSmaStat SSmaStat;
+
+static const char *TSDB_SMA_DNAME[] = {
+    "",      // TSDB_SMA_TYPE_BLOCK
+    "tsma",  // TSDB_SMA_TYPE_TIME_RANGE
+    "rsma",  // TSDB_SMA_TYPE_ROLLUP
+};
+
+#define SMA_TEST_INDEX_NAME "smaTestIndexName"  // TODO: just for test
+#define SMA_TEST_INDEX_UID  2000000001          // TODO: just for test
+#define SMA_STATE_HASH_SLOT 4
+
+#define RSMA_TASK_INFO_HASH_SLOT 8
+
+typedef struct SPoolMem {
+  int64_t          size;
+  struct SPoolMem *prev;
+  struct SPoolMem *next;
+} SPoolMem;
+
+// declaration of static functions
+
+// insert data
+
+static void tdGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]);
+
+// Pool Memory
+static SPoolMem *openPool();
+static void      clearPool(SPoolMem *pPool);
+static void      closePool(SPoolMem *pPool);
+static void     *poolMalloc(void *arg, size_t size);
+static void      poolFree(void *arg, void *ptr);
+
+// implementation
+
+static SPoolMem *openPool() {
+  SPoolMem *pPool = (SPoolMem *)taosMemoryMalloc(sizeof(*pPool));
+
+  pPool->prev = pPool->next = pPool;
+  pPool->size = 0;
+
+  return pPool;
+}
+
+static void clearPool(SPoolMem *pPool) {
+  if (!pPool) return;
+
+  SPoolMem *pMem;
+
+  do {
+    pMem = pPool->next;
+
+    if (pMem == pPool) break;
+
+    pMem->next->prev = pMem->prev;
+    pMem->prev->next = pMem->next;
+    pPool->size -= pMem->size;
+
+    taosMemoryFree(pMem);
+  } while (1);
+
+  assert(pPool->size == 0);
+}
+
+static void closePool(SPoolMem *pPool) {
+  if (pPool) {
+    clearPool(pPool);
+    taosMemoryFree(pPool);
+  }
+}
+
+static void *poolMalloc(void *arg, size_t size) {
+  void     *ptr = NULL;
+  SPoolMem *pPool = (SPoolMem *)arg;
+  SPoolMem *pMem;
+
+  pMem = (SPoolMem *)taosMemoryMalloc(sizeof(*pMem) + size);
+  if (!pMem) {
+    assert(0);
+  }
+
+  pMem->size = sizeof(*pMem) + size;
+  pMem->next = pPool->next;
+  pMem->prev = pPool;
+
+  pPool->next->prev = pMem;
+  pPool->next = pMem;
+  pPool->size += pMem->size;
+
+  ptr = (void *)(&pMem[1]);
+  return ptr;
+}
+
+static void poolFree(void *arg, void *ptr) {
+  SPoolMem *pPool = (SPoolMem *)arg;
+  SPoolMem *pMem;
+
+  pMem = &(((SPoolMem *)ptr)[-1]);
+
+  pMem->next->prev = pMem->prev;
+  pMem->prev->next = pMem->next;
+  pPool->size -= pMem->size;
+
+  taosMemoryFree(pMem);
+}
+
+int32_t tdInitSma(SSma *pSma) {
+  // tSma
+  int32_t numOfTSma = taosArrayGetSize(metaGetSmaTbUids(SMA_META(pSma), false));
+  if (numOfTSma > 0) {
+    atomic_store_16(&SMA_TSMA_NUM(pSma), (int16_t)numOfTSma);
+  }
+  // TODO: rSma
+  return TSDB_CODE_SUCCESS;
+}
+
+static void tdGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]) {
+  snprintf(dirName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s", TD_DIRSEP, vgId, TD_DIRSEP, TSDB_SMA_DNAME[smaType]);
+}
+
+static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path, SDiskID did) {
+  SSmaEnv *pEnv = NULL;
+
+  pEnv = (SSmaEnv *)taosMemoryCalloc(1, sizeof(SSmaEnv));
+  if (!pEnv) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  SMA_ENV_TYPE(pEnv) = smaType;
+
+  int code = taosThreadRwlockInit(&(pEnv->lock), NULL);
+  if (code) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    taosMemoryFree(pEnv);
+    return NULL;
+  }
+
+  ASSERT(path && (strlen(path) > 0));
+  SMA_ENV_PATH(pEnv) = strdup(path);
+  if (!SMA_ENV_PATH(pEnv)) {
+    tdFreeSmaEnv(pEnv);
+    return NULL;
+  }
+
+  SMA_ENV_DID(pEnv) = did;
+
+  if (tdInitSmaStat(&SMA_ENV_STAT(pEnv), smaType) != TSDB_CODE_SUCCESS) {
+    tdFreeSmaEnv(pEnv);
+    return NULL;
+  }
+
+  char aname[TSDB_FILENAME_LEN] = {0};
+  tfsAbsoluteName(SMA_TFS(pSma), did, path, aname);
+  if (smaOpenDBEnv(&pEnv->dbEnv, aname) != TSDB_CODE_SUCCESS) {
+    tdFreeSmaEnv(pEnv);
+    return NULL;
+  }
+
+  if (!(pEnv->pPool = openPool())) {
+    tdFreeSmaEnv(pEnv);
+    return NULL;
+  }
+
+  return pEnv;
+}
+
+static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SDiskID did, SSmaEnv **pEnv) {
+  if (!pEnv) {
+    terrno = TSDB_CODE_INVALID_PTR;
+    return TSDB_CODE_FAILED;
+  }
+
+  if (!(*pEnv)) {
+    if (!(*pEnv = tdNewSmaEnv(pSma, smaType, path, did))) {
+      return TSDB_CODE_FAILED;
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief Release resources allocated for its member fields, not including itself.
+ *
+ * @param pSmaEnv
+ * @return int32_t
+ */
+void tdDestroySmaEnv(SSmaEnv *pSmaEnv) {
+  if (pSmaEnv) {
+    tdDestroySmaState(pSmaEnv->pStat, SMA_ENV_TYPE(pSmaEnv));
+    taosMemoryFreeClear(pSmaEnv->pStat);
+    taosMemoryFreeClear(pSmaEnv->path);
+    taosThreadRwlockDestroy(&(pSmaEnv->lock));
+    smaCloseDBEnv(pSmaEnv->dbEnv);
+    closePool(pSmaEnv->pPool);
+  }
+}
+
+void *tdFreeSmaEnv(SSmaEnv *pSmaEnv) {
+  tdDestroySmaEnv(pSmaEnv);
+  taosMemoryFreeClear(pSmaEnv);
+  return NULL;
+}
+
+int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat) {
+  if (!pStat) return 0;
+
+  int ref = T_REF_INC(pStat);
+  smaDebug("vgId:%d ref sma stat:%p, val:%d", SMA_VID(pSma), pStat, ref);
+  return 0;
+}
+
+int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) {
+  if (!pStat) return 0;
+
+  int ref = T_REF_DEC(pStat);
+  smaDebug("vgId:%d unref sma stat:%p, val:%d", SMA_VID(pSma), pStat, ref);
+  return 0;
+}
+
+static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType) {
+  ASSERT(pSmaStat != NULL);
+
+  if (*pSmaStat) {  // no lock
+    return TSDB_CODE_SUCCESS;
+  }
+
+  /**
+   *  1. Lazy mode utilized when init SSmaStat to update expired window(or hungry mode when tdNew).
+   *  2. Currently, there is mutex lock when init SSmaEnv, thus no need add lock on SSmaStat, and please add lock if
+   * tdInitSmaStat invoked in other multithread environment later.
+   */
+  if (!(*pSmaStat)) {
+    *pSmaStat = (SSmaStat *)taosMemoryCalloc(1, sizeof(SSmaStat));
+    if (!(*pSmaStat)) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return TSDB_CODE_FAILED;
+    }
+
+    if (smaType == TSDB_SMA_TYPE_ROLLUP) {
+      SMA_STAT_INFO_HASH(*pSmaStat) = taosHashInit(
+          RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
+
+      if (!SMA_STAT_INFO_HASH(*pSmaStat)) {
+        taosMemoryFreeClear(*pSmaStat);
+        return TSDB_CODE_FAILED;
+      }
+    } else if (smaType == TSDB_SMA_TYPE_TIME_RANGE) {
+      SMA_STAT_ITEMS(*pSmaStat) =
+          taosHashInit(SMA_STATE_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+
+      if (!SMA_STAT_ITEMS(*pSmaStat)) {
+        taosMemoryFreeClear(*pSmaStat);
+        return TSDB_CODE_FAILED;
+      }
+    } else {
+      ASSERT(0);
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+void *tdFreeSmaStatItem(SSmaStatItem *pSmaStatItem) {
+  if (pSmaStatItem) {
+    tdDestroyTSma(pSmaStatItem->pTSma);
+    taosMemoryFreeClear(pSmaStatItem->pTSma);
+    taosHashCleanup(pSmaStatItem->expiredWindows);
+    taosMemoryFreeClear(pSmaStatItem);
+  }
+  return NULL;
+}
+
+/**
+ * @brief Release resources allocated for its member fields, not including itself.
+ *
+ * @param pSmaStat
+ * @return int32_t
+ */
+int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) {
+  if (pSmaStat) {
+    // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready.
+    if (smaType == TSDB_SMA_TYPE_TIME_RANGE) {
+      void *item = taosHashIterate(SMA_STAT_ITEMS(pSmaStat), NULL);
+      while (item) {
+        SSmaStatItem *pItem = *(SSmaStatItem **)item;
+        tdFreeSmaStatItem(pItem);
+        item = taosHashIterate(SMA_STAT_ITEMS(pSmaStat), item);
+      }
+      taosHashCleanup(SMA_STAT_ITEMS(pSmaStat));
+    } else if (smaType == TSDB_SMA_TYPE_ROLLUP) {
+      void *infoHash = taosHashIterate(SMA_STAT_INFO_HASH(pSmaStat), NULL);
+      while (infoHash) {
+        SRSmaInfo *pInfoHash = *(SRSmaInfo **)infoHash;
+        tdFreeRSmaInfo(pInfoHash);
+        infoHash = taosHashIterate(SMA_STAT_INFO_HASH(pSmaStat), infoHash);
+      }
+      taosHashCleanup(SMA_STAT_INFO_HASH(pSmaStat));
+    } else {
+      ASSERT(0);
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t tdLockSma(SSma *pSma) {
+  int code = taosThreadMutexLock(&pSma->mutex);
+  if (code != 0) {
+    smaError("vgId:%d failed to lock td since %s", SMA_VID(pSma), strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  pSma->locked = true;
+  return 0;
+}
+
+int32_t tdUnLockSma(SSma *pSma) {
+  ASSERT(SMA_LOCKED(pSma));
+  pSma->locked = false;
+  int code = taosThreadMutexUnlock(&pSma->mutex);
+  if (code != 0) {
+    smaError("vgId:%d failed to unlock td since %s", SMA_VID(pSma), strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) {
+  SSmaEnv *pEnv = NULL;
+
+  // return if already init
+  switch (smaType) {
+    case TSDB_SMA_TYPE_TIME_RANGE:
+      if ((pEnv = (SSmaEnv *)atomic_load_ptr(&SMA_TSMA_ENV(pSma)))) {
+        return TSDB_CODE_SUCCESS;
+      }
+      break;
+    case TSDB_SMA_TYPE_ROLLUP:
+      if ((pEnv = (SSmaEnv *)atomic_load_ptr(&SMA_RSMA_ENV(pSma)))) {
+        return TSDB_CODE_SUCCESS;
+      }
+      break;
+    default:
+      TASSERT(0);
+      return TSDB_CODE_FAILED;
+  }
+
+  // init sma env
+  tdLockSma(pSma);
+  pEnv = (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_load_ptr(&SMA_TSMA_ENV(pSma))
+                                               : atomic_load_ptr(&SMA_RSMA_ENV(pSma));
+  if (!pEnv) {
+    char rname[TSDB_FILENAME_LEN] = {0};
+
+    SDiskID did = {0};
+    if (tfsAllocDisk(SMA_TFS(pSma), TFS_PRIMARY_LEVEL, &did) < 0) {
+      tdUnLockSma(pSma);
+      return TSDB_CODE_FAILED;
+    }
+
+    if (did.level < 0 || did.id < 0) {
+      tdUnLockSma(pSma);
+      smaError("vgId:%d init sma env failed since invalid did(%d,%d)", SMA_VID(pSma), did.level, did.id);
+      return TSDB_CODE_FAILED;
+    }
+
+    tdGetSmaDir(SMA_VID(pSma), smaType, rname);
+
+    if (tfsMkdirRecurAt(SMA_TFS(pSma), rname, did) < 0) {
+      tdUnLockSma(pSma);
+      return TSDB_CODE_FAILED;
+    }
+
+    if (tdInitSmaEnv(pSma, smaType, rname, did, &pEnv) < 0) {
+      tdUnLockSma(pSma);
+      return TSDB_CODE_FAILED;
+    }
+
+    (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_store_ptr(&SMA_TSMA_ENV(pSma), pEnv)
+                                          : atomic_store_ptr(&SMA_RSMA_ENV(pSma), pEnv);
+  }
+  tdUnLockSma(pSma);
+
+  return TSDB_CODE_SUCCESS;
+};
+
+int32_t tdSmaBeginCommit(SSmaEnv *pEnv) {
+  TXN *pTxn = &pEnv->txn;
+  // start a new txn
+  tdbTxnOpen(pTxn, 0, poolMalloc, poolFree, pEnv->pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+  if (tdbBegin(pEnv->dbEnv, pTxn) != 0) {
+    smaWarn("tdSma tdb begin commit fail");
+    return -1;
+  }
+  return 0;
+}
+
+int32_t tdSmaEndCommit(SSmaEnv *pEnv) {
+  TXN *pTxn = &pEnv->txn;
+
+  // Commit current txn
+  if (tdbCommit(pEnv->dbEnv, pTxn) != 0) {
+    smaWarn("tdSma tdb end commit fail");
+    return -1;
+  }
+  tdbTxnClose(pTxn);
+  clearPool(pEnv->pPool);
+  return 0;
+}
+
+#if 0
+/**
+ * @brief Get the start TS key of the last data block of one interval/sliding.
+ *
+ * @param pSma
+ * @param param
+ * @param result
+ * @return int32_t
+ *         1) Return 0 and fill the result if the check procedure is normal;
+ *         2) Return -1 if error occurs during the check procedure.
+ */
+int32_t tdGetTSmaStatus(SSma *pSma, void *smaIndex, void *result) {
+  const char *procedure = "";
+  if (strncmp(procedure, "get the start TS key of the last data block", 100) != 0) {
+    return -1;
+  }
+  // fill the result
+  return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief Remove the tSma data files related to param between pWin.
+ *
+ * @param pSma
+ * @param param
+ * @param pWin
+ * @return int32_t
+ */
+int32_t tdRemoveTSmaData(SSma *pSma, void *smaIndex, STimeWindow *pWin) {
+  // for ("tSmaFiles of param-interval-sliding between pWin") {
+  //   // remove the tSmaFile
+  // }
+  return TSDB_CODE_SUCCESS;
+}
+#endif

source/dnode/vnode/src/sma/smaOpen.c

@@ -0,0 +1,137 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "sma.h"
+#include "tsdb.h"
+
+static int32_t smaEvalDays(SRetention *r, int8_t precision);
+static int32_t smaSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type);
+
+#define SMA_SET_KEEP_CFG(l)                                                                       \
+  do {                                                                                            \
+    SRetention *r = &pCfg->retentions[l];                                                         \
+    pKeepCfg->keep2 = convertTimeFromPrecisionToUnit(r->keep, pCfg->precision, TIME_UNIT_MINUTE); \
+    pKeepCfg->keep0 = pKeepCfg->keep2;                                                            \
+    pKeepCfg->keep1 = pKeepCfg->keep2;                                                            \
+    pKeepCfg->days = smaEvalDays(r, pCfg->precision);                                             \
+  } while (0)
+
+#define SMA_OPEN_RSMA_IMPL(v, l)                                                   \
+  do {                                                                             \
+    SRetention *r = (SRetention *)VND_RETENTIONS(v) + l;                           \
+    if (!RETENTION_VALID(r)) {                                                     \
+      if (l == 0) {                                                                \
+        goto _err;                                                                 \
+      }                                                                            \
+      break;                                                                       \
+    }                                                                              \
+    smaSetKeepCfg(&keepCfg, pCfg, TSDB_TYPE_RSMA_L##l);                            \
+    if (tsdbOpen(v, &SMA_RSMA_TSDB##l(pSma), VNODE_RSMA##l##_DIR, &keepCfg) < 0) { \
+      goto _err;                                                                   \
+    }                                                                              \
+  } while (0)
+
+#define RETENTION_DAYS_SPLIT_RATIO 10
+#define RETENTION_DAYS_SPLIT_MIN   1
+#define RETENTION_DAYS_SPLIT_MAX   30
+
+static int32_t smaEvalDays(SRetention *r, int8_t precision) {
+  int32_t keepDays = convertTimeFromPrecisionToUnit(r->keep, precision, TIME_UNIT_DAY);
+  int32_t freqDays = convertTimeFromPrecisionToUnit(r->freq, precision, TIME_UNIT_DAY);
+
+  int32_t days = keepDays / RETENTION_DAYS_SPLIT_RATIO;
+  if (days <= RETENTION_DAYS_SPLIT_MIN) {
+    days = RETENTION_DAYS_SPLIT_MIN;
+    if (days < freqDays) {
+      days = freqDays + 1;
+    }
+  } else {
+    if (days > RETENTION_DAYS_SPLIT_MAX) {
+      days = RETENTION_DAYS_SPLIT_MAX;
+    }
+    if (days < freqDays) {
+      days = freqDays + 1;
+    }
+  }
+  return days * 1440;
+}
+
+int smaSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type) {
+  pKeepCfg->precision = pCfg->precision;
+  switch (type) {
+    case TSDB_TYPE_TSMA:
+      ASSERT(0);
+      break;
+    case TSDB_TYPE_RSMA_L0:
+      SMA_SET_KEEP_CFG(0);
+      break;
+    case TSDB_TYPE_RSMA_L1:
+      SMA_SET_KEEP_CFG(1);
+      break;
+    case TSDB_TYPE_RSMA_L2:
+      SMA_SET_KEEP_CFG(2);
+      break;
+    default:
+      ASSERT(0);
+      break;
+  }
+  return 0;
+}
+
+int32_t smaOpen(SVnode *pVnode) {
+  STsdbCfg *pCfg = &pVnode->config.tsdbCfg;
+
+  ASSERT(!pVnode->pSma);
+
+  SSma *pSma = taosMemoryCalloc(1, sizeof(SSma));
+  if (!pSma) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+  pSma->pVnode = pVnode;
+  taosThreadMutexInit(&pSma->mutex, NULL);
+  pSma->locked = false;
+
+  if (vnodeIsRollup(pVnode)) {
+    STsdbKeepCfg keepCfg = {0};
+    for (int i = 0; i < TSDB_RETENTION_MAX; ++i) {
+      if (i == TSDB_RETENTION_L0) {
+        SMA_OPEN_RSMA_IMPL(pVnode, 0);
+      } else if (i == TSDB_RETENTION_L1) {
+        SMA_OPEN_RSMA_IMPL(pVnode, 1);
+      } else if (i == TSDB_RETENTION_L2) {
+        SMA_OPEN_RSMA_IMPL(pVnode, 2);
+      } else {
+        ASSERT(0);
+      }
+    }
+  }
+
+  pVnode->pSma = pSma;
+  return 0;
+_err:
+  taosMemoryFreeClear(pSma);
+  return -1;
+}
+
+int32_t smaClose(SSma *pSma) {
+  if (pSma) {
+    taosThreadMutexDestroy(&pSma->mutex);
+    if SMA_RSMA_TSDB0 (pSma) tsdbClose(&SMA_RSMA_TSDB0(pSma));
+    if SMA_RSMA_TSDB1 (pSma) tsdbClose(&SMA_RSMA_TSDB1(pSma));
+    if SMA_RSMA_TSDB2 (pSma) tsdbClose(&SMA_RSMA_TSDB2(pSma));
+  }
+  return 0;
+}

source/dnode/vnode/src/sma/smaRollup.c

@@ -0,0 +1,484 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "sma.h"
+
+static FORCE_INLINE int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
+static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids);
+static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, qTaskInfo_t *taskInfo,
+                                              STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, int8_t level);
+
+struct SRSmaInfo {
+  void *taskInfo[TSDB_RETENTION_L2];  // qTaskInfo_t
+};
+
+static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle) {
+  // Note: free/kill may in RC
+  qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle);
+  if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) {
+    qDestroyTask(otaskHandle);
+  }
+}
+
+void *tdFreeRSmaInfo(SRSmaInfo *pInfo) {
+  for (int32_t i = 0; i < TSDB_RETENTION_MAX; ++i) {
+    if (pInfo->taskInfo[i]) {
+      tdFreeTaskHandle(pInfo->taskInfo[i]);
+    }
+  }
+  return NULL;
+}
+
+static FORCE_INLINE int32_t tdUidStoreInit(STbUidStore **pStore) {
+  ASSERT(*pStore == NULL);
+  *pStore = taosMemoryCalloc(1, sizeof(STbUidStore));
+  if (*pStore == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return TSDB_CODE_FAILED;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) {
+  SSmaEnv   *pEnv = SMA_RSMA_ENV(pSma);
+  SSmaStat  *pStat = SMA_ENV_STAT(pEnv);
+  SRSmaInfo *pRSmaInfo = NULL;
+
+  if (!suid || !tbUids) {
+    terrno = TSDB_CODE_INVALID_PTR;
+    smaError("vgId:%d failed to get rsma info for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno));
+    return TSDB_CODE_FAILED;
+  }
+
+  pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), suid, sizeof(tb_uid_t));
+  if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
+    smaError("vgId:%d failed to get rsma info for uid:%" PRIi64, SMA_VID(pSma), *suid);
+    terrno = TSDB_CODE_TDB_INVALID_SMA_STAT;
+    return TSDB_CODE_FAILED;
+  }
+
+  if (pRSmaInfo->taskInfo[0] && (qUpdateQualifiedTableId(pRSmaInfo->taskInfo[0], tbUids, true) != 0)) {
+    smaError("vgId:%d update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno));
+    return TSDB_CODE_FAILED;
+  } else {
+    smaDebug("vgId:%d update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma),
+             pRSmaInfo->taskInfo[0], *suid, *(int64_t *)taosArrayGet(tbUids, 0));
+  }
+
+  if (pRSmaInfo->taskInfo[1] && (qUpdateQualifiedTableId(pRSmaInfo->taskInfo[1], tbUids, true) != 0)) {
+    smaError("vgId:%d update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno));
+    return TSDB_CODE_FAILED;
+  } else {
+    smaDebug("vgId:%d update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma),
+             pRSmaInfo->taskInfo[1], *suid, *(int64_t *)taosArrayGet(tbUids, 0));
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t tdUpdateTbUidList(SSma *pSma, STbUidStore *pStore) {
+  if (!pStore || (taosArrayGetSize(pStore->tbUids) == 0)) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (tdUpdateTbUidListImpl(pSma, &pStore->suid, pStore->tbUids) != TSDB_CODE_SUCCESS) {
+    return TSDB_CODE_FAILED;
+  }
+
+  void *pIter = taosHashIterate(pStore->uidHash, NULL);
+  while (pIter) {
+    tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL);
+    SArray   *pTbUids = *(SArray **)pIter;
+
+    if (tdUpdateTbUidListImpl(pSma, pTbSuid, pTbUids) != TSDB_CODE_SUCCESS) {
+      taosHashCancelIterate(pStore->uidHash, pIter);
+      return TSDB_CODE_FAILED;
+    }
+
+    pIter = taosHashIterate(pStore->uidHash, pIter);
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief fetch suid/uids when create child tables of rollup SMA
+ *
+ * @param pTsdb
+ * @param ppStore
+ * @param suid
+ * @param uid
+ * @return int32_t
+ */
+int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_uid_t uid) {
+  SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
+
+  // only applicable to rollup SMA ctables
+  if (!pEnv) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SSmaStat *pStat = SMA_ENV_STAT(pEnv);
+  SHashObj *infoHash = NULL;
+  if (!pStat || !(infoHash = SMA_STAT_INFO_HASH(pStat))) {
+    terrno = TSDB_CODE_TDB_INVALID_SMA_STAT;
+    return TSDB_CODE_FAILED;
+  }
+
+  // info cached when create rsma stable and return directly for non-rsma ctables
+  if (!taosHashGet(infoHash, &suid, sizeof(tb_uid_t))) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  ASSERT(ppStore != NULL);
+
+  if (!(*ppStore)) {
+    if (tdUidStoreInit(ppStore) != 0) {
+      return TSDB_CODE_FAILED;
+    }
+  }
+
+  if (tdUidStorePut(*ppStore, suid, &uid) != 0) {
+    *ppStore = tdUidStoreFree(*ppStore);
+    return TSDB_CODE_FAILED;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam.
+ *
+ * @param pTsdb
+ * @param pMeta
+ * @param pReq
+ * @return int32_t
+ */
+int32_t tdProcessRSmaCreate(SSma *pSma, SMeta *pMeta, SVCreateStbReq *pReq, SMsgCb *pMsgCb) {
+  if (!pReq->rollup) {
+    smaTrace("vgId:%d return directly since no rollup for stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SRSmaParam *param = &pReq->pRSmaParam;
+
+  if ((param->qmsg1Len == 0) && (param->qmsg2Len == 0)) {
+    smaWarn("vgId:%d no qmsg1/qmsg2 for rollup stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_ROLLUP) != TSDB_CODE_SUCCESS) {
+    terrno = TSDB_CODE_TDB_INIT_FAILED;
+    return TSDB_CODE_FAILED;
+  }
+
+  SSmaEnv   *pEnv = SMA_RSMA_ENV(pSma);
+  SSmaStat  *pStat = SMA_ENV_STAT(pEnv);
+  SRSmaInfo *pRSmaInfo = NULL;
+
+  pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t));
+  if (pRSmaInfo) {
+    smaWarn("vgId:%d rsma info already exists for stb: %s, %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  pRSmaInfo = (SRSmaInfo *)taosMemoryCalloc(1, sizeof(SRSmaInfo));
+  if (!pRSmaInfo) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return TSDB_CODE_FAILED;
+  }
+
+  STqReadHandle *pReadHandle = tqInitSubmitMsgScanner(pMeta);
+  if (!pReadHandle) {
+    taosMemoryFree(pRSmaInfo);
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return TSDB_CODE_FAILED;
+  }
+
+  SReadHandle handle = {
+      .reader = pReadHandle,
+      .meta = pMeta,
+      .pMsgCb = pMsgCb,
+  };
+
+  if (param->qmsg1) {
+    pRSmaInfo->taskInfo[0] = qCreateStreamExecTaskInfo(param->qmsg1, &handle);
+    if (!pRSmaInfo->taskInfo[0]) {
+      taosMemoryFree(pRSmaInfo);
+      taosMemoryFree(pReadHandle);
+      return TSDB_CODE_FAILED;
+    }
+  }
+
+  if (param->qmsg2) {
+    pRSmaInfo->taskInfo[1] = qCreateStreamExecTaskInfo(param->qmsg2, &handle);
+    if (!pRSmaInfo->taskInfo[1]) {
+      taosMemoryFree(pRSmaInfo);
+      taosMemoryFree(pReadHandle);
+      return TSDB_CODE_FAILED;
+    }
+  }
+
+  if (taosHashPut(SMA_STAT_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) !=
+      TSDB_CODE_SUCCESS) {
+    return TSDB_CODE_FAILED;
+  } else {
+    smaDebug("vgId:%d register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), pReq->suid);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief store suid/[uids], prefer to use array and then hash
+ *
+ * @param pStore
+ * @param suid
+ * @param uid
+ * @return int32_t
+ */
+static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid) {
+  // prefer to store suid/uids in array
+  if ((suid == pStore->suid) || (pStore->suid == 0)) {
+    if (pStore->suid == 0) {
+      pStore->suid = suid;
+    }
+    if (uid) {
+      if (!pStore->tbUids) {
+        if (!(pStore->tbUids = taosArrayInit(1, sizeof(tb_uid_t)))) {
+          terrno = TSDB_CODE_OUT_OF_MEMORY;
+          return TSDB_CODE_FAILED;
+        }
+      }
+      if (!taosArrayPush(pStore->tbUids, uid)) {
+        return TSDB_CODE_FAILED;
+      }
+    }
+  } else {
+    // store other suid/uids in hash when multiple stable/table included in 1 batch of request
+    if (!pStore->uidHash) {
+      pStore->uidHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
+      if (!pStore->uidHash) {
+        return TSDB_CODE_FAILED;
+      }
+    }
+    if (uid) {
+      SArray *uidArray = taosHashGet(pStore->uidHash, &suid, sizeof(tb_uid_t));
+      if (uidArray && ((uidArray = *(SArray **)uidArray))) {
+        taosArrayPush(uidArray, uid);
+      } else {
+        SArray *pUidArray = taosArrayInit(1, sizeof(tb_uid_t));
+        if (!pUidArray) {
+          terrno = TSDB_CODE_OUT_OF_MEMORY;
+          return TSDB_CODE_FAILED;
+        }
+        if (!taosArrayPush(pUidArray, uid)) {
+          terrno = TSDB_CODE_OUT_OF_MEMORY;
+          return TSDB_CODE_FAILED;
+        }
+        if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), &pUidArray, sizeof(pUidArray)) != 0) {
+          return TSDB_CODE_FAILED;
+        }
+      }
+    } else {
+      if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), NULL, 0) != 0) {
+        return TSDB_CODE_FAILED;
+      }
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+void tdUidStoreDestory(STbUidStore *pStore) {
+  if (pStore) {
+    if (pStore->uidHash) {
+      if (pStore->tbUids) {
+        // When pStore->tbUids not NULL, the pStore->uidHash has k/v; otherwise pStore->uidHash only has keys.
+        void *pIter = taosHashIterate(pStore->uidHash, NULL);
+        while (pIter) {
+          SArray *arr = *(SArray **)pIter;
+          taosArrayDestroy(arr);
+          pIter = taosHashIterate(pStore->uidHash, pIter);
+        }
+      }
+      taosHashCleanup(pStore->uidHash);
+    }
+    taosArrayDestroy(pStore->tbUids);
+  }
+}
+
+void *tdUidStoreFree(STbUidStore *pStore) {
+  if (pStore) {
+    tdUidStoreDestory(pStore);
+    taosMemoryFree(pStore);
+  }
+  return NULL;
+}
+
+static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) {
+  if (!pReq) {
+    terrno = TSDB_CODE_INVALID_PTR;
+    return TSDB_CODE_FAILED;
+  }
+
+  SSubmitReq *pSubmitReq = (SSubmitReq *)pReq;
+
+  if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) {
+    return TSDB_CODE_FAILED;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) {
+  ASSERT(pMsg != NULL);
+  SSubmitMsgIter msgIter = {0};
+  SSubmitBlk    *pBlock = NULL;
+  SSubmitBlkIter blkIter = {0};
+  STSRow        *row = NULL;
+
+  terrno = TSDB_CODE_SUCCESS;
+
+  if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1;
+  while (true) {
+    if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1;
+
+    if (!pBlock) break;
+    tdUidStorePut(pStore, msgIter.suid, NULL);
+    pStore->uid = msgIter.uid;  // TODO: remove, just for debugging
+  }
+
+  if (terrno != TSDB_CODE_SUCCESS) return -1;
+  return 0;
+}
+
+static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, qTaskInfo_t *taskInfo,
+                                              STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, int8_t level) {
+  SArray *pResult = NULL;
+
+  if (!taskInfo) {
+    smaDebug("vgId:%d no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  smaDebug("vgId:%d execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level, taskInfo, suid);
+
+  qSetStreamInput(taskInfo, pMsg, inputType);
+  while (1) {
+    SSDataBlock *output = NULL;
+    uint64_t     ts;
+    if (qExecTask(taskInfo, &output, &ts) < 0) {
+      ASSERT(false);
+    }
+    if (!output) {
+      break;
+    }
+    if (!pResult) {
+      pResult = taosArrayInit(0, sizeof(SSDataBlock));
+      if (!pResult) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        return TSDB_CODE_FAILED;
+      }
+    }
+
+    taosArrayPush(pResult, output);
+  }
+
+  if (taosArrayGetSize(pResult) > 0) {
+    blockDebugShowData(pResult);
+    STsdb      *sinkTsdb = (level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb1 : pSma->pRSmaTsdb2);
+    SSubmitReq *pReq = NULL;
+    if (buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, SMA_VID(pSma), uid, suid) != 0) {
+      taosArrayDestroy(pResult);
+      return TSDB_CODE_FAILED;
+    }
+    if (tdProcessSubmitReq(sinkTsdb, INT64_MAX, pReq) != 0) {
+      taosArrayDestroy(pResult);
+      taosMemoryFreeClear(pReq);
+      return TSDB_CODE_FAILED;
+    }
+    taosMemoryFreeClear(pReq);
+  } else {
+    smaWarn("vgId:%d no rsma % " PRIi8 " data generated since %s", SMA_VID(pSma), level, tstrerror(terrno));
+  }
+
+  taosArrayDestroy(pResult);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid, tb_uid_t uid) {
+  SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
+  if (!pEnv) {
+    // only applicable when rsma env exists
+    return TSDB_CODE_SUCCESS;
+  }
+
+  ASSERT(uid != 0);  // TODO: remove later
+
+  SSmaStat  *pStat = SMA_ENV_STAT(pEnv);
+  SRSmaInfo *pRSmaInfo = NULL;
+
+  pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
+
+  if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
+    smaDebug("vgId:%d no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid);
+    return TSDB_CODE_SUCCESS;
+  }
+  if (!pRSmaInfo->taskInfo[0]) {
+    smaDebug("vgId:%d no rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), suid);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
+    // TODO: use the proper schema instead of 0, and cache STSchema in cache
+    STSchema *pTSchema = metaGetTbTSchema(SMA_META(pSma), suid, 0);
+    if (!pTSchema) {
+      terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
+      return TSDB_CODE_FAILED;
+    }
+    tdExecuteRSmaImpl(pSma, pMsg, inputType, pRSmaInfo->taskInfo[0], pTSchema, suid, uid, TSDB_RETENTION_L1);
+    tdExecuteRSmaImpl(pSma, pMsg, inputType, pRSmaInfo->taskInfo[1], pTSchema, suid, uid, TSDB_RETENTION_L2);
+    taosMemoryFree(pTSchema);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) {
+  SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
+  if (!pEnv) {
+    // only applicable when rsma env exists
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
+    STbUidStore uidStore = {0};
+    tdFetchSubmitReqSuids(pMsg, &uidStore);
+
+    if (uidStore.suid != 0) {
+      tdExecuteRSma(pSma, pMsg, inputType, uidStore.suid, uidStore.uid);
+
+      void *pIter = taosHashIterate(uidStore.uidHash, NULL);
+      while (pIter) {
+        tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL);
+        tdExecuteRSma(pSma, pMsg, inputType, *pTbSuid, 0);
+        pIter = taosHashIterate(uidStore.uidHash, pIter);
+      }
+
+      tdUidStoreDestory(&uidStore);
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}

source/dnode/vnode/src/sma/smaTDBImpl.c

@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define ALLOW_FORBID_FUNC
+
+#include "sma.h"
+
+int32_t smaOpenDBEnv(TENV **ppEnv, const char *path) {
+  int ret = 0;
+
+  if (path == NULL) return -1;
+
+  ret = tdbEnvOpen(path, 4096, 256, ppEnv);  // use as param
+
+  if (ret != 0) {
+    smaError("failed to create tsdb db env, ret = %d", ret);
+    return -1;
+  }
+
+  return 0;
+}
+
+int32_t smaCloseDBEnv(TENV *pEnv) { return tdbEnvClose(pEnv); }
+
+static inline int tdSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) {
+  const SSmaKey *pKey1 = (const SSmaKey *)arg1;
+  const SSmaKey *pKey2 = (const SSmaKey *)arg2;
+
+  ASSERT(len1 == len2 && len1 == sizeof(SSmaKey));
+
+  if (pKey1->skey < pKey2->skey) {
+    return -1;
+  } else if (pKey1->skey > pKey2->skey) {
+    return 1;
+  }
+  if (pKey1->groupId < pKey2->groupId) {
+    return -1;
+  } else if (pKey1->groupId > pKey2->groupId) {
+    return 1;
+  }
+
+  return 0;
+}
+
+static int32_t smaOpenDBDb(TDB **ppDB, TENV *pEnv, const char *pFName) {
+  int           ret;
+  tdb_cmpr_fn_t compFunc;
+
+  // Create a database
+  compFunc = tdSmaKeyCmpr;
+  ret = tdbDbOpen(pFName, -1, -1, compFunc, pEnv, ppDB);
+
+  return 0;
+}
+
+static int32_t smaCloseDBDb(TDB *pDB) { return tdbDbClose(pDB); }
+
+int32_t smaOpenDBF(TENV *pEnv, SDBFile *pDBF) {
+  // TEnv is shared by a group of SDBFile
+  if (!pEnv || !pDBF) {
+    terrno = TSDB_CODE_INVALID_PTR;
+    return -1;
+  }
+
+  // Open DBF
+  if (smaOpenDBDb(&(pDBF->pDB), pEnv, pDBF->path) < 0) {
+    terrno = TSDB_CODE_TDB_INIT_FAILED;
+    smaCloseDBDb(pDBF->pDB);
+    return -1;
+  }
+
+  return 0;
+}
+
+int32_t smaCloseDBF(SDBFile *pDBF) {
+  int32_t ret = 0;
+  if (pDBF->pDB) {
+    ret = smaCloseDBDb(pDBF->pDB);
+    pDBF->pDB = NULL;
+  }
+  taosMemoryFreeClear(pDBF->path);
+  return ret;
+}
+
+int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) {
+  int32_t ret;
+
+  ret = tdbDbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn);
+  if (ret < 0) {
+    smaError("failed to create insert sma data into db, ret = %d", ret);
+    return -1;
+  }
+
+  return 0;
+}
+
+void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen) {
+  void *pVal = NULL;
+  int   ret;
+
+  ret = tdbDbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen);
+
+  if (ret < 0) {
+    smaError("failed to get sma data from db, ret = %d", ret);
+    return NULL;
+  }
+
+  ASSERT(*valLen >= 0);
+
+  // TODO: lock?
+  // TODO: Would the key/value be destoryed during return the data?
+  // TODO: How about the key is updated while value length is changed? The original value buffer would be freed
+  // automatically?
+
+  return pVal;
+}

source/dnode/vnode/src/tq/tq.c

@@ -14,6 +14,7 @@
  */
 
 #include "tq.h"
+#include "tqueue.h"
 
 int32_t tqInit() {
   //
@@ -234,7 +235,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
   if (msgType != TDMT_VND_SUBMIT) return 0;
 
   // make sure msgType == TDMT_VND_SUBMIT
-  if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg, ver) != 0) {
+  if (tdUpdateExpireWindow(pTq->pVnode->pSma, msg, ver) != 0) {
     return -1;
   }
 
@@ -1032,6 +1033,59 @@ int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId)
   return 0;
 }
 
+int32_t tqProcessStreamTrigger2(STQ* pTq, SSubmitReq* pReq, int64_t ver) {
+  void* pIter = NULL;
+  bool  failed = false;
+
+  SStreamDataSubmit* pSubmit = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM);
+  if (pSubmit == NULL) {
+    failed = true;
+  }
+  pSubmit->dataRef = taosMemoryMalloc(sizeof(int32_t));
+  if (pSubmit->dataRef == NULL) {
+    failed = true;
+  }
+
+  pSubmit->type = STREAM_DATA_TYPE_SUBMIT_BLOCK;
+  pSubmit->sourceVer = ver;
+  pSubmit->sourceVg = pTq->pVnode->config.vgId;
+  pSubmit->data = pReq;
+  *pSubmit->dataRef = 1;
+
+  while (1) {
+    pIter = taosHashIterate(pTq->pStreamTasks, pIter);
+    if (pIter == NULL) break;
+    SStreamTask* pTask = (SStreamTask*)pIter;
+    if (pTask->inputType != STREAM_INPUT__DATA_SUBMIT) continue;
+
+    int8_t inputStatus = atomic_load_8(&pTask->inputStatus);
+    if (inputStatus == TASK_INPUT_STATUS__NORMAL) {
+      if (failed) {
+        atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED);
+        continue;
+      }
+
+      streamDataSubmitRefInc(pSubmit);
+      taosWriteQitem(pTask->inputQ, pSubmit);
+
+      int8_t execStatus = atomic_load_8(&pTask->status);
+      if (execStatus == TASK_STATUS__IDLE || execStatus == TASK_STATUS__CLOSING) {
+        // TODO dispatch task launch msg to fetch queue
+      }
+
+    } else {
+      // blocked or stopped, do nothing
+    }
+  }
+
+  if (!failed) {
+    streamDataSubmitRefDec(pSubmit);
+    return 0;
+  } else {
+    return -1;
+  }
+}
+
 int32_t tqProcessTaskExec2(STQ* pTq, char* msg, int32_t msgLen) {
   SStreamTaskExecReq req = {0};
   tDecodeSStreamTaskExecReq(msg, &req);
@@ -1051,7 +1105,7 @@ int32_t tqProcessTaskExec2(STQ* pTq, char* msg, int32_t msgLen) {
   // try exec
   int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING);
   if (execStatus == TASK_STATUS__IDLE) {
-    if (streamTaskExecNew(pTask) < 0) {
+    if (streamTaskRun(pTask) < 0) {
       atomic_store_8(&pTask->status, TASK_STATUS__CLOSING);
 
       goto FAIL;

source/dnode/vnode/src/tq/tqRead.c

@@ -91,7 +91,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
   // TODO set to real sversion
   *pUid = 0;
 
-  int32_t sversion = 0;
+  int32_t sversion = 1;
   if (pHandle->sver != sversion || pHandle->cachedSchemaUid != pHandle->msgIter.suid) {
     pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion);
 

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -465,7 +465,7 @@ static int tsdbCreateCommitIters(SCommitH *pCommith) {
     pTbData = (STbData *)pNode->pData;
 
     pCommitIter = pCommith->iters + i;
-    pTSchema = metaGetTbTSchema(REPO_META(pRepo), pTbData->uid, 0);  // TODO: schema version
+    pTSchema = metaGetTbTSchema(REPO_META(pRepo), pTbData->uid, 1);  // TODO: schema version
 
     if (pTSchema) {
       pCommitIter->pIter = tSkipListCreateIter(pTbData->pData);
@@ -912,7 +912,7 @@ static int tsdbMoveBlkIdx(SCommitH *pCommith, SBlockIdx *pIdx) {
   while (bidx < nBlocks) {
     if (!pTSchema && !tsdbCommitIsSameFile(pCommith, bidx)) {
       // Set commit table
-      pTSchema = metaGetTbTSchema(REPO_META(pTsdb), pIdx->uid, 0);  // TODO: schema version
+      pTSchema = metaGetTbTSchema(REPO_META(pTsdb), pIdx->uid, 1);  // TODO: schema version
       if (!pTSchema) {
         terrno = TSDB_CODE_OUT_OF_MEMORY;
         return -1;

source/dnode/vnode/src/tsdb/tsdbCommit2.c

@@ -16,6 +16,8 @@
 #include "tsdb.h"
 
 int tsdbBegin(STsdb *pTsdb) {
+  if (!pTsdb) return 0;
+  
   STsdbMemTable *pMem;
 
   if (tsdbMemTableCreate(pTsdb, &pTsdb->mem) < 0) {

source/dnode/vnode/src/tsdb/tsdbFS.c

@@ -37,12 +37,12 @@ static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired);
 // static int  tsdbProcessExpiredFS(STsdb *pRepo);
 // static int  tsdbCreateMeta(STsdb *pRepo);
 
-static void tsdbGetRootDir(int repoid, int8_t level, char dirName[]) {
+static void tsdbGetRootDir(int repoid, const char* dir, char dirName[]) {
-  snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s", repoid, TSDB_LEVEL_DNAME[level]);
+  snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s", repoid, dir);
 }
 
-static void tsdbGetDataDir(int repoid, int8_t level, char dirName[]) {
+static void tsdbGetDataDir(int repoid,  const char* dir, char dirName[]) {
-  snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data", repoid, TSDB_LEVEL_DNAME[level]);
+  snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data", repoid, dir);
 }
 
 // For backward compatibility
@@ -591,7 +591,7 @@ static int tsdbComparFidFSet(const void *arg1, const void *arg2) {
 
 static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) {
   snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/%s/%s", tfsGetPrimaryPath(REPO_TFS(pRepo)), REPO_ID(pRepo),
-           TSDB_LEVEL_DNAME[REPO_LEVEL(pRepo)], tsdbTxnFname[ftype]);
+           pRepo->dir, tsdbTxnFname[ftype]);
 }
 
 static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
@@ -721,7 +721,7 @@ static int tsdbScanRootDir(STsdb *pRepo) {
   STsdbFS        *pfs = REPO_FS(pRepo);
   const STfsFile *pf;
 
-  tsdbGetRootDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), rootDir);
+  tsdbGetRootDir(REPO_ID(pRepo), pRepo->dir, rootDir);
   STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), rootDir);
   if (tdir == NULL) {
     tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno));
@@ -755,7 +755,7 @@ static int tsdbScanDataDir(STsdb *pRepo) {
   STsdbFS        *pfs = REPO_FS(pRepo);
   const STfsFile *pf;
 
-  tsdbGetDataDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), dataDir);
+  tsdbGetDataDir(REPO_ID(pRepo), pRepo->dir, dataDir);
   STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), dataDir);
   if (tdir == NULL) {
     tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno));
@@ -803,7 +803,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) {
   regex_t         regex;
   STsdbFS        *pfs = REPO_FS(pRepo);
 
-  tsdbGetDataDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), dataDir);
+  tsdbGetDataDir(REPO_ID(pRepo), pRepo->dir, dataDir);
 
   // Resource allocation and init
   regcomp(&regex, pattern, REG_EXTENDED);

source/dnode/vnode/src/tsdb/tsdbFile.c

@@ -23,14 +23,6 @@ static const char *TSDB_FNAME_SUFFIX[] = {
     "smal",  // TSDB_FILE_SMAL
     "",      // TSDB_FILE_MAX
     "meta",  // TSDB_FILE_META
-    "tsma",  // TSDB_FILE_TSMA
-    "rsma",  // TSDB_FILE_RSMA
-};
-
-const char *TSDB_LEVEL_DNAME[] = {
-    "tsdb",
-    "rsma1",
-    "rsma2",
 };
 
 static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char* dname, char *fname);
@@ -51,7 +43,7 @@ void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t
   pDFile->info.magic = TSDB_FILE_INIT_MAGIC;
   pDFile->info.fver = tsdbGetDFSVersion(ftype);
 
-  tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, TSDB_LEVEL_DNAME[pRepo->level], fname);
+  tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, pRepo->dir, fname);
   tfsInitFile(REPO_TFS(pRepo), &(pDFile->f), did, fname);
 }
 

source/dnode/vnode/src/tsdb/tsdbOpen.c

@@ -15,100 +15,17 @@
 
 #include "tsdb.h"
 
-#define TSDB_OPEN_RSMA_IMPL(v, l)                                                                \
+static int tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg);
-  do {                                                                                           \
-    SRetention *r = VND_RETENTIONS(v)[0];                                                        \
-    if (RETENTION_VALID(r)) {                                                                    \
-      return tsdbOpenImpl((v), type, &VND_RSMA##l(v), VNODE_RSMA##l##_DIR, TSDB_RETENTION_L##l); \
-    }                                                                                            \
-  } while (0)
 
-#define TSDB_SET_KEEP_CFG(l)                                                                      \
-  do {                                                                                            \
-    SRetention *r = &pCfg->retentions[l];                                                         \
-    pKeepCfg->keep2 = convertTimeFromPrecisionToUnit(r->keep, pCfg->precision, TIME_UNIT_MINUTE); \
-    pKeepCfg->keep0 = pKeepCfg->keep2;                                                            \
-    pKeepCfg->keep1 = pKeepCfg->keep2;                                                            \
-    pKeepCfg->days = tsdbEvalDays(r, pCfg->precision);                                            \
-  } while (0)
 
-#define RETENTION_DAYS_SPLIT_RATIO 10
+// implementation
-#define RETENTION_DAYS_SPLIT_MIN   1
-#define RETENTION_DAYS_SPLIT_MAX   30
 
-static int32_t tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int8_t type);
+static int tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg) {
-static int32_t tsdbEvalDays(SRetention *r, int8_t precision);
-static int32_t tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir, int8_t level);
-
-int tsdbOpen(SVnode *pVnode, int8_t type) {
-  switch (type) {
-    case TSDB_TYPE_TSDB:
-      return tsdbOpenImpl(pVnode, type, &VND_TSDB(pVnode), VNODE_TSDB_DIR, TSDB_RETENTION_L0);
-    case TSDB_TYPE_TSMA:
-      ASSERT(0);
-      break;
-    case TSDB_TYPE_RSMA_L0:
-      TSDB_OPEN_RSMA_IMPL(pVnode, 0);
-      break;
-    case TSDB_TYPE_RSMA_L1:
-      TSDB_OPEN_RSMA_IMPL(pVnode, 1);
-      break;
-    case TSDB_TYPE_RSMA_L2:
-      TSDB_OPEN_RSMA_IMPL(pVnode, 2);
-      break;
-    default:
-      ASSERT(0);
-      break;
-  }
-  return 0;
-}
-
-static int32_t tsdbEvalDays(SRetention *r, int8_t precision) {
-  int32_t keepDays = convertTimeFromPrecisionToUnit(r->keep, precision, TIME_UNIT_DAY);
-  int32_t freqDays = convertTimeFromPrecisionToUnit(r->freq, precision, TIME_UNIT_DAY);
-
-  int32_t days = keepDays / RETENTION_DAYS_SPLIT_RATIO;
-  if (days <= RETENTION_DAYS_SPLIT_MIN) {
-    days = RETENTION_DAYS_SPLIT_MIN;
-    if (days < freqDays) {
-      days = freqDays + 1;
-    }
-  } else {
-    if (days > RETENTION_DAYS_SPLIT_MAX) {
-      days = RETENTION_DAYS_SPLIT_MAX;
-    }
-    if (days < freqDays) {
-      days = freqDays + 1;
-    }
-  }
-  return days * 1440;
-}
-
-static int32_t tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int8_t type) {
   pKeepCfg->precision = pCfg->precision;
-  switch (type) {
+  pKeepCfg->days = pCfg->days;
-    case TSDB_TYPE_TSDB:
+  pKeepCfg->keep0 = pCfg->keep0;
-      pKeepCfg->days = pCfg->days;
+  pKeepCfg->keep1 = pCfg->keep1;
-      pKeepCfg->keep0 = pCfg->keep0;
+  pKeepCfg->keep2 = pCfg->keep2;
-      pKeepCfg->keep1 = pCfg->keep1;
-      pKeepCfg->keep2 = pCfg->keep2;
-      break;
-    case TSDB_TYPE_TSMA:
-      ASSERT(0);
-      break;
-    case TSDB_TYPE_RSMA_L0:
-      TSDB_SET_KEEP_CFG(0);
-      break;
-    case TSDB_TYPE_RSMA_L1:
-      TSDB_SET_KEEP_CFG(1);
-      break;
-    case TSDB_TYPE_RSMA_L2:
-      TSDB_SET_KEEP_CFG(2);
-      break;
-    default:
-      ASSERT(0);
-      break;
-  }
   return 0;
 }
 
@@ -116,18 +33,16 @@ static int32_t tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int8_t typ
  * @brief
  *
  * @param pVnode
- * @param type
  * @param ppTsdb
  * @param dir
- * @param level retention level
  * @return int
  */
-int32_t tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir, int8_t level) {
+int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKeepCfg) {
   STsdb *pTsdb = NULL;
   int    slen = 0;
 
   *ppTsdb = NULL;
-  slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(dir) + 3;
+  slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(dir) + TSDB_DATA_DIR_LEN + 3;
 
   // create handle
   pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen);
@@ -136,13 +51,18 @@ int32_t tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *di
     return -1;
   }
 
+  ASSERT(strlen(dir) < TSDB_DATA_DIR_LEN);
+  memcpy(pTsdb->dir, dir, strlen(dir));
   pTsdb->path = (char *)&pTsdb[1];
   sprintf(pTsdb->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP, dir);
   pTsdb->pVnode = pVnode;
-  pTsdb->level = level;
   pTsdb->repoLocked = false;
   taosThreadMutexInit(&pTsdb->mutex, NULL);
-  tsdbSetKeepCfg(REPO_KEEP_CFG(pTsdb), REPO_CFG(pTsdb), type);
+  if (!pKeepCfg) {
+    tsdbSetKeepCfg(&pTsdb->keepCfg, &pVnode->config.tsdbCfg);
+  } else {
+    memcpy(&pTsdb->keepCfg, pKeepCfg, sizeof(STsdbKeepCfg));
+  }
   pTsdb->fs = tsdbNewFS(REPO_KEEP_CFG(pTsdb));
 
   // create dir (TODO: use tfsMkdir)
@@ -163,12 +83,13 @@ _err:
   return -1;
 }
 
-int tsdbClose(STsdb *pTsdb) {
+int tsdbClose(STsdb **pTsdb) {
-  if (pTsdb) {
+  if (*pTsdb) {
     // TODO: destroy mem/imem
-    tsdbCloseFS(pTsdb);
+    taosThreadMutexDestroy(&(*pTsdb)->mutex);
-    tsdbFreeFS(pTsdb->fs);
+    tsdbCloseFS(*pTsdb);
-    taosMemoryFree(pTsdb);
+    tsdbFreeFS((*pTsdb)->fs);
+    taosMemoryFreeClear(*pTsdb);
   }
   return 0;
 }

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -490,7 +490,7 @@ tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableG
 
   STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, 0);
 
-  pTsdbReadHandle->pSchema = metaGetTbTSchema(pVnode->pMeta, pCheckInfo->tableId, 0);
+  pTsdbReadHandle->pSchema = metaGetTbTSchema(pVnode->pMeta, pCheckInfo->tableId, 1);
   int32_t  numOfCols = taosArrayGetSize(pTsdbReadHandle->suppInfo.defaultLoadColumn);
   int16_t* ids = pTsdbReadHandle->suppInfo.defaultLoadColumn->pData;
 
@@ -1271,7 +1271,6 @@ _error:
 static int32_t getEndPosInDataBlock(STsdbReadHandle* pTsdbReadHandle, SDataBlockInfo* pBlockInfo);
 static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows,
                                        int32_t start, int32_t end);
-static void    moveDataToFront(STsdbReadHandle* pTsdbReadHandle, int32_t numOfRows, int32_t numOfCols);
 static void    doCheckGeneratedBlockRange(STsdbReadHandle* pTsdbReadHandle);
 static void    copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo,
                                               SDataBlockInfo* pBlockInfo, int32_t endPos);
@@ -1301,7 +1300,7 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock*
     if ((ascScan && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) ||
         (!ascScan && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) {
       // do not load file block into buffer
-      int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1;
+      int32_t step = ascScan ? 1 : -1;
 
       TSKEY maxKey =
           ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? (binfo.window.skey - step) : (binfo.window.ekey - step);
@@ -1618,7 +1617,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa
   if (pSchema1 == NULL) {
     // pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row1));
     // TODO: use the real schemaVersion
-    pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 0);
+    pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 1);
   }
 
 #ifdef TD_DEBUG_PRINT_ROW
@@ -1637,7 +1636,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa
     if (pSchema2 == NULL) {
       // pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row2));
       // TODO: use the real schemaVersion
-      pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 0);
+      pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 1);
     }
     if (isRow2DataRow) {
       numOfColsOfRow2 = schemaNCols(pSchema2);
@@ -1790,22 +1789,6 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa
 #endif
 }
 
-static void moveDataToFront(STsdbReadHandle* pTsdbReadHandle, int32_t numOfRows, int32_t numOfCols) {
-  if (numOfRows == 0 || ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
-    return;
-  }
-
-  // if the buffer is not full in case of descending order query, move the data in the front of the buffer
-  if (numOfRows < pTsdbReadHandle->outputCapacity) {
-    int32_t emptySize = pTsdbReadHandle->outputCapacity - numOfRows;
-    for (int32_t i = 0; i < numOfCols; ++i) {
-      SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
-      memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes,
-              numOfRows * pColInfo->info.bytes);
-    }
-  }
-}
-
 static void getQualifiedRowsPos(STsdbReadHandle* pTsdbReadHandle, int32_t startPos, int32_t endPos,
                                 int32_t numOfExisted, int32_t* start, int32_t* end) {
   *start = -1;
@@ -1891,9 +1874,6 @@ static void copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, STa
   cur->lastKey = tsArray[endPos] + step;
   cur->blockCompleted = true;
 
-  // if the buffer is not full in case of descending order query, move the data in the front of the buffer
-  moveDataToFront(pTsdbReadHandle, numOfRows, numOfCols);
-
   // The value of pos may be -1 or pBlockInfo->rows, and it is invalid in both cases.
   pos = endPos + step;
   updateInfoAfterMerge(pTsdbReadHandle, pCheckInfo, numOfRows, pos);
@@ -1944,18 +1924,18 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
   assert(pCols->numOfRows == pBlock->numOfRows && tsArray[0] == pBlock->keyFirst &&
          tsArray[pBlock->numOfRows - 1] == pBlock->keyLast);
 
+  bool ascScan = ASCENDING_TRAVERSE(pTsdbReadHandle->order);
+  int32_t step = ascScan ? 1 : -1;
+
   // for search the endPos, so the order needs to reverse
-  int32_t order = (pTsdbReadHandle->order == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC;
+  int32_t order = ascScan ? TSDB_ORDER_DESC : TSDB_ORDER_ASC;
 
-  int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1;
   int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pTsdbReadHandle));
-
-  STable* pTable = NULL;
   int32_t endPos = getEndPosInDataBlock(pTsdbReadHandle, &blockInfo);
 
+  STimeWindow* pWin = &blockInfo.window;
   tsdbDebug("%p uid:%" PRIu64 " start merge data block, file block range:%" PRIu64 "-%" PRIu64
-            " rows:%d, start:%d, end:%d, %s",
+            " rows:%d, start:%d, end:%d, %s", pTsdbReadHandle, pCheckInfo->tableId, pWin->skey, pWin->ekey, blockInfo.rows,
-            pTsdbReadHandle, pCheckInfo->tableId, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows,
             cur->pos, endPos, pTsdbReadHandle->idStr);
 
   // compared with the data from in-memory buffer, to generate the correct timestamp array list
@@ -1986,20 +1966,16 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
       }
 
       TSKEY key = TD_ROW_KEY(row1);
-      if ((key > pTsdbReadHandle->window.ekey && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
+      if ((key > pTsdbReadHandle->window.ekey && ascScan) || (key < pTsdbReadHandle->window.ekey && !ascScan)) {
-          (key < pTsdbReadHandle->window.ekey && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) {
         break;
       }
 
-      if (((pos > endPos || tsArray[pos] > pTsdbReadHandle->window.ekey) &&
+      if (((pos > endPos || tsArray[pos] > pTsdbReadHandle->window.ekey) && ascScan) ||
-           ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
+          ((pos < endPos || tsArray[pos] < pTsdbReadHandle->window.ekey) && !ascScan)) {
-          ((pos < endPos || tsArray[pos] < pTsdbReadHandle->window.ekey) &&
-           !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) {
         break;
       }
 
-      if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
+      if ((key < tsArray[pos] && ascScan) || (key > tsArray[pos] && !ascScan)) {
-          (key > tsArray[pos] && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) {
         if (rv1 != TD_ROW_SVER(row1)) {
           //          pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1));
           rv1 = TD_ROW_SVER(row1);
@@ -2054,23 +2030,19 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
         }
 #endif
         if (TD_SUPPORT_UPDATE(pCfg->update)) {
-          if (lastKeyAppend != key) {
-            lastKeyAppend = key;
-            ++curRow;
-          }
           numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, curRow, pos, pos);
+          lastKeyAppend = key;
 
           if (rv1 != TD_ROW_SVER(row1)) {
-            //            pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1));
             rv1 = TD_ROW_SVER(row1);
           }
           if (row2 && rv2 != TD_ROW_SVER(row2)) {
-            //            pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2));
             rv2 = TD_ROW_SVER(row2);
           }
-          numOfRows +=
+
-              mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols,
+          // still assign data into current row
-                                 pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastKeyAppend);
+          mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols,
+                             pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastKeyAppend);
 
           if (cur->win.skey == TSKEY_INITIAL_VAL) {
             cur->win.skey = key;
@@ -2081,12 +2053,13 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
           cur->mixBlock = true;
 
           moveToNextRowInMem(pCheckInfo);
+          ++curRow;
+
           pos += step;
         } else {
           moveToNextRowInMem(pCheckInfo);
         }
-      } else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
+      } else if ((key > tsArray[pos] && ascScan) || (key < tsArray[pos] && !ascScan)) {
-                 (key < tsArray[pos] && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) {
         if (cur->win.skey == TSKEY_INITIAL_VAL) {
           cur->win.skey = tsArray[pos];
         }
@@ -2112,17 +2085,17 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
         int32_t qstart = 0, qend = 0;
         getQualifiedRowsPos(pTsdbReadHandle, pos, end, numOfRows, &qstart, &qend);
 
-        if ((lastKeyAppend != TSKEY_INITIAL_VAL) &&
+        if ((lastKeyAppend != TSKEY_INITIAL_VAL) && (lastKeyAppend != (ascScan ? tsArray[qstart] : tsArray[qend]))) {
-            (lastKeyAppend != (ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? tsArray[qstart] : tsArray[qend]))) {
           ++curRow;
         }
+
         numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, curRow, qstart, qend);
         pos += (qend - qstart + 1) * step;
         if (numOfRows > 0) {
           curRow = numOfRows - 1;
         }
 
-        cur->win.ekey = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? tsArray[qend] : tsArray[qstart];
+        cur->win.ekey = ascScan ? tsArray[qend] : tsArray[qstart];
         cur->lastKey = cur->win.ekey + step;
         lastKeyAppend = cur->win.ekey;
       }
@@ -2134,10 +2107,8 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
        * copy them all to result buffer, since it may be overlapped with file data block.
        */
       if (node == NULL ||
-          ((TD_ROW_KEY((STSRow*)SL_GET_NODE_DATA(node)) > pTsdbReadHandle->window.ekey) &&
+          ((TD_ROW_KEY((STSRow*)SL_GET_NODE_DATA(node)) > pTsdbReadHandle->window.ekey) && ascScan) ||
-           ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
+          ((TD_ROW_KEY((STSRow*)SL_GET_NODE_DATA(node)) < pTsdbReadHandle->window.ekey) && !ascScan)) {
-          ((TD_ROW_KEY((STSRow*)SL_GET_NODE_DATA(node)) < pTsdbReadHandle->window.ekey) &&
-           !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) {
         // no data in cache or data in cache is greater than the ekey of time window, load data from file block
         if (cur->win.skey == TSKEY_INITIAL_VAL) {
           cur->win.skey = tsArray[pos];
@@ -2149,22 +2120,20 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
         numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, start, end);
         pos += (end - start + 1) * step;
 
-        cur->win.ekey = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? tsArray[end] : tsArray[start];
+        cur->win.ekey = ascScan ? tsArray[end] : tsArray[start];
         cur->lastKey = cur->win.ekey + step;
         cur->mixBlock = true;
       }
     }
   }
 
-  cur->blockCompleted =
+  cur->blockCompleted = (((pos > endPos || cur->lastKey > pTsdbReadHandle->window.ekey) && ascScan) ||
-      (((pos > endPos || cur->lastKey > pTsdbReadHandle->window.ekey) && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
+       ((pos < endPos || cur->lastKey < pTsdbReadHandle->window.ekey) && !ascScan));
-       ((pos < endPos || cur->lastKey < pTsdbReadHandle->window.ekey) && !ASCENDING_TRAVERSE(pTsdbReadHandle->order)));
 
-  if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
+  if (!ascScan) {
     TSWAP(cur->win.skey, cur->win.ekey);
   }
 
-  moveDataToFront(pTsdbReadHandle, numOfRows, numOfCols);
   updateInfoAfterMerge(pTsdbReadHandle, pCheckInfo, numOfRows, pos);
   doCheckGeneratedBlockRange(pTsdbReadHandle);
 
@@ -2755,7 +2724,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
 
     win->ekey = key;
     if (rv != TD_ROW_SVER(row)) {
-      pSchema = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), pCheckInfo->tableId, 0);
+      pSchema = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), pCheckInfo->tableId, 1);
       rv = TD_ROW_SVER(row);
     }
     numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, &curRows, row, NULL, numOfCols, pCheckInfo->tableId,
@@ -3877,7 +3846,7 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch
 
   // NOTE: not add ref count for super table
   SArray*         res = taosArrayInit(8, sizeof(STableKeyInfo));
-  SSchemaWrapper* pTagSchema = metaGetTableSchema(pMeta, uid, 0, true);
+  SSchemaWrapper* pTagSchema = metaGetTableSchema(pMeta, uid, 1, true);
 
   // no tags and tbname condition, all child tables of this stable are involved
   if (tbnameCond == NULL && (pTagCond == NULL || len == 0)) {

source/dnode/vnode/src/tsdb/tsdbSma.c

@@ -2084,7 +2084,7 @@ static int32_t tsdbExecuteRSma(STsdb *pTsdb, const void *pMsg, int32_t inputType
 
   if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
     // TODO: use the proper schema instead of 0, and cache STSchema in cache
-    STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, suid, 0);
+    STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, suid, 1);
     if (!pTSchema) {
       terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
       return TSDB_CODE_FAILED;

source/dnode/vnode/src/vnd/vnodeCommit.c

@@ -47,7 +47,7 @@ int vnodeBegin(SVnode *pVnode) {
   }
 
   // begin tsdb
-  if (vnodeIsRollup(pVnode)) {
+  if (pVnode->pSma) {
     if (tsdbBegin(VND_RSMA0(pVnode)) < 0) {
       vError("vgId:%d failed to begin rsma0 since %s", TD_VID(pVnode), tstrerror(terrno));
       return -1;

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -96,26 +96,15 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
   }
 
   // open tsdb
-  if (vnodeIsRollup(pVnode)) {
+  if (!vnodeIsRollup(pVnode) && tsdbOpen(pVnode, &VND_TSDB(pVnode), VNODE_TSDB_DIR, TSDB_TYPE_TSDB) < 0) {
-    if (tsdbOpen(pVnode, TSDB_TYPE_RSMA_L0) < 0) {
+    vError("vgId:%d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
-      vError("vgId:%d failed to open vnode rsma0 since %s", TD_VID(pVnode), tstrerror(terrno));
+    goto _err;
-      goto _err;
+  }
-    }
 
-    if (tsdbOpen(pVnode, TSDB_TYPE_RSMA_L1) < 0) {
+  // open sma
-      vError("vgId:%d failed to open vnode rsma1 since %s", TD_VID(pVnode), tstrerror(terrno));
+  if (smaOpen(pVnode)) {
-      goto _err;
+    vError("vgId:%d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
-    }
+    goto _err;
-
-    if (tsdbOpen(pVnode, TSDB_TYPE_RSMA_L2) < 0) {
-      vError("vgId:%d failed to open vnode rsma2 since %s", TD_VID(pVnode), tstrerror(terrno));
-      goto _err;
-    }
-  } else {
-    if (tsdbOpen(pVnode, TSDB_TYPE_TSDB) < 0) {
-      vError("vgId:%d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
-      goto _err;
-    }
   }
 
   // open wal
@@ -161,10 +150,10 @@ _err:
   if (pVnode->pQuery) vnodeQueryClose(pVnode);
   if (pVnode->pTq) tqClose(pVnode->pTq);
   if (pVnode->pWal) walClose(pVnode->pWal);
-  if (pVnode->pTsdb) tsdbClose(pVnode->pTsdb);
+  if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb);
   if (pVnode->pMeta) metaClose(pVnode->pMeta);
-  tsdbClose(VND_RSMA1(pVnode));
+  if (pVnode->pSma) smaClose(pVnode->pSma);
-  tsdbClose(VND_RSMA2(pVnode));
+
   tsem_destroy(&(pVnode->canCommit));
   taosMemoryFree(pVnode);
   return NULL;
@@ -177,9 +166,8 @@ void vnodeClose(SVnode *pVnode) {
     vnodeQueryClose(pVnode);
     walClose(pVnode->pWal);
     tqClose(pVnode->pTq);
-    tsdbClose(VND_TSDB(pVnode));
+    if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb);
-    tsdbClose(VND_RSMA1(pVnode));
+    smaClose(pVnode->pSma);
-    tsdbClose(VND_RSMA2(pVnode));
     metaClose(pVnode->pMeta);
     vnodeCloseBufPool(pVnode);
     // destroy handle

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -16,12 +16,13 @@
 #include "vnd.h"
 
 static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp);
-static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp);
-static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp);
 
 int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) {
 #if 0
@@ -72,7 +73,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
       if (vnodeProcessCreateStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
       break;
     case TDMT_VND_ALTER_STB:
-      if (vnodeProcessAlterStbReq(pVnode, pReq, len, pRsp) < 0) goto _err;
+      if (vnodeProcessAlterStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
       break;
     case TDMT_VND_DROP_STB:
       if (vnodeProcessDropStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
@@ -81,15 +82,13 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
       if (vnodeProcessCreateTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
       break;
     case TDMT_VND_ALTER_TABLE:
-      if (vnodeProcessAlterTbReq(pVnode, pReq, len, pRsp) < 0) goto _err;
+      if (vnodeProcessAlterTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
       break;
     case TDMT_VND_DROP_TABLE:
       if (vnodeProcessDropTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
       break;
-    case TDMT_VND_CREATE_SMA: {  // timeRangeSMA
+    case TDMT_VND_CREATE_SMA: {
-      if (tsdbCreateTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
+      if (vnodeProcessCreateTSmaReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
-        // TODO
-      }
     } break;
     /* TSDB */
     case TDMT_VND_SUBMIT:
@@ -195,10 +194,12 @@ void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) {
   // TODO
 
   // blockDebugShowData(data);
-  tsdbInsertTSmaData(((SVnode *)pVnode)->pTsdb, smaId, (const char *)data);
+  tdProcessTSmaInsert(((SVnode *)pVnode)->pSma, smaId, (const char *)data);
 }
 
 int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
+  int32_t ret = TAOS_SYNC_PROPOSE_OTHER_ERROR;
+  
   if (syncEnvIsStart()) {
     SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync);
     assert(pSyncNode != NULL);
@@ -220,67 +221,70 @@ int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
       SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pRpcMsg);
       assert(pSyncMsg != NULL);
 
-      syncNodeOnTimeoutCb(pSyncNode, pSyncMsg);
+      ret = syncNodeOnTimeoutCb(pSyncNode, pSyncMsg);
       syncTimeoutDestroy(pSyncMsg);
 
     } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING) {
       SyncPing *pSyncMsg = syncPingFromRpcMsg2(pRpcMsg);
       assert(pSyncMsg != NULL);
 
-      syncNodeOnPingCb(pSyncNode, pSyncMsg);
+      ret = syncNodeOnPingCb(pSyncNode, pSyncMsg);
       syncPingDestroy(pSyncMsg);
 
     } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING_REPLY) {
       SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pRpcMsg);
       assert(pSyncMsg != NULL);
 
-      syncNodeOnPingReplyCb(pSyncNode, pSyncMsg);
+      ret = syncNodeOnPingReplyCb(pSyncNode, pSyncMsg);
       syncPingReplyDestroy(pSyncMsg);
 
     } else if (pRpcMsg->msgType == TDMT_VND_SYNC_CLIENT_REQUEST) {
       SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg);
       assert(pSyncMsg != NULL);
 
-      syncNodeOnClientRequestCb(pSyncNode, pSyncMsg);
+      ret = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg);
       syncClientRequestDestroy(pSyncMsg);
 
     } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE) {
       SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pRpcMsg);
       assert(pSyncMsg != NULL);
 
-      syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg);
+      ret = syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg);
       syncRequestVoteDestroy(pSyncMsg);
 
     } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE_REPLY) {
       SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pRpcMsg);
       assert(pSyncMsg != NULL);
 
-      syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg);
+      ret = syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg);
       syncRequestVoteReplyDestroy(pSyncMsg);
 
     } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES) {
       SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pRpcMsg);
       assert(pSyncMsg != NULL);
 
-      syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg);
+      ret = syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg);
       syncAppendEntriesDestroy(pSyncMsg);
 
     } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES_REPLY) {
       SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pRpcMsg);
       assert(pSyncMsg != NULL);
 
-      syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg);
+      ret = syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg);
       syncAppendEntriesReplyDestroy(pSyncMsg);
 
     } else {
       vError("==vnodeProcessSyncReq== error msg type:%d", pRpcMsg->msgType);
+      ret = TAOS_SYNC_PROPOSE_OTHER_ERROR;
     }
 
     syncNodeRelease(pSyncNode);
   } else {
     vError("==vnodeProcessSyncReq== error syncEnv stop");
+    ret = TAOS_SYNC_PROPOSE_OTHER_ERROR;
   }
-  return 0;
+
+  return ret;
 }
 
 static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp) {
@@ -305,7 +309,7 @@ static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq,
     goto _err;
   }
 
-  tsdbRegisterRSma(pVnode->pTsdb, pVnode->pMeta, &req, &pVnode->msgCb);
+  tdProcessRSmaCreate(pVnode->pSma, pVnode->pMeta, &req, &pVnode->msgCb);
 
   tDecoderClear(&coder);
   return 0;
@@ -366,7 +370,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
       }
     } else {
       cRsp.code = TSDB_CODE_SUCCESS;
-      tsdbFetchTbUidList(pVnode->pTsdb, &pStore, pCreateReq->ctb.suid, pCreateReq->uid);
+      tdFetchTbUidList(pVnode->pSma, &pStore, pCreateReq->ctb.suid, pCreateReq->uid);
     }
 
     taosArrayPush(rsp.pArray, &cRsp);
@@ -374,8 +378,8 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
 
   tDecoderClear(&decoder);
 
-  tsdbUpdateTbUidList(pVnode->pTsdb, pStore);
+  tdUpdateTbUidList(pVnode->pSma, pStore);
-  tsdbUidStoreFree(pStore);
+  tdUidStoreFree(pStore);
 
   // prepare rsp
   SEncoder encoder = {0};
@@ -398,20 +402,32 @@ _exit:
   return rcode;
 }
 
-static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
-  // ASSERT(0);
+  SVCreateStbReq req = {0};
-#if 0
+  SDecoder       dc = {0};
-  SVCreateTbReq vAlterTbReq = {0};
+
-  vTrace("vgId:%d, process alter stb req", TD_VID(pVnode));
+  pRsp->msgType = TDMT_VND_ALTER_STB_RSP;
-  tDeserializeSVCreateTbReq(pReq, &vAlterTbReq);
+  pRsp->code = TSDB_CODE_SUCCESS;
-  // TODO: to encapsule a free API
+  pRsp->pCont = NULL;
-  taosMemoryFree(vAlterTbReq.stbCfg.pSchema);
+  pRsp->contLen = 0;
-  taosMemoryFree(vAlterTbReq.stbCfg.pTagSchema);
+
-  if (vAlterTbReq.stbCfg.pRSmaParam) {
+  tDecoderInit(&dc, pReq, len);
-    taosMemoryFree(vAlterTbReq.stbCfg.pRSmaParam);
+
+  // decode req
+  if (tDecodeSVCreateStbReq(&dc, &req) < 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    tDecoderClear(&dc);
+    return -1;
   }
-  taosMemoryFree(vAlterTbReq.name);
+
-#endif
+  if (metaAlterSTable(pVnode->pMeta, version, &req) < 0) {
+    pRsp->code = terrno;
+    tDecoderClear(&dc);
+    return -1;
+  }
+
+  tDecoderClear(&dc);
+
   return 0;
 }
 
@@ -444,9 +460,32 @@ _exit:
   return 0;
 }
 
-static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
-  // TODO
+  SVAlterTbReq vAlterTbReq = {0};
-  ASSERT(0);
+  SDecoder     dc = {0};
+
+  pRsp->msgType = TDMT_VND_ALTER_TABLE_RSP;
+  pRsp->pCont = NULL;
+  pRsp->contLen = 0;
+  pRsp->code = TSDB_CODE_SUCCESS;
+
+  tDecoderInit(&dc, pReq, len);
+
+  // decode
+  if (tDecodeSVAlterTbReq(&dc, &vAlterTbReq) < 0) {
+    pRsp->code = TSDB_CODE_INVALID_MSG;
+    tDecoderClear(&dc);
+    return -1;
+  }
+
+  // process
+  if (metaAlterTable(pVnode->pMeta, version, &vAlterTbReq) < 0) {
+    pRsp->code = terrno;
+    tDecoderClear(&dc);
+    return -1;
+  }
+
+  tDecoderClear(&dc);
   return 0;
 }
 
@@ -514,7 +553,7 @@ static int vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, SSub
     if (pSchema) {
       taosMemoryFreeClear(pSchema);
     }
-    pSchema = metaGetTbTSchema(pMeta, msgIter->suid, 0);  // TODO: use the real schema
+    pSchema = metaGetTbTSchema(pMeta, msgIter->suid, 1);  // TODO: use the real schema
     if (pSchema) {
       suid = msgIter->suid;
     }
@@ -649,8 +688,38 @@ _exit:
   // TODO: refactor
   if ((terrno == TSDB_CODE_SUCCESS || terrno == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) &&
       (pRsp->code == TSDB_CODE_SUCCESS)) {
-    tsdbTriggerRSma(pVnode->pTsdb, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK);
+    tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK);
   }
 
   return 0;
 }
+
+static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp) {
+  SVCreateTSmaReq req = {0};
+  SDecoder        coder;
+
+  pRsp->msgType = TDMT_VND_CREATE_SMA_RSP;
+  pRsp->code = TSDB_CODE_SUCCESS;
+  pRsp->pCont = NULL;
+  pRsp->contLen = 0;
+
+  // decode and process req
+  tDecoderInit(&coder, pReq, len);
+
+  if (tDecodeSVCreateTSmaReq(&coder, &req) < 0) {
+    pRsp->code = terrno;
+    goto _err;
+  }
+
+  if (metaCreateTSma(pVnode->pMeta, version, &req) < 0) {
+    pRsp->code = terrno;
+    goto _err;
+  }
+
+  tDecoderClear(&coder);
+  return 0;
+
+_err:
+  tDecoderClear(&coder);
+  return -1;
+}

source/libs/catalog/src/catalog.c

@@ -2576,12 +2576,6 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) {
       CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
     }
 
-    SHashObj *metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
-    if (NULL == metaCache) {
-      qError("taosHashInit failed, num:%d", gCtgMgmt.cfg.maxTblCacheNum);
-      CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
-    }
-    
     code = taosHashPut(gCtgMgmt.pCluster, &clusterId, sizeof(clusterId), &clusterCtg, POINTER_BYTES);
     if (code) {
       if (HASH_NODE_EXIST(code)) {

source/libs/executor/src/executorimpl.c

@@ -13,7 +13,6 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <libs/function/function.h>
 #include "filter.h"
 #include "function.h"
 #include "functionMgt.h"
@@ -1601,9 +1600,6 @@ void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p) {
   }
 }
 
-static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId);
-static void         doSetTagValueInParam(void* pTable, int32_t tagColId, SVariant* tag, int16_t type, int16_t bytes);
-
 static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) {
   SqlFunctionCtx* pCtx = pTableScanInfo->pCtx;
   uint32_t        status = BLK_DATA_NOT_LOAD;
@@ -1771,100 +1767,6 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc
   return TSDB_CODE_SUCCESS;
 }
 
-/*
- * set tag value in SqlFunctionCtx
- * e.g.,tag information into input buffer
- */
-static void doSetTagValueInParam(void* pTable, int32_t tagColId, SVariant* tag, int16_t type, int16_t bytes) {
-  taosVariantDestroy(tag);
-
-  char* val = NULL;
-  //  if (tagColId == TSDB_TBNAME_COLUMN_INDEX) {
-  //    val = tsdbGetTableName(pTable);
-  //    assert(val != NULL);
-  //  } else {
-  //    val = tsdbGetTableTagVal(pTable, tagColId, type, bytes);
-  //  }
-
-  if (val == NULL || isNull(val, type)) {
-    tag->nType = TSDB_DATA_TYPE_NULL;
-    return;
-  }
-
-  if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
-    int32_t maxLen = bytes - VARSTR_HEADER_SIZE;
-    int32_t len = (varDataLen(val) > maxLen) ? maxLen : varDataLen(val);
-    taosVariantCreateFromBinary(tag, varDataVal(val), len, type);
-    // taosVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), type);
-  } else {
-    taosVariantCreateFromBinary(tag, val, bytes, type);
-  }
-}
-
-static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId) {
-  assert(pTagColList != NULL && numOfTags > 0);
-
-  for (int32_t i = 0; i < numOfTags; ++i) {
-    if (pTagColList[i].colId == colId) {
-      return &pTagColList[i];
-    }
-  }
-
-  return NULL;
-}
-
-void setTagValue(SOperatorInfo* pOperatorInfo, void* pTable, SqlFunctionCtx* pCtx, int32_t numOfOutput) {
-  SExprInfo* pExpr = pOperatorInfo->pExpr;
-  SExprInfo* pExprInfo = &pExpr[0];
-  int32_t    functionId = getExprFunctionId(pExprInfo);
-#if 0
-  if (pQueryAttr->numOfOutput == 1 && functionId == FUNCTION_TS_COMP && pQueryAttr->stableQuery) {
-    assert(pExprInfo->base.numOfParams == 1);
-
-    //    int16_t      tagColId = (int16_t)pExprInfo->base.param[0].i;
-    int16_t      tagColId = -1;
-    SColumnInfo* pColInfo = doGetTagColumnInfoById(pQueryAttr->tagColList, pQueryAttr->numOfTags, tagColId);
-
-    doSetTagValueInParam(pTable, tagColId, &pCtx[0].tag, pColInfo->type, pColInfo->bytes);
-
-  } else {
-    // set tag value, by which the results are aggregated.
-    int32_t offset = 0;
-    memset(pRuntimeEnv->tagVal, 0, pQueryAttr->tagLen);
-
-    for (int32_t idx = 0; idx < numOfOutput; ++idx) {
-      SExprInfo* pLocalExprInfo = &pExpr[idx];
-
-      // ts_comp column required the tag value for join filter
-      if (!TSDB_COL_IS_TAG(pLocalExprInfo->base.pParam[0].pCol->flag)) {
-        continue;
-      }
-
-      // todo use tag column index to optimize performance
-      doSetTagValueInParam(pTable, pLocalExprInfo->base.pParam[0].pCol->colId, &pCtx[idx].tag,
-                           pLocalExprInfo->base.resSchema.type, pLocalExprInfo->base.resSchema.bytes);
-
-      if (IS_NUMERIC_TYPE(pLocalExprInfo->base.resSchema.type) ||
-          pLocalExprInfo->base.resSchema.type == TSDB_DATA_TYPE_BOOL ||
-          pLocalExprInfo->base.resSchema.type == TSDB_DATA_TYPE_TIMESTAMP) {
-        memcpy(pRuntimeEnv->tagVal + offset, &pCtx[idx].tag.i, pLocalExprInfo->base.resSchema.bytes);
-      } else {
-        if (pCtx[idx].tag.pz != NULL) {
-          memcpy(pRuntimeEnv->tagVal + offset, pCtx[idx].tag.pz, pCtx[idx].tag.nLen);
-        }
-      }
-
-      offset += pLocalExprInfo->base.resSchema.bytes;
-    }
-  }
-
-  // set the tsBuf start position before check each data block
-  if (pRuntimeEnv->pTsBuf != NULL) {
-    setCtxTagForJoin(pRuntimeEnv, &pCtx[0], pExprInfo, pTable);
-  }
-#endif
-}
-
 void copyToSDataBlock(SSDataBlock* pBlock, int32_t* offset, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pResBuf) {
   pBlock->info.rows = 0;
 
@@ -4038,12 +3940,6 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
   if (pProjectInfo->existDataBlock) {  // TODO refactor
     SSDataBlock* pBlock = pProjectInfo->existDataBlock;
     pProjectInfo->existDataBlock = NULL;
-    *newgroup = true;
-
-    // todo dynamic set tags
-    //    if (pTableQueryInfo != NULL) {
-    //      setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfExprs);
-    //    }
 
     // the pDataBlock are always the same one, no need to call this again
     setInputDataBlock(pOperator, pInfo->pCtx, pBlock, TSDB_ORDER_ASC);
@@ -4084,13 +3980,6 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
       }
     }
 
-    // todo set tags
-
-    //    STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current;
-    //    if (pTableQueryInfo != NULL) {
-    //      setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfExprs);
-    //    }
-
     // the pDataBlock are always the same one, no need to call this again
     int32_t code = getTableScanInfo(pOperator->pDownstream[0], &order, &scanFlag);
 
@@ -4430,10 +4319,6 @@ void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) {
   doDestroyBasicInfo(pInfo, numOfOutput);
 }
 
-void destroyMergeJoinOperator(void* param, int32_t numOfOutput) {
-  SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param;
-}
-
 void destroyAggOperatorInfo(void* param, int32_t numOfOutput) {
   SAggOperatorInfo* pInfo = (SAggOperatorInfo*)param;
   doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
@@ -4778,7 +4663,6 @@ static SArray* extractColumnInfo(SNodeList* pNodeList);
 
 static SArray* createSortInfo(SNodeList* pNodeList);
 static SArray* extractPartitionColInfo(SNodeList* pNodeList);
-static void    setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode);
 
 SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle,
                                   uint64_t queryId, uint64_t taskId, STableGroupInfo* pTableGroupInfo) {
@@ -5447,150 +5331,3 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo
 
   return TSDB_CODE_SUCCESS;
 }
-
-static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) {
-  SJoinOperatorInfo* pJoinInfo = pOperator->info;
-
-  SSDataBlock* pRes = pJoinInfo->pRes;
-  blockDataCleanup(pRes);
-  blockDataEnsureCapacity(pRes, 4096);
-
-  int32_t nrows = 0;
-
-  while (1) {
-    if (pJoinInfo->pLeft == NULL || pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) {
-      SOperatorInfo* ds1 = pOperator->pDownstream[0];
-      publishOperatorProfEvent(ds1, QUERY_PROF_BEFORE_OPERATOR_EXEC);
-      pJoinInfo->pLeft = ds1->fpSet.getNextFn(ds1);
-      publishOperatorProfEvent(ds1, QUERY_PROF_AFTER_OPERATOR_EXEC);
-
-      pJoinInfo->leftPos = 0;
-      if (pJoinInfo->pLeft == NULL) {
-        setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
-        break;
-      }
-    }
-
-    if (pJoinInfo->pRight == NULL || pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) {
-      SOperatorInfo* ds2 = pOperator->pDownstream[1];
-      publishOperatorProfEvent(ds2, QUERY_PROF_BEFORE_OPERATOR_EXEC);
-      pJoinInfo->pRight = ds2->fpSet.getNextFn(ds2);
-      publishOperatorProfEvent(ds2, QUERY_PROF_AFTER_OPERATOR_EXEC);
-
-      pJoinInfo->rightPos = 0;
-      if (pJoinInfo->pRight == NULL) {
-        setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
-        break;
-      }
-    }
-
-    SColumnInfoData* pLeftCol = taosArrayGet(pJoinInfo->pLeft->pDataBlock, pJoinInfo->leftCol.slotId);
-    char*            pLeftVal = colDataGetData(pLeftCol, pJoinInfo->leftPos);
-
-    SColumnInfoData* pRightCol = taosArrayGet(pJoinInfo->pRight->pDataBlock, pJoinInfo->rightCol.slotId);
-    char*            pRightVal = colDataGetData(pRightCol, pJoinInfo->rightPos);
-
-    // only the timestamp match support for ordinary table
-    ASSERT(pLeftCol->info.type == TSDB_DATA_TYPE_TIMESTAMP);
-    if (*(int64_t*)pLeftVal == *(int64_t*)pRightVal) {
-      for (int32_t i = 0; i < pOperator->numOfExprs; ++i) {
-        SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, i);
-
-        SExprInfo* pExprInfo = &pOperator->pExpr[i];
-
-        int32_t blockId = pExprInfo->base.pParam[0].pCol->dataBlockId;
-        int32_t slotId = pExprInfo->base.pParam[0].pCol->slotId;
-        int32_t rowIndex = -1;
-
-        SColumnInfoData* pSrc = NULL;
-        if (pJoinInfo->pLeft->info.blockId == blockId) {
-          pSrc = taosArrayGet(pJoinInfo->pLeft->pDataBlock, slotId);
-          rowIndex = pJoinInfo->leftPos;
-        } else {
-          pSrc = taosArrayGet(pJoinInfo->pRight->pDataBlock, slotId);
-          rowIndex = pJoinInfo->rightPos;
-        }
-
-        if (colDataIsNull_s(pSrc, rowIndex)) {
-          colDataAppendNULL(pDst, nrows);
-        } else {
-          char* p = colDataGetData(pSrc, rowIndex);
-          colDataAppend(pDst, nrows, p, false);
-        }
-      }
-
-      pJoinInfo->leftPos += 1;
-      pJoinInfo->rightPos += 1;
-
-      nrows += 1;
-    } else if (*(int64_t*)pLeftVal < *(int64_t*)pRightVal) {
-      pJoinInfo->leftPos += 1;
-
-      if (pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) {
-        continue;
-      }
-    } else if (*(int64_t*)pLeftVal > *(int64_t*)pRightVal) {
-      pJoinInfo->rightPos += 1;
-      if (pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) {
-        continue;
-      }
-    }
-
-    // the pDataBlock are always the same one, no need to call this again
-    pRes->info.rows = nrows;
-    if (pRes->info.rows >= pOperator->resultInfo.threshold) {
-      break;
-    }
-  }
-
-  return (pRes->info.rows > 0) ? pRes : NULL;
-}
-
-SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo,
-                                           int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition,
-                                           SExecTaskInfo* pTaskInfo) {
-  SJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SJoinOperatorInfo));
-  SOperatorInfo*     pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
-  if (pOperator == NULL || pInfo == NULL) {
-    goto _error;
-  }
-
-  initResultSizeInfo(pOperator, 4096);
-
-  pInfo->pRes = pResBlock;
-  pOperator->name = "MergeJoinOperator";
-  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_JOIN;
-  pOperator->blocking = false;
-  pOperator->status = OP_NOT_OPENED;
-  pOperator->pExpr = pExprInfo;
-  pOperator->numOfExprs = numOfCols;
-  pOperator->info = pInfo;
-  pOperator->pTaskInfo = pTaskInfo;
-
-  SOperatorNode* pNode = (SOperatorNode*)pOnCondition;
-  setJoinColumnInfo(&pInfo->leftCol, (SColumnNode*)pNode->pLeft);
-  setJoinColumnInfo(&pInfo->rightCol, (SColumnNode*)pNode->pRight);
-
-  pOperator->fpSet =
-      createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, NULL, destroyMergeJoinOperator, NULL, NULL, NULL);
-  int32_t code = appendDownstream(pOperator, pDownstream, numOfDownstream);
-  if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
-  }
-
-  return pOperator;
-
-_error:
-  taosMemoryFree(pInfo);
-  taosMemoryFree(pOperator);
-  pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
-  return NULL;
-}
-
-void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) {
-  pColumn->slotId = pColumnNode->slotId;
-  pColumn->type = pColumnNode->node.resType.type;
-  pColumn->bytes = pColumnNode->node.resType.bytes;
-  pColumn->precision = pColumnNode->node.resType.precision;
-  pColumn->scale = pColumnNode->node.resType.scale;
-}

source/libs/executor/src/joinoperator.c

@@ -0,0 +1,199 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "function.h"
+#include "os.h"
+#include "querynodes.h"
+#include "tdatablock.h"
+#include "tmsg.h"
+#include "executorimpl.h"
+#include "tcompare.h"
+#include "thash.h"
+#include "ttypes.h"
+
+static void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode);
+static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator);
+static void destroyMergeJoinOperator(void* param, int32_t numOfOutput);
+static void extractTimeCondition(SJoinOperatorInfo *Info, SLogicConditionNode* pLogicConditionNode);
+
+SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo,
+                                           int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition,
+                                           SExecTaskInfo* pTaskInfo) {
+  SJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SJoinOperatorInfo));
+  SOperatorInfo*     pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+  if (pOperator == NULL || pInfo == NULL) {
+    goto _error;
+  }
+
+  initResultSizeInfo(pOperator, 4096);
+
+  pInfo->pRes           = pResBlock;
+  pOperator->name       = "MergeJoinOperator";
+  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_JOIN;
+  pOperator->blocking   = false;
+  pOperator->status     = OP_NOT_OPENED;
+  pOperator->pExpr      = pExprInfo;
+  pOperator->numOfExprs = numOfCols;
+  pOperator->info       = pInfo;
+  pOperator->pTaskInfo  = pTaskInfo;
+
+  if (nodeType(pOnCondition) == QUERY_NODE_OPERATOR) {
+    SOperatorNode* pNode = (SOperatorNode*)pOnCondition;
+    setJoinColumnInfo(&pInfo->leftCol, (SColumnNode*)pNode->pLeft);
+    setJoinColumnInfo(&pInfo->rightCol, (SColumnNode*)pNode->pRight);
+  } else if (nodeType(pOnCondition) == QUERY_NODE_LOGIC_CONDITION) {
+    extractTimeCondition(pInfo, (SLogicConditionNode*) pOnCondition);
+  }
+
+  pOperator->fpSet =
+      createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, NULL, destroyMergeJoinOperator, NULL, NULL, NULL);
+  int32_t code = appendDownstream(pOperator, pDownstream, numOfDownstream);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
+
+  return pOperator;
+
+  _error:
+  taosMemoryFree(pInfo);
+  taosMemoryFree(pOperator);
+  pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
+  return NULL;
+}
+
+void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) {
+  pColumn->slotId = pColumnNode->slotId;
+  pColumn->type = pColumnNode->node.resType.type;
+  pColumn->bytes = pColumnNode->node.resType.bytes;
+  pColumn->precision = pColumnNode->node.resType.precision;
+  pColumn->scale = pColumnNode->node.resType.scale;
+}
+
+void destroyMergeJoinOperator(void* param, int32_t numOfOutput) {
+  SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param;
+}
+
+SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) {
+  SJoinOperatorInfo* pJoinInfo = pOperator->info;
+
+  SSDataBlock* pRes = pJoinInfo->pRes;
+  blockDataCleanup(pRes);
+  blockDataEnsureCapacity(pRes, 4096);
+
+  int32_t nrows = 0;
+
+  while (1) {
+    // todo extract method
+    if (pJoinInfo->pLeft == NULL || pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) {
+      SOperatorInfo* ds1 = pOperator->pDownstream[0];
+      publishOperatorProfEvent(ds1, QUERY_PROF_BEFORE_OPERATOR_EXEC);
+      pJoinInfo->pLeft = ds1->fpSet.getNextFn(ds1);
+      publishOperatorProfEvent(ds1, QUERY_PROF_AFTER_OPERATOR_EXEC);
+
+      pJoinInfo->leftPos = 0;
+      if (pJoinInfo->pLeft == NULL) {
+        setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
+        break;
+      }
+    }
+
+    if (pJoinInfo->pRight == NULL || pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) {
+      SOperatorInfo* ds2 = pOperator->pDownstream[1];
+      publishOperatorProfEvent(ds2, QUERY_PROF_BEFORE_OPERATOR_EXEC);
+      pJoinInfo->pRight = ds2->fpSet.getNextFn(ds2);
+      publishOperatorProfEvent(ds2, QUERY_PROF_AFTER_OPERATOR_EXEC);
+
+      pJoinInfo->rightPos = 0;
+      if (pJoinInfo->pRight == NULL) {
+        setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
+        break;
+      }
+    }
+
+    SColumnInfoData* pLeftCol = taosArrayGet(pJoinInfo->pLeft->pDataBlock, pJoinInfo->leftCol.slotId);
+    char*            pLeftVal = colDataGetData(pLeftCol, pJoinInfo->leftPos);
+
+    SColumnInfoData* pRightCol = taosArrayGet(pJoinInfo->pRight->pDataBlock, pJoinInfo->rightCol.slotId);
+    char*            pRightVal = colDataGetData(pRightCol, pJoinInfo->rightPos);
+
+    // only the timestamp match support for ordinary table
+    ASSERT(pLeftCol->info.type == TSDB_DATA_TYPE_TIMESTAMP);
+    if (*(int64_t*)pLeftVal == *(int64_t*)pRightVal) {
+      for (int32_t i = 0; i < pOperator->numOfExprs; ++i) {
+        SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, i);
+
+        SExprInfo* pExprInfo = &pOperator->pExpr[i];
+
+        int32_t blockId = pExprInfo->base.pParam[0].pCol->dataBlockId;
+        int32_t slotId = pExprInfo->base.pParam[0].pCol->slotId;
+        int32_t rowIndex = -1;
+
+        SColumnInfoData* pSrc = NULL;
+        if (pJoinInfo->pLeft->info.blockId == blockId) {
+          pSrc = taosArrayGet(pJoinInfo->pLeft->pDataBlock, slotId);
+          rowIndex = pJoinInfo->leftPos;
+        } else {
+          pSrc = taosArrayGet(pJoinInfo->pRight->pDataBlock, slotId);
+          rowIndex = pJoinInfo->rightPos;
+        }
+
+        if (colDataIsNull_s(pSrc, rowIndex)) {
+          colDataAppendNULL(pDst, nrows);
+        } else {
+          char* p = colDataGetData(pSrc, rowIndex);
+          colDataAppend(pDst, nrows, p, false);
+        }
+      }
+
+      pJoinInfo->leftPos += 1;
+      pJoinInfo->rightPos += 1;
+
+      nrows += 1;
+    } else if (*(int64_t*)pLeftVal < *(int64_t*)pRightVal) {
+      pJoinInfo->leftPos += 1;
+
+      if (pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) {
+        continue;
+      }
+    } else if (*(int64_t*)pLeftVal > *(int64_t*)pRightVal) {
+      pJoinInfo->rightPos += 1;
+      if (pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) {
+        continue;
+      }
+    }
+
+    // the pDataBlock are always the same one, no need to call this again
+    pRes->info.rows = nrows;
+    if (pRes->info.rows >= pOperator->resultInfo.threshold) {
+      break;
+    }
+  }
+
+  return (pRes->info.rows > 0) ? pRes : NULL;
+}
+
+static void extractTimeCondition(SJoinOperatorInfo *pInfo, SLogicConditionNode* pLogicConditionNode) {
+  int32_t len = LIST_LENGTH(pLogicConditionNode->pParameterList);
+
+  for(int32_t i = 0; i < len; ++i) {
+    SNode* pNode = nodesListGetNode(pLogicConditionNode->pParameterList, i);
+    if (nodeType(pNode) == QUERY_NODE_OPERATOR) {
+      SOperatorNode* pn1 = (SOperatorNode*)pNode;
+      setJoinColumnInfo(&pInfo->leftCol, (SColumnNode*)pn1->pLeft);
+      setJoinColumnInfo(&pInfo->rightCol, (SColumnNode*)pn1->pRight);
+      break;
+    }
+  }
+}

source/libs/executor/src/timewindowoperator.c

@@ -773,7 +773,6 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
       break;
     }
 
-    //    setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfExprs);
     // the pDataBlock are always the same one, no need to call this again
     setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, MAIN_SCAN, true);
     STableQueryInfo* pTableQueryInfo = pInfo->pCurrent;
@@ -1062,8 +1061,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
 
     // The timewindows that overlaps the timestamps of the input pBlock need to be recalculated and return to the
     // caller. Note that all the time window are not close till now.
-
-    //    setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfExprs);
     // the pDataBlock are always the same one, no need to call this again
     setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, MAIN_SCAN, true);
     if (pInfo->invertible) {
@@ -1377,7 +1374,6 @@ static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator) {
       break;
     }
 
-    //    setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfExprs);
     // the pDataBlock are always the same one, no need to call this again
     setInputDataBlock(pOperator, pSliceInfo->binfo.pCtx, pBlock, order, MAIN_SCAN, true);
     //    hashAllIntervalAgg(pOperator, &pSliceInfo->binfo.resultRowInfo, pBlock, 0);

source/libs/nodes/src/nodesCodeFuncs.c

@@ -584,6 +584,37 @@ static int32_t jsonToLogicProjectNode(const SJson* pJson, void* pObj) {
   return code;
 }
 
+static const char* jkExchangeLogicPlanSrcGroupId = "SrcGroupId";
+static const char* jkExchangeLogicPlanSrcPrecision = "Precision";
+
+static int32_t logicExchangeNodeToJson(const void* pObj, SJson* pJson) {
+  const SExchangeLogicNode* pNode = (const SExchangeLogicNode*)pObj;
+
+  int32_t code = logicPlanNodeToJson(pObj, pJson);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkExchangeLogicPlanSrcGroupId, pNode->srcGroupId);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkExchangeLogicPlanSrcPrecision, pNode->precision);
+  }
+
+  return code;
+}
+
+static int32_t jsonToLogicExchangeNode(const SJson* pJson, void* pObj) {
+  SExchangeLogicNode* pNode = (SExchangeLogicNode*)pObj;
+
+  int32_t code = jsonToLogicPlanNode(pJson, pObj);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetIntValue(pJson, jkExchangeLogicPlanSrcGroupId, &pNode->srcGroupId);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetUTinyIntValue(pJson, jkExchangeLogicPlanSrcPrecision, &pNode->precision);
+  }
+
+  return code;
+}
+
 static const char* jkFillLogicPlanMode = "Mode";
 static const char* jkFillLogicPlanWStartTs = "WStartTs";
 static const char* jkFillLogicPlanValues = "Values";
@@ -2987,6 +3018,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
       return logicProjectNodeToJson(pObj, pJson);
     case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF:
       break;
+    case QUERY_NODE_LOGIC_PLAN_EXCHANGE:
+      return logicExchangeNodeToJson(pObj, pJson);
     case QUERY_NODE_LOGIC_PLAN_FILL:
       return logicFillNodeToJson(pObj, pJson);
     case QUERY_NODE_LOGIC_PLAN_SORT:
@@ -3083,6 +3116,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
       return jsonToLogicScanNode(pJson, pObj);
     case QUERY_NODE_LOGIC_PLAN_PROJECT:
       return jsonToLogicProjectNode(pJson, pObj);
+    case QUERY_NODE_LOGIC_PLAN_EXCHANGE:
+      return jsonToLogicExchangeNode(pJson, pObj);
     case QUERY_NODE_LOGIC_PLAN_FILL:
       return jsonToLogicFillNode(pJson, pObj);
     case QUERY_NODE_LOGIC_PLAN_SORT:

source/libs/parser/src/parAuthenticator.c

@@ -14,6 +14,7 @@
  */
 
 #include "catalog.h"
+#include "cmdnodes.h"
 #include "parInt.h"
 
 typedef struct SAuthCxt {
@@ -65,8 +66,8 @@ static int32_t authSetOperator(SAuthCxt* pCxt, SSetOperator* pSetOper) {
   return code;
 }
 
-static int32_t authDropUser(SAuthCxt* pCxt, SDropUserReq* pStmt) {
+static int32_t authDropUser(SAuthCxt* pCxt, SDropUserStmt* pStmt) {
-  if (!pCxt->pParseCxt->isSuperUser || 0 == strcmp(pStmt->user, TSDB_DEFAULT_USER)) {
+  if (!pCxt->pParseCxt->isSuperUser || 0 == strcmp(pStmt->useName, TSDB_DEFAULT_USER)) {
     return TSDB_CODE_PAR_PERMISSION_DENIED;
   }
   return TSDB_CODE_SUCCESS;
@@ -92,7 +93,7 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) {
     case QUERY_NODE_ALTER_USER_STMT:
       break;
     case QUERY_NODE_DROP_USER_STMT: {
-      return authDropUser(pCxt, (SDropUserReq*)pStmt);
+      return authDropUser(pCxt, (SDropUserStmt*)pStmt);
     }
     case QUERY_NODE_USE_DATABASE_STMT:
     case QUERY_NODE_CREATE_DNODE_STMT:

source/libs/parser/src/parInsert.c

@@ -1041,18 +1041,6 @@ static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) {
   destroyCreateSubTbReq(&pCxt->createTblReq);
 }
 
-static void destroyDataBlock(STableDataBlocks* pDataBlock) {
-  if (pDataBlock == NULL) {
-    return;
-  }
-
-  taosMemoryFreeClear(pDataBlock->pData);
-  if (!pDataBlock->cloned) {
-    destroyBoundColumnInfo(&pDataBlock->boundColumnInfo);
-  }
-  taosMemoryFreeClear(pDataBlock);
-}
-
 static void destroyInsertParseContext(SInsertParseContext* pCxt) {
   destroyInsertParseContextForTable(pCxt);
   taosHashCleanup(pCxt->pVgroupsHashObj);
@@ -1301,6 +1289,7 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash
 
   CHECK_CODE(buildOutput(&insertCtx));
 
+  destroyBlockArrayList(insertCtx.pVgDataBlocks);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -1580,16 +1569,25 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD** fields
 
 // schemaless logic start
 
-typedef struct SmlExecHandle {
+typedef struct SmlExecTableHandle {
-  SHashObj* pBlockHash;
-
   SParsedDataColInfo tags;          // each table
   SKVRowBuilder      tagsBuilder;   // each table
   SVCreateTbReq      createTblReq;  // each table
+} SmlExecTableHandle;
 
-  SQuery* pQuery;
+typedef struct SmlExecHandle {
+  SHashObj*           pBlockHash;
+  SmlExecTableHandle  tableExecHandle;
+  SQuery              *pQuery;
 } SSmlExecHandle;
 
+static void smlDestroyTableHandle(void* pHandle) {
+  SmlExecTableHandle* handle = (SmlExecTableHandle*)pHandle;
+  tdDestroyKVRowBuilder(&handle->tagsBuilder);
+  destroyBoundColumnInfo(&handle->tags);
+  destroyCreateSubTbReq(&handle->createTblReq);
+}
+
 static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SSchema* pSchema) {
   col_id_t nCols = pColList->numOfCols;
 
@@ -1692,25 +1690,26 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsSchema, SArray *cols
   SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen};
 
   SSmlExecHandle* smlHandle = (SSmlExecHandle*)handle;
+  smlDestroyTableHandle(&smlHandle->tableExecHandle);   // free for each table
   SSchema*        pTagsSchema = getTableTagSchema(pTableMeta);
-  setBoundColumnInfo(&smlHandle->tags, pTagsSchema, getNumOfTags(pTableMeta));
+  setBoundColumnInfo(&smlHandle->tableExecHandle.tags, pTagsSchema, getNumOfTags(pTableMeta));
-  int ret = smlBoundColumnData(tags, &smlHandle->tags, pTagsSchema);
+  int ret = smlBoundColumnData(tags, &smlHandle->tableExecHandle.tags, pTagsSchema);
   if (ret != TSDB_CODE_SUCCESS) {
     buildInvalidOperationMsg(&pBuf, "bound tags error");
     return ret;
   }
   SKVRow row = NULL;
-  ret = smlBuildTagRow(tags, &smlHandle->tagsBuilder, &smlHandle->tags, pTagsSchema, &row, &pBuf);
+  ret = smlBuildTagRow(tags, &smlHandle->tableExecHandle.tagsBuilder, &smlHandle->tableExecHandle.tags, pTagsSchema, &row, &pBuf);
   if (ret != TSDB_CODE_SUCCESS) {
     return ret;
   }
 
-  buildCreateTbReq(&smlHandle->createTblReq, tableName, row, pTableMeta->suid);
+  buildCreateTbReq(&smlHandle->tableExecHandle.createTblReq, tableName, row, pTableMeta->suid);
 
   STableDataBlocks* pDataBlock = NULL;
   ret = getDataBlockFromList(smlHandle->pBlockHash, &pTableMeta->uid, sizeof(pTableMeta->uid),
                              TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), getTableInfo(pTableMeta).rowSize,
-                             pTableMeta, &pDataBlock, NULL, &smlHandle->createTblReq);
+                             pTableMeta, &pDataBlock, NULL, &smlHandle->tableExecHandle.createTblReq);
   if (ret != TSDB_CODE_SUCCESS) {
     buildInvalidOperationMsg(&pBuf, "create data block error");
     return ret;
@@ -1826,9 +1825,7 @@ void smlDestroyHandle(void* pHandle) {
   if (!pHandle) return;
   SSmlExecHandle* handle = (SSmlExecHandle*)pHandle;
   destroyBlockHashmap(handle->pBlockHash);
-  tdDestroyKVRowBuilder(&handle->tagsBuilder);
+  smlDestroyTableHandle(&handle->tableExecHandle);
-  destroyBoundColumnInfo(&handle->tags);
-  destroyCreateSubTbReq(&handle->createTblReq);
   taosMemoryFree(handle);
 }
 

source/libs/parser/src/parInsertData.c

@@ -237,9 +237,7 @@ static void destroyDataBlock(STableDataBlocks* pDataBlock) {
   taosMemoryFreeClear(pDataBlock->pData);
   if (!pDataBlock->cloned) {
     // free the refcount for metermeta
-    if (pDataBlock->pTableMeta != NULL) {
+    taosMemoryFreeClear(pDataBlock->pTableMeta);
-      taosMemoryFreeClear(pDataBlock->pTableMeta);
-    }
 
     destroyBoundColumnInfo(&pDataBlock->boundColumnInfo);
   }

source/libs/parser/src/parTranslater.c

@@ -2949,6 +2949,8 @@ static int32_t translateCreateIndex(STranslateContext* pCxt, SCreateIndexStmt* p
 }
 
 static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt) {
+  SEncoder       encoder = {0};
+  int32_t        contLen = 0;
   SVDropTSmaReq dropSmaReq = {0};
   strcpy(dropSmaReq.indexName, pStmt->indexName);
 
@@ -2956,16 +2958,26 @@ static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt
   if (NULL == pCxt->pCmdMsg) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
+  
+  int32_t ret = 0;
+  tEncodeSize(tEncodeSVDropTSmaReq, &dropSmaReq, contLen, ret);
+  if (ret < 0) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
   pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet;
   pCxt->pCmdMsg->msgType = TDMT_VND_DROP_SMA;
-  pCxt->pCmdMsg->msgLen = tSerializeSVDropTSmaReq(NULL, &dropSmaReq);
+  pCxt->pCmdMsg->msgLen = contLen;
   pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen);
   if (NULL == pCxt->pCmdMsg->pMsg) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
   void* pBuf = pCxt->pCmdMsg->pMsg;
-  tSerializeSVDropTSmaReq(&pBuf, &dropSmaReq);
+  if (tEncodeSVDropTSmaReq(&encoder, &dropSmaReq) < 0) {
-
+    tEncoderClear(&encoder);
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  tEncoderClear(&encoder);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -3713,7 +3725,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt*
   req.type = TD_NORMAL_TABLE;
   req.name = strdup(pStmt->tableName);
   req.ntb.schema.nCols = LIST_LENGTH(pStmt->pCols);
-  req.ntb.schema.sver = 0;
+  req.ntb.schema.sver = 1;
   req.ntb.schema.pSchema = taosMemoryCalloc(req.ntb.schema.nCols, sizeof(SSchema));
   if (NULL == req.name || NULL == req.ntb.schema.pSchema) {
     destroyCreateTbReq(&req);

source/libs/parser/test/parSelectTest.cpp

@@ -235,11 +235,17 @@ TEST_F(ParserSelectTest, semanticError) {
 TEST_F(ParserSelectTest, setOperator) {
   useDb("root", "test");
 
-  // run("SELECT * FROM t1 UNION ALL SELECT * FROM t1");
+  run("SELECT * FROM t1 UNION ALL SELECT * FROM t1");
 
-  // run("(SELECT * FROM t1) UNION ALL (SELECT * FROM t1)");
+  run("(SELECT * FROM t1) UNION ALL (SELECT * FROM t1)");
 
   run("SELECT c1 FROM (SELECT c1 FROM t1 UNION ALL SELECT c1 FROM t1)");
 }
 
+TEST_F(ParserSelectTest, informationSchema) {
+  useDb("root", "test");
+
+  run("SELECT * FROM information_schema.user_databases WHERE name = 'information_schema'");
+}
+
 }  // namespace ParserTest

source/libs/planner/src/planOptimizer.c

@@ -392,7 +392,8 @@ static int32_t cpdCalcTimeRange(SScanLogicNode* pScan, SNode** pPrimaryKeyCond,
 }
 
 static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* pScan) {
-  if (NULL == pScan->node.pConditions || OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD)) {
+  if (NULL == pScan->node.pConditions || OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD) ||
+      TSDB_SYSTEM_TABLE == pScan->pMeta->tableType) {
     return TSDB_CODE_SUCCESS;
   }
 

source/libs/planner/src/planSpliter.c

@@ -303,7 +303,7 @@ static SLogicNode* unMatchByNode(SLogicNode* pNode) {
   }
   SNode* pChild;
   FOREACH(pChild, pNode->pChildren) {
-    SLogicNode* pSplitNode = uaMatchByNode((SLogicNode*)pChild);
+    SLogicNode* pSplitNode = unMatchByNode((SLogicNode*)pChild);
     if (NULL != pSplitNode) {
       return pSplitNode;
     }
@@ -318,7 +318,7 @@ static int32_t unCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan
   }
   pExchange->srcGroupId = pCxt->groupId;
   // pExchange->precision = pScan->pMeta->tableInfo.precision;
-  pExchange->node.pTargets = nodesCloneList(pAgg->node.pTargets);
+  pExchange->node.pTargets = nodesCloneList(pAgg->pGroupKeys);
   if (NULL == pExchange->node.pTargets) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }

source/libs/planner/test/planOtherTest.cpp

@@ -47,4 +47,10 @@ TEST_F(PlanOtherTest, explain) {
   run("explain analyze SELECT * FROM t1");
 
   run("explain analyze verbose true ratio 0.01 SELECT * FROM t1");
-}
+}
+
+TEST_F(PlanOtherTest, show) {
+  useDb("root", "test");
+
+  run("SHOW DATABASES");
+}

source/libs/planner/test/planStateTest.cpp

@@ -23,13 +23,13 @@ class PlanStateTest : public PlannerTestBase {};
 TEST_F(PlanStateTest, basic) {
   useDb("root", "test");
 
-  run("select count(*) from t1 state_window(c1)");
+  run("SELECT COUNT(*) FROM t1 STATE_WINDOW(c1)");
 }
 
 TEST_F(PlanStateTest, stateExpr) {
   useDb("root", "test");
 
-  run("select count(*) from t1 state_window(c1 + 10)");
+  run("SELECT COUNT(*) FROM t1 STATE_WINDOW(c1 + 10)");
 }
 
 TEST_F(PlanStateTest, selectFunc) {

source/libs/planner/test/planSubqueryTest.cpp

@@ -25,11 +25,9 @@ TEST_F(PlanSubqeuryTest, basic) {
 
   if (0 == g_skipSql) {
     run("SELECT * FROM (SELECT * FROM t1)");
-
-    run("SELECT LAST(c1) FROM (SELECT * FROM t1)");
   }
 
-  run("SELECT c1 FROM (SELECT c1 FROM t1 UNION ALL SELECT c1 FROM t1)");
+  run("SELECT LAST(c1) FROM (SELECT * FROM t1)");
 }
 
 TEST_F(PlanSubqeuryTest, doubleGroupBy) {
@@ -39,3 +37,11 @@ TEST_F(PlanSubqeuryTest, doubleGroupBy) {
       "SELECT c1 + c3 a, c1 + COUNT(*) b FROM t1 WHERE c2 = 'abc' GROUP BY c1, c3) "
       "WHERE a > 100 GROUP BY b");
 }
+
+TEST_F(PlanSubqeuryTest, withSetOperator) {
+  useDb("root", "test");
+
+  run("SELECT c1 FROM (SELECT c1 FROM t1 UNION ALL SELECT c1 FROM t1)");
+
+  run("SELECT c1 FROM (SELECT c1 FROM t1 UNION SELECT c1 FROM t1)");
+}

source/libs/planner/test/planSysTbTest.cpp

@@ -27,8 +27,8 @@ TEST_F(PlanSysTableTest, show) {
   run("show stables");
 }
 
-TEST_F(PlanSysTableTest, information) {
+TEST_F(PlanSysTableTest, informationSchema) {
   useDb("root", "information_schema");
 
-  run("show tables");
+  run("SELECT * FROM information_schema.user_databases WHERE name = 'information_schema'");
 }

source/libs/qworker/src/qworker.c

@@ -812,13 +812,11 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu
   }
 
   if (ctx->rspCode) {
-    QW_TASK_ELOG("task already failed at phase %s, error:%x - %s", qwPhaseStr(phase), ctx->rspCode,
+    QW_TASK_ELOG("task already failed at phase %s, code:%s", qwPhaseStr(phase), tstrerror(ctx->rspCode));
-                 tstrerror(ctx->rspCode));
     QW_ERR_JRET(ctx->rspCode);
   }
 
 _return:
-
   if (ctx) {
     QW_UPDATE_RSP_CODE(ctx, code);
 
@@ -836,7 +834,11 @@ _return:
     QW_TASK_DLOG("cancel rsp send, handle:%p, code:%x - %s", cancelConnection->handle, code, tstrerror(code));
   }
 
-  QW_TASK_DLOG("end to handle event at phase %s, code:%x - %s", qwPhaseStr(phase), code, tstrerror(code));
+  if (code != TSDB_CODE_SUCCESS) {
+    QW_TASK_ELOG("end to handle event at phase %s, code:%s", qwPhaseStr(phase), tstrerror(code));
+  } else {
+    QW_TASK_DLOG("end to handle event at phase %s, code:%s", qwPhaseStr(phase), tstrerror(code));
+  }
 
   QW_RET(code);
 }

source/libs/stream/src/tstream.c

@@ -133,7 +133,144 @@ int32_t streamEnqueueDataBlk(SStreamTask* pTask, SStreamDataBlock* input) {
   return inputStatus;
 }
 
-int32_t streamTaskProcessTriggerReq(SStreamTask* pTask, SMsgCb* pMsgCb, char* msg, int32_t msgLen) {
+int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) {
+  void* exec = pTask->exec.runners[0].executor;
+
+  // set input
+  if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) {
+    SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data;
+    ASSERT(pSubmit->type == STREAM_INPUT__DATA_SUBMIT);
+
+    qSetStreamInput(exec, pSubmit->data, STREAM_DATA_TYPE_SUBMIT_BLOCK);
+  } else if (pTask->inputType == STREAM_INPUT__DATA_BLOCK) {
+    SStreamDataBlock* pBlock = (SStreamDataBlock*)data;
+    ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK);
+
+    SArray* blocks = pBlock->blocks;
+    qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK);
+  }
+
+  // exec
+  while (1) {
+    SSDataBlock* output;
+    uint64_t     ts = 0;
+    if (qExecTask(exec, &output, &ts) < 0) {
+      ASSERT(false);
+    }
+    if (output == NULL) break;
+    taosArrayPush(pRes, &output);
+  }
+
+  // destroy
+  if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) {
+    streamDataSubmitRefDec((SStreamDataSubmit*)data);
+  } else {
+    taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock);
+  }
+  return 0;
+}
+
+// TODO: handle version
+int32_t streamTaskExec2(SStreamTask* pTask, SMsgCb* pMsgCb) {
+  SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
+  if (pRes == NULL) return -1;
+  while (1) {
+    int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING);
+    void*  exec = pTask->exec.runners[0].executor;
+    if (execStatus == TASK_STATUS__IDLE) {
+      // first run, from qall, handle failure from last exec
+      while (1) {
+        void* data = NULL;
+        taosGetQitem(pTask->inputQAll, &data);
+        if (data == NULL) break;
+
+        streamTaskExecImpl(pTask, data, pRes);
+
+        taosFreeQitem(data);
+
+        if (taosArrayGetSize(pRes) != 0) {
+          SStreamDataBlock* resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM);
+          resQ->type = STREAM_INPUT__DATA_BLOCK;
+          resQ->blocks = pRes;
+          taosWriteQitem(pTask->outputQ, resQ);
+          pRes = taosArrayInit(0, sizeof(SSDataBlock));
+          if (pRes == NULL) goto FAIL;
+        }
+      }
+      // second run, from inputQ
+      taosReadAllQitems(pTask->inputQ, pTask->inputQAll);
+      while (1) {
+        void* data = NULL;
+        taosGetQitem(pTask->inputQAll, &data);
+        if (data == NULL) break;
+
+        streamTaskExecImpl(pTask, data, pRes);
+
+        taosFreeQitem(data);
+
+        if (taosArrayGetSize(pRes) != 0) {
+          SStreamDataBlock* resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM);
+          resQ->type = STREAM_INPUT__DATA_BLOCK;
+          resQ->blocks = pRes;
+          taosWriteQitem(pTask->outputQ, resQ);
+          pRes = taosArrayInit(0, sizeof(SSDataBlock));
+          if (pRes == NULL) goto FAIL;
+        }
+      }
+      // set status closing
+      atomic_store_8(&pTask->status, TASK_STATUS__CLOSING);
+      // third run, make sure all inputQ is cleared
+      taosReadAllQitems(pTask->inputQ, pTask->inputQAll);
+      while (1) {
+        void* data = NULL;
+        taosGetQitem(pTask->inputQAll, &data);
+        if (data == NULL) break;
+
+        streamTaskExecImpl(pTask, data, pRes);
+
+        taosFreeQitem(data);
+
+        if (taosArrayGetSize(pRes) != 0) {
+          SStreamDataBlock* resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM);
+          resQ->type = STREAM_INPUT__DATA_BLOCK;
+          resQ->blocks = pRes;
+          taosWriteQitem(pTask->outputQ, resQ);
+          pRes = taosArrayInit(0, sizeof(SSDataBlock));
+          if (pRes == NULL) goto FAIL;
+        }
+      }
+      // set status closing
+      atomic_store_8(&pTask->status, TASK_STATUS__CLOSING);
+      // third run, make sure all inputQ is cleared
+      taosReadAllQitems(pTask->inputQ, pTask->inputQAll);
+      while (1) {
+        void* data = NULL;
+        taosGetQitem(pTask->inputQAll, &data);
+        if (data == NULL) break;
+      }
+
+      atomic_store_8(&pTask->status, TASK_STATUS__IDLE);
+      break;
+    } else if (execStatus == TASK_STATUS__CLOSING) {
+      continue;
+    } else if (execStatus == TASK_STATUS__EXECUTING) {
+      break;
+    } else {
+      ASSERT(0);
+    }
+  }
+  return 0;
+FAIL:
+  atomic_store_8(&pTask->status, TASK_STATUS__IDLE);
+  return -1;
+}
+
+int32_t streamTaskDispatchDown(SStreamTask* pTask, SMsgCb* pMsgCb) {
+  //
+  return 0;
+}
+
+int32_t streamTaskSink(SStreamTask* pTask) {
   //
   return 0;
 }
@@ -156,8 +293,8 @@ int32_t streamTaskProcessInputReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDat
   // 2.3. closing: keep trying
   while (1) {
     int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING);
-    void*  exec = pTask->exec.runners[0].executor;
     if (execStatus == TASK_STATUS__IDLE) {
+      void*         exec = pTask->exec.runners[0].executor;
       SArray*       pRes = taosArrayInit(0, sizeof(void*));
       const SArray* blocks = pBlock->blocks;
       qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK);
@@ -278,7 +415,7 @@ int32_t streamTaskProcessRecoverReq(SStreamTask* pTask, char* msg) {
   return 0;
 }
 
-int32_t streamTaskExecNew(SStreamTask* pTask) {
+int32_t streamTaskRun(SStreamTask* pTask) {
   SArray* pRes = NULL;
   if (pTask->execType == TASK_EXEC__PIPE || pTask->execType == TASK_EXEC__MERGE) {
     // TODO remove multi runner
@@ -494,11 +631,16 @@ SStreamTask* tNewSStreamTask(int64_t streamId) {
 
   pTask->inputQ = taosOpenQueue();
   pTask->outputQ = taosOpenQueue();
-  if (pTask->inputQ == NULL || pTask->outputQ == NULL) goto FAIL;
+  pTask->inputQAll = taosAllocateQall();
+  pTask->outputQAll = taosAllocateQall();
+  if (pTask->inputQ == NULL || pTask->outputQ == NULL || pTask->inputQAll == NULL || pTask->outputQAll == NULL)
+    goto FAIL;
   return pTask;
 FAIL:
   if (pTask->inputQ) taosCloseQueue(pTask->inputQ);
   if (pTask->outputQ) taosCloseQueue(pTask->outputQ);
+  if (pTask->inputQAll) taosFreeQall(pTask->inputQAll);
+  if (pTask->outputQAll) taosFreeQall(pTask->outputQAll);
   if (pTask) taosMemoryFree(pTask);
   return NULL;
 }
@@ -507,6 +649,7 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
   /*if (tStartEncode(pEncoder) < 0) return -1;*/
   if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1;
   if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1;
+  if (tEncodeI8(pEncoder, pTask->inputType) < 0) return -1;
   if (tEncodeI8(pEncoder, pTask->status) < 0) return -1;
   if (tEncodeI8(pEncoder, pTask->sourceType) < 0) return -1;
   if (tEncodeI8(pEncoder, pTask->execType) < 0) return -1;
@@ -552,6 +695,7 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
   /*if (tStartDecode(pDecoder) < 0) return -1;*/
   if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1;
   if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pTask->inputType) < 0) return -1;
   if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1;
   if (tDecodeI8(pDecoder, &pTask->sourceType) < 0) return -1;
   if (tDecodeI8(pDecoder, &pTask->execType) < 0) return -1;

source/libs/sync/inc/syncIndexMgr.h

@@ -35,6 +35,7 @@ typedef struct SSyncIndexMgr {
 } SSyncIndexMgr;
 
 SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode);
+void           syncIndexMgrUpdate(SSyncIndexMgr *pSyncIndexMgr, SSyncNode *pSyncNode);
 void           syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr);
 void           syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr);
 void           syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncIndex index);

source/libs/sync/inc/syncInt.h

@@ -247,6 +247,7 @@ typedef struct SSyncNode {
 // open/close --------------
 SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo);
 void       syncNodeStart(SSyncNode* pSyncNode);
+void       syncNodeStartStandBy(SSyncNode* pSyncNode);
 void       syncNodeClose(SSyncNode* pSyncNode);
 
 // ping --------------
@@ -271,7 +272,7 @@ int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, S
 cJSON*  syncNode2Json(const SSyncNode* pSyncNode);
 char*   syncNode2Str(const SSyncNode* pSyncNode);
 char*   syncNode2SimpleStr(const SSyncNode* pSyncNode);
-void    syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg *newConfig);
+void    syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig);
 
 SSyncNode* syncNodeAcquire(int64_t rid);
 void       syncNodeRelease(SSyncNode* pNode);

source/libs/sync/inc/syncUtil.h

@@ -62,7 +62,6 @@ bool        syncUtilUserPreCommit(tmsg_t msgType);
 bool        syncUtilUserCommit(tmsg_t msgType);
 bool        syncUtilUserRollback(tmsg_t msgType);
 
-
 #ifdef __cplusplus
 }
 #endif

source/libs/sync/src/syncAppendEntries.c

@@ -15,11 +15,11 @@
 
 #include "syncAppendEntries.h"
 #include "syncInt.h"
+#include "syncRaftCfg.h"
 #include "syncRaftLog.h"
 #include "syncRaftStore.h"
 #include "syncUtil.h"
 #include "syncVoteMgr.h"
-#include "syncRaftCfg.h"
 
 // TLA+ Spec
 // HandleAppendEntriesRequest(i, j, m) ==
@@ -200,7 +200,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
             SSyncRaftEntry* pRollBackEntry = logStoreGetEntry(ths->pLogStore, index);
             assert(pRollBackEntry != NULL);
 
-            //if (pRollBackEntry->msgType != TDMT_VND_SYNC_NOOP) {
+            // if (pRollBackEntry->msgType != TDMT_VND_SYNC_NOOP) {
             if (syncUtilUserRollback(pRollBackEntry->msgType)) {
               SRpcMsg rpcMsg;
               syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg);
@@ -229,7 +229,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
         SRpcMsg rpcMsg;
         syncEntry2OriginalRpc(pAppendEntry, &rpcMsg);
         if (ths->pFsm != NULL) {
-          //if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+          // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
           if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) {
             SFsmCbMeta cbMeta;
             cbMeta.index = pAppendEntry->index;
@@ -261,7 +261,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
       SRpcMsg rpcMsg;
       syncEntry2OriginalRpc(pAppendEntry, &rpcMsg);
       if (ths->pFsm != NULL) {
-        //if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+        // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
         if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) {
           SFsmCbMeta cbMeta;
           cbMeta.index = pAppendEntry->index;
@@ -324,7 +324,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
               SRpcMsg rpcMsg;
               syncEntry2OriginalRpc(pEntry, &rpcMsg);
 
-              //if (ths->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+              // if (ths->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
               if (ths->pFsm->FpCommitCb != NULL && syncUtilUserCommit(pEntry->originalRpcType)) {
                 SFsmCbMeta cbMeta;
                 cbMeta.index = pEntry->index;
@@ -338,10 +338,15 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
               // config change
               if (pEntry->originalRpcType == TDMT_VND_SYNC_CONFIG_CHANGE) {
                 SSyncCfg newSyncCfg;
-                int32_t ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg);
+                int32_t  ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg);
                 ASSERT(ret == 0);
 
                 syncNodeUpdateConfig(ths, &newSyncCfg);
+                if (ths->state == TAOS_SYNC_STATE_LEADER) {
+                  syncNodeBecomeLeader(ths);
+                } else {
+                  syncNodeBecomeFollower(ths);
+                }
               }
 
               rpcFreeCont(rpcMsg.pCont);

source/libs/sync/src/syncCommit.c

@@ -16,10 +16,10 @@
 #include "syncCommit.h"
 #include "syncIndexMgr.h"
 #include "syncInt.h"
+#include "syncRaftCfg.h"
 #include "syncRaftLog.h"
 #include "syncRaftStore.h"
 #include "syncUtil.h"
-#include "syncRaftCfg.h"
 
 // \* Leader i advances its commitIndex.
 // \* This is done as a separate step from handling AppendEntries responses,
@@ -102,7 +102,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
           SRpcMsg rpcMsg;
           syncEntry2OriginalRpc(pEntry, &rpcMsg);
 
-          //if (pSyncNode->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+          // if (pSyncNode->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
           if (pSyncNode->pFsm->FpCommitCb != NULL && syncUtilUserCommit(pEntry->originalRpcType)) {
             SFsmCbMeta cbMeta;
             cbMeta.index = pEntry->index;
@@ -114,12 +114,17 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
           }
 
           // config change
-            if (pEntry->originalRpcType == TDMT_VND_SYNC_CONFIG_CHANGE) {
+          if (pEntry->originalRpcType == TDMT_VND_SYNC_CONFIG_CHANGE) {
-              SSyncCfg newSyncCfg;
+            SSyncCfg newSyncCfg;
-              int32_t ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg);
+            int32_t  ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg);
-              ASSERT(ret == 0);
+            ASSERT(ret == 0);
 
-              syncNodeUpdateConfig(pSyncNode, &newSyncCfg);
+            syncNodeUpdateConfig(pSyncNode, &newSyncCfg);
+            if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
+              syncNodeBecomeLeader(pSyncNode);
+            } else {
+              syncNodeBecomeFollower(pSyncNode);
+            }
           }
 
           rpcFreeCont(rpcMsg.pCont);

source/libs/sync/src/syncIO.c

@@ -15,6 +15,7 @@
 
 #include "syncIO.h"
 #include <tdatablock.h>
+#include "os.h"
 #include "syncMessage.h"
 #include "syncUtil.h"
 #include "tglobal.h"
@@ -198,6 +199,7 @@ static int32_t syncIOStartInternal(SSyncIO *io) {
   {
     SRpcInit rpcInit;
     memset(&rpcInit, 0, sizeof(rpcInit));
+    snprintf(rpcInit.localFqdn, sizeof(rpcInit.localFqdn), "%s", "127.0.0.1");
     rpcInit.localPort = io->myAddr.eps[0].port;
     rpcInit.label = "SYNC-IO-SERVER";
     rpcInit.numOfThreads = 1;

source/libs/sync/src/syncIndexMgr.c

@@ -31,6 +31,13 @@ SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode) {
   return pSyncIndexMgr;
 }
 
+void syncIndexMgrUpdate(SSyncIndexMgr *pSyncIndexMgr, SSyncNode *pSyncNode) {
+  pSyncIndexMgr->replicas = &(pSyncNode->replicasId);
+  pSyncIndexMgr->replicaNum = pSyncNode->replicaNum;
+  pSyncIndexMgr->pSyncNode = pSyncNode;
+  syncIndexMgrClear(pSyncIndexMgr);
+}
+
 void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr) {
   if (pSyncIndexMgr != NULL) {
     taosMemoryFree(pSyncIndexMgr);

source/libs/sync/src/syncMain.c

@@ -103,6 +103,16 @@ void syncStart(int64_t rid) {
   taosReleaseRef(tsNodeRefId, pSyncNode->rid);
 }
 
+void syncStartStandBy(int64_t rid) {
+  SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid);
+  if (pSyncNode == NULL) {
+    return;
+  }
+  syncNodeStartStandBy(pSyncNode);
+
+  taosReleaseRef(tsNodeRefId, pSyncNode->rid);
+}
+
 void syncStop(int64_t rid) {
   SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid);
   if (pSyncNode == NULL) {
@@ -116,7 +126,7 @@ void syncStop(int64_t rid) {
 
 int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) {
   int32_t ret = 0;
-  char *configChange = syncCfg2Str((SSyncCfg*)pSyncCfg);
+  char*   configChange = syncCfg2Str((SSyncCfg*)pSyncCfg);
   SRpcMsg rpcMsg = {0};
   rpcMsg.msgType = TDMT_VND_SYNC_CONFIG_CHANGE;
   rpcMsg.noResp = 1;
@@ -188,10 +198,9 @@ void syncGetEpSet(int64_t rid, SEpSet* pEpSet) {
     (pEpSet->numOfEps)++;
 
     sInfo("syncGetEpSet index:%d %s:%d", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port);
-
   }
   pEpSet->inUse = pSyncNode->pRaftCfg->cfg.myIndex;
-  
+
   sInfo("syncGetEpSet pEpSet->inUse:%d ", pEpSet->inUse);
 
   taosReleaseRef(tsNodeRefId, pSyncNode->rid);
@@ -524,6 +533,17 @@ void syncNodeStart(SSyncNode* pSyncNode) {
   assert(ret == 0);
 }
 
+void syncNodeStartStandBy(SSyncNode* pSyncNode) {
+  // state change
+  pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER;
+  syncNodeStopHeartbeatTimer(pSyncNode);
+
+  // reset elect timer, long enough
+  int32_t electMS = TIMER_MAX_MS;
+  int32_t ret = syncNodeRestartElectTimer(pSyncNode, electMS);
+  ASSERT(ret == 0);
+}
+
 void syncNodeClose(SSyncNode* pSyncNode) {
   int32_t ret;
   assert(pSyncNode != NULL);
@@ -858,7 +878,7 @@ char* syncNode2SimpleStr(const SSyncNode* pSyncNode) {
   return s;
 }
 
-void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg *newConfig) {
+void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig) {
   pSyncNode->pRaftCfg->cfg = *newConfig;
   int32_t ret = raftCfgPersist(pSyncNode->pRaftCfg);
   ASSERT(ret == 0);
@@ -885,6 +905,11 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg *newConfig) {
   for (int i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) {
     syncUtilnodeInfo2raftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]);
   }
+
+  syncIndexMgrUpdate(pSyncNode->pNextIndex, pSyncNode);
+  syncIndexMgrUpdate(pSyncNode->pMatchIndex, pSyncNode);
+
+  syncNodeLog2("==syncNodeUpdateConfig==", pSyncNode);
 }
 
 SSyncNode* syncNodeAcquire(int64_t rid) {
@@ -1245,7 +1270,7 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) {
     syncEntry2OriginalRpc(pEntry, &rpcMsg);
 
     if (ths->pFsm != NULL) {
-      //if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+      // if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
       if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) {
         SFsmCbMeta cbMeta;
         cbMeta.index = pEntry->index;
@@ -1267,7 +1292,7 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) {
     syncEntry2OriginalRpc(pEntry, &rpcMsg);
 
     if (ths->pFsm != NULL) {
-      //if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+      // if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
       if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) {
         SFsmCbMeta cbMeta;
         cbMeta.index = pEntry->index;

source/libs/sync/src/syncRaftLog.c

@@ -58,14 +58,15 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
   syncMeta.term = pEntry->term;
   code = walWriteWithSyncInfo(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen);
   if (code != 0) {
-    int32_t err = terrno;
+    int32_t     err = terrno;
-    const char *errStr = tstrerror(err);
+    const char* errStr = tstrerror(err);
-    int32_t linuxErr = errno;
+    int32_t     linuxErr = errno;
-    const char *linuxErrMsg = strerror(errno);
+    const char* linuxErrMsg = strerror(errno);
-    sError("walWriteWithSyncInfo error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg);
+    sError("walWriteWithSyncInfo error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr,
+           linuxErrMsg);
     ASSERT(0);
-  }    
+  }
-  //assert(code == 0);
+  // assert(code == 0);
 
   walFsync(pWal, true);
   return code;
@@ -77,16 +78,17 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
 
   if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) {
     SWalReadHandle* pWalHandle = walOpenReadHandle(pWal);
-    int32_t code = walReadWithHandle(pWalHandle, index);
+    int32_t         code = walReadWithHandle(pWalHandle, index);
     if (code != 0) {
-      int32_t err = terrno;
+      int32_t     err = terrno;
-      const char *errStr = tstrerror(err);
+      const char* errStr = tstrerror(err);
-      int32_t linuxErr = errno;
+      int32_t     linuxErr = errno;
-      const char *linuxErrMsg = strerror(errno);
+      const char* linuxErrMsg = strerror(errno);
-      sError("walReadWithHandle error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg);
+      sError("walReadWithHandle error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr,
+             linuxErrMsg);
       ASSERT(0);
-    }    
+    }
-    //assert(walReadWithHandle(pWalHandle, index) == 0);
+    // assert(walReadWithHandle(pWalHandle, index) == 0);
 
     SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen);
     assert(pEntry != NULL);
@@ -112,16 +114,17 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
 int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) {
   SSyncLogStoreData* pData = pLogStore->data;
   SWal*              pWal = pData->pWal;
-  //assert(walRollback(pWal, fromIndex) == 0);
+  // assert(walRollback(pWal, fromIndex) == 0);
   int32_t code = walRollback(pWal, fromIndex);
   if (code != 0) {
-    int32_t err = terrno;
+    int32_t     err = terrno;
-    const char *errStr = tstrerror(err);
+    const char* errStr = tstrerror(err);
-    int32_t linuxErr = errno;
+    int32_t     linuxErr = errno;
-    const char *linuxErrMsg = strerror(errno);
+    const char* linuxErrMsg = strerror(errno);
-    sError("walRollback error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg);
+    sError("walRollback error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr,
+           linuxErrMsg);
     ASSERT(0);
-  } 
+  }
   return 0;  // to avoid compiler error
 }
 
@@ -145,16 +148,16 @@ SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore) {
 int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) {
   SSyncLogStoreData* pData = pLogStore->data;
   SWal*              pWal = pData->pWal;
-  //assert(walCommit(pWal, index) == 0);
+  // assert(walCommit(pWal, index) == 0);
   int32_t code = walCommit(pWal, index);
   if (code != 0) {
-    int32_t err = terrno;
+    int32_t     err = terrno;
-    const char *errStr = tstrerror(err);
+    const char* errStr = tstrerror(err);
-    int32_t linuxErr = errno;
+    int32_t     linuxErr = errno;
-    const char *linuxErrMsg = strerror(errno);
+    const char* linuxErrMsg = strerror(errno);
     sError("walCommit error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg);
     ASSERT(0);
-  } 
+  }
   return 0;  // to avoid compiler error
 }
 

source/libs/sync/test/CMakeLists.txt

@@ -37,6 +37,7 @@ add_executable(syncRaftCfgTest "")
 add_executable(syncRespMgrTest "")
 add_executable(syncSnapshotTest "")
 add_executable(syncApplyMsgTest "")
+add_executable(syncConfigChangeTest "")
 
 
 target_sources(syncTest
@@ -195,6 +196,10 @@ target_sources(syncApplyMsgTest
     PRIVATE
     "syncApplyMsgTest.cpp"
 )
+target_sources(syncConfigChangeTest
+    PRIVATE
+    "syncConfigChangeTest.cpp"
+)
 
 
 target_include_directories(syncTest
@@ -392,6 +397,11 @@ target_include_directories(syncApplyMsgTest
     "${TD_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
+target_include_directories(syncConfigChangeTest
+    PUBLIC
+    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
 
 
 target_link_libraries(syncTest
@@ -550,6 +560,10 @@ target_link_libraries(syncApplyMsgTest
     sync
     gtest_main
 )
+target_link_libraries(syncConfigChangeTest
+    sync
+    gtest_main
+)
 
 
 enable_testing()

source/libs/sync/test/syncConfigChangeTest.cpp

@@ -0,0 +1,259 @@
+#include <gtest/gtest.h>
+#include <stdio.h>
+#include "os.h"
+#include "syncEnv.h"
+#include "syncIO.h"
+#include "syncInt.h"
+#include "syncUtil.h"
+#include "wal.h"
+
+void logTest() {
+  sTrace("--- sync log test: trace");
+  sDebug("--- sync log test: debug");
+  sInfo("--- sync log test: info");
+  sWarn("--- sync log test: warn");
+  sError("--- sync log test: error");
+  sFatal("--- sync log test: fatal");
+}
+
+uint16_t    gPorts[] = {7010, 7110, 7210, 7310, 7410};
+const char* gDir = "./syncReplicateTest";
+int32_t     gVgId = 1234;
+SyncIndex   gSnapshotLastApplyIndex;
+
+void init() {
+  int code = walInit();
+  assert(code == 0);
+
+  code = syncInit();
+  assert(code == 0);
+
+  sprintf(tsTempDir, "%s", ".");
+}
+
+void cleanup() { walCleanUp(); }
+
+void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {
+  SyncIndex beginIndex = SYNC_INDEX_INVALID;
+  if (pFsm->FpGetSnapshot != NULL) {
+    SSnapshot snapshot;
+    pFsm->FpGetSnapshot(pFsm, &snapshot);
+    beginIndex = snapshot.lastApplyIndex;
+  }
+
+  if (cbMeta.index > beginIndex) {
+    char logBuf[256];
+    snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n",
+             pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state));
+    syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg);
+  } else {
+    sTrace("==callback== ==CommitCb== do not apply again %ld", cbMeta.index);
+  }
+}
+
+void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {
+  char logBuf[256];
+  snprintf(logBuf, sizeof(logBuf),
+           "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index,
+           cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state));
+  syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg);
+}
+
+void RollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {
+  char logBuf[256];
+  snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n",
+           pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state));
+  syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg);
+}
+
+int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) {
+  pSnapshot->data = NULL;
+  pSnapshot->lastApplyIndex = gSnapshotLastApplyIndex;
+  pSnapshot->lastApplyTerm = 100;
+  return 0;
+}
+
+SSyncFSM* createFsm() {
+  SSyncFSM* pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM));
+  pFsm->FpCommitCb = CommitCb;
+  pFsm->FpPreCommitCb = PreCommitCb;
+  pFsm->FpRollBackCb = RollBackCb;
+  pFsm->FpGetSnapshot = GetSnapshotCb;
+  return pFsm;
+}
+
+SWal* createWal(char* path, int32_t vgId) {
+  SWalCfg walCfg;
+  memset(&walCfg, 0, sizeof(SWalCfg));
+  walCfg.vgId = vgId;
+  walCfg.fsyncPeriod = 1000;
+  walCfg.retentionPeriod = 1000;
+  walCfg.rollPeriod = 1000;
+  walCfg.retentionSize = 1000;
+  walCfg.segSize = 1000;
+  walCfg.level = TAOS_WAL_FSYNC;
+  SWal* pWal = walOpen(path, &walCfg);
+  assert(pWal != NULL);
+  return pWal;
+}
+
+int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* pWal, char* path, bool isStandBy) {
+  SSyncInfo syncInfo;
+  syncInfo.vgId = vgId;
+  syncInfo.rpcClient = gSyncIO->clientRpc;
+  syncInfo.FpSendMsg = syncIOSendMsg;
+  syncInfo.queue = gSyncIO->pMsgQ;
+  syncInfo.FpEqMsg = syncIOEqMsg;
+  syncInfo.pFsm = createFsm();
+  snprintf(syncInfo.path, sizeof(syncInfo.path), "%s_sync_replica%d_index%d", path, replicaNum, myIndex);
+  syncInfo.pWal = pWal;
+
+  SSyncCfg* pCfg = &syncInfo.syncCfg;
+
+  if (isStandBy) {
+    pCfg->myIndex = 0;
+    pCfg->replicaNum = 1;
+    pCfg->nodeInfo[0].nodePort = gPorts[myIndex];
+    taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
+
+  } else {
+    pCfg->myIndex = myIndex;
+    pCfg->replicaNum = replicaNum;
+
+    for (int i = 0; i < replicaNum; ++i) {
+      pCfg->nodeInfo[i].nodePort = gPorts[i];
+      taosGetFqdn(pCfg->nodeInfo[i].nodeFqdn);
+      // snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1");
+    }
+  }
+
+  int64_t rid = syncOpen(&syncInfo);
+  assert(rid > 0);
+
+  SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid);
+  assert(pSyncNode != NULL);
+  gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
+  gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
+  gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote;
+  gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply;
+  gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries;
+  gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply;
+  gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
+  gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
+  gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout;
+  gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest;
+  gSyncIO->pSyncNode = pSyncNode;
+  syncNodeRelease(pSyncNode);
+
+  return rid;
+}
+
+void configChange(int64_t rid, int32_t replicaNum, int32_t myIndex) {
+  SSyncCfg syncCfg;
+
+  syncCfg.myIndex = myIndex;
+  syncCfg.replicaNum = replicaNum;
+
+  for (int i = 0; i < replicaNum; ++i) {
+    syncCfg.nodeInfo[i].nodePort = gPorts[i];
+    taosGetFqdn(syncCfg.nodeInfo[i].nodeFqdn);
+  }
+
+  syncReconfig(rid, &syncCfg);
+}
+
+void usage(char* exe) {
+  printf("usage: %s replicaNum myIndex lastApplyIndex writeRecordNum isStandBy isConfigChange \n", exe);
+}
+
+SRpcMsg* createRpcMsg(int i, int count, int myIndex) {
+  SRpcMsg* pMsg = (SRpcMsg*)taosMemoryMalloc(sizeof(SRpcMsg));
+  memset(pMsg, 0, sizeof(SRpcMsg));
+  pMsg->msgType = 9999;
+  pMsg->contLen = 256;
+  pMsg->pCont = rpcMallocCont(pMsg->contLen);
+  snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-%ld", myIndex, i, count, taosGetTimestampMs());
+  return pMsg;
+}
+
+int main(int argc, char** argv) {
+  tsAsyncLog = 0;
+  sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE;
+  if (argc != 7) {
+    usage(argv[0]);
+    exit(-1);
+  }
+
+  int32_t replicaNum = atoi(argv[1]);
+  int32_t myIndex = atoi(argv[2]);
+  int32_t lastApplyIndex = atoi(argv[3]);
+  int32_t writeRecordNum = atoi(argv[4]);
+  bool    isStandBy = atoi(argv[5]);
+  bool    isConfigChange = atoi(argv[6]);
+  gSnapshotLastApplyIndex = lastApplyIndex;
+
+  if (!isStandBy) {
+    assert(replicaNum >= 1 && replicaNum <= 5);
+    assert(myIndex >= 0 && myIndex < replicaNum);
+    assert(lastApplyIndex >= -1);
+    assert(writeRecordNum >= 0);
+  }
+
+  init();
+  int32_t ret = syncIOStart((char*)"127.0.0.1", gPorts[myIndex]);
+  assert(ret == 0);
+
+  char walPath[128];
+  snprintf(walPath, sizeof(walPath), "%s_wal_replica%d_index%d", gDir, replicaNum, myIndex);
+  SWal* pWal = createWal(walPath, gVgId);
+
+  int64_t rid = createSyncNode(replicaNum, myIndex, gVgId, pWal, (char*)gDir, isStandBy);
+  assert(rid > 0);
+
+  if (isStandBy) {
+    syncStartStandBy(rid);
+  } else {
+    syncStart(rid);
+  }
+
+  SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid);
+  assert(pSyncNode != NULL);
+
+  if (isConfigChange) {
+    configChange(rid, 3, myIndex);
+  }
+
+  //---------------------------
+  int32_t alreadySend = 0;
+  while (1) {
+    char* s = syncNode2SimpleStr(pSyncNode);
+
+    if (alreadySend < writeRecordNum) {
+      SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex);
+      int32_t  ret = syncPropose(rid, pRpcMsg, false);
+      if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) {
+        sTrace("%s value%d write not leader", s, alreadySend);
+      } else {
+        assert(ret == 0);
+        sTrace("%s value%d write ok", s, alreadySend);
+      }
+      alreadySend++;
+
+      rpcFreeCont(pRpcMsg->pCont);
+      taosMemoryFree(pRpcMsg);
+    } else {
+      sTrace("%s", s);
+    }
+
+    taosMsleep(1000);
+    taosMemoryFree(s);
+    taosMsleep(1000);
+  }
+
+  syncNodeRelease(pSyncNode);
+  syncStop(rid);
+  walClose(pWal);
+  syncIOStop();
+  cleanup();
+  return 0;
+}

source/libs/transport/src/transSrv.c

@@ -137,7 +137,9 @@ static void destroySmsg(SSrvMsg* smsg);
 // check whether already read complete packet
 static SSrvConn* createConn(void* hThrd);
 static void      destroyConn(SSrvConn* conn, bool clear /*clear handle or not*/);
-static int       reallocConnRefHandle(SSrvConn* conn);
+static void      destroyConnRegArg(SSrvConn* conn);
+
+static int reallocConnRefHandle(SSrvConn* conn);
 
 static void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd);
 static void uvHandleRelease(SSrvMsg* msg, SWorkThrdObj* thrd);
@@ -429,6 +431,8 @@ static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) {
     if (smsg->type == Release) {
       pHead->msgType = 0;
       pConn->status = ConnNormal;
+
+      destroyConnRegArg(pConn);
       transUnrefSrvHandle(pConn);
     } else {
       pHead->msgType = pMsg->msgType;
@@ -800,6 +804,12 @@ static void destroyConn(SSrvConn* conn, bool clear) {
     // uv_shutdown(req, (uv_stream_t*)conn->pTcp, uvShutDownCb);
   }
 }
+static void destroyConnRegArg(SSrvConn* conn) {
+  if (conn->regArg.init == 1) {
+    transFreeMsg(conn->regArg.msg.pCont);
+    conn->regArg.init = 0;
+  }
+}
 static int reallocConnRefHandle(SSrvConn* conn) {
   uvReleaseExHandle(conn->refId);
   uvRemoveExHandle(conn->refId);
@@ -827,16 +837,9 @@ static void uvDestroyConn(uv_handle_t* handle) {
   // uv_timer_stop(&conn->pTimer);
   transQueueDestroy(&conn->srvMsgs);
 
-  if (conn->regArg.init == 1) {
-    transFreeMsg(conn->regArg.msg.pCont);
-    conn->regArg.init = 0;
-  }
   QUEUE_REMOVE(&conn->queue);
   taosMemoryFree(conn->pTcp);
-  if (conn->regArg.init == 1) {
+  destroyConnRegArg(conn);
-    transFreeMsg(conn->regArg.msg.pCont);
-    conn->regArg.init = 0;
-  }
   taosMemoryFree(conn);
 
   if (thrd->quit && QUEUE_IS_EMPTY(&thrd->conn)) {

source/util/src/terror.c

@@ -322,6 +322,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_TB_NOT_EXIST,             "Table not exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_SMA_NOT_EXIST,            "SMA not exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_HASH_MISMATCH,            "Hash value mismatch")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_TABLE_NOT_EXIST,          "Table does not exists")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_TABLE_ACTION,     "Invalid table action")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_ALREADY_EXISTS,       "Table column already exists")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_TABLE_COL_NOT_EXISTS,     "Table column not exists")
+
 
 // tsdb
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID,         "Invalid table ID")
@@ -354,6 +358,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECREATED,          "Table re-created")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR,       "TDB env open error")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_SMA_INDEX_IN_META,     "No sma index in meta")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_SMA_STAT,         "Invalid sma state")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TSMA_ALREADY_EXIST,       "Tsma already exists")
 
 
 // query

source/util/src/tlog.c

@@ -94,6 +94,7 @@ int32_t tqDebugFlag = 135;
 int32_t fsDebugFlag = 135;
 int32_t metaDebugFlag = 135;
 int32_t fnDebugFlag = 135;
+int32_t smaDebugFlag = 135;
 
 int64_t dbgEmptyW = 0;
 int64_t dbgWN = 0;
@@ -755,6 +756,7 @@ void taosSetAllDebugFlag(int32_t flag) {
   tqDebugFlag = flag;
   fsDebugFlag = flag;
   fnDebugFlag = flag;
+  smaDebugFlag = flag;
 
   uInfo("all debug flag are set to %d", flag);
 }

source/util/test/CMakeLists.txt

@@ -33,13 +33,13 @@ ENDIF()
 
 INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/src/util/inc)
 
-# freelistTest
+# # freelistTest
-add_executable(freelistTest "")
+# add_executable(freelistTest "")
-target_sources(freelistTest
+# target_sources(freelistTest
-    PRIVATE
+#     PRIVATE
-    "freelistTest.cpp"
+#     "freelistTest.cpp"
-)
+# )
-target_link_libraries(freelistTest os util gtest gtest_main)
+# target_link_libraries(freelistTest os util gtest gtest_main)
 
 # # encodeTest
 # add_executable(encodeTest "encodeTest.cpp")

source/util/test/freelistTest.cpp

@@ -1,17 +0,0 @@
-#include <gtest/gtest.h>
-
-#include "tfreelist.h"
-
-TEST(TD_UTIL_FREELIST_TEST, simple_test) {
-  SFreeList fl;
-
-  tFreeListInit(&fl);
-
-  for (size_t i = 0; i < 1000; i++) {
-    void *ptr = NULL;
-    TFL_MALLOC(ptr, void*, 1024, &fl);
-    GTEST_ASSERT_NE(ptr, nullptr);
-  }
-
-  tFreeListClear(&fl);
-}