Merge branch '3.0' into enh/killQuery

Jenkinsfile2

@@ -127,6 +127,25 @@ def pre_test(){
     '''
     return 1
 }
+def pre_test_build_mac() {
+    sh '''
+        hostname
+        date
+    '''
+    sh '''
+        cd ${WK}
+        rm -rf debug
+        mkdir debug
+    '''
+    sh '''
+        cd ${WK}/debug
+        cmake ..
+        make -j8
+    '''
+    sh '''
+        date
+    '''
+}
 def pre_test_win(){
     bat '''
         hostname
@@ -334,6 +353,17 @@ pipeline {
                         }
                     }
                 }
+                stage('mac test') {
+                    agent{label " Mac_catalina "}
+                    steps {
+                        catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
+                            timeout(time: 20, unit: 'MINUTES'){
+                                pre_test()
+                                pre_test_build_mac()
+                            }
+                        }
+                    }
+                }
                 stage('linux test') {
                     agent{label " worker03 || slave215 || slave217 || slave219 "}
                     options { skipDefaultCheckout() } 

cmake/cmake.define

@@ -98,12 +98,12 @@ ELSE ()
     ENDIF ()
 
     IF (${SANITIZER} MATCHES "true")
-        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}     -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3")
+        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}     -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
-        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3")
+        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
 	MESSAGE(STATUS "Will compile with Address Sanitizer!")
     ELSE ()
-        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3")
+        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=0")
-        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3")
+        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=0")
     ENDIF ()
 
     MESSAGE("System processor ID: ${CMAKE_SYSTEM_PROCESSOR}")

include/common/tcommon.h

@@ -56,7 +56,8 @@ typedef enum EStreamType {
   STREAM_CLEAR,
   STREAM_INVALID,
   STREAM_GET_ALL,
-  STREAM_DELETE,
+  STREAM_DELETE_RESULT,
+  STREAM_DELETE_DATA,
   STREAM_RETRIEVE,
   STREAM_PULL_DATA,
   STREAM_PULL_OVER,

include/libs/nodes/nodes.h

@@ -22,8 +22,8 @@ extern "C" {
 
 #include "tdef.h"
 
-#define nodeType(nodeptr)          (((const SNode*)(nodeptr))->type)
+#define nodeType(nodeptr)              (((const SNode*)(nodeptr))->type)
-#define setNodeType(nodeptr, type) (((SNode*)(nodeptr))->type = (type))
+#define setNodeType(nodeptr, nodetype) (((SNode*)(nodeptr))->type = (nodetype))
 
 #define LIST_LENGTH(l) (NULL != (l) ? (l)->length : 0)
 
@@ -118,6 +118,7 @@ typedef enum ENodeType {
   QUERY_NODE_DROP_TABLE_STMT,
   QUERY_NODE_DROP_SUPER_TABLE_STMT,
   QUERY_NODE_ALTER_TABLE_STMT,
+  QUERY_NODE_ALTER_SUPER_TABLE_STMT,
   QUERY_NODE_CREATE_USER_STMT,
   QUERY_NODE_ALTER_USER_STMT,
   QUERY_NODE_DROP_USER_STMT,

include/libs/qcom/query.h

@@ -65,7 +65,7 @@ typedef struct SQueryExecRes {
 } SQueryExecRes;
 
 typedef struct SIndexMeta {
-#ifdef WINDOWS
+#if defined(WINDOWS) || defined(_TD_DARWIN_64)
   size_t avoidCompilationErrors;
 #endif
 

include/libs/scalar/scalar.h

@@ -25,6 +25,8 @@ extern "C" {
 
 typedef struct SFilterInfo SFilterInfo;
 
+int32_t scalarGetOperatorResultType(SDataType left, SDataType right, EOperatorType op, SDataType* pRes);
+
 /*
 pNode will be freed in API;
 *pRes need to freed in caller

include/os/os.h

@@ -41,7 +41,6 @@ extern "C" {
 #include <sys/types.h>
 #include <termios.h>
 #include <sys/statvfs.h>
-#include <sys/prctl.h>
 #include <sys/shm.h>
 #include <sys/wait.h>
 

include/os/osAtomic.h

@@ -63,7 +63,7 @@ int8_t  atomic_add_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val);
-void   *atomic_add_fetch_ptr(void *ptr, void *val);
+void   *atomic_add_fetch_ptr(void *ptr, int64_t val);
 int8_t  atomic_fetch_add_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val);
@@ -73,7 +73,7 @@ int8_t  atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val);
-void   *atomic_sub_fetch_ptr(void *ptr, void *val);
+void   *atomic_sub_fetch_ptr(void *ptr, int64_t val);
 int8_t  atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val);

include/os/osMath.h

@@ -20,6 +20,12 @@
 extern "C" {
 #endif
 
+// If the error is in a third-party library, place this header file under the third-party library header file.
+// When you want to use this feature, you should find or add the same function in the following sectio
+#ifndef ALLOW_FORBID_FUNC
+#define qsort  QSORT_FUNC_TAOS_FORBID
+#endif
+
 #define TPOW2(x) ((x) * (x))
 #define TABS(x) ((x) > 0 ? (x) : -(x))
 

include/os/osSemaphore.h

@@ -24,7 +24,9 @@ extern "C" {
 
 #if defined(_TD_DARWIN_64)
 
-typedef struct tsem_s *tsem_t;
+// typedef struct tsem_s *tsem_t;
+typedef struct bosal_sem_t *tsem_t;
+
 
 int tsem_init(tsem_t *sem, int pshared, unsigned int value);
 int tsem_wait(tsem_t *sem);
@@ -51,11 +53,11 @@ int tsem_timewait(tsem_t *sim, int64_t nanosecs);
 //  #define taosThreadRwlockRdlock(lock) taosThreadMutexLock(lock)
 //  #define taosThreadRwlockUnlock(lock) taosThreadMutexUnlock(lock)
 
-#define TdThreadSpinlock               TdThreadMutex
+// #define TdThreadSpinlock               TdThreadMutex
-#define taosThreadSpinInit(lock, NULL) taosThreadMutexInit(lock, NULL)
+// #define taosThreadSpinInit(lock, NULL) taosThreadMutexInit(lock, NULL)
-#define taosThreadSpinDestroy(lock)    taosThreadMutexDestroy(lock)
+// #define taosThreadSpinDestroy(lock)    taosThreadMutexDestroy(lock)
-#define taosThreadSpinLock(lock)       taosThreadMutexLock(lock)
+// #define taosThreadSpinLock(lock)       taosThreadMutexLock(lock)
-#define taosThreadSpinUnlock(lock)     taosThreadMutexUnlock(lock)
+// #define taosThreadSpinUnlock(lock)     taosThreadMutexUnlock(lock)
 #endif
 
 bool    taosCheckPthreadValid(TdThread thread);

include/os/osSocket.h

@@ -64,7 +64,6 @@
 #include <osEok.h>
 #else
 #include <netinet/in.h>
-#include <sys/epoll.h>
 #endif
 #endif
 
@@ -77,15 +76,12 @@ typedef int socklen_t;
 #define TAOS_EPOLL_WAIT_TIME 100
 typedef SOCKET eventfd_t;
 #define eventfd(a, b)      -1
-#define EpollClose(pollFd) epoll_close(pollFd)
 #ifndef EPOLLWAKEUP
 #define EPOLLWAKEUP (1u << 29)
 #endif
 #elif defined(_TD_DARWIN_64)
 #define TAOS_EPOLL_WAIT_TIME 500
 typedef int32_t SOCKET;
-typedef SOCKET  EpollFd;
-#define EpollClose(pollFd)   epoll_close(pollFd)
 #else
 #define TAOS_EPOLL_WAIT_TIME 500
 typedef int32_t SOCKET;
@@ -122,14 +118,6 @@ typedef SOCKET  EpollFd;
 typedef int32_t  SocketFd;
 typedef SocketFd EpollFd;
 
-typedef struct TdSocket {
-#if SOCKET_WITH_LOCK
-  TdThreadRwlock rwlock;
-#endif
-  int      refId;
-  SocketFd fd;
-} * TdSocketPtr, TdSocket;
-
 typedef struct TdSocketServer *TdSocketServerPtr;
 typedef struct TdSocket *      TdSocketPtr;
 typedef struct TdEpoll *       TdEpollPtr;
@@ -181,11 +169,6 @@ void        taosSetMaskSIGPIPE();
 uint32_t    taosInetAddr(const char *ipAddr);
 const char *taosInetNtoa(struct in_addr ipInt);
 
-TdEpollPtr taosCreateEpoll(int32_t size);
-int32_t    taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocket, struct epoll_event *event);
-int32_t    taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxEvents, int32_t timeout);
-int32_t    taosCloseEpoll(TdEpollPtr *ppEpoll);
-
 #ifdef __cplusplus
 }
 #endif

include/os/osString.h

@@ -67,7 +67,7 @@ bool    taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs
 int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes);
 TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4);
 bool    taosValidateEncodec(const char *encodec);
-int32_t taosHexEncode(const char *src, char *dst, int32_t len);
+int32_t taosHexEncode(const unsigned char *src, char *dst, int32_t len);
 int32_t taosHexDecode(const char *src, char *dst, int32_t len);
 
 int32_t taosWcharWidth(TdWchar wchar);

include/os/osThread.h

@@ -188,27 +188,27 @@ int32_t taosThreadJoin(TdThread thread, void **valuePtr);
 int32_t taosThreadKeyCreate(TdThreadKey * key, void(*destructor)(void *));
 int32_t taosThreadKeyDelete(TdThreadKey key);
 int32_t taosThreadKill(TdThread thread, int32_t sig);
-int32_t taosThreadMutexConsistent(TdThreadMutex* mutex);
+// int32_t taosThreadMutexConsistent(TdThreadMutex* mutex);
 int32_t taosThreadMutexDestroy(TdThreadMutex * mutex);
 int32_t taosThreadMutexInit(TdThreadMutex * mutex, const TdThreadMutexAttr * attr);
 int32_t taosThreadMutexLock(TdThreadMutex * mutex);
-int32_t taosThreadMutexTimedLock(TdThreadMutex * mutex, const struct timespec *abstime);
+// int32_t taosThreadMutexTimedLock(TdThreadMutex * mutex, const struct timespec *abstime);
 int32_t taosThreadMutexTryLock(TdThreadMutex * mutex);
 int32_t taosThreadMutexUnlock(TdThreadMutex * mutex);
 int32_t taosThreadMutexAttrDestroy(TdThreadMutexAttr * attr);
 int32_t taosThreadMutexAttrGetPshared(const TdThreadMutexAttr * attr, int32_t *pshared);
-int32_t taosThreadMutexAttrGetRobust(const TdThreadMutexAttr * attr, int32_t * robust);
+// int32_t taosThreadMutexAttrGetRobust(const TdThreadMutexAttr * attr, int32_t * robust);
 int32_t taosThreadMutexAttrGetType(const TdThreadMutexAttr * attr, int32_t *kind);
 int32_t taosThreadMutexAttrInit(TdThreadMutexAttr * attr);
 int32_t taosThreadMutexAttrSetPshared(TdThreadMutexAttr * attr, int32_t pshared);
-int32_t taosThreadMutexAttrSetRobust(TdThreadMutexAttr * attr, int32_t robust);
+// int32_t taosThreadMutexAttrSetRobust(TdThreadMutexAttr * attr, int32_t robust);
 int32_t taosThreadMutexAttrSetType(TdThreadMutexAttr * attr, int32_t kind);
 int32_t taosThreadOnce(TdThreadOnce * onceControl, void(*initRoutine)(void));
 int32_t taosThreadRwlockDestroy(TdThreadRwlock * rwlock);
 int32_t taosThreadRwlockInit(TdThreadRwlock * rwlock, const TdThreadRwlockAttr * attr);
 int32_t taosThreadRwlockRdlock(TdThreadRwlock * rwlock);
-int32_t taosThreadRwlockTimedRdlock(TdThreadRwlock * rwlock, const struct timespec *abstime);
+// int32_t taosThreadRwlockTimedRdlock(TdThreadRwlock * rwlock, const struct timespec *abstime);
-int32_t taosThreadRwlockTimedWrlock(TdThreadRwlock * rwlock, const struct timespec *abstime);
+// int32_t taosThreadRwlockTimedWrlock(TdThreadRwlock * rwlock, const struct timespec *abstime);
 int32_t taosThreadRwlockTryRdlock(TdThreadRwlock * rwlock);
 int32_t taosThreadRwlockTryWrlock(TdThreadRwlock * rwlock);
 int32_t taosThreadRwlockUnlock(TdThreadRwlock * rwlock);

include/util/taoserror.h

@@ -578,6 +578,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INVALID_TABLE_OPTION      TAOS_DEF_ERROR_CODE(0, 0x265C)
 #define TSDB_CODE_PAR_INVALID_INTERP_CLAUSE     TAOS_DEF_ERROR_CODE(0, 0x265D)
 #define TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN      TAOS_DEF_ERROR_CODE(0, 0x265E)
+#define TSDB_CODE_PAR_ONLY_SUPPORT_SINGLE_TABLE TAOS_DEF_ERROR_CODE(0, 0x265F)
 
 //planner
 #define TSDB_CODE_PLAN_INTERNAL_ERROR           TAOS_DEF_ERROR_CODE(0, 0x2700)
@@ -623,9 +624,12 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_RSMA_INVALID_ENV               TAOS_DEF_ERROR_CODE(0, 0x3150)
 #define TSDB_CODE_RSMA_INVALID_STAT              TAOS_DEF_ERROR_CODE(0, 0x3151)
 #define TSDB_CODE_RSMA_QTASKINFO_CREATE          TAOS_DEF_ERROR_CODE(0, 0x3152)
+#define TSDB_CODE_RSMA_FILE_CORRUPTED            TAOS_DEF_ERROR_CODE(0, 0x3153)
 
 //index
 #define TSDB_CODE_INDEX_REBUILDING               TAOS_DEF_ERROR_CODE(0, 0x3200)
+#define TSDB_CODE_INDEX_INVALID_FILE             TAOS_DEF_ERROR_CODE(0, 0x3201)
+
 
 //tmq
 #define TSDB_CODE_TMQ_INVALID_MSG                TAOS_DEF_ERROR_CODE(0, 0x4000)

include/util/tdef.h

@@ -287,7 +287,7 @@ typedef enum ELogicConditionType {
 #define TSDB_MULTI_TABLEMETA_MAX_NUM 100000  // maximum batch size allowed to load table meta
 
 #define TSDB_MIN_VNODES_PER_DB          1
-#define TSDB_MAX_VNODES_PER_DB          4096
+#define TSDB_MAX_VNODES_PER_DB          1024
 #define TSDB_DEFAULT_VN_PER_DB          2
 #define TSDB_MIN_BUFFER_PER_VNODE       3      // unit MB
 #define TSDB_MAX_BUFFER_PER_VNODE       16384  // unit MB

include/util/types.h

@@ -83,6 +83,7 @@ typedef uint16_t VarDataLenT;  // maxVarDataLen: 32767
 
 #define varDataLen(v) ((VarDataLenT *)(v))[0]
 #define varDataVal(v) ((char *)(v) + VARSTR_HEADER_SIZE)
+#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v))
 
 #define NCHAR_WIDTH_TO_BYTES(n)  ((n) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE)
 

source/common/src/systable.c

@@ -108,13 +108,11 @@ static const SSysDbTableSchema userFuncSchema[] = {
 };
 
 static const SSysDbTableSchema userIdxSchema[] = {
+    {.name = "index_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "index_database", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "index_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-    {.name = "column_name", .bytes = SYSTABLE_SCH_COL_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "index_type", .bytes = 10, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "index_extensions", .bytes = 256, .type = TSDB_DATA_TYPE_VARCHAR},
 };
 
 static const SSysDbTableSchema userStbsSchema[] = {
@@ -313,7 +311,7 @@ static const SSysDbTableSchema querySchema[] = {
     {.name = "query_id", .bytes = TSDB_QUERY_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "req_id", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
     {.name = "connId", .bytes = 4, .type = TSDB_DATA_TYPE_UINT},
-    {.name = "app", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},    
+    {.name = "app", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "end_point", .bytes = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
@@ -329,17 +327,17 @@ static const SSysDbTableSchema appSchema[] = {
     {.name = "app_id", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
     {.name = "ip", .bytes = TSDB_IPv4ADDR_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "name", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},    
+    {.name = "name", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "start_time", .bytes = 8 , .type = TSDB_DATA_TYPE_TIMESTAMP},
+    {.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-    {.name = "insert_req", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
+    {.name = "insert_req", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
-    {.name = "insert_row", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
+    {.name = "insert_row", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
-    {.name = "insert_time", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
+    {.name = "insert_time", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
-    {.name = "insert_bytes", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
+    {.name = "insert_bytes", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
-    {.name = "fetch_bytes", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
+    {.name = "fetch_bytes", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
-    {.name = "query_time", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
+    {.name = "query_time", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
-    {.name = "show_query", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
+    {.name = "show_query", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
-    {.name = "total_req", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
+    {.name = "total_req", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
-    {.name = "current_req", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
+    {.name = "current_req", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
     {.name = "last_access", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
 };
 
@@ -353,8 +351,7 @@ static const SSysTableMeta perfsMeta[] = {
     {TSDB_PERFS_TABLE_TRANS, transSchema, tListLen(transSchema)},
     {TSDB_PERFS_TABLE_SMAS, smaSchema, tListLen(smaSchema)},
     {TSDB_PERFS_TABLE_STREAMS, streamSchema, tListLen(streamSchema)},
-    {TSDB_PERFS_TABLE_APPS, appSchema, tListLen(appSchema)}
+    {TSDB_PERFS_TABLE_APPS, appSchema, tListLen(appSchema)}};
-};
 
 void getInfosDbMeta(const SSysTableMeta** pInfosTableMeta, size_t* size) {
   *pInfosTableMeta = infosMeta;

source/common/src/tdatablock.c

@@ -895,7 +895,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
         SBlockOrderInfo* pOrder = taosArrayGet(pOrderInfo, 0);
 
         int64_t p0 = taosGetTimestampUs();
-        
+
         __compar_fn_t fn = getKeyComparFunc(pColInfoData->info.type, pOrder->order);
         taosSort(pColInfoData->pData, pDataBlock->info.rows, pColInfoData->info.bytes, fn);
 
@@ -923,8 +923,9 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
     pInfo->pColData = taosArrayGet(pDataBlock->pDataBlock, pInfo->slotId);
   }
 
+  terrno = 0;
   taosqsort(index, rows, sizeof(int32_t), &helper, dataBlockCompar);
-  if(terrno) return terrno;
+  if (terrno) return terrno;
 
   int64_t p1 = taosGetTimestampUs();
 
@@ -1259,6 +1260,7 @@ int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src) {
 
   dst->info.rows = src->info.rows;
   dst->info.window = src->info.window;
+  dst->info.type = src->info.type;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -1437,21 +1439,21 @@ static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) {
   }
 }
 
-static int32_t colDataMoveVarData(SColumnInfoData* pColInfoData, size_t start, size_t end){
+static int32_t colDataMoveVarData(SColumnInfoData* pColInfoData, size_t start, size_t end) {
   int32_t dataOffset = -1;
   int32_t dataLen = 0;
   int32_t beigin = start;
-  while(beigin < end){
+  while (beigin < end) {
     int32_t offset = pColInfoData->varmeta.offset[beigin];
-    if(offset == -1) {
+    if (offset == -1) {
       beigin++;
       continue;
     }
-    if(start != 0) {
+    if (start != 0) {
       pColInfoData->varmeta.offset[beigin] = dataLen;
     }
-    char *data = pColInfoData->pData + offset;
+    char* data = pColInfoData->pData + offset;
-    if(dataOffset == -1) dataOffset = offset;   // mark the begin of data
+    if (dataOffset == -1) dataOffset = offset;  // mark the begin of data
     int32_t type = pColInfoData->info.type;
     if (type == TSDB_DATA_TYPE_JSON) {
       dataLen += getJsonValueLen(data);
@@ -1460,7 +1462,7 @@ static int32_t colDataMoveVarData(SColumnInfoData* pColInfoData, size_t start, s
     }
     beigin++;
   }
-  if(dataOffset > 0){
+  if (dataOffset > 0) {
     memmove(pColInfoData->pData, pColInfoData->pData + dataOffset, dataLen);
     memmove(pColInfoData->varmeta.offset, &pColInfoData->varmeta.offset[start], (end - start) * sizeof(int32_t));
   }

source/common/src/ttime.c

@@ -589,7 +589,7 @@ int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec
       return TSDB_CODE_FAILED;
     }
     newColData[len] = 0;
-    int32_t ret = taosParseTime(newColData, timeVal, len + 1, (int32_t)timePrec, tsDaylight);
+    int32_t ret = taosParseTime(newColData, timeVal, len, (int32_t)timePrec, tsDaylight);
     if (ret != TSDB_CODE_SUCCESS) {
       taosMemoryFree(newColData);
       return ret;

source/common/src/ttszip.c

@@ -845,11 +845,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) {
 
   int64_t offset = getDataStartOffset();
   int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset;
-#if defined(_TD_DARWIN_64)
-  int64_t written = taosFSendFile(pDestBuf->pFile->fp, pSrcBuf->pFile->fp, &offset, size);
-#else
   int64_t written = taosFSendFile(pDestBuf->pFile, pSrcBuf->pFile, &offset, size);
-#endif
 
   if (written == -1 || written != size) {
     return -1;

source/dnode/mgmt/mgmt_vnode/src/vmInt.c

@@ -21,7 +21,7 @@ SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId) {
 
   taosThreadRwlockRdlock(&pMgmt->lock);
   taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode);
-  if (pVnode == NULL) {
+  if (pVnode == NULL || pVnode->dropped) {
     terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
   } else {
     int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
@@ -81,16 +81,18 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
 
   vmReleaseVnode(pMgmt, pVnode);
   while (pVnode->refCount > 0) taosMsleep(10);
+  dTrace("vgId:%d, wait for vnode queue is empty", pVnode->vgId);
+
   while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10);
   while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10);
   while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10);
   while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10);
   while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10);
+  dTrace("vgId:%d, vnode-fetch queue is empty", pVnode->vgId);
 
   vmFreeQueue(pMgmt, pVnode);
   vnodeClose(pVnode->pImpl);
   pVnode->pImpl = NULL;
-
   dDebug("vgId:%d, vnode is closed", pVnode->vgId);
 
   if (pVnode->dropped) {

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -107,7 +107,7 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf
     const STraceId *trace = &pMsg->info.traceId;
     dGTrace("vgId:%d, msg:%p get from vnode-sync queue", pVnode->vgId, pMsg);
 
-    int32_t code = vnodeProcessSyncReq(pVnode->pImpl, pMsg, NULL); // no response here
+    int32_t code = vnodeProcessSyncReq(pVnode->pImpl, pMsg, NULL);  // no response here
     dGTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code);
     rpcFreeCont(pMsg->pCont);
     taosFreeQitem(pMsg);
@@ -146,8 +146,8 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
 
   SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
   if (pVnode == NULL) {
-    dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s", pHead->vgId, pMsg, terrstr(),
+    dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, msgtype:%s qtype:%d", pHead->vgId, pMsg,
-            TMSG_INFO(pMsg->msgType));
+            terrstr(), TMSG_INFO(pMsg->msgType), qtype);
     return terrno != 0 ? terrno : -1;
   }
 

source/dnode/mgmt/node_mgmt/src/dmProc.c

@@ -87,8 +87,8 @@ static SProcQueue *dmInitProcQueue(SProc *proc, char *ptr, int32_t size) {
 static void dmCleanupProcQueue(SProcQueue *queue) {}
 
 static inline int32_t dmPushToProcQueue(SProc *proc, SProcQueue *queue, SRpcMsg *pMsg, EProcFuncType ftype) {
-  const void   *pHead = pMsg;
+  const void *  pHead = pMsg;
-  const void   *pBody = pMsg->pCont;
+  const void *  pBody = pMsg->pCont;
   const int16_t rawHeadLen = sizeof(SRpcMsg);
   const int32_t rawBodyLen = pMsg->contLen;
   const int16_t headLen = CEIL8(rawHeadLen);
@@ -257,7 +257,7 @@ int32_t dmInitProc(struct SMgmtWrapper *pWrapper) {
   proc->wrapper = pWrapper;
   proc->name = pWrapper->name;
 
-  SShm   *shm = &proc->shm;
+  SShm *  shm = &proc->shm;
   int32_t cstart = 0;
   int32_t csize = CEIL8(shm->size / 2);
   int32_t pstart = csize;
@@ -281,13 +281,13 @@ int32_t dmInitProc(struct SMgmtWrapper *pWrapper) {
 }
 
 static void *dmConsumChildQueue(void *param) {
-  SProc        *proc = param;
+  SProc *       proc = param;
   SMgmtWrapper *pWrapper = proc->wrapper;
-  SProcQueue   *queue = proc->cqueue;
+  SProcQueue *  queue = proc->cqueue;
   int32_t       numOfMsgs = 0;
   int32_t       code = 0;
   EProcFuncType ftype = DND_FUNC_REQ;
-  SRpcMsg      *pMsg = NULL;
+  SRpcMsg *     pMsg = NULL;
 
   dDebug("node:%s, start to consume from cqueue", proc->name);
   do {
@@ -324,13 +324,13 @@ static void *dmConsumChildQueue(void *param) {
 }
 
 static void *dmConsumParentQueue(void *param) {
-  SProc        *proc = param;
+  SProc *       proc = param;
   SMgmtWrapper *pWrapper = proc->wrapper;
-  SProcQueue   *queue = proc->pqueue;
+  SProcQueue *  queue = proc->pqueue;
   int32_t       numOfMsgs = 0;
   int32_t       code = 0;
   EProcFuncType ftype = DND_FUNC_REQ;
-  SRpcMsg      *pMsg = NULL;
+  SRpcMsg *     pMsg = NULL;
 
   dDebug("node:%s, start to consume from pqueue", proc->name);
   do {
@@ -353,7 +353,7 @@ static void *dmConsumParentQueue(void *param) {
       rpcRegisterBrokenLinkArg(pMsg);
     } else if (ftype == DND_FUNC_RELEASE) {
       dmRemoveProcRpcHandle(proc, pMsg->info.handle);
-      rpcReleaseHandle(pMsg->info.handle, (int8_t)pMsg->code);
+      rpcReleaseHandle(&pMsg->info, TAOS_CONN_SERVER);
     } else {
       dError("node:%s, invalid ftype:%d from pqueue", proc->name, ftype);
       rpcFreeCont(pMsg->pCont);

source/dnode/mgmt/node_mgmt/src/dmTransport.c

@@ -245,7 +245,7 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) {
     SRpcMsg msg = {.code = type, .info = *pHandle};
     dmPutToProcPQueue(&pWrapper->proc, &msg, DND_FUNC_RELEASE);
   } else {
-    rpcReleaseHandle(pHandle->handle, type);
+    rpcReleaseHandle(pHandle, type);
   }
 }
 

source/dnode/mnode/impl/inc/mndDef.h

@@ -556,7 +556,7 @@ typedef struct {
   int64_t uid;
   int8_t  status;
   // config
-  int8_t  dropPolicy;
+  int8_t  igExpired;
   int8_t  trigger;
   int64_t triggerParam;
   int64_t watermark;

source/dnode/mnode/impl/inc/mndInt.h

@@ -87,7 +87,7 @@ typedef struct {
 } STelemMgmt;
 
 typedef struct {
-  sem_t    syncSem;
+  tsem_t    syncSem;
   int64_t  sync;
   bool     standby;
   SReplica replica;

source/dnode/mnode/impl/inc/mndStream.h

@@ -38,6 +38,8 @@ int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
 int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
 int32_t mndPersistDropStreamLog(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
 
+int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/mnode/impl/src/mndDb.c

@@ -21,6 +21,7 @@
 #include "mndShow.h"
 #include "mndSma.h"
 #include "mndStb.h"
+#include "mndStream.h"
 #include "mndSubscribe.h"
 #include "mndTopic.h"
 #include "mndTrans.h"
@@ -548,18 +549,33 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
   terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED;
 
   if (pAlter->buffer > 0 && pAlter->buffer != pDb->cfg.buffer) {
+#if 1
+    terrno = TSDB_CODE_OPS_NOT_SUPPORT;
+    return terrno;
+#else
     pDb->cfg.buffer = pAlter->buffer;
     terrno = 0;
+#endif
   }
 
   if (pAlter->pages > 0 && pAlter->pages != pDb->cfg.pages) {
+#if 1
+    terrno = TSDB_CODE_OPS_NOT_SUPPORT;
+    return terrno;
+#else
     pDb->cfg.pages = pAlter->pages;
     terrno = 0;
+#endif
   }
 
   if (pAlter->pageSize > 0 && pAlter->pageSize != pDb->cfg.pageSize) {
+#if 1
+    terrno = TSDB_CODE_OPS_NOT_SUPPORT;
+    return terrno;
+#else
     pDb->cfg.pageSize = pAlter->pageSize;
     terrno = 0;
+#endif
   }
 
   if (pAlter->daysPerFile > 0 && pAlter->daysPerFile != pDb->cfg.daysPerFile) {
@@ -593,8 +609,12 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
   }
 
   if (pAlter->strict >= 0 && pAlter->strict != pDb->cfg.strict) {
+#if 1
+    terrno = TSDB_CODE_OPS_NOT_SUPPORT;
+#else
     pDb->cfg.strict = pAlter->strict;
     terrno = 0;
+#endif
   }
 
   if (pAlter->cacheLastRow >= 0 && pAlter->cacheLastRow != pDb->cfg.cacheLastRow) {
@@ -603,9 +623,13 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
   }
 
   if (pAlter->replications > 0 && pAlter->replications != pDb->cfg.replications) {
+#if 1
+    terrno = TSDB_CODE_OPS_NOT_SUPPORT;
+#else
     pDb->cfg.replications = pAlter->replications;
     pDb->vgVersion++;
     terrno = 0;
+#endif
   }
 
   return terrno;
@@ -927,6 +951,7 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) {
   if (mndDropOffsetByDB(pMnode, pTrans, pDb) != 0) goto _OVER;
   if (mndDropSubByDB(pMnode, pTrans, pDb) != 0) goto _OVER;
   if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER;
+  if (mndDropStreamByDb(pMnode, pTrans, pDb) != 0) goto _OVER;
   if (mndDropSmasByDb(pMnode, pTrans, pDb) != 0) goto _OVER;
   if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto _OVER;
 
@@ -947,7 +972,6 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) {
   mndTransSetRpcRsp(pTrans, pRsp, rspLen);
 
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
-
   code = 0;
 
 _OVER:

source/dnode/mnode/impl/src/mndDef.c

@@ -28,7 +28,7 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
   if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1;
   if (tEncodeI8(pEncoder, pObj->status) < 0) return -1;
 
-  if (tEncodeI8(pEncoder, pObj->dropPolicy) < 0) return -1;
+  if (tEncodeI8(pEncoder, pObj->igExpired) < 0) return -1;
   if (tEncodeI8(pEncoder, pObj->trigger) < 0) return -1;
   if (tEncodeI64(pEncoder, pObj->triggerParam) < 0) return -1;
   if (tEncodeI64(pEncoder, pObj->watermark) < 0) return -1;
@@ -73,7 +73,7 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) {
   if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1;
   if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1;
 
-  if (tDecodeI8(pDecoder, &pObj->dropPolicy) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pObj->igExpired) < 0) return -1;
   if (tDecodeI8(pDecoder, &pObj->trigger) < 0) return -1;
   if (tDecodeI64(pDecoder, &pObj->triggerParam) < 0) return -1;
   if (tDecodeI64(pDecoder, &pObj->watermark) < 0) return -1;

source/dnode/mnode/impl/src/mndDnode.c

@@ -758,6 +758,11 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
     }
   }
 
+  if (numOfVnodes > 0) {
+    terrno = TSDB_CODE_OPS_NOT_SUPPORT;
+    goto _OVER;
+  }
+
   code = mndDropDnode(pMnode, pReq, pDnode, pMObj, pQObj, pSObj, numOfVnodes);
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
 

source/dnode/mnode/impl/src/mndShow.c

@@ -71,7 +71,7 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
   } else if (strncasecmp(name, TSDB_INS_TABLE_USER_FUNCTIONS, len) == 0) {
     type = TSDB_MGMT_TABLE_FUNC;
   } else if (strncasecmp(name, TSDB_INS_TABLE_USER_INDEXES, len) == 0) {
-    //    type = TSDB_MGMT_TABLE_INDEX;
+    type = TSDB_MGMT_TABLE_INDEX;
   } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, len) == 0) {
     type = TSDB_MGMT_TABLE_STB;
   } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, len) == 0) {

source/dnode/mnode/impl/src/mndSma.c

@@ -523,6 +523,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
   streamObj.updateTime = streamObj.createTime;
   streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name));
   streamObj.sourceDbUid = pDb->uid;
+  streamObj.targetDbUid = pDb->uid;
   streamObj.version = 1;
   streamObj.sql = pCreate->sql;
   streamObj.smaId = smaObj.uid;
@@ -853,36 +854,26 @@ _OVER:
 }
 
 int32_t mndDropSmasByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
-  SSdb    *pSdb = pMnode->pSdb;
+  SSdb *pSdb = pMnode->pSdb;
-  SSmaObj *pSma = NULL;
+  void *pIter = NULL;
-  void    *pIter = NULL;
-  SVgObj  *pVgroup = NULL;
-  int32_t  code = -1;
 
   while (1) {
+    SSmaObj *pSma = NULL;
     pIter = sdbFetch(pSdb, SDB_SMA, pIter, (void **)&pSma);
     if (pIter == NULL) break;
 
     if (pSma->dbUid == pDb->uid) {
-      pVgroup = mndAcquireVgroup(pMnode, pSma->dstVgId);
+      if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) {
-      if (pVgroup == NULL) goto _OVER;
+        sdbRelease(pSdb, pSma);
-      if (mndSetDropSmaVgroupCommitLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER;
+        sdbCancelFetch(pSdb, pSma);
-      if (mndSetDropSmaVgroupRedoActions(pMnode, pTrans, pDb, pVgroup) != 0) goto _OVER;
+        return -1;
-      if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER;
+      }
-      mndReleaseVgroup(pMnode, pVgroup);
-      pVgroup = NULL;
     }
 
     sdbRelease(pSdb, pSma);
   }
 
-  code = 0;
+  return 0;
-
-_OVER:
-  sdbCancelFetch(pSdb, pIter);
-  sdbRelease(pSdb, pSma);
-  mndReleaseVgroup(pMnode, pVgroup);
-  return code;
 }
 
 static int32_t mndProcessDropSmaReq(SRpcMsg *pReq) {
@@ -1156,29 +1147,32 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
 
     SName smaName = {0};
     tNameFromString(&smaName, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
+    char n1[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
+    STR_TO_VARSTR(n1, (char *)tNameGetTableName(&smaName));
 
-    char n[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
+    char n2[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
-    STR_TO_VARSTR(n, (char *)tNameGetTableName(&smaName));
+    STR_TO_VARSTR(n2, (char *)mndGetDbStr(pDb->name));
-    cols++;
 
     SName stbName = {0};
     tNameFromString(&stbName, pSma->stb, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
-
+    char n3[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
-    char n1[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
+    STR_TO_VARSTR(n3, (char *)tNameGetTableName(&stbName));
-    STR_TO_VARSTR(n1, (char *)tNameGetTableName(&stbName));
 
     SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)n, false);
-
-    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pSma->createdTime, false);
-
-    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, numOfRows, (const char *)n1, false);
 
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)n2, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)n3, false);
+
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, numOfRows, (const char *)&pSma->dstVgId, false);
 
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pSma->createdTime, false);
+
     numOfRows++;
     sdbRelease(pSdb, pSma);
   }

source/dnode/mnode/impl/src/mndSnode.c

@@ -273,6 +273,9 @@ _OVER:
 }
 
 static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) {
+#if 1
+  return TSDB_CODE_OPS_NOT_SUPPORT;
+#else
   SMnode          *pMnode = pReq->info.node;
   int32_t          code = -1;
   SSnodeObj       *pObj = NULL;
@@ -315,6 +318,7 @@ _OVER:
   mndReleaseSnode(pMnode, pObj);
   mndReleaseDnode(pMnode, pDnode);
   return code;
+#endif
 }
 
 static int32_t mndSetDropSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) {
@@ -386,9 +390,12 @@ _OVER:
 }
 
 static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) {
-  SMnode        *pMnode = pReq->info.node;
+#if 1
-  int32_t        code = -1;
+  return TSDB_CODE_OPS_NOT_SUPPORT;
-  SSnodeObj     *pObj = NULL;
+#else
+  SMnode *pMnode = pReq->info.node;
+  int32_t code = -1;
+  SSnodeObj *pObj = NULL;
   SMDropSnodeReq dropReq = {0};
 
   if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
@@ -422,6 +429,7 @@ _OVER:
 
   mndReleaseSnode(pMnode, pObj);
   return code;
+#endif
 }
 
 static int32_t mndRetrieveSnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {

source/dnode/mnode/impl/src/mndStream.c

@@ -247,8 +247,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj,
   pObj->uid = mndGenerateUid(pObj->name, strlen(pObj->name));
   pObj->status = 0;
 
-  // TODO
+  pObj->igExpired = pCreate->igExpired;
-  pObj->dropPolicy = 0;
   pObj->trigger = pCreate->triggerType;
   pObj->triggerParam = pCreate->maxDelay;
   pObj->watermark = pCreate->watermark;
@@ -301,6 +300,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj,
       .streamQuery = true,
       .triggerType = pObj->trigger == STREAM_TRIGGER_MAX_DELAY ? STREAM_TRIGGER_WINDOW_CLOSE : pObj->trigger,
       .watermark = pObj->watermark,
+      .igExpired = pObj->igExpired,
   };
 
   // using ast and param to build physical plan
@@ -673,27 +673,29 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
 
 int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
   SSdb *pSdb = pMnode->pSdb;
+  void *pIter = NULL;
 
-  void       *pIter = NULL;
-  SStreamObj *pStream = NULL;
   while (1) {
+    SStreamObj *pStream = NULL;
     pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
     if (pIter == NULL) break;
 
     if (pStream->sourceDbUid == pDb->uid || pStream->targetDbUid == pDb->uid) {
       if (pStream->sourceDbUid != pStream->targetDbUid) {
         sdbRelease(pSdb, pStream);
+        sdbCancelFetch(pSdb, pIter);
+        mError("db:%s, failed to drop stream:%s since sourceDbUid:%" PRId64 " not match with targetDbUid:%" PRId64,
+               pDb->name, pStream->name, pStream->sourceDbUid, pStream->targetDbUid);
+        terrno = TSDB_CODE_MND_STREAM_ALREADY_EXIST;
         return -1;
       } else {
         // TODO drop all task on snode
         if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) {
           sdbRelease(pSdb, pStream);
+          sdbCancelFetch(pSdb, pIter);
           return -1;
         }
       }
-    } else {
-      sdbRelease(pSdb, pStream);
-      continue;
     }
 
 #if 0

source/dnode/mnode/impl/src/mndTrans.c

@@ -90,7 +90,7 @@ static int32_t mndTransGetActionsSize(SArray *pArray) {
   for (int32_t i = 0; i < actionNum; ++i) {
     STransAction *pAction = taosArrayGet(pArray, i);
     if (pAction->actionType == TRANS_ACTION_RAW) {
-      rawDataLen += (sdbGetRawTotalSize(pAction->pRaw) + sizeof(int32_t));
+      rawDataLen += (sizeof(STransAction) + sdbGetRawTotalSize(pAction->pRaw));
     } else if (pAction->actionType == TRANS_ACTION_MSG) {
       rawDataLen += (sizeof(STransAction) + pAction->contLen);
     } else {
@@ -105,7 +105,7 @@ static int32_t mndTransGetActionsSize(SArray *pArray) {
 static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
   terrno = TSDB_CODE_OUT_OF_MEMORY;
 
-  int32_t rawDataLen = sizeof(STrans) + TRANS_RESERVE_SIZE;
+  int32_t rawDataLen = sizeof(STrans) + TRANS_RESERVE_SIZE + pTrans->paramLen;
   rawDataLen += mndTransGetActionsSize(pTrans->redoActions);
   rawDataLen += mndTransGetActionsSize(pTrans->undoActions);
   rawDataLen += mndTransGetActionsSize(pTrans->commitActions);
@@ -226,7 +226,8 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
 
 _OVER:
   if (terrno != 0) {
-    mError("trans:%d, failed to encode to raw:%p len:%d since %s", pTrans->id, pRaw, dataPos, terrstr());
+    mError("trans:%d, failed to encode to raw:%p maxlen:%d len:%d since %s", pTrans->id, pRaw, sdbGetRawTotalSize(pRaw),
+           dataPos, terrstr());
     sdbFreeRaw(pRaw);
     return NULL;
   }
@@ -1025,7 +1026,7 @@ static int32_t mndTransExecNullMsg(SMnode *pMnode, STrans *pTrans, STransAction
   pTrans->lastAction = pAction->id;
   pTrans->lastMsgType = pAction->msgType;
   pTrans->lastEpset = pAction->epSet;
-  pTrans->lastErrorNo == 0;
+  pTrans->lastErrorNo = 0;
   return 0;
 }
 

source/dnode/mnode/impl/src/mndVgroup.c

@@ -1219,6 +1219,9 @@ _OVER:
 }
 
 static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) {
+#if 1
+  return TSDB_CODE_OPS_NOT_SUPPORT;
+#else
   SMnode    *pMnode = pReq->info.node;
   SDnodeObj *pNew1 = NULL;
   SDnodeObj *pNew2 = NULL;
@@ -1412,6 +1415,7 @@ _OVER:
   mndReleaseDb(pMnode, pDb);
 
   return code;
+#endif  
 }
 
 int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) {
@@ -1711,6 +1715,9 @@ _OVER:
 }
 
 static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) {
+#if 1
+  return TSDB_CODE_OPS_NOT_SUPPORT;
+#else
   SMnode *pMnode = pReq->info.node;
   int32_t code = -1;
   SArray *pArray = NULL;
@@ -1759,6 +1766,7 @@ _OVER:
 
   taosArrayDestroy(pArray);
   return code;
+#endif 
 }
 
 bool mndVgroupInDb(SVgObj *pVgroup, int64_t dbUid) { return !pVgroup->isTsma && pVgroup->dbUid == dbUid; }

source/dnode/mnode/impl/test/db/db.cpp

@@ -93,7 +93,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, 0);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT);
   }
 
   test.SendShowReq(TSDB_MGMT_TABLE_DB, "user_databases", "");

source/dnode/vnode/CMakeLists.txt

@@ -31,6 +31,8 @@ target_sources(
     "src/sma/smaEnv.c"
     "src/sma/smaUtil.c"
     "src/sma/smaOpen.c"
+    "src/sma/smaCommit.c"
+    "src/sma/smaSnapshot.c"
     "src/sma/smaRollup.c"
     "src/sma/smaTimeRange.c"
 

source/dnode/vnode/src/inc/sma.h

@@ -62,12 +62,10 @@ struct STSmaStat {
 
 struct SRSmaStat {
   SSma     *pSma;
-  int64_t   refId;         // shared by persistence/fetch tasks
+  int64_t   refId;         // shared by fetch tasks
-  void     *tmrHandle;     // for persistence task
+  void     *tmrHandle;     // shared by fetch tasks
-  tmr_h     tmrId;         // for persistence task
+  int8_t    triggerStat;   // shared by fetch tasks
-  int32_t   tmrSeconds;    // for persistence task
+  int8_t    runningStat;   // for persistence task 
-  int8_t    triggerStat;   // for persistence task
-  int8_t    runningStat;   // for persistence task
   SHashObj *rsmaInfoHash;  // key: stbUid, value: SRSmaInfo;
 };
 
@@ -82,7 +80,6 @@ struct SSmaStat {
 #define SMA_TSMA_STAT(s)     (&(s)->tsmaStat)
 #define SMA_RSMA_STAT(s)     (&(s)->rsmaStat)
 #define RSMA_INFO_HASH(r)    ((r)->rsmaInfoHash)
-#define RSMA_TMR_ID(r)       ((r)->tmrId)
 #define RSMA_TMR_HANDLE(r)   ((r)->tmrHandle)
 #define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat)
 #define RSMA_RUNNING_STAT(r) (&(r)->runningStat)
@@ -92,8 +89,9 @@ enum {
   TASK_TRIGGER_STAT_INIT = 0,
   TASK_TRIGGER_STAT_ACTIVE = 1,
   TASK_TRIGGER_STAT_INACTIVE = 2,
-  TASK_TRIGGER_STAT_CANCELLED = 3,
+  TASK_TRIGGER_STAT_PAUSED = 3,
-  TASK_TRIGGER_STAT_FINISHED = 4,
+  TASK_TRIGGER_STAT_CANCELLED = 4,
+  TASK_TRIGGER_STAT_FINISHED = 5,
 };
 void  tdDestroySmaEnv(SSmaEnv *pSmaEnv);
 void *tdFreeSmaEnv(SSmaEnv *pSmaEnv);
@@ -184,9 +182,11 @@ static FORCE_INLINE void tdSmaStatSetDropped(STSmaStat *pTStat) {
 static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType);
 void          *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType);
 void          *tdFreeRSmaInfo(SRSmaInfo *pInfo);
+int32_t        tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat);
 
 int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName);
 int32_t tdProcessRSmaRestoreImpl(SSma *pSma);
+
 int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg);
 int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg);
 int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);
@@ -214,25 +214,22 @@ struct STFInfo {
 };
 
 struct STFile {
-  STFInfo   info;
-  STfsFile  f;
-  TdFilePtr pFile;
   uint8_t   state;
+  STFInfo   info;
+  char     *fname;
+  TdFilePtr pFile;
 };
 
-#define TD_TFILE_F(tf)            (&((tf)->f))
 #define TD_TFILE_PFILE(tf)        ((tf)->pFile)
 #define TD_TFILE_OPENED(tf)       (TD_TFILE_PFILE(tf) != NULL)
-#define TD_TFILE_FULL_NAME(tf)    (TD_TFILE_F(tf)->aname)
+#define TD_TFILE_FULL_NAME(tf)    ((tf)->fname)
-#define TD_TFILE_REL_NAME(tf)     (TD_TFILE_F(tf)->rname)
 #define TD_TFILE_OPENED(tf)       (TD_TFILE_PFILE(tf) != NULL)
 #define TD_TFILE_CLOSED(tf)       (!TD_TFILE_OPENED(tf))
 #define TD_TFILE_SET_CLOSED(f)    (TD_TFILE_PFILE(f) = NULL)
 #define TD_TFILE_SET_STATE(tf, s) ((tf)->state = (s))
-#define TD_TFILE_DID(tf)          (TD_TFILE_F(tf)->did)
 
-int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname);
+int32_t tdInitTFile(STFile *pTFile, const char *dname, const char *fname);
-int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fType);
+int32_t tdCreateTFile(STFile *pTFile, bool updateHeader, int8_t fType);
 int32_t tdOpenTFile(STFile *pTFile, int flags);
 int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte);
 int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence);
@@ -244,8 +241,10 @@ int32_t tdLoadTFileHeader(STFile *pTFile, STFInfo *pInfo);
 int32_t tdUpdateTFileHeader(STFile *pTFile);
 void    tdUpdateTFileMagic(STFile *pTFile, void *pCksm);
 void    tdCloseTFile(STFile *pTFile);
+void    tdDestroyTFile(STFile *pTFile);
 
-void tdGetVndFileName(int32_t vgId, const char *dname, const char *fname, char *outputName);
+void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version, char *outputName);
+void tdGetVndDirName(int32_t vgId,const char *pdname,  const char *dname, bool endWithSep, char *outputName);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/src/inc/vnodeInt.h

@@ -163,8 +163,11 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool
 
 // sma
 int32_t smaOpen(SVnode* pVnode);
-int32_t smaCloseEnv(SSma* pSma);
+int32_t smaClose(SSma* pSma);
-int32_t smaCloseEx(SSma* pSma);
+int32_t smaBegin(SSma* pSma);
+int32_t smaPreCommit(SSma* pSma);
+int32_t smaCommit(SSma* pSma);
+int32_t smaPostCommit(SSma* pSma);
 
 int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg);
 int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg);
@@ -238,7 +241,7 @@ struct SVnode {
   tsem_t     canCommit;
   int64_t    sync;
   int32_t    syncCount;
-  sem_t      syncSem;
+  tsem_t     syncSem;
   SQHandle*  pQuery;
 };
 

source/dnode/vnode/src/sma/smaCommit.c

@@ -0,0 +1,225 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "sma.h"
+
+static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma);
+static int32_t tdProcessRSmaCommitImpl(SSma *pSma);
+static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma);
+
+/**
+ * @brief Only applicable to Rollup SMA
+ *
+ * @param pSma
+ * @return int32_t
+ */
+int32_t smaPreCommit(SSma *pSma) { return tdProcessRSmaPreCommitImpl(pSma); }
+
+/**
+ * @brief Only applicable to Rollup SMA
+ *
+ * @param pSma
+ * @return int32_t
+ */
+int32_t smaCommit(SSma *pSma) { return tdProcessRSmaCommitImpl(pSma); }
+
+/**
+ * @brief Only applicable to Rollup SMA
+ *
+ * @param pSma
+ * @return int32_t
+ */
+int32_t smaPostCommit(SSma *pSma) { return tdProcessRSmaPostCommitImpl(pSma); }
+
+/**
+ * @brief set rsma trigger stat active
+ *
+ * @param pSma
+ * @return int32_t
+ */
+int32_t smaBegin(SSma *pSma) {
+  SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma);
+  if (!pSmaEnv) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SSmaStat  *pStat = SMA_ENV_STAT(pSmaEnv);
+  SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat);
+
+  int8_t rsmaTriggerStat =
+      atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED, TASK_TRIGGER_STAT_ACTIVE);
+  switch (rsmaTriggerStat) {
+    case TASK_TRIGGER_STAT_PAUSED: {
+      smaDebug("vgId:%d rsma trigger stat from paused to active", SMA_VID(pSma));
+      break;
+    }
+    case TASK_TRIGGER_STAT_INIT: {
+      atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_ACTIVE);
+      smaDebug("vgId:%d rsma trigger stat from init to active", SMA_VID(pSma));
+      break;
+    }
+    default: {
+      atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_ACTIVE);
+      smaWarn("vgId:%d rsma trigger stat %" PRIi8 " is unexpected", SMA_VID(pSma), rsmaTriggerStat);
+      ASSERT(0);
+      break;
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief pre-commit for rollup sma.
+ *  1) set trigger stat of rsma timer TASK_TRIGGER_STAT_PAUSED.
+ *  2) perform persist task for qTaskInfo
+ *  3) wait all triggered fetch tasks finished
+ *
+ * @param pSma
+ * @return int32_t
+ */
+static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma) {
+  SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma);
+  if (!pSmaEnv) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SSmaStat  *pStat = SMA_ENV_STAT(pSmaEnv);
+  SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat);
+
+
+  // step 1: set persistence task paused
+  atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED);
+
+  // step 2: perform persist task for qTaskInfo
+  tdRSmaPersistExecImpl(pRSmaStat);
+
+  // step 3: wait all triggered fetch tasks finished
+  int32_t nLoops = 0;
+  while (1) {
+    if (T_REF_VAL_GET(pStat) == 0) {
+      smaDebug("vgId:%d, rsma fetch tasks all finished", SMA_VID(pSma));
+      break;
+    } else {
+      smaDebug("vgId:%d, rsma fetch tasks not all finished yet", SMA_VID(pSma));
+    }
+    ++nLoops;
+    if (nLoops > 1000) {
+      sched_yield();
+      nLoops = 0;
+    }
+  }
+
+  smaDebug("vgId:%d, rsma pre commit succeess", SMA_VID(pSma));
+
+  return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief commit for rollup sma
+ *
+ * @param pSma
+ * @return int32_t
+ */
+static int32_t tdProcessRSmaCommitImpl(SSma *pSma) {
+  SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma);
+  if (!pSmaEnv) {
+    return TSDB_CODE_SUCCESS;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief post-commit for rollup sma
+ *  1) clean up the outdated qtaskinfo files
+ *
+ * @param pSma
+ * @return int32_t
+ */
+static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma) {
+  SVnode *pVnode = pSma->pVnode;
+
+  if (!VND_IS_RSMA(pVnode)) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  int64_t       committed = pVnode->state.committed;
+  TdDirPtr      pDir = NULL;
+  TdDirEntryPtr pDirEntry = NULL;
+  char          dir[TSDB_FILENAME_LEN];
+  const char   *pattern = "v[0-9]+qtaskinfo\\.ver([0-9]+)?$";
+  regex_t       regex;
+  int           code = 0;
+
+  tdGetVndDirName(TD_VID(pVnode), tfsGetPrimaryPath(pVnode->pTfs), VNODE_RSMA_DIR, true, dir);
+
+  // Resource allocation and init
+  if ((code = regcomp(&regex, pattern, REG_EXTENDED)) != 0) {
+    char errbuf[128];
+    regerror(code, &regex, errbuf, sizeof(errbuf));
+    smaWarn("vgId:%d, rsma post commit, regcomp for %s failed since %s", TD_VID(pVnode), dir, errbuf);
+    return TSDB_CODE_FAILED;
+  }
+
+  if ((pDir = taosOpenDir(dir)) == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    smaWarn("vgId:%d, rsma post commit, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr());
+    return TSDB_CODE_FAILED;
+  }
+
+  int32_t    dirLen = strlen(dir);
+  char      *dirEnd = POINTER_SHIFT(dir, dirLen);
+  regmatch_t regMatch[2];
+  while ((pDirEntry = taosReadDir(pDir)) != NULL) {
+    char *entryName = taosGetDirEntryName(pDirEntry);
+    if (!entryName) {
+      continue;
+    }
+
+    code = regexec(&regex, entryName, 2, regMatch, 0);
+
+    if (code == 0) {
+      // match
+      int64_t version = -1;
+      sscanf((const char *)POINTER_SHIFT(entryName, regMatch[1].rm_so), "%" PRIi64, &version);
+      if ((version < committed) && (version > -1)) {
+        strncpy(dirEnd, entryName, TSDB_FILENAME_LEN - dirLen);
+        if (taosRemoveFile(dir) != 0) {
+          terrno = TAOS_SYSTEM_ERROR(errno);
+          smaWarn("vgId:%d, committed version:%" PRIi64 ", failed to remove %s since %s", TD_VID(pVnode), committed,
+                  dir, terrstr());
+        } else {
+          smaDebug("vgId:%d, committed version:%" PRIi64 ", success to remove %s", TD_VID(pVnode), committed, dir);
+        }
+      }
+    } else if (code == REG_NOMATCH) {
+      // not match
+      smaTrace("vgId:%d, rsma post commit, not match %s", TD_VID(pVnode), entryName);
+      continue;
+    } else {
+      // has other error
+      char errbuf[128];
+      regerror(code, &regex, errbuf, sizeof(errbuf));
+      smaWarn("vgId:%d, rsma post commit, regexec failed since %s", TD_VID(pVnode), errbuf);
+
+      taosCloseDir(&pDir);
+      regfree(&regex);
+      return TSDB_CODE_FAILED;
+    }
+  }
+
+  taosCloseDir(&pDir);
+  regfree(&regex);
+  return TSDB_CODE_SUCCESS;
+}

source/dnode/vnode/src/sma/smaEnv.c

@@ -132,6 +132,7 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS
     if (smaType == TSDB_SMA_TYPE_ROLLUP) {
       SRSmaStat *pRSmaStat = (SRSmaStat *)(*pSmaStat);
       pRSmaStat->pSma = (SSma *)pSma;
+      atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_INIT);
 
       // init smaMgmt
       smaMgmt.smaRef = taosOpenRef(SMA_MGMT_REF_NUM, tdDestroyRSmaStat);
@@ -192,22 +193,20 @@ static void *tdFreeTSmaStat(STSmaStat *pStat) {
 static void tdDestroyRSmaStat(void *pRSmaStat) {
   if (pRSmaStat) {
     SRSmaStat *pStat = (SRSmaStat *)pRSmaStat;
-    smaDebug("vgId:%d %s:%d destroy rsma stat %p", SMA_VID(pStat->pSma), __func__, __LINE__, pRSmaStat);
+    SSma      *pSma = pStat->pSma;
-    // step 1: set persistence task cancelled
+    smaDebug("vgId:%d, destroy rsma stat %p", SMA_VID(pSma), pRSmaStat);
+    // step 1: set rsma trigger stat cancelled
     atomic_store_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_CANCELLED);
 
-    // step 2: stop the persistence timer
+    // step 2: wait the persistence thread to finish
-    taosTmrStopA(&RSMA_TMR_ID(pStat));
-
-    // step 3: wait the persistence thread to finish
     int32_t nLoops = 0;
     if (atomic_load_8(RSMA_RUNNING_STAT(pStat)) == 1) {
       while (1) {
         if (atomic_load_8(RSMA_TRIGGER_STAT(pStat)) == TASK_TRIGGER_STAT_FINISHED) {
-          smaDebug("rsma, persist task finished already");
+          smaDebug("vgId:%d, rsma persist task finished already", SMA_VID(pSma));
           break;
         } else {
-          smaDebug("rsma, persist task not finished yet since rsma stat in %" PRIi8,
+          smaDebug("vgId:%d, rsma persist task not finished yet since rsma stat in %" PRIi8, SMA_VID(pSma),
                    atomic_load_8(RSMA_TRIGGER_STAT(pStat)));
         }
         ++nLoops;
@@ -218,13 +217,15 @@ static void tdDestroyRSmaStat(void *pRSmaStat) {
       }
     }
 
-    // step 4: destroy the rsma info and associated fetch tasks
+    // step 3: destroy the rsma info and associated fetch tasks
     // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready.
-    void *infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), NULL);
+    if (taosHashGetSize(RSMA_INFO_HASH(pStat)) > 0) {
-    while (infoHash) {
+      void *infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), NULL);
-      SRSmaInfo *pSmaInfo = *(SRSmaInfo **)infoHash;
+      while (infoHash) {
-      tdFreeRSmaInfo(pSmaInfo);
+        SRSmaInfo *pSmaInfo = *(SRSmaInfo **)infoHash;
-      infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), infoHash);
+        tdFreeRSmaInfo(pSmaInfo);
+        infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), infoHash);
+      }
     }
     taosHashCleanup(RSMA_INFO_HASH(pStat));
 
@@ -232,10 +233,10 @@ static void tdDestroyRSmaStat(void *pRSmaStat) {
     nLoops = 0;
     while (1) {
       if (T_REF_VAL_GET((SSmaStat *)pStat) == 0) {
-        smaDebug("rsma, all fetch task finished already");
+        smaDebug("vgId:%d, rsma fetch tasks all finished", SMA_VID(pSma));
         break;
       } else {
-        smaDebug("rsma, fetch tasks not all finished yet");
+        smaDebug("vgId:%d, rsma fetch tasks not all finished yet", SMA_VID(pSma));
       }
       ++nLoops;
       if (nLoops > 1000) {
@@ -275,7 +276,7 @@ int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) {
     } else if (smaType == TSDB_SMA_TYPE_ROLLUP) {
       SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSmaStat);
       if (taosRemoveRef(smaMgmt.smaRef, RSMA_REF_ID(pRSmaStat)) < 0) {
-        smaError("remove refId from smaRef failed, refId:0x%" PRIx64, RSMA_REF_ID(pRSmaStat));
+        smaError("remove refId from rsmaRef:0x%" PRIx64 " failed since %s", RSMA_REF_ID(pRSmaStat), terrstr());
       }
     } else {
       ASSERT(0);

source/dnode/vnode/src/sma/smaOpen.c

@@ -135,17 +135,11 @@ _err:
   return -1;
 }
 
-int32_t smaCloseEnv(SSma *pSma) {
+int32_t smaClose(SSma *pSma) {
-  if (pSma) {
-    SMA_TSMA_ENV(pSma) = tdFreeSmaEnv(SMA_TSMA_ENV(pSma));
-    SMA_RSMA_ENV(pSma) = tdFreeSmaEnv(SMA_RSMA_ENV(pSma));
-  }
-  return 0;
-}
-
-int32_t smaCloseEx(SSma *pSma) {
   if (pSma) {
     taosThreadMutexDestroy(&pSma->mutex);
+    SMA_TSMA_ENV(pSma) = tdFreeSmaEnv(SMA_TSMA_ENV(pSma));
+    SMA_RSMA_ENV(pSma) = tdFreeSmaEnv(SMA_RSMA_ENV(pSma));
     if SMA_RSMA_TSDB0 (pSma) tsdbClose(&SMA_RSMA_TSDB0(pSma));
     if SMA_RSMA_TSDB1 (pSma) tsdbClose(&SMA_RSMA_TSDB1(pSma));
     if SMA_RSMA_TSDB2 (pSma) tsdbClose(&SMA_RSMA_TSDB2(pSma));

source/dnode/vnode/src/sma/smaRollup.c

@@ -15,16 +15,14 @@
 
 #include "sma.h"
 
-#define RSMA_QTASKINFO_PERSIST_MS 7200000
+#define RSMA_QTASKINFO_BUFSIZE  32768
-#define RSMA_QTASKINFO_BUFSIZE    32768
+#define RSMA_QTASKINFO_HEAD_LEN (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t))  // len + type + suid
-#define RSMA_QTASKINFO_HEAD_LEN   (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t))  // len + type + suid
 
 SSmaMgmt smaMgmt = {
     .smaRef = -1,
 };
 
-typedef enum { TD_QTASK_TMP_F = 0, TD_QTASK_CUR_F } TD_QTASK_FILE_T;
+#define TD_QTASKINFO_FNAME_PREFIX "qtaskinfo.ver"
-static const char                *tdQTaskInfoFname[] = {"qtaskinfo.t", "qtaskinfo"};
 typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem;
 typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter;
 
@@ -37,16 +35,16 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType
 static void    tdRSmaFetchTrigger(void *param, void *tmrId);
 static void    tdRSmaPersistTrigger(void *param, void *tmrId);
 static void   *tdRSmaPersistExec(void *param);
-static void    tdRSmaQTaskInfoGetFName(int32_t vid, int8_t ftype, char *outputName);
+static void    tdRSmaQTaskInfoGetFName(int32_t vid, int64_t version, char *outputName);
 
 static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile);
 static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish);
 static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter);
 static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *infoItem);
 
-static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma);
+static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables);
-static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma);
+static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int64_t *committed);
-static int32_t tdRSmaRestoreTSDataReload(SSma *pSma);
+static int32_t tdRSmaRestoreTSDataReload(SSma *pSma, int64_t committed);
 
 struct SRSmaInfoItem {
   SRSmaInfo *pRsmaInfo;
@@ -88,8 +86,8 @@ struct SRSmaQTaskInfoIter {
   int32_t nBufPos;
 };
 
-static void tdRSmaQTaskInfoGetFName(int32_t vgId, int8_t ftype, char *outputName) {
+static void tdRSmaQTaskInfoGetFName(int32_t vgId, int64_t version, char *outputName) {
-  tdGetVndFileName(vgId, VNODE_RSMA_DIR, tdQTaskInfoFname[ftype], outputName);
+  tdGetVndFileName(vgId, NULL, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, version, outputName);
 }
 
 static FORCE_INLINE int32_t tdRSmaQTaskInfoContLen(int32_t lenWithHead) {
@@ -115,12 +113,14 @@ void *tdFreeRSmaInfo(SRSmaInfo *pInfo) {
     for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
       SRSmaInfoItem *pItem = &pInfo->items[i];
       if (pItem->taskInfo) {
-        smaDebug("vgId:%d, stb %" PRIi64 " stop fetch-timer %p level %d", SMA_VID(pSma), pInfo->suid, pItem->tmrId,
+        if (pItem->tmrId) {
-                 i + 1);
+          smaDebug("vgId:%d, table %" PRIi64 " stop fetch timer %p level %d", SMA_VID(pSma), pInfo->suid, pItem->tmrId,
-        taosTmrStopA(&pItem->tmrId);
+                   i + 1);
+          taosTmrStopA(&pItem->tmrId);
+        }
         tdFreeTaskHandle(&pItem->taskInfo, SMA_VID(pSma), i + 1);
       } else {
-        smaDebug("vgId:%d, stb %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", SMA_VID(pSma),
+        smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", SMA_VID(pSma),
                  pInfo->suid, i + 1);
       }
     }
@@ -359,13 +359,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
     goto _err;
   }
 
-  smaDebug("vgId:%d, register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), suid);
+  smaDebug("vgId:%d, register rsma info succeed for table %" PRIi64, SMA_VID(pSma), suid);
-
-  // start the persist timer
-  if (TASK_TRIGGER_STAT_INIT ==
-      atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_INIT, TASK_TRIGGER_STAT_ACTIVE)) {
-    taosTmrStart(tdRSmaPersistTrigger, RSMA_QTASKINFO_PERSIST_MS, pStat, RSMA_TMR_HANDLE(pStat));
-  }
 
   return TSDB_CODE_SUCCESS;
 _err:
@@ -493,7 +487,6 @@ static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) {
 }
 
 static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) {
-  ASSERT(pMsg != NULL);
   SSubmitMsgIter msgIter = {0};
   SSubmitBlk    *pBlock = NULL;
   SSubmitBlkIter blkIter = {0};
@@ -501,19 +494,26 @@ static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) {
 
   terrno = TSDB_CODE_SUCCESS;
 
-  if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1;
+  if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) {
+    return -1;
+  }
   while (true) {
-    if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1;
+    if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) {
+      return -1;
+    }
 
     if (!pBlock) break;
     tdUidStorePut(pStore, msgIter.suid, NULL);
   }
 
-  if (terrno != TSDB_CODE_SUCCESS) return -1;
+  if (terrno != TSDB_CODE_SUCCESS) {
+    return -1;
+  }
   return 0;
 }
 
 static void tdDestroySDataBlockArray(SArray *pArray) {
+  // TODO
 #if 0
   for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
     SSDataBlock *pDataBlock = taosArrayGet(pArray, i);
@@ -598,33 +598,54 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
 
   pSma = RSMA_INFO_SMA(pItem->pRsmaInfo);
 
-  // if rsma trigger stat in cancelled or finished, not start fetch task anymore
+  // if rsma trigger stat in paused, cancelled or finished, not start fetch task
   int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat));
-  if (rsmaTriggerStat == TASK_TRIGGER_STAT_CANCELLED || rsmaTriggerStat == TASK_TRIGGER_STAT_FINISHED) {
+  switch (rsmaTriggerStat) {
-    taosReleaseRef(smaMgmt.smaRef, pItem->refId);
+    case TASK_TRIGGER_STAT_PAUSED:
-    smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is cancelled",
+    case TASK_TRIGGER_STAT_CANCELLED:
-             SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
+    case TASK_TRIGGER_STAT_FINISHED: {
-    return;
+      taosReleaseRef(smaMgmt.smaRef, pItem->refId);
+      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is cancelled",
+               SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
+      return;
+    }
+    default:
+      break;
   }
 
   int8_t fetchTriggerStat =
       atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
-  if (fetchTriggerStat == TASK_TRIGGER_STAT_ACTIVE) {
+  switch (fetchTriggerStat) {
-    smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma),
+    case TASK_TRIGGER_STAT_ACTIVE: {
-             pItem->level, pItem->pRsmaInfo->suid);
+      smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma),
+               pItem->level, pItem->pRsmaInfo->suid);
 
-    tdRefSmaStat(pSma, (SSmaStat *)pStat);
+      tdRefSmaStat(pSma, (SSmaStat *)pStat);
 
-    SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL};
+      SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL};
-    qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false);
+      qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false);
-    tdFetchAndSubmitRSmaResult(pItem, STREAM_INPUT__DATA_BLOCK);
+      tdFetchAndSubmitRSmaResult(pItem, STREAM_INPUT__DATA_BLOCK);
 
-    tdUnRefSmaStat(pSma, (SSmaStat *)pStat);
+      tdUnRefSmaStat(pSma, (SSmaStat *)pStat);
-
+    } break;
-  } else {
+    case TASK_TRIGGER_STAT_PAUSED: {
-    smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is inactive",
+      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused",
-             SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
+               SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
+    } break;
+    case TASK_TRIGGER_STAT_INACTIVE: {
+      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is inactive",
+               SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
+    } break;
+    case TASK_TRIGGER_STAT_INIT: {
+      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is init", SMA_VID(pSma),
+               pItem->level, pItem->pRsmaInfo->suid);
+    } break;
+    default: {
+      smaWarn("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is unknown",
+              SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
+    } break;
   }
+
 _end:
   taosReleaseRef(smaMgmt.smaRef, pItem->refId);
 }
@@ -722,7 +743,7 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma) {
+static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) {
   SVnode *pVnode = pSma->pVnode;
 
   SArray *suidList = taosArrayInit(1, sizeof(tb_uid_t));
@@ -732,7 +753,12 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma) {
     return TSDB_CODE_FAILED;
   }
 
-  int32_t arrSize = taosArrayGetSize(suidList);
+  int64_t arrSize = taosArrayGetSize(suidList);
+
+  if (nTables) {
+    *nTables = arrSize;
+  }
+
   if (arrSize == 0) {
     taosArrayDestroy(suidList);
     smaDebug("vgId:%d, no need to restore rsma env since empty stb id list", TD_VID(pVnode));
@@ -741,9 +767,9 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma) {
 
   SMetaReader mr = {0};
   metaReaderInit(&mr, SMA_META(pSma), 0);
-  for (int32_t i = 0; i < arrSize; ++i) {
+  for (int64_t i = 0; i < arrSize; ++i) {
     tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i);
-    smaDebug("vgId:%d, rsma restore, suid[%d] is %" PRIi64, TD_VID(pVnode), i, suid);
+    smaDebug("vgId:%d, rsma restore, suid is %" PRIi64, TD_VID(pVnode), suid);
     if (metaGetTableEntryByUid(&mr, suid) < 0) {
       smaError("vgId:%d, rsma restore, failed to get table meta for %" PRIi64 " since %s", TD_VID(pVnode), suid,
                terrstr());
@@ -777,17 +803,24 @@ _err:
   return TSDB_CODE_FAILED;
 }
 
-static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma) {
+static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int64_t *committed) {
   SVnode *pVnode = pSma->pVnode;
   STFile  tFile = {0};
-  char    qTaskInfoFName[TSDB_FILENAME_LEN];
+  char    qTaskInfoFName[TSDB_FILENAME_LEN] = {0};
 
-  tdRSmaQTaskInfoGetFName(TD_VID(pVnode), TD_QTASK_TMP_F, qTaskInfoFName);
+  tdRSmaQTaskInfoGetFName(TD_VID(pVnode), pVnode->state.committed, qTaskInfoFName);
-  if (tdInitTFile(&tFile, pVnode->pTfs, qTaskInfoFName) < 0) {
+  if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) {
     goto _err;
   }
 
   if (!taosCheckExistFile(TD_TFILE_FULL_NAME(&tFile))) {
+    if (pVnode->state.committed > 0) {
+      smaWarn("vgId:%d, rsma restore for version %" PRIi64 ", not start as %s not exist", TD_VID(pVnode),
+              pVnode->state.committed, TD_TFILE_FULL_NAME(&tFile));
+    } else {
+      smaDebug("vgId:%d, rsma restore for version %" PRIi64 ", no need as %s not exist", TD_VID(pVnode),
+               pVnode->state.committed, TD_TFILE_FULL_NAME(&tFile));
+    }
     return TSDB_CODE_SUCCESS;
   }
 
@@ -799,20 +832,28 @@ static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma) {
   if (tdRSmaQTaskInfoIterInit(&fIter, &tFile) < 0) {
     tdRSmaQTaskInfoIterDestroy(&fIter);
     tdCloseTFile(&tFile);
+    tdDestroyTFile(&tFile);
     goto _err;
   }
 
   if (tdRSmaQTaskInfoRestore(pSma, &fIter) < 0) {
     tdRSmaQTaskInfoIterDestroy(&fIter);
     tdCloseTFile(&tFile);
+    tdDestroyTFile(&tFile);
     goto _err;
   }
 
   tdRSmaQTaskInfoIterDestroy(&fIter);
   tdCloseTFile(&tFile);
+  tdDestroyTFile(&tFile);
+
+  // restored successfully from committed
+  *committed = pVnode->state.committed;
+
   return TSDB_CODE_SUCCESS;
 _err:
-  smaError("rsma restore, qtaskinfo reload failed since %s", terrstr());
+  smaError("vgId:%d, rsma restore for version %" PRIi64 ", qtaskinfo reload failed since %s", TD_VID(pVnode),
+           pVnode->state.committed, terrstr());
   return TSDB_CODE_FAILED;
 }
 
@@ -820,35 +861,45 @@ _err:
  * @brief reload ts data from checkpoint
  *
  * @param pSma
+ * @param committed restore from committed version
  * @return int32_t
  */
-static int32_t tdRSmaRestoreTSDataReload(SSma *pSma) {
+static int32_t tdRSmaRestoreTSDataReload(SSma *pSma, int64_t committed) {
   // TODO
+  smaDebug("vgId:%d, rsma restore from %" PRIi64 ", ts data reload success", SMA_VID(pSma), committed);
   return TSDB_CODE_SUCCESS;
 _err:
-  smaError("rsma restore, ts data reload failed since %s", terrstr());
+  smaError("vgId:%d, rsma restore from %" PRIi64 ", ts data reload failed since %s", SMA_VID(pSma), committed,
+           terrstr());
   return TSDB_CODE_FAILED;
 }
 
 int32_t tdProcessRSmaRestoreImpl(SSma *pSma) {
   // step 1: iterate all stables to restore the rsma env
-  if (tdRSmaRestoreQTaskInfoInit(pSma) < 0) {
+  int64_t nTables = 0;
+  if (tdRSmaRestoreQTaskInfoInit(pSma, &nTables) < 0) {
     goto _err;
   }
 
+  if (nTables <= 0) {
+    smaDebug("vgId:%d, no need to restore rsma task since no tables", SMA_VID(pSma));
+    return TSDB_CODE_SUCCESS;
+  }
+
   // step 2: retrieve qtaskinfo items from the persistence file(rsma/qtaskinfo) and restore
-  if (tdRSmaRestoreQTaskInfoReload(pSma) < 0) {
+  int64_t committed = -1;
+  if (tdRSmaRestoreQTaskInfoReload(pSma, &committed) < 0) {
     goto _err;
   }
 
   // step 3: reload ts data from checkpoint
-  if (tdRSmaRestoreTSDataReload(pSma) < 0) {
+  if (tdRSmaRestoreTSDataReload(pSma, committed) < 0) {
     goto _err;
   }
 
   return TSDB_CODE_SUCCESS;
 _err:
-  smaError("failed to restore rsma task since %s", terrstr());
+  smaError("vgId:%d failed to restore rsma task since %s", SMA_VID(pSma), terrstr());
   return TSDB_CODE_FAILED;
 }
 
@@ -864,9 +915,9 @@ static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *
     return TSDB_CODE_SUCCESS;
   }
 
-  if (pItem->type == 1) {
+  if (pItem->type == TSDB_RETENTION_L1) {
     qTaskInfo = pRSmaInfo->items[0].taskInfo;
-  } else if (pItem->type == 2) {
+  } else if (pItem->type == TSDB_RETENTION_L2) {
     qTaskInfo = pRSmaInfo->items[1].taskInfo;
   } else {
     ASSERT(0);
@@ -931,19 +982,21 @@ static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isF
   }
 
   if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET) < 0) {
-    ASSERT(0);
     return TSDB_CODE_FAILED;
   }
 
   if (tdReadTFile(pTFile, pIter->qBuf, nBytes) != nBytes) {
-    ASSERT(0);
     return TSDB_CODE_FAILED;
   }
 
   int32_t infoLen = 0;
   taosDecodeFixedI32(pIter->qBuf, &infoLen);
   if (infoLen > nBytes) {
-    ASSERT(infoLen > RSMA_QTASKINFO_BUFSIZE);
+    if (infoLen <= RSMA_QTASKINFO_BUFSIZE) {
+      terrno = TSDB_CODE_RSMA_FILE_CORRUPTED;
+      smaError("iterate rsma qtaskinfo file %s failed since %s", TD_TFILE_FULL_NAME(pIter->pTFile), terrstr());
+      return TSDB_CODE_FAILED;
+    }
     pIter->nAlloc = infoLen;
     void *pBuf = taosMemoryRealloc(pIter->pBuf, infoLen);
     if (!pBuf) {
@@ -955,12 +1008,10 @@ static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isF
     nBytes = infoLen;
 
     if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET)) {
-      ASSERT(0);
       return TSDB_CODE_FAILED;
     }
 
     if (tdReadTFile(pTFile, pIter->pBuf, nBytes) != nBytes) {
-      ASSERT(0);
       return TSDB_CODE_FAILED;
     }
   }
@@ -977,7 +1028,6 @@ static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter) {
     // block iter
     bool isFinish = false;
     if (tdRSmaQTaskInfoIterNextBlock(pIter, &isFinish) < 0) {
-      ASSERT(0);
       return TSDB_CODE_FAILED;
     }
     if (isFinish) {
@@ -989,6 +1039,8 @@ static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter) {
     pIter->qBuf = taosDecodeFixedI32(pIter->qBuf, &qTaskInfoLenWithHead);
     if (qTaskInfoLenWithHead < RSMA_QTASKINFO_HEAD_LEN) {
       terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+      smaError("vgId:%d, restore rsma qtaskinfo file %s failed since %s", SMA_VID(pSma),
+               TD_TFILE_FULL_NAME(pIter->pTFile), terrstr());
       return TSDB_CODE_FAILED;
     }
 
@@ -1025,22 +1077,20 @@ static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter) {
   return TSDB_CODE_SUCCESS;
 }
 
-static void *tdRSmaPersistExec(void *param) {
+int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat) {
-  setThreadName("rsma-task-persist");
+  SSma   *pSma = pRSmaStat->pSma;
-  SRSmaStat *pRSmaStat = param;
+  SVnode *pVnode = pSma->pVnode;
-  SSma      *pSma = pRSmaStat->pSma;
+  int32_t vid = SMA_VID(pSma);
-  STfs      *pTfs = pSma->pVnode->pTfs;
+  int64_t toffset = 0;
-  int32_t    vid = SMA_VID(pSma);
+  bool    isFileCreated = false;
-  int64_t    toffset = 0;
-  bool       isFileCreated = false;
 
-  if (TASK_TRIGGER_STAT_CANCELLED == atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat))) {
+  if (taosHashGetSize(RSMA_INFO_HASH(pRSmaStat)) <= 0) {
-    goto _end;
+    return TSDB_CODE_SUCCESS;
   }
 
   void *infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL);
   if (!infoHash) {
-    goto _end;
+    return TSDB_CODE_SUCCESS;
   }
 
   STFile tFile = {0};
@@ -1074,9 +1124,17 @@ static void *tdRSmaPersistExec(void *param) {
 
       if (!isFileCreated) {
         char qTaskInfoFName[TSDB_FILENAME_LEN];
-        tdRSmaQTaskInfoGetFName(vid, TD_QTASK_TMP_F, qTaskInfoFName);
+        tdRSmaQTaskInfoGetFName(vid, pSma->pVnode->state.applied, qTaskInfoFName);
-        tdInitTFile(&tFile, pTfs, qTaskInfoFName);
+        if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) {
-        tdCreateTFile(&tFile, pTfs, true, -1);
+          smaError("vgId:%d, rsma persit, init %s failed since %s", vid, qTaskInfoFName, terrstr());
+          goto _err;
+        }
+        if (tdCreateTFile(&tFile, true, -1) < 0) {
+          smaError("vgId:%d, rsma persit, create %s failed since %s", vid, TD_TFILE_FULL_NAME(&tFile), terrstr());
+          goto _err;
+        }
+        smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d serialize qTaskInfo, file %s created", vid, pRSmaInfo->suid,
+                 i + 1, TD_TFILE_FULL_NAME(&tFile));
 
         isFileCreated = true;
       }
@@ -1101,49 +1159,56 @@ static void *tdRSmaPersistExec(void *param) {
 
     infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), infoHash);
   }
-_normal:
+
   if (isFileCreated) {
     if (tdUpdateTFileHeader(&tFile) < 0) {
       smaError("vgId:%d, rsma, failed to update tfile %s header since %s", vid, TD_TFILE_FULL_NAME(&tFile),
                tstrerror(terrno));
-      tdCloseTFile(&tFile);
-      tdRemoveTFile(&tFile);
       goto _err;
     } else {
       smaDebug("vgId:%d, rsma, succeed to update tfile %s header", vid, TD_TFILE_FULL_NAME(&tFile));
     }
 
     tdCloseTFile(&tFile);
-
+    tdDestroyTFile(&tFile);
-    char newFName[TSDB_FILENAME_LEN];
-    strncpy(newFName, TD_TFILE_FULL_NAME(&tFile), TSDB_FILENAME_LEN);
-    char *pos = strstr(newFName, tdQTaskInfoFname[TD_QTASK_TMP_F]);
-    strncpy(pos, tdQTaskInfoFname[TD_QTASK_TMP_F], TSDB_FILENAME_LEN - POINTER_DISTANCE(pos, newFName));
-    if (taosRenameFile(TD_TFILE_FULL_NAME(&tFile), newFName) != 0) {
-      smaError("vgId:%d, rsma, failed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName);
-      goto _err;
-    } else {
-      smaDebug("vgId:%d, rsma, succeed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName);
-    }
   }
-  goto _end;
+  return TSDB_CODE_SUCCESS;
 _err:
+  smaError("vgId:%d, rsma persit failed since %s", vid, terrstr());
   if (isFileCreated) {
     tdRemoveTFile(&tFile);
+    tdDestroyTFile(&tFile);
   }
+  return TSDB_CODE_FAILED;
+}
+
+static void *tdRSmaPersistExec(void *param) {
+  setThreadName("rsma-task-persist");
+  SRSmaStat *pRSmaStat = param;
+  SSma      *pSma = pRSmaStat->pSma;
+
+  int8_t triggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat));
+
+  if (TASK_TRIGGER_STAT_CANCELLED == triggerStat || TASK_TRIGGER_STAT_PAUSED == triggerStat) {
+    goto _end;
+  }
+
+  // execution
+  tdRSmaPersistExecImpl(pRSmaStat);
+
 _end:
   if (TASK_TRIGGER_STAT_INACTIVE == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat),
                                                                   TASK_TRIGGER_STAT_INACTIVE,
                                                                   TASK_TRIGGER_STAT_ACTIVE)) {
-    smaDebug("vgId:%d, rsma persist task is active again", vid);
+    smaDebug("vgId:%d, rsma persist task is active again", SMA_VID(pSma));
   } else if (TASK_TRIGGER_STAT_CANCELLED == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat),
                                                                           TASK_TRIGGER_STAT_CANCELLED,
                                                                           TASK_TRIGGER_STAT_FINISHED)) {
-    smaDebug("vgId:%d, rsma persist task is cancelled", vid);
+    smaDebug("vgId:%d, rsma persist task is cancelled", SMA_VID(pSma));
   } else {
-    smaWarn("vgId:%d, rsma persist task in abnormal stat %" PRIi8, vid, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)));
+    smaWarn("vgId:%d, rsma persist task in stat %" PRIi8, SMA_VID(pSma), atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)));
-    ASSERT(0);
   }
+
   atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
   taosReleaseRef(smaMgmt.smaRef, pRSmaStat->refId);
   taosThreadExit(NULL);
@@ -1166,9 +1231,8 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) {
                                                                             TASK_TRIGGER_STAT_FINISHED)) {
       smaDebug("vgId:%d, persist task is cancelled and set finished", SMA_VID(pRSmaStat->pSma));
     } else {
-      smaWarn("vgId:%d, persist task in abnormal stat %" PRIi8, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)),
+      smaWarn("vgId:%d, persist task in abnormal stat %" PRIi8, SMA_VID(pRSmaStat->pSma),
-              SMA_VID(pRSmaStat->pSma));
+              atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)));
-      ASSERT(0);
     }
     atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
     taosReleaseRef(smaMgmt.smaRef, pRSmaStat->refId);
@@ -1205,8 +1269,8 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) {
         // start persist task
         tdRSmaPersistTask(pRSmaStat);
 
-        taosTmrReset(tdRSmaPersistTrigger, RSMA_QTASKINFO_PERSIST_MS, pRSmaStat, pRSmaStat->tmrHandle,
+        // taosTmrReset(tdRSmaPersistTrigger, 5000, pRSmaStat, pRSmaStat->tmrHandle,
-                     &pRSmaStat->tmrId);
+        //              RSMA_TMR_ID(pRSmaStat));
       } else {
         atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
       }
@@ -1216,6 +1280,9 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) {
       atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_FINISHED);
       smaDebug("rsma persistence not start since cancelled and finished");
     } break;
+    case TASK_TRIGGER_STAT_PAUSED: {
+      smaDebug("rsma persistence not start since paused");
+    } break;
     case TASK_TRIGGER_STAT_INACTIVE: {
       smaDebug("rsma persistence not start since inactive");
     } break;

source/dnode/vnode/src/sma/smaSnapshot.c

@@ -0,0 +1,16 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "sma.h"

source/dnode/vnode/src/sma/smaUtil.c

@@ -140,7 +140,7 @@ int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset)
     return -1;
   }
 
-#if 1
+#if 0
   smaDebug("append to file %s, offset:%" PRIi64 " nbyte:%" PRIi64 " fsize:%" PRIi64, TD_TFILE_FULL_NAME(pTFile),
            toffset, nbyte, toffset + nbyte);
 #endif
@@ -179,52 +179,85 @@ void tdCloseTFile(STFile *pTFile) {
   }
 }
 
-void tdGetVndFileName(int32_t vgId, const char *dname, const char *fname, char *outputName) {
+void tdDestroyTFile(STFile *pTFile) { taosMemoryFreeClear(TD_TFILE_FULL_NAME(pTFile)); }
-  snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/%s", vgId, dname, fname);
+
+void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version,
+                      char *outputName) {
+  if (version < 0) {
+    if (pdname) {
+      snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%sv%d%s", pdname, TD_DIRSEP, TD_DIRSEP, vgId,
+               TD_DIRSEP, dname, TD_DIRSEP, vgId, fname);
+    } else {
+      snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%sv%d%s", TD_DIRSEP, vgId, TD_DIRSEP, dname, TD_DIRSEP,
+               vgId, fname);
+    }
+  } else {
+    if (pdname) {
+      snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%sv%d%s%" PRIi64, pdname, TD_DIRSEP, TD_DIRSEP,
+               vgId, TD_DIRSEP, dname, TD_DIRSEP, vgId, fname, version);
+    } else {
+      snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%sv%d%s%" PRIi64, TD_DIRSEP, vgId, TD_DIRSEP, dname,
+               TD_DIRSEP, vgId, fname, version);
+    }
+  }
 }
 
-int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname) {
+void tdGetVndDirName(int32_t vgId, const char *pdname, const char *dname, bool endWithSep, char *outputName) {
-  char    fullname[TSDB_FILENAME_LEN];
+  if (pdname) {
-  SDiskID did = {0};
+    if (endWithSep) {
+      snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%s", pdname, TD_DIRSEP, TD_DIRSEP, vgId, TD_DIRSEP,
+               dname, TD_DIRSEP);
+    } else {
+      snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s", pdname, TD_DIRSEP, TD_DIRSEP, vgId, TD_DIRSEP,
+               dname);
+    }
+  } else {
+    if (endWithSep) {
+      snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%s", TD_DIRSEP, vgId, TD_DIRSEP, dname, TD_DIRSEP);
+    } else {
+      snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s", TD_DIRSEP, vgId, TD_DIRSEP, dname);
+    }
+  }
+}
 
+int32_t tdInitTFile(STFile *pTFile, const char *dname, const char *fname) {
   TD_TFILE_SET_STATE(pTFile, TD_FILE_STATE_OK);
   TD_TFILE_SET_CLOSED(pTFile);
 
   memset(&(pTFile->info), 0, sizeof(pTFile->info));
   pTFile->info.magic = TD_FILE_INIT_MAGIC;
 
-  if (tfsAllocDisk(pTfs, 0, &did) < 0) {
+  char tmpName[TSDB_FILENAME_LEN * 2 + 32] = {0};
-    terrno = TSDB_CODE_NO_AVAIL_DISK;
+  snprintf(tmpName, TSDB_FILENAME_LEN * 2 + 32, "%s%s%s", dname, TD_DIRSEP, fname);
+  int32_t tmpNameLen = strlen(tmpName) + 1;
+  pTFile->fname = taosMemoryMalloc(tmpNameLen);
+  if (!pTFile->fname) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
-
+  tstrncpy(pTFile->fname, tmpName, tmpNameLen);
-  tfsInitFile(pTfs, &(pTFile->f), did, fname);
 
   return 0;
 }
 
-int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fType) {
+int32_t tdCreateTFile(STFile *pTFile, bool updateHeader, int8_t fType) {
   ASSERT(pTFile->info.fsize == 0 && pTFile->info.magic == TD_FILE_INIT_MAGIC);
-
   pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pTFile->pFile == NULL) {
     if (errno == ENOENT) {
       // Try to create directory recursively
-      char *s = strdup(TD_TFILE_REL_NAME(pTFile));
+      char *s = strdup(TD_TFILE_FULL_NAME(pTFile));
-      if (tfsMkdirRecurAt(pTfs, taosDirName(s), TD_TFILE_DID(pTFile)) < 0) {
+      if (taosMulMkDir(taosDirName(s)) != 0) {
-        taosMemoryFreeClear(s);
+        terrno = TAOS_SYSTEM_ERROR(errno);
+        taosMemoryFree(s);
         return -1;
       }
-      taosMemoryFreeClear(s);
+      taosMemoryFree(s);
-
       pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
       if (pTFile->pFile == NULL) {
         terrno = TAOS_SYSTEM_ERROR(errno);
         return -1;
       }
-    } else {
-      terrno = TAOS_SYSTEM_ERROR(errno);
-      return -1;
     }
   }
 
@@ -244,7 +277,13 @@ int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fTyp
   return 0;
 }
 
-int32_t tdRemoveTFile(STFile *pTFile) { return tfsRemoveFile(TD_TFILE_F(pTFile)); }
+int32_t tdRemoveTFile(STFile *pTFile) {
+  if (taosRemoveFile(TD_TFILE_FULL_NAME(pTFile)) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  };
+  return 0;
+}
 
 // smaXXXUtil ================
 // ...

source/dnode/vnode/src/tq/tq.c

@@ -183,13 +183,15 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) {
   } else {
     ASSERT(0);
   }
-  STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, offset.subKey);
+  /*STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, offset.subKey);*/
-  if (pOffset == NULL || pOffset->val.version < offset.val.version) {
+  /*if (pOffset != NULL) {*/
-    if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) {
+  /*if (pOffset->val.type == TMQ_OFFSET__LOG && pOffset->val.version < offset.val.version) {*/
-      ASSERT(0);
+  if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) {
-      return -1;
+    ASSERT(0);
-    }
+    return -1;
   }
+  /*}*/
+  /*}*/
 
   return 0;
 }
@@ -375,8 +377,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
 
     taosMemoryFree(pCkHead);
   } else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_DATA) {
-    tqInfo("retrieve using snapshot req offset: uid %ld ts %ld, actual offset: uid %ld ts %ld", dataRsp.reqOffset.uid,
+    tqInfo("retrieve using snapshot actual offset: uid %ld ts %ld", fetchOffsetNew.uid, fetchOffsetNew.ts);
-           dataRsp.reqOffset.ts, fetchOffsetNew.uid, fetchOffsetNew.ts);
     if (tqScanSnapshot(pTq, &pHandle->execHandle, &dataRsp, fetchOffsetNew, workerId) < 0) {
       ASSERT(0);
     }
@@ -405,193 +406,6 @@ OVER:
   return code;
 }
 
-#if 0
-int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
-  SMqPollReq* pReq = pMsg->pCont;
-  int64_t     consumerId = pReq->consumerId;
-  int64_t     timeout = pReq->timeout;
-  int32_t     reqEpoch = pReq->epoch;
-  int64_t     fetchOffset;
-  int32_t     code = 0;
-
-  // get offset to fetch message
-  if (pReq->currentOffset >= 0) {
-    fetchOffset = pReq->currentOffset + 1;
-  } else {
-    STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pReq->subKey);
-    if (pOffset != NULL) {
-      ASSERT(pOffset->val.type == TMQ_OFFSET__LOG);
-      tqDebug("consumer %ld, restore offset of %s on vg %d, offset(type:log) version: %ld", consumerId, pReq->subKey,
-              TD_VID(pTq->pVnode), pOffset->val.version);
-      fetchOffset = pOffset->val.version + 1;
-    } else {
-      if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) {
-        fetchOffset = walGetFirstVer(pTq->pWal);
-      } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) {
-        fetchOffset = walGetCommittedVer(pTq->pWal);
-      } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__NONE) {
-        tqError("tmq poll: no offset committed for consumer %ld in vg %d, subkey %s", consumerId, TD_VID(pTq->pVnode),
-                pReq->subKey);
-        terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET;
-        return -1;
-      }
-      tqDebug("consumer %ld, restore offset of %s on vg %d failed, config is %ld, set to %ld", consumerId, pReq->subKey,
-              TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset);
-    }
-  }
-
-  tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req offset %ld fetch offset %ld", consumerId,
-          pReq->epoch, TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset);
-
-  STqHandle* pHandle = taosHashGet(pTq->handles, pReq->subKey, strlen(pReq->subKey));
-  /*ASSERT(pHandle);*/
-  if (pHandle == NULL) {
-    tqError("tmq poll: no consumer handle for consumer %ld in vg %d, subkey %s", consumerId, TD_VID(pTq->pVnode),
-            pReq->subKey);
-    return -1;
-  }
-
-  if (pHandle->consumerId != consumerId) {
-    tqError("tmq poll: consumer handle mismatch for consumer %ld in vg %d, subkey %s, handle consumer id %ld",
-            consumerId, TD_VID(pTq->pVnode), pReq->subKey, pHandle->consumerId);
-    return -1;
-  }
-
-  int32_t consumerEpoch = atomic_load_32(&pHandle->epoch);
-  while (consumerEpoch < reqEpoch) {
-    consumerEpoch = atomic_val_compare_exchange_32(&pHandle->epoch, consumerEpoch, reqEpoch);
-  }
-
-  SMqDataBlkRsp rsp = {0};
-  rsp.reqOffset = pReq->currentOffset;
-
-  rsp.blockData = taosArrayInit(0, sizeof(void*));
-  rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
-
-  if (rsp.blockData == NULL || rsp.blockDataLen == NULL) {
-    return -1;
-  }
-
-  rsp.withTbName = pReq->withTbName;
-  if (rsp.withTbName) {
-    rsp.blockTbName = taosArrayInit(0, sizeof(void*));
-  }
-
-  if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
-    rsp.withSchema = false;
-  } else {
-    rsp.withSchema = true;
-    rsp.blockSchema = taosArrayInit(0, sizeof(void*));
-  }
-
-#if 1
-  if (pReq->useSnapshot) {
-    // TODO set ver into snapshot
-    int64_t lastVer = walGetCommittedVer(pTq->pWal);
-    if (rsp.reqOffset < lastVer) {
-      tqInfo("retrieve using snapshot req offset %ld last ver %ld", rsp.reqOffset, lastVer);
-      tqScanSnapshot(pTq, &pHandle->execHandle, &rsp, workerId);
-
-      if (rsp.blockNum != 0) {
-        rsp.withTbName = false;
-        rsp.rspOffset = lastVer;
-        tqInfo("direct send by snapshot req offset %ld rsp offset %ld", rsp.reqOffset, rsp.rspOffset);
-        fetchOffset = lastVer;
-        goto SEND_RSP;
-      }
-    }
-  }
-#endif
-
-  SWalHead* pHeadWithCkSum = taosMemoryMalloc(sizeof(SWalHead) + 2048);
-  if (pHeadWithCkSum == NULL) {
-    return -1;
-  }
-
-  walSetReaderCapacity(pHandle->pWalReader, 2048);
-
-  while (1) {
-    consumerEpoch = atomic_load_32(&pHandle->epoch);
-    if (consumerEpoch > reqEpoch) {
-      tqWarn("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d, discard req epoch %d",
-             consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch);
-      break;
-    }
-
-    if (tqFetchLog(pTq, pHandle, &fetchOffset, &pHeadWithCkSum) < 0) {
-      // TODO add push mgr
-      break;
-    }
-
-    SWalCont* pHead = &pHeadWithCkSum->head;
-
-    tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
-            TD_VID(pTq->pVnode), fetchOffset, pHead->msgType);
-
-    if (pHead->msgType == TDMT_VND_SUBMIT) {
-      SSubmitReq* pCont = (SSubmitReq*)&pHead->body;
-
-      if (tqDataExec(pTq, &pHandle->execHandle, pCont, &rsp, workerId) < 0) {
-        /*ASSERT(0);*/
-      }
-    } else {
-      ASSERT(pHandle->fetchMeta);
-      ASSERT(IS_META_MSG(pHead->msgType));
-      tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType);
-      SMqMetaRsp metaRsp = {0};
-      metaRsp.reqOffset = pReq->currentOffset;
-      metaRsp.rspOffset = fetchOffset;
-      metaRsp.resMsgType = pHead->msgType;
-      metaRsp.metaRspLen = pHead->bodyLen;
-      metaRsp.metaRsp = pHead->body;
-      if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) {
-        code = -1;
-        goto OVER;
-      }
-      code = 0;
-      goto OVER;
-    }
-
-    // TODO batch optimization:
-    // TODO continue scan until meeting batch requirement
-    if (rsp.blockNum > 0 /* threshold */) {
-      break;
-    } else {
-      fetchOffset++;
-    }
-  }
-
-  taosMemoryFree(pHeadWithCkSum);
-
-SEND_RSP:
-  ASSERT(taosArrayGetSize(rsp.blockData) == rsp.blockNum);
-  ASSERT(taosArrayGetSize(rsp.blockDataLen) == rsp.blockNum);
-  if (rsp.withSchema) {
-    ASSERT(taosArrayGetSize(rsp.blockSchema) == rsp.blockNum);
-  }
-
-  rsp.rspOffset = fetchOffset;
-
-  if (tqSendDataRsp(pTq, pMsg, pReq, &rsp) < 0) {
-    code = -1;
-  }
-OVER:
-  // TODO wrap in destroy func
-  taosArrayDestroy(rsp.blockDataLen);
-  taosArrayDestroyP(rsp.blockData, (FDelete)taosMemoryFree);
-
-  if (rsp.withSchema) {
-    taosArrayDestroyP(rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper);
-  }
-
-  if (rsp.withTbName) {
-    taosArrayDestroyP(rsp.blockTbName, (FDelete)taosMemoryFree);
-  }
-
-  return code;
-}
-#endif
-
 int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen) {
   SMqVDeleteReq* pReq = (SMqVDeleteReq*)msg;
 

source/dnode/vnode/src/tq/tqRead.c

@@ -120,7 +120,9 @@ bool tqNextDataBlock(SStreamReader* pHandle) {
       return true;
     }
     void* ret = taosHashGet(pHandle->tbIdHash, &pHandle->msgIter.uid, sizeof(int64_t));
+    /*tqDebug("search uid %ld", pHandle->msgIter.uid);*/
     if (ret != NULL) {
+      /*tqDebug("find   uid %ld", pHandle->msgIter.uid);*/
       return true;
     }
   }

source/dnode/vnode/src/tq/tqSink.c

@@ -185,5 +185,7 @@ void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) {
       .contLen = ntohl(pReq->length),
   };
 
-  ASSERT(tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) == 0);
+  if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) {
+    tqDebug("failed to put into write-queue since %s", terrstr());
+  }
 }

source/dnode/vnode/src/vnd/vnodeCfg.c

@@ -28,12 +28,12 @@ const SVnodeCfg vnodeCfgDefault = {
                 .update = 1,
                 .compression = 2,
                 .slLevel = 5,
-                .days = 10,
+                .days = 14400,
                 .minRows = 100,
                 .maxRows = 4096,
-                .keep2 = 3650,
+                .keep2 = 5256000,
-                .keep0 = 3650,
+                .keep0 = 5256000,
-                .keep1 = 3650},
+                .keep1 = 5256000},
     .walCfg =
         {.vgId = -1, .fsyncPeriod = 0, .retentionPeriod = 0, .rollPeriod = 0, .segSize = 0, .level = TAOS_WAL_WRITE},
     .hashBegin = 0,

source/dnode/vnode/src/vnd/vnodeCommit.c

@@ -69,6 +69,9 @@ int vnodeBegin(SVnode *pVnode) {
     }
   }
 
+  // begin sma
+  smaBegin(pVnode->pSma);  // TODO: refactor to include the rsma1/rsma2 tsdbBegin() after tsdb_refact branch merged
+
   return 0;
 }
 
@@ -229,6 +232,9 @@ int vnodeCommit(SVnode *pVnode) {
     return -1;
   }
 
+  // preCommit
+  smaPreCommit(pVnode->pSma);
+
   // commit each sub-system
   if (metaCommit(pVnode->pMeta) < 0) {
     ASSERT(0);
@@ -269,6 +275,9 @@ int vnodeCommit(SVnode *pVnode) {
   
   pVnode->state.committed = info.state.committed;
 
+  // postCommit
+  smaPostCommit(pVnode->pSma);
+
   // apply the commit (TODO)
   vnodeBufPoolReset(pVnode->onCommit);
   pVnode->onCommit->next = pVnode->pPool;

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -152,12 +152,11 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
   return pVnode;
 
 _err:
-  if (pVnode->pSma) smaCloseEnv(pVnode->pSma);
   if (pVnode->pQuery) vnodeQueryClose(pVnode);
   if (pVnode->pTq) tqClose(pVnode->pTq);
   if (pVnode->pWal) walClose(pVnode->pWal);
   if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb);
-  if (pVnode->pSma) smaCloseEx(pVnode->pSma);
+  if (pVnode->pSma) smaClose(pVnode->pSma);
   if (pVnode->pMeta) metaClose(pVnode->pMeta);
 
   tsem_destroy(&(pVnode->canCommit));
@@ -167,14 +166,13 @@ _err:
 
 void vnodeClose(SVnode *pVnode) {
   if (pVnode) {
-    smaCloseEnv(pVnode->pSma);
     vnodeCommit(pVnode);
     vnodeSyncClose(pVnode);
     vnodeQueryClose(pVnode);
     walClose(pVnode->pWal);
     tqClose(pVnode->pTq);
     if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb);
-    smaCloseEx(pVnode->pSma);
+    smaClose(pVnode->pSma);
     metaClose(pVnode->pMeta);
     vnodeCloseBufPool(pVnode);
     // destroy handle

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -779,6 +779,7 @@ _exit:
   taosArrayDestroy(submitRsp.pArray);
 
   // TODO: the partial success scenario and the error case
+  // => If partial success, extract the success submitted rows and reconstruct a new submit msg, and push to level 1/level 2.
   // TODO: refactor
   if ((terrno == TSDB_CODE_SUCCESS) && (pRsp->code == TSDB_CODE_SUCCESS)) {
     tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_INPUT__DATA_SUBMIT);

source/libs/catalog/inc/catalogInt.h

@@ -278,7 +278,7 @@ typedef struct SCtgAsyncFps {
 
 typedef struct SCtgApiStat {
 
-#ifdef WINDOWS
+#if defined(WINDOWS) || defined(_TD_DARWIN_64)
   size_t avoidCompilationErrors;
 #endif
 

source/libs/catalog/test/CMakeLists.txt

@@ -1,23 +1,25 @@
 
 MESSAGE(STATUS "build catalog unit test")
 
-# GoogleTest requires at least C++11
+IF(NOT TD_DARWIN)
-SET(CMAKE_CXX_STANDARD 11)
+        # GoogleTest requires at least C++11
-AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+        SET(CMAKE_CXX_STANDARD 11)
+        AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
-ADD_EXECUTABLE(catalogTest ${SOURCE_LIST})
+        ADD_EXECUTABLE(catalogTest ${SOURCE_LIST})
-TARGET_LINK_LIBRARIES(
+        TARGET_LINK_LIBRARIES(
-        catalogTest
+                catalogTest
-        PUBLIC os util common catalog transport gtest qcom taos_static
+                PUBLIC os util common catalog transport gtest qcom taos_static
-)
+        )
 
-TARGET_INCLUDE_DIRECTORIES(
+        TARGET_INCLUDE_DIRECTORIES(
-        catalogTest
+                catalogTest
-        PUBLIC "${TD_SOURCE_DIR}/include/libs/catalog/"
+                PUBLIC "${TD_SOURCE_DIR}/include/libs/catalog/"
-        PRIVATE "${TD_SOURCE_DIR}/source/libs/catalog/inc"
+                PRIVATE "${TD_SOURCE_DIR}/source/libs/catalog/inc"
-)
+        )
 
-# add_test(
+        # add_test(
-#     NAME catalogTest
+        #     NAME catalogTest
-#     COMMAND catalogTest
+        #     COMMAND catalogTest
-# )
+        # )
+ENDIF()

source/libs/executor/inc/executorimpl.h

@@ -356,6 +356,7 @@ typedef struct SStreamBlockScanInfo {
   SUpdateInfo*    pUpdateInfo;
 
   EStreamScanMode scanMode;
+  SOperatorInfo* pStreamScanOp;
   SOperatorInfo* pSnapshotReadOp;
   SArray*        childIds;
   SessionWindowSupporter sessionSup;
@@ -427,7 +428,7 @@ typedef struct SIntervalAggOperatorInfo {
   STimeWindowAggSupp twAggSup;
   bool               invertible;
   SArray*            pPrevValues;        //  SArray<SGroupKeys> used to keep the previous not null value for interpolation.
-  bool               ignoreCloseWindow;
+  bool               ignoreExpiredData;
 } SIntervalAggOperatorInfo;
 
 typedef struct SStreamFinalIntervalOperatorInfo {
@@ -449,7 +450,7 @@ typedef struct SStreamFinalIntervalOperatorInfo {
   SArray*            pPullWins;          // SPullWindowInfo
   int32_t            pullIndex;
   SSDataBlock*       pPullDataRes;
-  bool               ignoreCloseWindow;
+  bool               ignoreExpiredData;
 } SStreamFinalIntervalOperatorInfo;
 
 typedef struct SAggOperatorInfo {
@@ -587,7 +588,7 @@ typedef struct SStreamSessionAggOperatorInfo {
   SArray*              pChildren;       // cache for children's result; final stream operator
   SPhysiNode*          pPhyNode;           // create new child
   bool                 isFinal;
-  bool                 ignoreCloseWindow;
+  bool                 ignoreExpiredData;
 } SStreamSessionAggOperatorInfo;
 
 typedef struct STimeSliceOperatorInfo {
@@ -631,7 +632,7 @@ typedef struct SStreamStateAggOperatorInfo {
   void*                pDelIterator;
   SArray*              pScanWindow;
   SArray*              pChildren;       // cache for children's result;
-  bool                 ignoreCloseWindow;
+  bool                 ignoreExpiredData;
 } SStreamStateAggOperatorInfo;
 
 typedef struct SSortedMergeOperatorInfo {

source/libs/executor/src/dataSinkMgt.c

@@ -35,7 +35,7 @@ int32_t dsDataSinkGetCacheSize(SDataSinkStat *pStat) {
 
 
 int32_t dsCreateDataSinker(const SDataSinkNode *pDataSink, DataSinkHandle* pHandle, void* pParam) {
-  switch (nodeType(pDataSink)) {
+  switch ((int)nodeType(pDataSink)) {
     case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
       return createDataDispatcher(&gDataSinkManager, pDataSink, pHandle);
     case QUERY_NODE_PHYSICAL_PLAN_DELETE:

source/libs/executor/src/executor.c

@@ -145,10 +145,12 @@ static SArray* filterQualifiedChildTables(const SStreamBlockScanInfo* pScanInfo,
       continue;
     }
 
+    // TODO handle ntb case
     if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pScanInfo->tableUid) {
       continue;
     }
-    // TODO handle ntb case
+    /*pScanInfo->pStreamScanOp->pTaskInfo->tableqinfoList.*/
+    // handle multiple partition
 
     taosArrayPush(qa, id);
   }

source/libs/executor/src/executorimpl.c

@@ -1422,7 +1422,7 @@ void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t
     return;
   }
 #ifdef BUF_PAGE_DEBUG
-  qDebug("page_setbuf, groupId:%"PRIu64, groupId);
+  qDebug("page_setbuf, groupId:%" PRIu64, groupId);
 #endif
   doSetTableGroupOutputBuf(pOperator, pAggInfo, numOfOutput, groupId);
 
@@ -1570,9 +1570,9 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI
 
     releaseBufPage(pBuf, page);
     pBlock->info.rows += pRow->numOfRows;
-//    if (pBlock->info.rows >= pBlock->info.capacity) {  // output buffer is full
+    //    if (pBlock->info.rows >= pBlock->info.capacity) {  // output buffer is full
-//      break;
+    //      break;
-//    }
+    //    }
   }
 
   qDebug("%s result generated, rows:%d, groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows,
@@ -2027,8 +2027,9 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf
 
   ASSERT(pDataInfo->status == EX_SOURCE_DATA_NOT_READY);
 
-  qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu, GET_TASKID(pTaskInfo),
+  qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu,
-         pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, pSource->execId, sourceIndex, totalSources);
+         GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, pSource->execId,
+         sourceIndex, totalSources);
 
   pMsg->header.vgId = htonl(pSource->addr.nodeId);
   pMsg->sId = htobe64(pSource->schedId);
@@ -2163,8 +2164,8 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx
       SSDataBlock*         pRes = pExchangeInfo->pResult;
       SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo;
       if (pRsp->numOfRows == 0) {
-        qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64
+        qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64
-               ", completed:%d try next %d/%" PRIzu,
+               ", totalRows:%" PRIu64 ", completed:%d try next %d/%" PRIzu,
                GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pDataInfo->totalRows,
                pExchangeInfo->loadInfo.totalRows, completed + 1, i + 1, totalSources);
         pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
@@ -2183,18 +2184,19 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx
       }
 
       if (pRsp->completed == 1) {
-        qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d"
+        qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64
+               " execId:%d"
                " index:%d completed, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64
                ", completed:%d try next %d/%" PRIzu,
-               GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRes->info.rows, pDataInfo->totalRows,
+               GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRes->info.rows,
-               pLoadInfo->totalRows, pLoadInfo->totalSize, completed + 1, i + 1, totalSources);
+               pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, completed + 1, i + 1, totalSources);
         completed += 1;
         pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
       } else {
         qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64
                ", totalBytes:%" PRIu64,
-               GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, pLoadInfo->totalRows,
+               GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows,
-               pLoadInfo->totalSize);
+               pLoadInfo->totalRows, pLoadInfo->totalSize);
       }
 
       taosMemoryFreeClear(pDataInfo->pRsp);
@@ -2267,8 +2269,8 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) {
     SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current);
 
     if (pDataInfo->code != TSDB_CODE_SUCCESS) {
-      qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo), pSource->addr.nodeId,
+      qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo),
-             pSource->taskId, pSource->execId, tstrerror(pDataInfo->code));
+             pSource->addr.nodeId, pSource->taskId, pSource->execId, tstrerror(pDataInfo->code));
       pOperator->pTaskInfo->code = pDataInfo->code;
       return NULL;
     }
@@ -2276,8 +2278,8 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) {
     SRetrieveTableRsp*   pRsp = pDataInfo->pRsp;
     SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo;
     if (pRsp->numOfRows == 0) {
-      qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64
+      qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64
-             " try next",
+             ", totalRows:%" PRIu64 " try next",
              GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pExchangeInfo->current + 1,
              pDataInfo->totalRows, pLoadInfo->totalRows);
 
@@ -2296,16 +2298,17 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) {
     if (pRsp->completed == 1) {
       qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64
              ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu,
-             GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, pDataInfo->totalRows,
+             GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows,
-             pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, totalSources);
+             pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1,
+             totalSources);
 
       pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
       pExchangeInfo->current += 1;
     } else {
       qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64
              ", totalBytes:%" PRIu64,
-             GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, pLoadInfo->totalRows,
+             GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows,
-             pLoadInfo->totalSize);
+             pLoadInfo->totalRows, pLoadInfo->totalSize);
     }
 
     pOperator->resultInfo.totalRows += pRes->info.rows;
@@ -2865,7 +2868,24 @@ int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts) {
       pInfo->cond.twindows[0].skey = oldSkey;
       pInfo->scanTimes = 0;
       pInfo->curTWinIdx = 0;
+
+      SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+
+      int32_t tableSz = taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList);
+      bool    found = false;
+      for (int32_t i = 0; i < tableSz; i++) {
+        STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, i);
+        if (pTableInfo->uid == uid) {
+          found = true;
+          pInfo->currentTable = i;
+        }
+      }
+      // TODO after processing drop,
+      ASSERT(found);
+      qDebug("tsdb reader offset seek to uid %ld ts %ld, table cur set to %d , all table num %d", uid, ts,
+             pInfo->currentTable, tableSz);
     }
+
     return TSDB_CODE_SUCCESS;
 
   } else {
@@ -3250,6 +3270,10 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
       doSetOperatorCompleted(pOperator);
       break;
     }
+    if (pBlock->info.type == STREAM_RETRIEVE) {
+      // for stream interval
+      return pBlock;
+    }
 
     // the pDataBlock are always the same one, no need to call this again
     int32_t code = getTableScanInfo(pOperator->pDownstream[0], &order, &scanFlag);
@@ -4094,14 +4118,14 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
       ASSERT(nodeType(pNew) == QUERY_NODE_VALUE);
       SValueNode* pValue = (SValueNode*)pNew;
 
-      if (pValue->node.resType.type == TSDB_DATA_TYPE_NULL) {
+      if (pValue->node.resType.type == TSDB_DATA_TYPE_NULL || pValue->isNull) {
         isNull[index++] = 1;
         continue;
       } else {
         isNull[index++] = 0;
         char* data = nodesGetValueFromNode(pValue);
-        if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON){
+        if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON) {
-          if(tTagIsJson(data)){
+          if (tTagIsJson(data)) {
             terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR;
             taosMemoryFree(keyBuf);
             nodesClearList(groupNew);
@@ -4166,7 +4190,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) {
       STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode;
       int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
-      if(code){
+      if (code) {
         pTaskInfo->code = code;
         return NULL;
       }
@@ -4195,7 +4219,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       };
       if (pHandle) {
         int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
-        if(code){
+        if (code) {
           pTaskInfo->code = code;
           return NULL;
         }

source/libs/executor/src/scanoperator.c

@@ -807,6 +807,23 @@ static bool isStateWindow(SStreamBlockScanInfo* pInfo) {
   return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
 }
 
+static void setGroupId(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, int32_t groupColIndex, int32_t rowIndex) {
+  ASSERT(rowIndex < pBlock->info.rows);
+  switch (pBlock->info.type)
+  {
+  case STREAM_RETRIEVE: {
+    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, groupColIndex);
+    uint64_t* groupCol = (uint64_t*)pColInfo->pData;
+    pInfo->groupId = groupCol[rowIndex];
+  }
+    break;
+  case STREAM_DELETE_DATA:
+    break;
+  default:
+    break;
+  }
+}
+
 static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
   STimeWindow win = {
       .skey = INT64_MIN,
@@ -829,6 +846,7 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int3
     } else {
       win =
           getActiveTimeWindow(NULL, &dumyInfo, tsCols[(*pRowIndex)], &pInfo->interval, pInfo->interval.precision, NULL);
+      setGroupId(pInfo, pSDB, 2, *pRowIndex);
       (*pRowIndex) += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, (*pRowIndex), win.ekey, binarySearchForKey, NULL,
                                                TSDB_ORDER_ASC);
     }
@@ -1031,10 +1049,12 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
       }
       pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
     } else {
-      if (isStateWindow(pInfo) && taosArrayGetSize(pInfo->sessionSup.pStreamAggSup->pScanWindow) > 0) {
+      if (isStateWindow(pInfo)) {
         pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
         pInfo->updateResIndex = pInfo->pUpdateRes->info.rows;
-        prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
+        if (!prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex)) {
+          pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
+        }
       }
       if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER) {
         SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
@@ -1274,6 +1294,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
   pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1};
   pInfo->groupId = 0;
   pInfo->pPullDataRes = createPullDataBlock();
+  pInfo->pStreamScanOp = pOperator;
 
   pOperator->name = "StreamBlockScanOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN;
@@ -1308,6 +1329,13 @@ static void destroySysScanOperator(void* param, int32_t numOfOutput) {
   taosArrayDestroy(pInfo->scanCols);
 }
 
+static int32_t getSysTableDbNameColId(const char* pTable) {
+  // if (0 == strcmp(TSDB_INS_TABLE_USER_INDEXES, pTable)) {
+  //   return 1;
+  // }
+  return TSDB_INS_USER_STABLES_DBNAME_COLID;
+}
+
 EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) {
   int32_t   code = TSDB_CODE_SUCCESS;
   ENodeType nType = nodeType(pNode);
@@ -1329,7 +1357,7 @@ EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) {
       }
 
       SColumnNode* node = (SColumnNode*)pNode;
-      if (TSDB_INS_USER_STABLES_DBNAME_COLID == node->colId) {
+      if (getSysTableDbNameColId(node->tableName) == node->colId) {
         *(int32_t*)pContext = 2;
         return DEAL_RES_CONTINUE;
       }

source/libs/executor/src/timewindowoperator.c

@@ -838,7 +838,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
   STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval,
                                         pInfo->interval.precision, &pInfo->win);
   int32_t     ret = TSDB_CODE_SUCCESS;
-  if (!pInfo->ignoreCloseWindow || !isCloseWindow(&win, &pInfo->twAggSup)) {
+  if (!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) {
     ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
                                  numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
     if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
@@ -871,7 +871,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
     doWindowBorderInterpolation(pInfo, pBlock, pResult, &win, startPos, forwardRows, pSup);
   }
 
-  if (!pInfo->ignoreCloseWindow || !isCloseWindow(&win, &pInfo->twAggSup)) {
+  if (!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) {
     updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
     doApplyFunctions(pTaskInfo, pSup->pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols,
                      pBlock->info.rows, numOfOutput, pInfo->order);
@@ -886,7 +886,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
     if (startPos < 0) {
       break;
     }
-    if (pInfo->ignoreCloseWindow && isCloseWindow(&nextWin, &pInfo->twAggSup)) {
+    if (pInfo->ignoreExpiredData && isCloseWindow(&nextWin, &pInfo->twAggSup)) {
       ekey = ascScan ? nextWin.ekey : nextWin.skey;
       forwardRows =
           getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order);
@@ -1535,7 +1535,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo*
   pInfo->interval = *pInterval;
   pInfo->execModel = pTaskInfo->execModel;
   pInfo->twAggSup = *pTwAggSupp;
-  pInfo->ignoreCloseWindow = false;
+  pInfo->ignoreExpiredData = pPhyNode->window.igExpired;
 
   if (pPhyNode->window.pExprs != NULL) {
     int32_t    numOfScalar = 0;
@@ -2292,7 +2292,7 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc
                                             pInfo->interval.precision, NULL);
   while (1) {
     bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);
-    if (pInfo->ignoreCloseWindow && isClosed) {
+    if (pInfo->ignoreExpiredData && isClosed) {
       startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
       if (startPos < 0) {
         break;
@@ -2710,7 +2710,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
   _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
   pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK);
   pInfo->pPullDataRes = createPullDataBlock();
-  pInfo->ignoreCloseWindow = false;
+  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
 
   pOperator->operatorType = pPhyNode->type;
   pOperator->blocking = true;
@@ -2852,12 +2852,12 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
   pInfo->pStDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
   pInfo->pDelIterator = NULL;
   pInfo->pDelRes = createOneDataBlock(pResBlock, false);
-  pInfo->pDelRes->info.type = STREAM_DELETE;
+  pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;
   blockDataEnsureCapacity(pInfo->pDelRes, 64);
   pInfo->pChildren = NULL;
   pInfo->isFinal = false;
   pInfo->pPhyNode = pPhyNode;
-  pInfo->ignoreCloseWindow = false;
+  pInfo->ignoreExpiredData = pSessionNode->window.igExpired;
 
   pOperator->name = "StreamSessionWindowAggOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
@@ -3133,7 +3133,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData
 
   SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
   for (int32_t i = 0; i < pSDataBlock->info.rows;) {
-    if (pInfo->ignoreCloseWindow && isOverdue(endTsCols[i], &pInfo->twAggSup)) {
+    if (pInfo->ignoreExpiredData && isOverdue(endTsCols[i], &pInfo->twAggSup)) {
       i++;
       continue;
     }
@@ -3413,8 +3413,8 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
   pOperator->status = OP_RES_TO_RETURN;
 
   closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
-      getResWinForSession, pInfo->ignoreCloseWindow);
+      getResWinForSession, pInfo->ignoreExpiredData);
-  closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreCloseWindow);
+  closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData);
   copyUpdateResult(pStUpdated, pUpdated);
   taosHashCleanup(pStUpdated);
 
@@ -3822,7 +3822,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
   SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
   SColumnInfoData*     pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
   for (int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) {
-    if (pInfo->ignoreCloseWindow && isOverdue(tsCols[i], &pInfo->twAggSup)) {
+    if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) {
       i++;
       continue;
     }
@@ -3866,12 +3866,14 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
   if (pOperator->status == OP_RES_TO_RETURN) {
     doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
     if (pInfo->pDelRes->info.rows > 0) {
+      printDataBlock(pInfo->pDelRes, "single state");
       return pInfo->pDelRes;
     }
     doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
     if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
       doSetOperatorCompleted(pOperator);
     }
+    printDataBlock(pBInfo->pRes, "single state");
     return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
   }
 
@@ -3884,6 +3886,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
     if (pBlock == NULL) {
       break;
     }
+    printDataBlock(pBlock, "single state recv");
 
     if (pBlock->info.type == STREAM_CLEAR) {
       doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, &pInfo->stateCol, pInfo->stateCol.slotId,
@@ -3903,8 +3906,8 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
   pOperator->status = OP_RES_TO_RETURN;
 
   closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
-      getResWinForState, pInfo->ignoreCloseWindow);
+      getResWinForState, pInfo->ignoreExpiredData);
-  closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreCloseWindow);
+  closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData);
   copyUpdateResult(pSeUpdated, pUpdated);
   taosHashCleanup(pSeUpdated);
 
@@ -3914,9 +3917,11 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
   blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
   doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
   if (pInfo->pDelRes->info.rows > 0) {
+    printDataBlock(pInfo->pDelRes, "single state");
     return pInfo->pDelRes;
   }
   doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
+  printDataBlock(pBInfo->pRes, "single state");
   return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
 }
 
@@ -3975,10 +3980,10 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
   pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
   pInfo->pDelIterator = NULL;
   pInfo->pDelRes = createOneDataBlock(pResBlock, false);
-  pInfo->pDelRes->info.type = STREAM_DELETE;
+  pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;
   blockDataEnsureCapacity(pInfo->pDelRes, 64);
   pInfo->pChildren = NULL;
-  pInfo->ignoreCloseWindow = false;
+  pInfo->ignoreExpiredData = pStateNode->window.igExpired;
 
   pOperator->name = "StreamStateAggOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;

source/libs/function/src/builtins.c

@@ -18,6 +18,7 @@
 #include "querynodes.h"
 #include "scalar.h"
 #include "taoserror.h"
+#include "cJSON.h"
 
 static int32_t buildFuncErrMsg(char* pErrBuf, int32_t len, int32_t errCode, const char* pFormat, ...) {
   va_list vArgList;
@@ -39,6 +40,174 @@ static int32_t invaildFuncParaValueErrMsg(char* pErrBuf, int32_t len, const char
   return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_PARA_VALUE, "Invalid parameter value : %s", pFuncName);
 }
 
+#define TIME_UNIT_INVALID 1
+#define TIME_UNIT_TOO_SMALL 2
+
+static int32_t validateTimeUnitParam(uint8_t dbPrec, const SValueNode* pVal) {
+  if (!pVal->isDuration) {
+    return TIME_UNIT_INVALID;
+  }
+
+  if (TSDB_TIME_PRECISION_MILLI == dbPrec && 0 == strcasecmp(pVal->literal, "1u")) {
+    return TIME_UNIT_TOO_SMALL;
+  }
+
+  if (pVal->literal[0] != '1' || (pVal->literal[1] != 'u' && pVal->literal[1] != 'a' &&
+                                  pVal->literal[1] != 's' && pVal->literal[1] != 'm' &&
+                                  pVal->literal[1] != 'h' && pVal->literal[1] != 'd' &&
+                                  pVal->literal[1] != 'w')) {
+    return TIME_UNIT_INVALID;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+/* Following are valid ISO-8601 timezone format:
+ * 1 z/Z
+ * 2 ±hh:mm
+ * 3 ±hhmm
+ * 4 ±hh
+ *
+ */
+
+static bool validateHourRange(int8_t hour) {
+  if (hour < 0 || hour > 12) {
+    return false;
+  }
+
+  return true;
+}
+
+static bool validateMinuteRange(int8_t hour, int8_t minute, char sign) {
+  if (minute == 0 || (minute == 30 && (hour == 3 || hour == 5) && sign == '+')) {
+    return true;
+  }
+
+  return false;
+}
+
+static bool validateTimestampDigits(const SValueNode* pVal) {
+  if (!IS_INTEGER_TYPE(pVal->node.resType.type)) {
+    return false;
+  }
+
+  int64_t tsVal = pVal->datum.i;
+  char    fraction[20] = {0};
+  NUM_TO_STRING(pVal->node.resType.type, &tsVal, sizeof(fraction), fraction);
+  int32_t tsDigits = (int32_t)strlen(fraction);
+
+  if (tsDigits > TSDB_TIME_PRECISION_SEC_DIGITS) {
+    if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS || tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS ||
+        tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+static bool validateTimezoneFormat(const SValueNode* pVal) {
+  if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) {
+    return false;
+  }
+
+  char*   tz = varDataVal(pVal->datum.p);
+  int32_t len = varDataLen(pVal->datum.p);
+
+  char   buf[3] = {0};
+  int8_t hour = -1, minute = -1;
+  if (len == 0) {
+    return false;
+  } else if (len == 1 && (tz[0] == 'z' || tz[0] == 'Z')) {
+    return true;
+  } else if ((tz[0] == '+' || tz[0] == '-')) {
+    switch (len) {
+      case 3:
+      case 5: {
+        for (int32_t i = 1; i < len; ++i) {
+          if (!isdigit(tz[i])) {
+            return false;
+          }
+
+          if (i == 2) {
+            memcpy(buf, &tz[i - 1], 2);
+            hour = taosStr2Int8(buf, NULL, 10);
+            if (!validateHourRange(hour)) {
+              return false;
+            }
+          } else if (i == 4) {
+            memcpy(buf, &tz[i - 1], 2);
+            minute = taosStr2Int8(buf, NULL, 10);
+            if (!validateMinuteRange(hour, minute, tz[0])) {
+              return false;
+            }
+          }
+        }
+        break;
+      }
+      case 6: {
+        for (int32_t i = 1; i < len; ++i) {
+          if (i == 3) {
+            if (tz[i] != ':') {
+              return false;
+            }
+            continue;
+          }
+
+          if (!isdigit(tz[i])) {
+            return false;
+          }
+
+          if (i == 2) {
+            memcpy(buf, &tz[i - 1], 2);
+            hour = taosStr2Int8(buf, NULL, 10);
+            if (!validateHourRange(hour)) {
+              return false;
+            }
+          } else if (i == 5) {
+            memcpy(buf, &tz[i - 1], 2);
+            minute = taosStr2Int8(buf, NULL, 10);
+            if (!validateMinuteRange(hour, minute, tz[0])) {
+              return false;
+            }
+          }
+        }
+        break;
+      }
+      default: {
+        return false;
+      }
+    }
+  } else {
+    return false;
+  }
+
+  return true;
+}
+
+void static addTimezoneParam(SNodeList* pList) {
+  char       buf[6] = {0};
+  time_t     t = taosTime(NULL);
+  struct tm* tmInfo = taosLocalTime(&t, NULL);
+  strftime(buf, sizeof(buf), "%z", tmInfo);
+  int32_t len = (int32_t)strlen(buf);
+
+  SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
+  pVal->literal = strndup(buf, len);
+  pVal->isDuration = false;
+  pVal->translate = true;
+  pVal->node.resType.type = TSDB_DATA_TYPE_BINARY;
+  pVal->node.resType.bytes = len + VARSTR_HEADER_SIZE;
+  pVal->node.resType.precision = TSDB_TIME_PRECISION_MILLI;
+  pVal->datum.p = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE + 1);
+  varDataSetLen(pVal->datum.p, len);
+  strncpy(varDataVal(pVal->datum.p), pVal->literal, len);
+
+  nodesListAppend(pList, (SNode*)pVal);
+}
+
 void static addDbPrecisonParam(SNodeList** pList, uint8_t precision) {
   SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
   pVal->literal = NULL;
@@ -525,9 +694,15 @@ static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len
       return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
     }
 
-    if (pValue->datum.i == 0) {
+    uint8_t dbPrec = pFunc->node.resType.precision;
+
+    int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode *)nodesListGetNode(pFunc->pParameterList, 1));
+    if (ret == TIME_UNIT_TOO_SMALL) {
       return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
                              "ELAPSED function time unit parameter should be greater than db precision");
+    } else if (ret == TIME_UNIT_INVALID) {
+      return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                             "ELAPSED function time unit parameter should be one of the following: [1u, 1a, 1s, 1m, 1h, 1d, 1w]");
     }
   }
 
@@ -622,6 +797,165 @@ static int32_t translateLeastSQR(SFunctionNode* pFunc, char* pErrBuf, int32_t le
   return TSDB_CODE_SUCCESS;
 }
 
+typedef enum { UNKNOWN_BIN = 0, USER_INPUT_BIN, LINEAR_BIN, LOG_BIN } EHistoBinType;
+
+static int8_t validateHistogramBinType(char* binTypeStr) {
+  int8_t binType;
+  if (strcasecmp(binTypeStr, "user_input") == 0) {
+    binType = USER_INPUT_BIN;
+  } else if (strcasecmp(binTypeStr, "linear_bin") == 0) {
+    binType = LINEAR_BIN;
+  } else if (strcasecmp(binTypeStr, "log_bin") == 0) {
+    binType = LOG_BIN;
+  } else {
+    binType = UNKNOWN_BIN;
+  }
+
+  return binType;
+}
+
+static bool validateHistogramBinDesc(char* binDescStr, int8_t binType, char* errMsg, int32_t msgLen) {
+  const char *msg1 = "HISTOGRAM function requires four parameters";
+  const char *msg3 = "HISTOGRAM function invalid format for binDesc parameter";
+  const char *msg4 = "HISTOGRAM function binDesc parameter \"count\" should be in range [1, 1000]";
+  const char *msg5 = "HISTOGRAM function bin/parameter should be in range [-DBL_MAX, DBL_MAX]";
+  const char *msg6 = "HISTOGRAM function binDesc parameter \"width\" cannot be 0";
+  const char *msg7 = "HISTOGRAM function binDesc parameter \"start\" cannot be 0 with \"log_bin\" type";
+  const char *msg8 = "HISTOGRAM function binDesc parameter \"factor\" cannot be negative or equal to 0/1";
+
+  cJSON*  binDesc = cJSON_Parse(binDescStr);
+  int32_t numOfBins;
+  double* intervals;
+  if (cJSON_IsObject(binDesc)) { /* linaer/log bins */
+    int32_t numOfParams = cJSON_GetArraySize(binDesc);
+    int32_t startIndex;
+    if (numOfParams != 4) {
+      snprintf(errMsg, msgLen, "%s", msg1);
+      return false;
+    }
+
+    cJSON* start = cJSON_GetObjectItem(binDesc, "start");
+    cJSON* factor = cJSON_GetObjectItem(binDesc, "factor");
+    cJSON* width = cJSON_GetObjectItem(binDesc, "width");
+    cJSON* count = cJSON_GetObjectItem(binDesc, "count");
+    cJSON* infinity = cJSON_GetObjectItem(binDesc, "infinity");
+
+    if (!cJSON_IsNumber(start) || !cJSON_IsNumber(count) || !cJSON_IsBool(infinity)) {
+      snprintf(errMsg, msgLen, "%s", msg3);
+      return false;
+    }
+
+    if (count->valueint <= 0 || count->valueint > 1000) {  // limit count to 1000
+      snprintf(errMsg, msgLen, "%s", msg4);
+      return false;
+    }
+
+    if (isinf(start->valuedouble) || (width != NULL && isinf(width->valuedouble)) ||
+        (factor != NULL && isinf(factor->valuedouble)) || (count != NULL && isinf(count->valuedouble))) {
+      snprintf(errMsg, msgLen, "%s", msg5);
+      return false;
+    }
+
+    int32_t counter = (int32_t)count->valueint;
+    if (infinity->valueint == false) {
+      startIndex = 0;
+      numOfBins = counter + 1;
+    } else {
+      startIndex = 1;
+      numOfBins = counter + 3;
+    }
+
+    intervals = taosMemoryCalloc(numOfBins, sizeof(double));
+    if (cJSON_IsNumber(width) && factor == NULL && binType == LINEAR_BIN) {
+      // linear bin process
+      if (width->valuedouble == 0) {
+        snprintf(errMsg, msgLen, "%s", msg6);
+        taosMemoryFree(intervals);
+        return false;
+      }
+      for (int i = 0; i < counter + 1; ++i) {
+        intervals[startIndex] = start->valuedouble + i * width->valuedouble;
+        if (isinf(intervals[startIndex])) {
+          snprintf(errMsg, msgLen, "%s", msg5);
+          taosMemoryFree(intervals);
+          return false;
+        }
+        startIndex++;
+      }
+    } else if (cJSON_IsNumber(factor) && width == NULL && binType == LOG_BIN) {
+      // log bin process
+      if (start->valuedouble == 0) {
+        snprintf(errMsg, msgLen, "%s", msg7);
+        taosMemoryFree(intervals);
+        return false;
+      }
+      if (factor->valuedouble < 0 || factor->valuedouble == 0 || factor->valuedouble == 1) {
+        snprintf(errMsg, msgLen, "%s", msg8);
+        taosMemoryFree(intervals);
+        return false;
+      }
+      for (int i = 0; i < counter + 1; ++i) {
+        intervals[startIndex] = start->valuedouble * pow(factor->valuedouble, i * 1.0);
+        if (isinf(intervals[startIndex])) {
+          snprintf(errMsg, msgLen, "%s", msg5);
+          taosMemoryFree(intervals);
+          return false;
+        }
+        startIndex++;
+      }
+    } else {
+      snprintf(errMsg, msgLen, "%s", msg3);
+      taosMemoryFree(intervals);
+      return false;
+    }
+
+    if (infinity->valueint == true) {
+      intervals[0] = -INFINITY;
+      intervals[numOfBins - 1] = INFINITY;
+      // in case of desc bin orders, -inf/inf should be swapped
+      ASSERT(numOfBins >= 4);
+      if (intervals[1] > intervals[numOfBins - 2]) {
+        TSWAP(intervals[0], intervals[numOfBins - 1]);
+      }
+    }
+  } else if (cJSON_IsArray(binDesc)) { /* user input bins */
+    if (binType != USER_INPUT_BIN) {
+      snprintf(errMsg, msgLen, "%s", msg3);
+      return false;
+    }
+    numOfBins = cJSON_GetArraySize(binDesc);
+    intervals = taosMemoryCalloc(numOfBins, sizeof(double));
+    cJSON* bin = binDesc->child;
+    if (bin == NULL) {
+      snprintf(errMsg, msgLen, "%s", msg3);
+      taosMemoryFree(intervals);
+      return false;
+    }
+    int i = 0;
+    while (bin) {
+      intervals[i] = bin->valuedouble;
+      if (!cJSON_IsNumber(bin)) {
+        snprintf(errMsg, msgLen, "%s", msg3);
+        taosMemoryFree(intervals);
+        return false;
+      }
+      if (i != 0 && intervals[i] <= intervals[i - 1]) {
+        snprintf(errMsg, msgLen, "%s", msg3);
+        taosMemoryFree(intervals);
+        return false;
+      }
+      bin = bin->next;
+      i++;
+    }
+  } else {
+    snprintf(errMsg, msgLen, "%s", msg3);
+    return false;
+  }
+
+  taosMemoryFree(intervals);
+  return true;
+}
+
 static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
   if (4 != numOfParams) {
@@ -640,6 +974,8 @@ static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t l
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
+  int8_t binType;
+  char*  binDesc;
   for (int32_t i = 1; i < numOfParams; ++i) {
     SNode* pParamNode = nodesListGetNode(pFunc->pParameterList, i);
     if (QUERY_NODE_VALUE != nodeType(pParamNode)) {
@@ -650,6 +986,23 @@ static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t l
 
     pValue->notReserved = true;
 
+    if (i == 1) {
+      binType = validateHistogramBinType(varDataVal(pValue->datum.p));
+      if (binType == UNKNOWN_BIN) {
+        return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                               "HISTOGRAM function binType parameter should be "
+                               "\"user_input\", \"log_bin\" or \"linear_bin\"");
+      }
+    }
+
+    if (i == 2) {
+      char errMsg[128] = {0};
+      binDesc = varDataVal(pValue->datum.p);
+      if (!validateHistogramBinDesc(binDesc, binType, errMsg, (int32_t)sizeof(errMsg))) {
+        return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, errMsg);
+      }
+    }
+
     if (i == 3 && pValue->datum.i != 1 && pValue->datum.i != 0) {
         return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
                                "HISTOGRAM function normalized parameter should be 0/1");
@@ -679,6 +1032,8 @@ static int32_t translateHistogramImpl(SFunctionNode* pFunc, char* pErrBuf, int32
       return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
     }
 
+    int8_t binType;
+    char*  binDesc;
     for (int32_t i = 1; i < numOfParams; ++i) {
       SNode* pParamNode = nodesListGetNode(pFunc->pParameterList, i);
       if (QUERY_NODE_VALUE != nodeType(pParamNode)) {
@@ -689,6 +1044,23 @@ static int32_t translateHistogramImpl(SFunctionNode* pFunc, char* pErrBuf, int32
 
       pValue->notReserved = true;
 
+      if (i == 1) {
+        binType = validateHistogramBinType(varDataVal(pValue->datum.p));
+        if (binType == UNKNOWN_BIN) {
+          return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                                 "HISTOGRAM function binType parameter should be "
+                                 "\"user_input\", \"log_bin\" or \"linear_bin\"");
+        }
+      }
+
+      if (i == 2) {
+        char errMsg[128] = {0};
+        binDesc = varDataVal(pValue->datum.p);
+        if (!validateHistogramBinDesc(binDesc, binType, errMsg, (int32_t)sizeof(errMsg))) {
+          return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, errMsg);
+        }
+      }
+
       if (i == 3 && pValue->datum.i != 1 && pValue->datum.i != 0) {
           return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
                                  "HISTOGRAM function normalized parameter should be 0/1");
@@ -843,6 +1215,19 @@ static int32_t translateStateDuration(SFunctionNode* pFunc, char* pErrBuf, int32
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
+  if (numOfParams == 4) {
+    uint8_t dbPrec = pFunc->node.resType.precision;
+
+    int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode *)nodesListGetNode(pFunc->pParameterList, 3));
+    if (ret == TIME_UNIT_TOO_SMALL) {
+      return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                             "STATEDURATION function time unit parameter should be greater than db precision");
+    } else if (ret == TIME_UNIT_INVALID) {
+      return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                             "STATEDURATION function time unit parameter should be one of the following: [1u, 1a, 1s, 1m, 1h, 1d, 1w]");
+    }
+  }
+
   // set result type
   pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
   return TSDB_CODE_SUCCESS;
@@ -1259,19 +1644,9 @@ static int32_t translateSubstr(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
 
 static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   // The number of parameters has been limited by the syntax definition
-  // uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
 
   // The function return type has been set during syntax parsing
   uint8_t para2Type = pFunc->node.resType.type;
-  // if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT &&
-  //     para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR &&
-  //     para2Type != TSDB_DATA_TYPE_TIMESTAMP) {
-  //   return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
-  // }
-  // if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) ||
-  //     (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) {
-  //   return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
-  // }
 
   int32_t para2Bytes = pFunc->node.resType.bytes;
   if (IS_VAR_DATA_TYPE(para2Type)) {
@@ -1281,155 +1656,14 @@ static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
     return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
                            "CAST function converted length should be in range [0, 1000]");
   }
+
+  // add database precision as param
+  uint8_t dbPrec = pFunc->node.resType.precision;
+  addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
+
   return TSDB_CODE_SUCCESS;
 }
 
-/* Following are valid ISO-8601 timezone format:
- * 1 z/Z
- * 2 ±hh:mm
- * 3 ±hhmm
- * 4 ±hh
- *
- */
-
-static bool validateHourRange(int8_t hour) {
-  if (hour < 0 || hour > 12) {
-    return false;
-  }
-
-  return true;
-}
-
-static bool validateMinuteRange(int8_t hour, int8_t minute, char sign) {
-  if (minute == 0 || (minute == 30 && (hour == 3 || hour == 5) && sign == '+')) {
-    return true;
-  }
-
-  return false;
-}
-
-static bool validateTimestampDigits(const SValueNode* pVal) {
-  if (!IS_INTEGER_TYPE(pVal->node.resType.type)) {
-    return false;
-  }
-
-  int64_t tsVal = pVal->datum.i;
-  char    fraction[20] = {0};
-  NUM_TO_STRING(pVal->node.resType.type, &tsVal, sizeof(fraction), fraction);
-  int32_t tsDigits = (int32_t)strlen(fraction);
-
-  if (tsDigits > TSDB_TIME_PRECISION_SEC_DIGITS) {
-    if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS || tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS ||
-        tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
-      return true;
-    } else {
-      return false;
-    }
-  }
-
-  return true;
-}
-
-static bool validateTimezoneFormat(const SValueNode* pVal) {
-  if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) {
-    return false;
-  }
-
-  char*   tz = varDataVal(pVal->datum.p);
-  int32_t len = varDataLen(pVal->datum.p);
-
-  char   buf[3] = {0};
-  int8_t hour = -1, minute = -1;
-  if (len == 0) {
-    return false;
-  } else if (len == 1 && (tz[0] == 'z' || tz[0] == 'Z')) {
-    return true;
-  } else if ((tz[0] == '+' || tz[0] == '-')) {
-    switch (len) {
-      case 3:
-      case 5: {
-        for (int32_t i = 1; i < len; ++i) {
-          if (!isdigit(tz[i])) {
-            return false;
-          }
-
-          if (i == 2) {
-            memcpy(buf, &tz[i - 1], 2);
-            hour = taosStr2Int8(buf, NULL, 10);
-            if (!validateHourRange(hour)) {
-              return false;
-            }
-          } else if (i == 4) {
-            memcpy(buf, &tz[i - 1], 2);
-            minute = taosStr2Int8(buf, NULL, 10);
-            if (!validateMinuteRange(hour, minute, tz[0])) {
-              return false;
-            }
-          }
-        }
-        break;
-      }
-      case 6: {
-        for (int32_t i = 1; i < len; ++i) {
-          if (i == 3) {
-            if (tz[i] != ':') {
-              return false;
-            }
-            continue;
-          }
-
-          if (!isdigit(tz[i])) {
-            return false;
-          }
-
-          if (i == 2) {
-            memcpy(buf, &tz[i - 1], 2);
-            hour = taosStr2Int8(buf, NULL, 10);
-            if (!validateHourRange(hour)) {
-              return false;
-            }
-          } else if (i == 5) {
-            memcpy(buf, &tz[i - 1], 2);
-            minute = taosStr2Int8(buf, NULL, 10);
-            if (!validateMinuteRange(hour, minute, tz[0])) {
-              return false;
-            }
-          }
-        }
-        break;
-      }
-      default: {
-        return false;
-      }
-    }
-  } else {
-    return false;
-  }
-
-  return true;
-}
-
-void static addTimezoneParam(SNodeList* pList) {
-  char       buf[6] = {0};
-  time_t     t = taosTime(NULL);
-  struct tm* tmInfo = taosLocalTime(&t, NULL);
-  strftime(buf, sizeof(buf), "%z", tmInfo);
-  int32_t len = (int32_t)strlen(buf);
-
-  SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
-  pVal->literal = strndup(buf, len);
-  pVal->isDuration = false;
-  pVal->translate = true;
-  pVal->node.resType.type = TSDB_DATA_TYPE_BINARY;
-  pVal->node.resType.bytes = len + VARSTR_HEADER_SIZE;
-  pVal->node.resType.precision = TSDB_TIME_PRECISION_MILLI;
-  pVal->datum.p = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE + 1);
-  varDataSetLen(pVal->datum.p, len);
-  strncpy(varDataVal(pVal->datum.p), pVal->literal, len);
-
-  nodesListAppend(pList, (SNode*)pVal);
-}
-
 static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
   if (1 != numOfParams && 2 != numOfParams) {
@@ -1498,6 +1732,16 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_
 
   // add database precision as param
   uint8_t dbPrec = pFunc->node.resType.precision;
+
+  int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode *)nodesListGetNode(pFunc->pParameterList, 1));
+  if (ret == TIME_UNIT_TOO_SMALL) {
+    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                           "TIMETRUNCATE function time unit parameter should be greater than db precision");
+  } else if (ret == TIME_UNIT_INVALID) {
+    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                           "TIMETRUNCATE function time unit parameter should be one of the following: [1u, 1a, 1s, 1m, 1h, 1d, 1w]");
+  }
+
   addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
 
   pFunc->node.resType =
@@ -1526,6 +1770,18 @@ static int32_t translateTimeDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t le
 
   // add database precision as param
   uint8_t dbPrec = pFunc->node.resType.precision;
+
+  if (3 == numOfParams) {
+    int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode *)nodesListGetNode(pFunc->pParameterList, 2));
+    if (ret == TIME_UNIT_TOO_SMALL) {
+      return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                             "TIMEDIFF function time unit parameter should be greater than db precision");
+    } else if (ret == TIME_UNIT_INVALID) {
+      return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                             "TIMEDIFF function time unit parameter should be one of the following: [1u, 1a, 1s, 1m, 1h, 1d, 1w]");
+    }
+  }
+
   addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
 
   pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
@@ -2054,7 +2310,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "histogram",
     .type = FUNCTION_TYPE_HISTOGRAM,
-    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_FORBID_FILL_FUNC,
+    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_FORBID_FILL_FUNC,
     .translateFunc = translateHistogram,
     .getEnvFunc   = getHistogramFuncEnv,
     .initFunc     = histogramFunctionSetup,

source/libs/function/src/functionMgt.c

@@ -260,7 +260,7 @@ bool fmIsSameInOutType(int32_t funcId) {
 }
 
 static int32_t getFuncInfo(SFunctionNode* pFunc) {
-  char msg[64] = {0};
+  char msg[128] = {0};
   return fmGetFuncInfo(pFunc, msg, sizeof(msg));
 }
 

source/libs/index/CMakeLists.txt

@@ -35,7 +35,7 @@ if (${BUILD_WITH_INVERTEDINDEX})
 endif(${BUILD_WITH_INVERTEDINDEX})
 
 
-if (${BUILD_TEST})
+# if (${BUILD_TEST})
-  add_subdirectory(test)
+#   add_subdirectory(test)
-endif(${BUILD_TEST})
+# endif(${BUILD_TEST})
 

source/libs/index/inc/indexFst.h

@@ -21,7 +21,7 @@ extern "C" {
 #endif
 
 #include "indexFstAutomation.h"
-#include "indexFstCountingWriter.h"
+#include "indexFstFile.h"
 #include "indexFstNode.h"
 #include "indexFstRegistry.h"
 #include "indexFstUtil.h"
@@ -90,8 +90,8 @@ FstBuilderNode*     fstUnFinishedNodesPopEmpty(FstUnFinishedNodes* nodes);
 uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes* node, FstSlice bs, Output in, Output* out);
 
 typedef struct FstBuilder {
-  FstCountingWriter*  wrt;         // The FST raw data is written directly to `wtr`.
+  IdxFstFile*         wrt;         // The FST raw data is written directly to `wtr`.
-  FstUnFinishedNodes* unfinished;  // The stack of unfinished nodes
+  FstUnFinishedNodes* unfinished;  // The stack of unfinished  nodes
   FstRegistry*        registry;    // A map of finished nodes.
   FstSlice            last;        // The last word added
   CompiledAddr        lastAddr;    // The address of the last compiled node
@@ -125,9 +125,9 @@ FstState fstStateCreateFrom(FstSlice* data, CompiledAddr addr);
 FstState fstStateCreate(State state);
 
 // compile
-void fstStateCompileForOneTransNext(FstCountingWriter* w, CompiledAddr addr, uint8_t inp);
+void fstStateCompileForOneTransNext(IdxFstFile* w, CompiledAddr addr, uint8_t inp);
-void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTransition* trn);
+void fstStateCompileForOneTrans(IdxFstFile* w, CompiledAddr addr, FstTransition* trn);
-void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuilderNode* node);
+void fstStateCompileForAnyTrans(IdxFstFile* w, CompiledAddr addr, FstBuilderNode* node);
 
 // set_comm_input
 void fstStateSetCommInput(FstState* state, uint8_t inp);
@@ -282,7 +282,7 @@ FStmSt* stmBuilderIntoStm(FStmBuilder* sb);
 bool fstVerify(Fst* fst);
 
 // refactor this function
-bool fstBuilderNodeCompileTo(FstBuilderNode* b, FstCountingWriter* wrt, CompiledAddr lastAddr, CompiledAddr startAddr);
+bool fstBuilderNodeCompileTo(FstBuilderNode* b, IdxFstFile* wrt, CompiledAddr lastAddr, CompiledAddr startAddr);
 
 typedef struct StreamState {
   FstNode*  node;

source/libs/index/inc/indexFstCountingWriter.h

@@ -1,96 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef __INDEX_FST_COUNTING_WRITER_H__
-#define __INDEX_FST_COUNTING_WRITER_H__
-
-#include "indexInt.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-//#define USE_MMAP 1
-
-#define DefaultMem 1024 * 1024
-
-static char tmpFile[] = "./index";
-typedef enum WriterType { TMemory, TFile } WriterType;
-
-typedef struct WriterCtx {
-  int (*write)(struct WriterCtx* ctx, uint8_t* buf, int len);
-  int (*read)(struct WriterCtx* ctx, uint8_t* buf, int len);
-  int (*flush)(struct WriterCtx* ctx);
-  int (*readFrom)(struct WriterCtx* ctx, uint8_t* buf, int len, int32_t offset);
-  int (*size)(struct WriterCtx* ctx);
-  WriterType type;
-  union {
-    struct {
-      TdFilePtr pFile;
-      bool readOnly;
-      char buf[256];
-      int  size;
-#ifdef USE_MMAP
-      char* ptr;
-#endif
-    } file;
-    struct {
-      int32_t capa;
-      char*   buf;
-    } mem;
-  };
-  int32_t offset;
-  int32_t limit;
-} WriterCtx;
-
-static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len);
-static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len);
-static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t offset);
-static int writeCtxDoFlush(WriterCtx* ctx);
-
-WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int32_t capacity);
-void       writerCtxDestroy(WriterCtx* w, bool remove);
-
-typedef uint32_t CheckSummer;
-
-typedef struct FstCountingWriter {
-  void*       wrt;  // wrap any writer that counts and checksum bytes written
-  uint64_t    count;
-  CheckSummer summer;
-} FstCountingWriter;
-
-int fstCountingWriterWrite(FstCountingWriter* write, uint8_t* buf, uint32_t len);
-
-int fstCountingWriterRead(FstCountingWriter* write, uint8_t* buf, uint32_t len);
-
-int fstCountingWriterFlush(FstCountingWriter* write);
-
-uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter* write);
-
-FstCountingWriter* fstCountingWriterCreate(void* wtr);
-void               fstCountingWriterDestroy(FstCountingWriter* w);
-
-void    fstCountingWriterPackUintIn(FstCountingWriter* writer, uint64_t n, uint8_t nBytes);
-uint8_t fstCountingWriterPackUint(FstCountingWriter* writer, uint64_t n);
-
-#define FST_WRITER_COUNT(writer) (writer->count)
-#define FST_WRITER_INTER_WRITER(writer) (writer->wtr)
-#define FST_WRITE_CHECK_SUMMER(writer) (writer->summer)
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

source/libs/index/inc/indexFstFile.h

@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __INDEX_FST_FILE_H__
+#define __INDEX_FST_FILE_H__
+
+#include "indexInt.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+//#define USE_MMAP 1
+
+#define DefaultMem 1024 * 1024
+
+static char tmpFile[] = "./index";
+typedef enum WriterType { TMemory, TFile } WriterType;
+
+typedef struct IFileCtx {
+  int (*write)(struct IFileCtx* ctx, uint8_t* buf, int len);
+  int (*read)(struct IFileCtx* ctx, uint8_t* buf, int len);
+  int (*flush)(struct IFileCtx* ctx);
+  int (*readFrom)(struct IFileCtx* ctx, uint8_t* buf, int len, int32_t offset);
+  int (*size)(struct IFileCtx* ctx);
+  WriterType type;
+  union {
+    struct {
+      TdFilePtr pFile;
+      bool      readOnly;
+      char      buf[256];
+      int64_t   size;
+#ifdef USE_MMAP
+      char* ptr;
+#endif
+    } file;
+    struct {
+      int32_t cap;
+      char*   buf;
+    } mem;
+  };
+  int32_t offset;
+  int32_t limit;
+} IFileCtx;
+
+static int idxFileCtxDoWrite(IFileCtx* ctx, uint8_t* buf, int len);
+static int idxFileCtxDoRead(IFileCtx* ctx, uint8_t* buf, int len);
+static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t offset);
+static int idxFileCtxDoFlush(IFileCtx* ctx);
+
+IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int32_t capacity);
+void      idxFileCtxDestroy(IFileCtx* w, bool remove);
+
+typedef uint32_t CheckSummer;
+
+typedef struct IdxFstFile {
+  void*       wrt;  // wrap any writer that counts and checksum bytes written
+  uint64_t    count;
+  CheckSummer summer;
+} IdxFstFile;
+
+int idxFileWrite(IdxFstFile* write, uint8_t* buf, uint32_t len);
+
+int idxFileRead(IdxFstFile* write, uint8_t* buf, uint32_t len);
+
+int idxFileFlush(IdxFstFile* write);
+
+uint32_t idxFileMaskedCheckSum(IdxFstFile* write);
+
+IdxFstFile* idxFileCreate(void* wtr);
+void        idxFileDestroy(IdxFstFile* w);
+
+void    idxFilePackUintIn(IdxFstFile* writer, uint64_t n, uint8_t nBytes);
+uint8_t idxFilePackUint(IdxFstFile* writer, uint64_t n);
+
+#define FST_WRITER_COUNT(writer)        (writer->count)
+#define FST_WRITER_INTER_WRITER(writer) (writer->wtr)
+#define FST_WRITE_CHECK_SUMMER(writer)  (writer->summer)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

source/libs/index/inc/indexFstNode.h

@@ -20,12 +20,12 @@
 extern "C" {
 #endif
 
-#include "indexFstCountingWriter.h"
+#include "indexFstFile.h"
 #include "indexFstUtil.h"
 #include "indexInt.h"
 
-#define FST_BUILDER_NODE_IS_FINAL(bn) (bn->isFinal)
+#define FST_BUILDER_NODE_IS_FINAL(bn)           (bn->isFinal)
-#define FST_BUILDER_NODE_TRANS_ISEMPTY(bn) (taosArrayGetSize(bn->trans) == 0)
+#define FST_BUILDER_NODE_TRANS_ISEMPTY(bn)      (taosArrayGetSize(bn->trans) == 0)
 #define FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn) (bn->finalOutput == 0)
 
 typedef struct FstTransition {
@@ -46,7 +46,7 @@ FstBuilderNode* fstBuilderNodeClone(FstBuilderNode* src);
 
 void fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src);
 
-// bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt,
+// bool fstBuilderNodeCompileTo(FstBuilderNode *b, IdxFile' *wrt,
 // CompiledAddr lastAddr, CompiledAddr startAddr);
 bool fstBuilderNodeEqual(FstBuilderNode* n1, FstBuilderNode* n2);
 

source/libs/index/inc/indexTfile.h

@@ -16,7 +16,7 @@
 #define __INDEX_TFILE_H__
 
 #include "indexFst.h"
-#include "indexFstCountingWriter.h"
+#include "indexFstFile.h"
 #include "indexInt.h"
 #include "indexTfile.h"
 #include "indexUtil.h"
@@ -59,7 +59,7 @@ typedef struct TFileCache {
 
 typedef struct TFileWriter {
   FstBuilder* fb;
-  WriterCtx*  ctx;
+  IFileCtx*   ctx;
   TFileHeader header;
   uint32_t    offset;
 } TFileWriter;
@@ -68,7 +68,7 @@ typedef struct TFileWriter {
 typedef struct TFileReader {
   T_REF_DECLARE()
   Fst*        fst;
-  WriterCtx*  ctx;
+  IFileCtx*   ctx;
   TFileHeader header;
   bool        remove;
 } TFileReader;
@@ -103,7 +103,7 @@ void         tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* read
 TFileReader* tfileGetReaderByCol(IndexTFile* tf, uint64_t suid, char* colName);
 
 TFileReader* tfileReaderOpen(char* path, uint64_t suid, int64_t version, const char* colName);
-TFileReader* tfileReaderCreate(WriterCtx* ctx);
+TFileReader* tfileReaderCreate(IFileCtx* ctx);
 void         tfileReaderDestroy(TFileReader* reader);
 int          tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTRslt* tr);
 void         tfileReaderRef(TFileReader* reader);
@@ -111,7 +111,7 @@ void         tfileReaderUnRef(TFileReader* reader);
 
 TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int64_t version, const char* colName, uint8_t type);
 void         tfileWriterClose(TFileWriter* tw);
-TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header);
+TFileWriter* tfileWriterCreate(IFileCtx* ctx, TFileHeader* header);
 void         tfileWriterDestroy(TFileWriter* tw);
 int          tfileWriterPut(TFileWriter* tw, void* data, bool order);
 int          tfileWriterFinish(TFileWriter* tw);

source/libs/index/src/index.c

@@ -39,7 +39,7 @@
 #define INDEX_DATA_BIGINT_NULL    0x8000000000000000LL
 #define INDEX_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL
 
-#define INDEX_DATA_FLOAT_NULL    0x7FF00000           // it is an NAN
+#define INDEX_DATA_FLOAT_NULL    0x7FF00000            // it is an NAN
 #define INDEX_DATA_DOUBLE_NULL   0x7FFFFF0000000000LL  // an NAN
 #define INDEX_DATA_NCHAR_NULL    0xFFFFFFFF
 #define INDEX_DATA_BINARY_NULL   0xFF
@@ -614,7 +614,7 @@ static int idxGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) {
   return ret;
 END:
   if (tw != NULL) {
-    writerCtxDestroy(tw->ctx, true);
+    idxFileCtxDestroy(tw->ctx, true);
     taosMemoryFree(tw);
   }
   return -1;

source/libs/index/src/indexFst.c

@@ -19,11 +19,11 @@
 #include "tchecksum.h"
 #include "tcoding.h"
 
-static void fstPackDeltaIn(FstCountingWriter* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr, uint8_t nBytes) {
+static void fstPackDeltaIn(IdxFstFile* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr, uint8_t nBytes) {
   CompiledAddr deltaAddr = (transAddr == EMPTY_ADDRESS) ? EMPTY_ADDRESS : nodeAddr - transAddr;
-  fstCountingWriterPackUintIn(wrt, deltaAddr, nBytes);
+  idxFilePackUintIn(wrt, deltaAddr, nBytes);
 }
-static uint8_t fstPackDetla(FstCountingWriter* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr) {
+static uint8_t fstPackDetla(IdxFstFile* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr) {
   uint8_t nBytes = packDeltaSize(nodeAddr, transAddr);
   fstPackDeltaIn(wrt, nodeAddr, transAddr, nBytes);
   return nBytes;
@@ -208,7 +208,7 @@ FstState fstStateCreate(State state) {
   return fstStateDict[idx];
 }
 // compile
-void fstStateCompileForOneTransNext(FstCountingWriter* w, CompiledAddr addr, uint8_t inp) {
+void fstStateCompileForOneTransNext(IdxFstFile* w, CompiledAddr addr, uint8_t inp) {
   FstState s = fstStateCreate(OneTransNext);
   fstStateSetCommInput(&s, inp);
 
@@ -216,21 +216,21 @@ void fstStateCompileForOneTransNext(FstCountingWriter* w, CompiledAddr addr, uin
   uint8_t v = fstStateCommInput(&s, &null);
   if (null) {
     // w->write_all(&[inp])
-    fstCountingWriterWrite(w, &inp, 1);
+    idxFileWrite(w, &inp, 1);
   }
-  fstCountingWriterWrite(w, &(s.val), 1);
+  idxFileWrite(w, &(s.val), 1);
   // w->write_all(&[s.val])
   return;
 }
-void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTransition* trn) {
+void fstStateCompileForOneTrans(IdxFstFile* w, CompiledAddr addr, FstTransition* trn) {
   Output    out = trn->out;
-  uint8_t   outPackSize = (out == 0 ? 0 : fstCountingWriterPackUint(w, out));
+  uint8_t   outPackSize = (out == 0 ? 0 : idxFilePackUint(w, out));
   uint8_t   transPackSize = fstPackDetla(w, addr, trn->addr);
   PackSizes packSizes = 0;
 
   FST_SET_OUTPUT_PACK_SIZE(packSizes, outPackSize);
   FST_SET_TRANSITION_PACK_SIZE(packSizes, transPackSize);
-  fstCountingWriterWrite(w, (char*)&packSizes, sizeof(packSizes));
+  idxFileWrite(w, (char*)&packSizes, sizeof(packSizes));
 
   FstState st = fstStateCreate(OneTrans);
 
@@ -239,12 +239,12 @@ void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTran
   bool    null = false;
   uint8_t inp = fstStateCommInput(&st, &null);
   if (null == true) {
-    fstCountingWriterWrite(w, (char*)&trn->inp, sizeof(trn->inp));
+    idxFileWrite(w, (char*)&trn->inp, sizeof(trn->inp));
   }
-  fstCountingWriterWrite(w, (char*)(&(st.val)), sizeof(st.val));
+  idxFileWrite(w, (char*)(&(st.val)), sizeof(st.val));
   return;
 }
-void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuilderNode* node) {
+void fstStateCompileForAnyTrans(IdxFstFile* w, CompiledAddr addr, FstBuilderNode* node) {
   int32_t sz = taosArrayGetSize(node->trans);
   assert(sz <= 256);
 
@@ -275,11 +275,11 @@ void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuil
 
   if (anyOuts) {
     if (FST_BUILDER_NODE_IS_FINAL(node)) {
-      fstCountingWriterPackUintIn(w, node->finalOutput, oSize);
+      idxFilePackUintIn(w, node->finalOutput, oSize);
     }
     for (int32_t i = sz - 1; i >= 0; i--) {
       FstTransition* t = taosArrayGet(node->trans, i);
-      fstCountingWriterPackUintIn(w, t->out, oSize);
+      idxFilePackUintIn(w, t->out, oSize);
     }
   }
   for (int32_t i = sz - 1; i >= 0; i--) {
@@ -288,7 +288,7 @@ void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuil
   }
   for (int32_t i = sz - 1; i >= 0; i--) {
     FstTransition* t = taosArrayGet(node->trans, i);
-    fstCountingWriterWrite(w, (char*)&t->inp, 1);
+    idxFileWrite(w, (char*)&t->inp, 1);
     // fstPackDeltaIn(w, addr, t->addr, tSize);
   }
   if (sz > TRANS_INDEX_THRESHOLD) {
@@ -306,10 +306,10 @@ void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuil
       index[t->inp] = i;
       // fstPackDeltaIn(w, addr, t->addr, tSize);
     }
-    fstCountingWriterWrite(w, (char*)index, 256);
+    idxFileWrite(w, (char*)index, 256);
     taosMemoryFree(index);
   }
-  fstCountingWriterWrite(w, (char*)&packSizes, 1);
+  idxFileWrite(w, (char*)&packSizes, 1);
   bool null = false;
   fstStateStateNtrans(&st, &null);
   if (null == true) {
@@ -318,12 +318,12 @@ void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuil
     // encoded in the state byte.
     uint8_t v = 1;
     if (sz == 256) {
-      fstCountingWriterWrite(w, (char*)&v, 1);
+      idxFileWrite(w, (char*)&v, 1);
     } else {
-      fstCountingWriterWrite(w, (char*)&sz, 1);
+      idxFileWrite(w, (char*)&sz, 1);
     }
   }
-  fstCountingWriterWrite(w, (char*)(&(st.val)), 1);
+  idxFileWrite(w, (char*)(&(st.val)), 1);
   return;
 }
 
@@ -753,7 +753,7 @@ bool fstNodeCompile(FstNode* node, void* w, CompiledAddr lastAddr, CompiledAddr
   return true;
 }
 
-bool fstBuilderNodeCompileTo(FstBuilderNode* b, FstCountingWriter* wrt, CompiledAddr lastAddr, CompiledAddr startAddr) {
+bool fstBuilderNodeCompileTo(FstBuilderNode* b, IdxFstFile* wrt, CompiledAddr lastAddr, CompiledAddr startAddr) {
   return fstNodeCompile(NULL, wrt, lastAddr, startAddr, b);
 }
 
@@ -763,7 +763,7 @@ FstBuilder* fstBuilderCreate(void* w, FstType ty) {
     return b;
   }
 
-  b->wrt = fstCountingWriterCreate(w);
+  b->wrt = idxFileCreate(w);
   b->unfinished = fstUnFinishedNodesCreate();
   b->registry = fstRegistryCreate(10000, 2);
   b->last = fstSliceCreate(NULL, 0);
@@ -773,12 +773,12 @@ FstBuilder* fstBuilderCreate(void* w, FstType ty) {
   char  buf64[8] = {0};
   void* pBuf64 = buf64;
   taosEncodeFixedU64(&pBuf64, VERSION);
-  fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64));
+  idxFileWrite(b->wrt, buf64, sizeof(buf64));
 
   pBuf64 = buf64;
   memset(buf64, 0, sizeof(buf64));
   taosEncodeFixedU64(&pBuf64, ty);
-  fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64));
+  idxFileWrite(b->wrt, buf64, sizeof(buf64));
 
   return b;
 }
@@ -787,7 +787,7 @@ void fstBuilderDestroy(FstBuilder* b) {
     return;
   }
 
-  fstCountingWriterDestroy(b->wrt);
+  idxFileDestroy(b->wrt);
   fstUnFinishedNodesDestroy(b->unfinished);
   fstRegistryDestroy(b->registry);
   fstSliceDestroy(&b->last);
@@ -905,21 +905,19 @@ void* fstBuilderInsertInner(FstBuilder* b) {
 
   void* pBuf64 = buf64;
   taosEncodeFixedU64(&pBuf64, b->len);
-  fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64));
+  idxFileWrite(b->wrt, buf64, sizeof(buf64));
 
   pBuf64 = buf64;
   taosEncodeFixedU64(&pBuf64, rootAddr);
-  fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64));
+  idxFileWrite(b->wrt, buf64, sizeof(buf64));
 
   char     buf32[4] = {0};
   void*    pBuf32 = buf32;
-  uint32_t sum = fstCountingWriterMaskedCheckSum(b->wrt);
+  uint32_t sum = idxFileMaskedCheckSum(b->wrt);
   taosEncodeFixedU32(&pBuf32, sum);
-  fstCountingWriterWrite(b->wrt, buf32, sizeof(buf32));
+  idxFileWrite(b->wrt, buf32, sizeof(buf32));
 
-  fstCountingWriterFlush(b->wrt);
+  idxFileFlush(b->wrt);
-  // fstCountingWriterDestroy(b->wrt);
-  // b->wrt = NULL;
   return b->wrt;
 }
 void fstBuilderFinish(FstBuilder* b) { fstBuilderInsertInner(b); }

source/libs/index/src/indexFstDfa.c

@@ -61,9 +61,10 @@ void dfaBuilderDestroy(FstDfaBuilder *builder) {
     pIter = taosHashIterate(builder->cache, pIter);
   }
   taosHashCleanup(builder->cache);
+  taosMemoryFree(builder);
 }
 
-FstDfa *dfaBuilderBuild(FstDfaBuilder *builder) {
+FstDfa *dfaBuilder(FstDfaBuilder *builder) {
   uint32_t      sz = taosArrayGetSize(builder->dfa->insts);
   FstSparseSet *cur = sparSetCreate(sz);
   FstSparseSet *nxt = sparSetCreate(sz);

source/libs/index/src/indexFstFile.c

@@ -13,13 +13,13 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "indexFstCountingWriter.h"
+#include "indexFstFile.h"
 #include "indexFstUtil.h"
 #include "indexInt.h"
 #include "os.h"
 #include "tutil.h"
 
-static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len) {
+static int idxFileCtxDoWrite(IFileCtx* ctx, uint8_t* buf, int len) {
   if (ctx->type == TFile) {
     assert(len == taosWriteFile(ctx->file.pFile, buf, len));
   } else {
@@ -28,7 +28,7 @@ static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len) {
   ctx->offset += len;
   return len;
 }
-static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) {
+static int idxFileCtxDoRead(IFileCtx* ctx, uint8_t* buf, int len) {
   int nRead = 0;
   if (ctx->type == TFile) {
 #ifdef USE_MMAP
@@ -44,7 +44,7 @@ static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) {
 
   return nRead;
 }
-static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t offset) {
+static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t offset) {
   int nRead = 0;
   if (ctx->type == TFile) {
     // tfLseek(ctx->file.pFile, offset, 0);
@@ -61,7 +61,7 @@ static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t off
   }
   return nRead;
 }
-static int writeCtxGetSize(WriterCtx* ctx) {
+static int idxFileCtxGetSize(IFileCtx* ctx) {
   if (ctx->type == TFile) {
     int64_t file_size = 0;
     taosStatFile(ctx->file.buf, &file_size, NULL);
@@ -69,7 +69,7 @@ static int writeCtxGetSize(WriterCtx* ctx) {
   }
   return 0;
 }
-static int writeCtxDoFlush(WriterCtx* ctx) {
+static int idxFileCtxDoFlush(IFileCtx* ctx) {
   if (ctx->type == TFile) {
     // taosFsyncFile(ctx->file.pFile);
     taosFsyncFile(ctx->file.pFile);
@@ -80,8 +80,8 @@ static int writeCtxDoFlush(WriterCtx* ctx) {
   return 1;
 }
 
-WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int32_t capacity) {
+IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int32_t capacity) {
-  WriterCtx* ctx = taosMemoryCalloc(1, sizeof(WriterCtx));
+  IFileCtx* ctx = taosMemoryCalloc(1, sizeof(IFileCtx));
   if (ctx == NULL) {
     return NULL;
   }
@@ -90,39 +90,36 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int
   if (ctx->type == TFile) {
     // ugly code, refactor later
     ctx->file.readOnly = readOnly;
+    memcpy(ctx->file.buf, path, strlen(path));
     if (readOnly == false) {
-      // ctx->file.pFile = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
       ctx->file.pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
       taosFtruncateFile(ctx->file.pFile, 0);
-      int64_t file_size;
+      taosStatFile(path, &ctx->file.size, NULL);
-      taosStatFile(path, &file_size, NULL);
+      // ctx->file.size = (int)size;
-      ctx->file.size = (int)file_size;
 
     } else {
-      // ctx->file.pFile = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO);
       ctx->file.pFile = taosOpenFile(path, TD_FILE_READ);
 
-      int64_t file_size = 0;
+      int64_t size = 0;
-      taosFStatFile(ctx->file.pFile, &file_size, NULL);
+      taosFStatFile(ctx->file.pFile, &ctx->file.size, NULL);
-      ctx->file.size = (int)file_size;
+      ctx->file.size = (int)size;
 #ifdef USE_MMAP
       ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.pFile, ctx->file.size);
 #endif
     }
-    memcpy(ctx->file.buf, path, strlen(path));
     if (ctx->file.pFile == NULL) {
       indexError("failed to open file, error %d", errno);
       goto END;
     }
   } else if (ctx->type == TMemory) {
     ctx->mem.buf = taosMemoryCalloc(1, sizeof(char) * capacity);
-    ctx->mem.capa = capacity;
+    ctx->mem.cap = capacity;
   }
-  ctx->write = writeCtxDoWrite;
+  ctx->write = idxFileCtxDoWrite;
-  ctx->read = writeCtxDoRead;
+  ctx->read = idxFileCtxDoRead;
-  ctx->flush = writeCtxDoFlush;
+  ctx->flush = idxFileCtxDoFlush;
-  ctx->readFrom = writeCtxDoReadFrom;
+  ctx->readFrom = idxFileCtxDoReadFrom;
-  ctx->size = writeCtxGetSize;
+  ctx->size = idxFileCtxGetSize;
 
   ctx->offset = 0;
   ctx->limit = capacity;
@@ -135,7 +132,7 @@ END:
   taosMemoryFree(ctx);
   return NULL;
 }
-void writerCtxDestroy(WriterCtx* ctx, bool remove) {
+void idxFileCtxDestroy(IFileCtx* ctx, bool remove) {
   if (ctx->type == TMemory) {
     taosMemoryFree(ctx->mem.buf);
   } else {
@@ -149,9 +146,6 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) {
     if (ctx->file.readOnly == false) {
       int64_t file_size = 0;
       taosStatFile(ctx->file.buf, &file_size, NULL);
-      // struct stat fstat;
-      // stat(ctx->file.buf, &fstat);
-      // indexError("write file size: %d", (int)(fstat.st_size));
     }
     if (remove) {
       unlink(ctx->file.buf);
@@ -160,30 +154,29 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) {
   taosMemoryFree(ctx);
 }
 
-FstCountingWriter* fstCountingWriterCreate(void* wrt) {
+IdxFstFile* idxFileCreate(void* wrt) {
-  FstCountingWriter* cw = taosMemoryCalloc(1, sizeof(FstCountingWriter));
+  IdxFstFile* cw = taosMemoryCalloc(1, sizeof(IdxFstFile));
   if (cw == NULL) {
     return NULL;
   }
 
   cw->wrt = wrt;
-  //(void *)(writerCtxCreate(TFile, readOnly));
   return cw;
 }
-void fstCountingWriterDestroy(FstCountingWriter* cw) {
+void idxFileDestroy(IdxFstFile* cw) {
   // free wrt object: close fd or free mem
-  fstCountingWriterFlush(cw);
+  idxFileFlush(cw);
-  // writerCtxDestroy((WriterCtx *)(cw->wrt));
+  // idxFileCtxDestroy((IFileCtx *)(cw->wrt));
   taosMemoryFree(cw);
 }
 
-int fstCountingWriterWrite(FstCountingWriter* write, uint8_t* buf, uint32_t len) {
+int idxFileWrite(IdxFstFile* write, uint8_t* buf, uint32_t len) {
   if (write == NULL) {
     return 0;
   }
   // update checksum
   // write data to file/socket or mem
-  WriterCtx* ctx = write->wrt;
+  IFileCtx* ctx = write->wrt;
 
   int nWrite = ctx->write(ctx, buf, len);
   assert(nWrite == len);
@@ -192,42 +185,41 @@ int fstCountingWriterWrite(FstCountingWriter* write, uint8_t* buf, uint32_t len)
   write->summer = taosCalcChecksum(write->summer, buf, len);
   return len;
 }
-int fstCountingWriterRead(FstCountingWriter* write, uint8_t* buf, uint32_t len) {
+int idxFileRead(IdxFstFile* write, uint8_t* buf, uint32_t len) {
   if (write == NULL) {
     return 0;
   }
-  WriterCtx* ctx = write->wrt;
+  IFileCtx* ctx = write->wrt;
-  int        nRead = ctx->read(ctx, buf, len);
+  int       nRead = ctx->read(ctx, buf, len);
   // assert(nRead == len);
   return nRead;
 }
 
-uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter* write) {
+uint32_t idxFileMaskedCheckSum(IdxFstFile* write) {
   // opt
   return write->summer;
 }
 
-int fstCountingWriterFlush(FstCountingWriter* write) {
+int idxFileFlush(IdxFstFile* write) {
-  WriterCtx* ctx = write->wrt;
+  IFileCtx* ctx = write->wrt;
   ctx->flush(ctx);
-  // write->wtr->flush
   return 1;
 }
 
-void fstCountingWriterPackUintIn(FstCountingWriter* writer, uint64_t n, uint8_t nBytes) {
+void idxFilePackUintIn(IdxFstFile* writer, uint64_t n, uint8_t nBytes) {
   assert(1 <= nBytes && nBytes <= 8);
   uint8_t* buf = taosMemoryCalloc(8, sizeof(uint8_t));
   for (uint8_t i = 0; i < nBytes; i++) {
     buf[i] = (uint8_t)n;
     n = n >> 8;
   }
-  fstCountingWriterWrite(writer, buf, nBytes);
+  idxFileWrite(writer, buf, nBytes);
   taosMemoryFree(buf);
   return;
 }
 
-uint8_t fstCountingWriterPackUint(FstCountingWriter* writer, uint64_t n) {
+uint8_t idxFilePackUint(IdxFstFile* writer, uint64_t n) {
   uint8_t nBytes = packSize(n);
-  fstCountingWriterPackUintIn(writer, n, nBytes);
+  idxFilePackUintIn(writer, n, nBytes);
   return nBytes;
 }

source/libs/index/src/indexFstNode.c

@@ -95,7 +95,7 @@ void fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src) {
   }
 }
 
-// bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr
+// bool fstBuilderNodeCompileTo(FstBuilderNode *b, IdxFile *wrt, CompiledAddr lastAddr, CompiledAddr
 // startAddr) {
 
 // size_t sz = taosArrayGetSize(b->trans);

source/libs/index/src/indexFstSparse.c

@@ -17,7 +17,7 @@
 
 FstSparseSet *sparSetCreate(int32_t sz) {
   FstSparseSet *ss = taosMemoryCalloc(1, sizeof(FstSparseSet));
-  if (ss = NULL) {
+  if (ss == NULL) {
     return NULL;
   }
 

source/libs/index/src/indexFstUtil.c

@@ -75,7 +75,6 @@ CompiledAddr unpackDelta(char* data, uint64_t len, uint64_t nodeAddr) {
 }
 
 // fst slice func
-//
 
 FstSlice fstSliceCreate(uint8_t* data, uint64_t len) {
   FstString* str = (FstString*)taosMemoryMalloc(sizeof(FstString));
@@ -164,16 +163,3 @@ int fstSliceCompare(FstSlice* a, FstSlice* b) {
     return 0;
   }
 }
-
-// FstStack* fstStackCreate(size_t elemSize, StackFreeElem freeFn) {
-//  FstStack *s = taosMemoryCalloc(1, sizeof(FstStack));
-//  if (s == NULL) { return NULL; }
-//  s->
-//  s->freeFn
-//
-//}
-// void  *fstStackPush(FstStack *s, void *elem);
-// void  *fstStackTop(FstStack *s);
-// size_t fstStackLen(FstStack *s);
-// void  *fstStackGetAt(FstStack *s, size_t i);
-// void   fstStackDestory(FstStack *);

source/libs/index/src/indexTfile.c

@@ -16,7 +16,7 @@
 #include "index.h"
 #include "indexComm.h"
 #include "indexFst.h"
-#include "indexFstCountingWriter.h"
+#include "indexFstFile.h"
 #include "indexUtil.h"
 #include "taosdef.h"
 #include "taoserror.h"
@@ -103,7 +103,7 @@ TFileCache* tfileCacheCreate(const char* path) {
   for (size_t i = 0; i < taosArrayGetSize(files); i++) {
     char* file = taosArrayGetP(files, i);
 
-    WriterCtx* wc = writerCtxCreate(TFile, file, true, 1024 * 1024 * 64);
+    IFileCtx* wc = idxFileCtxCreate(TFile, file, true, 1024 * 1024 * 64);
     if (wc == NULL) {
       indexError("failed to open index:%s", file);
       goto End;
@@ -175,7 +175,7 @@ void tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader) {
   tfileReaderRef(reader);
   return;
 }
-TFileReader* tfileReaderCreate(WriterCtx* ctx) {
+TFileReader* tfileReaderCreate(IFileCtx* ctx) {
   TFileReader* reader = taosMemoryCalloc(1, sizeof(TFileReader));
   if (reader == NULL) {
     return NULL;
@@ -216,7 +216,7 @@ void tfileReaderDestroy(TFileReader* reader) {
   } else {
     indexInfo("%s is not removed", reader->ctx->file.buf);
   }
-  writerCtxDestroy(reader->ctx, reader->remove);
+  idxFileCtxDestroy(reader->ctx, reader->remove);
 
   taosMemoryFree(reader);
 }
@@ -490,7 +490,7 @@ TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int64_t version, const c
   char fullname[256] = {0};
   tfileGenFileFullName(fullname, path, suid, colName, version);
   // indexInfo("open write file name %s", fullname);
-  WriterCtx* wcx = writerCtxCreate(TFile, fullname, false, 1024 * 1024 * 64);
+  IFileCtx* wcx = idxFileCtxCreate(TFile, fullname, false, 1024 * 1024 * 64);
   if (wcx == NULL) {
     return NULL;
   }
@@ -507,18 +507,18 @@ TFileReader* tfileReaderOpen(char* path, uint64_t suid, int64_t version, const c
   char fullname[256] = {0};
   tfileGenFileFullName(fullname, path, suid, colName, version);
 
-  WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024);
+  IFileCtx* wc = idxFileCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024);
   if (wc == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     indexError("failed to open readonly file: %s, reason: %s", fullname, terrstr());
     return NULL;
   }
-  indexTrace("open read file name:%s, file size: %d", wc->file.buf, wc->file.size);
+  indexTrace("open read file name:%s, file size: %" PRId64 "", wc->file.buf, wc->file.size);
 
   TFileReader* reader = tfileReaderCreate(wc);
   return reader;
 }
-TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) {
+TFileWriter* tfileWriterCreate(IFileCtx* ctx, TFileHeader* header) {
   TFileWriter* tw = taosMemoryCalloc(1, sizeof(TFileWriter));
   if (tw == NULL) {
     indexError("index: %" PRIu64 " failed to alloc TFilerWriter", header->suid);
@@ -609,14 +609,14 @@ void tfileWriterClose(TFileWriter* tw) {
   if (tw == NULL) {
     return;
   }
-  writerCtxDestroy(tw->ctx, false);
+  idxFileCtxDestroy(tw->ctx, false);
   taosMemoryFree(tw);
 }
 void tfileWriterDestroy(TFileWriter* tw) {
   if (tw == NULL) {
     return;
   }
-  writerCtxDestroy(tw->ctx, false);
+  idxFileCtxDestroy(tw->ctx, false);
   taosMemoryFree(tw);
 }
 
@@ -892,8 +892,8 @@ static int tfileReaderLoadHeader(TFileReader* reader) {
   return 0;
 }
 static int tfileReaderLoadFst(TFileReader* reader) {
-  WriterCtx* ctx = reader->ctx;
+  IFileCtx* ctx = reader->ctx;
-  int        size = ctx->size(ctx);
+  int       size = ctx->size(ctx);
 
   // current load fst into memory, refactor it later
   int   fstSize = size - reader->header.fstOffset - sizeof(tfileMagicNumber);
@@ -905,8 +905,9 @@ static int tfileReaderLoadFst(TFileReader* reader) {
   int64_t ts = taosGetTimestampUs();
   int32_t nread = ctx->readFrom(ctx, buf, fstSize, reader->header.fstOffset);
   int64_t cost = taosGetTimestampUs() - ts;
-  indexInfo("nread = %d, and fst offset=%d, fst size: %d, filename: %s, file size: %d, time cost: %" PRId64 "us", nread,
+  indexInfo("nread = %d, and fst offset=%d, fst size: %d, filename: %s, file size: %" PRId64 ", time cost: %" PRId64
-            reader->header.fstOffset, fstSize, ctx->file.buf, ctx->file.size, cost);
+            "us",
+            nread, reader->header.fstOffset, fstSize, ctx->file.buf, ctx->file.size, cost);
   // we assuse fst size less than FST_MAX_SIZE
   assert(nread > 0 && nread <= fstSize);
 
@@ -919,7 +920,7 @@ static int tfileReaderLoadFst(TFileReader* reader) {
 }
 static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result) {
   // TODO(yihao): opt later
-  WriterCtx* ctx = reader->ctx;
+  IFileCtx* ctx = reader->ctx;
   // add block cache
   char    block[4096] = {0};
   int32_t nread = ctx->readFrom(ctx, block, sizeof(block), offset);
@@ -952,7 +953,7 @@ static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray*
 }
 static int tfileReaderVerify(TFileReader* reader) {
   // just validate header and Footer, file corrupted also shuild be verified later
-  WriterCtx* ctx = reader->ctx;
+  IFileCtx* ctx = reader->ctx;
 
   uint64_t tMagicNumber = 0;
 

source/libs/index/test/fstTest.cc

@@ -7,7 +7,6 @@
 #include "index.h"
 #include "indexCache.h"
 #include "indexFst.h"
-#include "indexFstCountingWriter.h"
 #include "indexFstUtil.h"
 #include "indexInt.h"
 #include "indexTfile.h"
@@ -20,7 +19,7 @@ class FstWriter {
  public:
   FstWriter() {
     taosRemoveFile(fileName.c_str());
-    _wc = writerCtxCreate(TFile, fileName.c_str(), false, 64 * 1024 * 1024);
+    _wc = idxFileCtxCreate(TFile, fileName.c_str(), false, 64 * 1024 * 1024);
     _b = fstBuilderCreate(_wc, 0);
   }
   bool Put(const std::string& key, uint64_t val) {
@@ -38,25 +37,25 @@ class FstWriter {
     fstBuilderFinish(_b);
     fstBuilderDestroy(_b);
 
-    writerCtxDestroy(_wc, false);
+    idxFileCtxDestroy(_wc, false);
   }
 
  private:
   FstBuilder* _b;
-  WriterCtx*  _wc;
+  IFileCtx*   _wc;
 };
 
 class FstReadMemory {
  public:
   FstReadMemory(int32_t size, const std::string& fileName = TD_TMP_DIR_PATH "tindex.tindex") {
-    _wc = writerCtxCreate(TFile, fileName.c_str(), true, 64 * 1024);
+    _wc = idxFileCtxCreate(TFile, fileName.c_str(), true, 64 * 1024);
-    _w = fstCountingWriterCreate(_wc);
+    _w = idxFileCreate(_wc);
     _size = size;
     memset((void*)&_s, 0, sizeof(_s));
   }
   bool init() {
     char* buf = (char*)taosMemoryCalloc(1, sizeof(char) * _size);
-    int   nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size);
+    int   nRead = idxFileRead(_w, (uint8_t*)buf, _size);
     if (nRead <= 0) {
       return false;
     }
@@ -141,18 +140,18 @@ class FstReadMemory {
   }
 
   ~FstReadMemory() {
-    fstCountingWriterDestroy(_w);
+    idxFileDestroy(_w);
     fstDestroy(_fst);
     fstSliceDestroy(&_s);
-    writerCtxDestroy(_wc, false);
+    idxFileCtxDestroy(_wc, false);
   }
 
  private:
-  FstCountingWriter* _w;
+  IdxFstFile* _w;
-  Fst*               _fst;
+  Fst*        _fst;
-  FstSlice           _s;
+  FstSlice    _s;
-  WriterCtx*         _wc;
+  IFileCtx*   _wc;
-  int32_t            _size;
+  int32_t     _size;
 };
 
 #define L 100

source/libs/index/test/fstUT.cc

@@ -8,7 +8,6 @@
 #include "index.h"
 #include "indexCache.h"
 #include "indexFst.h"
-#include "indexFstCountingWriter.h"
 #include "indexFstUtil.h"
 #include "indexInt.h"
 #include "indexTfile.h"
@@ -40,7 +39,7 @@ static void EnvCleanup() {}
 class FstWriter {
  public:
   FstWriter() {
-    _wc = writerCtxCreate(TFile, tindex, false, 64 * 1024 * 1024);
+    _wc = idxFileCtxCreate(TFile, tindex, false, 64 * 1024 * 1024);
     _b = fstBuilderCreate(_wc, 0);
   }
   bool Put(const std::string& key, uint64_t val) {
@@ -58,25 +57,25 @@ class FstWriter {
     fstBuilderFinish(_b);
     fstBuilderDestroy(_b);
 
-    writerCtxDestroy(_wc, false);
+    idxFileCtxDestroy(_wc, false);
   }
 
  private:
   FstBuilder* _b;
-  WriterCtx*  _wc;
+  IFileCtx*   _wc;
 };
 
 class FstReadMemory {
  public:
   FstReadMemory(size_t size) {
-    _wc = writerCtxCreate(TFile, tindex, true, 64 * 1024);
+    _wc = idxFileCtxCreate(TFile, tindex, true, 64 * 1024);
-    _w = fstCountingWriterCreate(_wc);
+    _w = idxFileCreate(_wc);
     _size = size;
     memset((void*)&_s, 0, sizeof(_s));
   }
   bool init() {
     char* buf = (char*)taosMemoryCalloc(1, sizeof(char) * _size);
-    int   nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size);
+    int   nRead = idxFileRead(_w, (uint8_t*)buf, _size);
     if (nRead <= 0) {
       return false;
     }
@@ -130,18 +129,18 @@ class FstReadMemory {
   }
 
   ~FstReadMemory() {
-    fstCountingWriterDestroy(_w);
+    idxFileDestroy(_w);
     fstDestroy(_fst);
     fstSliceDestroy(&_s);
-    writerCtxDestroy(_wc, false);
+    idxFileCtxDestroy(_wc, false);
   }
 
  private:
-  FstCountingWriter* _w;
+  IdxFstFile* _w;
-  Fst*               _fst;
+  Fst*        _fst;
-  FstSlice           _s;
+  FstSlice    _s;
-  WriterCtx*         _wc;
+  IFileCtx*   _wc;
-  size_t             _size;
+  size_t      _size;
 };
 
 class FstWriterEnv : public ::testing::Test {

source/libs/index/test/indexTests.cc

@@ -20,7 +20,6 @@
 #include "index.h"
 #include "indexCache.h"
 #include "indexFst.h"
-#include "indexFstCountingWriter.h"
 #include "indexFstUtil.h"
 #include "indexInt.h"
 #include "indexTfile.h"
@@ -51,7 +50,7 @@ class DebugInfo {
 class FstWriter {
  public:
   FstWriter() {
-    _wc = writerCtxCreate(TFile, TD_TMP_DIR_PATH "tindex", false, 64 * 1024 * 1024);
+    _wc = idxFileCtxCreate(TFile, TD_TMP_DIR_PATH "tindex", false, 64 * 1024 * 1024);
     _b = fstBuilderCreate(NULL, 0);
   }
   bool Put(const std::string& key, uint64_t val) {
@@ -64,25 +63,25 @@ class FstWriter {
     fstBuilderFinish(_b);
     fstBuilderDestroy(_b);
 
-    writerCtxDestroy(_wc, false);
+    idxFileCtxDestroy(_wc, false);
   }
 
  private:
   FstBuilder* _b;
-  WriterCtx*  _wc;
+  IFileCtx*   _wc;
 };
 
 class FstReadMemory {
  public:
   FstReadMemory(size_t size) {
-    _wc = writerCtxCreate(TFile, TD_TMP_DIR_PATH "tindex", true, 64 * 1024);
+    _wc = idxFileCtxCreate(TFile, TD_TMP_DIR_PATH "tindex", true, 64 * 1024);
-    _w = fstCountingWriterCreate(_wc);
+    _w = idxFileCreate(_wc);
     _size = size;
     memset((void*)&_s, 0, sizeof(_s));
   }
   bool init() {
     char* buf = (char*)taosMemoryCalloc(1, sizeof(char) * _size);
-    int   nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size);
+    int   nRead = idxFileRead(_w, (uint8_t*)buf, _size);
     if (nRead <= 0) {
       return false;
     }
@@ -124,18 +123,18 @@ class FstReadMemory {
   }
 
   ~FstReadMemory() {
-    fstCountingWriterDestroy(_w);
+    idxFileDestroy(_w);
     fstDestroy(_fst);
     fstSliceDestroy(&_s);
-    writerCtxDestroy(_wc, true);
+    idxFileCtxDestroy(_wc, true);
   }
 
  private:
-  FstCountingWriter* _w;
+  IdxFstFile* _w;
-  Fst*               _fst;
+  Fst*        _fst;
-  FstSlice           _s;
+  FstSlice    _s;
-  WriterCtx*         _wc;
+  IFileCtx*   _wc;
-  size_t             _size;
+  size_t      _size;
 };
 
 #define L 100
@@ -392,13 +391,13 @@ class TFileObj {
 
     fileName_ = path;
 
-    WriterCtx* ctx = writerCtxCreate(TFile, path.c_str(), false, 64 * 1024 * 1024);
+    IFileCtx* ctx = idxFileCtxCreate(TFile, path.c_str(), false, 64 * 1024 * 1024);
 
     writer_ = tfileWriterCreate(ctx, &header);
     return writer_ != NULL ? true : false;
   }
   bool InitReader() {
-    WriterCtx* ctx = writerCtxCreate(TFile, fileName_.c_str(), true, 64 * 1024 * 1024);
+    IFileCtx* ctx = idxFileCtxCreate(TFile, fileName_.c_str(), true, 64 * 1024 * 1024);
     reader_ = tfileReaderCreate(ctx);
     return reader_ != NULL ? true : false;
   }

source/libs/index/test/jsonUT.cc

@@ -7,7 +7,6 @@
 #include "index.h"
 #include "indexCache.h"
 #include "indexFst.h"
-#include "indexFstCountingWriter.h"
 #include "indexFstUtil.h"
 #include "indexInt.h"
 #include "indexTfile.h"

source/libs/index/test/utilUT.cc

@@ -8,7 +8,6 @@
 #include "indexCache.h"
 #include "indexComm.h"
 #include "indexFst.h"
-#include "indexFstCountingWriter.h"
 #include "indexFstUtil.h"
 #include "indexInt.h"
 #include "indexTfile.h"

source/libs/monitor/test/monTest.cpp

@@ -75,18 +75,18 @@ void MonitorTest::GetSysInfo(SMonSysInfo *pInfo) {
   pInfo->cpu_engine = 2.1;
   pInfo->cpu_system = 2.1;
   pInfo->cpu_cores = 2;
-  pInfo->mem_engine = 3.1;
+  pInfo->mem_engine = 3;
-  pInfo->mem_system = 3.2;
+  pInfo->mem_system = 3;
-  pInfo->mem_total = 3.3;
+  pInfo->mem_total = 3;
-  pInfo->disk_engine = 4.1;
+  pInfo->disk_engine = 4;
-  pInfo->disk_used = 4.2;
+  pInfo->disk_used = 4;
-  pInfo->disk_total = 4.3;
+  pInfo->disk_total = 4;
-  pInfo->net_in = 5.1;
+  pInfo->net_in = 5;
-  pInfo->net_out = 5.2;
+  pInfo->net_out = 5;
-  pInfo->io_read = 6.1;
+  pInfo->io_read = 6;
-  pInfo->io_write = 6.2;
+  pInfo->io_write = 6;
-  pInfo->io_read_disk = 7.1;
+  pInfo->io_read_disk = 7;
-  pInfo->io_write_disk = 7.2;
+  pInfo->io_write_disk = 7;
 }
 
 void MonitorTest::GetClusterInfo(SMonClusterInfo *pInfo) {

source/libs/nodes/src/nodesUtilFuncs.c

@@ -19,8 +19,8 @@
 #include "querynodes.h"
 #include "taos.h"
 #include "taoserror.h"
-#include "thash.h"
 #include "tdatablock.h"
+#include "thash.h"
 
 static SNode* makeNode(ENodeType type, size_t size) {
   SNode* p = taosMemoryCalloc(1, size);
@@ -1497,13 +1497,18 @@ typedef struct SCollectFuncsCxt {
   int32_t         errCode;
   FFuncClassifier classifier;
   SNodeList*      pFuncs;
+  SHashObj*       pAliasName;
 } SCollectFuncsCxt;
 
 static EDealRes collectFuncs(SNode* pNode, void* pContext) {
   SCollectFuncsCxt* pCxt = (SCollectFuncsCxt*)pContext;
   if (QUERY_NODE_FUNCTION == nodeType(pNode) && pCxt->classifier(((SFunctionNode*)pNode)->funcId) &&
       !(((SExprNode*)pNode)->orderAlias)) {
-    pCxt->errCode = nodesListStrictAppend(pCxt->pFuncs, nodesCloneNode(pNode));
+    SExprNode* pExpr = (SExprNode*)pNode;
+    if (NULL == taosHashGet(pCxt->pAliasName, pExpr->aliasName, strlen(pExpr->aliasName))) {
+      pCxt->errCode = nodesListStrictAppend(pCxt->pFuncs, nodesCloneNode(pNode));
+      taosHashPut(pCxt->pAliasName, pExpr->aliasName, strlen(pExpr->aliasName), &pExpr, POINTER_BYTES);
+    }
     return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR);
   }
   return DEAL_RES_CONTINUE;
@@ -1515,23 +1520,27 @@ int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, FFuncClassifi
   }
 
   SCollectFuncsCxt cxt = {
-      .errCode = TSDB_CODE_SUCCESS, .classifier = classifier, .pFuncs = (NULL == *pFuncs ? nodesMakeList() : *pFuncs)};
+      .errCode = TSDB_CODE_SUCCESS,
+      .classifier = classifier,
+      .pFuncs = (NULL == *pFuncs ? nodesMakeList() : *pFuncs),
+      .pAliasName = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, false)};
   if (NULL == cxt.pFuncs) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
   *pFuncs = NULL;
   nodesWalkSelectStmt(pSelect, clause, collectFuncs, &cxt);
-  if (TSDB_CODE_SUCCESS != cxt.errCode) {
+  if (TSDB_CODE_SUCCESS == cxt.errCode) {
-    nodesDestroyList(cxt.pFuncs);
+    if (LIST_LENGTH(cxt.pFuncs) > 0) {
-    return cxt.errCode;
+      *pFuncs = cxt.pFuncs;
-  }
+    } else {
-  if (LIST_LENGTH(cxt.pFuncs) > 0) {
+      nodesDestroyList(cxt.pFuncs);
-    *pFuncs = cxt.pFuncs;
+    }
   } else {
     nodesDestroyList(cxt.pFuncs);
   }
+  taosHashCleanup(cxt.pAliasName);
 
-  return TSDB_CODE_SUCCESS;
+  return cxt.errCode;
 }
 
 typedef struct SCollectSpecialNodesCxt {

source/libs/parser/inc/parAst.h

@@ -154,6 +154,7 @@ SNode* createAlterTableDropCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_
 SNode* createAlterTableRenameCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, SToken* pOldColName,
                                  SToken* pNewColName);
 SNode* createAlterTableSetTag(SAstCreateContext* pCxt, SNode* pRealTable, SToken* pTagName, SNode* pVal);
+SNode* setAlterSuperTableType(SNode* pStmt);
 SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName);
 SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type);
 SNode* createShowStmtWithCond(SAstCreateContext* pCxt, ENodeType type, SNode* pDbName, SNode* pTbName,

source/libs/parser/inc/parUtil.h

@@ -53,6 +53,7 @@ typedef struct SParseMetaCache {
 } SParseMetaCache;
 
 int32_t generateSyntaxErrMsg(SMsgBuf* pBuf, int32_t errCode, ...);
+int32_t generateSyntaxErrMsgExt(SMsgBuf* pBuf, int32_t errCode, const char* pFormat, ...);
 int32_t buildInvalidOperationMsg(SMsgBuf* pMsgBuf, const char* msg);
 int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr);
 

source/libs/parser/inc/sql.y

@@ -232,7 +232,7 @@ cmd ::= DROP TABLE multi_drop_clause(A).
 cmd ::= DROP STABLE exists_opt(A) full_table_name(B).                             { pCxt->pRootNode = createDropSuperTableStmt(pCxt, A, B); }
 
 cmd ::= ALTER TABLE alter_table_clause(A).                                        { pCxt->pRootNode = A; }
-cmd ::= ALTER STABLE alter_table_clause(A).                                       { pCxt->pRootNode = A; }
+cmd ::= ALTER STABLE alter_table_clause(A).                                       { pCxt->pRootNode = setAlterSuperTableType(A); }
 
 alter_table_clause(A) ::= full_table_name(B) alter_table_options(C).              { A = createAlterTableModifyOptions(pCxt, B, C); }
 alter_table_clause(A) ::=
@@ -259,7 +259,7 @@ multi_create_clause(A) ::= multi_create_clause(B) create_subtable_clause(C).
 
 create_subtable_clause(A) ::=
   not_exists_opt(B) full_table_name(C) USING full_table_name(D)
-  specific_tags_opt(E) TAGS NK_LP literal_list(F) NK_RP table_options(G).         { A = createCreateSubTableClause(pCxt, B, C, D, E, F, G); }
+  specific_tags_opt(E) TAGS NK_LP expression_list(F) NK_RP table_options(G).      { A = createCreateSubTableClause(pCxt, B, C, D, E, F, G); }
 
 %type multi_drop_clause                                                           { SNodeList* }
 %destructor multi_drop_clause                                                     { nodesDestroyList($$); }

source/libs/parser/src/parAstCreater.c

@@ -1127,6 +1127,11 @@ SNode* createAlterTableSetTag(SAstCreateContext* pCxt, SNode* pRealTable, SToken
   return createAlterTableStmtFinalize(pRealTable, pStmt);
 }
 
+SNode* setAlterSuperTableType(SNode* pStmt) {
+  setNodeType(pStmt, QUERY_NODE_ALTER_SUPER_TABLE_STMT);
+  return pStmt;
+}
+
 SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName) {
   CHECK_PARSER_STATUS(pCxt);
   if (!checkDbName(pCxt, pDbName, false)) {

source/libs/parser/src/parAstParser.c

@@ -247,6 +247,10 @@ static int32_t collectMetaKeyFromAlterTable(SCollectMetaKeyCxt* pCxt, SAlterTabl
   return code;
 }
 
+static int32_t collectMetaKeyFromAlterStable(SCollectMetaKeyCxt* pCxt, SAlterTableStmt* pStmt) {
+  return reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache);
+}
+
 static int32_t collectMetaKeyFromUseDatabase(SCollectMetaKeyCxt* pCxt, SUseDatabaseStmt* pStmt) {
   return reserveDbVgVersionInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache);
 }
@@ -483,6 +487,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
       return collectMetaKeyFromDropTable(pCxt, (SDropTableStmt*)pStmt);
     case QUERY_NODE_ALTER_TABLE_STMT:
       return collectMetaKeyFromAlterTable(pCxt, (SAlterTableStmt*)pStmt);
+    case QUERY_NODE_ALTER_SUPER_TABLE_STMT:
+      return collectMetaKeyFromAlterStable(pCxt, (SAlterTableStmt*)pStmt);
     case QUERY_NODE_USE_DATABASE_STMT:
       return collectMetaKeyFromUseDatabase(pCxt, (SUseDatabaseStmt*)pStmt);
     case QUERY_NODE_CREATE_INDEX_STMT:

source/libs/parser/src/parInsert.c

@@ -48,6 +48,12 @@
     pSql += sToken.n;                         \
   } while (TK_NK_SPACE == sToken.type)
 
+typedef struct SInsertParseBaseContext {
+  SParseContext* pComCxt;
+  char*          pSql;
+  SMsgBuf        msg;
+} SInsertParseBaseContext;
+
 typedef struct SInsertParseContext {
   SParseContext*     pComCxt;             // input
   char*              pSql;                // input
@@ -1105,6 +1111,32 @@ static int32_t storeTableMeta(SInsertParseContext* pCxt, SHashObj* pHash, SName*
   return taosHashPut(pHash, pName, len, &pBackup, POINTER_BYTES);
 }
 
+static int32_t skipParentheses(SInsertParseSyntaxCxt* pCxt) {
+  SToken  sToken;
+  int32_t expectRightParenthesis = 1;
+  while (1) {
+    NEXT_TOKEN(pCxt->pSql, sToken);
+    if (TK_NK_LP == sToken.type) {
+      ++expectRightParenthesis;
+    } else if (TK_NK_RP == sToken.type && 0 == --expectRightParenthesis) {
+      break;
+    }
+    if (0 == sToken.n) {
+      return buildSyntaxErrMsg(&pCxt->msg, ") expected", NULL);
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t skipBoundColumns(SInsertParseSyntaxCxt* pCxt) { return skipParentheses(pCxt); }
+
+static int32_t ignoreBoundColumns(SInsertParseContext* pCxt) {
+  SInsertParseSyntaxCxt cxt = {.pComCxt = pCxt->pComCxt, .pSql = pCxt->pSql, .msg = pCxt->msg, .pMetaCache = NULL};
+  int32_t               code = skipBoundColumns(&cxt);
+  pCxt->pSql = cxt.pSql;
+  return code;
+}
+
 static int32_t skipUsingClause(SInsertParseSyntaxCxt* pCxt);
 
 // pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)
@@ -1453,12 +1485,29 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
     tNameGetFullDbName(&name, dbFName);
     CHECK_CODE(taosHashPut(pCxt->pDbFNameHashObj, dbFName, strlen(dbFName), dbFName, sizeof(dbFName)));
 
+    bool existedUsing = false;
     // USING clause
+    if (TK_USING == sToken.type) {
+      existedUsing = true;
+      CHECK_CODE(parseUsingClause(pCxt, &name, tbFName));
+      NEXT_TOKEN(pCxt->pSql, sToken);
+      autoCreateTbl = true;
+    }
+
+    char* pBoundColsStart = NULL;
+    if (TK_NK_LP == sToken.type) {
+      // pSql -> field1_name, ...)
+      pBoundColsStart = pCxt->pSql;
+      CHECK_CODE(ignoreBoundColumns(pCxt));
+      // CHECK_CODE(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pCxt->pTableMeta)));
+      NEXT_TOKEN(pCxt->pSql, sToken);
+    }
+
     if (TK_USING == sToken.type) {
       CHECK_CODE(parseUsingClause(pCxt, &name, tbFName));
       NEXT_TOKEN(pCxt->pSql, sToken);
       autoCreateTbl = true;
-    } else {
+    } else if (!existedUsing) {
       CHECK_CODE(getTableMeta(pCxt, &name, dbFName));
     }
 
@@ -1467,10 +1516,11 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
                                     sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta,
                                     &dataBuf, NULL, &pCxt->createTblReq));
 
-    if (TK_NK_LP == sToken.type) {
+    if (NULL != pBoundColsStart) {
-      // pSql -> field1_name, ...)
+      char* pCurrPos = pCxt->pSql;
+      pCxt->pSql = pBoundColsStart;
       CHECK_CODE(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pCxt->pTableMeta)));
-      NEXT_TOKEN(pCxt->pSql, sToken);
+      pCxt->pSql = pCurrPos;
     }
 
     if (TK_VALUES == sToken.type) {
@@ -1610,25 +1660,6 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery, SParseMetaCache
   return code;
 }
 
-static int32_t skipParentheses(SInsertParseSyntaxCxt* pCxt) {
-  SToken  sToken;
-  int32_t expectRightParenthesis = 1;
-  while (1) {
-    NEXT_TOKEN(pCxt->pSql, sToken);
-    if (TK_NK_LP == sToken.type) {
-      ++expectRightParenthesis;
-    } else if (TK_NK_RP == sToken.type && 0 == --expectRightParenthesis) {
-      break;
-    }
-    if (0 == sToken.n) {
-      return buildSyntaxErrMsg(&pCxt->msg, ") expected", NULL);
-    }
-  }
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t skipBoundColumns(SInsertParseSyntaxCxt* pCxt) { return skipParentheses(pCxt); }
-
 // pSql -> (field1_value, ...) [(field1_value2, ...) ...]
 static int32_t skipValuesClause(SInsertParseSyntaxCxt* pCxt) {
   int32_t numOfRows = 0;
@@ -1717,8 +1748,25 @@ static int32_t parseInsertBodySyntax(SInsertParseSyntaxCxt* pCxt) {
     SToken tbnameToken = sToken;
     NEXT_TOKEN(pCxt->pSql, sToken);
 
+    bool existedUsing = false;
     // USING clause
     if (TK_USING == sToken.type) {
+      existedUsing = true;
+      CHECK_CODE(collectAutoCreateTableMetaKey(pCxt, &tbnameToken));
+      NEXT_TOKEN(pCxt->pSql, sToken);
+      CHECK_CODE(collectTableMetaKey(pCxt, &sToken));
+      CHECK_CODE(skipUsingClause(pCxt));
+      NEXT_TOKEN(pCxt->pSql, sToken);
+    }
+
+    if (TK_NK_LP == sToken.type) {
+      // pSql -> field1_name, ...)
+      CHECK_CODE(skipBoundColumns(pCxt));
+      NEXT_TOKEN(pCxt->pSql, sToken);
+    }
+
+    if (TK_USING == sToken.type && !existedUsing) {
+      existedUsing = true;
       CHECK_CODE(collectAutoCreateTableMetaKey(pCxt, &tbnameToken));
       NEXT_TOKEN(pCxt->pSql, sToken);
       CHECK_CODE(collectTableMetaKey(pCxt, &sToken));
@@ -1728,12 +1776,6 @@ static int32_t parseInsertBodySyntax(SInsertParseSyntaxCxt* pCxt) {
       CHECK_CODE(collectTableMetaKey(pCxt, &tbnameToken));
     }
 
-    if (TK_NK_LP == sToken.type) {
-      // pSql -> field1_name, ...)
-      CHECK_CODE(skipBoundColumns(pCxt));
-      NEXT_TOKEN(pCxt->pSql, sToken);
-    }
-
     if (TK_VALUES == sToken.type) {
       // pSql -> (field1_value, ...) [(field1_value2, ...) ...]
       CHECK_CODE(skipValuesClause(pCxt));

source/libs/parser/src/parTranslater.c

@@ -121,8 +121,8 @@ static int32_t getTableMetaImpl(STranslateContext* pCxt, const SName* pName, STa
     }
   }
   if (TSDB_CODE_SUCCESS != code) {
-    parserError("catalogGetTableMeta error, code:%s, dbName:%s, tbName:%s", tstrerror(code), pName->dbname,
+    parserError("0x%" PRIx64 " catalogGetTableMeta error, code:%s, dbName:%s, tbName:%s", pCxt->pParseCxt->requestId,
-                pName->tname);
+                tstrerror(code), pName->dbname, pName->tname);
   }
   return code;
 }
@@ -150,8 +150,8 @@ static int32_t getTableCfg(STranslateContext* pCxt, const SName* pName, STableCf
     }
   }
   if (TSDB_CODE_SUCCESS != code) {
-    parserError("catalogRefreshGetTableCfg error, code:%s, dbName:%s, tbName:%s", tstrerror(code), pName->dbname,
+    parserError("0x%" PRIx64 " catalogRefreshGetTableCfg error, code:%s, dbName:%s, tbName:%s",
-                pName->tname);
+                pCxt->pParseCxt->requestId, tstrerror(code), pName->dbname, pName->tname);
   }
   return code;
 }
@@ -173,8 +173,8 @@ static int32_t refreshGetTableMeta(STranslateContext* pCxt, const char* pDbName,
     code = catalogRefreshGetTableMeta(pParCxt->pCatalog, &conn, &name, pMeta, false);
   }
   if (TSDB_CODE_SUCCESS != code) {
-    parserError("catalogRefreshGetTableMeta error, code:%s, dbName:%s, tbName:%s", tstrerror(code), pDbName,
+    parserError("0x%" PRIx64 " catalogRefreshGetTableMeta error, code:%s, dbName:%s, tbName:%s",
-                pTableName);
+                pCxt->pParseCxt->requestId, tstrerror(code), pDbName, pTableName);
   }
   return code;
 }
@@ -196,7 +196,8 @@ static int32_t getDBVgInfoImpl(STranslateContext* pCxt, const SName* pName, SArr
     }
   }
   if (TSDB_CODE_SUCCESS != code) {
-    parserError("catalogGetDBVgInfo error, code:%s, dbFName:%s", tstrerror(code), fullDbName);
+    parserError("0x%" PRIx64 " catalogGetDBVgInfo error, code:%s, dbFName:%s", pCxt->pParseCxt->requestId,
+                tstrerror(code), fullDbName);
   }
   return code;
 }
@@ -227,8 +228,8 @@ static int32_t getTableHashVgroupImpl(STranslateContext* pCxt, const SName* pNam
     }
   }
   if (TSDB_CODE_SUCCESS != code) {
-    parserError("catalogGetTableHashVgroup error, code:%s, dbName:%s, tbName:%s", tstrerror(code), pName->dbname,
+    parserError("0x%" PRIx64 " catalogGetTableHashVgroup error, code:%s, dbName:%s, tbName:%s",
-                pName->tname);
+                pCxt->pParseCxt->requestId, tstrerror(code), pName->dbname, pName->tname);
   }
   return code;
 }
@@ -251,7 +252,8 @@ static int32_t getDBVgVersion(STranslateContext* pCxt, const char* pDbFName, int
     }
   }
   if (TSDB_CODE_SUCCESS != code) {
-    parserError("catalogGetDBVgVersion error, code:%s, dbFName:%s", tstrerror(code), pDbFName);
+    parserError("0x%" PRIx64 " catalogGetDBVgVersion error, code:%s, dbFName:%s", pCxt->pParseCxt->requestId,
+                tstrerror(code), pDbFName);
   }
   return code;
 }
@@ -276,7 +278,8 @@ static int32_t getDBCfg(STranslateContext* pCxt, const char* pDbName, SDbCfgInfo
     }
   }
   if (TSDB_CODE_SUCCESS != code) {
-    parserError("catalogGetDBCfg error, code:%s, dbFName:%s", tstrerror(code), dbFname);
+    parserError("0x%" PRIx64 " catalogGetDBCfg error, code:%s, dbFName:%s", pCxt->pParseCxt->requestId, tstrerror(code),
+                dbFname);
   }
   return code;
 }
@@ -303,6 +306,10 @@ static int32_t getUdfInfo(STranslateContext* pCxt, SFunctionNode* pFunc) {
     pFunc->udfBufSize = funcInfo.bufSize;
     tFreeSFuncInfo(&funcInfo);
   }
+  if (TSDB_CODE_SUCCESS != code) {
+    parserError("0x%" PRIx64 " catalogGetUdfInfo error, code:%s, funcName:%s", pCxt->pParseCxt->requestId,
+                tstrerror(code), pFunc->functionName);
+  }
   return code;
 }
 
@@ -323,7 +330,8 @@ static int32_t getTableIndex(STranslateContext* pCxt, const SName* pName, SArray
     code = catalogGetTableIndex(pParCxt->pCatalog, &conn, pName, pIndexes);
   }
   if (TSDB_CODE_SUCCESS != code) {
-    parserError("getTableIndex error, code:%s, dbName:%s, tbName:%s", tstrerror(code), pName->dbname, pName->tname);
+    parserError("0x%" PRIx64 " getTableIndex error, code:%s, dbName:%s, tbName:%s", pCxt->pParseCxt->requestId,
+                tstrerror(code), pName->dbname, pName->tname);
   }
   return code;
 }
@@ -341,7 +349,7 @@ static int32_t getDnodeList(STranslateContext* pCxt, SArray** pDnodes) {
     code = catalogGetDnodeList(pParCxt->pCatalog, &conn, pDnodes);
   }
   if (TSDB_CODE_SUCCESS != code) {
-    parserError("getDnodeList error, code:%s", tstrerror(code));
+    parserError("0x%" PRIx64 " getDnodeList error, code:%s", pCxt->pParseCxt->requestId, tstrerror(code));
   }
   return code;
 }
@@ -707,7 +715,7 @@ static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** p
 }
 
 static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) {
-  if (isSelectStmt(pCxt->pCurrStmt) && NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) {
+  if (NULL == pCxt->pCurrStmt || isSelectStmt(pCxt->pCurrStmt) && NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) {
     return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_COLUMN, (*pCol)->colName);
   }
 
@@ -1248,6 +1256,22 @@ static int32_t translateForbidGroupByFunc(STranslateContext* pCxt, SFunctionNode
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t translateRepeatScanFunc(STranslateContext* pCxt, SFunctionNode* pFunc) {
+  if (!fmIsRepeatScanFunc(pFunc->funcId)) {
+    return TSDB_CODE_SUCCESS;
+  }
+  if (isSelectStmt(pCxt->pCurrStmt) && NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) {
+    SNode* pTable = ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable;
+    if (QUERY_NODE_REAL_TABLE == nodeType(pTable) &&
+        (TSDB_CHILD_TABLE == ((SRealTableNode*)pTable)->pMeta->tableType ||
+         TSDB_NORMAL_TABLE == ((SRealTableNode*)pTable)->pMeta->tableType)) {
+      return TSDB_CODE_SUCCESS;
+    }
+  }
+  return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_SUPPORT_SINGLE_TABLE,
+                                 "%s is only supported in single table query", pFunc->functionName);
+}
+
 static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) {
   if (NULL != pCurrStmt && QUERY_NODE_SELECT_STMT == nodeType(pCurrStmt)) {
     SSelectStmt* pSelect = (SSelectStmt*)pCurrStmt;
@@ -1370,6 +1394,9 @@ static int32_t translateNoramlFunction(STranslateContext* pCxt, SFunctionNode* p
   if (TSDB_CODE_SUCCESS == code) {
     code = translateForbidGroupByFunc(pCxt, pFunc);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = translateRepeatScanFunc(pCxt, pFunc);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     setFuncClassification(pCxt->pCurrStmt, pFunc);
   }
@@ -2760,6 +2787,7 @@ static int32_t translateDeleteWhere(STranslateContext* pCxt, SDeleteStmt* pDelet
 }
 
 static int32_t translateDelete(STranslateContext* pCxt, SDeleteStmt* pDelete) {
+  pCxt->pCurrStmt = (SNode*)pDelete;
   int32_t code = translateFrom(pCxt, pDelete->pFromTable);
   if (TSDB_CODE_SUCCESS == code) {
     code = translateDeleteWhere(pCxt, pDelete);
@@ -3743,12 +3771,24 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt*
   return TSDB_CODE_SUCCESS;
 }
 
-static SSchema* getColSchema(STableMeta* pTableMeta, const char* pTagName) {
+static SSchema* getColSchema(STableMeta* pTableMeta, const char* pColName) {
   int32_t numOfFields = getNumOfTags(pTableMeta) + getNumOfColumns(pTableMeta);
   for (int32_t i = 0; i < numOfFields; ++i) {
-    SSchema* pTagSchema = pTableMeta->schema + i;
+    SSchema* pSchema = pTableMeta->schema + i;
-    if (0 == strcmp(pTagName, pTagSchema->name)) {
+    if (0 == strcmp(pColName, pSchema->name)) {
-      return pTagSchema;
+      return pSchema;
+    }
+  }
+  return NULL;
+}
+
+static SSchema* getTagSchema(STableMeta* pTableMeta, const char* pTagName) {
+  int32_t  numOfTags = getNumOfTags(pTableMeta);
+  SSchema* pTagsSchema = getTableTagSchema(pTableMeta);
+  for (int32_t i = 0; i < numOfTags; ++i) {
+    SSchema* pSchema = pTagsSchema + i;
+    if (0 == strcmp(pTagName, pSchema->name)) {
+      return pSchema;
     }
   }
   return NULL;
@@ -3756,22 +3796,48 @@ static SSchema* getColSchema(STableMeta* pTableMeta, const char* pTagName) {
 
 static int32_t checkAlterSuperTable(STranslateContext* pCxt, SAlterTableStmt* pStmt) {
   if (TSDB_ALTER_TABLE_UPDATE_TAG_VAL == pStmt->alterType || TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME == pStmt->alterType) {
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE,
+                                   "Set tag value only available for child table");
+  }
+
+  if (pStmt->alterType == TSDB_ALTER_TABLE_UPDATE_OPTIONS && -1 != pStmt->pOptions->ttl) {
     return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE);
   }
+
+  if (pStmt->dataType.type == TSDB_DATA_TYPE_JSON && pStmt->alterType == TSDB_ALTER_TABLE_ADD_TAG) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG);
+  }
+
+  if (pStmt->dataType.type == TSDB_DATA_TYPE_JSON && pStmt->alterType == TSDB_ALTER_TABLE_ADD_COLUMN) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COL_JSON);
+  }
+
+  STableMeta* pTableMeta = NULL;
+  int32_t     code = getTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pTableMeta);
+  if (TSDB_CODE_SUCCESS != code) {
+    return code;
+  }
+
+  SSchema* pTagsSchema = getTableTagSchema(pTableMeta);
+  if (getNumOfTags(pTableMeta) == 1 && pTagsSchema->type == TSDB_DATA_TYPE_JSON &&
+      (pStmt->alterType == TSDB_ALTER_TABLE_ADD_TAG || pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG ||
+       pStmt->alterType == TSDB_ALTER_TABLE_UPDATE_TAG_BYTES)) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG);
+  }
+
   if (TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES == pStmt->alterType ||
       TSDB_ALTER_TABLE_UPDATE_TAG_BYTES == pStmt->alterType) {
-    STableMeta* pTableMeta = NULL;
+    if (TSDB_SUPER_TABLE != pTableMeta->tableType) {
-    int32_t     code = getTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pTableMeta);
+      return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "Table is not super table");
-    if (TSDB_CODE_SUCCESS == code) {
+    }
-      SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName);
+
-      if (NULL == pSchema) {
+    SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName);
-        code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName);
+    if (NULL == pSchema) {
-      } else if (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type ||
+      return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName);
-                 pSchema->bytes >= calcTypeBytes(pStmt->dataType)) {
+    } else if (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type ||
-        code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_MODIFY_COL);
+               pSchema->bytes >= calcTypeBytes(pStmt->dataType)) {
-      }
+      return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_MODIFY_COL);
     }
-    return code;
   }
   return TSDB_CODE_SUCCESS;
 }
@@ -4455,6 +4521,7 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
       code = translateDropSuperTable(pCxt, (SDropSuperTableStmt*)pNode);
       break;
     case QUERY_NODE_ALTER_TABLE_STMT:
+    case QUERY_NODE_ALTER_SUPER_TABLE_STMT:
       code = translateAlterSuperTable(pCxt, (SAlterTableStmt*)pNode);
       break;
     case QUERY_NODE_CREATE_USER_STMT:
@@ -5222,30 +5289,62 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, S
   }
 }
 
-static int32_t createValueFromFunction(STranslateContext* pCxt, SFunctionNode* pFunc, SValueNode** pVal) {
-  int32_t code = getFuncInfo(pCxt, pFunc);
-  if (TSDB_CODE_SUCCESS == code) {
-    code = scalarCalculateConstants((SNode*)pFunc, (SNode**)pVal);
-  }
-  return code;
-}
-
 static SDataType schemaToDataType(uint8_t precision, SSchema* pSchema) {
   SDataType dt = {.type = pSchema->type, .bytes = pSchema->bytes, .precision = precision, .scale = 0};
   return dt;
 }
 
-static int32_t translateTagVal(STranslateContext* pCxt, uint8_t precision, SSchema* pSchema, SNode* pNode,
+static int32_t createCastFuncForTag(STranslateContext* pCxt, SNode* pNode, SDataType dt, SNode** pCast) {
-                               SValueNode** pVal) {
+  SNode* pExpr = nodesCloneNode(pNode);
-  if (QUERY_NODE_FUNCTION == nodeType(pNode)) {
+  if (NULL == pExpr) {
-    return createValueFromFunction(pCxt, (SFunctionNode*)pNode, pVal);
+    return TSDB_CODE_OUT_OF_MEMORY;
-  } else if (QUERY_NODE_VALUE == nodeType(pNode)) {
+  }
-    return (DEAL_RES_ERROR == translateValueImpl(pCxt, (SValueNode*)pNode, schemaToDataType(precision, pSchema))
+  int32_t code = translateExpr(pCxt, &pExpr);
-                ? pCxt->errCode
+  if (TSDB_CODE_SUCCESS == code) {
-                : TSDB_CODE_SUCCESS);
+    code = createCastFunc(pCxt, pExpr, dt, pCast);
+  }
+  if (TSDB_CODE_SUCCESS != code) {
+    nodesDestroyNode(pExpr);
+  }
+  return code;
+}
+
+static int32_t createTagValFromExpr(STranslateContext* pCxt, SDataType targetDt, SNode* pNode, SValueNode** pVal) {
+  SNode*  pCast = NULL;
+  int32_t code = createCastFuncForTag(pCxt, pNode, targetDt, &pCast);
+  SNode*  pNew = NULL;
+  if (TSDB_CODE_SUCCESS == code) {
+    code = scalarCalculateConstants(pCast, &pNew);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    pCast = pNew;
+    if (QUERY_NODE_VALUE != nodeType(pCast)) {
+      code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)pNode)->aliasName);
+    }
+  }
+
+  if (TSDB_CODE_SUCCESS == code) {
+    *pVal = (SValueNode*)pCast;
   } else {
-    // return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)pNode)->aliasName);
+    nodesDestroyNode(pCast);
-    return TSDB_CODE_FAILED;
+  }
+  return code;
+}
+
+static int32_t createTagValFromVal(STranslateContext* pCxt, SDataType targetDt, SNode* pNode, SValueNode** pVal) {
+  *pVal = (SValueNode*)nodesCloneNode(pNode);
+  if (NULL == *pVal) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  return DEAL_RES_ERROR == translateValueImpl(pCxt, *pVal, targetDt) ? pCxt->errCode : TSDB_CODE_SUCCESS;
+}
+
+static int32_t createTagVal(STranslateContext* pCxt, uint8_t precision, SSchema* pSchema, SNode* pNode,
+                            SValueNode** pVal) {
+  if (QUERY_NODE_VALUE == nodeType(pNode)) {
+    return createTagValFromVal(pCxt, schemaToDataType(precision, pSchema), pNode, pVal);
+  } else {
+    return createTagValFromExpr(pCxt, schemaToDataType(precision, pSchema), pNode, pVal);
   }
 }
 
@@ -5285,56 +5384,45 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla
   if (NULL == pTagArray) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
-  int32_t  code = TSDB_CODE_SUCCESS;
+
-  SSchema* pTagSchema = getTableTagSchema(pSuperTableMeta);
+  int32_t code = TSDB_CODE_SUCCESS;
-  SNode *  pTag = NULL, *pNode = NULL;
+
-  bool     isJson = false;
+  bool       isJson = false;
+  SNodeList* pVals = NULL;
+  SNode *    pTag = NULL, *pNode = NULL;
   FORBOTH(pTag, pStmt->pSpecificTags, pNode, pStmt->pValsOfTags) {
     SColumnNode* pCol = (SColumnNode*)pTag;
-    SSchema*     pSchema = NULL;
+    SSchema*     pSchema = getTagSchema(pSuperTableMeta, pCol->colName);
-    for (int32_t i = 0; i < numOfTags; ++i) {
-      if (0 == strcmp(pCol->colName, pTagSchema[i].name)) {
-        pSchema = pTagSchema + i;
-        break;
-      }
-    }
     if (NULL == pSchema) {
       code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, pCol->colName);
-      goto end;
     }
     SValueNode* pVal = NULL;
-    code = translateTagVal(pCxt, pSuperTableMeta->tableInfo.precision, pSchema, pNode, &pVal);
+    if (TSDB_CODE_SUCCESS == code) {
-    if (TSDB_CODE_SUCCESS != code) {
+      code = createTagVal(pCxt, pSuperTableMeta->tableInfo.precision, pSchema, pNode, &pVal);
-      goto end;
     }
-
+    if (TSDB_CODE_SUCCESS == code) {
-    if (NULL == pVal) {
+      if (pSchema->type == TSDB_DATA_TYPE_JSON) {
-      pVal = (SValueNode*)pNode;
+        isJson = true;
-    } else {
+        code = buildJsonTagVal(pCxt, pSchema, pVal, pTagArray, ppTag);
-      REPLACE_LIST2_NODE(pVal);
+      } else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) {
-    }
+        code = buildNormalTagVal(pCxt, pSchema, pVal, pTagArray);
-
-    if (pSchema->type == TSDB_DATA_TYPE_JSON) {
-      isJson = true;
-      code = buildJsonTagVal(pCxt, pSchema, pVal, pTagArray, ppTag);
-    } else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) {
-      code = buildNormalTagVal(pCxt, pSchema, pVal, pTagArray);
-    }
-  }
-
-  if (!isJson) code = tTagNew(pTagArray, 1, false, ppTag);
-
-end:
-  if (isJson) {
-    for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) {
-      STagVal* p = (STagVal*)taosArrayGet(pTagArray, i);
-      if (IS_VAR_DATA_TYPE(p->type)) {
-        taosMemoryFree(p->pData);
       }
     }
+    if (TSDB_CODE_SUCCESS == code) {
+      code = nodesListMakeAppend(&pVals, (SNode*)pVal);
+    }
+    if (TSDB_CODE_SUCCESS != code) {
+      break;
+    }
   }
+
+  if (TSDB_CODE_SUCCESS == code && !isJson) {
+    code = tTagNew(pTagArray, 1, false, ppTag);
+  }
+
+  nodesDestroyList(pVals);
   taosArrayDestroy(pTagArray);
-  return TSDB_CODE_SUCCESS;
+  return code;
 }
 
 static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClause* pStmt, STableMeta* pSuperTableMeta,
@@ -5343,64 +5431,52 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau
     return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TAGS_NOT_MATCHED);
   }
 
-  SSchema* pTagSchemas = getTableTagSchema(pSuperTableMeta);
+  SArray* pTagArray = taosArrayInit(LIST_LENGTH(pStmt->pValsOfTags), sizeof(STagVal));
-  SNode*   pNode;
+  if (NULL == pTagArray) {
-  int32_t  code = TSDB_CODE_SUCCESS;
+    return TSDB_CODE_OUT_OF_MEMORY;
-  int32_t  index = 0;
-  SArray*  pTagArray = taosArrayInit(LIST_LENGTH(pStmt->pValsOfTags), sizeof(STagVal));
-  if (!pTagArray) {
-    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_TSC_OUT_OF_MEMORY);
   }
 
-  bool isJson = false;
+  int32_t code = TSDB_CODE_SUCCESS;
+
+  bool       isJson = false;
+  int32_t    index = 0;
+  SSchema*   pTagSchemas = getTableTagSchema(pSuperTableMeta);
+  SNodeList* pVals = NULL;
+  SNode*     pNode;
   FOREACH(pNode, pStmt->pValsOfTags) {
     SValueNode* pVal = NULL;
     SSchema*    pTagSchema = pTagSchemas + index;
-    code = translateTagVal(pCxt, pSuperTableMeta->tableInfo.precision, pTagSchema, pNode, &pVal);
+    code = createTagVal(pCxt, pSuperTableMeta->tableInfo.precision, pTagSchema, pNode, &pVal);
+    if (TSDB_CODE_SUCCESS == code) {
+      if (pTagSchema->type == TSDB_DATA_TYPE_JSON) {
+        isJson = true;
+        code = buildJsonTagVal(pCxt, pTagSchema, pVal, pTagArray, ppTag);
+      } else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL && !pVal->isNull) {
+        char*   tmpVal = nodesGetValueFromNode(pVal);
+        STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type};
+        if (IS_VAR_DATA_TYPE(pTagSchema->type)) {
+          val.pData = varDataVal(tmpVal);
+          val.nData = varDataLen(tmpVal);
+        } else {
+          memcpy(&val.i64, tmpVal, pTagSchema->bytes);
+        }
+        taosArrayPush(pTagArray, &val);
+      }
+    }
+    if (TSDB_CODE_SUCCESS == code) {
+      code = nodesListMakeAppend(&pVals, (SNode*)pVal);
+    }
     if (TSDB_CODE_SUCCESS != code) {
-      goto end;
+      break;
-    }
-    if (NULL == pVal) {
-      pVal = (SValueNode*)pNode;
-    } else {
-      REPLACE_NODE(pVal);
-    }
-    if (pTagSchema->type == TSDB_DATA_TYPE_JSON) {
-      if (pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
-        code = buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal);
-        goto end;
-      }
-
-      isJson = true;
-      code = parseJsontoTagData(pVal->literal, pTagArray, ppTag, &pCxt->msgBuf);
-      if (code != TSDB_CODE_SUCCESS) {
-        goto end;
-      }
-    } else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL && !pVal->isNull) {
-      char*   tmpVal = nodesGetValueFromNode(pVal);
-      STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type};
-      if (IS_VAR_DATA_TYPE(pTagSchema->type)) {
-        val.pData = varDataVal(tmpVal);
-        val.nData = varDataLen(tmpVal);
-      } else {
-        memcpy(&val.i64, tmpVal, pTagSchema->bytes);
-      }
-      taosArrayPush(pTagArray, &val);
     }
     ++index;
   }
-  if (!isJson) code = tTagNew(pTagArray, 1, false, ppTag);
 
-end:
+  if (TSDB_CODE_SUCCESS == code && !isJson) {
-  if (isJson) {
+    code = tTagNew(pTagArray, 1, false, ppTag);
-    for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) {
-      STagVal* p = (STagVal*)taosArrayGet(pTagArray, i);
-      if (IS_VAR_DATA_TYPE(p->type)) {
-        taosMemoryFree(p->pData);
-      }
-    }
   }
 
+  nodesDestroyList(pVals);
   taosArrayDestroy(pTagArray);
   return code;
 }
@@ -5898,15 +5974,6 @@ static int32_t rewriteAlterTableImpl(STranslateContext* pCxt, SAlterTableStmt* p
   }
 
   if (TSDB_SUPER_TABLE == pTableMeta->tableType) {
-    SSchema* pTagsSchema = getTableTagSchema(pTableMeta);
-    if (getNumOfTags(pTableMeta) == 1 && pTagsSchema->type == TSDB_DATA_TYPE_JSON &&
-        (pStmt->alterType == TSDB_ALTER_TABLE_ADD_TAG || pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG ||
-         pStmt->alterType == TSDB_ALTER_TABLE_UPDATE_TAG_BYTES)) {
-      return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG);
-    }
-    if (pStmt->alterType == TSDB_ALTER_TABLE_UPDATE_OPTIONS && -1 != pStmt->pOptions->ttl) {
-      return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE);
-    }
     return TSDB_CODE_SUCCESS;
   } else if (TSDB_CHILD_TABLE != pTableMeta->tableType && TSDB_NORMAL_TABLE != pTableMeta->tableType) {
     return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE);
@@ -5929,10 +5996,6 @@ static int32_t rewriteAlterTableImpl(STranslateContext* pCxt, SAlterTableStmt* p
 static int32_t rewriteAlterTable(STranslateContext* pCxt, SQuery* pQuery) {
   SAlterTableStmt* pStmt = (SAlterTableStmt*)pQuery->pRoot;
 
-  if (pStmt->dataType.type == TSDB_DATA_TYPE_JSON && pStmt->alterType == TSDB_ALTER_TABLE_ADD_TAG) {
-    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG);
-  }
-
   if (pStmt->dataType.type == TSDB_DATA_TYPE_JSON && pStmt->alterType == TSDB_ALTER_TABLE_ADD_COLUMN) {
     return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COL_JSON);
   }

source/libs/parser/src/parUtil.c

@@ -215,13 +215,21 @@ int32_t generateSyntaxErrMsg(SMsgBuf* pBuf, int32_t errCode, ...) {
   return errCode;
 }
 
+int32_t generateSyntaxErrMsgExt(SMsgBuf* pBuf, int32_t errCode, const char* pFormat, ...) {
+  va_list vArgList;
+  va_start(vArgList, pFormat);
+  vsnprintf(pBuf->buf, pBuf->len, pFormat, vArgList);
+  va_end(vArgList);
+  return errCode;
+}
+
 int32_t buildInvalidOperationMsg(SMsgBuf* pBuf, const char* msg) {
   strncpy(pBuf->buf, msg, pBuf->len);
   return TSDB_CODE_TSC_INVALID_OPERATION;
 }
 
 int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr) {
-  if(pBuf == NULL) return TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
+  if (pBuf == NULL) return TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
   const char* msgFormat1 = "syntax error near \'%s\'";
   const char* msgFormat2 = "syntax error near \'%s\' (%s)";
   const char* msgFormat3 = "%s";

source/libs/parser/test/CMakeLists.txt

@@ -1,32 +1,34 @@
 
 MESSAGE(STATUS "build parser unit test")
 
-# GoogleTest requires at least C++11
+IF(NOT TD_DARWIN)
-SET(CMAKE_CXX_STANDARD 11)
+    # GoogleTest requires at least C++11
-AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+    SET(CMAKE_CXX_STANDARD 11)
+    AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
-ADD_EXECUTABLE(parserTest ${SOURCE_LIST})
+    ADD_EXECUTABLE(parserTest ${SOURCE_LIST})
 
-TARGET_INCLUDE_DIRECTORIES(
+    TARGET_INCLUDE_DIRECTORIES(
-        parserTest
+            parserTest
-        PUBLIC "${TD_SOURCE_DIR}/include/libs/parser/"
+            PUBLIC "${TD_SOURCE_DIR}/include/libs/parser/"
-        PRIVATE "${TD_SOURCE_DIR}/source/libs/parser/inc"
+            PRIVATE "${TD_SOURCE_DIR}/source/libs/parser/inc"
-)
-
-TARGET_LINK_LIBRARIES(
-        parserTest
-        PUBLIC os util common nodes parser catalog transport gtest function planner qcom
-)
-
-if(${BUILD_WINGETOPT})
-    target_include_directories(
-        parserTest
-        PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src"
     )
-    target_link_libraries(parserTest PUBLIC wingetopt)
-endif()
 
-add_test(
+    TARGET_LINK_LIBRARIES(
-    NAME parserTest
+            parserTest
-    COMMAND parserTest
+            PUBLIC os util common nodes parser catalog transport gtest function planner qcom
-)
+    )
+
+    if(${BUILD_WINGETOPT})
+        target_include_directories(
+            parserTest
+            PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src"
+        )
+        target_link_libraries(parserTest PUBLIC wingetopt)
+    endif()
+
+    add_test(
+        NAME parserTest
+        COMMAND parserTest
+    )
+ENDIF()

source/libs/parser/test/parInitialATest.cpp

@@ -77,8 +77,6 @@ TEST_F(ParserInitialATest, alterLocal) {
   clearAlterLocal();
 }
 
-// todo ALTER stable
-
 /*
  * ALTER TABLE [db_name.]tb_name alter_table_clause
  *
@@ -157,7 +155,7 @@ TEST_F(ParserInitialATest, alterSTable) {
   };
 
   setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) {
-    ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_ALTER_TABLE_STMT);
+    ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_ALTER_SUPER_TABLE_STMT);
     SMAlterStbReq req = {0};
     ASSERT_EQ(tDeserializeSMAlterStbReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req), TSDB_CODE_SUCCESS);
     ASSERT_EQ(std::string(req.name), std::string(expect.name));
@@ -181,44 +179,44 @@ TEST_F(ParserInitialATest, alterSTable) {
   });
 
   //  setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_OPTIONS, 0, nullptr, 0, 0, nullptr, nullptr, 10);
-  //  run("ALTER TABLE st1 TTL 10");
+  //  run("ALTER STABLE st1 TTL 10");
   //  clearAlterStbReq();
 
   setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_OPTIONS, 0, nullptr, 0, 0, nullptr, "test");
-  run("ALTER TABLE st1 COMMENT 'test'");
+  run("ALTER STABLE st1 COMMENT 'test'");
   clearAlterStbReq();
 
   setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_ADD_COLUMN, 1, "cc1", TSDB_DATA_TYPE_BIGINT);
-  run("ALTER TABLE st1 ADD COLUMN cc1 BIGINT");
+  run("ALTER STABLE st1 ADD COLUMN cc1 BIGINT");
   clearAlterStbReq();
 
   setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_DROP_COLUMN, 1, "c1");
-  run("ALTER TABLE st1 DROP COLUMN c1");
+  run("ALTER STABLE st1 DROP COLUMN c1");
   clearAlterStbReq();
 
   setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, 1, "c2", TSDB_DATA_TYPE_VARCHAR,
                      30 + VARSTR_HEADER_SIZE);
-  run("ALTER TABLE st1 MODIFY COLUMN c2 VARCHAR(30)");
+  run("ALTER STABLE st1 MODIFY COLUMN c2 VARCHAR(30)");
   clearAlterStbReq();
 
   // setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, 2, "c1", 0, 0, "cc1");
-  // run("ALTER TABLE st1 RENAME COLUMN c1 cc1");
+  // run("ALTER STABLE st1 RENAME COLUMN c1 cc1");
 
   setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_ADD_TAG, 1, "tag11", TSDB_DATA_TYPE_BIGINT);
-  run("ALTER TABLE st1 ADD TAG tag11 BIGINT");
+  run("ALTER STABLE st1 ADD TAG tag11 BIGINT");
   clearAlterStbReq();
 
   setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_DROP_TAG, 1, "tag1");
-  run("ALTER TABLE st1 DROP TAG tag1");
+  run("ALTER STABLE st1 DROP TAG tag1");
   clearAlterStbReq();
 
   setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, 1, "tag2", TSDB_DATA_TYPE_VARCHAR,
                      30 + VARSTR_HEADER_SIZE);
-  run("ALTER TABLE st1 MODIFY TAG tag2 VARCHAR(30)");
+  run("ALTER STABLE st1 MODIFY TAG tag2 VARCHAR(30)");
   clearAlterStbReq();
 
   setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_TAG_NAME, 2, "tag1", 0, 0, "tag11");
-  run("ALTER TABLE st1 RENAME TAG tag1 tag11");
+  run("ALTER STABLE st1 RENAME TAG tag1 tag11");
   clearAlterStbReq();
 
   // todo
@@ -228,11 +226,11 @@ TEST_F(ParserInitialATest, alterSTable) {
 TEST_F(ParserInitialATest, alterSTableSemanticCheck) {
   useDb("root", "test");
 
-  run("ALTER TABLE st1 RENAME COLUMN c1 cc1", TSDB_CODE_PAR_INVALID_ALTER_TABLE);
+  run("ALTER STABLE st1 RENAME COLUMN c1 cc1", TSDB_CODE_PAR_INVALID_ALTER_TABLE);
 
-  run("ALTER TABLE st1 MODIFY COLUMN c2 NCHAR(10)", TSDB_CODE_PAR_INVALID_MODIFY_COL);
+  run("ALTER STABLE st1 MODIFY COLUMN c2 NCHAR(10)", TSDB_CODE_PAR_INVALID_MODIFY_COL);
 
-  run("ALTER TABLE st1 MODIFY TAG tag2 NCHAR(10)", TSDB_CODE_PAR_INVALID_MODIFY_COL);
+  run("ALTER STABLE st1 MODIFY TAG tag2 NCHAR(10)", TSDB_CODE_PAR_INVALID_MODIFY_COL);
 }
 
 TEST_F(ParserInitialATest, alterTable) {

source/libs/planner/src/planOptimizer.c

@@ -720,7 +720,7 @@ static int32_t pushDownCondOptDealAgg(SOptimizeContext* pCxt, SAggLogicNode* pAg
   // TODO: remove it after full implementation of pushing down to child
   if (1 != LIST_LENGTH(pAgg->node.pChildren) ||
       QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(nodesListGetNode(pAgg->node.pChildren, 0)) &&
-      QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(nodesListGetNode(pAgg->node.pChildren, 0))) {
+          QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(nodesListGetNode(pAgg->node.pChildren, 0))) {
     return TSDB_CODE_SUCCESS;
   }
 
@@ -1251,7 +1251,7 @@ static SNode* partTagsCreateWrapperFunc(const char* pFuncName, SNode* pNode) {
   }
 
   strcpy(pFunc->functionName, pFuncName);
-  if (QUERY_NODE_COLUMN == nodeType(pNode)) {
+  if (QUERY_NODE_COLUMN == nodeType(pNode) && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType) {
     SColumnNode* pCol = (SColumnNode*)pNode;
     partTagsSetAlias(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), pCol->tableAlias, pCol->colName);
   } else {
@@ -1868,6 +1868,8 @@ static EDealRes mergeProjectionsExpr(SNode** pNode, void* pContext) {
               pCxt->errCode = terrno;
               return DEAL_RES_ERROR;
             }
+            snprintf(((SExprNode*)pExpr)->aliasName, sizeof(((SExprNode*)pExpr)->aliasName), "%s",
+                     ((SExprNode*)*pNode)->aliasName);
             nodesDestroyNode(*pNode);
             *pNode = pExpr;
           }

source/libs/planner/src/planSpliter.c

@@ -986,6 +986,10 @@ static bool unionIsChildSubplan(SLogicNode* pLogicNode, int32_t groupId) {
     return ((SExchangeLogicNode*)pLogicNode)->srcGroupId == groupId;
   }
 
+  if (QUERY_NODE_LOGIC_PLAN_MERGE == nodeType(pLogicNode)) {
+    return ((SMergeLogicNode*)pLogicNode)->srcGroupId == groupId;
+  }
+
   SNode* pChild;
   FOREACH(pChild, pLogicNode->pChildren) {
     bool isChild = unionIsChildSubplan((SLogicNode*)pChild, groupId);

source/libs/planner/test/planOptimizeTest.cpp

@@ -68,6 +68,8 @@ TEST_F(PlanOptimizeTest, PartitionTags) {
   run("SELECT SUM(c1), tag1 FROM st1 GROUP BY tag1");
 
   run("SELECT SUM(c1), tag1 + 10 FROM st1 GROUP BY tag1 + 10");
+
+  run("SELECT SUM(c1), tbname FROM st1 GROUP BY tbname");
 }
 
 TEST_F(PlanOptimizeTest, eliminateProjection) {

source/libs/planner/test/planSetOpTest.cpp

@@ -97,7 +97,15 @@ TEST_F(PlanSetOpTest, unionSubquery) {
   run("SELECT * FROM (SELECT c1, c2 FROM t1 UNION SELECT c1, c2 FROM t1)");
 }
 
-TEST_F(PlanSetOpTest, bug001) {
+TEST_F(PlanSetOpTest, unionWithSubquery) {
+  useDb("root", "test");
+
+  run("SELECT c1 FROM (SELECT c1 FROM st1) UNION SELECT c2 FROM (SELECT c1 AS c2 FROM st2)");
+
+  run("SELECT c1 FROM (SELECT c1 FROM st1 ORDER BY c2) UNION SELECT c1 FROM (SELECT c1 FROM st2)");
+}
+
+TEST_F(PlanSetOpTest, unionDataTypeConversion) {
   useDb("root", "test");
 
   run("SELECT c2 FROM t1 WHERE c1 IS NOT NULL GROUP BY c2 "

source/libs/qworker/test/CMakeLists.txt

@@ -1,18 +1,19 @@
 
 MESSAGE(STATUS "build qworker unit test")
+IF(NOT TD_DARWIN)
+        # GoogleTest requires at least C++11
+        SET(CMAKE_CXX_STANDARD 11)
+        AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
-# GoogleTest requires at least C++11
+        ADD_EXECUTABLE(qworkerTest ${SOURCE_LIST})
-SET(CMAKE_CXX_STANDARD 11)
+        TARGET_LINK_LIBRARIES(
-AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+                qworkerTest
+                PUBLIC os util common transport gtest qcom nodes planner qworker executor
+        )
 
-ADD_EXECUTABLE(qworkerTest ${SOURCE_LIST})
+        TARGET_INCLUDE_DIRECTORIES(
-TARGET_LINK_LIBRARIES(
+                qworkerTest
-        qworkerTest
+                PUBLIC "${TD_SOURCE_DIR}/include/libs/qworker/"
-        PUBLIC os util common transport gtest qcom nodes planner qworker executor
+                PRIVATE "${TD_SOURCE_DIR}/source/libs/qworker/inc"
-)
+        )
-
+ENDIF()
-TARGET_INCLUDE_DIRECTORIES(
-        qworkerTest
-        PUBLIC "${TD_SOURCE_DIR}/include/libs/qworker/"
-        PRIVATE "${TD_SOURCE_DIR}/source/libs/qworker/inc"
-)