Merge branch '3.0' into cpwu/3.0

cmake/cmake.define

@@ -98,12 +98,12 @@ ELSE ()
     ENDIF ()
 
     IF (${SANITIZER} MATCHES "true")
-        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}     -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3")
+        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}     -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
-        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3")
+        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
 	MESSAGE(STATUS "Will compile with Address Sanitizer!")
     ELSE ()
-        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3")
+        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=0")
-        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3")
+        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=0")
     ENDIF ()
 
     MESSAGE("System processor ID: ${CMAKE_SYSTEM_PROCESSOR}")

include/common/tcommon.h

@@ -56,7 +56,8 @@ typedef enum EStreamType {
   STREAM_CLEAR,
   STREAM_INVALID,
   STREAM_GET_ALL,
-  STREAM_DELETE,
+  STREAM_DELETE_RESULT,
+  STREAM_DELETE_DATA,
   STREAM_RETRIEVE,
   STREAM_PULL_DATA,
   STREAM_PULL_OVER,

include/libs/qcom/query.h

@@ -65,7 +65,7 @@ typedef struct SQueryExecRes {
 } SQueryExecRes;
 
 typedef struct SIndexMeta {
-#ifdef WINDOWS
+#if defined(WINDOWS) || defined(_TD_DARWIN_64)
   size_t avoidCompilationErrors;
 #endif
 

include/os/os.h

@@ -41,7 +41,6 @@ extern "C" {
 #include <sys/types.h>
 #include <termios.h>
 #include <sys/statvfs.h>
-#include <sys/prctl.h>
 #include <sys/shm.h>
 #include <sys/wait.h>
 

include/os/osAtomic.h

@@ -63,7 +63,7 @@ int8_t  atomic_add_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val);
-void   *atomic_add_fetch_ptr(void *ptr, void *val);
+void   *atomic_add_fetch_ptr(void *ptr, int64_t val);
 int8_t  atomic_fetch_add_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val);
@@ -73,7 +73,7 @@ int8_t  atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val);
-void   *atomic_sub_fetch_ptr(void *ptr, void *val);
+void   *atomic_sub_fetch_ptr(void *ptr, int64_t val);
 int8_t  atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val);

include/os/osSemaphore.h

@@ -24,7 +24,9 @@ extern "C" {
 
 #if defined(_TD_DARWIN_64)
 
-typedef struct tsem_s *tsem_t;
+// typedef struct tsem_s *tsem_t;
+typedef struct bosal_sem_t *tsem_t;
+
 
 int tsem_init(tsem_t *sem, int pshared, unsigned int value);
 int tsem_wait(tsem_t *sem);
@@ -51,11 +53,11 @@ int tsem_timewait(tsem_t *sim, int64_t nanosecs);
 //  #define taosThreadRwlockRdlock(lock) taosThreadMutexLock(lock)
 //  #define taosThreadRwlockUnlock(lock) taosThreadMutexUnlock(lock)
 
-#define TdThreadSpinlock               TdThreadMutex
+// #define TdThreadSpinlock               TdThreadMutex
-#define taosThreadSpinInit(lock, NULL) taosThreadMutexInit(lock, NULL)
+// #define taosThreadSpinInit(lock, NULL) taosThreadMutexInit(lock, NULL)
-#define taosThreadSpinDestroy(lock)    taosThreadMutexDestroy(lock)
+// #define taosThreadSpinDestroy(lock)    taosThreadMutexDestroy(lock)
-#define taosThreadSpinLock(lock)       taosThreadMutexLock(lock)
+// #define taosThreadSpinLock(lock)       taosThreadMutexLock(lock)
-#define taosThreadSpinUnlock(lock)     taosThreadMutexUnlock(lock)
+// #define taosThreadSpinUnlock(lock)     taosThreadMutexUnlock(lock)
 #endif
 
 bool    taosCheckPthreadValid(TdThread thread);

include/os/osSocket.h

@@ -64,7 +64,6 @@
 #include <osEok.h>
 #else
 #include <netinet/in.h>
-#include <sys/epoll.h>
 #endif
 #endif
 
@@ -77,15 +76,12 @@ typedef int socklen_t;
 #define TAOS_EPOLL_WAIT_TIME 100
 typedef SOCKET eventfd_t;
 #define eventfd(a, b)      -1
-#define EpollClose(pollFd) epoll_close(pollFd)
 #ifndef EPOLLWAKEUP
 #define EPOLLWAKEUP (1u << 29)
 #endif
 #elif defined(_TD_DARWIN_64)
 #define TAOS_EPOLL_WAIT_TIME 500
 typedef int32_t SOCKET;
-typedef SOCKET  EpollFd;
-#define EpollClose(pollFd)   epoll_close(pollFd)
 #else
 #define TAOS_EPOLL_WAIT_TIME 500
 typedef int32_t SOCKET;
@@ -122,14 +118,6 @@ typedef SOCKET  EpollFd;
 typedef int32_t  SocketFd;
 typedef SocketFd EpollFd;
 
-typedef struct TdSocket {
-#if SOCKET_WITH_LOCK
-  TdThreadRwlock rwlock;
-#endif
-  int      refId;
-  SocketFd fd;
-} * TdSocketPtr, TdSocket;
-
 typedef struct TdSocketServer *TdSocketServerPtr;
 typedef struct TdSocket *      TdSocketPtr;
 typedef struct TdEpoll *       TdEpollPtr;
@@ -181,11 +169,6 @@ void        taosSetMaskSIGPIPE();
 uint32_t    taosInetAddr(const char *ipAddr);
 const char *taosInetNtoa(struct in_addr ipInt);
 
-TdEpollPtr taosCreateEpoll(int32_t size);
-int32_t    taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocket, struct epoll_event *event);
-int32_t    taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxEvents, int32_t timeout);
-int32_t    taosCloseEpoll(TdEpollPtr *ppEpoll);
-
 #ifdef __cplusplus
 }
 #endif

include/os/osThread.h

@@ -188,27 +188,27 @@ int32_t taosThreadJoin(TdThread thread, void **valuePtr);
 int32_t taosThreadKeyCreate(TdThreadKey * key, void(*destructor)(void *));
 int32_t taosThreadKeyDelete(TdThreadKey key);
 int32_t taosThreadKill(TdThread thread, int32_t sig);
-int32_t taosThreadMutexConsistent(TdThreadMutex* mutex);
+// int32_t taosThreadMutexConsistent(TdThreadMutex* mutex);
 int32_t taosThreadMutexDestroy(TdThreadMutex * mutex);
 int32_t taosThreadMutexInit(TdThreadMutex * mutex, const TdThreadMutexAttr * attr);
 int32_t taosThreadMutexLock(TdThreadMutex * mutex);
-int32_t taosThreadMutexTimedLock(TdThreadMutex * mutex, const struct timespec *abstime);
+// int32_t taosThreadMutexTimedLock(TdThreadMutex * mutex, const struct timespec *abstime);
 int32_t taosThreadMutexTryLock(TdThreadMutex * mutex);
 int32_t taosThreadMutexUnlock(TdThreadMutex * mutex);
 int32_t taosThreadMutexAttrDestroy(TdThreadMutexAttr * attr);
 int32_t taosThreadMutexAttrGetPshared(const TdThreadMutexAttr * attr, int32_t *pshared);
-int32_t taosThreadMutexAttrGetRobust(const TdThreadMutexAttr * attr, int32_t * robust);
+// int32_t taosThreadMutexAttrGetRobust(const TdThreadMutexAttr * attr, int32_t * robust);
 int32_t taosThreadMutexAttrGetType(const TdThreadMutexAttr * attr, int32_t *kind);
 int32_t taosThreadMutexAttrInit(TdThreadMutexAttr * attr);
 int32_t taosThreadMutexAttrSetPshared(TdThreadMutexAttr * attr, int32_t pshared);
-int32_t taosThreadMutexAttrSetRobust(TdThreadMutexAttr * attr, int32_t robust);
+// int32_t taosThreadMutexAttrSetRobust(TdThreadMutexAttr * attr, int32_t robust);
 int32_t taosThreadMutexAttrSetType(TdThreadMutexAttr * attr, int32_t kind);
 int32_t taosThreadOnce(TdThreadOnce * onceControl, void(*initRoutine)(void));
 int32_t taosThreadRwlockDestroy(TdThreadRwlock * rwlock);
 int32_t taosThreadRwlockInit(TdThreadRwlock * rwlock, const TdThreadRwlockAttr * attr);
 int32_t taosThreadRwlockRdlock(TdThreadRwlock * rwlock);
-int32_t taosThreadRwlockTimedRdlock(TdThreadRwlock * rwlock, const struct timespec *abstime);
+// int32_t taosThreadRwlockTimedRdlock(TdThreadRwlock * rwlock, const struct timespec *abstime);
-int32_t taosThreadRwlockTimedWrlock(TdThreadRwlock * rwlock, const struct timespec *abstime);
+// int32_t taosThreadRwlockTimedWrlock(TdThreadRwlock * rwlock, const struct timespec *abstime);
 int32_t taosThreadRwlockTryRdlock(TdThreadRwlock * rwlock);
 int32_t taosThreadRwlockTryWrlock(TdThreadRwlock * rwlock);
 int32_t taosThreadRwlockUnlock(TdThreadRwlock * rwlock);

include/util/tdef.h

@@ -287,7 +287,7 @@ typedef enum ELogicConditionType {
 #define TSDB_MULTI_TABLEMETA_MAX_NUM 100000  // maximum batch size allowed to load table meta
 
 #define TSDB_MIN_VNODES_PER_DB          1
-#define TSDB_MAX_VNODES_PER_DB          4096
+#define TSDB_MAX_VNODES_PER_DB          1024
 #define TSDB_DEFAULT_VN_PER_DB          2
 #define TSDB_MIN_BUFFER_PER_VNODE       3      // unit MB
 #define TSDB_MAX_BUFFER_PER_VNODE       16384  // unit MB

include/util/types.h

@@ -83,6 +83,7 @@ typedef uint16_t VarDataLenT;  // maxVarDataLen: 32767
 
 #define varDataLen(v) ((VarDataLenT *)(v))[0]
 #define varDataVal(v) ((char *)(v) + VARSTR_HEADER_SIZE)
+#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v))
 
 #define NCHAR_WIDTH_TO_BYTES(n)  ((n) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE)
 

source/common/src/systable.c

@@ -108,13 +108,11 @@ static const SSysDbTableSchema userFuncSchema[] = {
 };
 
 static const SSysDbTableSchema userIdxSchema[] = {
+    {.name = "index_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "index_database", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "index_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-    {.name = "column_name", .bytes = SYSTABLE_SCH_COL_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "index_type", .bytes = 10, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "index_extensions", .bytes = 256, .type = TSDB_DATA_TYPE_VARCHAR},
 };
 
 static const SSysDbTableSchema userStbsSchema[] = {
@@ -313,7 +311,7 @@ static const SSysDbTableSchema querySchema[] = {
     {.name = "query_id", .bytes = TSDB_QUERY_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "req_id", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
     {.name = "connId", .bytes = 4, .type = TSDB_DATA_TYPE_UINT},
-    {.name = "app", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},    
+    {.name = "app", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "end_point", .bytes = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
@@ -329,17 +327,17 @@ static const SSysDbTableSchema appSchema[] = {
     {.name = "app_id", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
     {.name = "ip", .bytes = TSDB_IPv4ADDR_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "name", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},    
+    {.name = "name", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "start_time", .bytes = 8 , .type = TSDB_DATA_TYPE_TIMESTAMP},
+    {.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-    {.name = "insert_req", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
+    {.name = "insert_req", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
-    {.name = "insert_row", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
+    {.name = "insert_row", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
-    {.name = "insert_time", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
+    {.name = "insert_time", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
-    {.name = "insert_bytes", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
+    {.name = "insert_bytes", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
-    {.name = "fetch_bytes", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
+    {.name = "fetch_bytes", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
-    {.name = "query_time", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
+    {.name = "query_time", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
-    {.name = "show_query", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
+    {.name = "show_query", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
-    {.name = "total_req", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
+    {.name = "total_req", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
-    {.name = "current_req", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
+    {.name = "current_req", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
     {.name = "last_access", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
 };
 
@@ -353,8 +351,7 @@ static const SSysTableMeta perfsMeta[] = {
     {TSDB_PERFS_TABLE_TRANS, transSchema, tListLen(transSchema)},
     {TSDB_PERFS_TABLE_SMAS, smaSchema, tListLen(smaSchema)},
     {TSDB_PERFS_TABLE_STREAMS, streamSchema, tListLen(streamSchema)},
-    {TSDB_PERFS_TABLE_APPS, appSchema, tListLen(appSchema)}
+    {TSDB_PERFS_TABLE_APPS, appSchema, tListLen(appSchema)}};
-};
 
 void getInfosDbMeta(const SSysTableMeta** pInfosTableMeta, size_t* size) {
   *pInfosTableMeta = infosMeta;

source/common/src/tdatablock.c

@@ -1259,6 +1259,7 @@ int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src) {
 
   dst->info.rows = src->info.rows;
   dst->info.window = src->info.window;
+  dst->info.type = src->info.type;
   return TSDB_CODE_SUCCESS;
 }
 

source/common/src/ttime.c

@@ -589,7 +589,7 @@ int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec
       return TSDB_CODE_FAILED;
     }
     newColData[len] = 0;
-    int32_t ret = taosParseTime(newColData, timeVal, len + 1, (int32_t)timePrec, tsDaylight);
+    int32_t ret = taosParseTime(newColData, timeVal, len, (int32_t)timePrec, tsDaylight);
     if (ret != TSDB_CODE_SUCCESS) {
       taosMemoryFree(newColData);
       return ret;

source/common/src/ttszip.c

@@ -845,11 +845,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) {
 
   int64_t offset = getDataStartOffset();
   int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset;
-#if defined(_TD_DARWIN_64)
-  int64_t written = taosFSendFile(pDestBuf->pFile->fp, pSrcBuf->pFile->fp, &offset, size);
-#else
   int64_t written = taosFSendFile(pDestBuf->pFile, pSrcBuf->pFile, &offset, size);
-#endif
 
   if (written == -1 || written != size) {
     return -1;

source/dnode/mnode/impl/inc/mndInt.h

@@ -87,7 +87,7 @@ typedef struct {
 } STelemMgmt;
 
 typedef struct {
-  sem_t    syncSem;
+  tsem_t    syncSem;
   int64_t  sync;
   bool     standby;
   SReplica replica;

source/dnode/mnode/impl/src/mndDb.c

@@ -549,18 +549,33 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
   terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED;
 
   if (pAlter->buffer > 0 && pAlter->buffer != pDb->cfg.buffer) {
+#if 1
+    terrno = TSDB_CODE_OPS_NOT_SUPPORT;
+    return terrno;
+#else
     pDb->cfg.buffer = pAlter->buffer;
     terrno = 0;
+#endif
   }
 
   if (pAlter->pages > 0 && pAlter->pages != pDb->cfg.pages) {
+#if 1
+    terrno = TSDB_CODE_OPS_NOT_SUPPORT;
+    return terrno;
+#else
     pDb->cfg.pages = pAlter->pages;
     terrno = 0;
+#endif
   }
 
   if (pAlter->pageSize > 0 && pAlter->pageSize != pDb->cfg.pageSize) {
+#if 1
+    terrno = TSDB_CODE_OPS_NOT_SUPPORT;
+    return terrno;
+#else
     pDb->cfg.pageSize = pAlter->pageSize;
     terrno = 0;
+#endif
   }
 
   if (pAlter->daysPerFile > 0 && pAlter->daysPerFile != pDb->cfg.daysPerFile) {
@@ -594,8 +609,12 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
   }
 
   if (pAlter->strict >= 0 && pAlter->strict != pDb->cfg.strict) {
+#if 1
+    terrno = TSDB_CODE_OPS_NOT_SUPPORT;
+#else
     pDb->cfg.strict = pAlter->strict;
     terrno = 0;
+#endif
   }
 
   if (pAlter->cacheLastRow >= 0 && pAlter->cacheLastRow != pDb->cfg.cacheLastRow) {
@@ -604,9 +623,13 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
   }
 
   if (pAlter->replications > 0 && pAlter->replications != pDb->cfg.replications) {
+#if 1
+    terrno = TSDB_CODE_OPS_NOT_SUPPORT;
+#else
     pDb->cfg.replications = pAlter->replications;
     pDb->vgVersion++;
     terrno = 0;
+#endif
   }
 
   return terrno;

source/dnode/mnode/impl/src/mndDnode.c

@@ -758,6 +758,11 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
     }
   }
 
+  if (numOfVnodes > 0) {
+    terrno = TSDB_CODE_OPS_NOT_SUPPORT;
+    goto _OVER;
+  }
+
   code = mndDropDnode(pMnode, pReq, pDnode, pMObj, pQObj, pSObj, numOfVnodes);
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
 

source/dnode/mnode/impl/src/mndShow.c

@@ -71,7 +71,7 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
   } else if (strncasecmp(name, TSDB_INS_TABLE_USER_FUNCTIONS, len) == 0) {
     type = TSDB_MGMT_TABLE_FUNC;
   } else if (strncasecmp(name, TSDB_INS_TABLE_USER_INDEXES, len) == 0) {
-    //    type = TSDB_MGMT_TABLE_INDEX;
+    type = TSDB_MGMT_TABLE_INDEX;
   } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, len) == 0) {
     type = TSDB_MGMT_TABLE_STB;
   } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, len) == 0) {

source/dnode/mnode/impl/src/mndSma.c

@@ -1147,29 +1147,32 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
 
     SName smaName = {0};
     tNameFromString(&smaName, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
+    char n1[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
+    STR_TO_VARSTR(n1, (char *)tNameGetTableName(&smaName));
 
-    char n[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
+    char n2[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
-    STR_TO_VARSTR(n, (char *)tNameGetTableName(&smaName));
+    STR_TO_VARSTR(n2, (char *)mndGetDbStr(pDb->name));
-    cols++;
 
     SName stbName = {0};
     tNameFromString(&stbName, pSma->stb, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
-
+    char n3[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
-    char n1[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
+    STR_TO_VARSTR(n3, (char *)tNameGetTableName(&stbName));
-    STR_TO_VARSTR(n1, (char *)tNameGetTableName(&stbName));
 
     SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)n, false);
-
-    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pSma->createdTime, false);
-
-    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, numOfRows, (const char *)n1, false);
 
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)n2, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)n3, false);
+
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, numOfRows, (const char *)&pSma->dstVgId, false);
 
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pSma->createdTime, false);
+
     numOfRows++;
     sdbRelease(pSdb, pSma);
   }

source/dnode/mnode/impl/src/mndSnode.c

@@ -273,6 +273,9 @@ _OVER:
 }
 
 static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) {
+#if 1
+  return TSDB_CODE_OPS_NOT_SUPPORT;
+#else
   SMnode          *pMnode = pReq->info.node;
   int32_t          code = -1;
   SSnodeObj       *pObj = NULL;
@@ -315,6 +318,7 @@ _OVER:
   mndReleaseSnode(pMnode, pObj);
   mndReleaseDnode(pMnode, pDnode);
   return code;
+#endif
 }
 
 static int32_t mndSetDropSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) {
@@ -386,9 +390,12 @@ _OVER:
 }
 
 static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) {
-  SMnode        *pMnode = pReq->info.node;
+#if 1
-  int32_t        code = -1;
+  return TSDB_CODE_OPS_NOT_SUPPORT;
-  SSnodeObj     *pObj = NULL;
+#else
+  SMnode *pMnode = pReq->info.node;
+  int32_t code = -1;
+  SSnodeObj *pObj = NULL;
   SMDropSnodeReq dropReq = {0};
 
   if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
@@ -422,6 +429,7 @@ _OVER:
 
   mndReleaseSnode(pMnode, pObj);
   return code;
+#endif
 }
 
 static int32_t mndRetrieveSnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {

source/dnode/mnode/impl/src/mndStream.c

@@ -247,7 +247,6 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj,
   pObj->uid = mndGenerateUid(pObj->name, strlen(pObj->name));
   pObj->status = 0;
 
-  // TODO
   pObj->igExpired = pCreate->igExpired;
   pObj->trigger = pCreate->triggerType;
   pObj->triggerParam = pCreate->maxDelay;

source/dnode/mnode/impl/src/mndVgroup.c

@@ -1219,6 +1219,9 @@ _OVER:
 }
 
 static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) {
+#if 1
+  return TSDB_CODE_OPS_NOT_SUPPORT;
+#else
   SMnode    *pMnode = pReq->info.node;
   SDnodeObj *pNew1 = NULL;
   SDnodeObj *pNew2 = NULL;
@@ -1412,6 +1415,7 @@ _OVER:
   mndReleaseDb(pMnode, pDb);
 
   return code;
+#endif  
 }
 
 int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) {
@@ -1711,6 +1715,9 @@ _OVER:
 }
 
 static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) {
+#if 1
+  return TSDB_CODE_OPS_NOT_SUPPORT;
+#else
   SMnode *pMnode = pReq->info.node;
   int32_t code = -1;
   SArray *pArray = NULL;
@@ -1759,6 +1766,7 @@ _OVER:
 
   taosArrayDestroy(pArray);
   return code;
+#endif 
 }
 
 bool mndVgroupInDb(SVgObj *pVgroup, int64_t dbUid) { return !pVgroup->isTsma && pVgroup->dbUid == dbUid; }

source/dnode/mnode/impl/test/db/db.cpp

@@ -93,7 +93,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, 0);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT);
   }
 
   test.SendShowReq(TSDB_MGMT_TABLE_DB, "user_databases", "");

source/dnode/vnode/src/inc/sma.h

@@ -62,12 +62,10 @@ struct STSmaStat {
 
 struct SRSmaStat {
   SSma     *pSma;
-  int64_t   refId;         // shared by persistence/fetch tasks
+  int64_t   refId;         // shared by fetch tasks
-  void     *tmrHandle;     // for persistence task
+  void     *tmrHandle;     // shared by fetch tasks
-  tmr_h     tmrId;         // for persistence task
+  int8_t    triggerStat;   // shared by fetch tasks
-  int32_t   tmrSeconds;    // for persistence task
+  int8_t    runningStat;   // for persistence task 
-  int8_t    triggerStat;   // for persistence task
-  int8_t    runningStat;   // for persistence task
   SHashObj *rsmaInfoHash;  // key: stbUid, value: SRSmaInfo;
 };
 
@@ -82,7 +80,6 @@ struct SSmaStat {
 #define SMA_TSMA_STAT(s)     (&(s)->tsmaStat)
 #define SMA_RSMA_STAT(s)     (&(s)->rsmaStat)
 #define RSMA_INFO_HASH(r)    ((r)->rsmaInfoHash)
-#define RSMA_TMR_ID(r)       ((r)->tmrId)
 #define RSMA_TMR_HANDLE(r)   ((r)->tmrHandle)
 #define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat)
 #define RSMA_RUNNING_STAT(r) (&(r)->runningStat)
@@ -185,9 +182,11 @@ static FORCE_INLINE void tdSmaStatSetDropped(STSmaStat *pTStat) {
 static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType);
 void          *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType);
 void          *tdFreeRSmaInfo(SRSmaInfo *pInfo);
+int32_t        tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat);
 
 int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName);
 int32_t tdProcessRSmaRestoreImpl(SSma *pSma);
+
 int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg);
 int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg);
 int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);
@@ -244,8 +243,8 @@ void    tdUpdateTFileMagic(STFile *pTFile, void *pCksm);
 void    tdCloseTFile(STFile *pTFile);
 void    tdDestroyTFile(STFile *pTFile);
 
-void tdGetVndFileName(int32_t vgId, const char *dname, const char *fname, int64_t version, char *outputName);
+void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version, char *outputName);
-void tdGetVndDirName(int32_t vgId, const char *dname, char *outputName);
+void tdGetVndDirName(int32_t vgId,const char *pdname,  const char *dname, bool endWithSep, char *outputName);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/src/inc/vnodeInt.h

@@ -163,8 +163,8 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool
 
 // sma
 int32_t smaOpen(SVnode* pVnode);
-int32_t smaCloseEnv(SSma* pSma);
+int32_t smaClose(SSma* pSma);
-int32_t smaCloseEx(SSma* pSma);
+int32_t smaBegin(SSma* pSma);
 int32_t smaPreCommit(SSma* pSma);
 int32_t smaCommit(SSma* pSma);
 int32_t smaPostCommit(SSma* pSma);
@@ -241,7 +241,7 @@ struct SVnode {
   tsem_t     canCommit;
   int64_t    sync;
   int32_t    syncCount;
-  sem_t      syncSem;
+  tsem_t     syncSem;
   SQHandle*  pQuery;
 };
 

source/dnode/vnode/src/sma/smaCommit.c

@@ -43,13 +43,48 @@ int32_t smaCommit(SSma *pSma) { return tdProcessRSmaCommitImpl(pSma); }
  */
 int32_t smaPostCommit(SSma *pSma) { return tdProcessRSmaPostCommitImpl(pSma); }
 
+/**
+ * @brief set rsma trigger stat active
+ *
+ * @param pSma
+ * @return int32_t
+ */
+int32_t smaBegin(SSma *pSma) {
+  SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma);
+  if (!pSmaEnv) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SSmaStat  *pStat = SMA_ENV_STAT(pSmaEnv);
+  SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat);
+
+  int8_t rsmaTriggerStat =
+      atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED, TASK_TRIGGER_STAT_ACTIVE);
+  switch (rsmaTriggerStat) {
+    case TASK_TRIGGER_STAT_PAUSED: {
+      smaDebug("vgId:%d rsma trigger stat from paused to active", SMA_VID(pSma));
+      break;
+    }
+    case TASK_TRIGGER_STAT_INIT: {
+      atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_ACTIVE);
+      smaDebug("vgId:%d rsma trigger stat from init to active", SMA_VID(pSma));
+      break;
+    }
+    default: {
+      atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_ACTIVE);
+      smaWarn("vgId:%d rsma trigger stat %" PRIi8 " is unexpected", SMA_VID(pSma), rsmaTriggerStat);
+      ASSERT(0);
+      break;
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 /**
  * @brief pre-commit for rollup sma.
  *  1) set trigger stat of rsma timer TASK_TRIGGER_STAT_PAUSED.
  *  2) perform persist task for qTaskInfo
  *  3) wait all triggered fetch tasks finished
- *  4) set trigger stat of rsma timer TASK_TRIGGER_STAT_ACTIVE.
- *  5) finish
  *
  * @param pSma
  * @return int32_t
@@ -63,10 +98,30 @@ static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma) {
   SSmaStat  *pStat = SMA_ENV_STAT(pSmaEnv);
   SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat);
 
-  // step 1
+
+  // step 1: set persistence task paused
   atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED);
 
-  // step 2
+  // step 2: perform persist task for qTaskInfo
+  tdRSmaPersistExecImpl(pRSmaStat);
+
+  // step 3: wait all triggered fetch tasks finished
+  int32_t nLoops = 0;
+  while (1) {
+    if (T_REF_VAL_GET(pStat) == 0) {
+      smaDebug("vgId:%d, rsma fetch tasks all finished", SMA_VID(pSma));
+      break;
+    } else {
+      smaDebug("vgId:%d, rsma fetch tasks not all finished yet", SMA_VID(pSma));
+    }
+    ++nLoops;
+    if (nLoops > 1000) {
+      sched_yield();
+      nLoops = 0;
+    }
+  }
+
+  smaDebug("vgId:%d, rsma pre commit succeess", SMA_VID(pSma));
 
   return TSDB_CODE_SUCCESS;
 }
@@ -103,48 +158,68 @@ static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma) {
   TdDirPtr      pDir = NULL;
   TdDirEntryPtr pDirEntry = NULL;
   char          dir[TSDB_FILENAME_LEN];
-  char          bname[TSDB_FILENAME_LEN];
+  const char   *pattern = "v[0-9]+qtaskinfo\\.ver([0-9]+)?$";
-  const char   *pattern = "^v[0-9]+qtaskinfo\\.ver([0-9]+)?$";
   regex_t       regex;
+  int           code = 0;
 
-  tdGetVndDirName(TD_VID(pVnode), VNODE_RSMA_DIR, dir);
+  tdGetVndDirName(TD_VID(pVnode), tfsGetPrimaryPath(pVnode->pTfs), VNODE_RSMA_DIR, true, dir);
 
   // Resource allocation and init
-  regcomp(&regex, pattern, REG_EXTENDED);
+  if ((code = regcomp(&regex, pattern, REG_EXTENDED)) != 0) {
-
+    char errbuf[128];
-  if ((pDir = taosOpenDir(dir)) == NULL) {
+    regerror(code, &regex, errbuf, sizeof(errbuf));
-    terrno = TAOS_SYSTEM_ERROR(errno);
+    smaWarn("vgId:%d, rsma post commit, regcomp for %s failed since %s", TD_VID(pVnode), dir, errbuf);
-    smaWarn("rsma post-commit open dir %s failed since %s", dir, terrstr());
     return TSDB_CODE_FAILED;
   }
 
+  if ((pDir = taosOpenDir(dir)) == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    smaWarn("vgId:%d, rsma post commit, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr());
+    return TSDB_CODE_FAILED;
+  }
+
+  int32_t    dirLen = strlen(dir);
+  char      *dirEnd = POINTER_SHIFT(dir, dirLen);
   regmatch_t regMatch[2];
   while ((pDirEntry = taosReadDir(pDir)) != NULL) {
     char *entryName = taosGetDirEntryName(pDirEntry);
     if (!entryName) {
       continue;
     }
-    char *fileName = taosDirEntryBaseName(entryName);
+
-    int   code = regexec(&regex, bname, 2, regMatch, 0);
+    code = regexec(&regex, entryName, 2, regMatch, 0);
 
     if (code == 0) {
       // match
-      printf("match 0 = %s\n", (char *)POINTER_SHIFT(fileName, regMatch[0].rm_so));
+      int64_t version = -1;
-      printf("match 1 = %s\n", (char *)POINTER_SHIFT(fileName, regMatch[1].rm_so));
+      sscanf((const char *)POINTER_SHIFT(entryName, regMatch[1].rm_so), "%" PRIi64, &version);
+      if ((version < committed) && (version > -1)) {
+        strncpy(dirEnd, entryName, TSDB_FILENAME_LEN - dirLen);
+        if (taosRemoveFile(dir) != 0) {
+          terrno = TAOS_SYSTEM_ERROR(errno);
+          smaWarn("vgId:%d, committed version:%" PRIi64 ", failed to remove %s since %s", TD_VID(pVnode), committed,
+                  dir, terrstr());
+        } else {
+          smaDebug("vgId:%d, committed version:%" PRIi64 ", success to remove %s", TD_VID(pVnode), committed, dir);
+        }
+      }
     } else if (code == REG_NOMATCH) {
       // not match
-      smaInfo("rsma post-commit not match %s", fileName);
+      smaTrace("vgId:%d, rsma post commit, not match %s", TD_VID(pVnode), entryName);
       continue;
     } else {
       // has other error
-      terrno = TAOS_SYSTEM_ERROR(code);
+      char errbuf[128];
-      smaWarn("rsma post-commit regexec failed since %s", terrstr());
+      regerror(code, &regex, errbuf, sizeof(errbuf));
+      smaWarn("vgId:%d, rsma post commit, regexec failed since %s", TD_VID(pVnode), errbuf);
 
       taosCloseDir(&pDir);
       regfree(&regex);
       return TSDB_CODE_FAILED;
     }
   }
+
   taosCloseDir(&pDir);
+  regfree(&regex);
   return TSDB_CODE_SUCCESS;
 }

source/dnode/vnode/src/sma/smaEnv.c

@@ -132,6 +132,7 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS
     if (smaType == TSDB_SMA_TYPE_ROLLUP) {
       SRSmaStat *pRSmaStat = (SRSmaStat *)(*pSmaStat);
       pRSmaStat->pSma = (SSma *)pSma;
+      atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_INIT);
 
       // init smaMgmt
       smaMgmt.smaRef = taosOpenRef(SMA_MGMT_REF_NUM, tdDestroyRSmaStat);
@@ -192,22 +193,20 @@ static void *tdFreeTSmaStat(STSmaStat *pStat) {
 static void tdDestroyRSmaStat(void *pRSmaStat) {
   if (pRSmaStat) {
     SRSmaStat *pStat = (SRSmaStat *)pRSmaStat;
-    smaDebug("vgId:%d %s:%d destroy rsma stat %p", SMA_VID(pStat->pSma), __func__, __LINE__, pRSmaStat);
+    SSma      *pSma = pStat->pSma;
-    // step 1: set persistence task cancelled
+    smaDebug("vgId:%d, destroy rsma stat %p", SMA_VID(pSma), pRSmaStat);
+    // step 1: set rsma trigger stat cancelled
     atomic_store_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_CANCELLED);
 
-    // step 2: stop the persistence timer
+    // step 2: wait the persistence thread to finish
-    taosTmrStopA(&RSMA_TMR_ID(pStat));
-
-    // step 3: wait the persistence thread to finish
     int32_t nLoops = 0;
     if (atomic_load_8(RSMA_RUNNING_STAT(pStat)) == 1) {
       while (1) {
         if (atomic_load_8(RSMA_TRIGGER_STAT(pStat)) == TASK_TRIGGER_STAT_FINISHED) {
-          smaDebug("rsma, persist task finished already");
+          smaDebug("vgId:%d, rsma persist task finished already", SMA_VID(pSma));
           break;
         } else {
-          smaDebug("rsma, persist task not finished yet since rsma stat in %" PRIi8,
+          smaDebug("vgId:%d, rsma persist task not finished yet since rsma stat in %" PRIi8, SMA_VID(pSma),
                    atomic_load_8(RSMA_TRIGGER_STAT(pStat)));
         }
         ++nLoops;
@@ -218,13 +217,15 @@ static void tdDestroyRSmaStat(void *pRSmaStat) {
       }
     }
 
-    // step 4: destroy the rsma info and associated fetch tasks
+    // step 3: destroy the rsma info and associated fetch tasks
     // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready.
-    void *infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), NULL);
+    if (taosHashGetSize(RSMA_INFO_HASH(pStat)) > 0) {
-    while (infoHash) {
+      void *infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), NULL);
-      SRSmaInfo *pSmaInfo = *(SRSmaInfo **)infoHash;
+      while (infoHash) {
-      tdFreeRSmaInfo(pSmaInfo);
+        SRSmaInfo *pSmaInfo = *(SRSmaInfo **)infoHash;
-      infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), infoHash);
+        tdFreeRSmaInfo(pSmaInfo);
+        infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), infoHash);
+      }
     }
     taosHashCleanup(RSMA_INFO_HASH(pStat));
 
@@ -232,10 +233,10 @@ static void tdDestroyRSmaStat(void *pRSmaStat) {
     nLoops = 0;
     while (1) {
       if (T_REF_VAL_GET((SSmaStat *)pStat) == 0) {
-        smaDebug("rsma, all fetch task finished already");
+        smaDebug("vgId:%d, rsma fetch tasks all finished", SMA_VID(pSma));
         break;
       } else {
-        smaDebug("rsma, fetch tasks not all finished yet");
+        smaDebug("vgId:%d, rsma fetch tasks not all finished yet", SMA_VID(pSma));
       }
       ++nLoops;
       if (nLoops > 1000) {
@@ -275,7 +276,7 @@ int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) {
     } else if (smaType == TSDB_SMA_TYPE_ROLLUP) {
       SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSmaStat);
       if (taosRemoveRef(smaMgmt.smaRef, RSMA_REF_ID(pRSmaStat)) < 0) {
-        smaError("remove refId from smaRef failed, refId:0x%" PRIx64, RSMA_REF_ID(pRSmaStat));
+        smaError("remove refId from rsmaRef:0x%" PRIx64 " failed since %s", RSMA_REF_ID(pRSmaStat), terrstr());
       }
     } else {
       ASSERT(0);

source/dnode/vnode/src/sma/smaOpen.c

@@ -135,17 +135,11 @@ _err:
   return -1;
 }
 
-int32_t smaCloseEnv(SSma *pSma) {
+int32_t smaClose(SSma *pSma) {
-  if (pSma) {
-    SMA_TSMA_ENV(pSma) = tdFreeSmaEnv(SMA_TSMA_ENV(pSma));
-    SMA_RSMA_ENV(pSma) = tdFreeSmaEnv(SMA_RSMA_ENV(pSma));
-  }
-  return 0;
-}
-
-int32_t smaCloseEx(SSma *pSma) {
   if (pSma) {
     taosThreadMutexDestroy(&pSma->mutex);
+    SMA_TSMA_ENV(pSma) = tdFreeSmaEnv(SMA_TSMA_ENV(pSma));
+    SMA_RSMA_ENV(pSma) = tdFreeSmaEnv(SMA_RSMA_ENV(pSma));
     if SMA_RSMA_TSDB0 (pSma) tsdbClose(&SMA_RSMA_TSDB0(pSma));
     if SMA_RSMA_TSDB1 (pSma) tsdbClose(&SMA_RSMA_TSDB1(pSma));
     if SMA_RSMA_TSDB2 (pSma) tsdbClose(&SMA_RSMA_TSDB2(pSma));

source/dnode/vnode/src/sma/smaRollup.c

@@ -15,9 +15,8 @@
 
 #include "sma.h"
 
-#define RSMA_QTASKINFO_PERSIST_MS 7200000
+#define RSMA_QTASKINFO_BUFSIZE  32768
-#define RSMA_QTASKINFO_BUFSIZE    32768
+#define RSMA_QTASKINFO_HEAD_LEN (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t))  // len + type + suid
-#define RSMA_QTASKINFO_HEAD_LEN   (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t))  // len + type + suid
 
 SSmaMgmt smaMgmt = {
     .smaRef = -1,
@@ -43,9 +42,9 @@ static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isF
 static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter);
 static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *infoItem);
 
-static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma);
+static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables);
-static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma);
+static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int64_t *committed);
-static int32_t tdRSmaRestoreTSDataReload(SSma *pSma);
+static int32_t tdRSmaRestoreTSDataReload(SSma *pSma, int64_t committed);
 
 struct SRSmaInfoItem {
   SRSmaInfo *pRsmaInfo;
@@ -88,7 +87,7 @@ struct SRSmaQTaskInfoIter {
 };
 
 static void tdRSmaQTaskInfoGetFName(int32_t vgId, int64_t version, char *outputName) {
-  tdGetVndFileName(vgId, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, version, outputName);
+  tdGetVndFileName(vgId, NULL, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, version, outputName);
 }
 
 static FORCE_INLINE int32_t tdRSmaQTaskInfoContLen(int32_t lenWithHead) {
@@ -114,12 +113,14 @@ void *tdFreeRSmaInfo(SRSmaInfo *pInfo) {
     for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
       SRSmaInfoItem *pItem = &pInfo->items[i];
       if (pItem->taskInfo) {
-        smaDebug("vgId:%d, stb %" PRIi64 " stop fetch-timer %p level %d", SMA_VID(pSma), pInfo->suid, pItem->tmrId,
+        if (pItem->tmrId) {
-                 i + 1);
+          smaDebug("vgId:%d, table %" PRIi64 " stop fetch timer %p level %d", SMA_VID(pSma), pInfo->suid, pItem->tmrId,
-        taosTmrStopA(&pItem->tmrId);
+                   i + 1);
+          taosTmrStopA(&pItem->tmrId);
+        }
         tdFreeTaskHandle(&pItem->taskInfo, SMA_VID(pSma), i + 1);
       } else {
-        smaDebug("vgId:%d, stb %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", SMA_VID(pSma),
+        smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", SMA_VID(pSma),
                  pInfo->suid, i + 1);
       }
     }
@@ -358,13 +359,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
     goto _err;
   }
 
-  smaDebug("vgId:%d, register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), suid);
+  smaDebug("vgId:%d, register rsma info succeed for table %" PRIi64, SMA_VID(pSma), suid);
-
-  // start the persist timer
-  if (TASK_TRIGGER_STAT_INIT ==
-      atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_INIT, TASK_TRIGGER_STAT_ACTIVE)) {
-    taosTmrStart(tdRSmaPersistTrigger, RSMA_QTASKINFO_PERSIST_MS, pStat, RSMA_TMR_HANDLE(pStat));
-  }
 
   return TSDB_CODE_SUCCESS;
 _err:
@@ -748,7 +743,7 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma) {
+static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) {
   SVnode *pVnode = pSma->pVnode;
 
   SArray *suidList = taosArrayInit(1, sizeof(tb_uid_t));
@@ -758,7 +753,12 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma) {
     return TSDB_CODE_FAILED;
   }
 
-  int32_t arrSize = taosArrayGetSize(suidList);
+  int64_t arrSize = taosArrayGetSize(suidList);
+
+  if (nTables) {
+    *nTables = arrSize;
+  }
+
   if (arrSize == 0) {
     taosArrayDestroy(suidList);
     smaDebug("vgId:%d, no need to restore rsma env since empty stb id list", TD_VID(pVnode));
@@ -767,9 +767,9 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma) {
 
   SMetaReader mr = {0};
   metaReaderInit(&mr, SMA_META(pSma), 0);
-  for (int32_t i = 0; i < arrSize; ++i) {
+  for (int64_t i = 0; i < arrSize; ++i) {
     tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i);
-    smaDebug("vgId:%d, rsma restore, suid[%d] is %" PRIi64, TD_VID(pVnode), i, suid);
+    smaDebug("vgId:%d, rsma restore, suid is %" PRIi64, TD_VID(pVnode), suid);
     if (metaGetTableEntryByUid(&mr, suid) < 0) {
       smaError("vgId:%d, rsma restore, failed to get table meta for %" PRIi64 " since %s", TD_VID(pVnode), suid,
                terrstr());
@@ -803,7 +803,7 @@ _err:
   return TSDB_CODE_FAILED;
 }
 
-static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma) {
+static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int64_t *committed) {
   SVnode *pVnode = pSma->pVnode;
   STFile  tFile = {0};
   char    qTaskInfoFName[TSDB_FILENAME_LEN] = {0};
@@ -814,6 +814,13 @@ static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma) {
   }
 
   if (!taosCheckExistFile(TD_TFILE_FULL_NAME(&tFile))) {
+    if (pVnode->state.committed > 0) {
+      smaWarn("vgId:%d, rsma restore for version %" PRIi64 ", not start as %s not exist", TD_VID(pVnode),
+              pVnode->state.committed, TD_TFILE_FULL_NAME(&tFile));
+    } else {
+      smaDebug("vgId:%d, rsma restore for version %" PRIi64 ", no need as %s not exist", TD_VID(pVnode),
+               pVnode->state.committed, TD_TFILE_FULL_NAME(&tFile));
+    }
     return TSDB_CODE_SUCCESS;
   }
 
@@ -839,9 +846,14 @@ static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma) {
   tdRSmaQTaskInfoIterDestroy(&fIter);
   tdCloseTFile(&tFile);
   tdDestroyTFile(&tFile);
+
+  // restored successfully from committed
+  *committed = pVnode->state.committed;
+
   return TSDB_CODE_SUCCESS;
 _err:
-  smaError("rsma restore, qtaskinfo reload failed since %s", terrstr());
+  smaError("vgId:%d, rsma restore for version %" PRIi64 ", qtaskinfo reload failed since %s", TD_VID(pVnode),
+           pVnode->state.committed, terrstr());
   return TSDB_CODE_FAILED;
 }
 
@@ -849,35 +861,45 @@ _err:
  * @brief reload ts data from checkpoint
  *
  * @param pSma
+ * @param committed restore from committed version
  * @return int32_t
  */
-static int32_t tdRSmaRestoreTSDataReload(SSma *pSma) {
+static int32_t tdRSmaRestoreTSDataReload(SSma *pSma, int64_t committed) {
   // TODO
+  smaDebug("vgId:%d, rsma restore from %" PRIi64 ", ts data reload success", SMA_VID(pSma), committed);
   return TSDB_CODE_SUCCESS;
 _err:
-  smaError("rsma restore, ts data reload failed since %s", terrstr());
+  smaError("vgId:%d, rsma restore from %" PRIi64 ", ts data reload failed since %s", SMA_VID(pSma), committed,
+           terrstr());
   return TSDB_CODE_FAILED;
 }
 
 int32_t tdProcessRSmaRestoreImpl(SSma *pSma) {
   // step 1: iterate all stables to restore the rsma env
-  if (tdRSmaRestoreQTaskInfoInit(pSma) < 0) {
+  int64_t nTables = 0;
+  if (tdRSmaRestoreQTaskInfoInit(pSma, &nTables) < 0) {
     goto _err;
   }
 
+  if (nTables <= 0) {
+    smaDebug("vgId:%d, no need to restore rsma task since no tables", SMA_VID(pSma));
+    return TSDB_CODE_SUCCESS;
+  }
+
   // step 2: retrieve qtaskinfo items from the persistence file(rsma/qtaskinfo) and restore
-  if (tdRSmaRestoreQTaskInfoReload(pSma) < 0) {
+  int64_t committed = -1;
+  if (tdRSmaRestoreQTaskInfoReload(pSma, &committed) < 0) {
     goto _err;
   }
 
   // step 3: reload ts data from checkpoint
-  if (tdRSmaRestoreTSDataReload(pSma) < 0) {
+  if (tdRSmaRestoreTSDataReload(pSma, committed) < 0) {
     goto _err;
   }
 
   return TSDB_CODE_SUCCESS;
 _err:
-  smaError("failed to restore rsma task since %s", terrstr());
+  smaError("vgId:%d failed to restore rsma task since %s", SMA_VID(pSma), terrstr());
   return TSDB_CODE_FAILED;
 }
 
@@ -1017,7 +1039,8 @@ static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter) {
     pIter->qBuf = taosDecodeFixedI32(pIter->qBuf, &qTaskInfoLenWithHead);
     if (qTaskInfoLenWithHead < RSMA_QTASKINFO_HEAD_LEN) {
       terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-      smaError("restore rsma qtaskinfo file %s failed since %s", TD_TFILE_FULL_NAME(pIter->pTFile), terrstr());
+      smaError("vgId:%d, restore rsma qtaskinfo file %s failed since %s", SMA_VID(pSma),
+               TD_TFILE_FULL_NAME(pIter->pTFile), terrstr());
       return TSDB_CODE_FAILED;
     }
 
@@ -1054,13 +1077,17 @@ static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat) {
+int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat) {
   SSma   *pSma = pRSmaStat->pSma;
   SVnode *pVnode = pSma->pVnode;
   int32_t vid = SMA_VID(pSma);
   int64_t toffset = 0;
   bool    isFileCreated = false;
 
+  if (taosHashGetSize(RSMA_INFO_HASH(pRSmaStat)) <= 0) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   void *infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL);
   if (!infoHash) {
     return TSDB_CODE_SUCCESS;
@@ -1099,11 +1126,15 @@ static int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat) {
         char qTaskInfoFName[TSDB_FILENAME_LEN];
         tdRSmaQTaskInfoGetFName(vid, pSma->pVnode->state.applied, qTaskInfoFName);
         if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) {
+          smaError("vgId:%d, rsma persit, init %s failed since %s", vid, qTaskInfoFName, terrstr());
           goto _err;
         }
         if (tdCreateTFile(&tFile, true, -1) < 0) {
+          smaError("vgId:%d, rsma persit, create %s failed since %s", vid, TD_TFILE_FULL_NAME(&tFile), terrstr());
           goto _err;
         }
+        smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d serialize qTaskInfo, file %s created", vid, pRSmaInfo->suid,
+                 i + 1, TD_TFILE_FULL_NAME(&tFile));
 
         isFileCreated = true;
       }
@@ -1143,6 +1174,7 @@ static int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat) {
   }
   return TSDB_CODE_SUCCESS;
 _err:
+  smaError("vgId:%d, rsma persit failed since %s", vid, terrstr());
   if (isFileCreated) {
     tdRemoveTFile(&tFile);
     tdDestroyTFile(&tFile);
@@ -1238,8 +1270,8 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) {
         // start persist task
         tdRSmaPersistTask(pRSmaStat);
 
-        taosTmrReset(tdRSmaPersistTrigger, RSMA_QTASKINFO_PERSIST_MS, pRSmaStat, pRSmaStat->tmrHandle,
+        // taosTmrReset(tdRSmaPersistTrigger, 5000, pRSmaStat, pRSmaStat->tmrHandle,
-                     &pRSmaStat->tmrId);
+        //              RSMA_TMR_ID(pRSmaStat));
       } else {
         atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
       }

source/dnode/vnode/src/sma/smaUtil.c

@@ -140,7 +140,7 @@ int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset)
     return -1;
   }
 
-#if 1
+#if 0
   smaDebug("append to file %s, offset:%" PRIi64 " nbyte:%" PRIi64 " fsize:%" PRIi64, TD_TFILE_FULL_NAME(pTFile),
            toffset, nbyte, toffset + nbyte);
 #endif
@@ -181,16 +181,43 @@ void tdCloseTFile(STFile *pTFile) {
 
 void tdDestroyTFile(STFile *pTFile) { taosMemoryFreeClear(TD_TFILE_FULL_NAME(pTFile)); }
 
-void tdGetVndFileName(int32_t vgId, const char *dname, const char *fname, int64_t version, char *outputName) {
+void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version,
+                      char *outputName) {
   if (version < 0) {
-    snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/v%d%s", vgId, dname, vgId, fname);
+    if (pdname) {
+      snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%sv%d%s", pdname, TD_DIRSEP, TD_DIRSEP, vgId,
+               TD_DIRSEP, dname, TD_DIRSEP, vgId, fname);
+    } else {
+      snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%sv%d%s", TD_DIRSEP, vgId, TD_DIRSEP, dname, TD_DIRSEP,
+               vgId, fname);
+    }
   } else {
-    snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/v%d%s%" PRIi64, vgId, dname, vgId, fname, version);
+    if (pdname) {
+      snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%sv%d%s%" PRIi64, pdname, TD_DIRSEP, TD_DIRSEP,
+               vgId, TD_DIRSEP, dname, TD_DIRSEP, vgId, fname, version);
+    } else {
+      snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%sv%d%s%" PRIi64, TD_DIRSEP, vgId, TD_DIRSEP, dname,
+               TD_DIRSEP, vgId, fname, version);
+    }
   }
 }
 
-void tdGetVndDirName(int32_t vgId, const char *dname, char *outputName) {
+void tdGetVndDirName(int32_t vgId, const char *pdname, const char *dname, bool endWithSep, char *outputName) {
-  snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s", vgId, dname);
+  if (pdname) {
+    if (endWithSep) {
+      snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%s", pdname, TD_DIRSEP, TD_DIRSEP, vgId, TD_DIRSEP,
+               dname, TD_DIRSEP);
+    } else {
+      snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s", pdname, TD_DIRSEP, TD_DIRSEP, vgId, TD_DIRSEP,
+               dname);
+    }
+  } else {
+    if (endWithSep) {
+      snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%s", TD_DIRSEP, vgId, TD_DIRSEP, dname, TD_DIRSEP);
+    } else {
+      snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s", TD_DIRSEP, vgId, TD_DIRSEP, dname);
+    }
+  }
 }
 
 int32_t tdInitTFile(STFile *pTFile, const char *dname, const char *fname) {
@@ -215,35 +242,36 @@ int32_t tdInitTFile(STFile *pTFile, const char *dname, const char *fname) {
 
 int32_t tdCreateTFile(STFile *pTFile, bool updateHeader, int8_t fType) {
   ASSERT(pTFile->info.fsize == 0 && pTFile->info.magic == TD_FILE_INIT_MAGIC);
-
   pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pTFile->pFile == NULL) {
     if (errno == ENOENT) {
       // Try to create directory recursively
-      if (taosMulMkDir(taosDirName(TD_TFILE_FULL_NAME(pTFile))) != 0) {
+      char *s = strdup(TD_TFILE_FULL_NAME(pTFile));
+      if (taosMulMkDir(taosDirName(s)) != 0) {
+        terrno = TAOS_SYSTEM_ERROR(errno);
+        taosMemoryFree(s);
+        return -1;
+      }
+      taosMemoryFree(s);
+      pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+      if (pTFile->pFile == NULL) {
         terrno = TAOS_SYSTEM_ERROR(errno);
         return -1;
-      } else {
-        pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-        if (pTFile->pFile == NULL) {
-          terrno = TAOS_SYSTEM_ERROR(errno);
-          return -1;
-        }
       }
     }
+  }
 
-    if (!updateHeader) {
+  if (!updateHeader) {
-      return 0;
+    return 0;
-    }
+  }
 
-    pTFile->info.fsize += TD_FILE_HEAD_SIZE;
+  pTFile->info.fsize += TD_FILE_HEAD_SIZE;
-    pTFile->info.fver = 0;
+  pTFile->info.fver = 0;
 
-    if (tdUpdateTFileHeader(pTFile) < 0) {
+  if (tdUpdateTFileHeader(pTFile) < 0) {
-      tdCloseTFile(pTFile);
+    tdCloseTFile(pTFile);
-      tdRemoveTFile(pTFile);
+    tdRemoveTFile(pTFile);
-      return -1;
+    return -1;
-    }
   }
 
   return 0;

source/dnode/vnode/src/tq/tq.c

@@ -406,193 +406,6 @@ OVER:
   return code;
 }
 
-#if 0
-int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
-  SMqPollReq* pReq = pMsg->pCont;
-  int64_t     consumerId = pReq->consumerId;
-  int64_t     timeout = pReq->timeout;
-  int32_t     reqEpoch = pReq->epoch;
-  int64_t     fetchOffset;
-  int32_t     code = 0;
-
-  // get offset to fetch message
-  if (pReq->currentOffset >= 0) {
-    fetchOffset = pReq->currentOffset + 1;
-  } else {
-    STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pReq->subKey);
-    if (pOffset != NULL) {
-      ASSERT(pOffset->val.type == TMQ_OFFSET__LOG);
-      tqDebug("consumer %ld, restore offset of %s on vg %d, offset(type:log) version: %ld", consumerId, pReq->subKey,
-              TD_VID(pTq->pVnode), pOffset->val.version);
-      fetchOffset = pOffset->val.version + 1;
-    } else {
-      if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) {
-        fetchOffset = walGetFirstVer(pTq->pWal);
-      } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) {
-        fetchOffset = walGetCommittedVer(pTq->pWal);
-      } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__NONE) {
-        tqError("tmq poll: no offset committed for consumer %ld in vg %d, subkey %s", consumerId, TD_VID(pTq->pVnode),
-                pReq->subKey);
-        terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET;
-        return -1;
-      }
-      tqDebug("consumer %ld, restore offset of %s on vg %d failed, config is %ld, set to %ld", consumerId, pReq->subKey,
-              TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset);
-    }
-  }
-
-  tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req offset %ld fetch offset %ld", consumerId,
-          pReq->epoch, TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset);
-
-  STqHandle* pHandle = taosHashGet(pTq->handles, pReq->subKey, strlen(pReq->subKey));
-  /*ASSERT(pHandle);*/
-  if (pHandle == NULL) {
-    tqError("tmq poll: no consumer handle for consumer %ld in vg %d, subkey %s", consumerId, TD_VID(pTq->pVnode),
-            pReq->subKey);
-    return -1;
-  }
-
-  if (pHandle->consumerId != consumerId) {
-    tqError("tmq poll: consumer handle mismatch for consumer %ld in vg %d, subkey %s, handle consumer id %ld",
-            consumerId, TD_VID(pTq->pVnode), pReq->subKey, pHandle->consumerId);
-    return -1;
-  }
-
-  int32_t consumerEpoch = atomic_load_32(&pHandle->epoch);
-  while (consumerEpoch < reqEpoch) {
-    consumerEpoch = atomic_val_compare_exchange_32(&pHandle->epoch, consumerEpoch, reqEpoch);
-  }
-
-  SMqDataBlkRsp rsp = {0};
-  rsp.reqOffset = pReq->currentOffset;
-
-  rsp.blockData = taosArrayInit(0, sizeof(void*));
-  rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
-
-  if (rsp.blockData == NULL || rsp.blockDataLen == NULL) {
-    return -1;
-  }
-
-  rsp.withTbName = pReq->withTbName;
-  if (rsp.withTbName) {
-    rsp.blockTbName = taosArrayInit(0, sizeof(void*));
-  }
-
-  if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
-    rsp.withSchema = false;
-  } else {
-    rsp.withSchema = true;
-    rsp.blockSchema = taosArrayInit(0, sizeof(void*));
-  }
-
-#if 1
-  if (pReq->useSnapshot) {
-    // TODO set ver into snapshot
-    int64_t lastVer = walGetCommittedVer(pTq->pWal);
-    if (rsp.reqOffset < lastVer) {
-      tqInfo("retrieve using snapshot req offset %ld last ver %ld", rsp.reqOffset, lastVer);
-      tqScanSnapshot(pTq, &pHandle->execHandle, &rsp, workerId);
-
-      if (rsp.blockNum != 0) {
-        rsp.withTbName = false;
-        rsp.rspOffset = lastVer;
-        tqInfo("direct send by snapshot req offset %ld rsp offset %ld", rsp.reqOffset, rsp.rspOffset);
-        fetchOffset = lastVer;
-        goto SEND_RSP;
-      }
-    }
-  }
-#endif
-
-  SWalHead* pHeadWithCkSum = taosMemoryMalloc(sizeof(SWalHead) + 2048);
-  if (pHeadWithCkSum == NULL) {
-    return -1;
-  }
-
-  walSetReaderCapacity(pHandle->pWalReader, 2048);
-
-  while (1) {
-    consumerEpoch = atomic_load_32(&pHandle->epoch);
-    if (consumerEpoch > reqEpoch) {
-      tqWarn("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d, discard req epoch %d",
-             consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch);
-      break;
-    }
-
-    if (tqFetchLog(pTq, pHandle, &fetchOffset, &pHeadWithCkSum) < 0) {
-      // TODO add push mgr
-      break;
-    }
-
-    SWalCont* pHead = &pHeadWithCkSum->head;
-
-    tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
-            TD_VID(pTq->pVnode), fetchOffset, pHead->msgType);
-
-    if (pHead->msgType == TDMT_VND_SUBMIT) {
-      SSubmitReq* pCont = (SSubmitReq*)&pHead->body;
-
-      if (tqDataExec(pTq, &pHandle->execHandle, pCont, &rsp, workerId) < 0) {
-        /*ASSERT(0);*/
-      }
-    } else {
-      ASSERT(pHandle->fetchMeta);
-      ASSERT(IS_META_MSG(pHead->msgType));
-      tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType);
-      SMqMetaRsp metaRsp = {0};
-      metaRsp.reqOffset = pReq->currentOffset;
-      metaRsp.rspOffset = fetchOffset;
-      metaRsp.resMsgType = pHead->msgType;
-      metaRsp.metaRspLen = pHead->bodyLen;
-      metaRsp.metaRsp = pHead->body;
-      if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) {
-        code = -1;
-        goto OVER;
-      }
-      code = 0;
-      goto OVER;
-    }
-
-    // TODO batch optimization:
-    // TODO continue scan until meeting batch requirement
-    if (rsp.blockNum > 0 /* threshold */) {
-      break;
-    } else {
-      fetchOffset++;
-    }
-  }
-
-  taosMemoryFree(pHeadWithCkSum);
-
-SEND_RSP:
-  ASSERT(taosArrayGetSize(rsp.blockData) == rsp.blockNum);
-  ASSERT(taosArrayGetSize(rsp.blockDataLen) == rsp.blockNum);
-  if (rsp.withSchema) {
-    ASSERT(taosArrayGetSize(rsp.blockSchema) == rsp.blockNum);
-  }
-
-  rsp.rspOffset = fetchOffset;
-
-  if (tqSendDataRsp(pTq, pMsg, pReq, &rsp) < 0) {
-    code = -1;
-  }
-OVER:
-  // TODO wrap in destroy func
-  taosArrayDestroy(rsp.blockDataLen);
-  taosArrayDestroyP(rsp.blockData, (FDelete)taosMemoryFree);
-
-  if (rsp.withSchema) {
-    taosArrayDestroyP(rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper);
-  }
-
-  if (rsp.withTbName) {
-    taosArrayDestroyP(rsp.blockTbName, (FDelete)taosMemoryFree);
-  }
-
-  return code;
-}
-#endif
-
 int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen) {
   SMqVDeleteReq* pReq = (SMqVDeleteReq*)msg;
 

source/dnode/vnode/src/vnd/vnodeCommit.c

@@ -69,6 +69,9 @@ int vnodeBegin(SVnode *pVnode) {
     }
   }
 
+  // begin sma
+  smaBegin(pVnode->pSma);  // TODO: refactor to include the rsma1/rsma2 tsdbBegin() after tsdb_refact branch merged
+
   return 0;
 }
 
@@ -230,8 +233,8 @@ int vnodeCommit(SVnode *pVnode) {
   }
 
   // preCommit
-  // TODO
+  smaPreCommit(pVnode->pSma);
-  
+
   // commit each sub-system
   if (metaCommit(pVnode->pMeta) < 0) {
     ASSERT(0);

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -152,12 +152,11 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
   return pVnode;
 
 _err:
-  if (pVnode->pSma) smaCloseEnv(pVnode->pSma);
   if (pVnode->pQuery) vnodeQueryClose(pVnode);
   if (pVnode->pTq) tqClose(pVnode->pTq);
   if (pVnode->pWal) walClose(pVnode->pWal);
   if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb);
-  if (pVnode->pSma) smaCloseEx(pVnode->pSma);
+  if (pVnode->pSma) smaClose(pVnode->pSma);
   if (pVnode->pMeta) metaClose(pVnode->pMeta);
 
   tsem_destroy(&(pVnode->canCommit));
@@ -167,14 +166,13 @@ _err:
 
 void vnodeClose(SVnode *pVnode) {
   if (pVnode) {
-    smaCloseEnv(pVnode->pSma);
     vnodeCommit(pVnode);
     vnodeSyncClose(pVnode);
     vnodeQueryClose(pVnode);
     walClose(pVnode->pWal);
     tqClose(pVnode->pTq);
     if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb);
-    smaCloseEx(pVnode->pSma);
+    smaClose(pVnode->pSma);
     metaClose(pVnode->pMeta);
     vnodeCloseBufPool(pVnode);
     // destroy handle

source/libs/catalog/inc/catalogInt.h

@@ -278,7 +278,7 @@ typedef struct SCtgAsyncFps {
 
 typedef struct SCtgApiStat {
 
-#ifdef WINDOWS
+#if defined(WINDOWS) || defined(_TD_DARWIN_64)
   size_t avoidCompilationErrors;
 #endif
 

source/libs/catalog/test/CMakeLists.txt

@@ -1,23 +1,25 @@
 
 MESSAGE(STATUS "build catalog unit test")
 
-# GoogleTest requires at least C++11
+IF(NOT TD_DARWIN)
-SET(CMAKE_CXX_STANDARD 11)
+        # GoogleTest requires at least C++11
-AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+        SET(CMAKE_CXX_STANDARD 11)
+        AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
-ADD_EXECUTABLE(catalogTest ${SOURCE_LIST})
+        ADD_EXECUTABLE(catalogTest ${SOURCE_LIST})
-TARGET_LINK_LIBRARIES(
+        TARGET_LINK_LIBRARIES(
-        catalogTest
+                catalogTest
-        PUBLIC os util common catalog transport gtest qcom taos_static
+                PUBLIC os util common catalog transport gtest qcom taos_static
-)
+        )
 
-TARGET_INCLUDE_DIRECTORIES(
+        TARGET_INCLUDE_DIRECTORIES(
-        catalogTest
+                catalogTest
-        PUBLIC "${TD_SOURCE_DIR}/include/libs/catalog/"
+                PUBLIC "${TD_SOURCE_DIR}/include/libs/catalog/"
-        PRIVATE "${TD_SOURCE_DIR}/source/libs/catalog/inc"
+                PRIVATE "${TD_SOURCE_DIR}/source/libs/catalog/inc"
-)
+        )
 
-# add_test(
+        # add_test(
-#     NAME catalogTest
+        #     NAME catalogTest
-#     COMMAND catalogTest
+        #     COMMAND catalogTest
-# )
+        # )
+ENDIF()

source/libs/executor/src/dataSinkMgt.c

@@ -35,7 +35,7 @@ int32_t dsDataSinkGetCacheSize(SDataSinkStat *pStat) {
 
 
 int32_t dsCreateDataSinker(const SDataSinkNode *pDataSink, DataSinkHandle* pHandle, void* pParam) {
-  switch (nodeType(pDataSink)) {
+  switch ((int)nodeType(pDataSink)) {
     case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
       return createDataDispatcher(&gDataSinkManager, pDataSink, pHandle);
     case QUERY_NODE_PHYSICAL_PLAN_DELETE:

source/libs/executor/src/executorimpl.c

@@ -1422,7 +1422,7 @@ void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t
     return;
   }
 #ifdef BUF_PAGE_DEBUG
-  qDebug("page_setbuf, groupId:%"PRIu64, groupId);
+  qDebug("page_setbuf, groupId:%" PRIu64, groupId);
 #endif
   doSetTableGroupOutputBuf(pOperator, pAggInfo, numOfOutput, groupId);
 
@@ -1570,9 +1570,9 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI
 
     releaseBufPage(pBuf, page);
     pBlock->info.rows += pRow->numOfRows;
-//    if (pBlock->info.rows >= pBlock->info.capacity) {  // output buffer is full
+    //    if (pBlock->info.rows >= pBlock->info.capacity) {  // output buffer is full
-//      break;
+    //      break;
-//    }
+    //    }
   }
 
   qDebug("%s result generated, rows:%d, groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows,
@@ -2868,7 +2868,24 @@ int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts) {
       pInfo->cond.twindows[0].skey = oldSkey;
       pInfo->scanTimes = 0;
       pInfo->curTWinIdx = 0;
+
+      SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+
+      int32_t tableSz = taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList);
+      bool    found = false;
+      for (int32_t i = 0; i < tableSz; i++) {
+        STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, i);
+        if (pTableInfo->uid == uid) {
+          found = true;
+          pInfo->currentTable = i;
+        }
+      }
+      // TODO after processing drop,
+      ASSERT(found);
+      qDebug("tsdb reader offset seek to uid %ld ts %ld, table cur set to %d , all table num %d", uid, ts,
+             pInfo->currentTable, tableSz);
     }
+
     return TSDB_CODE_SUCCESS;
 
   } else {
@@ -3253,6 +3270,10 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
       doSetOperatorCompleted(pOperator);
       break;
     }
+    if (pBlock->info.type == STREAM_RETRIEVE) {
+      // for stream interval
+      return pBlock;
+    }
 
     // the pDataBlock are always the same one, no need to call this again
     int32_t code = getTableScanInfo(pOperator->pDownstream[0], &order, &scanFlag);
@@ -4097,14 +4118,14 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
       ASSERT(nodeType(pNew) == QUERY_NODE_VALUE);
       SValueNode* pValue = (SValueNode*)pNew;
 
-      if (pValue->node.resType.type == TSDB_DATA_TYPE_NULL) {
+      if (pValue->node.resType.type == TSDB_DATA_TYPE_NULL || pValue->isNull) {
         isNull[index++] = 1;
         continue;
       } else {
         isNull[index++] = 0;
         char* data = nodesGetValueFromNode(pValue);
-        if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON){
+        if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON) {
-          if(tTagIsJson(data)){
+          if (tTagIsJson(data)) {
             terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR;
             taosMemoryFree(keyBuf);
             nodesClearList(groupNew);
@@ -4169,7 +4190,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) {
       STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode;
       int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
-      if(code){
+      if (code) {
         pTaskInfo->code = code;
         return NULL;
       }
@@ -4198,7 +4219,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       };
       if (pHandle) {
         int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
-        if(code){
+        if (code) {
           pTaskInfo->code = code;
           return NULL;
         }

source/libs/executor/src/scanoperator.c

@@ -807,6 +807,23 @@ static bool isStateWindow(SStreamBlockScanInfo* pInfo) {
   return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
 }
 
+static void setGroupId(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, int32_t groupColIndex, int32_t rowIndex) {
+  ASSERT(rowIndex < pBlock->info.rows);
+  switch (pBlock->info.type)
+  {
+  case STREAM_RETRIEVE: {
+    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, groupColIndex);
+    uint64_t* groupCol = (uint64_t*)pColInfo->pData;
+    pInfo->groupId = groupCol[rowIndex];
+  }
+    break;
+  case STREAM_DELETE_DATA:
+    break;
+  default:
+    break;
+  }
+}
+
 static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
   STimeWindow win = {
       .skey = INT64_MIN,
@@ -829,6 +846,7 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int3
     } else {
       win =
           getActiveTimeWindow(NULL, &dumyInfo, tsCols[(*pRowIndex)], &pInfo->interval, pInfo->interval.precision, NULL);
+      setGroupId(pInfo, pSDB, 2, *pRowIndex);
       (*pRowIndex) += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, (*pRowIndex), win.ekey, binarySearchForKey, NULL,
                                                TSDB_ORDER_ASC);
     }
@@ -1311,6 +1329,13 @@ static void destroySysScanOperator(void* param, int32_t numOfOutput) {
   taosArrayDestroy(pInfo->scanCols);
 }
 
+static int32_t getSysTableDbNameColId(const char* pTable) {
+  // if (0 == strcmp(TSDB_INS_TABLE_USER_INDEXES, pTable)) {
+  //   return 1;
+  // }
+  return TSDB_INS_USER_STABLES_DBNAME_COLID;
+}
+
 EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) {
   int32_t   code = TSDB_CODE_SUCCESS;
   ENodeType nType = nodeType(pNode);
@@ -1332,7 +1357,7 @@ EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) {
       }
 
       SColumnNode* node = (SColumnNode*)pNode;
-      if (TSDB_INS_USER_STABLES_DBNAME_COLID == node->colId) {
+      if (getSysTableDbNameColId(node->tableName) == node->colId) {
         *(int32_t*)pContext = 2;
         return DEAL_RES_CONTINUE;
       }

source/libs/executor/src/timewindowoperator.c

@@ -2852,7 +2852,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
   pInfo->pStDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
   pInfo->pDelIterator = NULL;
   pInfo->pDelRes = createOneDataBlock(pResBlock, false);
-  pInfo->pDelRes->info.type = STREAM_DELETE;
+  pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;
   blockDataEnsureCapacity(pInfo->pDelRes, 64);
   pInfo->pChildren = NULL;
   pInfo->isFinal = false;
@@ -3980,7 +3980,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
   pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
   pInfo->pDelIterator = NULL;
   pInfo->pDelRes = createOneDataBlock(pResBlock, false);
-  pInfo->pDelRes->info.type = STREAM_DELETE;
+  pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;
   blockDataEnsureCapacity(pInfo->pDelRes, 64);
   pInfo->pChildren = NULL;
   pInfo->ignoreExpiredData = pStateNode->window.igExpired;

source/libs/function/src/builtins.c

@@ -1446,19 +1446,9 @@ static int32_t translateSubstr(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
 
 static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   // The number of parameters has been limited by the syntax definition
-  // uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
 
   // The function return type has been set during syntax parsing
   uint8_t para2Type = pFunc->node.resType.type;
-  // if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT &&
-  //     para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR &&
-  //     para2Type != TSDB_DATA_TYPE_TIMESTAMP) {
-  //   return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
-  // }
-  // if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) ||
-  //     (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) {
-  //   return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
-  // }
 
   int32_t para2Bytes = pFunc->node.resType.bytes;
   if (IS_VAR_DATA_TYPE(para2Type)) {
@@ -1468,6 +1458,11 @@ static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
     return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
                            "CAST function converted length should be in range [0, 1000]");
   }
+
+  // add database precision as param
+  uint8_t dbPrec = pFunc->node.resType.precision;
+  addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
+
   return TSDB_CODE_SUCCESS;
 }
 

source/libs/index/src/indexFstSparse.c

@@ -17,7 +17,7 @@
 
 FstSparseSet *sparSetCreate(int32_t sz) {
   FstSparseSet *ss = taosMemoryCalloc(1, sizeof(FstSparseSet));
-  if (ss = NULL) {
+  if (ss == NULL) {
     return NULL;
   }
 

source/libs/monitor/test/monTest.cpp

@@ -75,18 +75,18 @@ void MonitorTest::GetSysInfo(SMonSysInfo *pInfo) {
   pInfo->cpu_engine = 2.1;
   pInfo->cpu_system = 2.1;
   pInfo->cpu_cores = 2;
-  pInfo->mem_engine = 3.1;
+  pInfo->mem_engine = 3;
-  pInfo->mem_system = 3.2;
+  pInfo->mem_system = 3;
-  pInfo->mem_total = 3.3;
+  pInfo->mem_total = 3;
-  pInfo->disk_engine = 4.1;
+  pInfo->disk_engine = 4;
-  pInfo->disk_used = 4.2;
+  pInfo->disk_used = 4;
-  pInfo->disk_total = 4.3;
+  pInfo->disk_total = 4;
-  pInfo->net_in = 5.1;
+  pInfo->net_in = 5;
-  pInfo->net_out = 5.2;
+  pInfo->net_out = 5;
-  pInfo->io_read = 6.1;
+  pInfo->io_read = 6;
-  pInfo->io_write = 6.2;
+  pInfo->io_write = 6;
-  pInfo->io_read_disk = 7.1;
+  pInfo->io_read_disk = 7;
-  pInfo->io_write_disk = 7.2;
+  pInfo->io_write_disk = 7;
 }
 
 void MonitorTest::GetClusterInfo(SMonClusterInfo *pInfo) {

source/libs/parser/test/CMakeLists.txt

@@ -1,32 +1,34 @@
 
 MESSAGE(STATUS "build parser unit test")
 
-# GoogleTest requires at least C++11
+IF(NOT TD_DARWIN)
-SET(CMAKE_CXX_STANDARD 11)
+    # GoogleTest requires at least C++11
-AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+    SET(CMAKE_CXX_STANDARD 11)
+    AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
-ADD_EXECUTABLE(parserTest ${SOURCE_LIST})
+    ADD_EXECUTABLE(parserTest ${SOURCE_LIST})
 
-TARGET_INCLUDE_DIRECTORIES(
+    TARGET_INCLUDE_DIRECTORIES(
-        parserTest
+            parserTest
-        PUBLIC "${TD_SOURCE_DIR}/include/libs/parser/"
+            PUBLIC "${TD_SOURCE_DIR}/include/libs/parser/"
-        PRIVATE "${TD_SOURCE_DIR}/source/libs/parser/inc"
+            PRIVATE "${TD_SOURCE_DIR}/source/libs/parser/inc"
-)
-
-TARGET_LINK_LIBRARIES(
-        parserTest
-        PUBLIC os util common nodes parser catalog transport gtest function planner qcom
-)
-
-if(${BUILD_WINGETOPT})
-    target_include_directories(
-        parserTest
-        PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src"
     )
-    target_link_libraries(parserTest PUBLIC wingetopt)
-endif()
 
-add_test(
+    TARGET_LINK_LIBRARIES(
-    NAME parserTest
+            parserTest
-    COMMAND parserTest
+            PUBLIC os util common nodes parser catalog transport gtest function planner qcom
-)
+    )
+
+    if(${BUILD_WINGETOPT})
+        target_include_directories(
+            parserTest
+            PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src"
+        )
+        target_link_libraries(parserTest PUBLIC wingetopt)
+    endif()
+
+    add_test(
+        NAME parserTest
+        COMMAND parserTest
+    )
+ENDIF()

source/libs/qworker/test/CMakeLists.txt

@@ -1,18 +1,19 @@
 
 MESSAGE(STATUS "build qworker unit test")
+IF(NOT TD_DARWIN)
+        # GoogleTest requires at least C++11
+        SET(CMAKE_CXX_STANDARD 11)
+        AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
-# GoogleTest requires at least C++11
+        ADD_EXECUTABLE(qworkerTest ${SOURCE_LIST})
-SET(CMAKE_CXX_STANDARD 11)
+        TARGET_LINK_LIBRARIES(
-AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+                qworkerTest
+                PUBLIC os util common transport gtest qcom nodes planner qworker executor
+        )
 
-ADD_EXECUTABLE(qworkerTest ${SOURCE_LIST})
+        TARGET_INCLUDE_DIRECTORIES(
-TARGET_LINK_LIBRARIES(
+                qworkerTest
-        qworkerTest
+                PUBLIC "${TD_SOURCE_DIR}/include/libs/qworker/"
-        PUBLIC os util common transport gtest qcom nodes planner qworker executor
+                PRIVATE "${TD_SOURCE_DIR}/source/libs/qworker/inc"
-)
+        )
-
+ENDIF()
-TARGET_INCLUDE_DIRECTORIES(
-        qworkerTest
-        PUBLIC "${TD_SOURCE_DIR}/include/libs/qworker/"
-        PRIVATE "${TD_SOURCE_DIR}/source/libs/qworker/inc"
-)

source/libs/scalar/src/filter.c

@@ -196,7 +196,7 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
         terrno = TSDB_CODE_QRY_JSON_IN_ERROR;
         return 0;
       default:
-        assert(0);
+        return 0;
     }
   }
 
@@ -222,7 +222,7 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
         terrno = TSDB_CODE_QRY_JSON_IN_ERROR;
         return 0;
       default:
-        assert(0);
+        return 0;
     }
   }
 

source/libs/scalar/src/scalar.c

@@ -109,9 +109,8 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) {
       }
 
       if (IS_VAR_DATA_TYPE(type)) {
-        char* data = colDataGetVarData(out.columnData, 0);
+        buf = colDataGetVarData(out.columnData, 0);
-        len = varDataLen(data);
+        len = varDataTLen(data);
-        buf = varDataVal(data);
       } else {
         len = tDataTypes[type].bytes;
         buf = out.columnData->pData;
@@ -119,8 +118,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) {
     } else {
       buf = nodesGetValueFromNode(valueNode);
       if (IS_VAR_DATA_TYPE(type)) {
-        len = varDataLen(buf);
+        len = varDataTLen(buf);
-        buf = varDataVal(buf);
       } else {
         len = valueNode->node.resType.bytes;
       }
@@ -194,7 +192,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
 
       param->numOfRows = 1;
       param->columnData = sclCreateColumnInfoData(&valueNode->node.resType, 1);
-      if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type) {
+      if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type || valueNode->isNull) {
         colDataAppendNULL(param->columnData, 0);
       } else {
         colDataAppend(param->columnData, 0, nodesGetValueFromNode(valueNode), false);
@@ -345,7 +343,7 @@ int32_t sclGetNodeType(SNode *pNode, SScalarCtx *ctx) {
     return -1;
   }
   
-  switch (nodeType(pNode)) {
+  switch ((int)nodeType(pNode)) {
     case QUERY_NODE_VALUE: {
       SValueNode *valueNode = (SValueNode *)pNode;
       return valueNode->node.resType.type;
@@ -538,6 +536,14 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp
   int32_t rowNum = 0;
   int32_t code = 0;
 
+  // json not support in in operator
+  if(nodeType(node->pLeft) == QUERY_NODE_VALUE){
+    SValueNode *valueNode = (SValueNode *)node->pLeft;
+    if(valueNode->node.resType.type == TSDB_DATA_TYPE_JSON && (node->opType == OP_TYPE_IN || node->opType == OP_TYPE_NOT_IN)){
+      SCL_RET(TSDB_CODE_QRY_JSON_IN_ERROR);
+    }
+  }
+
   SCL_ERR_RET(sclInitOperatorParams(&params, node, ctx, &rowNum));
   output->columnData = sclCreateColumnInfoData(&node->node.resType, rowNum);
   if (output->columnData == NULL) {
@@ -777,7 +783,12 @@ EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) {
   res->translate = true;
 
   if (colDataIsNull_s(output.columnData, 0)) {
-    res->node.resType.type = TSDB_DATA_TYPE_NULL;
+    if(node->node.resType.type != TSDB_DATA_TYPE_JSON){
+      res->node.resType.type = TSDB_DATA_TYPE_NULL;
+    }else{
+      res->node.resType = node->node.resType;
+      res->isNull = true;
+    }
   } else {
     res->node.resType = node->node.resType;
     int32_t type = output.columnData->info.type;

source/libs/scalar/src/sclfunc.c

@@ -934,9 +934,16 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
         break;
       }
       case TSDB_DATA_TYPE_TIMESTAMP: {
+        int64_t timeVal;
         if (inputType == TSDB_DATA_TYPE_BINARY || inputType == TSDB_DATA_TYPE_NCHAR) {
-          //convert to 0
+          int64_t timePrec;
-          *(int64_t *)output = 0;
+          GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData);
+          int32_t ret = convertStringToTimestamp(inputType, input, timePrec, &timeVal);
+          if (ret != TSDB_CODE_SUCCESS) {
+            *(int64_t *)output = 0;
+          } else {
+            *(int64_t *)output = timeVal;
+          }
         } else {
           GET_TYPED_DATA(*(int64_t *)output, int64_t, inputType, input);
         }

source/libs/scalar/test/filter/CMakeLists.txt

@@ -1,18 +1,20 @@
 
 MESSAGE(STATUS "build filter unit test")
 
-# GoogleTest requires at least C++11
+IF(NOT TD_DARWIN)
-SET(CMAKE_CXX_STANDARD 11)
+        # GoogleTest requires at least C++11
-AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+        SET(CMAKE_CXX_STANDARD 11)
+        AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
-ADD_EXECUTABLE(filterTest ${SOURCE_LIST})
+        ADD_EXECUTABLE(filterTest ${SOURCE_LIST})
-TARGET_LINK_LIBRARIES(
+        TARGET_LINK_LIBRARIES(
-        filterTest
+                filterTest
-        PUBLIC os util common gtest qcom function nodes scalar
+                PUBLIC os util common gtest qcom function nodes scalar
-)
+        )
 
-TARGET_INCLUDE_DIRECTORIES(
+        TARGET_INCLUDE_DIRECTORIES(
-        filterTest
+                filterTest
-        PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar/"
+                PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar/"
-        PRIVATE "${TD_SOURCE_DIR}/source/libs/scalar/inc"
+                PRIVATE "${TD_SOURCE_DIR}/source/libs/scalar/inc"
-)
+        )
+ENDIF()

source/libs/scalar/test/scalar/CMakeLists.txt

@@ -1,23 +1,25 @@
 
 MESSAGE(STATUS "build scalar unit test")
 
-# GoogleTest requires at least C++11
+IF(NOT TD_DARWIN)
-SET(CMAKE_CXX_STANDARD 11)
+        # GoogleTest requires at least C++11
-AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+        SET(CMAKE_CXX_STANDARD 11)
+        AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
-ADD_EXECUTABLE(scalarTest ${SOURCE_LIST})
+        ADD_EXECUTABLE(scalarTest ${SOURCE_LIST})
-TARGET_LINK_LIBRARIES(
+        TARGET_LINK_LIBRARIES(
-        scalarTest
+                scalarTest
-        PUBLIC os util common gtest qcom function nodes scalar parser catalog transport
+                PUBLIC os util common gtest qcom function nodes scalar parser catalog transport
-)
+        )
 
-TARGET_INCLUDE_DIRECTORIES(
+        TARGET_INCLUDE_DIRECTORIES(
-        scalarTest
+                scalarTest
-        PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar/"
+                PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar/"
-        PUBLIC "${TD_SOURCE_DIR}/source/libs/parser/inc"
+                PUBLIC "${TD_SOURCE_DIR}/source/libs/parser/inc"
-        PRIVATE "${TD_SOURCE_DIR}/source/libs/scalar/inc"
+                PRIVATE "${TD_SOURCE_DIR}/source/libs/scalar/inc"
-)
+        )
-add_test(
+        add_test(
-        NAME scalarTest
+                NAME scalarTest
-        COMMAND scalarTest
+                COMMAND scalarTest
-)
+        )
+ENDIF()

source/libs/scheduler/inc/schedulerInt.h

@@ -68,7 +68,7 @@ typedef struct SSchHbTrans {
 
 typedef struct SSchApiStat {
 
-#ifdef WINDOWS
+#if defined(WINDOWS) || defined(_TD_DARWIN_64)
   size_t avoidCompilationErrors;
 #endif
 
@@ -76,7 +76,7 @@ typedef struct SSchApiStat {
 
 typedef struct SSchRuntimeStat {
 
-#ifdef WINDOWS
+#if defined(WINDOWS) || defined(_TD_DARWIN_64)
   size_t avoidCompilationErrors;
 #endif
 
@@ -84,7 +84,7 @@ typedef struct SSchRuntimeStat {
 
 typedef struct SSchJobStat {
 
-#ifdef WINDOWS
+#if defined(WINDOWS) || defined(_TD_DARWIN_64)
   size_t avoidCompilationErrors;
 #endif
 

source/libs/scheduler/test/CMakeLists.txt

@@ -1,18 +1,20 @@
 
 MESSAGE(STATUS "build scheduler unit test")
 
-# GoogleTest requires at least C++11
+IF(NOT TD_DARWIN)
-SET(CMAKE_CXX_STANDARD 11)
+        # GoogleTest requires at least C++11
-AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+        SET(CMAKE_CXX_STANDARD 11)
+        AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
-ADD_EXECUTABLE(schedulerTest ${SOURCE_LIST})
+        ADD_EXECUTABLE(schedulerTest ${SOURCE_LIST})
-TARGET_LINK_LIBRARIES(
+        TARGET_LINK_LIBRARIES(
-        schedulerTest
+                schedulerTest
-        PUBLIC os util common catalog transport gtest qcom taos_static planner scheduler
+                PUBLIC os util common catalog transport gtest qcom taos_static planner scheduler
-)
+        )
 
-TARGET_INCLUDE_DIRECTORIES(
+        TARGET_INCLUDE_DIRECTORIES(
-        schedulerTest
+                schedulerTest
-        PUBLIC "${TD_SOURCE_DIR}/include/libs/scheduler/"
+                PUBLIC "${TD_SOURCE_DIR}/include/libs/scheduler/"
-        PRIVATE "${TD_SOURCE_DIR}/source/libs/scheduler/inc"
+                PRIVATE "${TD_SOURCE_DIR}/source/libs/scheduler/inc"
-)
+        )
+ENDIF()

source/libs/tdb/src/db/tdbTxn.c

@@ -18,7 +18,7 @@
 int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg,
                int flags) {
   // not support read-committed version at the moment
-  ASSERT(flags == 0 || flags == TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+  ASSERT(flags == 0 || flags == (TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED));
 
   pTxn->flags = flags;
   pTxn->txnId = txnid;

source/libs/transport/src/transCli.c

@@ -778,7 +778,6 @@ SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrd* pThrd, bool* ignore) {
       *ignore = true;
       destroyCmsg(pMsg);
       return NULL;
-      // assert(0);
     } else {
       conn = exh->handle;
       transReleaseExHandle(refId);
@@ -812,7 +811,6 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) {
 
   cliMayCvtFqdnToIp(&pCtx->epSet, &pThrd->cvtAddr);
 
-  // transPrintEpSet(&pCtx->epSet);
   bool      ignore = false;
   SCliConn* conn = cliGetConn(pMsg, pThrd, &ignore);
   if (ignore == true) {

source/os/CMakeLists.txt

@@ -39,12 +39,16 @@ endif()
 target_link_libraries(
     os PUBLIC pthread
 )
-if(NOT TD_WINDOWS)
+if(TD_WINDOWS)
-    target_link_libraries(
-        os PUBLIC dl m rt
-    )
-else()
     target_link_libraries(
         os PUBLIC ws2_32 iconv msvcregex wcwidth winmm
     )
-endif(NOT TD_WINDOWS)
+elseif(TD_DARWIN_64)
+    target_link_libraries(
+        os PUBLIC dl m iconv
+    )
+else()
+    target_link_libraries(
+        os PUBLIC dl m rt
+    )
+endif()

source/os/src/osAtomic.c

@@ -518,7 +518,7 @@ int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val) {
 #endif
 }
 
-void* atomic_add_fetch_ptr(void *ptr, void *val) {
+void* atomic_add_fetch_ptr(void *ptr, int64_t val) {
 #ifdef WINDOWS
   return interlocked_add_fetch_ptr((void* volatile*)(ptr), (void*)(val));
 #elif defined(_TD_NINGSI_60)
@@ -618,11 +618,13 @@ int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val) {
 #endif
 }
 
-void* atomic_sub_fetch_ptr(void *ptr, void* val) {
+void* atomic_sub_fetch_ptr(void *ptr, int64_t val) {
 #ifdef WINDOWS
   return interlocked_sub_fetch_ptr(ptr, val);
 #elif defined(_TD_NINGSI_60)
   return __sync_sub_and_fetch((ptr), (val));
+#elif defined(_TD_DARWIN_64)
+  return __atomic_sub_fetch((void **)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
 #else
   return __atomic_sub_fetch((void **)(ptr), (val), __ATOMIC_SEQ_CST);
 #endif
@@ -673,6 +675,8 @@ void* atomic_fetch_sub_ptr(void *ptr, void* val) {
   return interlocked_fetch_sub_ptr(ptr, val);
 #elif defined(_TD_NINGSI_60)
   return __sync_fetch_and_sub((ptr), (val));
+#elif defined(_TD_DARWIN_64)
+  return __atomic_fetch_sub((void **)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
 #else
   return __atomic_fetch_sub((void **)(ptr), (val), __ATOMIC_SEQ_CST);
 #endif
@@ -723,6 +727,8 @@ void* atomic_and_fetch_ptr(void *ptr, void *val) {
   return interlocked_and_fetch_ptr((void* volatile*)(ptr), (void*)(val));
 #elif defined(_TD_NINGSI_60)
   return __sync_and_and_fetch((ptr), (val));
+#elif defined(_TD_DARWIN_64)
+  return (void*)__atomic_and_fetch((size_t *)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
 #else
   return __atomic_and_fetch((void **)(ptr), (val), __ATOMIC_SEQ_CST);
 #endif
@@ -773,6 +779,8 @@ void* atomic_fetch_and_ptr(void *ptr, void *val) {
   return interlocked_fetch_and_ptr((void* volatile*)(ptr), (void*)(val));
 #elif defined(_TD_NINGSI_60)
   return __sync_fetch_and_and((ptr), (val));
+#elif defined(_TD_DARWIN_64)
+  return (void*)__atomic_fetch_and((size_t *)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
 #else
   return __atomic_fetch_and((void **)(ptr), (val), __ATOMIC_SEQ_CST);
 #endif
@@ -823,6 +831,8 @@ void* atomic_or_fetch_ptr(void *ptr, void *val) {
   return interlocked_or_fetch_ptr((void* volatile*)(ptr), (void*)(val));
 #elif defined(_TD_NINGSI_60)
   return __sync_or_and_fetch((ptr), (val));
+#elif defined(_TD_DARWIN_64)
+  return (void*)__atomic_or_fetch((size_t *)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
 #else
   return __atomic_or_fetch((void **)(ptr), (val), __ATOMIC_SEQ_CST);
 #endif
@@ -873,6 +883,8 @@ void* atomic_fetch_or_ptr(void *ptr, void *val) {
   return interlocked_fetch_or_ptr((void* volatile*)(ptr), (void*)(val));
 #elif defined(_TD_NINGSI_60)
   return __sync_fetch_and_or((ptr), (val));
+#elif defined(_TD_DARWIN_64)
+  return (void*)__atomic_fetch_or((size_t *)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
 #else
   return __atomic_fetch_or((void **)(ptr), (val), __ATOMIC_SEQ_CST);
 #endif
@@ -923,6 +935,8 @@ void* atomic_xor_fetch_ptr(void *ptr, void *val) {
   return interlocked_xor_fetch_ptr((void* volatile*)(ptr), (void*)(val));
 #elif defined(_TD_NINGSI_60)
   return __sync_xor_and_fetch((ptr), (val));
+#elif defined(_TD_DARWIN_64)
+  return (void*)__atomic_xor_fetch((size_t *)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
 #else
   return __atomic_xor_fetch((void **)(ptr), (val), __ATOMIC_SEQ_CST);
 #endif
@@ -973,6 +987,8 @@ void* atomic_fetch_xor_ptr(void *ptr, void *val) {
   return interlocked_fetch_xor_ptr((void* volatile*)(ptr), (void*)(val));
 #elif defined(_TD_NINGSI_60)
   return __sync_fetch_and_xor((ptr), (val));
+#elif defined(_TD_DARWIN_64)
+  return (void*)__atomic_fetch_xor((size_t *)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
 #else
   return __atomic_fetch_xor((void **)(ptr), (val), __ATOMIC_SEQ_CST);
 #endif

source/os/src/osFile.c

@@ -35,6 +35,8 @@
 #include <unistd.h>
 #define LINUX_FILE_NO_TEXT_OPTION 0
 #define O_TEXT                    LINUX_FILE_NO_TEXT_OPTION
+
+#define _SEND_FILE_STEP_ 1000
 #endif
 
 #if defined(WINDOWS)
@@ -612,28 +614,34 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in
 
 #elif defined(_TD_DARWIN_64)
 
-  int r = 0;
+  lseek(pFileIn->fd, (int32_t)(*offset), 0);
-  if (offset) {
+  int64_t writeLen = 0;
-    r = fseek(in_file, *offset, SEEK_SET);
+  uint8_t buffer[_SEND_FILE_STEP_] = {0};
-    if (r == -1) return -1;
+
-  }
+  for (int64_t len = 0; len < (size - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
-  off_t len = size;
+    size_t rlen = read(pFileIn->fd, (void *)buffer, _SEND_FILE_STEP_);
-  while (len > 0) {
+    if (rlen <= 0) {
-    char  buf[1024 * 16];
+      return writeLen;
-    off_t n = sizeof(buf);
+    } else if (rlen < _SEND_FILE_STEP_) {
-    if (len < n) n = len;
+      write(pFileOut->fd, (void *)buffer, (uint32_t)rlen);
-    size_t m = fread(buf, 1, n, in_file);
+      return (int64_t)(writeLen + rlen);
-    if (m < n) {
+    } else {
-      int e = ferror(in_file);
+      write(pFileOut->fd, (void *)buffer, _SEND_FILE_STEP_);
-      if (e) return -1;
+      writeLen += _SEND_FILE_STEP_;
     }
-    if (m == 0) break;
-    if (m != fwrite(buf, 1, m, out_file)) {
-      return -1;
-    }
-    len -= m;
   }
-  return size - len;
+
+  int64_t remain = size - writeLen;
+  if (remain > 0) {
+    size_t rlen = read(pFileIn->fd, (void *)buffer, (size_t)remain);
+    if (rlen <= 0) {
+      return writeLen;
+    } else {
+      write(pFileOut->fd, (void *)buffer, (uint32_t)remain);
+      writeLen += remain;
+    }
+  }
+  return writeLen;
 
 #else
 

source/os/src/osMemory.c

@@ -14,7 +14,11 @@
  */
 
 #define ALLOW_FORBID_FUNC
+#ifdef _TD_DARWIN_64
+#include <malloc/malloc.h>
+#else
 #include <malloc.h>
+#endif
 #include "os.h"
 
 #if defined(USE_TD_MEMORY) || defined(USE_ADDR2LINE)
@@ -323,6 +327,8 @@ int32_t taosMemorySize(void *ptr) {
 #else
 #ifdef WINDOWS
   return _msize(ptr);
+#elif defined(_TD_DARWIN_64)
+  return malloc_size(ptr);
 #else
   return malloc_usable_size(ptr);
 #endif

source/os/src/osSemaphore.c

@@ -13,8 +13,10 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#define ALLOW_FORBID_FUNC
 #define _DEFAULT_SOURCE
 #include "os.h"
+#include "pthread.h"
 
 #ifdef WINDOWS
 
@@ -111,289 +113,501 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) {
 
 // #define SEM_USE_PTHREAD
 // #define SEM_USE_POSIX
-#define SEM_USE_SEM
+// #define SEM_USE_SEM
 
-#ifdef SEM_USE_SEM
+// #ifdef SEM_USE_SEM
-#include <mach/mach_error.h>
+// #include <mach/mach_error.h>
-#include <mach/mach_init.h>
+// #include <mach/mach_init.h>
-#include <mach/semaphore.h>
+// #include <mach/semaphore.h>
-#include <mach/task.h>
+// #include <mach/task.h>
 
-static TdThread     sem_thread;
+// static TdThread     sem_thread;
-static TdThreadOnce sem_once;
+// static TdThreadOnce sem_once;
-static task_t       sem_port;
+// static task_t       sem_port;
-static volatile int sem_inited = 0;
+// static volatile int sem_inited = 0;
-static semaphore_t  sem_exit;
+// static semaphore_t  sem_exit;
 
-static void *sem_thread_routine(void *arg) {
+// static void *sem_thread_routine(void *arg) {
-  (void)arg;
+//   (void)arg;
-  setThreadName("sem_thrd");
+//   setThreadName("sem_thrd");
 
-  sem_port = mach_task_self();
+//   sem_port = mach_task_self();
-  kern_return_t ret = semaphore_create(sem_port, &sem_exit, SYNC_POLICY_FIFO, 0);
+//   kern_return_t ret = semaphore_create(sem_port, &sem_exit, SYNC_POLICY_FIFO, 0);
-  if (ret != KERN_SUCCESS) {
+//   if (ret != KERN_SUCCESS) {
-    fprintf(stderr, "==%s[%d]%s()==failed to create sem_exit\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__);
+//     fprintf(stderr, "==%s[%d]%s()==failed to create sem_exit\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__);
-    sem_inited = -1;
+//     sem_inited = -1;
-    return NULL;
+//     return NULL;
-  }
+//   }
-  sem_inited = 1;
+//   sem_inited = 1;
-  semaphore_wait(sem_exit);
+//   semaphore_wait(sem_exit);
-  return NULL;
+//   return NULL;
-}
+// }
 
-static void once_init(void) {
+// static void once_init(void) {
-  int r = 0;
+//   int r = 0;
-  r = taosThreadCreate(&sem_thread, NULL, sem_thread_routine, NULL);
+//   r = taosThreadCreate(&sem_thread, NULL, sem_thread_routine, NULL);
-  if (r) {
+//   if (r) {
-    fprintf(stderr, "==%s[%d]%s()==failed to create thread\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__);
+//     fprintf(stderr, "==%s[%d]%s()==failed to create thread\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__);
-    return;
+//     return;
-  }
+//   }
-  while (sem_inited == 0) {
+//   while (sem_inited == 0) {
-    ;
+//     ;
-  }
+//   }
-}
+// }
-#endif
+// #endif
 
-struct tsem_s {
+// struct tsem_s {
-#ifdef SEM_USE_PTHREAD
+// #ifdef SEM_USE_PTHREAD
-  TdThreadMutex    lock;
+//   TdThreadMutex    lock;
-  TdThreadCond     cond;
+//   TdThreadCond     cond;
-  volatile int64_t val;
+//   volatile int64_t val;
-#elif defined(SEM_USE_POSIX)
+// #elif defined(SEM_USE_POSIX)
-  size_t        id;
+//   size_t        id;
-  sem_t        *sem;
+//   sem_t        *sem;
-#elif defined(SEM_USE_SEM)
+// #elif defined(SEM_USE_SEM)
-  semaphore_t sem;
+//   semaphore_t sem;
-#else   // SEM_USE_PTHREAD
+// #else   // SEM_USE_PTHREAD
-  dispatch_semaphore_t sem;
+//   dispatch_semaphore_t sem;
-#endif  // SEM_USE_PTHREAD
+// #endif  // SEM_USE_PTHREAD
 
-  volatile unsigned int valid : 1;
+//   volatile unsigned int valid : 1;
-};
+// };
 
-int tsem_init(tsem_t *sem, int pshared, unsigned int value) {
+// int tsem_init(tsem_t *sem, int pshared, unsigned int value) {
-  // fprintf(stderr, "==%s[%d]%s():[%p]==creating\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
+//   // fprintf(stderr, "==%s[%d]%s():[%p]==creating\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
-  if (*sem) {
+//   if (*sem) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==already initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
+//     fprintf(stderr, "==%s[%d]%s():[%p]==already initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
-            sem);
+//             sem);
-    abort();
+//     abort();
-  }
+//   }
-  struct tsem_s *p = (struct tsem_s *)taosMemoryCalloc(1, sizeof(*p));
+//   struct tsem_s *p = (struct tsem_s *)taosMemoryCalloc(1, sizeof(*p));
-  if (!p) {
+//   if (!p) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==out of memory\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
+//     fprintf(stderr, "==%s[%d]%s():[%p]==out of memory\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
-    abort();
+//     abort();
-  }
+//   }
 
-#ifdef SEM_USE_PTHREAD
+// #ifdef SEM_USE_PTHREAD
-  int r = taosThreadMutexInit(&p->lock, NULL);
+//   int r = taosThreadMutexInit(&p->lock, NULL);
-  do {
+//   do {
-    if (r) break;
+//     if (r) break;
-    r = taosThreadCondInit(&p->cond, NULL);
+//     r = taosThreadCondInit(&p->cond, NULL);
-    if (r) {
+//     if (r) {
-      taosThreadMutexDestroy(&p->lock);
+//       taosThreadMutexDestroy(&p->lock);
-      break;
+//       break;
+//     }
+//     p->val = value;
+//   } while (0);
+//   if (r) {
+//     fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
+//     abort();
+//   }
+// #elif defined(SEM_USE_POSIX)
+//   static size_t tick = 0;
+//   do {
+//     size_t id = atomic_add_fetch_64(&tick, 1);
+//     if (id == SEM_VALUE_MAX) {
+//       atomic_store_64(&tick, 0);
+//       id = 0;
+//     }
+//     char name[NAME_MAX - 4];
+//     snprintf(name, sizeof(name), "/t%ld", id);
+//     p->sem = sem_open(name, O_CREAT | O_EXCL, pshared, value);
+//     p->id = id;
+//     if (p->sem != SEM_FAILED) break;
+//     int e = errno;
+//     if (e == EEXIST) continue;
+//     if (e == EINTR) continue;
+//     fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem,
+//             e, strerror(e));
+//     abort();
+//   } while (p->sem == SEM_FAILED);
+// #elif defined(SEM_USE_SEM)
+//   taosThreadOnce(&sem_once, once_init);
+//   if (sem_inited != 1) {
+//     fprintf(stderr, "==%s[%d]%s():[%p]==internal resource init failed\n", taosDirEntryBaseName(__FILE__), __LINE__,
+//             __func__, sem);
+//     errno = ENOMEM;
+//     return -1;
+//   }
+//   kern_return_t ret = semaphore_create(sem_port, &p->sem, SYNC_POLICY_FIFO, value);
+//   if (ret != KERN_SUCCESS) {
+//     fprintf(stderr, "==%s[%d]%s():[%p]==semophore_create failed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
+//             sem);
+//     // we fail-fast here, because we have less-doc about semaphore_create for the moment
+//     abort();
+//   }
+// #else   // SEM_USE_PTHREAD
+//   p->sem = dispatch_semaphore_create(value);
+//   if (p->sem == NULL) {
+//     fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
+//     abort();
+//   }
+// #endif  // SEM_USE_PTHREAD
+
+//   p->valid = 1;
+
+//   *sem = p;
+
+//   return 0;
+// }
+
+// int tsem_wait(tsem_t *sem) {
+//   if (!*sem) {
+//     fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
+//     abort();
+//   }
+//   struct tsem_s *p = *sem;
+//   if (!p->valid) {
+//     fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
+//     abort();
+//   }
+// #ifdef SEM_USE_PTHREAD
+//   if (taosThreadMutexLock(&p->lock)) {
+//     fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
+//             sem);
+//     abort();
+//   }
+//   p->val -= 1;
+//   if (p->val < 0) {
+//     if (taosThreadCondWait(&p->cond, &p->lock)) {
+//       fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
+//               sem);
+//       abort();
+//     }
+//   }
+//   if (taosThreadMutexUnlock(&p->lock)) {
+//     fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
+//             sem);
+//     abort();
+//   }
+//   return 0;
+// #elif defined(SEM_USE_POSIX)
+//   return sem_wait(p->sem);
+// #elif defined(SEM_USE_SEM)
+//   return semaphore_wait(p->sem);
+// #else   // SEM_USE_PTHREAD
+//   return dispatch_semaphore_wait(p->sem, DISPATCH_TIME_FOREVER);
+// #endif  // SEM_USE_PTHREAD
+// }
+
+// int tsem_post(tsem_t *sem) {
+//   if (!*sem) {
+//     fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
+//     abort();
+//   }
+//   struct tsem_s *p = *sem;
+//   if (!p->valid) {
+//     fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
+//     abort();
+//   }
+// #ifdef SEM_USE_PTHREAD
+//   if (taosThreadMutexLock(&p->lock)) {
+//     fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
+//             sem);
+//     abort();
+//   }
+//   p->val += 1;
+//   if (p->val <= 0) {
+//     if (taosThreadCondSignal(&p->cond)) {
+//       fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
+//               sem);
+//       abort();
+//     }
+//   }
+//   if (taosThreadMutexUnlock(&p->lock)) {
+//     fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
+//             sem);
+//     abort();
+//   }
+//   return 0;
+// #elif defined(SEM_USE_POSIX)
+//   return sem_post(p->sem);
+// #elif defined(SEM_USE_SEM)
+//   return semaphore_signal(p->sem);
+// #else   // SEM_USE_PTHREAD
+//   return dispatch_semaphore_signal(p->sem);
+// #endif  // SEM_USE_PTHREAD
+// }
+
+// int tsem_destroy(tsem_t *sem) {
+//   // fprintf(stderr, "==%s[%d]%s():[%p]==destroying\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
+//   if (!*sem) {
+//     // fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
+//     // abort();
+//     return 0;
+//   }
+//   struct tsem_s *p = *sem;
+//   if (!p->valid) {
+//     // fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
+//     // sem); abort();
+//     return 0;
+//   }
+// #ifdef SEM_USE_PTHREAD
+//   if (taosThreadMutexLock(&p->lock)) {
+//     fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
+//             sem);
+//     abort();
+//   }
+//   p->valid = 0;
+//   if (taosThreadCondDestroy(&p->cond)) {
+//     fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
+//             sem);
+//     abort();
+//   }
+//   if (taosThreadMutexUnlock(&p->lock)) {
+//     fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
+//             sem);
+//     abort();
+//   }
+//   if (taosThreadMutexDestroy(&p->lock)) {
+//     fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
+//             sem);
+//     abort();
+//   }
+// #elif defined(SEM_USE_POSIX)
+//   char name[NAME_MAX - 4];
+//   snprintf(name, sizeof(name), "/t%ld", p->id);
+//   int r = sem_unlink(name);
+//   if (r) {
+//     int e = errno;
+//     fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem,
+//             e, strerror(e));
+//     abort();
+//   }
+// #elif defined(SEM_USE_SEM)
+//   semaphore_destroy(sem_port, p->sem);
+// #else   // SEM_USE_PTHREAD
+// #endif  // SEM_USE_PTHREAD
+
+//   p->valid = 0;
+//   taosMemoryFree(p);
+
+//   *sem = NULL;
+//   return 0;
+// }
+typedef struct
+{
+    pthread_mutex_t count_lock;
+    pthread_cond_t  count_bump;
+    unsigned int count;
+}bosal_sem_t;
+
+int tsem_init(tsem_t *psem, int flags, unsigned int count)
+{
+    bosal_sem_t *pnewsem;
+    int result;
+
+    pnewsem = (bosal_sem_t *)malloc(sizeof(bosal_sem_t));
+    if (! pnewsem)
+    {
+        return -1;
     }
-    p->val = value;
+    result = pthread_mutex_init(&pnewsem->count_lock, NULL);
-  } while (0);
+    if (result)
-  if (r) {
+    {
-    fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
+        free(pnewsem);
-    abort();
+        return result;
-  }
-#elif defined(SEM_USE_POSIX)
-  static size_t tick = 0;
-  do {
-    size_t id = atomic_add_fetch_64(&tick, 1);
-    if (id == SEM_VALUE_MAX) {
-      atomic_store_64(&tick, 0);
-      id = 0;
     }
-    char name[NAME_MAX - 4];
+    result = pthread_cond_init(&pnewsem->count_bump, NULL);
-    snprintf(name, sizeof(name), "/t%ld", id);
+    if (result)
-    p->sem = sem_open(name, O_CREAT | O_EXCL, pshared, value);
+    {
-    p->id = id;
+        pthread_mutex_destroy(&pnewsem->count_lock);
-    if (p->sem != SEM_FAILED) break;
+        free(pnewsem);
-    int e = errno;
+        return result;
-    if (e == EEXIST) continue;
-    if (e == EINTR) continue;
-    fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem,
-            e, strerror(e));
-    abort();
-  } while (p->sem == SEM_FAILED);
-#elif defined(SEM_USE_SEM)
-  taosThreadOnce(&sem_once, once_init);
-  if (sem_inited != 1) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==internal resource init failed\n", taosDirEntryBaseName(__FILE__), __LINE__,
-            __func__, sem);
-    errno = ENOMEM;
-    return -1;
-  }
-  kern_return_t ret = semaphore_create(sem_port, &p->sem, SYNC_POLICY_FIFO, value);
-  if (ret != KERN_SUCCESS) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==semophore_create failed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
-            sem);
-    // we fail-fast here, because we have less-doc about semaphore_create for the moment
-    abort();
-  }
-#else   // SEM_USE_PTHREAD
-  p->sem = dispatch_semaphore_create(value);
-  if (p->sem == NULL) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
-    abort();
-  }
-#endif  // SEM_USE_PTHREAD
-
-  p->valid = 1;
-
-  *sem = p;
-
-  return 0;
-}
-
-int tsem_wait(tsem_t *sem) {
-  if (!*sem) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
-    abort();
-  }
-  struct tsem_s *p = *sem;
-  if (!p->valid) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
-    abort();
-  }
-#ifdef SEM_USE_PTHREAD
-  if (taosThreadMutexLock(&p->lock)) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
-            sem);
-    abort();
-  }
-  p->val -= 1;
-  if (p->val < 0) {
-    if (taosThreadCondWait(&p->cond, &p->lock)) {
-      fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
-              sem);
-      abort();
     }
-  }
+    pnewsem->count = count;
-  if (taosThreadMutexUnlock(&p->lock)) {
+    *psem = (tsem_t)pnewsem;
-    fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
-            sem);
-    abort();
-  }
-  return 0;
-#elif defined(SEM_USE_POSIX)
-  return sem_wait(p->sem);
-#elif defined(SEM_USE_SEM)
-  return semaphore_wait(p->sem);
-#else   // SEM_USE_PTHREAD
-  return dispatch_semaphore_wait(p->sem, DISPATCH_TIME_FOREVER);
-#endif  // SEM_USE_PTHREAD
-}
-
-int tsem_post(tsem_t *sem) {
-  if (!*sem) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
-    abort();
-  }
-  struct tsem_s *p = *sem;
-  if (!p->valid) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
-    abort();
-  }
-#ifdef SEM_USE_PTHREAD
-  if (taosThreadMutexLock(&p->lock)) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
-            sem);
-    abort();
-  }
-  p->val += 1;
-  if (p->val <= 0) {
-    if (taosThreadCondSignal(&p->cond)) {
-      fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
-              sem);
-      abort();
-    }
-  }
-  if (taosThreadMutexUnlock(&p->lock)) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
-            sem);
-    abort();
-  }
-  return 0;
-#elif defined(SEM_USE_POSIX)
-  return sem_post(p->sem);
-#elif defined(SEM_USE_SEM)
-  return semaphore_signal(p->sem);
-#else   // SEM_USE_PTHREAD
-  return dispatch_semaphore_signal(p->sem);
-#endif  // SEM_USE_PTHREAD
-}
-
-int tsem_destroy(tsem_t *sem) {
-  // fprintf(stderr, "==%s[%d]%s():[%p]==destroying\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
-  if (!*sem) {
-    // fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
-    // abort();
     return 0;
-  }
+}
-  struct tsem_s *p = *sem;
+
-  if (!p->valid) {
+int tsem_destroy(tsem_t *psem)
-    // fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
+{
-    // sem); abort();
+    bosal_sem_t *poldsem;
+
+    if (! psem)
+    {
+        return EINVAL;
+    }
+    poldsem = (bosal_sem_t *)*psem;
+
+    pthread_mutex_destroy(&poldsem->count_lock);
+    pthread_cond_destroy(&poldsem->count_bump);
+    free(poldsem);
     return 0;
-  }
-#ifdef SEM_USE_PTHREAD
-  if (taosThreadMutexLock(&p->lock)) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
-            sem);
-    abort();
-  }
-  p->valid = 0;
-  if (taosThreadCondDestroy(&p->cond)) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
-            sem);
-    abort();
-  }
-  if (taosThreadMutexUnlock(&p->lock)) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
-            sem);
-    abort();
-  }
-  if (taosThreadMutexDestroy(&p->lock)) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__,
-            sem);
-    abort();
-  }
-#elif defined(SEM_USE_POSIX)
-  char name[NAME_MAX - 4];
-  snprintf(name, sizeof(name), "/t%ld", p->id);
-  int r = sem_unlink(name);
-  if (r) {
-    int e = errno;
-    fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem,
-            e, strerror(e));
-    abort();
-  }
-#elif defined(SEM_USE_SEM)
-  semaphore_destroy(sem_port, p->sem);
-#else   // SEM_USE_PTHREAD
-#endif  // SEM_USE_PTHREAD
-
-  p->valid = 0;
-  taosMemoryFree(p);
-
-  *sem = NULL;
-  return 0;
 }
 
-bool taosCheckPthreadValid(TdThread thread) {
+int tsem_post(tsem_t *psem)
-  uint64_t id = 0;
+{
-  int      r = TdThreadhreadid_np(thread, &id);
+     bosal_sem_t *pxsem;
-  return r ? false : true;
+    int result, xresult;
+
+    if (! psem)
+    {
+        return EINVAL;
+    }
+    pxsem = (bosal_sem_t *)*psem;
+
+    result = pthread_mutex_lock(&pxsem->count_lock);
+    if (result)
+    {
+        return result;
+    }
+    pxsem->count = pxsem->count + 1;
+
+    xresult = pthread_cond_signal(&pxsem->count_bump);
+
+    result = pthread_mutex_unlock(&pxsem->count_lock);
+    if (result)
+    {
+        return result;
+    }
+    if (xresult)
+    {
+        errno = xresult;
+        return -1;
+    }
+    return 0;
 }
 
+int tsem_trywait(tsem_t *psem)
+{
+    bosal_sem_t *pxsem;
+    int result, xresult;
+
+    if (! psem)
+    {
+        return EINVAL;
+    }
+    pxsem = (bosal_sem_t *)*psem;
+
+    result = pthread_mutex_lock(&pxsem->count_lock);
+    if (result)
+    {
+        return result;
+    }
+    xresult = 0;
+
+    if (pxsem->count > 0)
+    {
+        pxsem->count--;
+    }
+    else
+    {
+        xresult = EAGAIN;
+    }
+    result = pthread_mutex_unlock(&pxsem->count_lock);
+    if (result)
+    {
+        return result;
+    }
+    if (xresult)
+    {
+        errno = xresult;
+        return -1;
+    }
+    return 0;
+}
+
+int tsem_wait(tsem_t *psem)
+{
+    bosal_sem_t *pxsem;
+    int result, xresult;
+
+    if (! psem)
+    {
+        return EINVAL;
+    }
+    pxsem = (bosal_sem_t *)*psem;
+
+    result = pthread_mutex_lock(&pxsem->count_lock);
+    if (result)
+    {
+        return result;
+    }
+    xresult = 0;
+
+    if (pxsem->count == 0)
+    {
+        xresult = pthread_cond_wait(&pxsem->count_bump, &pxsem->count_lock);
+    }
+    if (! xresult)
+    {
+        if (pxsem->count > 0)
+        {
+            pxsem->count--;
+        }
+    }
+    result = pthread_mutex_unlock(&pxsem->count_lock);
+    if (result)
+    {
+        return result;
+    }
+    if (xresult)
+    {
+        errno = xresult;
+        return -1;
+    }
+    return 0;
+}
+
+int tsem_timewait(tsem_t *psem, int64_t nanosecs)
+{
+  struct timespec abstim = {
+      .tv_sec = 0,
+      .tv_nsec = nanosecs,
+  };
+
+    bosal_sem_t *pxsem;
+    int result, xresult;
+
+    if (! psem)
+    {
+        return EINVAL;
+    }
+    pxsem = (bosal_sem_t *)*psem;
+
+    result = pthread_mutex_lock(&pxsem->count_lock);
+    if (result)
+    {
+        return result;
+    }
+    xresult = 0;
+
+    if (pxsem->count == 0)
+    {
+        xresult = pthread_cond_timedwait(&pxsem->count_bump, &pxsem->count_lock, &abstim);
+    }
+    if (! xresult)
+    {
+        if (pxsem->count > 0)
+        {
+            pxsem->count--;
+        }
+    }
+    result = pthread_mutex_unlock(&pxsem->count_lock);
+    if (result)
+    {
+        return result;
+    }
+    if (xresult)
+    {
+        errno = xresult;
+        return -1;
+    }
+    return 0;
+}
+
+bool taosCheckPthreadValid(TdThread thread) { 
+  int32_t ret = taosThreadKill(thread, 0);
+  if (ret == ESRCH) return false;
+  if (ret == EINVAL) return false;
+  // alive
+  return true;
+ }
+
 int64_t taosGetSelfPthreadId() {
-  uint64_t id;
+  TdThread thread = taosThreadSelf();
-  TdThreadhreadid_np(0, &id);
+  return (int64_t)thread;
-  return (int64_t)id;
 }
 
 int64_t taosGetPthreadId(TdThread thread) { return (int64_t)thread; }

source/os/src/osSignal.c

@@ -73,6 +73,10 @@ void taosIgnSignal(int32_t signum) { signal(signum, SIG_IGN); }
 
 void taosDflSignal(int32_t signum) { signal(signum, SIG_DFL); }
 
-void taosKillChildOnParentStopped() { prctl(PR_SET_PDEATHSIG, SIGKILL); }
+void taosKillChildOnParentStopped() { 
+#ifndef _TD_DARWIN_64
+  prctl(PR_SET_PDEATHSIG, SIGKILL); 
+#endif
+}
 
 #endif

source/os/src/osSocket.c

@@ -49,6 +49,14 @@
 #define INVALID_SOCKET -1
 #endif
 
+typedef struct TdSocket {
+#if SOCKET_WITH_LOCK
+  TdThreadRwlock rwlock;
+#endif
+  int      refId;
+  SocketFd fd;
+} * TdSocketPtr, TdSocket;
+
 typedef struct TdSocketServer {
 #if SOCKET_WITH_LOCK
   TdThreadRwlock rwlock;
@@ -1029,60 +1037,6 @@ int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, int *a
   return getsockname(pSocket->fd, destAddr, addrLen);
 }
 
-TdEpollPtr taosCreateEpoll(int32_t size) {
-  EpollFd fd = -1;
-#ifdef WINDOWS
-  assert(0);
-#else
-  fd = epoll_create(size);
-#endif
-  if (fd < 0) {
-    return NULL;
-  }
-
-  TdEpollPtr pEpoll = (TdEpollPtr)taosMemoryMalloc(sizeof(TdEpoll));
-  if (pEpoll == NULL) {
-    taosCloseSocketNoCheck1(fd);
-    return NULL;
-  }
-  pEpoll->fd = fd;
-  pEpoll->refId = 0;
-  return pEpoll;
-}
-int32_t taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocket, struct epoll_event *event) {
-  int32_t code = -1;
-  if (pEpoll == NULL || pEpoll->fd < 0) {
-    return -1;
-  }
-#ifdef WINDOWS
-  assert(0);
-#else
-  code = epoll_ctl(pEpoll->fd, epollOperate, pSocket->fd, event);
-#endif
-  return code;
-}
-int32_t taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxEvents, int32_t timeout) {
-  int32_t code = -1;
-  if (pEpoll == NULL || pEpoll->fd < 0) {
-    return -1;
-  }
-#ifdef WINDOWS
-  assert(0);
-#else
-  code = epoll_wait(pEpoll->fd, event, maxEvents, timeout);
-#endif
-  return code;
-}
-int32_t taosCloseEpoll(TdEpollPtr *ppEpoll) {
-  int32_t code;
-  if (ppEpoll == NULL || *ppEpoll == NULL || (*ppEpoll)->fd < 0) {
-    return -1;
-  }
-  code = taosCloseSocketNoCheck1((*ppEpoll)->fd);
-  (*ppEpoll)->fd = -1;
-  taosMemoryFree(*ppEpoll);
-  return code;
-}
 /*
  * Set TCP connection timeout per-socket level.
  * ref [https://github.com/libuv/help/issues/54]
@@ -1100,6 +1054,11 @@ int32_t taosCreateSocketWithTimeout(uint32_t timeout) {
   if (0 != setsockopt(fd, IPPROTO_TCP, TCP_MAXRT, (char *)&timeout, sizeof(timeout))) {
     return -1;
   }
+#elif defined(_TD_DARWIN_64)
+  uint32_t conn_timeout_ms = timeout * 1000;
+  if (0 != setsockopt(fd, IPPROTO_TCP, TCP_CONNECTIONTIMEOUT, (char *)&conn_timeout_ms, sizeof(conn_timeout_ms))) {
+    return -1;
+  }
 #else  // Linux like systems
   uint32_t conn_timeout_ms = timeout * 1000;
   if (0 != setsockopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, (char *)&conn_timeout_ms, sizeof(conn_timeout_ms))) {

source/os/src/osThread.c

@@ -157,9 +157,9 @@ int32_t taosThreadKill(TdThread thread, int32_t sig) {
   return pthread_kill(thread, sig);
 }
 
-int32_t taosThreadMutexConsistent(TdThreadMutex* mutex) {
+// int32_t taosThreadMutexConsistent(TdThreadMutex* mutex) {
-  return pthread_mutex_consistent(mutex);
+//   return pthread_mutex_consistent(mutex);
-}
+// }
 
 int32_t taosThreadMutexDestroy(TdThreadMutex * mutex) {
   return pthread_mutex_destroy(mutex);
@@ -173,9 +173,9 @@ int32_t taosThreadMutexLock(TdThreadMutex * mutex) {
   return pthread_mutex_lock(mutex);
 }
 
-int32_t taosThreadMutexTimedLock(TdThreadMutex * mutex, const struct timespec *abstime) {
+// int32_t taosThreadMutexTimedLock(TdThreadMutex * mutex, const struct timespec *abstime) {
-  return pthread_mutex_timedlock(mutex, abstime);
+//   return pthread_mutex_timedlock(mutex, abstime);
-}
+// }
 
 int32_t taosThreadMutexTryLock(TdThreadMutex * mutex) {
   return pthread_mutex_trylock(mutex);
@@ -193,9 +193,9 @@ int32_t taosThreadMutexAttrGetPshared(const TdThreadMutexAttr * attr, int32_t *p
   return pthread_mutexattr_getpshared(attr, pshared);
 }
 
-int32_t taosThreadMutexAttrGetRobust(const TdThreadMutexAttr * attr, int32_t * robust) {
+// int32_t taosThreadMutexAttrGetRobust(const TdThreadMutexAttr * attr, int32_t * robust) {
-  return pthread_mutexattr_getrobust(attr, robust);
+//   return pthread_mutexattr_getrobust(attr, robust);
-}
+// }
 
 int32_t taosThreadMutexAttrGetType(const TdThreadMutexAttr * attr, int32_t *kind) {
   return pthread_mutexattr_gettype(attr, kind);
@@ -209,9 +209,9 @@ int32_t taosThreadMutexAttrSetPshared(TdThreadMutexAttr * attr, int32_t pshared)
   return pthread_mutexattr_setpshared(attr, pshared);
 }
 
-int32_t taosThreadMutexAttrSetRobust(TdThreadMutexAttr * attr, int32_t robust) {
+// int32_t taosThreadMutexAttrSetRobust(TdThreadMutexAttr * attr, int32_t robust) {
-  return pthread_mutexattr_setrobust(attr, robust);
+//   return pthread_mutexattr_setrobust(attr, robust);
-}
+// }
 
 int32_t taosThreadMutexAttrSetType(TdThreadMutexAttr * attr, int32_t kind) {
   return pthread_mutexattr_settype(attr, kind);
@@ -233,13 +233,13 @@ int32_t taosThreadRwlockRdlock(TdThreadRwlock * rwlock) {
   return pthread_rwlock_rdlock(rwlock);
 }
 
-int32_t taosThreadRwlockTimedRdlock(TdThreadRwlock * rwlock, const struct timespec *abstime) {
+// int32_t taosThreadRwlockTimedRdlock(TdThreadRwlock * rwlock, const struct timespec *abstime) {
-  return pthread_rwlock_timedrdlock(rwlock, abstime);
+//   return pthread_rwlock_timedrdlock(rwlock, abstime);
-}
+// }
 
-int32_t taosThreadRwlockTimedWrlock(TdThreadRwlock * rwlock, const struct timespec *abstime) {
+// int32_t taosThreadRwlockTimedWrlock(TdThreadRwlock * rwlock, const struct timespec *abstime) {
-  return pthread_rwlock_timedwrlock(rwlock, abstime);
+//   return pthread_rwlock_timedwrlock(rwlock, abstime);
-}
+// }
 
 int32_t taosThreadRwlockTryRdlock(TdThreadRwlock * rwlock) {
   return pthread_rwlock_tryrdlock(rwlock);
@@ -303,7 +303,7 @@ int32_t taosThreadSpinDestroy(TdThreadSpinlock * lock) {
 
 int32_t taosThreadSpinInit(TdThreadSpinlock * lock, int32_t pshared) {
 #ifdef TD_USE_SPINLOCK_AS_MUTEX
-  assert(pshared == NULL);
+  assert(pshared == 0);
   return pthread_mutex_init((pthread_mutex_t*)lock, NULL);
 #else
   return pthread_spin_init((pthread_spinlock_t*)lock, pshared);

source/util/src/tcache.c

@@ -417,8 +417,8 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v
 
   if (pNode == NULL) {
     pushfrontNodeInEntryList(pe, pNode1);
-    atomic_add_fetch_64(&pCacheObj->numOfElems, 1);
+    atomic_add_fetch_ptr(&pCacheObj->numOfElems, 1);
-    atomic_add_fetch_64(&pCacheObj->sizeInBytes, pNode1->size);
+    atomic_add_fetch_ptr(&pCacheObj->sizeInBytes, pNode1->size);
     uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64
            ", totalNum:%d sizeInBytes:%" PRId64 "bytes size:%" PRId64 "bytes",
            pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime, (int32_t)pCacheObj->numOfElems,
@@ -667,7 +667,7 @@ void doTraverseElems(SCacheObj *pCacheObj, bool (*fp)(void *param, SCacheNode *p
         pEntry->next = next;
         pEntry->num -= 1;
 
-        atomic_sub_fetch_64(&pCacheObj->numOfElems, 1);
+        atomic_sub_fetch_ptr(&pCacheObj->numOfElems, 1);
         pNode = next;
       }
     }

source/util/src/tcompare.c

@@ -56,11 +56,11 @@ int32_t setChkNotInBytes8(const void *pLeft, const void *pRight) {
 }
 
 int32_t compareChkInString(const void *pLeft, const void *pRight) {
-  return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0;
+  return NULL != taosHashGet((SHashObj *)pRight, pLeft, varDataTLen(pLeft)) ? 1 : 0;
 }
 
 int32_t compareChkNotInString(const void *pLeft, const void *pRight) {
-  return NULL == taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0;
+  return NULL == taosHashGet((SHashObj *)pRight, pLeft, varDataTLen(pLeft)) ? 1 : 0;
 }
 
 int32_t compareInt8Val(const void *pLeft, const void *pRight) {

source/util/src/ttimer.c

@@ -132,7 +132,7 @@ static timer_map_t timerMap;
 static uintptr_t getNextTimerId() {
   uintptr_t id;
   do {
-    id = (uintptr_t)atomic_add_fetch_ptr((void **)&nextTimerId, (void*)1);
+    id = (uintptr_t)atomic_add_fetch_ptr((void **)&nextTimerId, 1);
   } while (id == 0);
   return id;
 }

tests/script/jenkins/basic.txt

@@ -9,8 +9,8 @@
 
 ## ---- db
 ./test.sh -f tsim/db/alter_option.sim
-./test.sh -f tsim/db/alter_replica_13.sim
+# ./test.sh -f tsim/db/alter_replica_13.sim
-./test.sh -f tsim/db/alter_replica_31.sim
+# ./test.sh -f tsim/db/alter_replica_31.sim
 ./test.sh -f tsim/db/basic1.sim
 ./test.sh -f tsim/db/basic2.sim
 ./test.sh -f tsim/db/basic3.sim
@@ -24,26 +24,26 @@
 ./test.sh -f tsim/db/taosdlog.sim
 
 # ---- dnode
-./test.sh -f tsim/dnode/balance_replica1.sim
+# ./test.sh -f tsim/dnode/balance_replica1.sim
-./test.sh -f tsim/dnode/balance_replica3.sim
+# ./test.sh -f tsim/dnode/balance_replica3.sim
-./test.sh -f tsim/dnode/balance1.sim
+# ./test.sh -f tsim/dnode/balance1.sim
-./test.sh -f tsim/dnode/balance2.sim
+# ./test.sh -f tsim/dnode/balance2.sim
-./test.sh -f tsim/dnode/balance3.sim
+# ./test.sh -f tsim/dnode/balance3.sim
-./test.sh -f tsim/dnode/balancex.sim
+# ./test.sh -f tsim/dnode/balancex.sim
 ./test.sh -f tsim/dnode/create_dnode.sim
 ./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim
-./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim
+# ./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim
-./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica1.sim
+# ./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica1.sim
-./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim
+# ./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim
-./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim
+# ./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim
-./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim
+# ./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim
 ./test.sh -f tsim/dnode/offline_reason.sim
-./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim
+# ./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim
-./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim
+# ./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim
-./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
+# ./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
-./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim
+# ./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim
-./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim
+# ./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim
-./test.sh -f tsim/dnode/vnode_clean.sim
+# ./test.sh -f tsim/dnode/vnode_clean.sim
 
 # ---- insert
 ./test.sh -f tsim/insert/basic0.sim
@@ -71,7 +71,7 @@
 ./test.sh -f tsim/qnode/basic1.sim
 
 # ---- snode
-./test.sh -f tsim/snode/basic1.sim
+# ./test.sh -f tsim/snode/basic1.sim
 
 # ---- bnode
 ./test.sh -f tsim/bnode/basic1.sim
@@ -104,8 +104,9 @@
 # ./test.sh -f tsim/stream/triggerSession0.sim
 ./test.sh -f tsim/stream/partitionby.sim
 ./test.sh -f tsim/stream/partitionby1.sim
-./test.sh -f tsim/stream/schedSnode.sim
+# ./test.sh -f tsim/stream/schedSnode.sim
 ./test.sh -f tsim/stream/windowClose.sim
+./test.sh -f tsim/stream/ignoreExpiredData.sim
 
 # ---- transaction
  ./test.sh -f tsim/trans/lossdata1.sim
@@ -168,13 +169,13 @@
 ./test.sh -f tsim/valgrind/checkError.sim -v
 
 # --- vnode
-#./test.sh -f tsim/vnode/replica3_basic.sim
+# ./test.sh -f tsim/vnode/replica3_basic.sim
-#./test.sh -f tsim/vnode/replica3_repeat.sim
+# ./test.sh -f tsim/vnode/replica3_repeat.sim
-./test.sh -f tsim/vnode/replica3_vgroup.sim
+# ./test.sh -f tsim/vnode/replica3_vgroup.sim
-#./test.sh -f tsim/vnode/replica3_many.sim
+# ./test.sh -f tsim/vnode/replica3_many.sim
-#./test.sh -f tsim/vnode/replica3_import.sim
+# ./test.sh -f tsim/vnode/replica3_import.sim
-./test.sh -f tsim/vnode/stable_balance_replica1.sim
+# ./test.sh -f tsim/vnode/stable_balance_replica1.sim
-./test.sh -f tsim/vnode/stable_dnode2_stop.sim
+# ./test.sh -f tsim/vnode/stable_dnode2_stop.sim
 ./test.sh -f tsim/vnode/stable_dnode2.sim
 ./test.sh -f tsim/vnode/stable_dnode3.sim
 ./test.sh -f tsim/vnode/stable_replica3_dnode6.sim

tests/script/tsim/sma/drop_sma.sim

@@ -50,6 +50,21 @@ sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) t
 print --> create sma
 sql create sma index sma_index_name1 on stb function(max(c1),max(c2),min(c1)) interval(6m,10s) sliding(6m);
 
+print --> show sma
+sql show indexes from stb from d1;
+if $rows != 1 then 
+  return -1
+endi
+if $data[0][0] != sma_index_name1 then 
+  return -1
+endi
+if $data[0][1] != d1 then 
+  return -1
+endi
+if $data[0][2] != stb then 
+  return -1
+endi
+
 print --> drop stb
 sql drop table stb; 
 
@@ -61,6 +76,21 @@ sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) t
 print --> create sma
 sql create sma index sma_index_name1 on stb function(max(c1),max(c2),min(c1)) interval(6m,10s) sliding(6m);
 
+print --> show sma
+sql show indexes from stb from d1;
+if $rows != 1 then 
+  return -1
+endi
+if $data[0][0] != sma_index_name1 then 
+  return -1
+endi
+if $data[0][1] != d1 then 
+  return -1
+endi
+if $data[0][2] != stb then 
+  return -1
+endi
+
 print --> drop stb
 sql drop table stb; 
 

tests/system-test/1-insert/alter_stable.py

@@ -16,135 +16,150 @@ import string
 from util.log import *
 from util.cases import *
 from util.sql import *
-
+from util.sqlset import *
+from util import constant
+from util.common import *
 class TDTestCase:
     def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor())
+        self.setsql = TDSetSql()
+        self.ntbname = 'ntb'
+        self.stbname = 'stb'
+        self.binary_length = 20 # the length of binary for column_dict
+        self.nchar_length = 20  # the length of nchar for column_dict
+        self.column_dict = {
+            'ts'  : 'timestamp',
+            'col1': 'tinyint',
+            'col2': 'smallint',
+            'col3': 'int',
+            'col4': 'bigint',
+            'col5': 'tinyint unsigned',
+            'col6': 'smallint unsigned',
+            'col7': 'int unsigned',
+            'col8': 'bigint unsigned',
+            'col9': 'float',
+            'col10': 'double',
+            'col11': 'bool',
+            'col12': f'binary({self.binary_length})',
+            'col13': f'nchar({self.nchar_length})'
+        }
+        self.tag_dict = {
+            'ts_tag'  : 'timestamp',
+            't1': 'tinyint',
+            't2': 'smallint',
+            't3': 'int',
+            't4': 'bigint',
+            't5': 'tinyint unsigned',
+            't6': 'smallint unsigned',
+            't7': 'int unsigned',
+            't8': 'bigint unsigned',
+            't9': 'float',
+            't10': 'double',
+            't11': 'bool',
+            't12': f'binary({self.binary_length})',
+            't13': f'nchar({self.nchar_length})'
+        }
+        self.tag_list = [
+            f'now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据"'
+        ]
+        self.tbnum = 1
+        self.values_list = [
+            f'now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据"'
+        ]
+        self.column_add_dict = {
+            'col_time'      : 'timestamp',
+            'col_tinyint'   : 'tinyint',
+            'col_smallint'  : 'smallint',
+            'col_int'       : 'int',
+            'col_bigint'    : 'bigint',
+            'col_untinyint' : 'tinyint unsigned',
+            'col_smallint'  : 'smallint unsigned',
+            'col_int'       : 'int unsigned',
+            'col_bigint'    : 'bigint unsigned',
+            'col_bool'      : 'bool',
+            'col_float'     : 'float',
+            'col_double'    : 'double',
+            'col_binary'    : f'binary({constant.BINARY_LENGTH_MAX})',
+            'col_nchar'     : f'nchar({constant.NCAHR_LENGTH_MAX})'
 
-    def get_long_name(self, length, mode="mixed"):
+        }
-        """
-        generate long name
-        mode could be numbers/letters/letters_mixed/mixed
-        """
-        if mode == "numbers":
-            population = string.digits
-        elif mode == "letters":
-            population = string.ascii_letters.lower()
-        elif mode == "letters_mixed":
-            population = string.ascii_letters.upper() + string.ascii_letters.lower()
-        else:
-            population = string.ascii_letters.lower() + string.digits
-        return "".join(random.choices(population, k=length))
-    def alter_stable_column_check(self,dbname,stbname,tbname):
-        tdSql.execute(f'create database if not exists {dbname}')
-        tdSql.execute(f'use {dbname}')
-        tdSql.execute(
-            f'create stable {stbname} (ts timestamp, c1 tinyint, c2 smallint, c3 int, \
-                c4 bigint, c5 tinyint unsigned, c6 smallint unsigned, c7 int unsigned, c8 bigint unsigned, c9 float, c10 double, c11 bool,c12 binary(20),c13 nchar(20)) tags(t0 int) ')
-        tdSql.execute(f'create table {tbname} using {stbname} tags(1)')
-        tdSql.execute(f'insert into {tbname} values (now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据")')
-        tdSql.execute(f'alter stable {stbname} add column c14 int')
-        tdSql.query(f'select c14 from {stbname}')
-        tdSql.checkRows(1)
-        tdSql.execute(f'alter stable {stbname} add column `c15` int')
-        tdSql.query(f'select c15 from {stbname}')
-        tdSql.checkRows(1)
-        tdSql.query(f'describe {stbname}')
-        tdSql.checkRows(17)
-        tdSql.execute(f'alter stable {stbname} drop column c14')
-        tdSql.query(f'describe {stbname}')
-        tdSql.checkRows(16)
-        tdSql.execute(f'alter stable {stbname} drop column `c15`')
-        tdSql.query(f'describe {stbname}')
-        tdSql.checkRows(15)
-        tdSql.execute(f'alter stable {stbname} modify column c12 binary(30)')
-        tdSql.query(f'describe {stbname}')
-        tdSql.checkData(12,2,30)
-        tdSql.execute(f'alter stable {stbname} modify column `c12` binary(35)')
-        tdSql.query(f'describe {stbname}')
-        tdSql.checkData(12,2,35)
-        tdSql.error(f'alter stable {stbname} modify column `c12` binary(34)')
-        tdSql.execute(f'alter stable {stbname} modify column c13 nchar(30)')
-        tdSql.query(f'describe {stbname}')
-        tdSql.checkData(13,2,30)
-        tdSql.error(f'alter stable {stbname} modify column c13 nchar(29)')
-        tdSql.error(f'alter stable {stbname} rename column c1 c21')
-        tdSql.error(f'alter stable {stbname} modify column c1 int')
-        tdSql.error(f'alter stable {stbname} modify column c4 int')
-        tdSql.error(f'alter stable {stbname} modify column c8 int')
-        tdSql.error(f'alter stable {stbname} modify column c1 unsigned int')
-        tdSql.error(f'alter stable {stbname} modify column c9 double')
-        tdSql.error(f'alter stable {stbname} modify column c10 float')
-        tdSql.error(f'alter stable {stbname} modify column c11 int')
-        tdSql.error(f'alter stable {stbname} drop tag t0')
-        tdSql.execute(f'drop database {dbname}')
-
-    def alter_stable_tag_check(self,dbname,stbname,tbname):
-        tdSql.execute(f'create database if not exists {dbname}')
-        tdSql.execute(f'use {dbname}')
-        tdSql.execute(
-            f'create stable {stbname} (ts timestamp, c1 int) tags(ts_tag timestamp, t1 tinyint, t2 smallint, t3 int, \
-                t4 bigint, t5 tinyint unsigned, t6 smallint unsigned, t7 int unsigned, t8 bigint unsigned, t9 float, t10 double, t11 bool,t12 binary(20),t13 nchar(20)) ')
-        tdSql.execute(f'create table {tbname} using {stbname} tags(now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据")')
-        tdSql.execute(f'insert into {tbname} values(now,1)')
-
-        tdSql.execute(f'alter stable {stbname} add tag t14 int')
-        tdSql.query(f'select t14 from {stbname}')
-        tdSql.checkRows(1)
-        tdSql.execute(f'alter stable {stbname} add tag `t15` int')
-        tdSql.query(f'select t14 from {stbname}')
-        tdSql.checkRows(1)
-        tdSql.query(f'describe {stbname}')
-        tdSql.checkRows(18)
-        tdSql.execute(f'alter stable {stbname} drop tag t14')
-        tdSql.query(f'describe {stbname}')
-        tdSql.checkRows(17)
-        tdSql.execute(f'alter stable {stbname} drop tag `t15`')
-        tdSql.query(f'describe {stbname}')
-        tdSql.checkRows(16)
-        tdSql.execute(f'alter stable {stbname} modify tag t12 binary(30)')
-        tdSql.query(f'describe {stbname}')
-        tdSql.checkData(14,2,30)
-        tdSql.execute(f'alter stable {stbname} modify tag `t12` binary(35)')
-        tdSql.query(f'describe {stbname}')
-        tdSql.checkData(14,2,35)
-        tdSql.error(f'alter stable {stbname} modify tag `t12` binary(34)')
-        tdSql.execute(f'alter stable {stbname} modify tag t13 nchar(30)')
-        tdSql.query(f'describe {stbname}')
-        tdSql.checkData(15,2,30)
-        tdSql.error(f'alter stable {stbname} modify tag t13 nchar(29)')
-        tdSql.execute(f'alter table {stbname} rename tag t1 t21')
-        tdSql.query(f'describe {stbname}')
-        tdSql.checkData(3,0,'t21')
-        tdSql.execute(f'alter table {stbname} rename tag `t21` t1')
-        tdSql.query(f'describe {stbname}')
-        tdSql.checkData(3,0,'t1')
-
-        for i in ['bigint','unsigned int','float','double','binary(10)','nchar(10)']:
-            for j in [1,2,3]:
-                tdSql.error(f'alter stable {stbname} modify tag t{j} {i}')
-        for i in ['int','unsigned int','float','binary(10)','nchar(10)']:
-            tdSql.error(f'alter stable {stbname} modify tag t8 {i}')
-        tdSql.error(f'alter stable {stbname} modify tag t4 int')
-        tdSql.error(f'alter stable {stbname} drop column t0')
-        #!bug TD-16410
-        # tdSql.error(f'alter stable {tbname} set tag t1=100 ')
-        # tdSql.execute(f'create table ntb (ts timestamp,c0 int)')
-        tdSql.error(f'alter stable ntb add column c2 ')
-        tdSql.execute(f'drop database {dbname}')
 
+    def alter_stable_check(self):
+        tdSql.prepare()
+        tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict))
+        tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict))
+        for i in self.values_list:
+            tdSql.execute(f'insert into {self.ntbname} values({i})')
+        for i in range(self.tbnum):
+            tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({self.tag_list[i]})')
+            for j in self.values_list:
+                tdSql.execute(f'insert into {self.stbname}_{i} values({j})')
+        for key,values in self.column_add_dict.items():
+            tdSql.execute(f'alter stable {self.stbname} add column {key} {values}')
+            tdSql.query(f'describe {self.stbname}')
+            tdSql.checkRows(len(self.column_dict)+len(self.tag_dict)+1)
+            for i in range(self.tbnum):
+                tdSql.query(f'describe {self.stbname}_{i}')
+                tdSql.checkRows(len(self.column_dict)+len(self.tag_dict)+1)
+                tdSql.query(f'select {key} from {self.stbname}_{i}')
+                tdSql.checkRows(len(self.values_list))
+            for i in range(self.tbnum):
+                tdSql.error(f'alter stable {self.stbname}_{i} add column {key} {values}')
+                tdSql.error(f'alter stable {self.stbname}_{i} drop column {key}')
+            #! bug TD-16921
+            #tdSql.error(f'alter stable {self.ntbname} add column {key} {values}')
+            #tdSql.error(f'alter stable {self.ntbname} drop column {key}')
+            tdSql.execute(f'alter stable {self.stbname} drop column {key}')
+            tdSql.query(f'describe {self.stbname}')
+            tdSql.checkRows(len(self.column_dict)+len(self.tag_dict))
+            for i in range(self.tbnum):
+                tdSql.query(f'describe {self.stbname}_{i}')
+                tdSql.checkRows(len(self.column_dict)+len(self.tag_dict))
+            tdSql.error(f'select {key} from {self.stbname} ')
+        for key,values in self.column_dict.items():
+            if 'binary' in values.lower():
+                v = f'binary({self.binary_length+1})'
+                v_error = f'binary({self.binary_length-1})'
+                tdSql.error(f'alter stable {self.stbname} modify column {key} {v_error}')
+                tdSql.execute(f'alter stable {self.stbname} modify column {key} {v}')
+                tdSql.query(f'describe {self.stbname}')
+                result = tdCom.getOneRow(1,'VARCHAR')
+                tdSql.checkEqual(result[0][2],self.binary_length+1)
+                for i in range(self.tbnum):
+                    tdSql.query(f'describe {self.stbname}_{i}')
+                    result = tdCom.getOneRow(1,'VARCHAR')
+                    tdSql.checkEqual(result[0][2],self.binary_length+1)
+                    tdSql.error(f'alter stable {self.stbname}_{i} modify column {key} {v}')
+               #! bug TD-16921
+               # tdSql.error(f'alter stable {self.ntbname} modify column {key} {v}')
+            elif 'nchar' in values.lower():
+                v = f'nchar({self.binary_length+1})'
+                v_error = f'nchar({self.binary_length-1})'
+                tdSql.error(f'alter stable {self.stbname} modify column {key} {v_error}')
+                tdSql.execute(f'alter stable {self.stbname} modify column {key} {v}')
+                tdSql.query(f'describe {self.stbname}')
+                result = tdCom.getOneRow(1,'NCHAR')
+                tdSql.checkEqual(result[0][2],self.binary_length+1)
+                for i in range(self.tbnum):
+                    tdSql.query(f'describe {self.stbname}_{i}')
+                    result = tdCom.getOneRow(1,'NCHAR')
+                    tdSql.checkEqual(result[0][2],self.binary_length+1)
+                    tdSql.error(f'alter stable {self.stbname}_{i} modify column {key} {v}')
+                #! bug TD-16921
+                #tdSql.error(f'alter stable {self.ntbname} modify column {key} {v}')
+            else:
+                for v in self.column_dict.values():
+                    tdSql.error(f'alter stable {self.stbname} modify column {key} {v}')
+                   # tdSql.error(f'alter stable {self.ntbname} modify column {key} {v}')
+                    for i in range(self.tbnum):
+                        tdSql.error(f'alter stable {self.stbname}_{i} modify column {key} {v}')
     def run(self):
 
-        dbname = self.get_long_name(length=10, mode="letters")
+        self.alter_stable_check()
-        stbname = self.get_long_name(length=5, mode="letters")
-        tbname = self.get_long_name(length=5, mode="letters")
-        self.alter_stable_column_check(dbname,stbname,tbname)
-        self.alter_stable_tag_check(dbname,stbname,tbname)
-
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)
 
 tdCases.addWindows(__file__, TDTestCase())
-tdCases.addLinux(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/system-test/1-insert/alter_table.py

@@ -126,6 +126,7 @@ class TDTestCase:
             tdSql.execute(f'alter table {self.ntbname} rename column {key} {rename_str}')
             tdSql.query(f'select {rename_str} from {self.ntbname}')
             tdSql.checkRows(1)
+            tdSql.error(f'select {key} from {self.ntbname}')
         
     def alter_check_tb(self):
         tag_tinyint = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX)
@@ -277,7 +278,11 @@ class TDTestCase:
             else:
                 for v in self.column_dict.values():
                     tdSql.error(f'alter table {self.stbname} modify column {key} {v}')
-
+        for key,values in self.column_dict.items():
+            rename_str = f'{tdCom.getLongName(constant.COL_NAME_LENGTH_MAX,"letters")}'
+            tdSql.error(f'alter table {self.stbname} rename column {key} {rename_str}')
+            for i in range(self.tbnum):
+                tdSql.error(f'alter table {self.stbname}_{i} rename column {key} {rename_str}')
     def run(self):
         self.alter_check_ntb()
         self.alter_check_tb()

tests/system-test/1-insert/test_stmt_set_tbname_tag.py

@@ -78,21 +78,29 @@ class TDTestCase:
         password = "taosdata"
         port =6030 
         con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port)
-        print(con)
+        tdLog.debug(con)
         return con
-        
+
+    def stmtExe(self,conn,sql,bindStat):
+            queryStat=conn.statement("%s"%sql)
+            queryStat.bind_param(bindStat)
+            queryStat.execute() 
+            result=queryStat.use_result()
+            rows=result.fetch_all()
+            return rows
+
     def test_stmt_set_tbname_tag(self,conn):
-        dbname = "stmt_set_tbname_tag"
+        dbname = "stmt_tag"
-        
+        stablename = 'log'
         try:
             conn.execute("drop database if exists %s" % dbname)
             conn.execute("create database if not exists %s PRECISION 'us' "  % dbname)
             conn.select_db(dbname)
-            conn.execute("create table if not exists log(ts timestamp, bo bool, nil tinyint, ti tinyint, si smallint, ii int,\
+            conn.execute("create table if not exists %s(ts timestamp, bo bool, nil tinyint, ti tinyint, si smallint, ii int,\
                 bi bigint, tu tinyint unsigned, su smallint unsigned, iu int unsigned, bu bigint unsigned, \
                 ff float, dd double, bb binary(100), nn nchar(100), tt timestamp , vc varchar(100)) tags (t1 timestamp, t2 bool,\
                 t3 tinyint, t4 tinyint, t5 smallint, t6 int, t7 bigint, t8 tinyint unsigned, t9 smallint unsigned, \
-                t10 int unsigned, t11 bigint unsigned, t12 float, t13 double, t14 binary(100), t15 nchar(100), t16 timestamp)")
+                t10 int unsigned, t11 bigint unsigned, t12 float, t13 double, t14 binary(100), t15 nchar(100), t16 timestamp)"%stablename)
             
             stmt = conn.statement("insert into ? using log tags (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) \
                 values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)")
@@ -139,142 +147,93 @@ class TDTestCase:
             assert stmt.affected_rows == 3
 
             #query all 
-            querystmt1=conn.statement("select * from log where bu < ?")
+            queryparam=new_bind_params(1)
-            queryparam1=new_bind_params(1)
+            queryparam[0].int(10)
-            print(type(queryparam1))
+            rows=self.stmtExe(conn,"select * from log where bu < ?",queryparam)
-            queryparam1[0].int(10)
+            tdLog.debug("assert 1st case %s"%rows)
-            querystmt1.bind_param(queryparam1)
+            assert str(rows[0][0]) == "2021-07-21 17:56:32.589111"
-            querystmt1.execute() 
+            assert rows[0][10] == 3 , '1st case is failed'
-            result1=querystmt1.use_result()
+            assert rows[1][10] == 4 , '1st case is failed'
-            rows1=result1.fetch_all()
-            print(rows1[0])
-            print(rows1[1])
-            print(rows1[2])
-            assert str(rows1[0][0]) == "2021-07-21 17:56:32.589111"
-            assert rows1[0][10] == 3
-            assert rows1[1][10] == 4
 
             #query: Numeric Functions
-            querystmt2=conn.statement("select abs(?) from log where bu < ?")
+            queryparam=new_bind_params(2)
-            queryparam2=new_bind_params(2)
+            queryparam[0].int(5)
-            print(type(queryparam2))
+            queryparam[1].int(5)
-            queryparam2[0].int(5)
+            rows=self.stmtExe(conn,"select abs(?) from log where bu < ?",queryparam)
-            queryparam2[1].int(5)
+            tdLog.debug("assert 2nd case %s"%rows)
-            querystmt2.bind_param(queryparam2)
+            assert rows[0][0] == 5 , '2nd case is failed'
-            querystmt2.execute() 
+            assert rows[1][0] == 5 , '2nd case is failed'
-            result2=querystmt2.use_result()
-            rows2=result2.fetch_all()
-            print("2",rows2)
-            assert rows2[0][0] == 5
-            assert rows2[1][0] == 5
 
 
             #query: Numeric Functions and escapes
+            queryparam=new_bind_params(1)
+            queryparam[0].int(5)
+            rows=self.stmtExe(conn,"select abs(?) from log where  nn= 'a? long string with 中文字符'",queryparam)
+            tdLog.debug("assert 3rd case %s"%rows)
+            assert rows == [] , '3rd case is failed'
 
-            querystmt3=conn.statement("select abs(?) from log where  nn= 'a? long string with 中文字符' ")
+            #query: string Functions
-            queryparam3=new_bind_params(1)
+            queryparam=new_bind_params(1)
-            print(type(queryparam3))
+            queryparam[0].binary('中文字符')
-            queryparam3[0].int(5)
+            rows=self.stmtExe(conn,"select CHAR_LENGTH(?) from log ",queryparam)
-            querystmt3.bind_param(queryparam3)
+            tdLog.debug("assert 4th case %s"%rows)
-            querystmt3.execute() 
+            assert rows[0][0] == 12, '4th case is failed'
-            result3=querystmt3.use_result()
+            assert rows[1][0] == 12, '4th case is failed'
-            rows3=result3.fetch_all()
-            print("3",rows3)
-            assert rows3 == []
 
-            # #query: string Functions
+            queryparam=new_bind_params(1)
+            queryparam[0].binary('123')
+            rows=self.stmtExe(conn,"select CHAR_LENGTH(?) from log ",queryparam)
+            tdLog.debug("assert 4th case %s"%rows)
+            assert rows[0][0] == 3, '4th.1 case is failed'
+            assert rows[1][0] == 3, '4th.1 case is failed'
 
-            # querystmt3=conn.statement("select CHAR_LENGTH(?) from log  ")
+            #query: conversion Functions
-            # queryparam3=new_bind_params(1)
+            queryparam=new_bind_params(1)
-            # print(type(queryparam3))
+            queryparam[0].binary('1232a')           
-            # queryparam3[0].binary('中文字符')
+            rows=self.stmtExe(conn,"select cast( ? as bigint) from log",queryparam)
-            # querystmt3.bind_param(queryparam3)
+            tdLog.debug("assert 5th case %s"%rows)
-            # querystmt3.execute() 
+            assert rows[0][0] == 1232, '5th.1 case is failed'
-            # result3=querystmt3.use_result()
+            assert rows[1][0] == 1232, '5th.1 case is failed'
-            # rows3=result3.fetch_all()
-            # print("4",rows3)
-            # assert rows3[0][0] == 12, 'fourth case is failed'
-            # assert rows3[1][0] == 12, 'fourth case is failed'
 
-            # #query: conversion Functions
+            querystmt4=conn.statement("select cast( ? as binary(10)) from log  ")
+            queryparam=new_bind_params(1)
+            queryparam[0].int(123)
+            rows=self.stmtExe(conn,"select cast( ? as bigint) from log",queryparam)
+            tdLog.debug("assert 6th case %s"%rows)
+            assert rows[0][0] == 123, '6th.1 case is failed'
+            assert rows[1][0] == 123, '6th.1 case is failed'
 
-            # querystmt4=conn.statement("select cast( ? as bigint) from log  ")
+            #query: datatime Functions
-            # queryparam4=new_bind_params(1)
+            queryparam=new_bind_params(1)
-            # print(type(queryparam4))
+            queryparam[0].timestamp(1626861392591112)
-            # queryparam4[0].binary('1232a')
+            rows=self.stmtExe(conn,"select timediff('2021-07-21 17:56:32.590111',?,1a)  from log",queryparam)
-            # querystmt4.bind_param(queryparam4)
+            tdLog.debug("assert 7th case %s"%rows)
-            # querystmt4.execute() 
+            assert rows[0][0] == 1, '7th case is failed'
-            # result4=querystmt4.use_result()
+            assert rows[1][0] == 1, '7th case is failed'
-            # rows4=result4.fetch_all()
-            # print("5",rows4)
-            # assert rows4[0][0] == 1232
-            # assert rows4[1][0] == 1232
-
-            # querystmt4=conn.statement("select cast( ? as binary(10)) from log  ")
-            # queryparam4=new_bind_params(1)
-            # print(type(queryparam4))
-            # queryparam4[0].int(123)
-            # querystmt4.bind_param(queryparam4)
-            # querystmt4.execute() 
-            # result4=querystmt4.use_result()
-            # rows4=result4.fetch_all()
-            # print("6",rows4)
-            # assert rows4[0][0] == '123'
-            # assert rows4[1][0] == '123'
-
-            # #query: datatime Functions
-
-            # querystmt4=conn.statement(" select timediff('2021-07-21 17:56:32.590111',?,1s)  from log  ")
-            # queryparam4=new_bind_params(1)
-            # print(type(queryparam4))
-            # queryparam4[0].timestamp(1626861392591111)
-            # querystmt4.bind_param(queryparam4)
-            # querystmt4.execute() 
-            # result4=querystmt4.use_result()
-            # rows4=result4.fetch_all()
-            # print("7",rows4)
-            # assert rows4[0][0] == 1, 'seventh case is failed'
-            # assert rows4[1][0] == 1, 'seventh case is failed'
             
             #query: aggregate Functions
+            queryparam=new_bind_params(1)
+            queryparam[0].int(123)
+            rows=self.stmtExe(conn,"select count(?)  from log ",queryparam)
+            tdLog.debug("assert 8th case %s"%rows)
+            assert rows[0][0] == 3, ' 8th case is failed'
 
-            querystmt4=conn.statement(" select count(?)  from log  ")
+            # #query: selector Functions 9
-            queryparam4=new_bind_params(1)
+            # queryparam=new_bind_params(1)
-            print(type(queryparam4))
+            # queryparam[0].int(2)
-            queryparam4[0].int(123)
+            # rows=self.stmtExe(conn,"select bottom(bu,?)  from log group by bu ; ",queryparam)
-            querystmt4.bind_param(queryparam4)
+            # tdLog.debug("assert 9th case %s"%rows)
-            querystmt4.execute() 
+            # assert rows[0][0] == 4, ' 9 case is failed'
-            result4=querystmt4.use_result()
+            # assert rows[1][0] == 3, ' 9 case is failed'
-            rows4=result4.fetch_all()
-            print("8",rows4)
-            assert rows4[0][0] == 3, ' 8 case is failed'
-
-            #query: selector Functions 9
-
-            querystmt4=conn.statement("  select bottom(bu,?)  from log group by bu ;  ")
-            queryparam4=new_bind_params(1)
-            print(type(queryparam4))
-            queryparam4[0].int(2)
-            querystmt4.bind_param(queryparam4)
-            querystmt4.execute() 
-            result4=querystmt4.use_result()
-            rows4=result4.fetch_all()
-            print("9",rows4)
-            assert rows4[0][0] == 4, ' 9 case is failed'
-            assert rows4[1][0] == 3, ' 9 case is failed'
 
             # #query: time-series specific Functions 10
 
-            querystmt4=conn.statement("  select twa(?)  from log;  ")
+            querystmt=conn.statement("  select twa(?)  from log;  ")
-            queryparam4=new_bind_params(1)
+            queryparam=new_bind_params(1)
-            print(type(queryparam4))
+            queryparam[0].int(15)
-            queryparam4[0].int(15)
+            rows=self.stmtExe(conn," select twa(?)  from log; ",queryparam)
-            querystmt4.bind_param(queryparam4)
+            tdLog.debug("assert 10th case %s"%rows)
-            querystmt4.execute() 
+            assert rows[0][0] == 15, ' 10th case is failed'
-            result4=querystmt4.use_result()
-            rows4=result4.fetch_all()
-            print("10",rows4)
-            assert rows4[0][0] == 15, ' 10 case is failed'
 
 
             # conn.execute("drop database if exists %s" % dbname)

tests/system-test/2-query/Now.py

@@ -40,6 +40,7 @@ class TDTestCase:
         self.time_unit = ['b','u','a','s','m','h','d','w']
         self.symbol = ['+','-','*','/']
         self.error_values = [1.5,'abc','"abc"','!@','today()']
+        self.db_percision = ['ms','us','ns']
     def tbtype_check(self,tb_type):
         if tb_type == 'normal table' or tb_type == 'child table':
             tdSql.checkRows(len(self.values_list))  
@@ -70,23 +71,29 @@ class TDTestCase:
                 tdSql.checkData(i,0,None)
 
     def now_check_ntb(self):
-        tdSql.prepare()
+        for time_unit in self.db_percision:
-        tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict))
+            tdSql.execute(f'create database db precision "{time_unit}"')
-        for value in self.values_list:
+            tdSql.execute('use db')
-            tdSql.execute(
+            tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict))
-                f'insert into {self.ntbname} values({value})')
+            for value in self.values_list:
-        self.data_check(self.ntbname,'normal table')
+                tdSql.execute(
+                    f'insert into {self.ntbname} values({value})')
+            self.data_check(self.ntbname,'normal table')
+            tdSql.execute('drop database db')
 
     def now_check_stb(self):
-        tdSql.prepare()
+        for time_unit in self.db_percision:
-        tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict))
+            tdSql.execute(f'create database db precision "{time_unit}"')
-        for i in range(self.tbnum):
+            tdSql.execute('use db')
-            tdSql.execute(f"create table {self.stbname}_{i} using {self.stbname} tags({self.tag_values[0]})")
+            tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict))
-            for value in self.values_list:
+            for i in range(self.tbnum):
-                tdSql.execute(f'insert into {self.stbname}_{i} values({value})')
+                tdSql.execute(f"create table {self.stbname}_{i} using {self.stbname} tags({self.tag_values[0]})")
-        for i in range(self.tbnum):
+                for value in self.values_list:
-            self.data_check(f'{self.stbname}_{i}','child table')
+                    tdSql.execute(f'insert into {self.stbname}_{i} values({value})')
-        self.data_check(self.stbname,'stable')
+            for i in range(self.tbnum):
+                self.data_check(f'{self.stbname}_{i}','child table')
+            self.data_check(self.stbname,'stable')
+            tdSql.execute('drop database db')
     def run(self):  # sourcery skip: extract-duplicate-method
 
         self.now_check_ntb()

tests/system-test/2-query/Today.py

@@ -5,6 +5,7 @@ from util.log import *
 from util.sql import *
 from util.cases import *
 import datetime
+import pandas as pd
 
 
 class TDTestCase:
@@ -12,8 +13,8 @@ class TDTestCase:
     def init(self, conn, logSql):
         tdLog.debug(f"start to excute {__file__}")
         tdSql.init(conn.cursor())
-        self.today_date = datetime.datetime.strptime(
+        self.today_date = datetime.datetime.strptime(datetime.datetime.now().strftime("%Y-%m-%d"), "%Y-%m-%d")
-            datetime.datetime.now().strftime("%Y-%m-%d"), "%Y-%m-%d")
+        self.today_ts = datetime.datetime.strptime(datetime.datetime.now().strftime("%Y-%m-%d"), "%Y-%m-%d").timestamp()
         self.time_unit = ['b','u','a','s','m','h','d','w']
         self.error_param = ['1.5','abc','!@#','"abc"','today()']
         self.arithmetic_operators = ['+','-','*','/']
@@ -41,7 +42,7 @@ class TDTestCase:
                         f'today(),3,3.333,333.333333,now()',
                         f'today()-1d,10,11.11,99.999999,now()',
                         f'today()+1d,1,1.55,100.555555,today()']
-                       
+        self.db_percision = ['ms','us','ns']
     def set_create_normaltable_sql(self, ntbname, column_dict):
         column_sql = ''
         for k, v in column_dict.items():
@@ -57,7 +58,8 @@ class TDTestCase:
             tag_sql += f"{k} {v},"
         create_stb_sql = f'create table {stbname} ({column_sql[:-1]}) tags({tag_sql[:-1]})'
         return create_stb_sql
-    def data_check(self,column_dict={},tbname = '',values_list = [],tb_num = 1,tb = 'tb'):
+        
+    def data_check(self,column_dict={},tbname = '',values_list = [],tb_num = 1,tb = 'tb',precision = 'ms'):
         for k,v in column_dict.items():
             num_up = 0
             num_down = 0
@@ -65,12 +67,27 @@ class TDTestCase:
             if v.lower() == 'timestamp':
                 tdSql.query(f'select {k} from {tbname}')
                 for i in tdSql.queryResult:
-                    if i[0] > self.today_date:
+                    if precision == 'ms':
-                        num_up += 1
+                        if int(i[0].timestamp())*1000 > int(self.today_ts)*1000:
-                    elif i[0] == self.today_date:
+                            num_up += 1
-                        num_same += 1
+                        elif int(i[0].timestamp())*1000 == int(self.today_ts)*1000:
-                    elif i[0] < self.today_date:
+                            num_same += 1
-                        num_down += 1
+                        elif int(i[0].timestamp())*1000 < int(self.today_ts)*1000:
+                            num_down += 1
+                    elif precision == 'us':
+                        if int(i[0].timestamp())*1000000 > int(self.today_ts)*1000000:
+                            num_up += 1
+                        elif int(i[0].timestamp())*1000000 == int(self.today_ts)*1000000:
+                            num_same += 1
+                        elif int(i[0].timestamp())*1000000 < int(self.today_ts)*1000000:
+                            num_down += 1
+                    elif precision == 'ns':
+                        if i[0] > int(self.today_ts)*1000000000:
+                            num_up += 1
+                        elif i[0] == int(self.today_ts)*1000000000:
+                            num_same += 1
+                        elif i[0] < int(self.today_ts)*1000000000:
+                            num_down += 1
                 tdSql.query(f"select today() from {tbname}")
                 tdSql.checkRows(len(values_list)*tb_num)
                 tdSql.checkData(0, 0, str(self.today_date))
@@ -130,32 +147,36 @@ class TDTestCase:
                     for i in range(num_same):
                         tdSql.checkData(i, 0, str(self.today_date))
     def today_check_ntb(self):
-        tdSql.prepare()
+        for time_unit in self.db_percision:
-        tdSql.execute(self.set_create_normaltable_sql(self.ntbname,self.column_dict))
+            print(time_unit)
-        for i in self.values_list:
+            tdSql.execute(f'create database db precision "{time_unit}"')
-            tdSql.execute(
+            tdSql.execute('use db')
-                f'insert into {self.ntbname} values({i})')
+            tdSql.execute(self.set_create_normaltable_sql(self.ntbname,self.column_dict))
-        self.data_check(self.column_dict,self.ntbname,self.values_list)
+            for i in self.values_list:
-        tdSql.execute('drop database db')
+                tdSql.execute(
+                    f'insert into {self.ntbname} values({i})')
+            self.data_check(self.column_dict,self.ntbname,self.values_list,1,'tb',time_unit)
+            tdSql.execute('drop database db')
     def today_check_stb_tb(self):
-        tdSql.prepare()
+        for time_unit in self.db_percision:
-        tdSql.execute(self.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict))
+            print(time_unit)
-        for i in range(self.tbnum):
+            tdSql.execute(f'create database db precision "{time_unit}"')
-            tdSql.execute(f'create table if not exists {self.stbname}_{i} using {self.stbname} tags({self.tag_values[i]})')
+            tdSql.execute('use db')
-            for j in self.values_list:
+            tdSql.execute(self.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict))
-                tdSql.execute(f'insert into {self.stbname}_{i} values ({j})')
+            for i in range(self.tbnum):
-        # check child table
+                tdSql.execute(f'create table if not exists {self.stbname}_{i} using {self.stbname} tags({self.tag_values[i]})')
-        for i in range(self.tbnum):
+                for j in self.values_list:
-            self.data_check(self.column_dict,f'{self.stbname}_{i}',self.values_list)
+                    tdSql.execute(f'insert into {self.stbname}_{i} values ({j})')
-        # check stable
+            # check child table
-        self.data_check(self.column_dict,self.stbname,self.values_list,self.tbnum,'stb')
+            for i in range(self.tbnum):
-        tdSql.execute('drop database db')
+                self.data_check(self.column_dict,f'{self.stbname}_{i}',self.values_list,1,'tb',time_unit)
+            # check stable
+            self.data_check(self.column_dict,self.stbname,self.values_list,self.tbnum,'stb',time_unit)
+            tdSql.execute('drop database db')
 
     def run(self):  # sourcery skip: extract-duplicate-method
         
-        tdLog.printNoPrefix("==========check today() for normal table ==========")
         self.today_check_ntb()
-        tdLog.printNoPrefix("==========check today() for stable and child table==========")
         self.today_check_stb_tb()
 
     def stop(self):

tests/system-test/7-tmq/stbFilter.py

@@ -20,18 +20,54 @@ class TDTestCase:
         tdSql.init(conn.cursor())
         #tdSql.init(conn.cursor(), logSql)  # output sql.txt file
 
-    def tmqCase1(self):
+    def prepareTestEnv(self):
-        tdLog.printNoPrefix("======== test case 1: ")
+        tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
-        paraDict = {'dbName':     'db1',
+        paraDict = {'dbName':     'dbt',
                     'dropFlag':   1,
                     'event':      '',
                     'vgroups':    4,
+                    'replica':    1,
                     'stbName':    'stb',
                     'colPrefix':  'c',
                     'tagPrefix':  't',
-                    'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'colSchema':   [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}],
                     'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
                     'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   100,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  10,
+                    'showMsg':    1,
+                    'showRow':    1}
+
+        tmqCom.initConsumerTable()
+        tmqCom.create_database(tsql=tdSql, dbName=paraDict["dbName"],dropFlag=paraDict["dropFlag"], vgroups=paraDict['vgroups'],replica=paraDict['replica'])
+        tdLog.info("create stb")
+        tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
+        tdLog.info("create ctb")
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum'])
+        tdLog.info("insert data")
+        tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
+                                               ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+                                               startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+        return
+
+    def tmqCase1(self):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    4,
+                    'replica':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
                     'ctbNum':     10,
                     'rowsPerTbl': 10000,
                     'batchNum':   100,
@@ -43,13 +79,6 @@ class TDTestCase:
         topicNameList = ['topic1', 'topic2', 'topic3']
         expectRowsList = []
         tmqCom.initConsumerTable()
-        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1)
-        tdLog.info("create stb")
-        tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
-        tdLog.info("create ctb")
-        tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
-        tdLog.info("insert data")
-        tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
         
         tdLog.info("create topics from stb with filter")
         queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 4 == 0" %(paraDict['dbName'], paraDict['stbName'])
@@ -134,16 +163,18 @@ class TDTestCase:
 
     def tmqCase2(self):
         tdLog.printNoPrefix("======== test case 2: ")
-        paraDict = {'dbName':     'db2',
+        paraDict = {'dbName':     'dbt',
                     'dropFlag':   1,
                     'event':      '',
                     'vgroups':    4,
+                    'replica':    1,
                     'stbName':    'stb',
                     'colPrefix':  'c',
                     'tagPrefix':  't',
                     'colSchema':   [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}],
                     'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
                     'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
                     'ctbNum':     10,
                     'rowsPerTbl': 10000,
                     'batchNum':   100,
@@ -155,13 +186,6 @@ class TDTestCase:
         topicNameList = ['topic1', 'topic2', 'topic3']
         expectRowsList = []
         tmqCom.initConsumerTable()
-        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1)
-        tdLog.info("create stb")
-        tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
-        tdLog.info("create ctb")
-        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum'])
-        tdLog.info("insert data")
-        tmqCom.insert_data_1(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
 
         tdLog.info("create topics from stb with filter")
         # sqlString = "create topic %s as select ts, sin(c1), pow(c2,3) from %s.%s where c2 >= 0" %(topicNameList[0], paraDict['dbName'], paraDict['stbName'])
@@ -247,6 +271,7 @@ class TDTestCase:
 
     def run(self):
         tdSql.prepare()
+        self.prepareTestEnv()
         self.tmqCase1()
         self.tmqCase2()
 

tests/system-test/7-tmq/tmqCommon.py

@@ -170,33 +170,42 @@ class TMQCom:
         tdLog.debug("complete to create database %s"%(dbName))
         return
 
+    # self.create_stable() and self.create_ctable() and self.insert_data_interlaceByMultiTbl() : The three functions are matched
+    # schema: (ts timestamp, c1 int, c2 bigint, c3 double, c4 binary(32), c5 nchar(32), c6 timestamp) tags (t1 int, t2 bigint, t3 double, t4 binary(32), t5 nchar(32))
     def create_stable(self,tsql, dbName,stbName):
-        tsql.execute("create table if not exists %s.%s (ts timestamp, c1 int, c2 int, c3 binary(16)) tags(t1 int, t2 binary(32))"%(dbName, stbName))
+        schemaString = "(ts timestamp, c1 int, c2 bigint, c3 double, c4 binary(32), c5 nchar(32), c6 timestamp) tags (t1 int, t2 bigint, t3 double, t4 binary(32), t5 nchar(32))"
+        tsql.execute("create table if not exists %s.%s %s"%(dbName, stbName, schemaString))
         tdLog.debug("complete to create %s.%s" %(dbName, stbName))
         return
 
     def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1,ctbStartIdx=0):
-        tsql.execute("use %s" %dbName)
+        # tsql.execute("use %s" %dbName)
         pre_create = "create table"
         sql = pre_create
         #tdLog.debug("doing create one  stable %s and %d  child table in %s  ..." %(stbname, count ,dbname))
+        batchNum = 10
+        tblBatched  = 0
         for i in range(ctbNum):
-            tagValue = 'beijing'
+            tagBinaryValue = 'beijing'
             if (i % 2 == 0):
-                tagValue = 'shanghai'
+                tagBinaryValue = 'shanghai'
             elif (i % 3 == 0):
-                tagValue = 'changsha'
+                tagBinaryValue = 'changsha'
             
-            sql += " %s%d using %s tags(%d, '%s')"%(ctbPrefix,i+ctbStartIdx,stbName,i+ctbStartIdx+1, tagValue)
+            sql += " %s.%s%d using %s.%s tags(%d, %d, %d, '%s', '%s')"%(dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,tagBinaryValue,tagBinaryValue)
-            if (i > 0) and (i%100 == 0):
+            tblBatched += 1
+            if (i == ctbNum-1 ) or (tblBatched == batchNum):
                 tsql.execute(sql)
+                tblBatched = 0
                 sql = pre_create
+
         if sql != pre_create:
             tsql.execute(sql)
         
-        tdLog.debug("complete to create %d child tables in %s.%s" %(ctbNum, dbName, stbName))
+        tdLog.debug("complete to create %d child tables by %s.%s" %(ctbNum, dbName, stbName))
-        return
+        return    
 
+    # schema: (ts timestamp, c1 int, c2 binary(16))
     def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=None):
         tdLog.debug("start to insert data ............")
         tsql.execute("use %s" %dbName)
@@ -208,11 +217,14 @@ class TMQCom:
             startTs = int(round(t * 1000))
         #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
         for i in range(ctbNum):
+            rowsBatched = 0
             sql += " %s%d values "%(stbName,i)
             for j in range(rowsPerTbl):
                 sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j)
-                if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)):
+                rowsBatched += 1
+                if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)):
                     tsql.execute(sql)
+                    rowsBatched = 0
                     if j < rowsPerTbl - 1:
                         sql = "insert into %s%d values " %(stbName,i)
                     else:
@@ -224,6 +236,7 @@ class TMQCom:
         tdLog.debug("insert data ............ [OK]")
         return        
 
+    # schema: (ts timestamp, c1 int, c2 int, c3 binary(16))
     def insert_data_1(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs):
         tdLog.debug("start to insert data ............")
         tsql.execute("use %s" %dbName)
@@ -234,14 +247,17 @@ class TMQCom:
         startTs = int(round(t * 1000))
         #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
         for i in range(ctbNum):
+            rowsBatched = 0
             sql += " %s%d values "%(ctbPrefix,i)
             for j in range(rowsPerTbl):
                 if (j % 2 == 0):
                     sql += "(%d, %d, %d, 'tmqrow_%d') "%(startTs + j, j, j, j)
                 else:
                     sql += "(%d, %d, %d, 'tmqrow_%d') "%(startTs + j, j, -j, j)
-                if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)):
+                rowsBatched += 1
+                if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)):
                     tsql.execute(sql)
+                    rowsBatched = 0
                     if j < rowsPerTbl - 1:
                         sql = "insert into %s%d values " %(ctbPrefix,i)
                     else:
@@ -253,6 +269,7 @@ class TMQCom:
         tdLog.debug("insert data ............ [OK]")
         return
 
+    # schema: (ts timestamp, c1 int, c2 int, c3 binary(16), c4 timestamp)
     def insert_data_2(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs,ctbStartIdx=0):
         tdLog.debug("start to insert data ............")
         tsql.execute("use %s" %dbName)
@@ -263,14 +280,17 @@ class TMQCom:
         startTs = int(round(t * 1000))
         #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
         for i in range(ctbNum):
+            rowsBatched = 0
             sql += " %s%d values "%(ctbPrefix,i+ctbStartIdx)
             for j in range(rowsPerTbl):
                 if (j % 2 == 0):
                     sql += "(%d, %d, %d, 'tmqrow_%d', now) "%(startTs + j, j, j, j)
                 else:
                     sql += "(%d, %d, %d, 'tmqrow_%d', now) "%(startTs + j, j, -j, j)
-                if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)):
+                rowsBatched += 1
+                if (rowsBatched == batchNum) or (j == rowsPerTbl - 1):
                     tsql.execute(sql)
+                    rowsBatched = 0
                     if j < rowsPerTbl - 1:
                         sql = "insert into %s%d values " %(ctbPrefix,i+ctbStartIdx)
                     else:
@@ -282,7 +302,8 @@ class TMQCom:
         tdLog.debug("insert data ............ [OK]")
         return
 
-    def insert_data_interlaceByMultiTbl(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0):
+    # schema: (ts timestamp, c1 int, c2 bigint, c3 double, c4 binary(32), c5 nchar(32), c6 timestamp) tags (t1 int, t2 bigint, t3 double, t4 binary(32), t5 nchar(32))
+    def insert_data_interlaceByMultiTbl(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0,ctbStartIdx=0):
         tdLog.debug("start to insert data ............")
         tsql.execute("use %s" %dbName)
         pre_insert = "insert into "
@@ -297,15 +318,22 @@ class TMQCom:
             ctbDict[i] = 0
 
         #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
-        rowsOfCtb = 0        
+        rowsOfCtb = 0
         while rowsOfCtb < rowsPerTbl:
             for i in range(ctbNum):
-                sql += " %s.%s_%d values "%(dbName,ctbPrefix,i)
+                sql += " %s.%s%d values "%(dbName,ctbPrefix,i+ctbStartIdx)
+                rowsBatched = 0
                 for k in range(batchNum):
-                    sql += "(%d, %d, 'tmqrow_%d') "%(startTs + ctbDict[i], ctbDict[i], ctbDict[i])
+                    if (k % 2 == 0):
+                        sql += "(%d, %d, %d, %d, 'binary_%d', 'nchar_%d', now) "%(startTs+ctbDict[i], ctbDict[i],ctbDict[i], ctbDict[i],i+ctbStartIdx,k)
+                    else:
+                        sql += "(%d, %d, %d, %d, 'binary_%d', 'nchar_%d', now) "%(startTs+ctbDict[i],-ctbDict[i],ctbDict[i],-ctbDict[i],i+ctbStartIdx,k)
+
+                    rowsBatched += 1
                     ctbDict[i] += 1
-                    if (0 == ctbDict[i]%batchNum) or (ctbDict[i] == rowsPerTbl):
+                    if (rowsBatched == batchNum) or (ctbDict[i] == rowsPerTbl):
                         tsql.execute(sql)
+                        rowsBatched = 0
                         sql = "insert into "
                         break
             rowsOfCtb = ctbDict[0]
@@ -313,7 +341,18 @@ class TMQCom:
         tdLog.debug("insert data ............ [OK]")
         return
 
-    def insert_data_with_autoCreateTbl(self,tsql,dbName,stbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0):
+    def threadFunctionForInsertByInterlace(self, **paraDict):
+        # create new connector for new tdSql instance in my thread
+        newTdSql = tdCom.newTdSql()
+        self.insert_data_interlaceByMultiTbl(newTdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"],paraDict["ctbStartIdx"])
+        return
+
+    def asyncInsertDataByInterlace(self, paraDict):
+        pThread = threading.Thread(target=self.threadFunctionForInsertByInterlace, kwargs=paraDict)
+        pThread.start()
+        return pThread
+
+    def insert_data_with_autoCreateTbl(self,tsql,dbName,stbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0,ctbStartIdx=0):
         tdLog.debug("start to insert data wiht auto create child table ............")
         tsql.execute("use %s" %dbName)
         pre_insert = "insert into "
@@ -324,17 +363,17 @@ class TMQCom:
             startTs = int(round(t * 1000))
 
         #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
-        rowsOfSql = 0        
+        rowsBatched = 0        
         for i in range(ctbNum):
-            sql += " %s.%s_%d using %s.%s tags (%d) values "%(dbName,ctbPrefix,i,dbName,stbName,i)
+            sql += " %s.%s_%d using %s.%s tags (%d) values "%(dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,i)
             for j in range(rowsPerTbl):
                 sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j)
-                rowsOfSql += 1
+                rowsBatched += 1
-                if (j > 0) and ((rowsOfSql == batchNum) or (j == rowsPerTbl - 1)):
+                if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)):
                     tsql.execute(sql)
-                    rowsOfSql = 0
+                    rowsBatched = 0
                     if j < rowsPerTbl - 1:
-                        sql = "insert into %s.%s_%d using %s.%s tags (%d) values " %(dbName,ctbPrefix,i,dbName,stbName,i)
+                        sql = "insert into %s.%s_%d using %s.%s tags (%d) values " %(dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,i)
                     else:
                         sql = "insert into "
         #end sql

tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py

@@ -0,0 +1,242 @@
+
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+import math
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+class TDTestCase:
+    def __init__(self):
+        self.vgroups    = 4
+        self.ctbNum     = 10
+        self.rowsPerTbl = 10000
+        
+    def init(self, conn, logSql):
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor(), False)
+
+    def prepareTestEnv(self):
+        tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  3,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   1}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        
+        tmqCom.initConsumerTable()
+        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
+        tdLog.info("create stb")
+        tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
+        tdLog.info("create ctb")
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
+                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
+        tdLog.info("insert data")
+        tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
+                                               ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+                                               startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+        
+        tdLog.info("restart taosd to ensure that the data falls into the disk")
+        tdDnodes.stop(1)
+        # tdDnodes.start(1)
+        tdDnodes.starttaosd(1)
+        return
+
+    def tmqCase1(self):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  3,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   1}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        
+        topicNameList = ['topic1']
+        expectRowsList = []
+        tmqCom.initConsumerTable()
+        
+        tdLog.info("create topics from stb with filter")
+        queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
+        # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
+        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+        tdSql.query(queryString)        
+        expectRowsList.append(tdSql.getRows())
+
+        # init consume info, and start tmq_sim, then check consume result
+        tdLog.info("insert consume info to consume processor")
+        consumerId   = 0
+        expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"]
+        topicList    = topicNameList[0]
+        ifcheckdata  = 1
+        ifManualCommit = 1
+        keyList      = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        tdLog.info("start consume processor")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+        tdLog.info("wait the consume result") 
+        
+        expectRows = 1
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        
+        if expectRowsList[0] != resultList[0]:
+            tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
+            tdLog.exit("%d tmq consume rows error!"%consumerId)
+
+        # tmqCom.checkFileContent(consumerId, queryString)     
+
+        time.sleep(10)        
+        for i in range(len(topicNameList)):
+            tdSql.query("drop topic %s"%topicNameList[i])
+
+        tdLog.printNoPrefix("======== test case 1 end ...... ")
+
+    def tmqCase2(self):
+        tdLog.printNoPrefix("======== test case 2: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  3,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   1}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        
+        topicNameList = ['topic1']
+        expectRowsList = []
+        tmqCom.initConsumerTable()
+        
+        tdLog.info("create topics from stb with filter")
+        queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
+        # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
+        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+        tdSql.query(queryString)        
+        expectRowsList.append(tdSql.getRows())
+        totalRowsInserted = expectRowsList[0]
+
+        # init consume info, and start tmq_sim, then check consume result
+        tdLog.info("insert consume info to consume processor")
+        consumerId   = 1
+        expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] / 3)
+        topicList    = topicNameList[0]
+        ifcheckdata  = 1
+        ifManualCommit = 1
+        keyList      = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        tdLog.info("start consume processor 0")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+        tdLog.info("wait the consume result") 
+        
+        expectRows = 1
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        
+        if not (expectrowcnt <= resultList[0] and totalRowsInserted >= resultList[0]):
+            tdLog.info("act consume rows: %d, expect consume rows between %d and %d"%(resultList[0], expectrowcnt, totalRowsInserted))
+            tdLog.exit("%d tmq consume rows error!"%consumerId)
+            
+        firstConsumeRows = resultList[0]
+
+        # reinit consume info, and start tmq_sim, then check consume result
+        tmqCom.initConsumerTable()
+        consumerId   = 2
+        expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2/3)
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+        
+        tdLog.info("start consume processor 1")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+        tdLog.info("wait the consume result") 
+        
+        expectRows = 1
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        
+        actConsumeTotalRows = firstConsumeRows + resultList[0]
+        
+        if not (expectrowcnt >= resultList[0] and totalRowsInserted == actConsumeTotalRows):
+            tdLog.info("act consume rows, first: %d, second: %d "%(firstConsumeRows, resultList[0]))            
+            tdLog.info("and sum of two consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted))
+            tdLog.exit("%d tmq consume rows error!"%consumerId)
+
+        time.sleep(10)        
+        for i in range(len(topicNameList)):
+            tdSql.query("drop topic %s"%topicNameList[i])
+
+        tdLog.printNoPrefix("======== test case 2 end ...... ")
+
+    def run(self):
+        tdSql.prepare()
+        self.prepareTestEnv()
+        self.tmqCase1()
+        self.tmqCase2()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/7-tmq/tmqConsFromTsdb.py

@@ -25,6 +25,49 @@ class TDTestCase:
         tdLog.debug(f"start to excute {__file__}")
         tdSql.init(conn.cursor(), False)
 
+    def prepareTestEnv(self):
+        tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  3,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   1}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        
+        tmqCom.initConsumerTable()
+        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
+        tdLog.info("create stb")
+        tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
+        tdLog.info("create ctb")
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
+                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
+        tdLog.info("insert data")
+        tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
+                                               ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+                                               startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+        
+        tdLog.info("restart taosd to ensure that the data falls into the disk")
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        return
+
     def tmqCase1(self):
         tdLog.printNoPrefix("======== test case 1: ")
         paraDict = {'dbName':     'dbt',
@@ -34,10 +77,11 @@ class TDTestCase:
                     'stbName':    'stb',
                     'colPrefix':  'c',
                     'tagPrefix':  't',
-                    'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
-                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
                     'ctbPrefix':  'ctb',
-                    'ctbNum':     1,
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
                     'rowsPerTbl': 10000,
                     'batchNum':   10,
                     'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
@@ -53,16 +97,6 @@ class TDTestCase:
         topicNameList = ['topic1']
         expectRowsList = []
         tmqCom.initConsumerTable()
-        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
-        tdLog.info("create stb")
-        tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
-        tdLog.info("create ctb")
-        tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
-        tdLog.info("insert data")
-        tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
-        
-        tdDnodes.stop(1)
-        tdDnodes.start(1)
         
         tdLog.info("create topics from stb with filter")
         queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
@@ -111,10 +145,11 @@ class TDTestCase:
                     'stbName':    'stb',
                     'colPrefix':  'c',
                     'tagPrefix':  't',
-                    'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
-                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
                     'ctbPrefix':  'ctb',
-                    'ctbNum':     1,
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
                     'rowsPerTbl': 10000,
                     'batchNum':   10,
                     'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
@@ -130,16 +165,6 @@ class TDTestCase:
         topicNameList = ['topic1']
         expectRowsList = []
         tmqCom.initConsumerTable()
-        # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
-        # tdLog.info("create stb")
-        # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
-        # tdLog.info("create ctb")
-        # tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
-        # tdLog.info("insert data")
-        # tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
-        
-        # tdDnodes.stop(1)
-        # tdDnodes.start(1)
         
         tdLog.info("create topics from stb with filter")
         queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
@@ -200,188 +225,11 @@ class TDTestCase:
 
         tdLog.printNoPrefix("======== test case 2 end ...... ")
 
-    def tmqCase3(self):
-        tdLog.printNoPrefix("======== test case 3: ")
-        paraDict = {'dbName':     'dbt',
-                    'dropFlag':   1,
-                    'event':      '',
-                    'vgroups':    1,
-                    'stbName':    'stb',
-                    'colPrefix':  'c',
-                    'tagPrefix':  't',
-                    'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
-                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
-                    'ctbPrefix':  'ctb',
-                    'ctbNum':     1,
-                    'rowsPerTbl': 10000,
-                    'batchNum':   10,
-                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
-                    'pollDelay':  10,
-                    'showMsg':    1,
-                    'showRow':    1,
-                    'snapshot':   1}
-
-        paraDict['vgroups'] = self.vgroups
-        paraDict['ctbNum'] = self.ctbNum
-        paraDict['rowsPerTbl'] = self.rowsPerTbl
-        
-        topicNameList = ['topic1']
-        expectRowsList = []
-        tmqCom.initConsumerTable()
-        # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
-        # tdLog.info("create stb")
-        # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
-        # tdLog.info("create ctb")
-        # tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
-        # tdLog.info("insert data")
-        # tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
-        
-        # tdDnodes.stop(1)
-        # tdDnodes.start(1)
-        
-        tdLog.info("create topics from stb with filter")
-        queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
-        # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
-        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
-        tdLog.info("create topic sql: %s"%sqlString)
-        tdSql.execute(sqlString)
-        tdSql.query(queryString)        
-        expectRowsList.append(tdSql.getRows())
-        totalRowsInserted = expectRowsList[0]
-
-        # init consume info, and start tmq_sim, then check consume result
-        tdLog.info("insert consume info to consume processor")
-        consumerId   = 3
-        expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] / 3)
-        topicList    = topicNameList[0]
-        ifcheckdata  = 1
-        ifManualCommit = 1
-        keyList      = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
-        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
-
-        consumerId   = 4
-        expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2/3)
-        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
-
-        tdLog.info("start consume processor 0")
-        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
-        tdLog.info("wait the consume result") 
-        
-        expectRows = 2
-        resultList = tmqCom.selectConsumeResult(expectRows)
-        actConsumeTotalRows = resultList[0] + resultList[1]
-        
-        if not (totalRowsInserted == actConsumeTotalRows):
-            tdLog.info("sum of two consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted))
-            tdLog.exit("%d tmq consume rows error!"%consumerId)
-
-        time.sleep(10)        
-        for i in range(len(topicNameList)):
-            tdSql.query("drop topic %s"%topicNameList[i])
-
-        tdLog.printNoPrefix("======== test case 3 end ...... ")
-
-    def tmqCase4(self):
-        tdLog.printNoPrefix("======== test case 4: ")
-        paraDict = {'dbName':     'dbt',
-                    'dropFlag':   1,
-                    'event':      '',
-                    'vgroups':    1,
-                    'stbName':    'stb',
-                    'colPrefix':  'c',
-                    'tagPrefix':  't',
-                    'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
-                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
-                    'ctbPrefix':  'ctb',
-                    'ctbNum':     1,
-                    'rowsPerTbl': 10000,
-                    'batchNum':   10,
-                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
-                    'pollDelay':  10,
-                    'showMsg':    1,
-                    'showRow':    1,
-                    'snapshot':   1}
-
-        paraDict['vgroups'] = self.vgroups
-        paraDict['ctbNum'] = self.ctbNum
-        paraDict['rowsPerTbl'] = self.rowsPerTbl
-        
-        topicNameList = ['topic1']
-        expectRowsList = []
-        tmqCom.initConsumerTable()
-        # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
-        # tdLog.info("create stb")
-        # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
-        # tdLog.info("create ctb")
-        # tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
-        # tdLog.info("insert data")
-        # tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
-        
-        # tdDnodes.stop(1)
-        # tdDnodes.start(1)
-        
-        tdLog.info("create topics from stb with filter")
-        queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
-        # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
-        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
-        tdLog.info("create topic sql: %s"%sqlString)
-        tdSql.execute(sqlString)
-        tdSql.query(queryString)        
-        expectRowsList.append(tdSql.getRows())
-        totalRowsInserted = expectRowsList[0]
-
-        # init consume info, and start tmq_sim, then check consume result
-        tdLog.info("insert consume info to consume processor")
-        consumerId   = 5
-        expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"])
-        topicList    = topicNameList[0]
-        ifcheckdata  = 1
-        ifManualCommit = 1
-        keyList      = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
-        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
-
-        tdLog.info("start consume processor 0")
-        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
-        
-        tdLog.info("wait commit notify")
-        tmqCom.getStartCommitNotifyFromTmqsim()
-
-        tdLog.info("pkill consume processor")
-        tdCom.killProcessor("tmq_sim")
-        
-        # time.sleep(10)
-        
-        # reinit consume info, and start tmq_sim, then check consume result
-        tmqCom.initConsumerTable()
-        consumerId   = 6
-        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
-        
-        tdLog.info("start consume processor 1")
-        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
-        tdLog.info("wait the consume result")
-        
-        expectRows = 1
-        resultList = tmqCom.selectConsumeResult(expectRows)
-        
-        actConsumeTotalRows = resultList[0]
-        
-        if not (actConsumeTotalRows > 0 and actConsumeTotalRows < totalRowsInserted):
-            tdLog.info("act consume rows: %d"%(actConsumeTotalRows))            
-            tdLog.info("and second consume rows should be between 0 and %d"%(totalRowsInserted))
-            tdLog.exit("%d tmq consume rows error!"%consumerId)
-
-        time.sleep(10)        
-        for i in range(len(topicNameList)):
-            tdSql.query("drop topic %s"%topicNameList[i])
-
-        tdLog.printNoPrefix("======== test case 4 end ...... ")
-
     def run(self):
         tdSql.prepare()
+        self.prepareTestEnv()
         self.tmqCase1()
         self.tmqCase2()
-        self.tmqCase3()
-        self.tmqCase4()
 
     def stop(self):
         tdSql.close()

tests/system-test/7-tmq/tmqConsFromTsdb1.py

@@ -0,0 +1,241 @@
+
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+import math
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+class TDTestCase:
+    def __init__(self):
+        self.vgroups    = 1
+        self.ctbNum     = 10
+        self.rowsPerTbl = 10000
+        
+    def init(self, conn, logSql):
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor(), False)
+
+    def prepareTestEnv(self):
+        tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  3,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   1}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        
+        tmqCom.initConsumerTable()
+        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
+        tdLog.info("create stb")
+        tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
+        tdLog.info("create ctb")
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
+                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
+        tdLog.info("insert data")
+        tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
+                                               ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+                                               startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+        
+        tdLog.info("restart taosd to ensure that the data falls into the disk")
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        return
+
+    def tmqCase3(self):
+        tdLog.printNoPrefix("======== test case 3: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  10,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   1}
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        
+        topicNameList = ['topic1']
+        expectRowsList = []
+        tmqCom.initConsumerTable()
+        
+        tdLog.info("create topics from stb with filter")
+        queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
+        # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
+        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+        tdSql.query(queryString)        
+        expectRowsList.append(tdSql.getRows())
+        totalRowsInserted = expectRowsList[0]
+
+        # init consume info, and start tmq_sim, then check consume result
+        tdLog.info("insert consume info to consume processor")
+        consumerId   = 3
+        expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] / 3)
+        topicList    = topicNameList[0]
+        ifcheckdata  = 1
+        ifManualCommit = 1
+        keyList      = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        consumerId   = 4
+        expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2/3)
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        tdLog.info("start consume processor 0")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+        tdLog.info("wait the consume result") 
+        
+        expectRows = 2
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        actConsumeTotalRows = resultList[0] + resultList[1]
+        
+        if not (totalRowsInserted == actConsumeTotalRows):
+            tdLog.info("sum of two consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted))
+            tdLog.exit("%d tmq consume rows error!"%consumerId)
+
+        time.sleep(10)        
+        for i in range(len(topicNameList)):
+            tdSql.query("drop topic %s"%topicNameList[i])
+
+        tdLog.printNoPrefix("======== test case 3 end ...... ")
+
+    def tmqCase4(self):
+        tdLog.printNoPrefix("======== test case 4: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  10,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   1}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        
+        topicNameList = ['topic1']
+        expectRowsList = []
+        tmqCom.initConsumerTable()
+        
+        tdLog.info("create topics from stb with filter")
+        queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
+        # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
+        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+        tdSql.query(queryString)        
+        expectRowsList.append(tdSql.getRows())
+        totalRowsInserted = expectRowsList[0]
+
+        # init consume info, and start tmq_sim, then check consume result
+        tdLog.info("insert consume info to consume processor")
+        consumerId   = 5
+        expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"])
+        topicList    = topicNameList[0]
+        ifcheckdata  = 1
+        ifManualCommit = 1
+        keyList      = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        tdLog.info("start consume processor 0")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+        
+        tdLog.info("wait commit notify")
+        tmqCom.getStartCommitNotifyFromTmqsim()
+
+        tdLog.info("pkill consume processor")
+        tdCom.killProcessor("tmq_sim")
+        
+        # time.sleep(10)
+        
+        # reinit consume info, and start tmq_sim, then check consume result
+        tmqCom.initConsumerTable()
+        consumerId   = 6
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+        
+        tdLog.info("start consume processor 1")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+        tdLog.info("wait the consume result")
+        
+        expectRows = 1
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        
+        actConsumeTotalRows = resultList[0]
+        
+        if not (actConsumeTotalRows > 0 and actConsumeTotalRows < totalRowsInserted):
+            tdLog.info("act consume rows: %d"%(actConsumeTotalRows))            
+            tdLog.info("and second consume rows should be between 0 and %d"%(totalRowsInserted))
+            tdLog.exit("%d tmq consume rows error!"%consumerId)
+
+        time.sleep(10)        
+        for i in range(len(topicNameList)):
+            tdSql.query("drop topic %s"%topicNameList[i])
+
+        tdLog.printNoPrefix("======== test case 4 end ...... ")
+
+    def run(self):
+        tdSql.prepare()
+        self.prepareTestEnv()
+        self.tmqCase3()
+        self.tmqCase4()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/fulltest.sh

@@ -18,7 +18,8 @@ python3 ./test.py -f 0-others/fsync.py
 python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py
 python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py
 python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py
-# BUG python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py
+python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py
+python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py
 python3 ./test.py -f 1-insert/alter_stable.py
 python3 ./test.py -f 1-insert/alter_table.py
 python3 ./test.py -f 1-insert/insertWithMoreVgroup.py
@@ -119,10 +120,10 @@ python3 ./test.py -f 2-query/irate.py
 python3 ./test.py -f 2-query/function_null.py
 python3 ./test.py -f 2-query/queryQnode.py
 
-python3 ./test.py -f 6-cluster/5dnode1mnode.py
+#python3 ./test.py -f 6-cluster/5dnode1mnode.py
-python3 ./test.py -f 6-cluster/5dnode2mnode.py  -N 5 -M 3
+#python3 ./test.py -f 6-cluster/5dnode2mnode.py  -N 5 -M 3
-python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3
+#python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3
-python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3
+#python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3
 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3
 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 5 -M 3
 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py  -N 5 -M 3
@@ -158,3 +159,6 @@ python3 ./test.py -f 7-tmq/tmqUdf.py
 python3 ./test.py -f 7-tmq/tmqConsumerGroup.py
 python3 ./test.py -f 7-tmq/tmqShow.py
 python3 ./test.py -f 7-tmq/tmqAlterSchema.py
+python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py
+python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py
+python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg.py

tests/system-test/test.py

@@ -63,8 +63,9 @@ if __name__ == "__main__":
     mnodeNums = 0
     updateCfgDict = {}
     execCmd = ""
-    opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:', [
+    queryPolicy = 1
-        'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums'])
+    opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:', [
+        'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy'])
     for key, value in opts:
         if key in ['-h', '--help']:
             tdLog.printNoPrefix(
@@ -82,6 +83,7 @@ if __name__ == "__main__":
             tdLog.printNoPrefix('-e eval str to run')
             tdLog.printNoPrefix('-N create dnodes numbers in clusters')
             tdLog.printNoPrefix('-M create mnode numbers in clusters')
+            tdLog.printNoPrefix('-Q set queryPolicy in one dnode')
 
             sys.exit(0)
 
@@ -138,6 +140,9 @@ if __name__ == "__main__":
         if key in ['-M', '--mnodeNums']:
             mnodeNums = value
 
+        if key in ['-Q', '--queryPolicy']:
+            queryPolicy = value
+
     if not execCmd == "":
         tdDnodes.init(deployPath)
         print(execCmd)
@@ -276,6 +281,22 @@ if __name__ == "__main__":
             tdDnodes.deploy(1,updateCfgDict)
             tdDnodes.start(1)
             tdCases.logSql(logSql)
+            if queryPolicy != 1:
+                queryPolicy=int(queryPolicy)
+                conn = taos.connect(
+                host,
+                config=tdDnodes.getSimCfgPath())
+                tdSql.init(conn.cursor())
+                tdSql.execute("create qnode on dnode 1")
+                tdSql.execute('alter local "queryPolicy" "%d"'%queryPolicy)     
+                tdSql.query("show local variables;")
+                for i in range(tdSql.queryRows):
+                    if tdSql.queryResult[i][0] == "queryPolicy" :
+                        if int(tdSql.queryResult[i][1]) == int(queryPolicy):
+                            tdLog.success('alter queryPolicy to %d successfully'%queryPolicy)
+                        else :
+                            tdLog.debug(tdSql.queryResult)
+                            tdLog.exit("alter queryPolicy to  %d failed"%queryPolicy)      
         else :
             tdLog.debug("create an cluster  with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums))
             dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums,mnodeNums=mnodeNums)

tests/test/c/tmqSim.c

@@ -36,7 +36,6 @@
 #define MAX_CONSUMER_THREAD_CNT (16)
 #define MAX_VGROUP_CNT          (32)
 
-int64_t now;
 typedef enum {
   NOTIFY_CMD_START_CONSUM,
   NOTIFY_CMD_START_COMMIT,
@@ -91,6 +90,7 @@ typedef struct {
   int32_t     consumeDelay;  // unit s
   int32_t     numOfThread;
   int32_t     useSnapshot;
+  int64_t     nowTime;
   SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT];
 } SConfInfo;
 
@@ -199,6 +199,8 @@ void parseArgument(int32_t argc, char* argv[]) {
   g_stConfInfo.saveRowFlag = 0;
   g_stConfInfo.consumeDelay = 5;
 
+  g_stConfInfo.nowTime = taosGetTimestampMs();
+
   for (int32_t i = 1; i < argc; i++) {
     if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
       printHelp();
@@ -511,10 +513,8 @@ static void appNothing(void* param, TAOS_RES* res, int32_t numOfRows) {}
 int32_t notifyMainScript(SThreadInfo* pInfo, int32_t cmdId) {
   char sqlStr[1024] = {0};
 
-  int64_t now = taosGetTimestampMs();
-
   // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int
-  sprintf(sqlStr, "insert into %s.notifyinfo values (%" PRId64 ", %d, %d)", g_stConfInfo.cdbName, now, cmdId,
+  sprintf(sqlStr, "insert into %s.notifyinfo values (%" PRId64 ", %d, %d)", g_stConfInfo.cdbName, atomic_fetch_add_64(&g_stConfInfo.nowTime, 1), cmdId,
           pInfo->consumerId);
 
   taos_query_a(pInfo->taos, sqlStr, appNothing, NULL);
@@ -591,7 +591,7 @@ int32_t saveConsumeResult(SThreadInfo* pInfo) {
   char sqlStr[1024] = {0};
   // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int
   sprintf(sqlStr, "insert into %s.consumeresult values (%" PRId64 ", %d, %" PRId64 ", %" PRId64 ", %d)",
-          g_stConfInfo.cdbName, atomic_fetch_add_64(&now, 1), pInfo->consumerId, pInfo->consumeMsgCnt,
+          g_stConfInfo.cdbName, atomic_fetch_add_64(&g_stConfInfo.nowTime, 1), pInfo->consumerId, pInfo->consumeMsgCnt,
           pInfo->consumeRowCnt, pInfo->checkresult);
 
   char tmpString[128];
@@ -855,8 +855,6 @@ int32_t getConsumeInfo() {
 }
 
 int main(int32_t argc, char* argv[]) {
-  now = taosGetTimestampMs();
-
   parseArgument(argc, argv);
   getConsumeInfo();
   saveConfigToLogFile();

tools/shell/src/shellArguments.c

@@ -13,6 +13,10 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#ifdef _TD_DARWIN_64
+#include <pwd.h>
+#endif
+
 #include "shellInt.h"
 
 #define SHELL_HOST     "The auth string to use when connecting to the server."

tools/shell/src/shellEngine.c

@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#define ALLOW_FORBID_FUNC
 #define _BSD_SOURCE
 #define _GNU_SOURCE
 #define _XOPEN_SOURCE

tools/taos-tools

@@ -1 +1 @@
-Subproject commit 5fdd694621fbb7bd2d6102ff4feaec92a7001038
+Subproject commit 7105027650b51e701cfa1dac11b8fb42d447dd01