Merge remote-tracking branch 'origin/3.0' into fix/mnode

include/common/tmsg.h

@@ -70,7 +70,7 @@ typedef uint16_t tmsg_t;
 #define TSDB_IE_TYPE_DNODE_EXT   6
 #define TSDB_IE_TYPE_DNODE_STATE 7
 
-enum { CONN_TYPE__QUERY = 1, CONN_TYPE__TMQ, CONN_TYPE__MAX };
+enum { CONN_TYPE__QUERY = 1, CONN_TYPE__TMQ, CONN_TYPE__UDFD, CONN_TYPE__MAX };
 
 enum {
   HEARTBEAT_KEY_USER_AUTHINFO = 1,
@@ -2097,6 +2097,18 @@ enum {
   TOPIC_SUB_TYPE__TABLE,
 };
 
+typedef struct {
+  SMsgHead head;
+  int64_t  leftForVer;
+  int32_t  vgId;
+  int64_t  consumerId;
+  char     subKey[TSDB_SUBSCRIBE_KEY_LEN];
+} SMqVDeleteReq;
+
+typedef struct {
+  int8_t reserved;
+} SMqVDeleteRsp;
+
 typedef struct {
   int64_t leftForVer;
   int32_t vgId;
@@ -2255,20 +2267,22 @@ static FORCE_INLINE void tdDestroyTSma(STSma* pSma) {
   }
 }
 
-static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW) {
+static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW, bool deepCopy) {
   if (pSW) {
     if (pSW->tSma) {
-      for (uint32_t i = 0; i < pSW->number; ++i) {
+      if (deepCopy) {
-        tdDestroyTSma(pSW->tSma + i);
+        for (uint32_t i = 0; i < pSW->number; ++i) {
+          tdDestroyTSma(pSW->tSma + i);
+        }
       }
       taosMemoryFreeClear(pSW->tSma);
     }
   }
 }
 
-static FORCE_INLINE void* tdFreeTSmaWrapper(STSmaWrapper* pSW) {
+static FORCE_INLINE void* tdFreeTSmaWrapper(STSmaWrapper* pSW, bool deepCopy) {
-  tdDestroyTSmaWrapper(pSW);
+  tdDestroyTSmaWrapper(pSW, deepCopy);
-  taosMemoryFree(pSW);
+  taosMemoryFreeClear(pSW);
   return NULL;
 }
 
@@ -2532,11 +2546,15 @@ static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp* pRsp) {
 }
 
 typedef struct {
-  void* data;
+  int64_t streamId;
+  int32_t taskId;
+  int32_t sourceVg;
+  int64_t sourceVer;
+  SArray* data;  // SArray<SSDataBlock>
 } SStreamDispatchReq;
 
 typedef struct {
-  int8_t status;
+  int8_t inputStatus;
 } SStreamDispatchRsp;
 
 #define TD_AUTO_CREATE_TABLE 0x1

include/common/tmsgdef.h

@@ -178,6 +178,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONNECT, "vnode-mq-connect", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_DISCONNECT, "vnode-mq-disconnect", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_CHANGE, "vnode-mq-vg-change", SMqRebVgReq, SMqRebVgRsp)
+  TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_DELETE, "vnode-mq-vg-delete", SMqVDeleteReq, SMqVDeleteRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_TASK, "vnode-cancel-task", NULL, NULL)

include/libs/function/functionMgt.h

@@ -41,6 +41,7 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_SUM,
   FUNCTION_TYPE_TWA,
   FUNCTION_TYPE_HISTOGRAM,
+  FUNCTION_TYPE_HYPERLOGLOG,
 
   // nonstandard SQL function
   FUNCTION_TYPE_BOTTOM = 500,

include/libs/function/tudf.h

@@ -87,6 +87,7 @@ typedef struct SUdfInterBuf {
 } SUdfInterBuf;
 typedef void *UdfcFuncHandle;
 
+//low level APIs
 /**
  * setup udf
  * @param udf, in
@@ -115,6 +116,9 @@ int32_t doCallUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t
  */
 int32_t doTeardownUdf(UdfcFuncHandle handle);
 
+void freeUdfInterBuf(SUdfInterBuf *buf);
+
+//high level APIs
 bool udfAggGetEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
 int32_t udfAggProcess(struct SqlFunctionCtx *pCtx);

include/libs/nodes/querynodes.h

@@ -325,6 +325,7 @@ typedef struct SQuery {
   bool           showRewrite;
   int32_t        placeholderNum;
   SArray*        pPlaceholderValues;
+  SNode*         pContainPlaceholderRoot;
 } SQuery;
 
 void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext);

include/libs/parser/parser.h

@@ -62,7 +62,7 @@ int32_t     qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t
 void        qDestroyStmtDataBlock(void* pBlock);
 STableMeta* qGetTableMetaInDataBlock(void* pDataBlock);
 
-int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId);
+int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx);
 int32_t qStmtParseQuerySql(SParseContext* pCxt, SQuery* pQuery);
 int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen);
 int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx,
@@ -77,8 +77,8 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char*
 
 void*   smlInitHandle(SQuery* pQuery);
 void    smlDestroyHandle(void* pHandle);
-int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format,
+int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta,
-                    STableMeta* pTableMeta, char* tableName, char* msgBuf, int16_t msgBufLen);
+                    char* tableName, char* msgBuf, int16_t msgBufLen);
 int32_t smlBuildOutput(void* handle, SHashObj* pVgHash);
 
 #ifdef __cplusplus

include/libs/planner/planner.h

@@ -34,7 +34,6 @@ typedef struct SPlanContext {
   bool     showRewrite;
   int8_t   triggerType;
   int64_t  watermark;
-  int32_t  placeholderNum;
   char*    pMsg;
   int32_t  msgLen;
 } SPlanContext;
@@ -48,9 +47,6 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo
 // @pSource one execution location of this group of datasource subplans
 int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstreamSourceNode* pSource);
 
-int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId,
-                       bool* pEmptyResult);
-
 // Convert to subplan to string for the scheduler to send to the executor
 int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen);
 int32_t qStringToSubplan(const char* pStr, SSubplan** pSubplan);

include/os/os.h

@@ -58,9 +58,6 @@ extern "C" {
 #else
 #include <winsock.h>
 #endif
-
-#define __typeof(a) auto
-
 #endif
 
 #include <errno.h>

include/os/osFile.h

@@ -66,7 +66,7 @@ int32_t taosUnLockFile(TdFilePtr pFile);
 int32_t taosUmaskFile(int32_t maskVal);
  
 int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime);
-int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno);
+int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno);
 int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime);
 bool    taosCheckExistFile(const char *pathname);
  

include/os/osMath.h

@@ -23,11 +23,13 @@ extern "C" {
 #define TPOW2(x) ((x) * (x))
 #define TABS(x) ((x) > 0 ? (x) : -(x))
 
-#define TSWAP(a, b)             \
+#define TSWAP(a, b)                              \
-  do {                          \
+  do {                                           \
-    __typeof(a) __tmp = (a); \
+    char *__tmp = taosMemoryMalloc(sizeof(a));   \
-    (a) = (b);               \
+    memcpy(__tmp, &(a), sizeof(a));              \
-    (b) = __tmp;                \
+    memcpy(&(a), &(b), sizeof(a));               \
+    memcpy(&(b), __tmp, sizeof(a));              \
+    taosMemoryFree(__tmp);                       \
   } while (0)
 
 #ifdef WINDOWS

include/util/taoserror.h

@@ -281,6 +281,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03E8)
 #define TSDB_CODE_MND_OFFSET_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03E9)
 #define TSDB_CODE_MND_CONSUMER_NOT_READY        TAOS_DEF_ERROR_CODE(0, 0x03EA)
+#define TSDB_CODE_MND_TOPIC_SUBSCRIBED          TAOS_DEF_ERROR_CODE(0, 0x03EB)
 
 // mnode-stream
 #define TSDB_CODE_MND_STREAM_ALREADY_EXIST      TAOS_DEF_ERROR_CODE(0, 0x03F0)
@@ -643,6 +644,8 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INVALID_TIMELINE_FUNC     TAOS_DEF_ERROR_CODE(0, 0x2647)
 #define TSDB_CODE_PAR_INVALID_PASSWD            TAOS_DEF_ERROR_CODE(0, 0x2648)
 #define TSDB_CODE_PAR_INVALID_ALTER_TABLE       TAOS_DEF_ERROR_CODE(0, 0x2649)
+#define TSDB_CODE_PAR_CANNOT_DROP_PRIMARY_KEY   TAOS_DEF_ERROR_CODE(0, 0x264A)
+#define TSDB_CODE_PAR_INVALID_MODIFY_COL        TAOS_DEF_ERROR_CODE(0, 0x264B)
 
 //planner
 #define TSDB_CODE_PLAN_INTERNAL_ERROR           TAOS_DEF_ERROR_CODE(0, 0x2700)

include/util/tcoding.h

@@ -63,14 +63,14 @@ static FORCE_INLINE void *taosSkipFixedLen(const void *buf, size_t len) { return
 
 static FORCE_INLINE int32_t taosEncodeFixedBool(void **buf, bool value) {
   if (buf != NULL) {
-    ((int8_t *)(*buf))[0] = value ? 1 : 0;
+    ((int8_t *)(*buf))[0] = (value ? 1 : 0);
     *buf = POINTER_SHIFT(*buf, sizeof(int8_t));
   }
   return (int32_t)sizeof(int8_t);
 }
 
 static FORCE_INLINE void *taosDecodeFixedBool(const void *buf, bool *value) {
-  *value = ((int8_t *)buf)[0] == 0 ? false : true;
+  *value = ( (((int8_t *)buf)[0] == 0) ? false : true );
   return POINTER_SHIFT(buf, sizeof(int8_t));
 }
 

include/util/thash.h

@@ -40,6 +40,7 @@ typedef void (*_hash_free_fn_t)(void *);
  */
 uint32_t MurmurHash3_32(const char *key, uint32_t len);
 
+uint64_t MurmurHash3_64(const char *key, uint32_t len);
 /**
  *
  * @param key

include/util/tjson.h

@@ -25,7 +25,7 @@ extern "C" {
 #define tjsonGetNumberValue(pJson, pName, val, code)  \
   do {                                                \
     uint64_t _tmp = 0;                                \
-    code = tjsonGetUBigIntValue(pJson, pName, &_tmp); \
+    code = tjsonGetBigIntValue(pJson, pName, &_tmp); \
     val = _tmp;                                       \
   } while (0)
 

source/client/src/clientImpl.c

@@ -233,8 +233,7 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra
                       .pAstRoot = pQuery->pRoot,
                       .showRewrite = pQuery->showRewrite,
                       .pMsg = pRequest->msgBuf,
-                      .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE,
+                      .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE};
-                      .placeholderNum = pQuery->placeholderNum};
   SEpSet       mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
   SCatalog*    pCatalog = NULL;
   int32_t      code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
@@ -518,7 +517,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t
   if (pRequest->code != TSDB_CODE_SUCCESS) {
     const char* errorMsg =
         (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(pRequest->code);
-    fprintf(stderr,"failed to connect to server, reason: %s\n\n", errorMsg);
+    fprintf(stderr, "failed to connect to server, reason: %s\n\n", errorMsg);
 
     terrno = pRequest->code;
     destroyRequest(pRequest);
@@ -949,7 +948,8 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR
 
   // TODO handle the compressed case
   pResultInfo->totalRows += pResultInfo->numOfRows;
-  return setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows, convertUcs4);
+  return setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows,
+                          convertUcs4);
 }
 
 TSDB_SERVER_STATUS taos_check_server_status(const char* fqdn, int port, char* details, int maxlen) {

source/client/src/clientStmt.c

@@ -67,7 +67,7 @@ int32_t stmtGetTbName(TAOS_STMT* stmt, char** tbName) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
   pStmt->sql.type = STMT_TYPE_MULTI_INSERT;
-  
+
   if ('\0' == pStmt->bInfo.tbName[0]) {
     tscError("no table name set");
     STMT_ERR_RET(TSDB_CODE_TSC_STMT_TBNAME_ERROR);
@@ -126,7 +126,7 @@ int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags,
 
   strncpy(pStmt->bInfo.tbFName, tbFName, sizeof(pStmt->bInfo.tbFName) - 1);
   pStmt->bInfo.tbFName[sizeof(pStmt->bInfo.tbFName) - 1] = 0;
-  
+
   pStmt->bInfo.tbUid = pTableMeta->uid;
   pStmt->bInfo.tbSuid = pTableMeta->suid;
   pStmt->bInfo.tbType = pTableMeta->tableType;
@@ -146,18 +146,18 @@ int32_t stmtUpdateExecInfo(TAOS_STMT* stmt, SHashObj* pVgHash, SHashObj* pBlockH
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t stmtUpdateInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, char* tbFName, bool autoCreateTbl, SHashObj* pVgHash, SHashObj* pBlockHash) {
+int32_t stmtUpdateInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, char* tbFName, bool autoCreateTbl,
+                       SHashObj* pVgHash, SHashObj* pBlockHash) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
   STMT_ERR_RET(stmtUpdateBindInfo(stmt, pTableMeta, tags, tbFName));
   STMT_ERR_RET(stmtUpdateExecInfo(stmt, pVgHash, pBlockHash, autoCreateTbl));
 
   pStmt->sql.autoCreateTbl = autoCreateTbl;
-  
+
   return TSDB_CODE_SUCCESS;
 }
 
-
 int32_t stmtGetExecInfo(TAOS_STMT* stmt, SHashObj** pVgHash, SHashObj** pBlockHash) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
@@ -172,7 +172,7 @@ int32_t stmtCacheBlock(STscStmt* pStmt) {
     return TSDB_CODE_SUCCESS;
   }
 
-  uint64_t uid = pStmt->bInfo.tbUid; 
+  uint64_t uid = pStmt->bInfo.tbUid;
   uint64_t cacheUid = (TSDB_CHILD_TABLE == pStmt->bInfo.tbType) ? pStmt->bInfo.tbSuid : uid;
 
   if (taosHashGet(pStmt->sql.pTableCache, &cacheUid, sizeof(cacheUid))) {
@@ -180,8 +180,8 @@ int32_t stmtCacheBlock(STscStmt* pStmt) {
   }
 
   STableDataBlocks** pSrc = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
-  STableDataBlocks* pDst = NULL;
+  STableDataBlocks*  pDst = NULL;
-  
+
   STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc));
 
   SStmtTableCache cache = {
@@ -198,16 +198,16 @@ int32_t stmtCacheBlock(STscStmt* pStmt) {
   } else {
     pStmt->bInfo.boundTags = NULL;
   }
-  
+
   return TSDB_CODE_SUCCESS;
 }
 
 int32_t stmtParseSql(STscStmt* pStmt) {
   SStmtCallback stmtCb = {
-    .pStmt = pStmt, 
+      .pStmt = pStmt,
-    .getTbNameFn = stmtGetTbName, 
+      .getTbNameFn = stmtGetTbName,
-    .setInfoFn = stmtUpdateInfo,
+      .setInfoFn = stmtUpdateInfo,
-    .getExecInfoFn = stmtGetExecInfo,
+      .getExecInfoFn = stmtGetExecInfo,
   };
 
   if (NULL == pStmt->exec.pRequest) {
@@ -259,12 +259,12 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool freeRequest) {
   }
 
   size_t keyLen = 0;
-  void *pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL);
+  void*  pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL);
   while (pIter) {
-    STableDataBlocks* pBlocks = *(STableDataBlocks**)pIter;    
+    STableDataBlocks* pBlocks = *(STableDataBlocks**)pIter;
-    char *key = taosHashGetKey(pIter, &keyLen);
+    char*             key = taosHashGetKey(pIter, &keyLen);
-    STableMeta* pMeta = qGetTableMetaInDataBlock(pBlocks);
+    STableMeta*       pMeta = qGetTableMetaInDataBlock(pBlocks);
-    
+
     if (keepTable && (strlen(pStmt->bInfo.tbFName) == keyLen) && strncmp(pStmt->bInfo.tbFName, key, keyLen) == 0) {
       STMT_ERR_RET(qResetStmtDataBlock(pBlocks, true));
 
@@ -279,7 +279,7 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool freeRequest) {
   }
 
   pStmt->exec.autoCreateTbl = false;
-  
+
   if (keepTable) {
     return TSDB_CODE_SUCCESS;
   }
@@ -320,13 +320,15 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataBlocks *pDataBlock, STableDataBlocks **newBlock, uint64_t uid) {
+int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataBlocks* pDataBlock, STableDataBlocks** newBlock, uint64_t uid) {
-  SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp);
+  SEpSet      ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp);
   SVgroupInfo vgInfo = {0};
-  
+
-  STMT_ERR_RET(catalogGetTableHashVgroup(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &vgInfo));
+  STMT_ERR_RET(catalogGetTableHashVgroup(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname,
-  STMT_ERR_RET(taosHashPut(pStmt->exec.pVgHash, (const char*)&vgInfo.vgId, sizeof(vgInfo.vgId), (char*)&vgInfo, sizeof(vgInfo)));
+                                         &vgInfo));
-  
+  STMT_ERR_RET(
+      taosHashPut(pStmt->exec.pVgHash, (const char*)&vgInfo.vgId, sizeof(vgInfo.vgId), (char*)&vgInfo, sizeof(vgInfo)));
+
   STMT_ERR_RET(qRebuildStmtDataBlock(newBlock, pDataBlock, uid, vgInfo.vgId));
 
   return TSDB_CODE_SUCCESS;
@@ -335,8 +337,9 @@ int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataBlocks *pDataBlock, STab
 int32_t stmtGetFromCache(STscStmt* pStmt) {
   pStmt->bInfo.needParse = true;
   pStmt->bInfo.inExecCache = false;
-  
+
-  STableDataBlocks *pBlockInExec = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
+  STableDataBlocks* pBlockInExec =
+      taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
   if (pBlockInExec) {
     pStmt->bInfo.needParse = false;
     pStmt->bInfo.inExecCache = true;
@@ -352,7 +355,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
       pStmt->bInfo.needParse = false;
       return TSDB_CODE_SUCCESS;
     }
-    
+
     return TSDB_CODE_SUCCESS;
   }
 
@@ -367,24 +370,25 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
       pStmt->exec.autoCreateTbl = true;
 
       pStmt->bInfo.tbUid = 0;
-      
+
       STableDataBlocks* pNewBlock = NULL;
       STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataBlock, &pNewBlock, 0));
-      
+
-      if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, POINTER_BYTES)) {
+      if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock,
+                      POINTER_BYTES)) {
         STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
       }
-      
+
       return TSDB_CODE_SUCCESS;
     }
-    
+
     STMT_RET(stmtCleanBindInfo(pStmt));
   }
-  
 
-  STableMeta *pTableMeta = NULL;
+  STableMeta* pTableMeta = NULL;
-  SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp);
+  SEpSet      ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp);
-  int32_t code = catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta);
+  int32_t     code =
+      catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta);
   if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) {
     STMT_ERR_RET(stmtCleanBindInfo(pStmt));
 
@@ -398,7 +402,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
   int8_t   tableType = pTableMeta->tableType;
   taosMemoryFree(pTableMeta);
   uint64_t cacheUid = (TSDB_CHILD_TABLE == tableType) ? suid : uid;
-  
+
   if (uid == pStmt->bInfo.tbUid) {
     pStmt->bInfo.needParse = false;
 
@@ -408,8 +412,9 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
   if (pStmt->bInfo.inExecCache) {
     SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &cacheUid, sizeof(cacheUid));
     if (NULL == pCache) {
-      tscError("table [%s, %" PRIx64 ", %" PRIx64 "] found in exec blockHash, but not in sql blockHash", pStmt->bInfo.tbFName, uid, cacheUid);
+      tscError("table [%s, %" PRIx64 ", %" PRIx64 "] found in exec blockHash, but not in sql blockHash",
-      
+               pStmt->bInfo.tbFName, uid, cacheUid);
+
       STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR);
     }
 
@@ -437,7 +442,8 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
     STableDataBlocks* pNewBlock = NULL;
     STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataBlock, &pNewBlock, uid));
 
-    if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, POINTER_BYTES)) {
+    if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock,
+                    POINTER_BYTES)) {
       STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
 
@@ -521,10 +527,11 @@ int stmtSetTbName(TAOS_STMT* stmt, const char* tbName) {
   if (NULL == pStmt->exec.pRequest) {
     STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
   }
-  
+
-  STMT_ERR_RET(qCreateSName(&pStmt->bInfo.sname, tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen));
+  STMT_ERR_RET(qCreateSName(&pStmt->bInfo.sname, tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb,
+                            pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen));
   tNameExtractFullName(&pStmt->bInfo.sname, pStmt->bInfo.tbFName);
-  
+
   STMT_ERR_RET(stmtGetFromCache(pStmt));
 
   if (pStmt->bInfo.needParse) {
@@ -548,7 +555,8 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) {
     return TSDB_CODE_SUCCESS;
   }
 
-  STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
+  STableDataBlocks** pDataBlock =
+      (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
   if (NULL == pDataBlock) {
     tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName);
     STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
@@ -566,7 +574,8 @@ int32_t stmtFetchTagFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD** fiel
     STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
   }
 
-  STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
+  STableDataBlocks** pDataBlock =
+      (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
   if (NULL == pDataBlock) {
     tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName);
     STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
@@ -583,7 +592,8 @@ int32_t stmtFetchColFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD** fiel
     STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
   }
 
-  STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
+  STableDataBlocks** pDataBlock =
+      (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
   if (NULL == pDataBlock) {
     tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName);
     STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
@@ -618,8 +628,8 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
   }
 
   if (STMT_TYPE_QUERY == pStmt->sql.type) {
-    STMT_ERR_RET(qStmtBindParams(pStmt->sql.pQuery, bind, colIdx, pStmt->exec.pRequest->requestId));
+    STMT_ERR_RET(qStmtBindParams(pStmt->sql.pQuery, bind, colIdx));
-    
+
     SParseContext ctx = {.requestId = pStmt->exec.pRequest->requestId,
                          .acctId = pStmt->taos->acctId,
                          .db = pStmt->exec.pRequest->pDb,
@@ -633,27 +643,29 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
                          .pUser = pStmt->taos->user};
     ctx.mgmtEpSet = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp);
     STMT_ERR_RET(catalogGetHandle(pStmt->taos->pAppInfo->clusterId, &ctx.pCatalog));
-    
+
     STMT_ERR_RET(qStmtParseQuerySql(&ctx, pStmt->sql.pQuery));
 
     if (pStmt->sql.pQuery->haveResultSet) {
-      setResSchemaInfo(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->pResSchema, pStmt->sql.pQuery->numOfResCols);
+      setResSchemaInfo(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->pResSchema,
+                       pStmt->sql.pQuery->numOfResCols);
       setResPrecision(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->precision);
     }
-  
+
     TSWAP(pStmt->exec.pRequest->dbList, pStmt->sql.pQuery->pDbList);
-    TSWAP(pStmt->exec.pRequest->tableList, pStmt->sql.pQuery->pTableList);  
+    TSWAP(pStmt->exec.pRequest->tableList, pStmt->sql.pQuery->pTableList);
 
-    //if (STMT_TYPE_QUERY == pStmt->sql.queryRes) {
+    // if (STMT_TYPE_QUERY == pStmt->sql.queryRes) {
-    //  STMT_ERR_RET(stmtRestoreQueryFields(pStmt));
+    //   STMT_ERR_RET(stmtRestoreQueryFields(pStmt));
-    //}
+    // }
 
-    //STMT_ERR_RET(stmtBackupQueryFields(pStmt));
+    // STMT_ERR_RET(stmtBackupQueryFields(pStmt));
 
     return TSDB_CODE_SUCCESS;
   }
- 
+
-  STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
+  STableDataBlocks** pDataBlock =
+      (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
   if (NULL == pDataBlock) {
     tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName);
     STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
@@ -694,19 +706,19 @@ int stmtAddBatch(TAOS_STMT* stmt) {
   return TSDB_CODE_SUCCESS;
 }
 
-int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp *pRsp) {
+int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp* pRsp) {
   if (pRsp->nBlocks <= 0) {
     tscError("invalid submit resp block number %d", pRsp->nBlocks);
     STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR);
   }
 
-  size_t keyLen = 0;
+  size_t             keyLen = 0;
-  STableDataBlocks **pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL);
+  STableDataBlocks** pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL);
   while (pIter) {
-    STableDataBlocks *pBlock = *pIter;
+    STableDataBlocks* pBlock = *pIter;
-    char *key = taosHashGetKey(pIter, &keyLen);
+    char*             key = taosHashGetKey(pIter, &keyLen);
-  
+
-    STableMeta *pMeta = qGetTableMetaInDataBlock(pBlock);
+    STableMeta* pMeta = qGetTableMetaInDataBlock(pBlock);
     if (pMeta->uid != pStmt->bInfo.tbUid) {
       tscError("table uid %" PRIx64 " mis-match with current table uid %" PRIx64, pMeta->uid, pStmt->bInfo.tbUid);
       STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR);
@@ -717,24 +729,25 @@ int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp *pRsp) {
       continue;
     }
 
-    SSubmitBlkRsp *blkRsp = NULL;
+    SSubmitBlkRsp* blkRsp = NULL;
-    int32_t i = 0;
+    int32_t        i = 0;
     for (; i < pRsp->nBlocks; ++i) {
       blkRsp = pRsp->pBlocks + i;
       if (strlen(blkRsp->tblFName) != keyLen) {
         continue;
       }
-      
+
       if (strncmp(blkRsp->tblFName, key, keyLen)) {
         continue;
       }
-    
+
       break;
     }
 
     if (i < pRsp->nBlocks) {
-      tscDebug("auto created table %s uid updated from %" PRIx64 " to %" PRIx64, blkRsp->tblFName, pMeta->uid, blkRsp->uid);
+      tscDebug("auto created table %s uid updated from %" PRIx64 " to %" PRIx64, blkRsp->tblFName, pMeta->uid,
-      
+               blkRsp->uid);
+
       pMeta->uid = blkRsp->uid;
       pStmt->bInfo.tbUid = blkRsp->uid;
     } else {
@@ -748,11 +761,11 @@ int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp *pRsp) {
   return TSDB_CODE_SUCCESS;
 }
 
-int stmtExec(TAOS_STMT *stmt) {
+int stmtExec(TAOS_STMT* stmt) {
-  STscStmt* pStmt = (STscStmt*)stmt;
+  STscStmt*   pStmt = (STscStmt*)stmt;
-  int32_t code = 0;
+  int32_t     code = 0;
-  SSubmitRsp *pRsp = NULL;
+  SSubmitRsp* pRsp = NULL;
-  bool autoCreateTbl = pStmt->exec.autoCreateTbl;
+  bool        autoCreateTbl = pStmt->exec.autoCreateTbl;
 
   STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE));
 
@@ -760,7 +773,8 @@ int stmtExec(TAOS_STMT *stmt) {
     launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true, NULL);
   } else {
     STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->exec.pVgHash, pStmt->exec.pBlockHash));
-    launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true, (autoCreateTbl ? (void**)&pRsp : NULL));
+    launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true,
+                    (autoCreateTbl ? (void**)&pRsp : NULL));
   }
 
   if (pStmt->exec.pRequest->code && NEED_CLIENT_HANDLE_ERROR(pStmt->exec.pRequest->code)) {
@@ -787,10 +801,10 @@ _return:
       tscError("no submit resp got for auto create table");
       STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR);
     }
-    
+
     STMT_ERR_RET(stmtUpdateTableUid(pStmt, pRsp));
   }
-  
+
   ++pStmt->sql.runTimes;
 
   STMT_RET(code);

source/common/src/tglobal.c

@@ -434,7 +434,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
 
   tsRpcQueueMemoryAllowed = tsTotalMemoryKB * 1024 * 0.1;
   tsRpcQueueMemoryAllowed = TRANGE(tsRpcQueueMemoryAllowed, TSDB_MAX_WAL_SIZE * 10L, TSDB_MAX_WAL_SIZE * 10000L);
-  if (cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsRpcQueueMemoryAllowed, 1, INT64_MAX, 0) != 0) return -1;
+  if (cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsRpcQueueMemoryAllowed, TSDB_MAX_WAL_SIZE * 10L, INT64_MAX, 0) != 0)
+    return -1;
 
   if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, 0) != 0) return -1;

source/dnode/mgmt/mgmt_mnode/src/mmHandle.c

@@ -220,6 +220,7 @@ SArray *mmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_COMMIT_OFFSET, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_ASK_EP, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_STREAM, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_DEPLOY_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_GET_DB_CFG, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER;

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -308,6 +308,7 @@ SArray *vmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_SUBMIT_RSMA, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_CONSUME, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_DEPLOY, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_HEARTBEAT, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER;

source/dnode/mgmt/test/sut/src/sut.cpp

@@ -30,6 +30,7 @@ void Testbase::InitLog(const char* path) {
   tsdbDebugFlag = 0;
   tsLogEmbedded = 1;
   tsAsyncLog = 0;
+  tsRpcQueueMemoryAllowed = 1024 * 1024 * 64;
 
   taosRemoveDir(path);
   taosMkDir(path);
@@ -83,7 +84,7 @@ SRpcMsg* Testbase::SendReq(tmsg_t msgType, void* pCont, int32_t contLen) {
   return client.SendReq(&rpcMsg);
 }
 
-int32_t Testbase::SendShowReq(int8_t showType, const char *tb, const char* db) {
+int32_t Testbase::SendShowReq(int8_t showType, const char* tb, const char* db) {
   if (showRsp != NULL) {
     rpcFreeCont(showRsp);
     showRsp = NULL;

source/dnode/mnode/impl/inc/mndDef.h

@@ -457,6 +457,7 @@ typedef struct {
   char*          ast;
   char*          physicalPlan;
   SSchemaWrapper schema;
+  int32_t        refConsumerCnt;
 } SMqTopicObj;
 
 typedef struct {

source/dnode/mnode/impl/inc/mndOffset.h

@@ -38,6 +38,9 @@ static FORCE_INLINE int32_t mndMakePartitionKey(char *key, const char *cgroup, c
 }
 
 int32_t mndDropOffsetByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
+int32_t mndDropOffsetByTopic(SMnode *pMnode, STrans *pTrans, const char *topic);
+
+bool mndOffsetFromTopic(SMqOffsetObj *pOffset, const char *topic);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/inc/mndSubscribe.h

@@ -32,6 +32,7 @@ void             mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub);
 int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName);
 
 int32_t mndDropSubByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
+int32_t mndDropSubByTopic(SMnode *pMnode, STrans *pTrans, const char *topic);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/inc/mndTopic.h

@@ -35,6 +35,8 @@ int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
 
 const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]);
 
+int32_t mndSetTopicRedoLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/mnode/impl/src/mndConsumer.c

@@ -399,6 +399,9 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
 
   int32_t newTopicNum = taosArrayGetSize(newSub);
   // check topic existance
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_SUBSCRIBE, &pMsg->rpcMsg);
+  if (pTrans == NULL) goto SUBSCRIBE_OVER;
+
   for (int32_t i = 0; i < newTopicNum; i++) {
     char        *topic = taosArrayGetP(newSub, i);
     SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
@@ -406,7 +409,14 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
       terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST;
       goto SUBSCRIBE_OVER;
     }
-    // TODO lock topic to prevent drop
+
+    // ref topic to prevent drop
+    // TODO make topic complete
+    SMqTopicObj topicObj = {0};
+    memcpy(&topicObj, pTopic, sizeof(SMqTopicObj));
+    topicObj.refConsumerCnt = pTopic->refConsumerCnt + 1;
+    if (mndSetTopicRedoLogs(pMnode, pTrans, &topicObj) != 0) goto SUBSCRIBE_OVER;
+
     mndReleaseTopic(pMnode, pTopic);
   }
 
@@ -422,8 +432,6 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
       taosArrayPush(pConsumerNew->assignedTopics, &newTopicCopy);
     }
 
-    STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_SUBSCRIBE, pMsg);
-    if (pTrans == NULL) goto SUBSCRIBE_OVER;
     if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto SUBSCRIBE_OVER;
     if (mndTransPrepare(pMnode, pTrans) != 0) goto SUBSCRIBE_OVER;
 
@@ -494,8 +502,6 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
       goto SUBSCRIBE_OVER;
     }
 
-    STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_SUBSCRIBE, pMsg);
-    if (pTrans == NULL) goto SUBSCRIBE_OVER;
     if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto SUBSCRIBE_OVER;
     if (mndTransPrepare(pMnode, pTrans) != 0) goto SUBSCRIBE_OVER;
   }
@@ -503,6 +509,8 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
   code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
 
 SUBSCRIBE_OVER:
+  mndTransDrop(pTrans);
+
   if (pConsumerOld) {
     /*taosRUnLockLatch(&pConsumerOld->lock);*/
     mndReleaseConsumer(pMnode, pConsumerOld);

source/dnode/mnode/impl/src/mndOffset.c

@@ -50,6 +50,14 @@ int32_t mndInitOffset(SMnode *pMnode) {
 
 void mndCleanupOffset(SMnode *pMnode) {}
 
+bool mndOffsetFromTopic(SMqOffsetObj *pOffset, const char *topic) {
+  int32_t i = 0;
+  while (pOffset->key[i] != ':') i++;
+  while (pOffset->key[i] != ':') i++;
+  if (strcmp(&pOffset->key[i + 1], topic) == 0) return true;
+  return false;
+}
+
 SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset) {
   terrno = TSDB_CODE_OUT_OF_MEMORY;
   void   *buf = NULL;
@@ -134,10 +142,11 @@ int32_t mndCreateOffsets(STrans *pTrans, const char *cgroup, const char *topicNa
   int32_t sz = taosArrayGetSize(vgs);
   for (int32_t i = 0; i < sz; i++) {
     int32_t      vgId = *(int32_t *)taosArrayGet(vgs, i);
-    SMqOffsetObj offsetObj;
+    SMqOffsetObj offsetObj = {0};
     if (mndMakePartitionKey(offsetObj.key, cgroup, topicName, vgId) < 0) {
       return -1;
     }
+    // TODO assign db
     offsetObj.offset = -1;
     SSdbRaw *pOffsetRaw = mndOffsetActionEncode(&offsetObj);
     if (pOffsetRaw == NULL) {
@@ -240,6 +249,14 @@ static int32_t mndSetDropOffsetCommitLogs(SMnode *pMnode, STrans *pTrans, SMqOff
   return 0;
 }
 
+static int32_t mndSetDropOffsetRedoLogs(SMnode *pMnode, STrans *pTrans, SMqOffsetObj *pOffset) {
+  SSdbRaw *pRedoRaw = mndOffsetActionEncode(pOffset);
+  if (pRedoRaw == NULL) return -1;
+  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
+  if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPED) != 0) return -1;
+  return 0;
+}
+
 int32_t mndDropOffsetByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
   int32_t code = -1;
   SSdb   *pSdb = pMnode->pSdb;
@@ -247,7 +264,7 @@ int32_t mndDropOffsetByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
   void         *pIter = NULL;
   SMqOffsetObj *pOffset = NULL;
   while (1) {
-    pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, pIter, (void **)&pOffset);
+    pIter = sdbFetch(pSdb, SDB_OFFSET, pIter, (void **)&pOffset);
     if (pIter == NULL) break;
 
     if (pOffset->dbUid != pDb->uid) {
@@ -256,8 +273,39 @@ int32_t mndDropOffsetByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
     }
 
     if (mndSetDropOffsetCommitLogs(pMnode, pTrans, pOffset) < 0) {
+      sdbRelease(pSdb, pOffset);
       goto END;
     }
+
+    sdbRelease(pSdb, pOffset);
+  }
+
+  code = 0;
+END:
+  return code;
+}
+
+int32_t mndDropOffsetByTopic(SMnode *pMnode, STrans *pTrans, const char *topic) {
+  int32_t code = -1;
+  SSdb   *pSdb = pMnode->pSdb;
+
+  void         *pIter = NULL;
+  SMqOffsetObj *pOffset = NULL;
+  while (1) {
+    pIter = sdbFetch(pSdb, SDB_OFFSET, pIter, (void **)&pOffset);
+    if (pIter == NULL) break;
+
+    if (!mndOffsetFromTopic(pOffset, topic)) {
+      sdbRelease(pSdb, pOffset);
+      continue;
+    }
+
+    if (mndSetDropOffsetRedoLogs(pMnode, pTrans, pOffset) < 0) {
+      sdbRelease(pSdb, pOffset);
+      goto END;
+    }
+
+    sdbRelease(pSdb, pOffset);
   }
 
   code = 0;

source/dnode/mnode/impl/src/mndSma.c

@@ -530,6 +530,7 @@ static int32_t mndProcessMCreateSmaReq(SRpcMsg *pReq) {
   pStream = mndAcquireStream(pMnode, createReq.name);
   if (pStream != NULL) {
     mError("sma:%s, failed to create since stream:%s already exist", createReq.name, createReq.name);
+    terrno = TSDB_CODE_MND_STREAM_ALREADY_EXIST;
     goto _OVER;
   }
 
@@ -565,7 +566,7 @@ static int32_t mndProcessMCreateSmaReq(SRpcMsg *pReq) {
 
 _OVER:
   if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
-    mError("sma:%s, failed to create since %s", createReq.name, terrstr());
+    mError("sma:%s, failed to create since %s", createReq.name, terrstr(terrno));
   }
 
   mndReleaseStb(pMnode, pStb);

source/dnode/mnode/impl/src/mndStb.c

@@ -386,7 +386,7 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
   req.schema.sver = pStb->version;
   req.schema.pSchema = pStb->pColumns;
   req.schemaTag.nCols = pStb->numOfTags;
-  req.schemaTag.nCols = 0;
+  req.schemaTag.sver = 1;
   req.schemaTag.pSchema = pStb->pTags;
 
   if (req.rollup) {

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -73,6 +73,7 @@ int32_t mndInitSubscribe(SMnode *pMnode) {
                      .deleteFp = (SdbDeleteFp)mndSubActionDelete};
 
   mndSetMsgHandle(pMnode, TDMT_VND_MQ_VG_CHANGE_RSP, mndProcessSubscribeInternalRsp);
+  mndSetMsgHandle(pMnode, TDMT_VND_MQ_VG_DELETE_RSP, mndProcessSubscribeInternalRsp);
   mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessRebalanceReq);
 
   mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_SUBSCRIPTIONS, mndRetrieveSubscribe);
@@ -389,7 +390,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
 }
 
 static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOutputObj *pOutput) {
-  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_REBALANCE, pMsg);
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_REBALANCE, &pMsg->rpcMsg);
   if (pTrans == NULL) {
     return -1;
   }
@@ -458,6 +459,20 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu
       goto REB_FAIL;
     }
   }
+  if (consumerNum) {
+    char topic[TSDB_TOPIC_FNAME_LEN];
+    char cgroup[TSDB_CGROUP_LEN];
+    mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup, true);
+    SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
+    if (pTopic) {
+      // TODO make topic complete
+      SMqTopicObj topicObj = {0};
+      memcpy(&topicObj, pTopic, sizeof(SMqTopicObj));
+      topicObj.refConsumerCnt = pTopic->refConsumerCnt - consumerNum;
+      if (mndSetTopicRedoLogs(pMnode, pTrans, &topicObj) != 0) goto REB_FAIL;
+    }
+  }
+
   // 4. TODO commit log: modification log
 
   // 5. set cb
@@ -688,6 +703,14 @@ static int32_t mndProcessSubscribeInternalRsp(SRpcMsg *pRsp) {
   return 0;
 }
 
+static int32_t mndSetDropSubRedoLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) {
+  SSdbRaw *pRedoRaw = mndSubActionEncode(pSub);
+  if (pRedoRaw == NULL) return -1;
+  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
+  if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPED) != 0) return -1;
+  return 0;
+}
+
 static int32_t mndSetDropSubCommitLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) {
   SSdbRaw *pCommitRaw = mndSubActionEncode(pSub);
   if (pCommitRaw == NULL) return -1;
@@ -712,6 +735,57 @@ int32_t mndDropSubByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
     }
 
     if (mndSetDropSubCommitLogs(pMnode, pTrans, pSub) < 0) {
+      sdbRelease(pSdb, pSub);
+      goto END;
+    }
+  }
+
+  code = 0;
+END:
+  return code;
+}
+
+int32_t mndDropSubByTopic(SMnode *pMnode, STrans *pTrans, const char *topicName) {
+  int32_t code = -1;
+  SSdb   *pSdb = pMnode->pSdb;
+
+  void            *pIter = NULL;
+  SMqSubscribeObj *pSub = NULL;
+  while (1) {
+    pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, pIter, (void **)&pSub);
+    if (pIter == NULL) break;
+
+    char topic[TSDB_TOPIC_FNAME_LEN];
+    char cgroup[TSDB_CGROUP_LEN];
+    mndSplitSubscribeKey(pSub->key, topic, cgroup, true);
+    if (strcmp(topic, topicName) != 0) {
+      sdbRelease(pSdb, pSub);
+      continue;
+    }
+
+    // iter all vnode to delete handle
+    ASSERT(taosHashGetSize(pSub->consumerHash) == 0);
+    int32_t sz = taosArrayGetSize(pSub->unassignedVgs);
+    for (int32_t i = 0; i < sz; i++) {
+      SMqVgEp       *pVgEp = taosArrayGetP(pSub->unassignedVgs, i);
+      SMqVDeleteReq *pReq = taosMemoryCalloc(1, sizeof(SMqVDeleteReq));
+      pReq->head.vgId = htonl(pVgEp->vgId);
+      pReq->vgId = pVgEp->vgId;
+      pReq->consumerId = -1;
+      memcpy(pReq->subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN);
+      STransAction action = {0};
+      action.epSet = pVgEp->epSet;
+      action.pCont = pReq;
+      action.contLen = sizeof(SMqVDeleteReq);
+      action.msgType = TDMT_VND_MQ_VG_DELETE;
+      if (mndTransAppendRedoAction(pTrans, &action) != 0) {
+        taosMemoryFree(pReq);
+        return -1;
+      }
+    }
+
+    if (mndSetDropSubRedoLogs(pMnode, pTrans, pSub) < 0) {
+      sdbRelease(pSdb, pSub);
       goto END;
     }
   }

source/dnode/mnode/impl/src/mndTopic.c

@@ -18,8 +18,10 @@
 #include "mndDb.h"
 #include "mndDnode.h"
 #include "mndMnode.h"
+#include "mndOffset.h"
 #include "mndShow.h"
 #include "mndStb.h"
+#include "mndSubscribe.h"
 #include "mndTrans.h"
 #include "mndUser.h"
 #include "mndVgroup.h"
@@ -106,6 +108,7 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
   taosEncodeSSchemaWrapper(&aswBuf, &pTopic->schema);
   SDB_SET_INT32(pRaw, dataPos, schemaLen, TOPIC_ENCODE_OVER);
   SDB_SET_BINARY(pRaw, dataPos, swBuf, schemaLen, TOPIC_ENCODE_OVER);
+  SDB_SET_INT32(pRaw, dataPos, pTopic->refConsumerCnt, TOPIC_ENCODE_OVER);
 
   SDB_SET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_ENCODE_OVER);
   SDB_SET_DATALEN(pRaw, dataPos, TOPIC_ENCODE_OVER);
@@ -190,6 +193,8 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
     goto TOPIC_DECODE_OVER;
   }
 
+  SDB_GET_INT32(pRaw, dataPos, &pTopic->refConsumerCnt, TOPIC_DECODE_OVER);
+
   SDB_GET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_DECODE_OVER);
 
   terrno = TSDB_CODE_SUCCESS;
@@ -220,11 +225,13 @@ static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pOldTopic, SMqTopic
   atomic_exchange_64(&pOldTopic->updateTime, pNewTopic->updateTime);
   atomic_exchange_32(&pOldTopic->version, pNewTopic->version);
 
-  taosWLockLatch(&pOldTopic->lock);
+  atomic_store_32(&pOldTopic->refConsumerCnt, pNewTopic->refConsumerCnt);
+
+  /*taosWLockLatch(&pOldTopic->lock);*/
 
   // TODO handle update
 
-  taosWUnLockLatch(&pOldTopic->lock);
+  /*taosWUnLockLatch(&pOldTopic->lock);*/
   return 0;
 }
 
@@ -292,6 +299,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
   topicObj.version = 1;
   topicObj.sql = strdup(pCreate->sql);
   topicObj.sqlLen = strlen(pCreate->sql) + 1;
+  topicObj.refConsumerCnt = 0;
 
   if (pCreate->ast && pCreate->ast[0]) {
     topicObj.ast = strdup(pCreate->ast);
@@ -436,15 +444,7 @@ CREATE_TOPIC_OVER:
   return code;
 }
 
-static int32_t mndDropTopic(SMnode *pMnode, SRpcMsg *pReq, SMqTopicObj *pTopic) {
+static int32_t mndDropTopic(SMnode *pMnode, STrans *pTrans, SRpcMsg *pReq, SMqTopicObj *pTopic) {
-  // TODO: cannot drop when subscribed
-  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_TOPIC, pReq);
-  if (pTrans == NULL) {
-    mError("topic:%s, failed to drop since %s", pTopic->name, terrstr());
-    return -1;
-  }
-  mDebug("trans:%d, used to drop topic:%s", pTrans->id, pTopic->name);
-
   SSdbRaw *pRedoRaw = mndTopicActionEncode(pTopic);
   if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
     mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
@@ -464,31 +464,42 @@ static int32_t mndDropTopic(SMnode *pMnode, SRpcMsg *pReq, SMqTopicObj *pTopic)
 }
 
 static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
-  SMnode        *pMnode = pReq->info.node;
+  SMnode        *pMnode = pReq->pNode;
+  SSdb          *pSdb = pMnode->pSdb;
   SMDropTopicReq dropReq = {0};
 
   if (tDeserializeSMDropTopicReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
+    }
+  }
+
+  if (pTopic->refConsumerCnt != 0) {
+    terrno = TSDB_CODE_MND_TOPIC_SUBSCRIBED;
+    mError("topic:%s, failed to drop since %s", dropReq.name, terrstr());
     return -1;
   }
 
-  mDebug("topic:%s, start to drop", dropReq.name);
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_TOPIC, &pReq->rpcMsg);
-
+  if (pTrans == NULL) {
-  SMqTopicObj *pTopic = mndAcquireTopic(pMnode, dropReq.name);
+    mError("topic:%s, failed to drop since %s", pTopic->name, terrstr());
-  if (pTopic == NULL) {
+    return -1;
-    if (dropReq.igNotExists) {
-      mDebug("topic:%s, not exist, ignore not exist is set", dropReq.name);
-      return 0;
-    } else {
-      terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST;
-      mError("topic:%s, failed to drop since %s", dropReq.name, terrstr());
-      return -1;
-    }
   }
-  // TODO: check ref
 
-  int32_t code = mndDropTopic(pMnode, pReq, pTopic);
+  mDebug("trans:%d, used to drop topic:%s", pTrans->id, pTopic->name);
-  // TODO: iterate and drop related subscriptions and offsets
+
+#if 1
+  if (mndDropOffsetByTopic(pMnode, pTrans, dropReq.name) < 0) {
+    ASSERT(0);
+    return -1;
+  }
+#endif
+
+  if (mndDropSubByTopic(pMnode, pTrans, dropReq.name) < 0) {
+    ASSERT(0);
+    return -1;
+  }
+
+  int32_t code = mndDropTopic(pMnode, pTrans, pReq, pTopic);
   mndReleaseTopic(pMnode, pTopic);
 
   if (code != 0) {
@@ -577,6 +588,15 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
   return numOfRows;
 }
 
+int32_t mndSetTopicRedoLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic) {
+  SSdbRaw *pRedoRaw = mndTopicActionEncode(pTopic);
+  if (pRedoRaw == NULL) return -1;
+  if (mndTransAppendCommitlog(pTrans, pRedoRaw) != 0) return -1;
+  if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1;
+
+  return 0;
+}
+
 static int32_t mndSetDropTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic) {
   SSdbRaw *pCommitRaw = mndTopicActionEncode(pTopic);
   if (pCommitRaw == NULL) return -1;

source/dnode/vnode/inc/vnode.h

@@ -196,6 +196,8 @@ struct SMetaEntry {
       STSma *tsma;
     } smaEntry;
   };
+
+  uint8_t *pBuf;
 };
 
 struct SMetaReader {

source/dnode/vnode/src/inc/meta.h

@@ -24,7 +24,6 @@ extern "C" {
 
 typedef struct SMetaIdx    SMetaIdx;
 typedef struct SMetaDB     SMetaDB;
-typedef struct SMSmaCursor SMSmaCursor;
 
 // metaDebug ==================
 // clang-format off
@@ -114,22 +113,12 @@ typedef struct {
   int64_t  smaUid;
 } SSmaIdxKey;
 
-#if 1
-
-SMSmaCursor* metaOpenSmaCursor(SMeta* pMeta, tb_uid_t uid);
-void         metaCloseSmaCursor(SMSmaCursor* pSmaCur);
-int64_t      metaSmaCursorNext(SMSmaCursor* pSmaCur);
-
 #ifndef META_REFACT
 // SMetaDB
 int  metaOpenDB(SMeta* pMeta);
 void metaCloseDB(SMeta* pMeta);
 int  metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
 int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid);
-int metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg);
-int metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid);
-#endif
-
 #endif
 
 #ifdef __cplusplus

source/dnode/vnode/src/inc/sma.h

@@ -218,6 +218,11 @@ static int32_t  tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SDisk
 
 void *tdFreeRSmaInfo(SRSmaInfo *pInfo);
 
+int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg);
+int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version);
+// TODO: This is the basic params, and should wrap the params to a queryHandle.
+int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/inc/vnodeInt.h

@@ -89,11 +89,13 @@ STSchema*       metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver);
 int             metaGetTableEntryByName(SMetaReader* pReader, const char* name);
 int             metaGetTbNum(SMeta* pMeta);
 SMCtbCursor*    metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid);
-void            metaCloseCtbCurosr(SMCtbCursor* pCtbCur);
+void            metaCloseCtbCursor(SMCtbCursor* pCtbCur);
 tb_uid_t        metaCtbCursorNext(SMCtbCursor* pCtbCur);
-SArray*         metaGetSmaTbUids(SMeta* pMeta, bool isDup);
+STSma*          metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid);
-void*           metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid, bool isDecode);
+STSmaWrapper*   metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid, bool deepCopy);
-STSmaWrapper*   metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid);
+SArray*         metaGetSmaIdsByTable(SMeta* pMeta, tb_uid_t uid);
+SArray*         metaGetSmaTbUids(SMeta* pMeta);
+
 int32_t         metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg);
 int32_t         metaDropTSma(SMeta* pMeta, int64_t indexUid);
 
@@ -116,6 +118,7 @@ void    tqClose(STQ*);
 int     tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
 int     tqCommit(STQ*);
 int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen);
+int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen);
 int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId);
 int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen);
 int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId);
@@ -126,7 +129,7 @@ int32_t smaOpen(SVnode* pVnode);
 int32_t smaClose(SSma* pSma);
 
 int32_t tdUpdateExpireWindow(SSma* pSma, SSubmitReq* pMsg, int64_t version);
-int32_t tdProcessTSmaCreate(SSma* pSma, char* pMsg);
+int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg);
 int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg);
 
 int32_t tdProcessRSmaCreate(SSma* pSma, SMeta* pMeta, SVCreateStbReq* pReq, SMsgCb* pMsgCb);

source/dnode/vnode/src/meta/metaEntry.c

@@ -69,6 +69,11 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
     if (tDecodeI32v(pCoder, &pME->ntbEntry.ncid) < 0) return -1;
     if (tDecodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1;
   } else if (pME->type == TSDB_TSMA_TABLE) {
+    pME->smaEntry.tsma = taosMemoryCalloc(1, sizeof(STSma));
+    if(!pME->smaEntry.tsma) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return -1;
+    }
     if (tDecodeTSma(pCoder, pME->smaEntry.tsma) < 0) return -1;
   } else {
     ASSERT(0);

source/dnode/vnode/src/meta/metaQuery.c

@@ -225,7 +225,7 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) {
   return pCtbCur;
 }
 
-void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) {
+void metaCloseCtbCursor(SMCtbCursor *pCtbCur) {
   if (pCtbCur) {
     if (pCtbCur->pMeta) metaULock(pCtbCur->pMeta);
     if (pCtbCur->pCur) {
@@ -291,178 +291,268 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
   return pTSchema;
 }
 
-STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) {
-#if 0
-#ifdef META_TDB_SMA_TEST
-  STSmaWrapper *pSW = NULL;
-
-  pSW = taosMemoryCalloc(1, sizeof(*pSW));
-  if (pSW == NULL) {
-    return NULL;
-  }
-
-  SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid);
-  if (pCur == NULL) {
-    taosMemoryFree(pSW);
-    return NULL;
-  }
-
-  void       *pBuf = NULL;
-  SSmaIdxKey *pSmaIdxKey = NULL;
-
-  while (true) {
-    // TODO: lock during iterate?
-    if (tdbDbcNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) {
-      pSmaIdxKey = pCur->pKey;
-      ASSERT(pSmaIdxKey != NULL);
-
-      void *pSmaVal = metaGetSmaInfoByIndex(pMeta, pSmaIdxKey->smaUid, false);
-
-      if (pSmaVal == NULL) {
-        tsdbWarn("no tsma exists for indexUid: %" PRIi64, pSmaIdxKey->smaUid);
-        continue;
-      }
-
-      ++pSW->number;
-      STSma *tptr = (STSma *)taosMemoryRealloc(pSW->tSma, pSW->number * sizeof(STSma));
-      if (tptr == NULL) {
-        tdbFree(pSmaVal);
-        metaCloseSmaCursor(pCur);
-        tdDestroyTSmaWrapper(pSW);
-        taosMemoryFreeClear(pSW);
-        return NULL;
-      }
-      pSW->tSma = tptr;
-      pBuf = pSmaVal;
-      if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) {
-        tdbFree(pSmaVal);
-        metaCloseSmaCursor(pCur);
-        tdDestroyTSmaWrapper(pSW);
-        taosMemoryFreeClear(pSW);
-        return NULL;
-      }
-      tdbFree(pSmaVal);
-      continue;
-    }
-    break;
-  }
-
-  metaCloseSmaCursor(pCur);
-
-  return pSW;
-
-#endif
-#endif
-  return NULL;
-}
-
 int metaGetTbNum(SMeta *pMeta) {
   // TODO
   // ASSERT(0);
   return 0;
 }
 
-SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) {
+typedef struct {
-#if 0
+  SMeta   *pMeta;
-  // TODO
+  TDBC    *pCur;
-  // ASSERT(0); // comment this line to pass CI
+  tb_uid_t uid;
-  // return NULL:
-#ifdef META_TDB_SMA_TEST
-  SArray  *pUids = NULL;
-  SMetaDB *pDB = pMeta->pDB;
   void    *pKey;
+  void    *pVal;
+  int      kLen;
+  int      vLen;
+} SMSmaCursor;
 
-  // TODO: lock?
+SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) {
-  SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, 0);
+  SMSmaCursor *pSmaCur = NULL;
-  if (pCur == NULL) {
+  SSmaIdxKey   smaIdxKey;
+  int          ret;
+  int          c;
+
+  pSmaCur = (SMSmaCursor *)taosMemoryCalloc(1, sizeof(*pSmaCur));
+  if (pSmaCur == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
-  // TODO: lock?
 
-  SSmaIdxKey *pSmaIdxKey = NULL;
+  pSmaCur->pMeta = pMeta;
-  tb_uid_t    uid = 0;
+  pSmaCur->uid = uid;
-  while (true) {
+  metaRLock(pMeta);
-    // TODO: lock during iterate?
-    if (tdbDbcNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) {
-      ASSERT(pSmaIdxKey != NULL);
-      pSmaIdxKey = pCur->pKey;
 
-      if (pSmaIdxKey->uid == 0 || pSmaIdxKey->uid == uid) {
+  ret = tdbDbcOpen(pMeta->pSmaIdx, &pSmaCur->pCur, NULL);
-        continue;
+  if (ret < 0) {
-      }
+    metaULock(pMeta);
-      uid = pSmaIdxKey->uid;
+    taosMemoryFree(pSmaCur);
-
+    return NULL;
-      if (!pUids) {
-        pUids = taosArrayInit(16, sizeof(tb_uid_t));
-        if (!pUids) {
-          metaCloseSmaCursor(pCur);
-          return NULL;
-        }
-      }
-
-      taosArrayPush(pUids, &uid);
-
-      continue;
-    }
-    break;
   }
 
-  metaCloseSmaCursor(pCur);
+  // move to the suid
+  smaIdxKey.uid = uid;
+  smaIdxKey.smaUid = INT64_MIN;
+  tdbDbcMoveTo(pSmaCur->pCur, &smaIdxKey, sizeof(smaIdxKey), &c);
+  if (c > 0) {
+    tdbDbcMoveToNext(pSmaCur->pCur);
+  }
 
-  return pUids;
+  return pSmaCur;
-#endif
+}
-#endif
+
+void metaCloseSmaCursor(SMSmaCursor *pSmaCur) {
+  if (pSmaCur) {
+    if (pSmaCur->pMeta) metaULock(pSmaCur->pMeta);
+    if (pSmaCur->pCur) {
+      tdbDbcClose(pSmaCur->pCur);
+
+      tdbFree(pSmaCur->pKey);
+      tdbFree(pSmaCur->pVal);
+    }
+
+    taosMemoryFree(pSmaCur);
+  }
+}
+
+tb_uid_t metaSmaCursorNext(SMSmaCursor *pSmaCur) {
+  int         ret;
+  SSmaIdxKey *pSmaIdxKey;
+
+  ret = tdbDbcNext(pSmaCur->pCur, &pSmaCur->pKey, &pSmaCur->kLen, &pSmaCur->pVal, &pSmaCur->vLen);
+  if (ret < 0) {
+    return 0;
+  }
+
+  pSmaIdxKey = pSmaCur->pKey;
+  if (pSmaIdxKey->uid > pSmaCur->uid) {
+    return 0;
+  }
+
+  return pSmaIdxKey->uid;
+}
+
+STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) {
+  STSmaWrapper *pSW = NULL;
+  SArray       *pSmaIds = NULL;
+
+  if (!(pSmaIds = metaGetSmaIdsByTable(pMeta, uid))) {
+    return NULL;
+  }
+
+  pSW = taosMemoryCalloc(1, sizeof(*pSW));
+  if (!pSW) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+  pSW->number = taosArrayGetSize(pSmaIds);
+  pSW->tSma = taosMemoryCalloc(pSW->number, sizeof(STSma));
+
+  if (!pSW->tSma) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+  SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid);
+  if (pCur == NULL) {
+    goto _err;
+  }
+
+  SMetaReader mr = {0};
+  metaReaderInit(&mr, pMeta, 0);
+  int64_t smaId;
+  int     smaIdx = 0;
+  STSma  *pTSma = NULL;
+  for (int i = 0; i < pSW->number; ++i) {
+    smaId = *(tb_uid_t *)taosArrayGet(pSmaIds, i);
+    if (metaGetTableEntryByUid(&mr, smaId) < 0) {
+      metaWarn("vgId:%d no entry for tbId: %" PRIi64 ", smaId: %" PRIi64, TD_VID(pMeta->pVnode), uid, smaId);
+      continue;
+    }
+    pTSma = pSW->tSma + smaIdx;
+    memcpy(pTSma, mr.me.smaEntry.tsma, sizeof(STSma));
+    if (deepCopy) {
+      if (pTSma->exprLen > 0) {
+        if (!(pTSma->expr = taosMemoryCalloc(1, pTSma->exprLen))) {
+          terrno = TSDB_CODE_OUT_OF_MEMORY;
+          goto _err;
+        }
+        memcpy((void*)pTSma->expr, mr.me.smaEntry.tsma->expr, pTSma->exprLen);
+      }
+      if (pTSma->tagsFilterLen > 0) {
+        if (!(pTSma->tagsFilter = taosMemoryCalloc(1, pTSma->tagsFilterLen))) {
+          terrno = TSDB_CODE_OUT_OF_MEMORY;
+          goto _err;
+        }
+      }
+      memcpy((void*)pTSma->tagsFilter, mr.me.smaEntry.tsma->tagsFilter, pTSma->tagsFilterLen);
+    } else {
+      pTSma->exprLen = 0;
+      pTSma->expr = NULL;
+      pTSma->tagsFilterLen = 0;
+      pTSma->tagsFilter = NULL;
+    }
+    
+    ++smaIdx;
+  }
+
+  if (smaIdx <= 0) goto _err;
+  pSW->number = smaIdx;
+
+  metaReaderClear(&mr);
+  taosArrayDestroy(pSmaIds);
+  metaCloseSmaCursor(pCur);
+  return pSW;
+_err:
+  metaReaderClear(&mr);
+  taosArrayDestroy(pSmaIds);
+  metaCloseSmaCursor(pCur);
+  tdFreeTSmaWrapper(pSW, deepCopy);
   return NULL;
 }
 
-void *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid, bool isDecode) {
+STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) {
-#if 0
+  STSma      *pTSma = NULL;
-  // TODO
+  SMetaReader mr = {0};
-  // ASSERT(0);
+  metaReaderInit(&mr, pMeta, 0);
-  // return NULL;
+  if (metaGetTableEntryByUid(&mr, indexUid) < 0) {
-#ifdef META_TDB_SMA_TEST
+    metaWarn("vgId:%d failed to get table entry for smaId: %" PRIi64, TD_VID(pMeta->pVnode), indexUid);
-  SMetaDB *pDB = pMeta->pDB;
+    metaReaderClear(&mr);
-  void    *pKey = NULL;
+    return NULL;
-  void    *pVal = NULL;
+  }
-  int      kLen = 0;
+  pTSma = (STSma *)taosMemoryMalloc(sizeof(STSma));
-  int      vLen = 0;
+  if (!pTSma) {
-  int      ret = -1;
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
-
+    metaReaderClear(&mr);
-  // Set key
-  pKey = (void *)&indexUid;
-  kLen = sizeof(indexUid);
-
-  // Query
-  ret = tdbDbGet(pDB->pSmaDB, pKey, kLen, &pVal, &vLen);
-  if (ret != 0 || !pVal) {
     return NULL;
   }
 
-  if (!isDecode) {
+  memcpy(pTSma, mr.me.smaEntry.tsma, sizeof(STSma));
-    // return raw value
-    return pVal;
-  }
 
-  // Decode
+  metaReaderClear(&mr);
-  STSma *pCfg = (STSma *)taosMemoryCalloc(1, sizeof(STSma));
+  return pTSma;
-  if (pCfg == NULL) {
+}
-    taosMemoryFree(pVal);
+
+SArray *metaGetSmaIdsByTable(SMeta *pMeta, tb_uid_t uid) {
+  SArray     *pUids = NULL;
+  SSmaIdxKey *pSmaIdxKey = NULL;
+
+  SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid);
+  if (!pCur) {
     return NULL;
   }
 
-  void *pBuf = pVal;
+  while (1) {
-  if (tDecodeTSma(pBuf, pCfg) == NULL) {
+    tb_uid_t id = metaSmaCursorNext(pCur);
-    tdDestroyTSma(pCfg);
+    if (id == 0) {
-    taosMemoryFree(pCfg);
+      break;
-    tdbFree(pVal);
+    }
+
+    if (!pUids) {
+      pUids = taosArrayInit(16, sizeof(tb_uid_t));
+      if (!pUids) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        metaCloseSmaCursor(pCur);
+        return NULL;
+      }
+    }
+
+    pSmaIdxKey = (SSmaIdxKey *)pCur->pKey;
+
+    if (taosArrayPush(pUids, &pSmaIdxKey->smaUid) < 0) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      metaCloseSmaCursor(pCur);
+      taosArrayDestroy(pUids);
+      return NULL;
+    }
+  }
+
+  metaCloseSmaCursor(pCur);
+  return pUids;
+}
+
+SArray *metaGetSmaTbUids(SMeta *pMeta) {
+  SArray     *pUids = NULL;
+  SSmaIdxKey *pSmaIdxKey = NULL;
+  tb_uid_t    lastUid = 0;
+
+  SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, 0);
+  if (!pCur) {
     return NULL;
   }
 
-  tdbFree(pVal);
+  while (1) {
-  return pCfg;
+    tb_uid_t uid = metaSmaCursorNext(pCur);
-#endif
+    if (uid == 0) {
-#endif
+      break;
-  return NULL;
+    }
+
+    if (lastUid == uid) {
+      continue;
+    }
+
+    lastUid = uid;
+
+    if (!pUids) {
+      pUids = taosArrayInit(16, sizeof(tb_uid_t));
+      if (!pUids) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        metaCloseSmaCursor(pCur);
+        return NULL;
+      }
+    }
+
+    if (taosArrayPush(pUids, &uid) < 0) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      metaCloseSmaCursor(pCur);
+      taosArrayDestroy(pUids);
+      return NULL;
+    }
+  }
+
+  metaCloseSmaCursor(pCur);
+  return pUids;
 }
 
 #endif

source/dnode/vnode/src/meta/metaSma.c

@@ -16,7 +16,7 @@
 #include "meta.h"
 
 static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME);
-static int metaSaveSmaToDB(SMeta *pMeta,  const SMetaEntry *pME);
+static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME);
 
 int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) {
   // TODO: Validate the cfg
@@ -81,55 +81,6 @@ int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid) {
   return TSDB_CODE_SUCCESS;
 }
 
-// static int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) {
-//   int32_t ret = 0;
-//   void   *pBuf = NULL, *qBuf = NULL;
-//   void   *key = {0}, *val = {0};
-
-//   // save sma info
-//   int32_t len = tEncodeTSma(NULL, pSmaCfg);
-//   pBuf = taosMemoryCalloc(1, len);
-//   if (pBuf == NULL) {
-//     terrno = TSDB_CODE_OUT_OF_MEMORY;
-//     return -1;
-//   }
-
-//   key = (void *)&pSmaCfg->indexUid;
-//   qBuf = pBuf;
-//   tEncodeTSma(&qBuf, pSmaCfg);
-//   val = pBuf;
-
-//   int32_t kLen = sizeof(pSmaCfg->indexUid);
-//   int32_t vLen = POINTER_DISTANCE(qBuf, pBuf);
-
-//   ret = tdbDbInsert(pMeta->pTbDb, key, kLen, val, vLen, &pMeta->txn);
-//   if (ret < 0) {
-//     taosMemoryFreeClear(pBuf);
-//     return -1;
-//   }
-
-//   // add sma idx
-//   SSmaIdxKey smaIdxKey;
-//   smaIdxKey.uid = pSmaCfg->tableUid;
-//   smaIdxKey.smaUid = pSmaCfg->indexUid;
-//   key = &smaIdxKey;
-//   kLen = sizeof(smaIdxKey);
-//   val = NULL;
-//   vLen = 0;
-
-//   ret = tdbDbInsert(pMeta->pSmaIdx, key, kLen, val, vLen, &pMeta->txn);
-//   if (ret < 0) {
-//     taosMemoryFreeClear(pBuf);
-//     return -1;
-//   }
-
-//   // release
-//   taosMemoryFreeClear(pBuf);
-
-//   return 0;
-// }
-
-
 static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME) {
   STbDbKey tbDbKey;
   void    *pKey = NULL;
@@ -182,6 +133,10 @@ static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) {
   return tdbDbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn);
 }
 
+static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) {
+  return tdbDbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn);
+}
+
 static int metaUpdateSmaIdx(SMeta *pMeta, const SMetaEntry *pME) {
   SSmaIdxKey smaIdxKey = {.uid = pME->smaEntry.tsma->tableUid, .smaUid = pME->smaEntry.tsma->indexUid};
 
@@ -194,9 +149,13 @@ static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME) {
   // save to table.db
   if (metaSaveSmaToDB(pMeta, pME) < 0) goto _err;
 
-  // // update uid.idx
+  // update uid.idx
   if (metaUpdateUidIdx(pMeta, pME) < 0) goto _err;
 
+  // update name.idx
+  if (metaUpdateNameIdx(pMeta, pME) < 0) goto _err;
+
+  // update sma.idx
   if (metaUpdateSmaIdx(pMeta, pME) < 0) goto _err;
 
   metaULock(pMeta);

source/dnode/vnode/src/meta/metaTable.c

@@ -420,7 +420,8 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
   // get table entry
   SDecoder dc = {0};
   tDecoderInit(&dc, pData, nData);
-  metaDecodeEntry(&dc, &entry);
+  ret = metaDecodeEntry(&dc, &entry);
+  ASSERT(ret == 0);
 
   if (entry.type != TSDB_NORMAL_TABLE) {
     terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION;
@@ -468,11 +469,11 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
         goto _err;
       }
       pSchema->sver++;
-      pSchema->nCols--;
       tlen = (pSchema->nCols - iCol - 1) * sizeof(SSchema);
       if (tlen) {
         memmove(pColumn, pColumn + 1, tlen);
       }
+      pSchema->nCols--;
       break;
     case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:
       if (pColumn == NULL) {
@@ -498,6 +499,18 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
 
   entry.version = version;
 
+  // do actual write
+  metaWLock(pMeta);
+
+  // save to table db
+  metaSaveToTbDb(pMeta, &entry);
+
+  tdbDbcUpsert(pUidIdxc, &entry.uid, sizeof(tb_uid_t), &version, sizeof(version), 0);
+
+  metaSaveToSkmDb(pMeta, &entry);
+
+  metaULock(pMeta);
+
   tDecoderClear(&dc);
   tdbDbcClose(pTbDbc);
   tdbDbcClose(pUidIdxc);
@@ -511,8 +524,128 @@ _err:
 }
 
 static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
-  // TODO
+  SMetaEntry  ctbEntry = {0};
+  SMetaEntry  stbEntry = {0};
+  void       *pVal = NULL;
+  int         nVal = 0;
+  int         ret;
+  int         c;
+  tb_uid_t    uid;
+  int64_t     oversion;
+  const void *pData = NULL;
+  int         nData = 0;
+
+  // search name index
+  ret = tdbDbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal);
+  if (ret < 0) {
+    terrno = TSDB_CODE_VND_TABLE_NOT_EXIST;
+    return -1;
+  }
+
+  uid = *(tb_uid_t *)pVal;
+  tdbFree(pVal);
+  pVal = NULL;
+
+  // search uid index
+  TDBC *pUidIdxc = NULL;
+
+  tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn);
+  tdbDbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c);
+  ASSERT(c == 0);
+
+  tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData);
+  oversion = *(int64_t *)pData;
+
+  // search table.db
+  TDBC    *pTbDbc = NULL;
+  SDecoder dc = {0};
+
+  /* get ctbEntry */
+  tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn);
+  tdbDbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c);
+  ASSERT(c == 0);
+  tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData);
+
+  ctbEntry.pBuf = taosMemoryMalloc(nData);
+  memcpy(ctbEntry.pBuf, pData, nData);
+  tDecoderInit(&dc, ctbEntry.pBuf, nData);
+  metaDecodeEntry(&dc, &ctbEntry);
+  tDecoderClear(&dc);
+
+  /* get stbEntry*/
+  tdbDbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal);
+  tdbDbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = *(int64_t *)pVal}), sizeof(STbDbKey),
+           (void **)&stbEntry.pBuf, &nVal);
+  tdbFree(pVal);
+  tDecoderInit(&dc, stbEntry.pBuf, nVal);
+  metaDecodeEntry(&dc, &stbEntry);
+  tDecoderClear(&dc);
+
+  SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag;
+  SSchema        *pColumn = NULL;
+  int32_t         iCol = 0;
+  for (;;) {
+    pColumn = NULL;
+
+    if (iCol >= pTagSchema->nCols) break;
+    pColumn = &pTagSchema->pSchema[iCol];
+
+    if (strcmp(pColumn->name, pAlterTbReq->tagName) == 0) break;
+    iCol++;
+  }
+
+  if (pColumn == NULL) {
+    terrno = TSDB_CODE_VND_TABLE_COL_NOT_EXISTS;
+    goto _err;
+  }
+
+  if (iCol == 0) {
+    // TODO : need to update tag index
+  }
+
+  ctbEntry.version = version;
+  SKVRowBuilder kvrb = {0};
+  const SKVRow  pOldTag = (const SKVRow)ctbEntry.ctbEntry.pTags;
+  SKVRow        pNewTag = NULL;
+
+  tdInitKVRowBuilder(&kvrb);
+  for (int32_t i = 0; i < pTagSchema->nCols; i++) {
+    SSchema *pCol = &pTagSchema->pSchema[i];
+    if (iCol == i) {
+      tdAddColToKVRow(&kvrb, pCol->colId, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal);
+    } else {
+      void *p = tdGetKVRowValOfCol(pOldTag, pCol->colId);
+      if (p) {
+        if (IS_VAR_DATA_TYPE(pCol->type)) {
+          tdAddColToKVRow(&kvrb, pCol->colId, p, varDataTLen(p));
+        } else {
+          tdAddColToKVRow(&kvrb, pCol->colId, p, pCol->bytes);
+        }
+      }
+    }
+  }
+
+  ctbEntry.ctbEntry.pTags = tdGetKVRowFromBuilder(&kvrb);
+  tdDestroyKVRowBuilder(&kvrb);
+
+  // save to table.db
+  metaSaveToTbDb(pMeta, &ctbEntry);
+
+  // save to uid.idx
+  tdbDbUpsert(pMeta->pUidIdx, &ctbEntry.uid, sizeof(tb_uid_t), &version, sizeof(version), &pMeta->txn);
+
+  if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf);
+  if (stbEntry.pBuf) tdbFree(stbEntry.pBuf);
+  tdbDbcClose(pTbDbc);
+  tdbDbcClose(pUidIdxc);
   return 0;
+
+_err:
+  if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf);
+  if (stbEntry.pBuf) tdbFree(stbEntry.pBuf);
+  tdbDbcClose(pTbDbc);
+  tdbDbcClose(pUidIdxc);
+  return -1;
 }
 
 static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {

source/dnode/vnode/src/sma/sma.c

@@ -27,4 +27,28 @@ int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg) {
   return code;
 }
 
+int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg) {
+  int32_t code = TSDB_CODE_SUCCESS;
 
+  if ((code = tdProcessTSmaCreateImpl(pSma, version, msg)) < 0) {
+    smaWarn("vgId:%d create tsma failed since %s", SMA_VID(pSma), tstrerror(terrno));
+  }
+  // TODO: destroy SSDataBlocks(msg)
+  return code;
+}
+
+int32_t tdUpdateExpireWindow(SSma* pSma, SSubmitReq* pMsg, int64_t version) {
+  int32_t code = TSDB_CODE_SUCCESS;
+  if ((code = tdUpdateExpiredWindowImpl(pSma, pMsg, version)) < 0) {
+    smaWarn("vgId:%d update expired sma window failed since %s", SMA_VID(pSma), tstrerror(terrno));
+  }
+  return code;
+}
+
+int32_t tdGetTSmaData(SSma* pSma, char* pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) {
+  int32_t code = TSDB_CODE_SUCCESS;
+  if ((code = tdGetTSmaDataImpl(pSma, pData, indexUid, querySKey, nMaxResult)) < 0) {
+    smaWarn("vgId:%d get tSma data failed since %s", SMA_VID(pSma), tstrerror(terrno));
+  }
+  return code;
+}

source/dnode/vnode/src/sma/smaEnv.c

@@ -122,12 +122,10 @@ static void poolFree(void *arg, void *ptr) {
 }
 
 int32_t tdInitSma(SSma *pSma) {
-  // tSma
+  int32_t numOfTSma = taosArrayGetSize(metaGetSmaTbUids(SMA_META(pSma)));
-  int32_t numOfTSma = taosArrayGetSize(metaGetSmaTbUids(SMA_META(pSma), false));
   if (numOfTSma > 0) {
     atomic_store_16(&SMA_TSMA_NUM(pSma), (int16_t)numOfTSma);
   }
-  // TODO: rSma
   return TSDB_CODE_SUCCESS;
 }
 

source/dnode/vnode/src/sma/smaRollup.c

@@ -443,7 +443,7 @@ static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb
 
   if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
     // TODO: use the proper schema instead of 0, and cache STSchema in cache
-    STSchema *pTSchema = metaGetTbTSchema(SMA_META(pSma), suid, 0);
+    STSchema *pTSchema = metaGetTbTSchema(SMA_META(pSma), suid, 1);
     if (!pTSchema) {
       terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
       return TSDB_CODE_FAILED;

source/dnode/vnode/src/sma/smaTimeRange.c

@@ -70,15 +70,15 @@ static bool    tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey);
 static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen,
                                     TXN *txn);
 // expired window
-static int32_t  tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version);
+
+
 static int32_t  tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey,
                                      int64_t version);
 static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey);
 static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid);
 
 // read data
-// TODO: This is the basic params, and should wrap the params to a queryHandle.
+
-static int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult);
 
 // implementation
 
@@ -713,7 +713,7 @@ static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid) {
  * @param nMaxResult The query invoker should control the nMaxResult need to return to avoid OOM.
  * @return int32_t
  */
-static int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) {
+int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) {
   SSmaEnv  *pEnv = atomic_load_ptr(&SMA_TSMA_ENV(pSma));
   SSmaStat *pStat = NULL;
 
@@ -834,35 +834,15 @@ static int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKE
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t tdProcessTSmaCreate(SSma *pSma, char *pMsg) {
+int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) {
-  #if 0
+  SSmaCfg *pCfg = (SSmaCfg *)pMsg;
-  SSmaCfg vCreateSmaReq = {0};
-  if (!tDeserializeSVCreateTSmaReq(pMsg, &vCreateSmaReq)) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    smaWarn("vgId:%d tsma create msg received but deserialize failed since %s", SMA_VID(pSma), terrstr(terrno));
-    return -1;
-  }
 
-  smaDebug("vgId:%d tsma create msg %s:%" PRIi64 " for table %" PRIi64 " received", SMA_VID(pSma),
+  if (metaCreateTSma(SMA_META(pSma), version, pCfg) < 0) {
-            vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid, vCreateSmaReq.tSma.tableUid);
-
-  // record current timezone of server side
-  vCreateSmaReq.tSma.timezoneInt = tsTimezone;
-
-  if (metaCreateTSma(SMA_META(pSma), &vCreateSmaReq) < 0) {
-    // TODO: handle error
-    smaWarn("vgId:%d tsma %s:%" PRIi64 " create failed for table %" PRIi64 " since %s", SMA_VID(pSma),
-             vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid, vCreateSmaReq.tSma.tableUid, terrstr(terrno));
-    tdDestroyTSma(&vCreateSmaReq.tSma);
     return -1;
   }
 
   tdTSmaAdd(pSma, 1);
-
+  return 0;
-  tdDestroyTSma(&vCreateSmaReq.tSma);
-  // TODO: return directly or go on follow steps?
-#endif
-  return TSDB_CODE_SUCCESS;
 }
 
 int32_t tdDropTSma(SSma *pSma, char *pMsg) {
@@ -930,7 +910,7 @@ static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t inde
     }
 
     // cache smaMeta
-    STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid, true);
+    STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid);
     if (!pTSma) {
       terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META;
       taosHashCleanup(pItem->expiredWindows);
@@ -1031,25 +1011,25 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version)
 
     SSubmitBlkIter blkIter = {0};
     if (tInitSubmitBlkIter(&msgIter, pBlock, &blkIter) < 0) {
-      pSW = tdFreeTSmaWrapper(pSW);
+      pSW = tdFreeTSmaWrapper(pSW, false);
       break;
     }
 
     while (true) {
       STSRow *row = tGetSubmitBlkNext(&blkIter);
       if (!row) {
-        tdFreeTSmaWrapper(pSW);
+        pSW = tdFreeTSmaWrapper(pSW, false);
         break;
       }
-      if (!pSW || (pTSma->tableUid != pBlock->suid)) {
+      if (!pSW || (pTSma->tableUid != msgIter.suid)) {
         if (pSW) {
-          pSW = tdFreeTSmaWrapper(pSW);
+          pSW = tdFreeTSmaWrapper(pSW, false);
         }
-        if (!(pSW = metaGetSmaInfoByTable(SMA_META(pSma), pBlock->suid))) {
+        if (!(pSW = metaGetSmaInfoByTable(SMA_META(pSma), msgIter.suid, false))) {
           break;
         }
         if ((pSW->number) <= 0 || !pSW->tSma) {
-          pSW = tdFreeTSmaWrapper(pSW);
+          pSW = tdFreeTSmaWrapper(pSW, false);
           break;
         }
 
@@ -1068,6 +1048,7 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version)
       if (lastWinSKey != winSKey) {
         lastWinSKey = winSKey;
         if (tdSetExpiredWindow(pSma, pItemsHash, pTSma->indexUid, winSKey, version) < 0) {
+          pSW = tdFreeTSmaWrapper(pSW, false);
           tdUnRefSmaStat(pSma, pStat);
           return TSDB_CODE_FAILED;
         }
@@ -1083,21 +1064,3 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version)
   return TSDB_CODE_SUCCESS;
 }
 
-
-int32_t tdUpdateExpireWindow(SSma *pSma, SSubmitReq *pMsg, int64_t version) {
-  int32_t code = TSDB_CODE_SUCCESS;
-  if ((code = tdUpdateExpiredWindowImpl(pSma, pMsg, version)) < 0) {
-    smaWarn("vgId:%d update expired sma window failed since %s", SMA_VID(pSma), tstrerror(terrno));
-  }
-  return code;
-}
-
-int32_t tdGetTSmaData(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) {
-  int32_t code = TSDB_CODE_SUCCESS;
-  if ((code = tdGetTSmaDataImpl(pSma, pData, indexUid, querySKey, nMaxResult)) < 0) {
-    smaWarn("vgId:%d get tSma data failed since %s", SMA_VID(pSma), tstrerror(terrno));
-  }
-  return code;
-}
-
-

source/dnode/vnode/src/tq/tq.c

@@ -861,6 +861,14 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
 }
 #endif
 
+int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen) {
+  SMqVDeleteReq* pReq = (SMqVDeleteReq*)msg;
+
+  int32_t code = taosHashRemove(pTq->execs, pReq->subKey, strlen(pReq->subKey));
+  ASSERT(code == 0);
+  return 0;
+}
+
 // TODO: persist meta into tdb
 int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
   SMqRebVgReq req = {0};
@@ -1087,35 +1095,6 @@ int32_t tqProcessStreamTrigger2(STQ* pTq, SSubmitReq* pReq, int64_t ver) {
 }
 
 int32_t tqProcessTaskExec2(STQ* pTq, char* msg, int32_t msgLen) {
-  SStreamTaskExecReq req = {0};
+  //
-  tDecodeSStreamTaskExecReq(msg, &req);
-  int32_t taskId = req.taskId;
-
-  SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t));
-  ASSERT(pTask);
-  ASSERT(pTask->inputType == TASK_INPUT_TYPE__DATA_BLOCK);
-
-  // enqueue
-  int32_t inputStatus = streamEnqueueDataBlk(pTask, (SStreamDataBlock*)req.data);
-  if (inputStatus == TASK_INPUT_STATUS__BLOCKED) {
-    // TODO rsp blocked
-    return 0;
-  }
-
-  // try exec
-  int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING);
-  if (execStatus == TASK_STATUS__IDLE) {
-    if (streamTaskRun(pTask) < 0) {
-      atomic_store_8(&pTask->status, TASK_STATUS__CLOSING);
-
-      goto FAIL;
-    }
-  } else if (execStatus == TASK_STATUS__EXECUTING) {
-    return 0;
-  }
-
-  // TODO rsp success
   return 0;
-FAIL:
-  return -1;
 }

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -2760,7 +2760,7 @@ static int32_t getAllTableList(SMeta* pMeta, uint64_t uid, SArray* list) {
     taosArrayPush(list, &info);
   }
 
-  metaCloseCtbCurosr(pCur);
+  metaCloseCtbCursor(pCur);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -3055,7 +3055,8 @@ static bool loadDataBlockFromTableSeq(STsdbReadHandle* pTsdbReadHandle) {
 bool tsdbNextDataBlock(tsdbReaderT pHandle) {
   STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)pHandle;
 
-  for (int32_t i = 0; i < taosArrayGetSize(pTsdbReadHandle->pColumns); ++i) {
+  size_t numOfCols = taosArrayGetSize(pTsdbReadHandle->pColumns);
+  for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
     colInfoDataCleanup(pColInfo, pTsdbReadHandle->outputCapacity);
   }

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -101,6 +101,11 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
         // TODO: handle error
       }
       break;
+    case TDMT_VND_MQ_VG_DELETE:
+      if (tqProcessVgDeleteReq(pVnode->pTq, pMsg->pCont, pMsg->contLen) < 0) {
+        // TODO: handle error
+      }
+      break;
     case TDMT_VND_TASK_DEPLOY: {
       if (tqProcessTaskDeploy(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)),
                               pMsg->contLen - sizeof(SMsgHead)) < 0) {
@@ -199,7 +204,7 @@ void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) {
 
 int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
   int32_t ret = TAOS_SYNC_PROPOSE_OTHER_ERROR;
-  
+
   if (syncEnvIsStart()) {
     SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync);
     assert(pSyncNode != NULL);
@@ -462,7 +467,11 @@ _exit:
 
 static int vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
   SVAlterTbReq vAlterTbReq = {0};
+  SVAlterTbRsp vAlterTbRsp = {0};
   SDecoder     dc = {0};
+  int          rcode = 0;
+  int          ret;
+  SEncoder     ec = {0};
 
   pRsp->msgType = TDMT_VND_ALTER_TABLE_RSP;
   pRsp->pCont = NULL;
@@ -473,19 +482,27 @@ static int vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, i
 
   // decode
   if (tDecodeSVAlterTbReq(&dc, &vAlterTbReq) < 0) {
-    pRsp->code = TSDB_CODE_INVALID_MSG;
+    vAlterTbRsp.code = TSDB_CODE_INVALID_MSG;
     tDecoderClear(&dc);
-    return -1;
+    rcode = -1;
+    goto _exit;
   }
 
   // process
   if (metaAlterTable(pVnode->pMeta, version, &vAlterTbReq) < 0) {
-    pRsp->code = terrno;
+    vAlterTbRsp.code = TSDB_CODE_INVALID_MSG;
     tDecoderClear(&dc);
-    return -1;
+    rcode = -1;
+    goto _exit;
   }
-
   tDecoderClear(&dc);
+
+_exit:
+  tEncodeSize(tEncodeSVAlterTbRsp, &vAlterTbRsp, pRsp->contLen, ret);
+  pRsp->pCont = rpcMallocCont(pRsp->contLen);
+  tEncoderInit(&ec, pRsp->pCont, pRsp->contLen);
+  tEncodeSVAlterTbRsp(&ec, &vAlterTbRsp);
+  tEncoderClear(&ec);
   return 0;
 }
 
@@ -711,15 +728,22 @@ static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq
     goto _err;
   }
 
-  if (metaCreateTSma(pVnode->pMeta, version, &req) < 0) {
+  // record current timezone of server side
+  req.timezoneInt = tsTimezone;
+
+  if (tdProcessTSmaCreate(pVnode->pSma, version, (const char *)&req) < 0) {
     pRsp->code = terrno;
     goto _err;
   }
 
   tDecoderClear(&coder);
+  vDebug("vgId:%d success to create tsma %s:%" PRIi64 " for table %" PRIi64, TD_VID(pVnode), req.indexName,
+         req.indexUid, req.tableUid);
   return 0;
 
 _err:
   tDecoderClear(&coder);
+  vError("vgId:%d failed to create tsma %s:%" PRIi64 " for table %" PRIi64 " since %s", TD_VID(pVnode), req.indexName,
+         req.indexUid, req.tableUid, terrstr(terrno));
   return -1;
 }

source/libs/executor/inc/executil.h

@@ -139,7 +139,7 @@ typedef struct {
   int32_t colId;
 } SStddevInterResult;
 
-void    initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, bool sortGroupResult);
+void    initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int32_t order);
 void    initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList);
 
 void    cleanupGroupResInfo(SGroupResInfo* pGroupResInfo);

source/libs/executor/src/executil.c

@@ -184,7 +184,7 @@ void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) {
   pGroupResInfo->index     = 0;
 }
 
-static int32_t resultrowCompar1(const void* p1, const void* p2) {
+static int32_t resultrowComparAsc(const void* p1, const void* p2) {
   SResKeyPos* pp1 = *(SResKeyPos**) p1;
   SResKeyPos* pp2 = *(SResKeyPos**) p2;
 
@@ -202,7 +202,11 @@ static int32_t resultrowCompar1(const void* p1, const void* p2) {
   }
 }
 
-void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, bool sortGroupResult) {
+static int32_t resultrowComparDesc(const void* p1, const void* p2) {
+  return resultrowComparAsc(p2, p1);
+}
+
+void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int32_t order) {
   if (pGroupResInfo->pRows != NULL) {
     taosArrayDestroy(pGroupResInfo->pRows);
   }
@@ -224,8 +228,9 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, boo
     taosArrayPush(pGroupResInfo->pRows, &p);
   }
 
-  if (sortGroupResult) {
+  if (order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC) {
-    qsort(pGroupResInfo->pRows->pData, taosArrayGetSize(pGroupResInfo->pRows), POINTER_BYTES, resultrowCompar1);
+    __compar_fn_t fn = (order == TSDB_ORDER_ASC)? resultrowComparAsc:resultrowComparDesc;
+    qsort(pGroupResInfo->pRows->pData, taosArrayGetSize(pGroupResInfo->pRows), POINTER_BYTES, fn);
   }
 
   pGroupResInfo->index = 0;

source/libs/executor/src/executorimpl.c

@@ -3707,7 +3707,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
   finalizeMultiTupleQueryResult(pOperator->numOfExprs, pAggInfo->aggSup.pResultBuf, &pAggInfo->binfo.resultRowInfo,
                                 pAggInfo->binfo.rowCellInfoOffset);
 
-  initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, false);
+  initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, 0);
   OPTR_SET_OPENED(pOperator);
   return TSDB_CODE_SUCCESS;
 }

source/libs/executor/src/groupoperator.c

@@ -314,7 +314,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
   //  }
 
   blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity);
-  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, false);
+  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, 0);
 
   while(1) {
     doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);

source/libs/executor/src/scanoperator.c

@@ -102,7 +102,7 @@ static void getNextTimeWindow(SInterval* pInterval, STimeWindow* tw, int32_t ord
   tw->ekey -= 1;
 }
 
-static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockInfo) {
+static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockInfo, int32_t order) {
   STimeWindow w = {0};
 
   // 0 by default, which means it is not a interval operator of the upstream operator.
@@ -110,13 +110,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn
     return false;
   }
 
-  // todo handle the time range case
+  if (order == TSDB_ORDER_ASC) {
-  TSKEY sk = INT64_MIN;
-  TSKEY ek = INT64_MAX;
-  //  TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey);
-  //  TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey);
-
-  if (true) {
     getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.skey, &w);
     assert(w.ekey >= pBlockInfo->window.skey);
 
@@ -124,8 +118,8 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn
       return true;
     }
 
-    while (1) {  // todo handle the desc order scan case
+    while (1) {
-      getNextTimeWindow(pInterval, &w, TSDB_ORDER_ASC);
+      getNextTimeWindow(pInterval, &w, order);
       if (w.skey > pBlockInfo->window.ekey) {
         break;
       }
@@ -136,24 +130,24 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn
       }
     }
   } else {
-    //    getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.ekey, sk, ek, &w);
+    getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.ekey, &w);
-    //    assert(w.skey <= pBlockInfo->window.ekey);
+    assert(w.skey <= pBlockInfo->window.ekey);
-    //
+
-    //    if (w.skey > pBlockInfo->window.skey) {
+    if (w.skey > pBlockInfo->window.skey) {
-    //      return true;
+      return true;
-    //    }
+    }
-    //
+
-    //    while(1) {
+    while(1) {
-    //      getNextTimeWindow(pQueryAttr, &w);
+      getNextTimeWindow(pInterval, &w, order);
-    //      if (w.ekey < pBlockInfo->window.skey) {
+      if (w.ekey < pBlockInfo->window.skey) {
-    //        break;
+        break;
-    //      }
+      }
-    //
+
-    //      assert(w.skey < pBlockInfo->window.skey);
+      assert(w.skey < pBlockInfo->window.skey);
-    //      if (w.ekey < pBlockInfo->window.ekey && w.ekey >= pBlockInfo->window.skey) {
+      if (w.ekey < pBlockInfo->window.ekey && w.ekey >= pBlockInfo->window.skey) {
-    //        return true;
+        return true;
-    //      }
+      }
-    //    }
+    }
   }
 
   return false;
@@ -172,7 +166,8 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
   pCost->totalRows += pBlock->info.rows;
 
   *status = pInfo->dataBlockLoadFlag;
-  if (pTableScanInfo->pFilterNode != NULL || overlapWithTimeWindow(&pTableScanInfo->interval, &pBlock->info)) {
+  if (pTableScanInfo->pFilterNode != NULL ||
+      overlapWithTimeWindow(&pTableScanInfo->interval, &pBlock->info, pTableScanInfo->cond.order)) {
     (*status) = FUNC_DATA_REQUIRED_DATA_LOAD;
   }
 
@@ -188,6 +183,13 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
     qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
            pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
     pCost->skipBlocks += 1;
+
+    // clear all data in pBlock that are set when handing the previous block
+    for(int32_t i = 0; i < pBlockInfo->numOfCols; ++i) {
+      SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i);
+      pcol->pData = NULL;
+    }
+
     return TSDB_CODE_SUCCESS;
   } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) {
     pCost->loadBlockStatis += 1;
@@ -466,6 +468,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
   }
 
   pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]};
+//  pInfo->scanInfo = (SScanInfo){.numOfAsc = 0, .numOfDesc = 1}; // for debug purpose
 
   pInfo->readHandle        = *readHandle;
   pInfo->interval          = extractIntervalInfo(pTableScanNode);

source/libs/executor/src/timewindowoperator.c

@@ -54,8 +54,8 @@ static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols, int32_t rows,
   if (tsCols == NULL) {
     ts = ascQuery ? win->skey : win->ekey;
   } else {
-    int32_t offset = ascQuery ? 0 : rows - 1;
+//    int32_t offset = ascQuery ? 0 : rows - 1;
-    ts = tsCols[offset];
+    ts = tsCols[0];
   }
 
   return ts;
@@ -172,14 +172,22 @@ static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_se
       }
     }
   } else {
-    int32_t end = searchFn((char*)pData, pos + 1, ekey, order);
+    int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order);
     if (end >= 0) {
-      forwardStep = pos - end;
+      forwardStep = end;
 
-      if (pData[end] == ekey) {
+      if (pData[end + pos] == ekey) {
         forwardStep += 1;
       }
     }
+//    int32_t end = searchFn((char*)pData, pos + 1, ekey, order);
+//    if (end >= 0) {
+//      forwardStep = pos - end;
+//
+//      if (pData[end] == ekey) {
+//        forwardStep += 1;
+//      }
+//    }
   }
 
   assert(forwardStep >= 0);
@@ -203,17 +211,25 @@ int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) {
   if (order == TSDB_ORDER_DESC) {
     // find the first position which is smaller than the key
     while (1) {
-      if (key >= keyList[lastPos]) return lastPos;
+      if (key >= keyList[firstPos]) return firstPos;
-      if (key == keyList[firstPos]) return firstPos;
+      if (key == keyList[lastPos]) return lastPos;
-      if (key < keyList[firstPos]) return firstPos - 1;
+
+      if (key < keyList[lastPos]) {
+        lastPos += 1;
+        if (lastPos >= num) {
+          return -1;
+        } else {
+          return lastPos;
+        }
+      }
 
       numOfRows = lastPos - firstPos + 1;
       midPos = (numOfRows >> 1) + firstPos;
 
       if (key < keyList[midPos]) {
-        lastPos = midPos - 1;
-      } else if (key > keyList[midPos]) {
         firstPos = midPos + 1;
+      } else if (key > keyList[midPos]) {
+        lastPos = midPos - 1;
       } else {
         break;
       }
@@ -273,12 +289,12 @@ int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimary
     if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) {
       num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn);
       if (item != NULL) {
-        item->lastKey = pPrimaryColumn[startPos - (num - 1)] + step;
+        item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step;
       }
     } else {
-      num = startPos + 1;
+      num = pDataBlockInfo->rows - startPos;
       if (item != NULL) {
-        item->lastKey = pDataBlockInfo->window.skey + step;
+        item->lastKey = pDataBlockInfo->window.ekey + step;
       }
     }
   }
@@ -470,20 +486,17 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext,
     return -1;
   }
 
-  TSKEY   startKey = ascQuery ? pNext->skey : pNext->ekey;
+  TSKEY   skey = ascQuery ? pNext->skey : pNext->ekey;
   int32_t startPos = 0;
 
   // tumbling time window query, a special case of sliding time window query
   if (pInterval->sliding == pInterval->interval && prevPosition != -1) {
-    int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order);
+    startPos = prevPosition + 1;
-    startPos = prevPosition + factor;
   } else {
-    if (startKey <= pDataBlockInfo->window.skey && ascQuery) {
+    if ((skey <= pDataBlockInfo->window.skey && ascQuery) || (skey >= pDataBlockInfo->window.ekey && !ascQuery)) {
       startPos = 0;
-    } else if (startKey >= pDataBlockInfo->window.ekey && !ascQuery) {
-      startPos = pDataBlockInfo->rows - 1;
     } else {
-      startPos = binarySearchForKey((char*)primaryKeys, pDataBlockInfo->rows, startKey, order);
+      startPos = binarySearchForKey((char*)primaryKeys, pDataBlockInfo->rows, skey, order);
     }
   }
 
@@ -608,7 +621,7 @@ static void saveDataBlockLastRow(char** pRow, SArray* pDataBlock, int32_t rowInd
 }
 
 static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock,
-                               int32_t tableGroupId) {
+                               uint64_t tableGroupId) {
   SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
 
   SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
@@ -620,7 +633,7 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
   }
 
   int32_t step = 1;
-  bool    ascScan = true;
+  bool    ascScan = (pInfo->order == TSDB_ORDER_ASC);
 
   //  int32_t prevIndex = pResultRowInfo->curPos;
 
@@ -630,7 +643,7 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
     tsCols = (int64_t*)pColDataInfo->pData;
   }
 
-  int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1);
+  int32_t startPos = 0;
   TSKEY   ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascScan);
 
   STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval,
@@ -654,9 +667,10 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
   }
 
   int32_t forwardStep = 0;
-  TSKEY   ekey = win.ekey;
+  TSKEY   ekey = ascScan? win.ekey:win.skey;
   forwardStep =
-      getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
+      getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order);
+  ASSERT(forwardStep > 0);
 
   // prev time window not interpolation yet.
   //  int32_t curIndex = pResultRowInfo->curPos;
@@ -731,9 +745,9 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
       taosArrayPush(pUpdated, &pos);
     }
 
-    ekey = nextWin.ekey;  // reviseWindowEkey(pQueryAttr, &nextWin);
+    ekey = ascScan? nextWin.ekey:nextWin.skey;
     forwardStep =
-        getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
+        getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order);
 
     // window start(end) key interpolation
     doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep,
@@ -761,7 +775,8 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
   SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
   SIntervalAggOperatorInfo* pInfo = pOperator->info;
 
-  int32_t        order = TSDB_ORDER_ASC;
+  int32_t scanFlag = MAIN_SCAN;
+
   SOperatorInfo* downstream = pOperator->pDownstream[0];
 
   while (1) {
@@ -773,8 +788,10 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
       break;
     }
 
+    getTableScanInfo(pOperator, &pInfo->order, &scanFlag);
+
     // the pDataBlock are always the same one, no need to call this again
-    setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, MAIN_SCAN, true);
+    setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pInfo->order, scanFlag, true);
     STableQueryInfo* pTableQueryInfo = pInfo->pCurrent;
 
     setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window);
@@ -800,7 +817,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
   finalizeMultiTupleQueryResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo,
                                 pInfo->binfo.rowCellInfoOffset);
 
-  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true);
+  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, pInfo->order);
   OPTR_SET_OPENED(pOperator);
   return TSDB_CODE_SUCCESS;
 }
@@ -945,7 +962,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
   finalizeMultiTupleQueryResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo,
                                 pBInfo->rowCellInfoOffset);
 
-  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true);
+  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
   blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
   doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
   if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) {
@@ -1070,6 +1087,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
       doClearWindows(pInfo, pOperator->numOfExprs, pBlock);
       continue;
     }
+    pInfo->order = TSDB_ORDER_ASC;
     pUpdated = hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0);
   }
 
@@ -1119,7 +1137,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo*
 
   pInfo->order = TSDB_ORDER_ASC;
   pInfo->interval = *pInterval;
-  //  pInfo->execModel      = OPTR_EXEC_MODEL_STREAM;
   pInfo->execModel = pTaskInfo->execModel;
   pInfo->win = pTaskInfo->window;
   pInfo->twAggSup = *pTwAggSupp;
@@ -1338,7 +1355,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
   finalizeMultiTupleQueryResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo,
                                 pBInfo->rowCellInfoOffset);
 
-  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true);
+  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
   blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
   doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
   if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) {

source/libs/function/inc/builtinsimpl.h

@@ -90,6 +90,10 @@ bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultIn
 int32_t histogramFunction(SqlFunctionCtx* pCtx);
 int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 
+bool getHLLFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
+int32_t hllFunction(SqlFunctionCtx* pCtx);
+int32_t hllFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+
 bool getStateFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool stateFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t stateCountFunction(SqlFunctionCtx* pCtx);
@@ -105,7 +109,12 @@ int32_t mavgFunction(SqlFunctionCtx* pCtx);
 bool getSampleFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool sampleFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t sampleFunction(SqlFunctionCtx* pCtx);
-int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+//int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+
+bool getTailFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
+bool tailFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
+int32_t tailFunction(SqlFunctionCtx* pCtx);
+int32_t tailFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 
 bool getSelectivityFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 

source/libs/function/src/builtins.c

@@ -263,6 +263,21 @@ static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t l
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t translateHLL(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  if (1 != LIST_LENGTH(pFunc->pParameterList)) {
+    return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
+  }
+
+  SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0);
+  if (QUERY_NODE_COLUMN != nodeType(pPara)) {
+    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                           "The input parameter of HYPERLOGLOG function can only be column");
+  }
+
+  pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_UBIGINT].bytes, .type = TSDB_DATA_TYPE_UBIGINT};
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t translateStateCount(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   if (3 != LIST_LENGTH(pFunc->pParameterList)) {
     return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
@@ -386,6 +401,35 @@ static int32_t translateSample(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t translateTail(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
+  if (2 != numOfParams && 3 != numOfParams) {
+    return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
+  }
+
+  SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0);
+  if (QUERY_NODE_COLUMN != nodeType(pPara)) {
+    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                           "The input parameter of TAIL function can only be column");
+  }
+
+  for (int32_t i = 1; i < numOfParams; ++i) {
+    uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, i))->resType.type;
+    if (!IS_INTEGER_TYPE(paraType)) {
+      return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+  }
+
+  SExprNode* pCol = (SExprNode*)nodesListGetNode(pFunc->pParameterList, 0);
+  uint8_t colType = pCol->resType.type;
+  if (IS_VAR_DATA_TYPE(colType)) {
+    pFunc->node.resType = (SDataType){.bytes = pCol->resType.bytes, .type = colType};
+  } else {
+    pFunc->node.resType = (SDataType){.bytes = tDataTypes[colType].bytes, .type = colType};
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t translateLastRow(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   // todo
   return TSDB_CODE_SUCCESS;
@@ -800,6 +844,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .processFunc  = histogramFunction,
     .finalizeFunc = histogramFinalize
   },
+  {
+    .name = "hyperloglog",
+    .type = FUNCTION_TYPE_HYPERLOGLOG,
+    .classification = FUNC_MGT_AGG_FUNC,
+    .translateFunc = translateHLL,
+    .getEnvFunc   = getHLLFuncEnv,
+    .initFunc     = functionSetup,
+    .processFunc  = hllFunction,
+    .finalizeFunc = hllFinalize
+  },
   {
     .name = "state_count",
     .type = FUNCTION_TYPE_STATE_COUNT,
@@ -850,6 +904,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .processFunc  = sampleFunction,
     .finalizeFunc = NULL
   },
+  {
+    .name = "tail",
+    .type = FUNCTION_TYPE_TAIL,
+    .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC,
+    .translateFunc = translateTail,
+    .getEnvFunc   = getTailFuncEnv,
+    .initFunc     = tailFunctionSetup,
+    .processFunc  = tailFunction,
+    .finalizeFunc = tailFinalize
+  },
   {
     .name = "abs",
     .type = FUNCTION_TYPE_ABS,

source/libs/function/src/builtinsimpl.c

@@ -18,12 +18,21 @@
 #include "function.h"
 #include "querynodes.h"
 #include "taggfunction.h"
+#include "tcompare.h"
 #include "tdatablock.h"
 #include "tpercentile.h"
 
 #define HISTOGRAM_MAX_BINS_NUM   1000
 #define MAVG_MAX_POINTS_NUM      1000
 #define SAMPLE_MAX_POINTS_NUM    1000
+#define TAIL_MAX_POINTS_NUM      100
+#define TAIL_MAX_OFFSET          100
+
+#define HLL_BUCKET_BITS 14 // The bits of the bucket
+#define HLL_DATA_BITS (64-HLL_BUCKET_BITS)
+#define HLL_BUCKETS (1<<HLL_BUCKET_BITS)
+#define HLL_BUCKET_MASK (HLL_BUCKETS-1)
+#define HLL_ALPHA_INF 0.721347520444481703680 // constant for 0.5/ln(2)
 
 typedef struct SSumRes {
   union {
@@ -126,6 +135,11 @@ typedef enum {
   LOG_BIN
 } EHistoBinType;
 
+typedef struct SHLLFuncInfo {
+  uint64_t result;
+  uint8_t buckets[HLL_BUCKETS];
+} SHLLInfo;
+
 typedef struct SStateInfo {
   union {
     int64_t count;
@@ -161,6 +175,21 @@ typedef struct SSampleInfo {
   int64_t *timestamp;
 } SSampleInfo;
 
+typedef struct STailItem {
+  int64_t timestamp;
+  bool    isNull;
+  char    data[];
+} STailItem;
+
+typedef struct STailInfo {
+  int32_t   numOfPoints;
+  int32_t   numAdded;
+  int32_t   offset;
+  uint8_t   colType;
+  int16_t   colBytes;
+  STailItem **pItems;
+} STailInfo;
+
 #define SET_VAL(_info, numOfElem, res) \
   do {                                 \
     if ((numOfElem) <= 0) {            \
@@ -2711,6 +2740,140 @@ int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   return pResInfo->numOfRes;
 }
 
+bool getHLLFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
+  pEnv->calcMemSize = sizeof(SHLLInfo);
+  return true;
+}
+
+static uint8_t hllCountNum(void* data, int32_t bytes, int32_t *buk) {
+  uint64_t hash = MurmurHash3_64(data, bytes);
+  int32_t index = hash & HLL_BUCKET_MASK;
+  hash >>= HLL_BUCKET_BITS;
+  hash |= ((uint64_t)1 << HLL_DATA_BITS);
+  uint64_t bit = 1;
+  uint8_t count = 1;
+  while((hash & bit) == 0) {
+    count++;
+    bit <<= 1;
+  }
+  *buk = index;
+  return count;
+}
+
+static void hllBucketHisto(uint8_t *buckets, int32_t* bucketHisto) {
+  uint64_t *word = (uint64_t*) buckets;
+  uint8_t *bytes;
+
+  for (int32_t j = 0; j < HLL_BUCKETS>>3; j++) {
+    if (*word == 0) {
+      bucketHisto[0] += 8;
+    } else {
+      bytes = (uint8_t*) word;
+      bucketHisto[bytes[0]]++;
+      bucketHisto[bytes[1]]++;
+      bucketHisto[bytes[2]]++;
+      bucketHisto[bytes[3]]++;
+      bucketHisto[bytes[4]]++;
+      bucketHisto[bytes[5]]++;
+      bucketHisto[bytes[6]]++;
+      bucketHisto[bytes[7]]++;
+    }
+    word++;
+  }
+}
+static double hllTau(double x) {
+  if (x == 0. || x == 1.) return 0.;
+  double zPrime;
+  double y = 1.0;
+  double z = 1 - x;
+  do {
+    x = sqrt(x);
+    zPrime = z;
+    y *= 0.5;
+    z -= pow(1 - x, 2)*y;
+  } while(zPrime != z);
+  return z / 3;
+}
+
+static double hllSigma(double x) {
+  if (x == 1.0) return INFINITY;
+  double zPrime;
+  double y = 1;
+  double z = x;
+  do {
+    x *= x;
+    zPrime = z;
+    z += x * y;
+    y += y;
+  } while(zPrime != z);
+  return z;
+}
+
+// estimate the cardinality, the algorithm refer this paper: "New cardinality estimation algorithms for HyperLogLog sketches"
+static uint64_t hllCountCnt(uint8_t *buckets) {
+  double m = HLL_BUCKETS;
+  int32_t buckethisto[64] = {0};
+  hllBucketHisto(buckets,buckethisto);
+
+  double z = m * hllTau((m-buckethisto[HLL_DATA_BITS+1])/(double)m);
+  for (int j = HLL_DATA_BITS; j >= 1; --j) {
+    z += buckethisto[j];
+    z *= 0.5;
+  }
+  z += m * hllSigma(buckethisto[0]/(double)m);
+  double E = (double)llroundl(HLL_ALPHA_INF*m*m/z);
+
+  return (uint64_t) E;
+}
+
+
+int32_t hllFunction(SqlFunctionCtx *pCtx) {
+  SHLLInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+
+  SInputColumnInfoData* pInput = &pCtx->input;
+  SColumnInfoData*      pCol = pInput->pData[0];
+
+  int32_t type  = pCol->info.type;
+  int32_t bytes = pCol->info.bytes;
+
+  int32_t start = pInput->startRowIndex;
+  int32_t numOfRows = pInput->numOfRows;
+
+  int32_t numOfElems = 0;
+  for (int32_t i = start; i < numOfRows + start; ++i) {
+    if (pCol->hasNull && colDataIsNull_s(pCol, i)) {
+      continue;
+    }
+
+    numOfElems++;
+
+    char* data = colDataGetData(pCol, i);
+    if (IS_VAR_DATA_TYPE(type)) {
+      bytes = varDataLen(data);
+      data = varDataVal(data);
+    }
+
+    int32_t index = 0;
+    uint8_t count = hllCountNum(data, bytes, &index);
+    uint8_t oldcount = pInfo->buckets[index];
+    if (count > oldcount) {
+      pInfo->buckets[index] = count;
+    }
+
+  }
+
+  SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1);
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t hllFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
+  SHLLInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+
+  pInfo->result = hllCountCnt(pInfo->buckets);
+
+  return functionFinalize(pCtx, pBlock);
+}
+
 bool getStateFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
   pEnv->calcMemSize = sizeof(SStateInfo);
   return true;
@@ -3107,7 +3270,7 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) {
 
   int32_t startOffset = pCtx->offset;
   for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) {
-    if (colDataIsNull_f(pInputCol->nullbitmap, i)) {
+    if (colDataIsNull_s(pInputCol, i)) {
       //colDataAppendNULL(pOutput, i);
       continue;
     }
@@ -3141,3 +3304,132 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) {
 //
 //  return pResInfo->numOfRes;
 //}
+
+
+bool getTailFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
+  SColumnNode* pCol = (SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0);
+  SValueNode*  pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1);
+  int32_t numOfPoints = pVal->datum.i;
+  pEnv->calcMemSize = sizeof(STailInfo) + numOfPoints * (POINTER_BYTES + sizeof(STailItem) + pCol->node.resType.bytes);
+  return true;
+}
+
+bool tailFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo *pResultInfo) {
+  if (!functionSetup(pCtx, pResultInfo)) {
+    return false;
+  }
+
+  STailInfo *pInfo = GET_ROWCELL_INTERBUF(pResultInfo);
+  pInfo->numAdded = 0;
+  pInfo->numOfPoints = pCtx->param[1].param.i;
+  if (pCtx->numOfParams == 4) {
+    pInfo->offset = pCtx->param[2].param.i;
+  } else {
+    pInfo->offset = 0;
+  }
+  pInfo->colType = pCtx->resDataInfo.type;
+  pInfo->colBytes = pCtx->resDataInfo.bytes;
+  if ((pInfo->numOfPoints < 1 || pInfo->numOfPoints > TAIL_MAX_POINTS_NUM) ||
+      (pInfo->numOfPoints < 0 || pInfo->numOfPoints > TAIL_MAX_OFFSET)) {
+    return false;
+  }
+
+  pInfo->pItems = (STailItem **)((char *)pInfo + sizeof(STailInfo));
+  char *pItem = (char *)pInfo->pItems + pInfo->numOfPoints * POINTER_BYTES;
+
+  size_t unitSize = sizeof(STailItem) + pInfo->colBytes;
+  for (int32_t i = 0; i < pInfo->numOfPoints; ++i) {
+    pInfo->pItems[i] = (STailItem *)(pItem + i * unitSize);
+    pInfo->pItems[i]->isNull = false;
+  }
+
+  return true;
+}
+
+static void tailAssignResult(STailItem* pItem, char *data, int32_t colBytes, TSKEY ts, bool isNull) {
+  pItem->timestamp = ts;
+  if (isNull) {
+    pItem->isNull = true;
+  } else {
+    memcpy(pItem->data, data, colBytes);
+  }
+}
+
+static int32_t tailCompFn(const void *p1, const void *p2, const void *param) {
+  STailItem *d1 = *(STailItem **)p1;
+  STailItem *d2 = *(STailItem **)p2;
+  return compareInt64Val(&d1->timestamp, &d2->timestamp);
+}
+
+static void doTailAdd(STailInfo* pInfo, char *data, TSKEY ts, bool isNull) {
+  STailItem **pList = pInfo->pItems;
+  if (pInfo->numAdded < pInfo->numOfPoints) {
+    tailAssignResult(pList[pInfo->numAdded], data, pInfo->colBytes, ts, isNull);
+    taosheapsort((void *)pList, sizeof(STailItem **), pInfo->numAdded + 1, NULL, tailCompFn, 0);
+    pInfo->numAdded++;
+  } else if (pList[0]->timestamp < ts) {
+    tailAssignResult(pList[0], data, pInfo->colBytes, ts, isNull);
+    taosheapadjust((void *)pList, sizeof(STailItem **), 0, pInfo->numOfPoints - 1, NULL, tailCompFn, NULL, 0);
+  }
+}
+
+int32_t tailFunction(SqlFunctionCtx* pCtx) {
+  SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
+  STailInfo*         pInfo = GET_ROWCELL_INTERBUF(pResInfo);
+
+  SInputColumnInfoData* pInput = &pCtx->input;
+  TSKEY* tsList = (int64_t*)pInput->pPTS->pData;
+
+  SColumnInfoData* pInputCol = pInput->pData[0];
+  SColumnInfoData* pTsOutput = pCtx->pTsOutput;
+  SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput;
+
+  int32_t startOffset = pCtx->offset;
+  if (pInfo->offset >= pInput->numOfRows) {
+    return 0;
+  } else {
+    pInfo->numOfPoints = TMIN(pInfo->numOfPoints, pInput->numOfRows - pInfo->offset);
+  }
+  for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex - pInfo->offset; i += 1) {
+
+    char* data = colDataGetData(pInputCol, i);
+    doTailAdd(pInfo, data, tsList[i], colDataIsNull_s(pInputCol, i));
+  }
+
+  taosqsort(pInfo->pItems, pInfo->numOfPoints, POINTER_BYTES, NULL, tailCompFn);
+
+  for (int32_t i = 0; i < pInfo->numOfPoints; ++i) {
+    int32_t pos = startOffset + i;
+    STailItem *pItem = pInfo->pItems[i];
+    if (pItem->isNull) {
+      colDataAppendNULL(pOutput, pos);
+    } else {
+      colDataAppend(pOutput, pos, pItem->data, false);
+    }
+  }
+
+  return pInfo->numOfPoints;
+}
+
+int32_t tailFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
+  SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx);
+  STailInfo*                pInfo = GET_ROWCELL_INTERBUF(pEntryInfo);
+  pEntryInfo->complete = true;
+
+  int32_t type = pCtx->input.pData[0]->info.type;
+  int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
+
+  SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
+
+  // todo assign the tag value and the corresponding row data
+  int32_t currentRow = pBlock->info.rows;
+  for (int32_t i = 0; i < pEntryInfo->numOfRes; ++i) {
+    STailItem *pItem = pInfo->pItems[i];
+    colDataAppend(pCol, currentRow, pItem->data, false);
+
+    //setSelectivityValue(pCtx, pBlock, &pInfo->pItems[i].tuplePos, currentRow);
+    currentRow += 1;
+  }
+
+  return pEntryInfo->numOfRes;
+}

source/libs/function/src/tudf.c

@@ -312,6 +312,7 @@ typedef struct SUdfcFuncStub {
   char udfName[TSDB_FUNC_NAME_LEN];
   UdfcFuncHandle handle;
   int32_t refCount;
+  int64_t lastRefTime;
 } SUdfcFuncStub;
 
 typedef struct SUdfcProxy {
@@ -794,7 +795,6 @@ int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SS
   }
   output->info.hasVarCol = hasVarCol;
 
-  //TODO: free the array output->pDataBlock
   output->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
   for (int32_t i = 0; i < numOfCols; ++i) {
     taosArrayPush(output->pDataBlock, (input + i)->columnData);
@@ -808,8 +808,12 @@ int32_t convertDataBlockToScalarParm(SSDataBlock *input, SScalarParam *output) {
     return -1;
   }
   output->numOfRows = input->info.rows;
-  //TODO: memory
+
-  output->columnData = taosArrayGet(input->pDataBlock, 0);
+  output->columnData = taosMemoryMalloc(sizeof(SColumnInfoData));
+  memcpy(output->columnData,
+         taosArrayGet(input->pDataBlock, 0),
+         sizeof(SColumnInfoData));
+
   return 0;
 }
 
@@ -832,7 +836,7 @@ int32_t udfcGetUdfTaskResultFromUvTask(SClientUdfTask *task, SClientUvTaskNode *
   fnDebug("udfc get uv task result. task: %p, uvTask: %p", task, uvTask);
   if (uvTask->type == UV_TASK_REQ_RSP) {
     if (uvTask->rspBuf.base != NULL) {
-      SUdfResponse rsp;
+      SUdfResponse rsp = {0};
       void* buf = decodeUdfResponse(uvTask->rspBuf.base, &rsp);
       assert(uvTask->rspBuf.len == POINTER_DISTANCE(buf, uvTask->rspBuf.base));
       task->errCode = rsp.code;
@@ -1283,7 +1287,7 @@ int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) {
   task->type = UDF_TASK_SETUP;
 
   SUdfSetupRequest *req = &task->_setup.req;
-  memcpy(req->udfName, udfName, TSDB_FUNC_NAME_LEN);
+  strncpy(req->udfName, udfName, TSDB_FUNC_NAME_LEN);
 
   int32_t errCode = udfcRunUdfUvTask(task, UV_TASK_CONNECT);
   if (errCode != 0) {
@@ -1426,7 +1430,10 @@ int32_t doCallUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t
   int32_t err = callUdf(handle, callType, &inputBlock, NULL, NULL, &resultBlock, NULL);
   if (err == 0) {
     convertDataBlockToScalarParm(&resultBlock, output);
+    taosArrayDestroy(resultBlock.pDataBlock);
   }
+
+  taosArrayDestroy(inputBlock.pDataBlock);
   return err;
 }
 
@@ -1446,6 +1453,7 @@ int32_t accquireUdfFuncHandle(char* udfName, UdfcFuncHandle* pHandle) {
     uv_mutex_unlock(&gUdfdProxy.udfStubsMutex);
     *pHandle = foundStub->handle;
     ++foundStub->refCount;
+    foundStub->lastRefTime = taosGetTimestampUs();
     return 0;
   }
   *pHandle = NULL;
@@ -1455,6 +1463,7 @@ int32_t accquireUdfFuncHandle(char* udfName, UdfcFuncHandle* pHandle) {
     strcpy(stub.udfName, udfName);
     stub.handle = *pHandle;
     ++stub.refCount;
+    stub.lastRefTime = taosGetTimestampUs();
     taosArrayPush(gUdfdProxy.udfStubs, &stub);
     taosArraySort(gUdfdProxy.udfStubs, compareUdfcFuncSub);
   } else {
@@ -1505,16 +1514,15 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) {
 
   udfcRunUdfUvTask(task, UV_TASK_REQ_RSP);
 
-  SUdfTeardownResponse *rsp = &task->_teardown.rsp;
   int32_t err = task->errCode;
 
   udfcRunUdfUvTask(task, UV_TASK_DISCONNECT);
 
+  fnInfo("tear down udf. udf name: %s, udf func handle: %p", session->udfName, handle);
+
   taosMemoryFree(task->session);
   taosMemoryFree(task);
 
-  fnInfo("tear down udf. udf name: %s, udf func handle: %p", session->udfName, handle);
-
   return err;
 }
 
@@ -1561,6 +1569,7 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult
   }
   udfRes->interResNum = buf.numOfResult;
   memcpy(udfRes->interResBuf, buf.buf, buf.bufLen);
+  freeUdfInterBuf(&buf);
   return true;
 }
 
@@ -1618,7 +1627,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
   blockDataDestroy(inputBlock);
   taosArrayDestroy(tempBlock.pDataBlock);
 
-  taosMemoryFree(newState.buf);
+  freeUdfInterBuf(&newState);
   return udfCode;
 }
 
@@ -1647,6 +1656,8 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) {
     GET_RES_INFO(pCtx)->numOfRes = udfRes->finalResNum;
   }
 
+  freeUdfInterBuf(&resultBuf);
+
   int32_t numOfResults = functionFinalizeWithResultBuf(pCtx, pBlock, udfRes->finalResBuf);
   releaseUdfFuncHandle(pCtx->udfName);
   return udfCallCode == 0 ? numOfResults : udfCallCode;
@@ -1662,7 +1673,8 @@ int32_t cleanUpUdfs() {
       fnInfo("tear down udf. udf name: %s, handle: %p", stub->udfName, stub->handle);
       doTeardownUdf(stub->handle);
     } else {
-      fnInfo("udf still in use. udf name: %s, ref count: %d, handle: %p", stub->udfName, stub->refCount, stub->handle);
+      fnInfo("udf still in use. udf name: %s, ref count: %d, last ref time: %"PRId64", handle: %p",
+             stub->udfName, stub->refCount, stub->lastRefTime, stub->handle);
       taosArrayPush(udfStubs, stub);
     }
     ++i;

source/libs/function/src/udfd.c

@@ -96,10 +96,14 @@ int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf);
 
 int32_t udfdLoadUdf(char *udfName, SUdf *udf) {
   strcpy(udf->name, udfName);
+  int32_t err = 0;
+  err = udfdFillUdfInfoFromMNode(global.clientRpc, udf->name, udf);
+  if (err != 0) {
+    fnError("can not retrieve udf from mnode. udf name %s", udfName);
+    return TSDB_CODE_UDF_LOAD_UDF_FAILURE;
+  }
 
-  udfdFillUdfInfoFromMNode(global.clientRpc, udf->name, udf);
+  err = uv_dlopen(udf->path, &udf->lib);
-  //strcpy(udf->path, "/home/slzhou/TDengine/debug/build/lib/libudf1.so");
-  int err = uv_dlopen(udf->path, &udf->lib);
   if (err != 0) {
     fnError("can not load library %s. error: %s", udf->path, uv_strerror(err));
     return TSDB_CODE_UDF_LOAD_UDF_FAILURE;
@@ -142,7 +146,7 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) {
 
 void udfdProcessSetupRequest(SUvUdfWork* uvUdf, SUdfRequest* request) {
     // TODO: tracable id from client. connect, setup, call, teardown
-    fnInfo("%" PRId64 " setup request. udf name: %s", request->seqNum, request->setup.udfName);
+    fnInfo( "setup request. seq num: %" PRId64 ", udf name: %s", request->seqNum, request->setup.udfName);
     SUdfSetupRequest *setup = &request->setup;
     int32_t code = TSDB_CODE_SUCCESS;
     SUdf *udf = NULL;
@@ -222,7 +226,7 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
       SUdfDataBlock input = {0};
       convertDataBlockToUdfDataBlock(&call->block, &input);
       code = udf->scalarProcFunc(&input, &output);
-
+      freeUdfDataDataBlock(&input);
       convertUdfColumnToDataBlock(&output, &response.callRsp.resultData);
       freeUdfColumn(&output);
       break;
@@ -242,6 +246,8 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
                              .bufLen= udf->bufSize,
                              .numOfResult = 0};
       code = udf->aggProcFunc(&input, &call->interBuf, &outBuf);
+      freeUdfInterBuf(&call->interBuf);
+      freeUdfDataDataBlock(&input);
       subRsp->resultBuf = outBuf;
 
       break;
@@ -251,6 +257,7 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
                              .bufLen= udf->bufSize,
                              .numOfResult = 0};
       code = udf->aggFinishFunc(&call->interBuf, &outBuf);
+      freeUdfInterBuf(&call->interBuf);
       subRsp->resultBuf = outBuf;
       break;
     }
@@ -270,13 +277,37 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
   encodeUdfResponse(&buf, rsp);
   uvUdf->output = uv_buf_init(bufBegin, len);
 
+  switch (call->callType) {
+    case TSDB_UDF_CALL_SCALA_PROC: {
+      tDeleteSSDataBlock(&call->block);
+      tDeleteSSDataBlock(&subRsp->resultData);
+      break;
+    }
+    case TSDB_UDF_CALL_AGG_INIT: {
+      freeUdfInterBuf(&subRsp->resultBuf);
+      break;
+    }
+    case TSDB_UDF_CALL_AGG_PROC: {
+      tDeleteSSDataBlock(&call->block);
+      freeUdfInterBuf(&subRsp->resultBuf);
+      break;
+    }
+    case TSDB_UDF_CALL_AGG_FIN: {
+      freeUdfInterBuf(&subRsp->resultBuf);
+      break;
+    }
+    default:
+      break;
+
+  }
+
   taosMemoryFree(uvUdf->input.base);
   return;
 }
 
 void udfdProcessTeardownRequest(SUvUdfWork* uvUdf, SUdfRequest* request) {
   SUdfTeardownRequest *teardown = &request->teardown;
-  fnInfo("teardown. %" PRId64 "handle:%" PRIx64, request->seqNum, teardown->udfHandle);
+  fnInfo("teardown. seq number: %" PRId64 ", handle:%" PRIx64, request->seqNum, teardown->udfHandle);
   SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(teardown->udfHandle);
   SUdf *udf = handle->udf;
   bool  unloadUdf = false;
@@ -344,9 +375,8 @@ void udfdProcessRequest(uv_work_t *req) {
 void udfdOnWrite(uv_write_t *req, int status) {
   SUvUdfWork *work = (SUvUdfWork *)req->data;
   if (status < 0) {
-    // TODO:log error and process it.
+    fnError("udfd send response error, length: %zu code: %s", work->output.len, uv_err_name(status));
   }
-  fnDebug("send response. length:%zu, status: %s", work->output.len, uv_err_name(status));
   taosMemoryFree(work->output.base);
   taosMemoryFree(work);
   taosMemoryFree(req);
@@ -485,7 +515,147 @@ void udfdIntrSignalHandler(uv_signal_t *handle, int signum) {
   uv_stop(global.loop);
 }
 
-void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { return; }
+typedef enum EUdfdRpcReqRspType {
+  UDFD_RPC_MNODE_CONNECT = 0,
+  UDFD_RPC_RETRIVE_FUNC,
+} EUdfdRpcReqRspType;
+
+typedef struct SUdfdRpcSendRecvInfo {
+  EUdfdRpcReqRspType rpcType;
+  int32_t code;
+  void* param;
+  uv_sem_t resultSem;
+} SUdfdRpcSendRecvInfo;
+
+
+void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
+  SUdfdRpcSendRecvInfo *msgInfo = (SUdfdRpcSendRecvInfo *)pMsg->ahandle;
+  ASSERT(pMsg->ahandle != NULL);
+
+  if (pEpSet) {
+    if (!isEpsetEqual(&global.mgmtEp.epSet, pEpSet)) {
+      updateEpSet_s(&global.mgmtEp, pEpSet);
+    }
+  }
+
+  if (pMsg->code != TSDB_CODE_SUCCESS) {
+    fnError("udfd rpc error. code: %s", tstrerror(pMsg->code));
+    msgInfo->code = pMsg->code;
+    goto _return;
+  }
+
+  if (msgInfo->rpcType == UDFD_RPC_MNODE_CONNECT) {
+    SConnectRsp connectRsp = {0};
+    tDeserializeSConnectRsp(pMsg->pCont, pMsg->contLen, &connectRsp);
+    if (connectRsp.epSet.numOfEps == 0) {
+      msgInfo->code = TSDB_CODE_MND_APP_ERROR;
+      goto _return;
+    }
+
+    if (connectRsp.dnodeNum > 1 && !isEpsetEqual(&global.mgmtEp.epSet, &connectRsp.epSet)) {
+      updateEpSet_s(&global.mgmtEp, &connectRsp.epSet);
+    }
+    msgInfo->code = 0;
+  } else if (msgInfo->rpcType == UDFD_RPC_RETRIVE_FUNC) {
+    SRetrieveFuncRsp retrieveRsp = {0};
+    tDeserializeSRetrieveFuncRsp(pMsg->pCont, pMsg->contLen, &retrieveRsp);
+
+    SFuncInfo *pFuncInfo = (SFuncInfo *)taosArrayGet(retrieveRsp.pFuncInfos, 0);
+    SUdf* udf = msgInfo->param;
+    udf->funcType = pFuncInfo->funcType;
+    udf->scriptType = pFuncInfo->scriptType;
+    udf->outputType = pFuncInfo->funcType;
+    udf->outputLen = pFuncInfo->outputLen;
+    udf->bufSize = pFuncInfo->bufSize;
+
+    char path[PATH_MAX] = {0};
+    snprintf(path, sizeof(path), "%s/lib%s.so", "/tmp", pFuncInfo->name);
+    TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL);
+    // TODO check for failure of flush to disk
+    taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize);
+    taosCloseFile(&file);
+    strncpy(udf->path, path, strlen(path));
+    tFreeSFuncInfo(pFuncInfo);
+    taosArrayDestroy(retrieveRsp.pFuncInfos);
+    msgInfo->code = 0;
+  }
+
+_return:
+  rpcFreeCont(pMsg->pCont);
+  uv_sem_post(&msgInfo->resultSem);
+  return;
+}
+
+int32_t udfdConnectToMNode() {
+  SConnectReq connReq = {0};
+  connReq.connType = CONN_TYPE__UDFD;
+  tstrncpy(connReq.app, "udfd",sizeof(connReq.app));
+  tstrncpy(connReq.user, TSDB_DEFAULT_USER, sizeof(connReq.user));
+  char pass[TSDB_PASSWORD_LEN + 1] = {0};
+  taosEncryptPass_c((uint8_t *)(TSDB_DEFAULT_PASS), strlen(TSDB_DEFAULT_PASS), pass);
+  tstrncpy(connReq.passwd, pass, sizeof(connReq.passwd));
+  connReq.pid = htonl(taosGetPId());
+  connReq.startTime = htobe64(taosGetTimestampMs());
+
+  int32_t contLen = tSerializeSConnectReq(NULL, 0, &connReq);
+  void*   pReq = rpcMallocCont(contLen);
+  tSerializeSConnectReq(pReq, contLen, &connReq);
+
+  SUdfdRpcSendRecvInfo *msgInfo = taosMemoryCalloc(1, sizeof(SUdfdRpcSendRecvInfo));
+  msgInfo->rpcType = UDFD_RPC_MNODE_CONNECT;
+  uv_sem_init(&msgInfo->resultSem, 0);
+
+  SRpcMsg rpcMsg = {0};
+  rpcMsg.msgType = TDMT_MND_CONNECT;
+  rpcMsg.pCont = pReq;
+  rpcMsg.contLen = contLen;
+  rpcMsg.ahandle = msgInfo;
+  rpcSendRequest(global.clientRpc, &global.mgmtEp.epSet, &rpcMsg, NULL);
+
+  uv_sem_wait(&msgInfo->resultSem);
+  int32_t code = msgInfo->code;
+  uv_sem_destroy(&msgInfo->resultSem);
+  taosMemoryFree(msgInfo);
+  return code;
+}
+
+int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf) {
+  SRetrieveFuncReq retrieveReq = {0};
+  retrieveReq.numOfFuncs = 1;
+  retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
+  taosArrayPush(retrieveReq.pFuncNames, udfName);
+
+  int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
+  void   *pReq = rpcMallocCont(contLen);
+  tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq);
+  taosArrayDestroy(retrieveReq.pFuncNames);
+
+  SUdfdRpcSendRecvInfo* msgInfo = taosMemoryCalloc(1, sizeof(SUdfdRpcSendRecvInfo));
+  msgInfo->rpcType = UDFD_RPC_RETRIVE_FUNC;
+  msgInfo->param = udf;
+  uv_sem_init(&msgInfo->resultSem, 0);
+
+  SRpcMsg rpcMsg = {0};
+  rpcMsg.pCont = pReq;
+  rpcMsg.contLen = contLen;
+  rpcMsg.msgType = TDMT_MND_RETRIEVE_FUNC;
+  rpcMsg.ahandle = msgInfo;
+  rpcSendRequest(clientRpc, &global.mgmtEp.epSet, &rpcMsg, NULL);
+
+  uv_sem_wait(&msgInfo->resultSem);
+  uv_sem_destroy(&msgInfo->resultSem);
+  int32_t code = msgInfo->code;
+  taosMemoryFree(msgInfo);
+  return code;
+}
+
+static bool udfdRpcRfp(int32_t code) {
+  if (code == TSDB_CODE_RPC_REDIRECT) {
+    return true;
+  } else {
+    return false;
+  }
+}
 
 int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet) {
   pEpSet->version = 0;
@@ -528,69 +698,30 @@ int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSe
   return 0;
 }
 
-int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf) {
-  SRetrieveFuncReq retrieveReq = {0};
-  retrieveReq.numOfFuncs = 1;
-  retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
-  taosArrayPush(retrieveReq.pFuncNames, udfName);
-
-  int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
-  void   *pReq = rpcMallocCont(contLen);
-  tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq);
-  taosArrayDestroy(retrieveReq.pFuncNames);
-
-  SRpcMsg rpcMsg = {0};
-  rpcMsg.pCont = pReq;
-  rpcMsg.contLen = contLen;
-  rpcMsg.msgType = TDMT_MND_RETRIEVE_FUNC;
-
-  SRpcMsg rpcRsp = {0};
-  rpcSendRecv(clientRpc, &global.mgmtEp.epSet, &rpcMsg, &rpcRsp);
-  SRetrieveFuncRsp retrieveRsp = {0};
-  tDeserializeSRetrieveFuncRsp(rpcRsp.pCont, rpcRsp.contLen, &retrieveRsp);
-
-  SFuncInfo *pFuncInfo = (SFuncInfo *)taosArrayGet(retrieveRsp.pFuncInfos, 0);
-
-  udf->funcType = pFuncInfo->funcType;
-  udf->scriptType = pFuncInfo->scriptType;
-  udf->outputType = pFuncInfo->funcType;
-  udf->outputLen = pFuncInfo->outputLen;
-  udf->bufSize = pFuncInfo->bufSize;
-
-  char path[PATH_MAX] = {0};
-  snprintf(path, sizeof(path), "%s/lib%s.so", "/tmp", udfName);
-  TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL);
-  // TODO check for failure of flush to disk
-  taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize);
-  taosCloseFile(&file);
-  strncpy(udf->path, path, strlen(path));
-  taosArrayDestroy(retrieveRsp.pFuncInfos);
-
-  rpcFreeCont(rpcRsp.pCont);
-  return 0;
-}
 
 int32_t udfdOpenClientRpc() {
-  char *pass = "taosdata";
-  char *user = "root";
-  char  secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0};
-  taosEncryptPass_c((uint8_t *)pass, strlen(pass), secretEncrypt);
   SRpcInit rpcInit = {0};
-  rpcInit.label = (char *)"UDFD";
+  rpcInit.label = "UDFD";
   rpcInit.numOfThreads = 1;
-  rpcInit.cfp = udfdProcessRpcRsp;
+  rpcInit.cfp = (RpcCfp)udfdProcessRpcRsp;
   rpcInit.sessions = 1024;
   rpcInit.connType = TAOS_CONN_CLIENT;
-  rpcInit.idleTime = 30 * 1000;
+  rpcInit.idleTime = tsShellActivityTimer * 1000;
-  rpcInit.parent = &global;
+  rpcInit.user = TSDB_DEFAULT_USER;
-
+  rpcInit.ckey = "key";
-  rpcInit.user = (char *)user;
-  rpcInit.ckey = (char *)"key";
-  rpcInit.secret = (char *)secretEncrypt;
   rpcInit.spi = 1;
+  rpcInit.parent = &global;
+  rpcInit.rfp = udfdRpcRfp;
+
+  char pass[TSDB_PASSWORD_LEN + 1] = {0};
+  taosEncryptPass_c((uint8_t *)(TSDB_DEFAULT_PASS), strlen(TSDB_DEFAULT_PASS), pass);
+  rpcInit.secret = pass;
 
   global.clientRpc = rpcOpen(&rpcInit);
-
+  if (global.clientRpc == NULL) {
+    fnError("failed to init dnode rpc client");
+    return -1;
+  }
   return 0;
 }
 
@@ -696,28 +827,26 @@ static int32_t udfdUvInit() {
   return 0;
 }
 
+static void udfdCloseWalkCb(uv_handle_t* handle, void* arg) {
+  if (!uv_is_closing(handle)) {
+    uv_close(handle, NULL);
+  }
+}
+
 static int32_t udfdRun() {
   global.udfsHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
   uv_mutex_init(&global.udfsMutex);
 
-  // TOOD: client rpc to fetch udf function info from mnode
+  fnInfo("start udfd event loop");
-  if (udfdOpenClientRpc() != 0) {
+  uv_run(global.loop, UV_RUN_DEFAULT);
-    fnError("open rpc connection to mnode failure");
+  fnInfo("udfd event loop stopped.");
-    return -1;
-  }
 
-  if (udfdUvInit() != 0) {
+  uv_loop_close(global.loop);
-    fnError("uv init failure");
+
-    return -2;
+  uv_walk(global.loop, udfdCloseWalkCb, NULL);
-  }
+  uv_run(global.loop, UV_RUN_DEFAULT);
+  uv_loop_close(global.loop);
 
-  fnInfo("start the udfd");
-  int code = uv_run(global.loop, UV_RUN_DEFAULT);
-  fnInfo("udfd stopped. result: %s, code: %d", uv_err_name(code), code);
-  int codeClose = uv_loop_close(global.loop);
-  fnDebug("uv loop close. result: %s", uv_err_name(codeClose));
-  removeListeningPipe();
-  udfdCloseClientRpc();
   uv_mutex_destroy(&global.udfsMutex);
   taosHashCleanup(global.udfsHash);
   return 0;
@@ -746,9 +875,28 @@ int main(int argc, char *argv[]) {
 
   if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) {
     fnError("failed to start since read config error");
-    return -1;
+    return -2;
   }
 
   initEpSetFromCfg(tsFirst, tsSecond, &global.mgmtEp);
-  return udfdRun();
+  if (udfdOpenClientRpc() != 0) {
+    fnError("open rpc connection to mnode failure");
+    return -3;
+  }
+
+  if (udfdConnectToMNode() != 0) {
+    fnError("failed to start since can not connect to mnode");
+    return -4;
+  }
+
+  if (udfdUvInit() != 0) {
+    fnError("uv init failure");
+    return -5;
+  }
+
+  udfdRun();
+
+  removeListeningPipe();
+
+  udfdCloseClientRpc();
 }

source/libs/function/test/runUdf.c

@@ -34,20 +34,13 @@ static int32_t initLog() {
   return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, NULL, 0);
 }
 
-int main(int argc, char *argv[]) {
+int scalarFuncTest() {
-  parseArgs(argc, argv);
-  initLog();
-  if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) {
-    fnError("failed to start since read config error");
-    return -1;
-  }
-
-  udfcOpen();
-  uv_sleep(1000);
-
   UdfcFuncHandle handle;
 
-  doSetupUdf("udf1", &handle);
+  if (doSetupUdf("udf1", &handle) != 0) {
+    fnError("setup udf failure");
+    return -1;
+  }
 
   SSDataBlock  block = {0};
   SSDataBlock *pBlock = █
@@ -74,11 +67,78 @@ int main(int argc, char *argv[]) {
   input.columnData = taosArrayGet(pBlock->pDataBlock, 0);
   SScalarParam output = {0};
   doCallUdfScalarFunc(handle, &input, 1, &output);
-
+  taosArrayDestroy(pBlock->pDataBlock);
   SColumnInfoData *col = output.columnData;
   for (int32_t i = 0; i < output.numOfRows; ++i) {
     fprintf(stderr, "%d\t%d\n", i, *(int32_t *)(col->pData + i * sizeof(int32_t)));
   }
+
+  colDataDestroy(output.columnData);
+  taosMemoryFree(output.columnData);
+
   doTeardownUdf(handle);
+
+  return 0;
+}
+
+int aggregateFuncTest() {
+  UdfcFuncHandle handle;
+
+  if (doSetupUdf("udf2", &handle) != 0) {
+    fnError("setup udf failure");
+    return -1;
+  }
+
+  SSDataBlock  block = {0};
+  SSDataBlock *pBlock = &block;
+  pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
+  pBlock->info.numOfCols = 1;
+  pBlock->info.rows = 4;
+  char data[16] = {0};
+  char bitmap[4] = {0};
+  for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
+    SColumnInfoData colInfo = {0};
+    colInfo.info.type = TSDB_DATA_TYPE_INT;
+    colInfo.info.bytes = sizeof(int32_t);
+    colInfo.info.colId = 1;
+    colInfo.pData = data;
+    colInfo.nullbitmap = bitmap;
+    for (int32_t j = 0; j < pBlock->info.rows; ++j) {
+      colDataAppendInt32(&colInfo, j, &j);
+    }
+    taosArrayPush(pBlock->pDataBlock, &colInfo);
+  }
+
+  SUdfInterBuf buf = {0};
+  SUdfInterBuf newBuf = {0};
+  SUdfInterBuf resultBuf = {0};
+  doCallUdfAggInit(handle, &buf);
+  doCallUdfAggProcess(handle, pBlock, &buf, &newBuf);
+  taosArrayDestroy(pBlock->pDataBlock);
+
+  doCallUdfAggFinalize(handle, &newBuf, &resultBuf);
+  fprintf(stderr, "agg result: %f\n", *(double*)resultBuf.buf);
+
+  freeUdfInterBuf(&buf);
+  freeUdfInterBuf(&newBuf);
+  freeUdfInterBuf(&resultBuf);
+  doTeardownUdf(handle);
+
+  return 0;
+}
+
+int main(int argc, char *argv[]) {
+  parseArgs(argc, argv);
+  initLog();
+  if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) {
+    fnError("failed to start since read config error");
+    return -1;
+  }
+
+  udfcOpen();
+  uv_sleep(1000);
+
+  scalarFuncTest();
+  aggregateFuncTest();
   udfcClose();
 }

source/libs/index/inc/indexComm.h

@@ -33,11 +33,17 @@ typedef enum { MATCH, CONTINUE, BREAK } TExeCond;
 
 typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type);
 
-TExeCond tDoCommpare(__compar_fn_t func, int8_t comType, void* a, void* b);
+TExeCond tCompare(__compar_fn_t func, int8_t cmpType, void* a, void* b, int8_t dType);
+TExeCond tDoCompare(__compar_fn_t func, int8_t cmpType, void* a, void* b);
 
 _cache_range_compare indexGetCompare(RangeType ty);
 
 int32_t indexConvertData(void* src, int8_t type, void** dst);
+int32_t indexConvertDataToStr(void* src, int8_t type, void** dst);
+
+int32_t indexGetDataByteLen(int8_t type);
+
+char* indexInt2str(int64_t val, char* dst, int radix);
 
 #ifdef __cplusplus
 }

source/libs/index/src/index.c

@@ -109,17 +109,15 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) {
   taosThreadMutexInit(&sIdx->mtx, NULL);
 
   sIdx->refId = indexAddRef(sIdx);
-  taosAcquireRef(indexRefMgt, sIdx->refId);
+  indexAcquireRef(sIdx->refId);
 
   *index = sIdx;
-
   return 0;
 
 END:
   if (sIdx != NULL) {
     indexClose(sIdx);
   }
-
   *index = NULL;
   return -1;
 }
@@ -273,7 +271,7 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy
   tm->nColName = nColName;
 
   char*   buf = NULL;
-  int32_t len = indexConvertData((void*)colVal, INDEX_TYPE_GET_TYPE(colType), (void**)&buf);
+  int32_t len = indexConvertDataToStr((void*)colVal, INDEX_TYPE_GET_TYPE(colType), (void**)&buf);
   assert(len != -1);
 
   tm->colVal = buf;

source/libs/index/src/indexCache.c

@@ -282,8 +282,10 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTe
     if (0 != strncmp(c->colVal, pCt->colVal, skip)) {
       break;
     }
+    char* p = taosMemoryCalloc(1, strlen(c->colVal) + 1);
+    memcpy(p, c->colVal, strlen(c->colVal));
 
-    TExeCond cond = cmpFn(c->colVal + skip, term->colVal, dType);
+    TExeCond cond = cmpFn(p + skip, term->colVal, dType);
     if (cond == MATCH) {
       if (c->operaType == ADD_VALUE) {
         INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid)
@@ -297,6 +299,7 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTe
     } else if (cond == BREAK) {
       break;
     }
+    taosMemoryFree(p);
   }
 
   taosMemoryFree(pCt);
@@ -463,7 +466,6 @@ int indexCacheSchedToMerge(IndexCache* pCache) {
   // schedMsg.thandle = taosMemoryCalloc(1, sizeof(int64_t));
   // memcpy((char*)(schedMsg.thandle), (char*)&(pCache->index->refId), sizeof(int64_t));
   schedMsg.msg = NULL;
-
   indexAcquireRef(pCache->index->refId);
   taosScheduleTask(indexQhandle, &schedMsg);
 

source/libs/index/src/indexComm.c

@@ -19,10 +19,38 @@
 #include "tcoding.h"
 #include "tcompare.h"
 #include "tdataformat.h"
+#include "ttypes.h"
 
 char JSON_COLUMN[] = "JSON";
 char JSON_VALUE_DELIM = '&';
 
+char* indexInt2str(int64_t val, char* dst, int radix) {
+  char     buffer[65];
+  char*    p;
+  int64_t  new_val;
+  uint64_t uval = (uint64_t)val;
+
+  if (radix < 0) {
+    if (val < 0) {
+      *dst++ = '-';
+      uval = (uint64_t)0 - uval; /* Avoid integer overflow in (-val) for LLONG_MIN (BUG#31799). */
+    }
+  }
+  p = &buffer[sizeof(buffer) - 1];
+  *p = '\0';
+  new_val = (int64_t)(uval / 10);
+  *--p = '0' + (char)(uval - (uint64_t)new_val * 10);
+  val = new_val;
+
+  while (val != 0) {
+    new_val = val / 10;
+    *--p = '0' + (char)(val - new_val * 10);
+    val = new_val;
+  }
+  while ((*dst++ = *p++) != 0)
+    ;
+  return dst - 1;
+}
 static __compar_fn_t indexGetCompar(int8_t type) {
   if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
     return (__compar_fn_t)strcmp;
@@ -31,25 +59,49 @@ static __compar_fn_t indexGetCompar(int8_t type) {
 }
 static TExeCond tCompareLessThan(void* a, void* b, int8_t type) {
   __compar_fn_t func = indexGetCompar(type);
-  return tDoCommpare(func, QUERY_LESS_THAN, a, b);
+  return tCompare(func, QUERY_LESS_THAN, a, b, type);
 }
 static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) {
   __compar_fn_t func = indexGetCompar(type);
-  return tDoCommpare(func, QUERY_LESS_EQUAL, a, b);
+  return tCompare(func, QUERY_LESS_EQUAL, a, b, type);
 }
 static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) {
   __compar_fn_t func = indexGetCompar(type);
-  return tDoCommpare(func, QUERY_GREATER_THAN, a, b);
+  return tCompare(func, QUERY_GREATER_THAN, a, b, type);
 }
 static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) {
   __compar_fn_t func = indexGetCompar(type);
-  return tDoCommpare(func, QUERY_GREATER_EQUAL, a, b);
+  return tCompare(func, QUERY_GREATER_EQUAL, a, b, type);
 }
-
+TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t dtype) {
-TExeCond tDoCommpare(__compar_fn_t func, int8_t comType, void* a, void* b) {
+  if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR) {
+    return tDoCompare(func, cmptype, a, b);
+  }
+#if 1
+  int8_t bytes = tDataTypes[dtype].bytes;
+  if (bytes == 1) {
+    int8_t va = taosStr2int64(a);
+    int8_t vb = taosStr2int64(b);
+    return tDoCompare(func, cmptype, &va, &vb);
+  } else if (bytes == 2) {
+    int16_t va = taosStr2int64(a);
+    int16_t vb = taosStr2int64(b);
+    return tDoCompare(func, cmptype, &va, &vb);
+  } else if (bytes == 4) {
+    int32_t va = taosStr2int64(a);
+    int32_t vb = taosStr2int64(b);
+    return tDoCompare(func, cmptype, &va, &vb);
+  } else {
+    int64_t va = taosStr2int64(a);
+    int64_t vb = taosStr2int64(b);
+    return tDoCompare(func, cmptype, &va, &vb);
+  }
+#endif
+}
+TExeCond tDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) {
   // optime later
   int32_t ret = func(a, b);
-  switch (comType) {
+  switch (comparType) {
     case QUERY_LESS_THAN: {
       if (ret < 0) return MATCH;
     } break;
@@ -174,9 +226,9 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) {
       tlen = taosEncodeFixedU32(dst, *(uint32_t*)src);
       break;
     case TSDB_DATA_TYPE_BIGINT:
-      tlen = taosEncodeFixedI64(NULL, *(uint32_t*)src);
+      tlen = taosEncodeFixedI64(NULL, *(int64_t*)src);
       *dst = taosMemoryCalloc(1, tlen + 1);
-      tlen = taosEncodeFixedI64(dst, *(uint32_t*)src);
+      tlen = taosEncodeFixedI64(dst, *(int64_t*)src);
       break;
     case TSDB_DATA_TYPE_DOUBLE:
       tlen = taosEncodeBinary(NULL, src, sizeof(double));
@@ -184,9 +236,9 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) {
       tlen = taosEncodeBinary(dst, src, sizeof(double));
       break;
     case TSDB_DATA_TYPE_UBIGINT:
-      tlen = taosEncodeFixedU64(NULL, *(uint32_t*)src);
+      tlen = taosEncodeFixedU64(NULL, *(uint64_t*)src);
       *dst = taosMemoryCalloc(1, tlen + 1);
-      tlen = taosEncodeFixedU64(dst, *(uint32_t*)src);
+      tlen = taosEncodeFixedU64(dst, *(uint64_t*)src);
       break;
     case TSDB_DATA_TYPE_NCHAR: {
       tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src));
@@ -214,15 +266,95 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) {
       TASSERT(0);
       break;
   }
-  *dst = *dst - tlen;
+  *dst = (char*)*dst - tlen;
-  if (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_VARBINARY &&
+  // indexMayFillNumbericData(*dst, tlen);
-      type == TSDB_DATA_TYPE_VARCHAR) {
+  return tlen;
-    uint8_t* p = *dst;
+}
-    for (int i = 0; i < tlen; i++) {
+int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) {
-      if (p[i] == 0) {
+  int tlen = tDataTypes[type].bytes;
-        p[i] = (uint8_t)'0';
+
-      }
+  switch (type) {
+    case TSDB_DATA_TYPE_TIMESTAMP:
+      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      indexInt2str(*(int64_t*)src, *dst, -1);
+      break;
+    case TSDB_DATA_TYPE_BOOL:
+    case TSDB_DATA_TYPE_UTINYINT:
+      // tlen = taosEncodeFixedU8(NULL, *(uint8_t*)src);
+      //*dst = taosMemoryCalloc(1, tlen + 1);
+      // tlen = taosEncodeFixedU8(dst, *(uint8_t*)src);
+      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      indexInt2str(*(uint8_t*)src, *dst, 1);
+      break;
+    case TSDB_DATA_TYPE_TINYINT:
+      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      indexInt2str(*(int8_t*)src, *dst, 1);
+      break;
+    case TSDB_DATA_TYPE_SMALLINT:
+      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      indexInt2str(*(int16_t*)src, *dst, -1);
+      break;
+    case TSDB_DATA_TYPE_USMALLINT:
+      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      indexInt2str(*(uint16_t*)src, *dst, -1);
+      break;
+    case TSDB_DATA_TYPE_INT:
+      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      indexInt2str(*(int32_t*)src, *dst, -1);
+      break;
+    case TSDB_DATA_TYPE_FLOAT:
+      tlen = taosEncodeBinary(NULL, src, sizeof(float));
+      *dst = taosMemoryCalloc(1, tlen + 1);
+      tlen = taosEncodeBinary(dst, src, sizeof(float));
+      *dst = (char*) * dst - tlen;
+      break;
+    case TSDB_DATA_TYPE_UINT:
+      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      indexInt2str(*(uint32_t*)src, *dst, 1);
+      break;
+    case TSDB_DATA_TYPE_BIGINT:
+      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      indexInt2str(*(int64_t*)src, *dst, 1);
+      break;
+    case TSDB_DATA_TYPE_DOUBLE:
+      tlen = taosEncodeBinary(NULL, src, sizeof(double));
+      *dst = taosMemoryCalloc(1, tlen + 1);
+      tlen = taosEncodeBinary(dst, src, sizeof(double));
+      *dst = (char*) * dst - tlen;
+      break;
+    case TSDB_DATA_TYPE_UBIGINT:
+      assert(0);
+      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      indexInt2str(*(uint64_t*)src, *dst, 1);
+      break;
+    case TSDB_DATA_TYPE_NCHAR: {
+      tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src));
+      *dst = taosMemoryCalloc(1, tlen + 1);
+      tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src));
+      *dst = (char*) * dst - tlen;
+
+      break;
     }
+    case TSDB_DATA_TYPE_VARCHAR: {  // TSDB_DATA_TYPE_BINARY
+#if 1
+      tlen = taosEncodeBinary(NULL, src, strlen(src));
+      *dst = taosMemoryCalloc(1, tlen + 1);
+      tlen = taosEncodeBinary(dst, src, strlen(src));
+      *dst = (char*) * dst - tlen;
+      break;
+#endif
+    }
+    case TSDB_DATA_TYPE_VARBINARY:
+#if 1
+      tlen = taosEncodeBinary(NULL, src, strlen(src));
+      *dst = taosMemoryCalloc(1, tlen + 1);
+      tlen = taosEncodeBinary(dst, src, strlen(src));
+      *dst = (char*) * dst - tlen;
+      break;
+#endif
+    default:
+      TASSERT(0);
+      break;
   }
   return tlen;
 }

source/libs/index/src/indexFst.c

@@ -1324,7 +1324,7 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
       if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) {
         taosArrayPop(sws->inp);
       }
-      streamStateDestroy(p);
+      // streamStateDestroy(p);
       continue;
     }
     FstTransition trn;

source/libs/index/src/indexTfile.c

@@ -410,8 +410,9 @@ static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTempResult*
 
     ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total);
     cost = taosGetTimestampUs() - et;
-    indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, time cost: %" PRIu64 "us", tem->suid,
+    indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, offset: %" PRIu64
-              tem->colName, tem->colVal, cost);
+              ", size: %d, time cost: %" PRIu64 "us",
+              tem->suid, tem->colName, tem->colVal, offset, (int)taosArrayGetSize(tr->total), cost);
   }
   fstSliceDestroy(&key);
   return 0;
@@ -941,7 +942,7 @@ static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray*
   // TODO(yihao): opt later
   WriterCtx* ctx = reader->ctx;
   // add block cache
-  char    block[1024] = {0};
+  char    block[4096] = {0};
   int32_t nread = ctx->readFrom(ctx, block, sizeof(block), offset);
   assert(nread >= sizeof(uint32_t));
 

source/libs/index/test/jsonUT.cc

@@ -56,6 +56,29 @@ class JsonEnv : public ::testing::Test {
   SIndexJson*     index;
 };
 
+static void WriteData(SIndexJson* index, const std::string& colName, int8_t dtype, void* data, int dlen, int tableId,
+                      int8_t operType = ADD_VALUE) {
+  SIndexTerm* term =
+      indexTermCreate(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(), (const char*)data, dlen);
+  SIndexMultiTerm* terms = indexMultiTermCreate();
+  indexMultiTermAdd(terms, term);
+  tIndexJsonPut(index, terms, (int64_t)tableId);
+
+  indexMultiTermDestroy(terms);
+}
+static void Search(SIndexJson* index, const std::string& colNam, int8_t dtype, void* data, int dlen, int8_t filterType,
+                   SArray** result) {
+  std::string colName(colNam);
+
+  SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
+  SIndexTerm* q = indexTermCreate(1, ADD_VALUE, dtype, colName.c_str(), colName.size(), (const char*)data, dlen);
+
+  SArray* res = taosArrayInit(1, sizeof(uint64_t));
+  indexMultiTermQueryAdd(mq, q, (EIndexQueryType)filterType);
+  tIndexJsonSearch(index, mq, res);
+  indexMultiTermQueryDestroy(mq);
+  *result = res;
+}
 TEST_F(JsonEnv, testWrite) {
   {
     std::string colName("test");
@@ -204,9 +227,10 @@ TEST_F(JsonEnv, testWriteMillonData) {
 TEST_F(JsonEnv, testWriteJsonNumberData) {
   {
     std::string colName("test");
-    std::string colVal("10");
+    // std::string colVal("10");
+    int         val = 10;
     SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                       (const char*)&val, sizeof(val));
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -217,35 +241,9 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
   }
   {
     std::string colName("test2");
-    std::string colVal("20");
+    int         val = 20;
     SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                       (const char*)&val, sizeof(val));
-
-    SIndexMultiTerm* terms = indexMultiTermCreate();
-    indexMultiTermAdd(terms, term);
-    for (size_t i = 0; i < 1000; i++) {
-      tIndexJsonPut(index, terms, i);
-    }
-    indexMultiTermDestroy(terms);
-  }
-  {
-    std::string colName("test2");
-    std::string colVal("15");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
-
-    SIndexMultiTerm* terms = indexMultiTermCreate();
-    indexMultiTermAdd(terms, term);
-    for (size_t i = 0; i < 1000; i++) {
-      tIndexJsonPut(index, terms, i);
-    }
-    indexMultiTermDestroy(terms);
-  }
-  {
-    std::string colName("test2");
-    std::string colVal("15");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -256,11 +254,36 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
   }
   {
     std::string colName("test");
-    std::string colVal("10");
+    int         val = 15;
+    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+                                       (const char*)&val, sizeof(val));
 
+    SIndexMultiTerm* terms = indexMultiTermCreate();
+    indexMultiTermAdd(terms, term);
+    for (size_t i = 0; i < 1000; i++) {
+      tIndexJsonPut(index, terms, i);
+    }
+    indexMultiTermDestroy(terms);
+  }
+  {
+    std::string colName("test2");
+    const char* val = "test";
+    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+                                       (const char*)val, strlen(val));
+
+    SIndexMultiTerm* terms = indexMultiTermCreate();
+    indexMultiTermAdd(terms, term);
+    for (size_t i = 0; i < 1000; i++) {
+      tIndexJsonPut(index, terms, i);
+    }
+    indexMultiTermDestroy(terms);
+  }
+  {
+    std::string           colName("test");
+    int                   val = 15;
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    colVal.size());
+                                    (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_TERM);
@@ -270,11 +293,11 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
   }
   {
     std::string colName("test");
-    std::string colVal("10");
+    int         val = 15;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    colVal.size());
+                                    (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN);
@@ -284,11 +307,12 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
   }
   {
     std::string colName("test");
-    std::string colVal("10");
+    int         val = 10;
+    ;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    colVal.size());
+                                    (const char*)&val, sizeof(int));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL);
@@ -298,11 +322,12 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
   }
   {
     std::string colName("test");
-    std::string colVal("10");
+    int         val = 10;
+    // std::string colVal("10");
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    colVal.size());
+                                    (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_LESS_THAN);
@@ -312,11 +337,12 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
   }
   {
     std::string colName("test");
-    std::string colVal("10");
+    int         val = 10;
+    // std::string colVal("10");
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    colVal.size());
+                                    (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_LESS_EQUAL);
@@ -326,12 +352,12 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
   }
 }
 
-TEST_F(JsonEnv, testWriteJsonTfileAndCache) {
+TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
   {
     std::string colName("test1");
-    std::string colVal("10");
+    int         val = 10;
     SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                       (const char*)&val, sizeof(val));
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -355,11 +381,11 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) {
   }
   {
     std::string colName("test1");
-    std::string colVal("10");
+    int         val = 10;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    colVal.size());
+                                    (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_TERM);
@@ -369,11 +395,11 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) {
   }
   {
     std::string colName("test1");
-    std::string colVal("10");
+    int         val = 10;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    colVal.size());
+                                    (const char*)&val, sizeof(int));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN);
@@ -383,11 +409,12 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) {
   }
   {
     std::string colName("test1");
-    std::string colVal("10");
+    // std::string colVal("10");
+    int val = 10;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    colVal.size());
+                                    (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL);
@@ -397,11 +424,11 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) {
   }
   {
     std::string colName("test1");
-    std::string colVal("10");
+    int         val = 10;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    colVal.size());
+                                    (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN);
@@ -411,11 +438,11 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) {
   }
   {
     std::string colName("test1");
-    std::string colVal("10");
+    int         val = 10;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    colVal.size());
+                                    (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_LESS_EQUAL);
@@ -425,9 +452,10 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) {
   }
   {
     std::string colName("other_column");
-    std::string colVal("100");
+    int         val = 100;
+
     SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                       (const char*)&val, sizeof(val));
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -438,11 +466,12 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) {
   }
   {
     std::string colName("test1");
-    std::string colVal("10");
+    int         val = 10;
+    // std::string colVal("10");
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    colVal.size());
+                                    (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_LESS_THAN);
@@ -450,4 +479,102 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) {
     EXPECT_EQ(0, taosArrayGetSize(result));
     indexMultiTermQueryDestroy(mq);
   }
+  {
+    std::string colName("test1");
+    int         val = 15;
+    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+                                       (const char*)&val, sizeof(val));
+
+    SIndexMultiTerm* terms = indexMultiTermCreate();
+    indexMultiTermAdd(terms, term);
+    for (size_t i = 0; i < 1000; i++) {
+      tIndexJsonPut(index, terms, i + 1000);
+    }
+    indexMultiTermDestroy(terms);
+  }
+  {
+    std::string colName("test1");
+    int         val = 8;
+    // std::string colVal("10");
+
+    SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+                                    (const char*)&val, sizeof(val));
+
+    SArray* result = taosArrayInit(1, sizeof(uint64_t));
+    indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL);
+    tIndexJsonSearch(index, mq, result);
+    EXPECT_EQ(2000, taosArrayGetSize(result));
+    indexMultiTermQueryDestroy(mq);
+  }
+}
+TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT2) {
+  {
+    int         val = 10;
+    std::string colName("test1");
+    for (int i = 0; i < 10000; i++) {
+      val += 1;
+      WriteData(index, colName, TSDB_DATA_TYPE_INT, &val, sizeof(val), i);
+    }
+  }
+  {
+    int         val = 10;
+    std::string colName("test2xxx");
+    std::string colVal("xxxxxxxxxxxxxxx");
+    for (int i = 0; i < 100000; i++) {
+      val += 1;
+      WriteData(index, colName, TSDB_DATA_TYPE_BINARY, (void*)(colVal.c_str()), colVal.size(), i);
+    }
+  }
+  {
+    SArray*     res = NULL;
+    std::string colName("test1");
+    int         val = 9;
+    Search(index, colName, TSDB_DATA_TYPE_INT, &val, sizeof(val), QUERY_GREATER_EQUAL, &res);
+    EXPECT_EQ(10000, taosArrayGetSize(res));
+  }
+  {
+    SArray*     res = NULL;
+    std::string colName("test2xxx");
+    std::string colVal("xxxxxxxxxxxxxxx");
+    Search(index, colName, TSDB_DATA_TYPE_BINARY, (void*)(colVal.c_str()), colVal.size(), QUERY_TERM, &res);
+    EXPECT_EQ(100000, taosArrayGetSize(res));
+  }
+}
+TEST_F(JsonEnv, testWriteJsonTfileAndCache_FLOAT) {
+  {
+    float       val = 10.0;
+    std::string colName("test1");
+    for (int i = 0; i < 1000; i++) {
+      WriteData(index, colName, TSDB_DATA_TYPE_FLOAT, &val, sizeof(val), i);
+    }
+  }
+  {
+    float       val = 2.0;
+    std::string colName("test1");
+    for (int i = 0; i < 1000; i++) {
+      WriteData(index, colName, TSDB_DATA_TYPE_FLOAT, &val, sizeof(val), i);
+    }
+  }
+  {
+    SArray*     res = NULL;
+    std::string colName("test1");
+    float       val = 1.9;
+    Search(index, colName, TSDB_DATA_TYPE_FLOAT, &val, sizeof(val), QUERY_GREATER_EQUAL, &res);
+    EXPECT_EQ(2000, taosArrayGetSize(res));
+  }
+  {
+    SArray*     res = NULL;
+    std::string colName("test1");
+    float       val = 2.1;
+    Search(index, colName, TSDB_DATA_TYPE_FLOAT, &val, sizeof(val), QUERY_GREATER_EQUAL, &res);
+    EXPECT_EQ(1000, taosArrayGetSize(res));
+  }
+  {
+    std::string colName("test1");
+    SArray*     res = NULL;
+    float       val = 2.1;
+    Search(index, colName, TSDB_DATA_TYPE_FLOAT, &val, sizeof(val), QUERY_GREATER_EQUAL, &res);
+    EXPECT_EQ(1000, taosArrayGetSize(res));
+  }
 }

source/libs/index/test/utilUT.cc

@@ -6,12 +6,14 @@
 #include <vector>
 #include "index.h"
 #include "indexCache.h"
+#include "indexComm.h"
 #include "indexFst.h"
 #include "indexFstCountingWriter.h"
 #include "indexFstUtil.h"
 #include "indexInt.h"
 #include "indexTfile.h"
 #include "indexUtil.h"
+#include "tcoding.h"
 #include "tglobal.h"
 #include "tskiplist.h"
 #include "tutil.h"
@@ -305,3 +307,17 @@ TEST_F(UtilEnv, 01Except) {
   ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 0), 1);
   ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 1), 100);
 }
+TEST_F(UtilEnv, testFill) {
+  for (int i = 0; i < 10000000; i++) {
+    int64_t val = i;
+    char    buf[65] = {0};
+    indexInt2str(val, buf, 1);
+    EXPECT_EQ(val, taosStr2int64(buf));
+  }
+  for (int i = 0; i < 10000000; i++) {
+    int64_t val = 0 - i;
+    char    buf[65] = {0};
+    indexInt2str(val, buf, -1);
+    EXPECT_EQ(val, taosStr2int64(buf));
+  }
+}

source/libs/nodes/src/nodesUtilFuncs.c

@@ -529,6 +529,18 @@ void nodesDestroyNode(SNodeptr pNode) {
       nodesDestroyNode(pStmt->pTbNamePattern);
       break;
     }
+    case QUERY_NODE_QUERY: {
+      SQuery* pQuery = (SQuery*)pNode;
+      nodesDestroyNode(pQuery->pRoot);
+      taosMemoryFreeClear(pQuery->pResSchema);
+      if (NULL != pQuery->pCmdMsg) {
+        taosMemoryFreeClear(pQuery->pCmdMsg->pMsg);
+        taosMemoryFreeClear(pQuery->pCmdMsg);
+      }
+      taosArrayDestroy(pQuery->pDbList);
+      taosArrayDestroy(pQuery->pTableList);
+      break;
+    }
     case QUERY_NODE_LOGIC_PLAN_SCAN: {
       SScanLogicNode* pLogicNode = (SScanLogicNode*)pNode;
       destroyLogicNode((SLogicNode*)pLogicNode);

source/libs/parser/src/parTranslater.c

@@ -4254,7 +4254,135 @@ static int32_t rewriteDropTable(STranslateContext* pCxt, SQuery* pQuery) {
   return rewriteToVnodeModifyOpStmt(pQuery, pBufArray);
 }
 
-static int32_t buildAlterTbReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, SVAlterTbReq* pReq) {
+static SSchema* getColSchema(STableMeta* pTableMeta, const char* pTagName) {
+  int32_t numOfFields = getNumOfTags(pTableMeta) + getNumOfColumns(pTableMeta);
+  for (int32_t i = 0; i < numOfFields; ++i) {
+    SSchema* pTagSchema = pTableMeta->schema + i;
+    if (0 == strcmp(pTagName, pTagSchema->name)) {
+      return pTagSchema;
+    }
+  }
+  return NULL;
+}
+
+static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta,
+                                    SVAlterTbReq* pReq) {
+  SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName);
+  if (NULL == pSchema) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE);
+  }
+
+  pReq->tagName = strdup(pStmt->colName);
+  if (NULL == pReq->tagName) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  if (DEAL_RES_ERROR == translateValueImpl(pCxt, pStmt->pVal, schemaToDataType(pSchema))) {
+    return pCxt->errCode;
+  }
+
+  pReq->isNull = (TSDB_DATA_TYPE_NULL == pStmt->pVal->node.resType.type);
+  pReq->nTagVal = pStmt->pVal->node.resType.bytes;
+  char* pVal = nodesGetValueFromNode(pStmt->pVal);
+  pReq->pTagVal = IS_VAR_DATA_TYPE(pStmt->pVal->node.resType.type) ? pVal + VARSTR_HEADER_SIZE : pVal;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t buildAddColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta,
+                              SVAlterTbReq* pReq) {
+  if (NULL != getColSchema(pTableMeta, pStmt->colName)) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DUPLICATED_COLUMN);
+  }
+
+  pReq->colName = strdup(pStmt->colName);
+  if (NULL == pReq->colName) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  pReq->type = pStmt->dataType.type;
+  pReq->flags = COL_SMA_ON;
+  pReq->bytes = pStmt->dataType.bytes;
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t buildDropColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta,
+                               SVAlterTbReq* pReq) {
+  SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName);
+  if (NULL == pSchema) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName);
+  } else if (PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_CANNOT_DROP_PRIMARY_KEY);
+  }
+
+  pReq->colName = strdup(pStmt->colName);
+  if (NULL == pReq->colName) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t buildUpdateColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta,
+                                 SVAlterTbReq* pReq) {
+  pReq->colModBytes = calcTypeBytes(pStmt->dataType);
+
+  SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName);
+  if (NULL == pSchema) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName);
+  } else if (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->bytes >= pReq->colModBytes) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_MODIFY_COL);
+  }
+
+  pReq->colName = strdup(pStmt->colName);
+  if (NULL == pReq->colName) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t buildRenameColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta,
+                                 SVAlterTbReq* pReq) {
+  if (NULL == getColSchema(pTableMeta, pStmt->colName)) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName);
+  }
+  if (NULL != getColSchema(pTableMeta, pStmt->newColName)) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DUPLICATED_COLUMN);
+  }
+
+  pReq->colName = strdup(pStmt->colName);
+  pReq->colNewName = strdup(pStmt->newColName);
+  if (NULL == pReq->colName || NULL == pReq->colNewName) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t buildUpdateOptionsReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, SVAlterTbReq* pReq) {
+  int32_t code = TSDB_CODE_SUCCESS;
+
+  if (-1 != pStmt->pOptions->ttl) {
+    code = checkRangeOption(pCxt, "ttl", pStmt->pOptions->ttl, TSDB_MIN_TABLE_TTL, INT32_MAX);
+    if (TSDB_CODE_SUCCESS == code) {
+      pReq->updateTTL = true;
+      pReq->newTTL = pStmt->pOptions->ttl;
+    }
+  }
+
+  if (TSDB_CODE_SUCCESS == code && '\0' != pStmt->pOptions->comment[0]) {
+    pReq->updateComment = true;
+    pReq->newComment = strdup(pStmt->pOptions->comment);
+    if (NULL == pReq->newComment) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
+
+  return code;
+}
+
+static int32_t buildAlterTbReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta,
+                               SVAlterTbReq* pReq) {
   pReq->tbName = strdup(pStmt->tableName);
   if (NULL == pReq->tbName) {
     return TSDB_CODE_OUT_OF_MEMORY;
@@ -4268,60 +4396,22 @@ static int32_t buildAlterTbReq(STranslateContext* pCxt, SAlterTableStmt* pStmt,
     case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES:
       return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE);
     case TSDB_ALTER_TABLE_UPDATE_TAG_VAL:
-      pReq->tagName = strdup(pStmt->colName);
+      return buildUpdateTagValReq(pCxt, pStmt, pTableMeta, pReq);
-      if (NULL == pReq->tagName) {
-        return TSDB_CODE_OUT_OF_MEMORY;
-      }
-      if (DEAL_RES_ERROR == translateValue(pCxt, pStmt->pVal)) {
-        return pCxt->errCode;
-      }
-      pReq->isNull = (TSDB_DATA_TYPE_NULL == pStmt->pVal->node.resType.type);
-      pReq->nTagVal = pStmt->pVal->node.resType.bytes;
-      char* pVal = nodesGetValueFromNode(pStmt->pVal);
-      pReq->pTagVal = IS_VAR_DATA_TYPE(pStmt->pVal->node.resType.type) ? pVal + VARSTR_HEADER_SIZE : pVal;
-      break;
     case TSDB_ALTER_TABLE_ADD_COLUMN:
+      return buildAddColReq(pCxt, pStmt, pTableMeta, pReq);
     case TSDB_ALTER_TABLE_DROP_COLUMN:
-      pReq->colName = strdup(pStmt->colName);
+      return buildDropColReq(pCxt, pStmt, pTableMeta, pReq);
-      if (NULL == pReq->colName) {
-        return TSDB_CODE_OUT_OF_MEMORY;
-      }
-      pReq->type = pStmt->dataType.type;
-      pReq->flags = COL_SMA_ON;
-      pReq->bytes = pStmt->dataType.bytes;
-      break;
     case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:
-      pReq->colName = strdup(pStmt->colName);
+      return buildUpdateColReq(pCxt, pStmt, pTableMeta, pReq);
-      if (NULL == pReq->colName) {
-        return TSDB_CODE_OUT_OF_MEMORY;
-      }
-      pReq->colModBytes = calcTypeBytes(pStmt->dataType);
-      break;
     case TSDB_ALTER_TABLE_UPDATE_OPTIONS:
-      if (-1 != pStmt->pOptions->ttl) {
+      return buildUpdateOptionsReq(pCxt, pStmt, pReq);
-        pReq->updateTTL = true;
-        pReq->newTTL = pStmt->pOptions->ttl;
-      }
-      if ('\0' != pStmt->pOptions->comment[0]) {
-        pReq->updateComment = true;
-        pReq->newComment = strdup(pStmt->pOptions->comment);
-        if (NULL == pReq->newComment) {
-          return TSDB_CODE_OUT_OF_MEMORY;
-        }
-      }
-      break;
     case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME:
-      pReq->colName = strdup(pStmt->colName);
+      return buildRenameColReq(pCxt, pStmt, pTableMeta, pReq);
-      pReq->colNewName = strdup(pStmt->newColName);
-      if (NULL == pReq->colName || NULL == pReq->colNewName) {
-        return TSDB_CODE_OUT_OF_MEMORY;
-      }
-      break;
     default:
       break;
   }
 
-  return TSDB_CODE_SUCCESS;
+  return TSDB_CODE_FAILED;
 }
 
 static int32_t serializeAlterTbReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, SVAlterTbReq* pReq,
@@ -4394,7 +4484,7 @@ static int32_t rewriteAlterTable(STranslateContext* pCxt, SQuery* pQuery) {
   }
 
   SVAlterTbReq req = {0};
-  code = buildAlterTbReq(pCxt, pStmt, &req);
+  code = buildAlterTbReq(pCxt, pStmt, pTableMeta, &req);
 
   SArray* pArray = NULL;
   if (TSDB_CODE_SUCCESS == code) {

source/libs/parser/src/parUtil.c

@@ -154,6 +154,10 @@ static char* getSyntaxErrFormat(int32_t errCode) {
       return "Invalid password";
     case TSDB_CODE_PAR_INVALID_ALTER_TABLE:
       return "Invalid alter table statement";
+    case TSDB_CODE_PAR_CANNOT_DROP_PRIMARY_KEY:
+      return "Primary timestamp column cannot be dropped";
+    case TSDB_CODE_PAR_INVALID_MODIFY_COL:
+      return "Only binary/nchar column length could be modified";
     case TSDB_CODE_OUT_OF_MEMORY:
       return "Out of memory";
     default:

source/libs/parser/src/parser.c

@@ -39,10 +39,16 @@ static int32_t parseSqlIntoAst(SParseContext* pCxt, SQuery** pQuery) {
   if (TSDB_CODE_SUCCESS == code) {
     code = authenticate(pCxt, *pQuery);
   }
-  if (TSDB_CODE_SUCCESS == code && 0 == (*pQuery)->placeholderNum) {
+
+  if (TSDB_CODE_SUCCESS == code && (*pQuery)->placeholderNum > 0) {
+    // TSWAP((*pQuery)->pContainPlaceholderRoot, (*pQuery)->pRoot);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (TSDB_CODE_SUCCESS == code) {
     code = translate(pCxt, *pQuery);
   }
-  if (TSDB_CODE_SUCCESS == code && 0 == (*pQuery)->placeholderNum) {
+  if (TSDB_CODE_SUCCESS == code) {
     code = calculateConstant(pCxt, *pQuery);
   }
   return code;
@@ -142,26 +148,13 @@ int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery) {
   return code;
 }
 
-void qDestroyQuery(SQuery* pQueryNode) {
+void qDestroyQuery(SQuery* pQueryNode) { nodesDestroyNode(pQueryNode); }
-  if (NULL == pQueryNode) {
-    return;
-  }
-  nodesDestroyNode(pQueryNode->pRoot);
-  taosMemoryFreeClear(pQueryNode->pResSchema);
-  if (NULL != pQueryNode->pCmdMsg) {
-    taosMemoryFreeClear(pQueryNode->pCmdMsg->pMsg);
-    taosMemoryFreeClear(pQueryNode->pCmdMsg);
-  }
-  taosArrayDestroy(pQueryNode->pDbList);
-  taosArrayDestroy(pQueryNode->pTableList);
-  taosMemoryFreeClear(pQueryNode);
-}
 
 int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema) {
   return extractResultSchema(pRoot, numOfCols, pSchema);
 }
 
-int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId) {
+int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx) {
   int32_t code = TSDB_CODE_SUCCESS;
 
   if (colIdx < 0) {
@@ -184,6 +177,6 @@ int32_t qStmtParseQuerySql(SParseContext* pCxt, SQuery* pQuery) {
   if (TSDB_CODE_SUCCESS == code) {
     code = calculateConstant(pCxt, pQuery);
   }
-  
+
   return code;
 }

source/libs/parser/test/parInitialATest.cpp

@@ -283,13 +283,13 @@ TEST_F(ParserInitialATest, alterTable) {
   setAlterColFunc("t1", TSDB_ALTER_TABLE_DROP_COLUMN, "c1");
   run("ALTER TABLE t1 DROP COLUMN c1");
 
-  setAlterColFunc("t1", TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, "c1", TSDB_DATA_TYPE_VARCHAR, 20 + VARSTR_HEADER_SIZE);
+  setAlterColFunc("t1", TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, "c2", TSDB_DATA_TYPE_VARCHAR, 30 + VARSTR_HEADER_SIZE);
-  run("ALTER TABLE t1 MODIFY COLUMN c1 VARCHAR(20)");
+  run("ALTER TABLE t1 MODIFY COLUMN c2 VARCHAR(30)");
 
   setAlterColFunc("t1", TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, "c1", 0, 0, "cc1");
   run("ALTER TABLE t1 RENAME COLUMN c1 cc1");
 
-  int64_t val = 10;
+  int32_t val = 10;
   setAlterTagFunc("st1s1", "tag1", (const uint8_t*)&val, sizeof(val));
   run("ALTER TABLE st1s1 SET TAG tag1=10");
 

source/libs/planner/src/planner.c

@@ -18,28 +18,6 @@
 #include "planInt.h"
 #include "scalar.h"
 
-typedef struct SCollectPlaceholderValuesCxt {
-  int32_t errCode;
-  SArray* pValues;
-} SCollectPlaceholderValuesCxt;
-
-static EDealRes collectPlaceholderValuesImpl(SNode* pNode, void* pContext) {
-  if (QUERY_NODE_VALUE == nodeType(pNode) && ((SValueNode*)pNode)->placeholderNo > 0) {
-    SCollectPlaceholderValuesCxt* pCxt = pContext;
-    taosArrayInsert(pCxt->pValues, ((SValueNode*)pNode)->placeholderNo - 1, &pNode);
-    return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR;
-  }
-  return DEAL_RES_CONTINUE;
-}
-
-static int32_t collectPlaceholderValues(SPlanContext* pCxt, SQueryPlan* pPlan) {
-  pPlan->pPlaceholderValues = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
-
-  SCollectPlaceholderValuesCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pValues = pPlan->pPlaceholderValues};
-  nodesWalkPhysiPlan((SNode*)pPlan, collectPlaceholderValuesImpl, &cxt);
-  return cxt.errCode;
-}
-
 int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNodeList) {
   SLogicNode*      pLogicNode = NULL;
   SLogicSubplan*   pLogicSubplan = NULL;
@@ -58,9 +36,6 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo
   if (TSDB_CODE_SUCCESS == code) {
     code = createPhysiPlan(pCxt, pLogicPlan, pPlan, pExecNodeList);
   }
-  if (TSDB_CODE_SUCCESS == code && pCxt->placeholderNum > 0) {
-    code = collectPlaceholderValues(pCxt, *pPlan);
-  }
 
   nodesDestroyNode(pLogicNode);
   nodesDestroyNode(pLogicSubplan);
@@ -99,249 +74,6 @@ int32_t qSetSubplanExecutionNode(SSubplan* subplan, int32_t groupId, SDownstream
   return setSubplanExecutionNode(subplan->pNode, groupId, pSource);
 }
 
-static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) {
-  if (pParam->is_null && 1 == *(pParam->is_null)) {
-    pVal->node.resType.type = TSDB_DATA_TYPE_NULL;
-    pVal->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_NULL].bytes;
-    return TSDB_CODE_SUCCESS;
-  }
-  int32_t inputSize = (NULL != pParam->length ? *(pParam->length) : tDataTypes[pParam->buffer_type].bytes);
-  pVal->node.resType.type = pParam->buffer_type;
-  pVal->node.resType.bytes = inputSize;
-  switch (pParam->buffer_type) {
-    case TSDB_DATA_TYPE_BOOL:
-      pVal->datum.b = *((bool*)pParam->buffer);
-      break;
-    case TSDB_DATA_TYPE_TINYINT:
-      pVal->datum.i = *((int8_t*)pParam->buffer);
-      break;
-    case TSDB_DATA_TYPE_SMALLINT:
-      pVal->datum.i = *((int16_t*)pParam->buffer);
-      break;
-    case TSDB_DATA_TYPE_INT:
-      pVal->datum.i = *((int32_t*)pParam->buffer);
-      break;
-    case TSDB_DATA_TYPE_BIGINT:
-      pVal->datum.i = *((int64_t*)pParam->buffer);
-      break;
-    case TSDB_DATA_TYPE_FLOAT:
-      pVal->datum.d = *((float*)pParam->buffer);
-      break;
-    case TSDB_DATA_TYPE_DOUBLE:
-      pVal->datum.d = *((double*)pParam->buffer);
-      break;
-    case TSDB_DATA_TYPE_VARCHAR:
-    case TSDB_DATA_TYPE_VARBINARY:
-      pVal->datum.p = taosMemoryCalloc(1, pVal->node.resType.bytes + VARSTR_HEADER_SIZE + 1);
-      if (NULL == pVal->datum.p) {
-        return TSDB_CODE_OUT_OF_MEMORY;
-      }
-      varDataSetLen(pVal->datum.p, pVal->node.resType.bytes);
-      strncpy(varDataVal(pVal->datum.p), (const char*)pParam->buffer, pVal->node.resType.bytes);
-      break;
-    case TSDB_DATA_TYPE_NCHAR: {
-      pVal->node.resType.bytes *= TSDB_NCHAR_SIZE;
-      pVal->datum.p = taosMemoryCalloc(1, pVal->node.resType.bytes + VARSTR_HEADER_SIZE + 1);
-      if (NULL == pVal->datum.p) {
-        return TSDB_CODE_OUT_OF_MEMORY;
-      }
-
-      int32_t output = 0;
-      if (!taosMbsToUcs4(pParam->buffer, inputSize, (TdUcs4*)varDataVal(pVal->datum.p), pVal->node.resType.bytes,
-                         &output)) {
-        return errno;
-      }
-      varDataSetLen(pVal->datum.p, output);
-      pVal->node.resType.bytes = output;
-      break;
-    }
-    case TSDB_DATA_TYPE_TIMESTAMP:
-      pVal->datum.i = *((int64_t*)pParam->buffer);
-      break;
-    case TSDB_DATA_TYPE_UTINYINT:
-      pVal->datum.u = *((uint8_t*)pParam->buffer);
-      break;
-    case TSDB_DATA_TYPE_USMALLINT:
-      pVal->datum.u = *((uint16_t*)pParam->buffer);
-      break;
-    case TSDB_DATA_TYPE_UINT:
-      pVal->datum.u = *((uint32_t*)pParam->buffer);
-      break;
-    case TSDB_DATA_TYPE_UBIGINT:
-      pVal->datum.u = *((uint64_t*)pParam->buffer);
-      break;
-    case TSDB_DATA_TYPE_JSON:
-    case TSDB_DATA_TYPE_DECIMAL:
-    case TSDB_DATA_TYPE_BLOB:
-    case TSDB_DATA_TYPE_MEDIUMBLOB:
-      // todo
-    default:
-      break;
-  }
-  pVal->translate = true;
-  return TSDB_CODE_SUCCESS;
-}
-
-static EDealRes updatePlanQueryId(SNode* pNode, void* pContext) {
-  int64_t queryId = *(uint64_t*)pContext;
-
-  if (QUERY_NODE_PHYSICAL_PLAN == nodeType(pNode)) {
-    SQueryPlan* planNode = (SQueryPlan*)pNode;
-    planNode->queryId = queryId;
-  } else if (QUERY_NODE_PHYSICAL_SUBPLAN == nodeType(pNode)) {
-    SSubplan* subplanNode = (SSubplan*)pNode;
-    subplanNode->id.queryId = queryId;
-  }
-
-  return DEAL_RES_CONTINUE;
-}
-
-static int32_t calcConstNode(SNode** pNode) {
-  if (NULL == *pNode) {
-    return TSDB_CODE_SUCCESS;
-  }
-
-  SNode*  pNew = NULL;
-  int32_t code = scalarCalculateConstants(*pNode, &pNew);
-  if (TSDB_CODE_SUCCESS == code) {
-    *pNode = pNew;
-  }
-  return code;
-}
-
-static int32_t calcConstList(SNodeList* pList) {
-  SNode* pNode = NULL;
-  FOREACH(pNode, pList) {
-    SNode*  pNew = NULL;
-    int32_t code = scalarCalculateConstants(pNode, &pNew);
-    if (TSDB_CODE_SUCCESS == code) {
-      REPLACE_NODE(pNew);
-    } else {
-      return code;
-    }
-  }
-  return TSDB_CODE_SUCCESS;
-}
-
-static bool isEmptyResultCond(SNode** pCond) {
-  if (NULL == *pCond || QUERY_NODE_VALUE != nodeType(*pCond)) {
-    return false;
-  }
-  if (((SValueNode*)*pCond)->datum.b) {
-    nodesDestroyNode(*pCond);
-    *pCond = NULL;
-    return false;
-  }
-  return true;
-}
-
-static int32_t calcConstSpecificPhysiNode(SPhysiNode* pPhyNode) {
-  switch (nodeType(pPhyNode)) {
-    case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
-    case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:
-    case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN:
-    case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN:
-    case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:
-    case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:
-    case QUERY_NODE_PHYSICAL_PLAN_FILL:
-      return TSDB_CODE_SUCCESS;
-    case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
-      return calcConstList(((SProjectPhysiNode*)pPhyNode)->pProjections);
-    case QUERY_NODE_PHYSICAL_PLAN_JOIN:
-      return calcConstNode(&(((SJoinPhysiNode*)pPhyNode)->pOnConditions));
-    case QUERY_NODE_PHYSICAL_PLAN_AGG:
-      return calcConstList(((SAggPhysiNode*)pPhyNode)->pExprs);
-    case QUERY_NODE_PHYSICAL_PLAN_SORT:
-      return calcConstList(((SSortPhysiNode*)pPhyNode)->pExprs);
-    case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:
-    case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL:
-    case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:
-    case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW:
-      return calcConstList(((SWinodwPhysiNode*)pPhyNode)->pExprs);
-    case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
-      return calcConstList(((SPartitionPhysiNode*)pPhyNode)->pExprs);
-    default:
-      break;
-  }
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t calcConstSubplan(SPhysiNode* pPhyNode, bool* pEmptyResult) {
-  int32_t code = calcConstNode(&pPhyNode->pConditions);
-  if (TSDB_CODE_SUCCESS == code) {
-    code = calcConstSpecificPhysiNode(pPhyNode);
-  }
-  if (TSDB_CODE_SUCCESS != code) {
-    return code;
-  }
-
-  *pEmptyResult = isEmptyResultCond(&pPhyNode->pConditions);
-  if (*pEmptyResult) {
-    return TSDB_CODE_SUCCESS;
-  }
-
-  *pEmptyResult = true;
-
-  bool   subEmptyResult = false;
-  SNode* pChild = NULL;
-  FOREACH(pChild, pPhyNode->pChildren) {
-    code = calcConstSubplan((SPhysiNode*)pChild, &subEmptyResult);
-    if (TSDB_CODE_SUCCESS != code) {
-      return code;
-    }
-    if (!subEmptyResult) {
-      *pEmptyResult = false;
-    }
-  }
-
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t calcConstPhysiPlan(SQueryPlan* pPlan, bool* pEmptyResult) {
-  *pEmptyResult = true;
-
-  bool           subEmptyResult = false;
-  SNodeListNode* pNode = nodesListGetNode(pPlan->pSubplans, 0);
-  SNode*         pSubplan = NULL;
-  FOREACH(pSubplan, pNode->pNodeList) {
-    int32_t code = calcConstSubplan(((SSubplan*)pSubplan)->pNode, pEmptyResult);
-    if (TSDB_CODE_SUCCESS != code) {
-      return code;
-    }
-    if (!subEmptyResult) {
-      *pEmptyResult = false;
-    }
-  }
-  return TSDB_CODE_SUCCESS;
-}
-
-int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId,
-                       bool* pEmptyResult) {
-  int32_t size = taosArrayGetSize(pPlan->pPlaceholderValues);
-  int32_t code = 0;
-
-  if (colIdx < 0) {
-    for (int32_t i = 0; i < size; ++i) {
-      code = setValueByBindParam((SValueNode*)taosArrayGetP(pPlan->pPlaceholderValues, i), pParams + i);
-      if (code) {
-        return code;
-      }
-    }
-  } else {
-    code = setValueByBindParam((SValueNode*)taosArrayGetP(pPlan->pPlaceholderValues, colIdx), pParams);
-    if (code) {
-      return code;
-    }
-  }
-
-  if (colIdx < 0 || ((colIdx + 1) == size)) {
-    nodesWalkPhysiPlan((SNode*)pPlan, updatePlanQueryId, &queryId);
-    code = calcConstPhysiPlan(pPlan, pEmptyResult);
-  }
-
-  return code;
-}
-
 int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen) {
   if (SUBPLAN_TYPE_MODIFY == pSubplan->subplanType) {
     SDataInserterNode* insert = (SDataInserterNode*)pSubplan->pDataSink;

source/libs/planner/test/planStmtTest.cpp

@@ -20,35 +20,49 @@ using namespace std;
 
 class PlanStmtTest : public PlannerTestBase {
  public:
-  void prepare(const string& sql) {
+  void buildParam(TAOS_MULTI_BIND* pBindParams, int32_t index, void* pVal, int32_t type, int32_t bytes = 0) {
-    run(sql);
+    TAOS_MULTI_BIND* pBindParam = pBindParams + index;
-    // todo calloc pBindParams_
+    pBindParam->buffer_type = type;
-  }
+    pBindParam->num = 1;
+    pBindParam->buffer_length = bytes > 0 ? bytes : tDataTypes[type].bytes;
+    pBindParam->buffer = taosMemoryCalloc(1, pBindParam->buffer_length);
+    pBindParam->length = (int32_t*)taosMemoryCalloc(1, sizeof(int32_t));
+    pBindParam->is_null = (char*)taosMemoryCalloc(1, sizeof(char));
+    *(pBindParam->length) = bytes > 0 ? bytes : tDataTypes[type].bytes;
+    *(pBindParam->is_null) = 0;
 
-  void bindParam(int32_t val) {
+    switch (type) {
-    TAOS_MULTI_BIND* pBind = pBindParams_ + paramNo_++;
+      case TSDB_DATA_TYPE_BOOL:
-    pBind->buffer_type = TSDB_DATA_TYPE_INT;
+        *((bool*)pBindParam->buffer) = *(bool*)pVal;
-    pBind->num = 1;
+        break;
-    pBind->buffer_length = sizeof(int32_t);
+      case TSDB_DATA_TYPE_TINYINT:
-    pBind->buffer = taosMemoryCalloc(1, pBind->buffer_length);
+        *((int8_t*)pBindParam->buffer) = *(int64_t*)pVal;
-    pBind->length = (int32_t*)taosMemoryCalloc(1, sizeof(int32_t));
+        break;
-    pBind->is_null = (char*)taosMemoryCalloc(1, sizeof(char));
+      case TSDB_DATA_TYPE_SMALLINT:
-    *((int32_t*)pBind->buffer) = val;
+      case TSDB_DATA_TYPE_INT:
-    *(pBind->length) = sizeof(int32_t);
+      case TSDB_DATA_TYPE_BIGINT:
-    *(pBind->is_null) = 0;
+      case TSDB_DATA_TYPE_FLOAT:
+      case TSDB_DATA_TYPE_DOUBLE:
+      case TSDB_DATA_TYPE_VARCHAR:
+      case TSDB_DATA_TYPE_TIMESTAMP:
+      case TSDB_DATA_TYPE_NCHAR:
+      case TSDB_DATA_TYPE_UTINYINT:
+      case TSDB_DATA_TYPE_USMALLINT:
+      case TSDB_DATA_TYPE_UINT:
+      case TSDB_DATA_TYPE_UBIGINT:
+      case TSDB_DATA_TYPE_JSON:
+      case TSDB_DATA_TYPE_VARBINARY:
+      case TSDB_DATA_TYPE_DECIMAL:
+      case TSDB_DATA_TYPE_BLOB:
+      case TSDB_DATA_TYPE_MEDIUMBLOB:
+      default:
+        break;
+    }
   }
-
-  void exec() {
-    // todo
-  }
-
- private:
-  TAOS_MULTI_BIND* pBindParams_;
-  int32_t          paramNo_;
 };
 
 TEST_F(PlanStmtTest, stmt) {
   useDb("root", "test");
 
-  // run("select * from t1 where c1 = ?");
+  prepare("SELECT * FROM t1 WHERE c1 = ?");
 }

source/libs/planner/test/planTestUtil.cpp

@@ -108,6 +108,57 @@ class PlannerTestBaseImpl {
     }
   }
 
+  void prepare(const string& sql) {
+    reset();
+    try {
+      doParseSql(sql, &stmtEnv_.pQuery_, true);
+
+      dump(g_dumpModule);
+    } catch (...) {
+      dump(DUMP_MODULE_ALL);
+      throw;
+    }
+  }
+
+  void bindParams(TAOS_MULTI_BIND* pParams, int32_t colIdx) {
+    try {
+      doBindParams(stmtEnv_.pQuery_, pParams, colIdx);
+
+      SPlanContext cxt = {0};
+      setPlanContext(stmtEnv_.pQuery_, &cxt);
+
+      SLogicNode* pLogicNode = nullptr;
+      doCreateLogicPlan(&cxt, &pLogicNode);
+
+      doOptimizeLogicPlan(&cxt, pLogicNode);
+
+      SLogicSubplan* pLogicSubplan = nullptr;
+      doSplitLogicPlan(&cxt, pLogicNode, &pLogicSubplan);
+
+      SQueryLogicPlan* pLogicPlan = nullptr;
+      doScaleOutLogicPlan(&cxt, pLogicSubplan, &pLogicPlan);
+
+      SQueryPlan* pPlan = nullptr;
+      doCreatePhysiPlan(&cxt, pLogicPlan, &pPlan);
+
+      dump(g_dumpModule);
+    } catch (...) {
+      dump(DUMP_MODULE_ALL);
+      throw;
+    }
+  }
+
+  void exec() {
+    try {
+      doParseBoundSql(stmtEnv_.pQuery_);
+
+      dump(g_dumpModule);
+    } catch (...) {
+      dump(DUMP_MODULE_ALL);
+      throw;
+    }
+  }
+
  private:
   struct caseEnv {
     string acctId_;
@@ -117,10 +168,15 @@ class PlannerTestBaseImpl {
   struct stmtEnv {
     string            sql_;
     array<char, 1024> msgBuf_;
+    SQuery*           pQuery_;
+
+    ~stmtEnv() { qDestroyQuery(pQuery_); }
   };
 
   struct stmtRes {
     string         ast_;
+    string         prepareAst_;
+    string         boundAst_;
     string         rawLogicPlan_;
     string         optimizedLogicPlan_;
     string         splitLogicPlan_;
@@ -132,8 +188,10 @@ class PlannerTestBaseImpl {
   void reset() {
     stmtEnv_.sql_.clear();
     stmtEnv_.msgBuf_.fill(0);
+    qDestroyQuery(stmtEnv_.pQuery_);
 
     res_.ast_.clear();
+    res_.boundAst_.clear();
     res_.rawLogicPlan_.clear();
     res_.optimizedLogicPlan_.clear();
     res_.splitLogicPlan_.clear();
@@ -152,6 +210,9 @@ class PlannerTestBaseImpl {
     if (DUMP_MODULE_ALL == module || DUMP_MODULE_PARSER == module) {
       cout << "syntax tree : " << endl;
       cout << res_.ast_ << endl;
+
+      cout << "bound syntax tree : " << endl;
+      cout << res_.boundAst_ << endl;
     }
 
     if (DUMP_MODULE_ALL == module || DUMP_MODULE_LOGIC == module) {
@@ -187,7 +248,7 @@ class PlannerTestBaseImpl {
     }
   }
 
-  void doParseSql(const string& sql, SQuery** pQuery) {
+  void doParseSql(const string& sql, SQuery** pQuery, bool prepare = false) {
     stmtEnv_.sql_ = sql;
     transform(stmtEnv_.sql_.begin(), stmtEnv_.sql_.end(), stmtEnv_.sql_.begin(), ::tolower);
 
@@ -200,7 +261,31 @@ class PlannerTestBaseImpl {
     cxt.msgLen = stmtEnv_.msgBuf_.max_size();
 
     DO_WITH_THROW(qParseSql, &cxt, pQuery);
-    res_.ast_ = toString((*pQuery)->pRoot);
+    if (prepare) {
+      res_.prepareAst_ = toString((*pQuery)->pRoot);
+    } else {
+      res_.ast_ = toString((*pQuery)->pRoot);
+    }
+  }
+
+  void doBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx) {
+    DO_WITH_THROW(qStmtBindParams, pQuery, pParams, colIdx);
+    if (colIdx < 0 || pQuery->placeholderNum == colIdx + 1) {
+      res_.boundAst_ = toString(pQuery->pRoot);
+    }
+  }
+
+  void doParseBoundSql(SQuery* pQuery) {
+    SParseContext cxt = {0};
+    cxt.acctId = atoi(caseEnv_.acctId_.c_str());
+    cxt.db = caseEnv_.db_.c_str();
+    cxt.pSql = stmtEnv_.sql_.c_str();
+    cxt.sqlLen = stmtEnv_.sql_.length();
+    cxt.pMsg = stmtEnv_.msgBuf_.data();
+    cxt.msgLen = stmtEnv_.msgBuf_.max_size();
+
+    DO_WITH_THROW(qStmtParseQuerySql, &cxt, pQuery);
+    res_.ast_ = toString(pQuery->pRoot);
   }
 
   void doCreateLogicPlan(SPlanContext* pCxt, SLogicNode** pLogicNode) {
@@ -275,3 +360,11 @@ PlannerTestBase::~PlannerTestBase() {}
 void PlannerTestBase::useDb(const std::string& acctId, const std::string& db) { impl_->useDb(acctId, db); }
 
 void PlannerTestBase::run(const std::string& sql) { return impl_->run(sql); }
+
+void PlannerTestBase::prepare(const std::string& sql) { return impl_->prepare(sql); }
+
+void PlannerTestBase::bindParams(TAOS_MULTI_BIND* pParams, int32_t colIdx) {
+  return impl_->bindParams(pParams, colIdx);
+}
+
+void PlannerTestBase::exec() { return impl_->exec(); }

source/libs/planner/test/planTestUtil.h

@@ -19,6 +19,7 @@
 #include <gtest/gtest.h>
 
 class PlannerTestBaseImpl;
+struct TAOS_MULTI_BIND;
 
 class PlannerTestBase : public testing::Test {
  public:
@@ -27,6 +28,10 @@ class PlannerTestBase : public testing::Test {
 
   void useDb(const std::string& acctId, const std::string& db);
   void run(const std::string& sql);
+  // stmt mode APIs
+  void prepare(const std::string& sql);
+  void bindParams(TAOS_MULTI_BIND* pParams, int32_t colIdx);
+  void exec();
 
  private:
   std::unique_ptr<PlannerTestBaseImpl> impl_;

source/libs/scheduler/src/scheduler.c

@@ -1148,6 +1148,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch
       if (NULL == msg) {
         SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
       }
+      SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask));
       break;
     }
     case TDMT_VND_SUBMIT_RSP: {

source/libs/stream/src/tstream.c

@@ -35,7 +35,7 @@ void* streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput) {
   return (void*)buf;
 }
 
-static int32_t streamBuildDispatchMsg(SStreamTask* pTask, SArray* data, SRpcMsg* pMsg, SEpSet** ppEpSet) {
+static int32_t streamBuildExecMsg(SStreamTask* pTask, SArray* data, SRpcMsg* pMsg, SEpSet** ppEpSet) {
   SStreamTaskExecReq req = {
       .streamId = pTask->streamId,
       .data = data,
@@ -107,7 +107,7 @@ static int32_t streamShuffleDispatch(SStreamTask* pTask, SMsgCb* pMsgCb, SHashOb
     SArray* pData = *(SArray**)pIter;
     SRpcMsg dispatchMsg = {0};
     SEpSet* pEpSet;
-    if (streamBuildDispatchMsg(pTask, pData, &dispatchMsg, &pEpSet) < 0) {
+    if (streamBuildExecMsg(pTask, pData, &dispatchMsg, &pEpSet) < 0) {
       ASSERT(0);
       return -1;
     }
@@ -133,7 +133,7 @@ int32_t streamEnqueueDataBlk(SStreamTask* pTask, SStreamDataBlock* input) {
   return inputStatus;
 }
 
-int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) {
+static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) {
   void* exec = pTask->exec.runners[0].executor;
 
   // set input
@@ -265,87 +265,42 @@ FAIL:
   return -1;
 }
 
-int32_t streamTaskDispatchDown(SStreamTask* pTask, SMsgCb* pMsgCb) {
+int32_t streamTaskSink(SStreamTask* pTask, SMsgCb* pMsgCb) {
-  //
+  bool firstRun = 1;
-  return 0;
-}
-
-int32_t streamTaskSink(SStreamTask* pTask) {
-  //
-  return 0;
-}
-
-int32_t streamTaskProcessInputReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDataBlock* pBlock, SRpcMsg* pRsp) {
-  // 1. handle input
-  // 1.1 enqueue
-  taosWriteQitem(pTask->inputQ, pBlock);
-  // 1.2 calc back pressure
-  // 1.3 rsp by input status
-  int8_t              inputStatus = atomic_load_8(&pTask->inputStatus);
-  SStreamDispatchRsp* pCont = rpcMallocCont(sizeof(SStreamDispatchRsp));
-  pCont->status = inputStatus;
-  pRsp->pCont = pCont;
-  pRsp->contLen = sizeof(SStreamDispatchRsp);
-  tmsgSendRsp(pRsp);
-  // 2. try exec
-  // 2.1. idle: exec
-  // 2.2. executing: return
-  // 2.3. closing: keep trying
   while (1) {
-    int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING);
+    SStreamDataBlock* pBlock = NULL;
-    if (execStatus == TASK_STATUS__IDLE) {
+    if (!firstRun) {
-      void*         exec = pTask->exec.runners[0].executor;
+      taosReadAllQitems(pTask->outputQ, pTask->outputQAll);
-      SArray*       pRes = taosArrayInit(0, sizeof(void*));
+    }
-      const SArray* blocks = pBlock->blocks;
+    taosGetQitem(pTask->outputQAll, (void**)&pBlock);
-      qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK);
+    if (pBlock == NULL) {
-      while (1) {
+      if (firstRun) {
-        SSDataBlock* output;
+        firstRun = 0;
-        uint64_t     ts = 0;
+        continue;
-        if (qExecTask(exec, &output, &ts) < 0) {
+      } else {
-          ASSERT(false);
+        break;
-        }
+      }
-        if (output == NULL) break;
-        taosArrayPush(pRes, &output);
-      }
-      // TODO: wrap destroy block
-      taosArrayDestroyP(pBlock->blocks, (FDelete)blockDataDestroy);
-
-      if (taosArrayGetSize(pRes) != 0) {
-        SArray** resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM);
-        *resQ = pRes;
-        taosWriteQitem(pTask->outputQ, resQ);
-      }
-
-    } else if (execStatus == TASK_STATUS__CLOSING) {
-      continue;
-    } else if (execStatus == TASK_STATUS__EXECUTING)
-      break;
-    else {
-      ASSERT(0);
     }
-  }
-  // 3. handle output
-  // 3.1 check and set status
-  // 3.2 dispatch / sink
-  STaosQall* qall = taosAllocateQall();
-  taosReadAllQitems(pTask->outputQ, qall);
-  SArray** ppRes = NULL;
-  while (1) {
-    taosGetQitem(qall, (void**)&ppRes);
-    if (ppRes == NULL) break;
 
-    SArray* pRes = *ppRes;
+    SArray* pRes = pBlock->blocks;
+
+    // sink
     if (pTask->sinkType == TASK_SINK__TABLE) {
-      pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, pBlock->sourceVer, pRes);
+      // blockDebugShowData(pRes);
+      pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pRes);
     } else if (pTask->sinkType == TASK_SINK__SMA) {
       pTask->smaSink.smaSink(pTask->ahandle, pTask->smaSink.smaId, pRes);
+      //
+    } else if (pTask->sinkType == TASK_SINK__FETCH) {
+      //
     } else {
+      ASSERT(pTask->sinkType == TASK_SINK__NONE);
     }
 
     // dispatch
     if (pTask->dispatchType == TASK_DISPATCH__INPLACE) {
       SRpcMsg dispatchMsg = {0};
-      if (streamBuildDispatchMsg(pTask, pRes, &dispatchMsg, NULL) < 0) {
+      if (streamBuildExecMsg(pTask, pRes, &dispatchMsg, NULL) < 0) {
         ASSERT(0);
         return -1;
       }
@@ -366,7 +321,7 @@ int32_t streamTaskProcessInputReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDat
     } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) {
       SRpcMsg dispatchMsg = {0};
       SEpSet* pEpSet = NULL;
-      if (streamBuildDispatchMsg(pTask, pRes, &dispatchMsg, &pEpSet) < 0) {
+      if (streamBuildExecMsg(pTask, pRes, &dispatchMsg, &pEpSet) < 0) {
         ASSERT(0);
         return -1;
       }
@@ -401,12 +356,53 @@ int32_t streamTaskProcessInputReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDat
       ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE);
     }
   }
-  //
   return 0;
 }
 
-int32_t streamTaskProcessDispatchRsp(SStreamTask* pTask, char* msg, int32_t msgLen) {
+int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp) {
-  //
+  SStreamDataBlock* pBlock = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM);
+  int8_t            status;
+
+  // 1.1 update status
+  // TODO cal backpressure
+  if (pBlock == NULL) {
+    atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED);
+    status = TASK_INPUT_STATUS__FAILED;
+  } else {
+    status = atomic_load_8(&pTask->inputStatus);
+  }
+
+  // 1.2 enqueue
+  pBlock->type = STREAM_DATA_TYPE_SSDATA_BLOCK;
+  pBlock->sourceVg = pReq->sourceVg;
+  pBlock->sourceVer = pReq->sourceVer;
+  taosWriteQitem(pTask->inputQ, pBlock);
+
+  // 1.3 rsp by input status
+  SStreamDispatchRsp* pCont = rpcMallocCont(sizeof(SStreamDispatchRsp));
+  pCont->inputStatus = status;
+  pRsp->pCont = pCont;
+  pRsp->contLen = sizeof(SStreamDispatchRsp);
+  tmsgSendRsp(pRsp);
+
+  return 0;
+}
+
+int32_t streamTaskProcessDispatchReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchReq* pReq, SRpcMsg* pRsp) {
+  // 1. handle input
+  streamTaskEnqueue(pTask, pReq, pRsp);
+
+  // 2. try exec
+  // 2.1. idle: exec
+  // 2.2. executing: return
+  // 2.3. closing: keep trying
+  streamTaskExec2(pTask, pMsgCb);
+
+  // 3. handle output
+  // 3.1 check and set status
+  // 3.2 dispatch / sink
+  streamTaskSink(pTask, pMsgCb);
+
   return 0;
 }
 
@@ -415,64 +411,6 @@ int32_t streamTaskProcessRecoverReq(SStreamTask* pTask, char* msg) {
   return 0;
 }
 
-int32_t streamTaskRun(SStreamTask* pTask) {
-  SArray* pRes = NULL;
-  if (pTask->execType == TASK_EXEC__PIPE || pTask->execType == TASK_EXEC__MERGE) {
-    // TODO remove multi runner
-    void* exec = pTask->exec.runners[0].executor;
-
-    int8_t status = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING);
-    if (status == TASK_STATUS__IDLE) {
-      pRes = taosArrayInit(0, sizeof(void*));
-      if (pRes == NULL) {
-        return -1;
-      }
-
-      void* input = NULL;
-      taosWriteQitem(pTask->inputQ, &input);
-      if (input == NULL) return 0;
-
-      // TODO: fix type
-      if (pTask->sourceType == TASK_SOURCE__SCAN) {
-        SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)input;
-        qSetStreamInput(exec, pSubmit->data, STREAM_DATA_TYPE_SUBMIT_BLOCK);
-        while (1) {
-          SSDataBlock* output;
-          uint64_t     ts = 0;
-          if (qExecTask(exec, &output, &ts) < 0) {
-            ASSERT(false);
-          }
-          if (output == NULL) break;
-          taosArrayPush(pRes, &output);
-        }
-        streamDataSubmitRefDec(pSubmit);
-      } else {
-        SStreamDataBlock* pStreamBlock = (SStreamDataBlock*)input;
-        const SArray*     blocks = pStreamBlock->blocks;
-        qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK);
-        while (1) {
-          SSDataBlock* output;
-          uint64_t     ts = 0;
-          if (qExecTask(exec, &output, &ts) < 0) {
-            ASSERT(false);
-          }
-          if (output == NULL) break;
-          taosArrayPush(pRes, &output);
-        }
-        // TODO: wrap destroy block
-        taosArrayDestroyP(pStreamBlock->blocks, (FDelete)blockDataDestroy);
-      }
-
-      if (taosArrayGetSize(pRes) != 0) {
-        SArray** resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM);
-        *resQ = pRes;
-        taosWriteQitem(pTask->outputQ, resQ);
-      }
-    }
-  }
-  return 0;
-}
-
 int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId) {
   SArray* pRes = NULL;
   // source
@@ -545,7 +483,7 @@ int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, in
 
   if (pTask->dispatchType == TASK_DISPATCH__INPLACE) {
     SRpcMsg dispatchMsg = {0};
-    if (streamBuildDispatchMsg(pTask, pRes, &dispatchMsg, NULL) < 0) {
+    if (streamBuildExecMsg(pTask, pRes, &dispatchMsg, NULL) < 0) {
       ASSERT(0);
       return -1;
     }
@@ -566,7 +504,7 @@ int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, in
   } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) {
     SRpcMsg dispatchMsg = {0};
     SEpSet* pEpSet = NULL;
-    if (streamBuildDispatchMsg(pTask, pRes, &dispatchMsg, &pEpSet) < 0) {
+    if (streamBuildExecMsg(pTask, pRes, &dispatchMsg, &pEpSet) < 0) {
       ASSERT(0);
       return -1;
     }

source/libs/tdb/src/db/tdbPCache.c

@@ -28,9 +28,9 @@ struct SPCache {
   SPage       lru;
 };
 
-static inline int tdbPCachePageHash(const SPgid *pPgid) {
+static inline uint32_t tdbPCachePageHash(const SPgid *pPgid) {
-  u32 *t = (u32 *)((pPgid)->fileid);
+  uint32_t *t = (uint32_t *)((pPgid)->fileid);
-  return t[0] + t[1] + t[2] + t[3] + t[4] + t[5] + (pPgid)->pgno;
+  return (uint32_t)(t[0] + t[1] + t[2] + t[3] + t[4] + t[5] + (pPgid)->pgno);
 }
 #define PAGE_IS_PINNED(pPage) ((pPage)->pLruNext == NULL)
 

source/libs/tdb/src/db/tdbPager.c

@@ -72,7 +72,7 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
     return -1;
   }
 
-  ret = tdbGnrtFileID(pPager->dbFileName, pPager->fid, false);
+  ret = tdbGnrtFileID(pPager->fd, pPager->fid, false);
   if (ret < 0) {
     return -1;
   }

source/libs/tdb/src/db/tdbUtil.c

@@ -35,10 +35,10 @@ void tdbFree(void *p) {
   }
 }
 
-int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) {
+int tdbGnrtFileID(tdb_fd_t fd, uint8_t *fileid, bool unique) {
   int64_t stDev = 0, stIno = 0;
 
-  if (taosDevInoFile(fname, &stDev, &stIno) < 0) {
+  if (taosDevInoFile(fd, &stDev, &stIno) < 0) {
     return -1;
   }
 

source/libs/tdb/src/inc/tdbUtil.h

@@ -28,7 +28,7 @@ extern "C" {
 
 #define TDB_ROUND8(x) (((x) + 7) & ~7)
 
-int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique);
+int tdbGnrtFileID(tdb_fd_t fd, uint8_t *fileid, bool unique);
 int tdbGetFileSize(tdb_fd_t fd, int szPage, SPgno *size);
 
 void *tdbRealloc(void *ptr, size_t size);

source/os/src/osFile.c

@@ -110,7 +110,7 @@ void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, cha
 int64_t taosCopyFile(const char *from, const char *to) {
 #ifdef WINDOWS
   assert(0);
-  return 0;
+  return -1;
 #else
   char    buffer[4096];
   int64_t size = 0;
@@ -190,15 +190,35 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) {
 
   return 0;
 }
-int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno) {
+int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) {
+  if (pFile == NULL) {
+    return 0;
+  }
+  assert(pFile->fd >= 0);  // Please check if you have closed the file.
 
-  struct stat fileStat;
 #ifdef WINDOWS
-  int32_t     code = _stat(path, &fileStat);
+
+  BY_HANDLE_FILE_INFORMATION bhfi;
+  HANDLE handle = (HANDLE)_get_osfhandle(pFile->fd);
+  if (GetFileInformationByHandle(handle, &bhfi) == FALSE) {
+    printf("taosFStatFile get file info fail.");
+    return -1;
+  }
+
+  if (stDev != NULL) {
+    *stDev = (int64_t)(bhfi.dwVolumeSerialNumber);
+  }
+
+  if (stIno != NULL) {
+    *stIno = (int64_t)((((uint64_t)bhfi.nFileIndexHigh) << 32) + bhfi.nFileIndexLow);
+  }
+
 #else
-  int32_t     code = stat(path, &fileStat);
+  
-#endif
+  struct stat fileStat;
+  int32_t     code = fstat(pFile->fd, &fileStat);
   if (code < 0) {
+    printf("taosFStatFile run fstat fail.");
     return code;
   }
 
@@ -209,6 +229,7 @@ int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno) {
   if (stIno != NULL) {
     *stIno = fileStat.st_ino;
   }
+#endif
 
   return 0;
 }

source/os/src/osSysinfo.c

@@ -744,7 +744,8 @@ int32_t taosGetSystemUUID(char *uid, int32_t uidlen) {
 #ifdef WINDOWS
   GUID guid;
   CoCreateGuid(&guid);
-  memcpy(uid, &guid, uidlen);
+  snprintf(uid, uidlen, "%08X-%04X-%04X-%02X%02X-%02X%02X%02X%02X%02X%02X", guid.Data1, guid.Data2, guid.Data3, guid.Data4[0],
+          guid.Data4[1], guid.Data4[2], guid.Data4[3], guid.Data4[4], guid.Data4[5], guid.Data4[6], guid.Data4[7]);
 
   return 0;
 #elif defined(_TD_DARWIN_64)

source/util/src/terror.c

@@ -285,6 +285,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC_QUERY,      "Topic with invalid qu
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC_OPTION,     "Topic with invalid option")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_CONSUMER_NOT_EXIST,       "Consumer not exist")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_CONSUMER_NOT_READY,       "Consumer waiting for rebalance")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_SUBSCRIBED,         "Topic subscribed cannot be dropped")
 
 // mnode-sma
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_ALREADY_EXIST,        "SMA already exists")
@@ -358,7 +359,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECREATED,          "Table re-created")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR,       "TDB env open error")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_SMA_INDEX_IN_META,     "No sma index in meta")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_SMA_STAT,         "Invalid sma state")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TSMA_ALREADY_EXIST,       "Tsma already exists")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TSMA_ALREADY_EXIST,       "TSMA already exists")
 
 
 // query

source/util/src/thashutil.c

@@ -30,7 +30,7 @@
     (h) ^= (h) >> 13;  \
     (h) *= 0xc2b2ae35; \
     (h) ^= (h) >> 16; } while (0)
-  
+
 uint32_t MurmurHash3_32(const char *key, uint32_t len) {
   const uint8_t *data = (const uint8_t *)key;
   const int32_t nblocks = len >> 2u;
@@ -78,18 +78,54 @@ uint32_t MurmurHash3_32(const char *key, uint32_t len) {
   return h1;
 }
 
+uint64_t MurmurHash3_64(const char *key, uint32_t len) {
+  const uint64_t m = 0x87c37b91114253d5;
+  const int r = 47;
+  uint32_t seed = 0x12345678;
+  uint64_t h = seed ^ (len * m);
+  const uint8_t *data = (const uint8_t *)key;
+  const uint8_t *end = data + (len-(len&7));
+
+  while(data != end) {
+    uint64_t k = *((uint64_t*)data);
+
+    k *= m;
+    k ^= k >> r;
+    k *= m;
+    h ^= k;
+    h *= m;
+    data += 8;
+  }
+
+  switch(len & 7) {
+    case 7: h ^= (uint64_t)data[6] << 48; /* fall-thru */
+    case 6: h ^= (uint64_t)data[5] << 40; /* fall-thru */
+    case 5: h ^= (uint64_t)data[4] << 32; /* fall-thru */
+    case 4: h ^= (uint64_t)data[3] << 24; /* fall-thru */
+    case 3: h ^= (uint64_t)data[2] << 16; /* fall-thru */
+    case 2: h ^= (uint64_t)data[1] << 8; /* fall-thru */
+    case 1: h ^= (uint64_t)data[0];
+      h *= m; /* fall-thru */
+  };
+
+  h ^= h >> r;
+  h *= m;
+  h ^= h >> r;
+  return h;
+}
+
 uint32_t taosIntHash_32(const char *key, uint32_t UNUSED_PARAM(len)) { return *(uint32_t *)key; }
 uint32_t taosIntHash_16(const char *key, uint32_t UNUSED_PARAM(len)) { return *(uint16_t *)key; }
 uint32_t taosIntHash_8(const char *key, uint32_t UNUSED_PARAM(len)) { return *(uint8_t *)key; }
 uint32_t taosFloatHash(const char *key, uint32_t UNUSED_PARAM(len)) {
-  float f = GET_FLOAT_VAL(key); 
+  float f = GET_FLOAT_VAL(key);
   if (isnan(f)) {
     return 0x7fc00000;
   }
-   
+
   if (FLT_EQUAL(f, 0.0)) {
     return 0;
-  } 
+  }
   if (fabs(f) < FLT_MAX/BASE - DLT) {
    int32_t t = (int32_t)(round(BASE * (f + DLT)));
    return (uint32_t)t;
@@ -98,27 +134,27 @@ uint32_t taosFloatHash(const char *key, uint32_t UNUSED_PARAM(len)) {
   }
 }
 uint32_t taosDoubleHash(const char *key, uint32_t UNUSED_PARAM(len)) {
-  double f = GET_DOUBLE_VAL(key);  
+  double f = GET_DOUBLE_VAL(key);
   if (isnan(f)) {
     return 0x7fc00000;
   }
 
   if (FLT_EQUAL(f, 0.0)) {
     return 0;
-  } 
+  }
   if (fabs(f) < DBL_MAX/BASE - DLT) {
    int32_t t = (int32_t)(round(BASE * (f + DLT)));
    return (uint32_t)t;
   } else {
    return 0x7fc00000;
-  } 
+  }
 }
 uint32_t taosIntHash_64(const char *key, uint32_t UNUSED_PARAM(len)) {
   uint64_t val = *(uint64_t *)key;
 
   uint64_t hash = val >> 16U;
   hash += (val & 0xFFFFU);
-  
+
   return (uint32_t)hash;
 }
 
@@ -127,39 +163,39 @@ _hash_fn_t taosGetDefaultHashFunction(int32_t type) {
   switch(type) {
     case TSDB_DATA_TYPE_TIMESTAMP:
     case TSDB_DATA_TYPE_UBIGINT:
-    case TSDB_DATA_TYPE_BIGINT:   
+    case TSDB_DATA_TYPE_BIGINT:
       fn = taosIntHash_64;
       break;
-    case TSDB_DATA_TYPE_BINARY:   
+    case TSDB_DATA_TYPE_BINARY:
       fn = MurmurHash3_32;
       break;
-    case TSDB_DATA_TYPE_NCHAR:    
+    case TSDB_DATA_TYPE_NCHAR:
       fn = MurmurHash3_32;
       break;
     case TSDB_DATA_TYPE_UINT:
-    case TSDB_DATA_TYPE_INT:      
+    case TSDB_DATA_TYPE_INT:
-      fn = taosIntHash_32; 
+      fn = taosIntHash_32;
       break;
-    case TSDB_DATA_TYPE_SMALLINT: 
+    case TSDB_DATA_TYPE_SMALLINT:
-    case TSDB_DATA_TYPE_USMALLINT:  
+    case TSDB_DATA_TYPE_USMALLINT:
-      fn = taosIntHash_16; 
+      fn = taosIntHash_16;
       break;
     case TSDB_DATA_TYPE_BOOL:
     case TSDB_DATA_TYPE_UTINYINT:
-    case TSDB_DATA_TYPE_TINYINT:  
+    case TSDB_DATA_TYPE_TINYINT:
-      fn = taosIntHash_8; 
+      fn = taosIntHash_8;
       break;
-    case TSDB_DATA_TYPE_FLOAT:    
+    case TSDB_DATA_TYPE_FLOAT:
-      fn = taosFloatHash; 
+      fn = taosFloatHash;
-      break;                             
+      break;
-    case TSDB_DATA_TYPE_DOUBLE:   
+    case TSDB_DATA_TYPE_DOUBLE:
-      fn = taosDoubleHash; 
+      fn = taosDoubleHash;
-      break;                             
+      break;
-    default: 
+    default:
       fn = taosIntHash_32;
       break;
   }
-  
+
   return fn;
 }
 

source/util/src/tjson.c

@@ -183,8 +183,12 @@ int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal
   if (NULL == p) {
     return TSDB_CODE_FAILED;
   }
-
+#ifdef WINDOWS
+  sscanf(p,"%lld",pVal);
+#else
+  // sscanf(p,"%ld",pVal);
   *pVal = strtol(p, NULL, 10);
+#endif
   return TSDB_CODE_SUCCESS;
 }
 
@@ -214,8 +218,12 @@ int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pV
   if (NULL == p) {
     return TSDB_CODE_FAILED;
   }
-
+#ifdef WINDOWS
+  sscanf(p,"%llu",pVal);
+#else
+  // sscanf(p,"%ld",pVal);
   *pVal = strtoul(p, NULL, 10);
+#endif
   return TSDB_CODE_SUCCESS;
 }
 

source/util/src/tqueue.c

@@ -33,11 +33,11 @@ typedef struct STaosQnode {
 } STaosQnode;
 
 typedef struct STaosQueue {
-  STaosQnode   *head;
+  STaosQnode *  head;
-  STaosQnode   *tail;
+  STaosQnode *  tail;
-  STaosQueue   *next;     // for queue set
+  STaosQueue *  next;     // for queue set
-  STaosQset    *qset;     // for queue set
+  STaosQset *   qset;     // for queue set
-  void         *ahandle;  // for queue set
+  void *        ahandle;  // for queue set
   FItem         itemFp;
   FItems        itemsFp;
   TdThreadMutex mutex;
@@ -46,8 +46,8 @@ typedef struct STaosQueue {
 } STaosQueue;
 
 typedef struct STaosQset {
-  STaosQueue   *head;
+  STaosQueue *  head;
-  STaosQueue   *current;
+  STaosQueue *  current;
   TdThreadMutex mutex;
   tsem_t        sem;
   int32_t       numOfQueues;
@@ -85,7 +85,7 @@ void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp) {
 void taosCloseQueue(STaosQueue *queue) {
   if (queue == NULL) return;
   STaosQnode *pTemp;
-  STaosQset  *qset;
+  STaosQset * qset;
 
   taosThreadMutexLock(&queue->mutex);
   STaosQnode *pNode = queue->head;
@@ -152,7 +152,7 @@ void *taosAllocateQitem(int32_t size, EQItype itype) {
 
   if (itype == RPC_QITEM) {
     int64_t alloced = atomic_add_fetch_64(&tsRpcQueueMemoryUsed, size);
-    if (alloced > tsRpcQueueMemoryUsed) {
+    if (alloced > tsRpcQueueMemoryAllowed) {
       taosMemoryFree(pNode);
       terrno = TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE;
       return NULL;

tests/script/jenkins/basic.txt

@@ -104,7 +104,8 @@
 ./test.sh -f tsim/mnode/basic1.sim -m
 
 # --- sma
-# ./test.sh -f tsim/sma/tsmaCreateInsertData.sim
+./test.sh -f tsim/sma/tsmaCreateInsertData.sim
+./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim
 
 # --- valgrind
 ./test.sh -f tsim/valgrind/checkError.sim -v

tests/script/tsim/query/udf.sim

@@ -7,7 +7,7 @@ system sh/cfg.sh -n dnode1 -c udf -v 1
 
 print ========= start dnode1 as LEADER
 system sh/exec.sh -n dnode1 -s start
-sleep 2000
+sleep 1000
 sql connect
 
 print ======== step1 udf 

tests/script/tsim/sma/rsmaCreateInsertQuery.sim

@@ -0,0 +1,89 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sleep 50
+sql connect
+
+print =============== create database with retentions
+sql create database d0 retentions 15s:7d,1m:21d,15m:365d;
+sql use d0
+
+print =============== create super table and register rsma
+sql create table if not exists stb (ts timestamp, c1 int) tags (city binary(20),district binary(20)) rollup(min) file_factor 0.1 delay 2;
+
+sql show stables
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== create child table
+sql create table ct1 using stb tags("BeiJing", "ChaoYang");
+
+sql show tables
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== insert data and trigger rollup
+sql insert into ct1 values(now, 10);
+sql insert into ct1 values(now+1s, 1);
+sql insert into ct1 values(now+2s, 100);
+
+
+print =============== select * from retention level 2 from memory
+sql select * from ct1;
+print $data00 $data01
+if $rows > 2 then
+  print retention level 2 file rows $rows > 2
+  return -1
+endi
+print =============== select * from retention level 1 from memory
+sql select * from ct1 where ts > now-8d;
+print $data00 $data01
+if $rows > 2 then
+  print retention level 1 file rows $rows > 2
+  return -1
+endi
+print =============== select * from retention level 0 from memory
+sql select * from ct1 where ts > now-3d;
+print $data00 $data01
+print $data10 $data11
+print $data20 $data21
+if $rows < 1 then
+  print retention level 0 file rows $rows < 1
+  return -1
+endi
+#===================================================================
+
+
+#==================== reboot to trigger commit data to file
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s start
+
+print =============== select * from retention level 2 from file
+sql select * from ct1;
+print $data00 $data01
+if $rows > 2 then
+  print retention level 2 file rows $rows > 2
+  return -1
+endi
+
+print =============== select * from retention level 1 from file
+sql select * from ct1 where ts > now-8d;
+print $data00 $data01
+if $rows > 2 then
+  print retention level 1 file rows $rows > 2
+  return -1
+endi
+
+print =============== select * from retention level 0 from file
+sql select * from ct1 where ts > now-3d;
+print $data00 $data01 
+print $data10 $data11
+print $data20 $data21
+if $rows < 1 then
+  print retention level 0 file rows $rows < 1
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/sync/threeReplica1VgElect.sim

@@ -220,7 +220,6 @@ if $data[0][4] == LEADER then
   print ---- vgroup $data[0][0] leader switch to dnode $data[0][3]
 elif $data[0][6] == LEADER then
   print ---- vgroup $data[0][0] leader switch to dnode $data[0][5]
-endi
 elif $data[0][8] == LEADER then
   print ---- vgroup $data[0][0] leader switch to dnode $data[0][7]
 else 
@@ -342,7 +341,6 @@ elif $data[0][6] == LEADER then
     goto check_vg_ready_3
   endi
   print ---- vgroup $data[0][0] leader locating dnode $data[0][7]
-endi
 elif $data[0][8] == LEADER then
   if $data[0][4] == LEADER then
     goto check_vg_ready_3

tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim

@@ -420,7 +420,6 @@ elif $data[0][6] == LEADER then
     goto check_vg_ready_3
   endi
   print ---- vgroup $data[0][0] leader locating dnode $data[0][7]
-endi
 elif $data[0][8] == LEADER then
   if $data[0][4] == LEADER then
     goto check_vg_ready_3

tests/system-test/2-query/union.py

@@ -0,0 +1,412 @@
+import datetime
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+
+PRIMARY_COL = "ts"
+
+INT_COL     = "c1"
+BINT_COL    = "c2"
+SINT_COL    = "c3"
+TINT_COL    = "c4"
+FLOAT_COL   = "c5"
+DOUBLE_COL  = "c6"
+BOOL_COL    = "c7"
+
+BINARY_COL  = "c8"
+NCHAR_COL   = "c9"
+TS_COL      = "c10"
+
+NUM_COL     = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ]
+CHAR_COL    = [ BINARY_COL, NCHAR_COL, ]
+BOOLEAN_COL = [ BOOL_COL, ]
+TS_TYPE_COL = [ TS_COL, ]
+
+class TDTestCase:
+
+    def init(self, conn, logSql):
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor())
+
+    def __query_condition(self,tbname):
+        query_condition = []
+        for char_col in CHAR_COL:
+            query_condition.extend(
+                (
+                    f"{tbname}.{char_col}",
+                    f"upper( {tbname}.{char_col} )",
+                    f"char_length( {tbname}.{char_col} )",
+                    f"concat( {tbname}.{char_col}, {tbname}.{char_col} )",
+                    f"concat_ws( '_', {tbname}.{char_col}, {tbname}.{char_col} )",
+                    f"length( {tbname}.{char_col} )",
+                    f"lower( {tbname}.{char_col} )",
+                    f"ltrim( {tbname}.{char_col} )",
+                    f"rtrim( {tbname}.{char_col} )",
+                    f"substr( {tbname}.{char_col}, 1 )",
+                    f"count( {tbname}.{char_col} )",
+                    f"cast( {tbname}.{char_col} as nchar(3) )",
+                    f"cast( {tbname}.{char_col} as nchar(8) )",
+                )
+            )
+            query_condition.extend( f"cast( {tbname}.{un_char_col} as binary(16) ) " for un_char_col in NUM_COL)
+            query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{char_col_2} as binary(32) ) " for char_col_2 in CHAR_COL )
+            query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{un_char_col} as binary(32) ) " for un_char_col in NUM_COL )
+
+        for num_col in NUM_COL:
+            query_condition.extend(
+                (
+                    f"{tbname}.{num_col}",
+                    f"ceil( {tbname}.{num_col} )",
+                    f"abs( {tbname}.{num_col} )",
+                    f"acos( {tbname}.{num_col} )",
+                    f"asin( {tbname}.{num_col} )",
+                    f"atan( {tbname}.{num_col} )",
+                    f"cos( {tbname}.{num_col} )",
+                    f"floor( {tbname}.{num_col} )",
+                    f"log( {tbname}.{num_col},  {tbname}.{num_col})",
+                    f"sin( {tbname}.{num_col} )",
+                    f"sqrt( {tbname}.{num_col} )",
+                    f"tan( {tbname}.{num_col} )",
+                    f"round( {tbname}.{num_col} )",
+                    f"max( {tbname}.{num_col} )",
+                    f"sum( {tbname}.{num_col} )",
+                    f"count( {tbname}.{num_col} )",
+                    f"min( {tbname}.{num_col} )",
+                )
+            )
+            query_condition.extend( f"{tbname}.{num_col} + {tbname}.{num_col_2}" for num_col_2 in NUM_COL )
+            query_condition.extend( f"{tbname}.{num_col} + {tbname}.{char_col} " for char_col in CHAR_COL )
+
+        query_condition.extend(
+            (
+                ''' "test1234!@#$%^&*():'><?/.,][}{" ''',
+                ''' "test12" ''',
+                # 1010,
+            )
+        )
+
+        return query_condition
+
+    def __join_condition(self, tb_list, filter=PRIMARY_COL, INNER=False):
+        table_reference = tb_list[0]
+        join_condition = table_reference
+        join = "inner join" if INNER else "join"
+        for i in range(len(tb_list[1:])):
+            join_condition += f" {join} {tb_list[i+1]} on {table_reference}.{filter}={tb_list[i+1]}.{filter}"
+
+        return join_condition
+
+    def __where_condition(self, col=None, tbname=None, query_conditon=None):
+        if query_conditon and isinstance(query_conditon, str):
+            if query_conditon.startswith("count"):
+                query_conditon = query_conditon[6:-1]
+            elif query_conditon.startswith("max"):
+                query_conditon = query_conditon[4:-1]
+            elif query_conditon.startswith("sum"):
+                query_conditon = query_conditon[4:-1]
+            elif query_conditon.startswith("min"):
+                query_conditon = query_conditon[4:-1]
+
+
+        if query_conditon:
+            return f" where {query_conditon} is not null"
+        if col in NUM_COL:
+            return f" where abs( {tbname}.{col} ) >= 0"
+        if col in CHAR_COL:
+            return f" where lower( {tbname}.{col} ) like 'bina%' or lower( {tbname}.{col} ) like '_cha%' "
+        if col in BOOLEAN_COL:
+            return f" where {tbname}.{col} in (false, true)  "
+        if col in TS_TYPE_COL or col in PRIMARY_COL:
+            return f" where cast( {tbname}.{col} as binary(16) ) is not null "
+
+        return ""
+
+
+    def __group_condition(self, col, having = None):
+        if isinstance(col, str):
+            if col.startswith("count"):
+                col = col[6:-1]
+            elif col.startswith("max"):
+                col = col[4:-1]
+            elif col.startswith("sum"):
+                col = col[4:-1]
+            elif col.startswith("min"):
+                col = col[4:-1]
+        return f" group by {col} having {having}" if having else f" group by {col} "
+
+    def __single_sql(self, select_clause, from_clause, where_condition="", group_condition=""):
+        if isinstance(select_clause, str) and "on" not in from_clause and select_clause.split(".")[0] != from_clause.split(".")[0]:
+            return
+        return f"select {select_clause} from {from_clause} {where_condition} {group_condition}"
+
+
+    @property
+    def __join_tblist(self):
+        return [
+            ["ct1", "ct2"],
+            ["ct1", "ct4"],
+            ["ct1", "t1"],
+            ["ct2", "ct4"],
+            ["ct2", "t1"],
+            ["ct4", "t1"],
+            # ["ct1", "ct2", "ct4"],
+            # ["ct1", "ct2", "t1"],
+            # ["ct1", "ct4", "t1"],
+            # ["ct2", "ct4", "t1"],
+            # ["ct1", "ct2", "ct4", "t1"],
+        ]
+
+    @property
+    def __tb_liast(self):
+        return [
+            "ct1",
+            "ct2",
+            "ct4",
+            "t1",
+        ]
+
+    def sql_list(self):
+        sqls = []
+        __join_tblist = self.__join_tblist
+        for join_tblist in __join_tblist:
+            for join_tb in join_tblist:
+                select_claus_list = self.__query_condition(join_tb)
+                for select_claus in select_claus_list:
+                    group_claus = self.__group_condition( col=select_claus)
+                    where_claus = self.__where_condition(query_conditon=select_claus)
+                    having_claus = self.__group_condition( col=select_claus, having=f"{select_claus} is not null")
+                    sqls.extend(
+                        (
+                            self.__single_sql(select_claus, join_tb, where_claus, group_claus),
+                            self.__single_sql(select_claus, join_tb, where_claus, having_claus),
+                            self.__single_sql(select_claus, self.__join_condition(join_tblist), where_claus, having_claus),
+                            self.__single_sql(select_claus, self.__join_condition(join_tblist, INNER=True), where_claus, having_claus),
+                            self.__single_sql(select_claus, join_tb, where_claus),
+                            self.__single_sql(select_claus, join_tb, having_claus),
+                            self.__single_sql(select_claus, join_tb, group_claus),
+                            self.__single_sql(select_claus, join_tb),
+
+                        )
+                    )
+        __no_join_tblist = self.__tb_liast
+        for tb in __no_join_tblist:
+                select_claus_list = self.__query_condition(tb)
+                for select_claus in select_claus_list:
+                    group_claus = self.__group_condition(col=select_claus)
+                    where_claus = self.__where_condition(query_conditon=select_claus)
+                    having_claus = self.__group_condition(col=select_claus, having=f"{select_claus} is not null")
+                    sqls.extend(
+                        (
+                            self.__single_sql(select_claus, join_tb, where_claus, group_claus),
+                            self.__single_sql(select_claus, join_tb, where_claus, having_claus),
+                            self.__single_sql(select_claus, join_tb, where_claus),
+                            self.__single_sql(select_claus, join_tb, group_claus),
+                            self.__single_sql(select_claus, join_tb, having_claus),
+                            self.__single_sql(select_claus, join_tb),
+                        )
+                    )
+
+        return filter(None, sqls)
+        # return list(filter(None, sqls))
+
+    def __get_type(self, col):
+        if tdSql.cursor.istype(col, "BOOL"):
+            return "BOOL"
+        if tdSql.cursor.istype(col, "INT"):
+            return "INT"
+        if tdSql.cursor.istype(col, "BIGINT"):
+            return "BIGINT"
+        if tdSql.cursor.istype(col, "TINYINT"):
+            return "TINYINT"
+        if tdSql.cursor.istype(col, "SMALLINT"):
+            return "SMALLINT"
+        if tdSql.cursor.istype(col, "FLOAT"):
+            return "FLOAT"
+        if tdSql.cursor.istype(col, "DOUBLE"):
+            return "DOUBLE"
+        if tdSql.cursor.istype(col, "BINARY"):
+            return "BINARY"
+        if tdSql.cursor.istype(col, "NCHAR"):
+            return "NCHAR"
+        if tdSql.cursor.istype(col, "TIMESTAMP"):
+            return "TIMESTAMP"
+        if tdSql.cursor.istype(col, "JSON"):
+            return "JSON"
+        if tdSql.cursor.istype(col, "TINYINT UNSIGNED"):
+            return "TINYINT UNSIGNED"
+        if tdSql.cursor.istype(col, "SMALLINT UNSIGNED"):
+            return "SMALLINT UNSIGNED"
+        if tdSql.cursor.istype(col, "INT UNSIGNED"):
+            return "INT UNSIGNED"
+        if tdSql.cursor.istype(col, "BIGINT UNSIGNED"):
+            return "BIGINT UNSIGNED"
+
+    def union_check(self):
+        sqls = self.sql_list()
+        for sql1 in sqls:
+            tdSql.query(sql1)
+            res1_type = self.__get_type(0)
+            for sql2 in sqls:
+                tdSql.query(sql2)
+                union_type = False
+                res2_type =  self.__get_type(0)
+
+                if res1_type in  ( "BIGINT" , "NCHAR" ):
+                    union_type = True
+                elif res2_type == res1_type:
+                    union_type = True
+                elif res1_type == "TIMESAMP" and res2_type not in ("BINARY", "NCHAR"):
+                    union_type = True
+                elif res1_type == "BINARY" and res2_type != "NCHAR":
+                    union_type = True
+
+                if union_type:
+                    tdSql.query(f"{sql1} union {sql2}")
+                    tdSql.checkCols(1)
+                    tdSql.query(f"{sql1} union all {sql2}")
+                    tdSql.checkCols(1)
+                else:
+                    tdSql.error(f"{sql1} union {sql2}")
+
+    def __test_error(self):
+
+
+        tdSql.error( "show tables union show tables" )
+        tdSql.error( "create table errtb1 union all create table errtb2" )
+        tdSql.error( "drop table ct1 union all drop table ct3" )
+        tdSql.error( "select c1 from ct1 union all drop table ct3" )
+        tdSql.error( "select c1 from ct1 union all '' " )
+        tdSql.error( " '' union all select c1 from ct1 " )
+        tdSql.error( "select c1 from ct1 union select c1 from ct2 union select c1 from ct4 ")
+
+    def all_test(self):
+        self.__test_error()
+        self.union_check()
+
+
+    def __create_tb(self):
+
+        tdLog.printNoPrefix("==========step1:create table")
+        create_stb_sql  =  f'''create table stb1(
+                ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
+                 {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
+                 {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp
+            ) tags (t1 int)
+            '''
+        create_ntb_sql = f'''create table t1(
+                ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
+                 {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
+                 {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp
+            )
+            '''
+        tdSql.execute(create_stb_sql)
+        tdSql.execute(create_ntb_sql)
+
+        for i in range(4):
+            tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
+            { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}
+
+    def __insert_data(self, rows):
+        now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000)
+        for i in range(rows):
+            tdSql.execute(
+                f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
+            )
+            tdSql.execute(
+                f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
+            )
+            tdSql.execute(
+                f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i},  {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
+            )
+        tdSql.execute(
+            f'''insert into ct1 values
+            ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 } )
+            ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 } )
+            '''
+        )
+
+        tdSql.execute(
+            f'''insert into ct4 values
+            ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time +  7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            (
+                { now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127,
+                { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000}
+                )
+            (
+                { now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126,
+                { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000}
+                )
+            '''
+        )
+
+        tdSql.execute(
+            f'''insert into ct2 values
+            ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            (
+                { now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126,
+                { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }
+                )
+            (
+                { now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127,
+                { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }
+                )
+            '''
+        )
+
+        for i in range(rows):
+            insert_data = f'''insert into t1 values
+                ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2},
+                "binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i } )
+                '''
+            tdSql.execute(insert_data)
+        tdSql.execute(
+            f'''insert into t1 values
+            ( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127,
+                { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 },
+                "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }
+                )
+            (
+                { now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126,
+                { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 },
+                "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }
+                )
+            '''
+        )
+
+
+    def run(self):
+        tdSql.prepare()
+
+        tdLog.printNoPrefix("==========step1:create table")
+        self.__create_tb()
+
+        tdLog.printNoPrefix("==========step2:insert data")
+        self.rows = 10
+        self.__insert_data(self.rows)
+
+        tdLog.printNoPrefix("==========step3:all check")
+        self.all_test()
+
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+
+        tdSql.execute("use db")
+
+        tdLog.printNoPrefix("==========step4:after wal, all check again ")
+        self.all_test()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/7-tmq/basic5.py

@@ -360,7 +360,7 @@ class TDTestCase:
         # wait db ready
         while 1:
             tdSql.query("show databases")
-            if tdSql.getRows() == 4: 
+            if tdSql.getRows() == 5: 
                 print (tdSql.getData(0,0), tdSql.getData(1,0),tdSql.getData(2,0),)           
                 break
             else:

tests/system-test/7-tmq/subscribeDb.py

@@ -49,7 +49,38 @@ class TDTestCase:
         print(cur)
         return cur
 
-    def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg,showRow,cdbName,valgrind=0):
+    def initConsumerTable(self,cdbName='cdb'):        
+        tdLog.info("create consume database, and consume info table, and consume result table")
+        tdSql.query("create database if not exists %s vgroups 1"%(cdbName))
+        tdSql.query("drop table if exists %s.consumeinfo "%(cdbName))
+        tdSql.query("drop table if exists %s.consumeresult "%(cdbName))
+
+        tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName)
+        tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName)
+
+    def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,cdbName='cdb'):    
+        sql = "insert into %s.consumeinfo values "%cdbName
+        sql += "(now, %d, '%s', '%s', %d, %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit)
+        tdLog.info("consume info sql: %s"%sql)
+        tdSql.query(sql)
+
+    def selectConsumeResult(self,expectRows,cdbName='cdb'):
+        resultList=[]
+        while 1:
+            tdSql.query("select * from %s.consumeresult"%cdbName)
+            #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
+            if tdSql.getRows() == expectRows:
+                break
+            else:
+                time.sleep(5)
+        
+        for i in range(expectRows):
+            tdLog.info ("consume id: %d, consume msgs: %d, consume rows: %d"%(tdSql.getData(i , 1), tdSql.getData(i , 2), tdSql.getData(i , 3)))
+            resultList.append(tdSql.getData(i , 3))
+        
+        return resultList
+
+    def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0):
         shellCmd = 'nohup '
         if valgrind == 1:
             logFile = cfgPath + '/../log/valgrind-tmq.log'
@@ -58,7 +89,7 @@ class TDTestCase:
         
         shellCmd += buildPath + '/build/bin/tmq_sim -c ' + cfgPath
         shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) 
-        shellCmd += "> /dev/null 2>&1 &"        
+        shellCmd += "> /dev/null 2>&1 &"
         tdLog.info(shellCmd)
         os.system(shellCmd)
 
@@ -87,6 +118,8 @@ class TDTestCase:
         pre_insert = "insert into "
         sql = pre_insert
 
+        t = time.time()
+        startTs = int(round(t * 1000))
         #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
         for i in range(ctbNum):
             sql += " %s_%d values "%(stbName,i)
@@ -127,7 +160,7 @@ class TDTestCase:
         return
 
     def tmqCase1(self, cfgPath, buildPath):
-        tdLog.printNoPrefix("======== test case 1: Produce while one consume to subscribe one db")
+        tdLog.printNoPrefix("======== test case 1: Produce while one consume to subscribe one db, inclue 1 stb")
         tdLog.info("step 1: create database, stb, ctb and insert data")
         # create and start thread
         parameterDict = {'cfg':        '',       \
@@ -135,11 +168,13 @@ class TDTestCase:
                          'vgroups':    4,        \
                          'stbName':    'stb',    \
                          'ctbNum':     10,       \
-                         'rowsPerTbl': 100000,    \
+                         'rowsPerTbl': 10000,    \
-                         'batchNum':   200,      \
+                         'batchNum':   100,      \
                          'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
         parameterDict['cfg'] = cfgPath
 
+        self.initConsumerTable()
+
         tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups']))
 
         prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict)
@@ -149,23 +184,16 @@ class TDTestCase:
         topicName1 = 'topic_db1'
         
         tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName']))
-        
-        tdLog.info("create consume info table and consume result table")
-        cdbName = parameterDict["dbName"]
-        tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)"%cdbName)
-        tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName)
-
         consumerId   = 0
         expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"]
         topicList    = topicName1
         ifcheckdata  = 0
+        ifManualCommit = 0
         keyList      = 'group.id:cgrp1,\
                         enable.auto.commit:false,\
                         auto.commit.interval.ms:6000,\
                         auto.offset.reset:earliest'
-        sql = "insert into %s.consumeinfo values "%cdbName
+        self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
-        sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata)
-        tdSql.query(sql)
         
         event.wait()
 
@@ -173,32 +201,28 @@ class TDTestCase:
         pollDelay = 5
         showMsg   = 1
         showRow   = 1
-        self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow, cdbName)
+        self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
 
         # wait for data ready
         prepareEnvThread.join()
         
         tdLog.info("insert process end, and start to check consume result")
-        while 1:
+        expectRows = 1
-            tdSql.query("select * from %s.consumeresult"%cdbName)
+        resultList = self.selectConsumeResult(expectRows)
-            #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
+        totalConsumeRows = 0
-            if tdSql.getRows() == 1:
+        for i in range(expectRows):
-                break
+            totalConsumeRows += resultList[i]
-            else:
+        
-                time.sleep(5)
+        if totalConsumeRows != expectrowcnt:
-
+            tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
-        tdLog.info("consumer result: %d, %d"%(tdSql.getData(0 , 2), tdSql.getData(0 , 3)))
+            tdLog.exit("tmq consume rows error!")
-        tdSql.checkData(0 , 1, consumerId)
-         # mulit rows and mulit tables in one sql, this num of msg is not sure
-        #tdSql.checkData(0 , 2, expectmsgcnt)
-        tdSql.checkData(0 , 3, expectrowcnt+1)
 
         tdSql.query("drop topic %s"%topicName1)
 
         tdLog.printNoPrefix("======== test case 1 end ...... ")
 
     def tmqCase2(self, cfgPath, buildPath):
-        tdLog.printNoPrefix("======== test case 2: Produce while two consumers to subscribe one db")
+        tdLog.printNoPrefix("======== test case 2: Produce while two consumers to subscribe one db, inclue 1 stb")
         tdLog.info("step 1: create database, stb, ctb and insert data")
         # create and start thread
         parameterDict = {'cfg':        '',       \
@@ -206,11 +230,13 @@ class TDTestCase:
                          'vgroups':    4,        \
                          'stbName':    'stb',    \
                          'ctbNum':     10,       \
-                         'rowsPerTbl': 100000,    \
+                         'rowsPerTbl': 10000,    \
                          'batchNum':   100,      \
                          'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
         parameterDict['cfg'] = cfgPath
 
+        self.initConsumerTable()
+
         tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups']))
 
         prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict)
@@ -221,27 +247,19 @@ class TDTestCase:
         
         tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName']))
         
-        tdLog.info("create consume info table and consume result table")
-        cdbName = parameterDict["dbName"]
-        tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)"%cdbName)
-        tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName)
-
         consumerId   = 0
         expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"]
         topicList    = topicName1
         ifcheckdata  = 0
+        ifManualCommit = 1
         keyList      = 'group.id:cgrp1,\
                         enable.auto.commit:false,\
                         auto.commit.interval.ms:6000,\
                         auto.offset.reset:earliest'
-        sql = "insert into %s.consumeinfo values "%cdbName
+        self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
-        sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata)
+
-        tdSql.query(sql)
-        
         consumerId   = 1
-        sql = "insert into %s.consumeinfo values "%cdbName
+        self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
-        sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata)
-        tdSql.query(sql)
         
         event.wait()
 
@@ -249,30 +267,20 @@ class TDTestCase:
         pollDelay = 5
         showMsg   = 1
         showRow   = 1        
-        self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow, cdbName)
+        self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
 
         # wait for data ready
         prepareEnvThread.join()
         
         tdLog.info("insert process end, and start to check consume result")
-        while 1:
+        expectRows = 2
-            tdSql.query("select * from %s.consumeresult"%cdbName)
+        resultList = self.selectConsumeResult(expectRows)
-            #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
+        totalConsumeRows = 0
-            if tdSql.getRows() == 2:
+        for i in range(expectRows):
-                break
+            totalConsumeRows += resultList[i]
-            else:
+        
-                time.sleep(5)
+        if totalConsumeRows != expectrowcnt:
-
+            tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
-        consumerId0     = tdSql.getData(0 , 1)
-        consumerId1     = tdSql.getData(1 , 1)
-        actConsumeRows0 = tdSql.getData(0 , 3)
-        actConsumeRows1 = tdSql.getData(1 , 3)
-
-        tdLog.info("consumer %d rows: %d"%(consumerId0, actConsumeRows0))
-        tdLog.info("consumer %d rows: %d"%(consumerId1, actConsumeRows1))
-
-        totalConsumeRows = actConsumeRows0 + actConsumeRows1
-        if totalConsumeRows != expectrowcnt + 2:
             tdLog.exit("tmq consume rows error!")
 
         tdSql.query("drop topic %s"%topicName1)
@@ -288,11 +296,13 @@ class TDTestCase:
                          'vgroups':    4,        \
                          'stbName':    'stb',    \
                          'ctbNum':     10,       \
-                         'rowsPerTbl': 100000,    \
+                         'rowsPerTbl': 10000,    \
                          'batchNum':   100,      \
                          'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
         parameterDict['cfg'] = cfgPath
 
+        self.initConsumerTable()
+
         tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups']))
 
         prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict)
@@ -303,7 +313,7 @@ class TDTestCase:
                          'vgroups':    4,        \
                          'stbName':    'stb2',    \
                          'ctbNum':     10,       \
-                         'rowsPerTbl': 100000,    \
+                         'rowsPerTbl': 10000,    \
                          'batchNum':   100,      \
                          'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
         parameterDict['cfg'] = cfgPath
@@ -316,65 +326,377 @@ class TDTestCase:
         
         tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName']))
         
-        tdLog.info("create consume info table and consume result table")
-        cdbName = parameterDict["dbName"]
-        tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)"%cdbName)
-        tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName)
-
         consumerId   = 0
         expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] +  parameterDict2["rowsPerTbl"] * parameterDict2["ctbNum"]
         topicList    = topicName1
         ifcheckdata  = 0
+        ifManualCommit = 0
         keyList      = 'group.id:cgrp1,\
                         enable.auto.commit:false,\
                         auto.commit.interval.ms:6000,\
                         auto.offset.reset:earliest'
-        sql = "insert into %s.consumeinfo values "%cdbName
+        self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
-        sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata)
-        tdSql.query(sql)
         
         # consumerId   = 1
-        # sql = "insert into %s.consumeinfo values "%cdbName
+        # self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
-        # sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata)
-        # tdSql.query(sql)
         
         event.wait()
 
         tdLog.info("start consume processor")
         pollDelay = 5
         showMsg   = 1
-        showRow   = 1        
+        showRow   = 1
-        self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow, cdbName)
+        self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
 
         # wait for data ready
         prepareEnvThread.join()        
         prepareEnvThread2.join()
         
         tdLog.info("insert process end, and start to check consume result")
-        while 1:
+        expectRows = 1
-            tdSql.query("select * from %s.consumeresult"%cdbName)
+        resultList = self.selectConsumeResult(expectRows)
-            #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
+        totalConsumeRows = 0
-            if tdSql.getRows() == 1:
+        for i in range(expectRows):
-                break
+            totalConsumeRows += resultList[i]
-            else:
+        
-                time.sleep(5)
+        if totalConsumeRows != expectrowcnt:
-
+            tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
-        consumerId0     = tdSql.getData(0 , 1)
-        #consumerId1     = tdSql.getData(1 , 1)
-        actConsumeRows0 = tdSql.getData(0 , 3)
-        #actConsumeRows1 = tdSql.getData(1 , 3)
-
-        tdLog.info("consumer %d rows: %d"%(consumerId0, actConsumeRows0))
-        #tdLog.info("consumer %d rows: %d"%(consumerId1, actConsumeRows1))
-
-        #totalConsumeRows = actConsumeRows0 + actConsumeRows1
-        if actConsumeRows0 != expectrowcnt + 1:
             tdLog.exit("tmq consume rows error!")
 
         tdSql.query("drop topic %s"%topicName1)
 
         tdLog.printNoPrefix("======== test case 3 end ...... ")
 
+    def tmqCase4(self, cfgPath, buildPath):
+        tdLog.printNoPrefix("======== test case 4: Produce while two consumers to subscribe one db, include 2 stb")
+        tdLog.info("step 1: create database, stb, ctb and insert data")
+        # create and start thread
+        parameterDict = {'cfg':        '',       \
+                         'dbName':     'db4',    \
+                         'vgroups':    4,        \
+                         'stbName':    'stb',    \
+                         'ctbNum':     10,       \
+                         'rowsPerTbl': 10000,    \
+                         'batchNum':   100,      \
+                         'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
+        parameterDict['cfg'] = cfgPath
+
+        self.initConsumerTable()
+
+        tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups']))
+
+        prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict)
+        prepareEnvThread.start()
+
+        parameterDict2 = {'cfg':        '',       \
+                         'dbName':     'db4',    \
+                         'vgroups':    4,        \
+                         'stbName':    'stb2',    \
+                         'ctbNum':     10,       \
+                         'rowsPerTbl': 10000,    \
+                         'batchNum':   100,      \
+                         'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
+        parameterDict['cfg'] = cfgPath
+
+        prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2)
+        prepareEnvThread2.start()
+
+        tdLog.info("create topics from db")
+        topicName1 = 'topic_db1'
+        
+        tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName']))
+        
+        consumerId   = 0
+        expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] +  parameterDict2["rowsPerTbl"] * parameterDict2["ctbNum"]
+        topicList    = topicName1
+        ifcheckdata  = 0
+        ifManualCommit = 1
+        keyList      = 'group.id:cgrp1,\
+                        enable.auto.commit:false,\
+                        auto.commit.interval.ms:6000,\
+                        auto.offset.reset:earliest'
+        self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+        
+        consumerId   = 1
+        self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+        
+        event.wait()
+
+        tdLog.info("start consume processor")
+        pollDelay = 5
+        showMsg   = 1
+        showRow   = 1
+        self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
+
+        # wait for data ready
+        prepareEnvThread.join()        
+        prepareEnvThread2.join()
+        
+        tdLog.info("insert process end, and start to check consume result")
+        expectRows = 2
+        resultList = self.selectConsumeResult(expectRows)
+        totalConsumeRows = 0
+        for i in range(expectRows):
+            totalConsumeRows += resultList[i]
+        
+        if totalConsumeRows != expectrowcnt:
+            tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
+            tdLog.exit("tmq consume rows error!")
+
+        tdSql.query("drop topic %s"%topicName1)
+
+        tdLog.printNoPrefix("======== test case 4 end ...... ")
+
+    def tmqCase5(self, cfgPath, buildPath):
+        tdLog.printNoPrefix("======== test case 5: Produce while two consumers to subscribe one db, firstly create one stb, after start consume create other stb")
+        tdLog.info("step 1: create database, stb, ctb and insert data")
+        # create and start thread
+        parameterDict = {'cfg':        '',       \
+                         'dbName':     'db5',    \
+                         'vgroups':    4,        \
+                         'stbName':    'stb',    \
+                         'ctbNum':     10,       \
+                         'rowsPerTbl': 10000,    \
+                         'batchNum':   100,      \
+                         'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
+        parameterDict['cfg'] = cfgPath
+
+        self.initConsumerTable()
+
+        tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups']))
+
+        prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict)
+        prepareEnvThread.start()
+
+        parameterDict2 = {'cfg':        '',       \
+                         'dbName':     'db5',    \
+                         'vgroups':    4,        \
+                         'stbName':    'stb2',    \
+                         'ctbNum':     10,       \
+                         'rowsPerTbl': 10000,    \
+                         'batchNum':   100,      \
+                         'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
+        parameterDict['cfg'] = cfgPath
+
+        tdLog.info("create topics from db")
+        topicName1 = 'topic_db1'
+        
+        tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName']))
+        
+        consumerId   = 0
+        expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] +  parameterDict2["rowsPerTbl"] * parameterDict2["ctbNum"]
+        topicList    = topicName1
+        ifcheckdata  = 0
+        ifManualCommit = 1
+        keyList      = 'group.id:cgrp1,\
+                        enable.auto.commit:false,\
+                        auto.commit.interval.ms:6000,\
+                        auto.offset.reset:earliest'
+        self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+        
+        consumerId   = 1
+        self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+        
+        event.wait()
+
+        tdLog.info("start consume processor")
+        pollDelay = 5
+        showMsg   = 1
+        showRow   = 1
+        self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
+
+        prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2)
+        prepareEnvThread2.start()
+
+        # wait for data ready
+        prepareEnvThread.join()        
+        prepareEnvThread2.join()
+        
+        tdLog.info("insert process end, and start to check consume result")
+        expectRows = 2
+        resultList = self.selectConsumeResult(expectRows)
+        totalConsumeRows = 0
+        for i in range(expectRows):
+            totalConsumeRows += resultList[i]
+        
+        if totalConsumeRows != expectrowcnt:
+            tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
+            tdLog.exit("tmq consume rows error!")
+
+        tdSql.query("drop topic %s"%topicName1)
+
+        tdLog.printNoPrefix("======== test case 5 end ...... ")
+
+    def tmqCase6(self, cfgPath, buildPath):
+        tdLog.printNoPrefix("======== test case 6: Produce while one consumers to subscribe tow topic, Each contains one db")
+        tdLog.info("step 1: create database, stb, ctb and insert data")
+        # create and start thread
+        parameterDict = {'cfg':        '',       \
+                         'dbName':     'db60',    \
+                         'vgroups':    4,        \
+                         'stbName':    'stb',    \
+                         'ctbNum':     10,       \
+                         'rowsPerTbl': 10000,    \
+                         'batchNum':   100,      \
+                         'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
+        parameterDict['cfg'] = cfgPath
+
+        self.initConsumerTable()
+
+        tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups']))
+
+        prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict)
+        prepareEnvThread.start()
+
+        parameterDict2 = {'cfg':        '',       \
+                         'dbName':     'db61',    \
+                         'vgroups':    4,        \
+                         'stbName':    'stb2',    \
+                         'ctbNum':     10,       \
+                         'rowsPerTbl': 10000,    \
+                         'batchNum':   100,      \
+                         'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
+        parameterDict['cfg'] = cfgPath
+
+        tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict2['dbName'], parameterDict2['vgroups']))
+
+        prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2)
+        prepareEnvThread2.start()
+
+        tdLog.info("create topics from db")
+        topicName1 = 'topic_db60'
+        topicName2 = 'topic_db61'
+        
+        tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName']))        
+        tdSql.execute("create topic %s as %s" %(topicName2, parameterDict2['dbName']))
+        
+        consumerId   = 0
+        expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] +  parameterDict2["rowsPerTbl"] * parameterDict2["ctbNum"]
+        topicList    = topicName1 + ',' + topicName2
+        ifcheckdata  = 0
+        ifManualCommit = 0
+        keyList      = 'group.id:cgrp1,\
+                        enable.auto.commit:false,\
+                        auto.commit.interval.ms:6000,\
+                        auto.offset.reset:earliest'
+        self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+        
+        #consumerId   = 1
+        #self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+        
+        event.wait()
+
+        tdLog.info("start consume processor")
+        pollDelay = 5
+        showMsg   = 1
+        showRow   = 1
+        self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
+
+        # wait for data ready
+        prepareEnvThread.join()        
+        prepareEnvThread2.join()
+        
+        tdLog.info("insert process end, and start to check consume result")
+        expectRows = 1
+        resultList = self.selectConsumeResult(expectRows)
+        totalConsumeRows = 0
+        for i in range(expectRows):
+            totalConsumeRows += resultList[i]
+        
+        if totalConsumeRows != expectrowcnt:
+            tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
+            tdLog.exit("tmq consume rows error!")
+
+        tdSql.query("drop topic %s"%topicName1)
+        tdSql.query("drop topic %s"%topicName2)
+
+        tdLog.printNoPrefix("======== test case 6 end ...... ")
+
+    def tmqCase7(self, cfgPath, buildPath):
+        tdLog.printNoPrefix("======== test case 7: Produce while two consumers to subscribe tow topic, Each contains one db")
+        tdLog.info("step 1: create database, stb, ctb and insert data")
+        # create and start thread
+        parameterDict = {'cfg':        '',       \
+                         'dbName':     'db70',    \
+                         'vgroups':    4,        \
+                         'stbName':    'stb',    \
+                         'ctbNum':     10,       \
+                         'rowsPerTbl': 10000,    \
+                         'batchNum':   100,      \
+                         'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
+        parameterDict['cfg'] = cfgPath
+
+        self.initConsumerTable()
+
+        tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups']))
+
+        prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict)
+        prepareEnvThread.start()
+
+        parameterDict2 = {'cfg':        '',       \
+                         'dbName':     'db71',    \
+                         'vgroups':    4,        \
+                         'stbName':    'stb2',    \
+                         'ctbNum':     10,       \
+                         'rowsPerTbl': 10000,    \
+                         'batchNum':   100,      \
+                         'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
+        parameterDict['cfg'] = cfgPath
+
+        tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict2['dbName'], parameterDict2['vgroups']))
+
+        prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2)
+        prepareEnvThread2.start()
+
+        tdLog.info("create topics from db")
+        topicName1 = 'topic_db60'
+        topicName2 = 'topic_db61'
+        
+        tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName']))        
+        tdSql.execute("create topic %s as %s" %(topicName2, parameterDict2['dbName']))
+        
+        consumerId   = 0
+        expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] +  parameterDict2["rowsPerTbl"] * parameterDict2["ctbNum"]
+        topicList    = topicName1 + ',' + topicName2
+        ifcheckdata  = 0
+        ifManualCommit = 1
+        keyList      = 'group.id:cgrp1,\
+                        enable.auto.commit:false,\
+                        auto.commit.interval.ms:6000,\
+                        auto.offset.reset:earliest'
+        self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+        
+        consumerId   = 1
+        self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+        
+        event.wait()
+
+        tdLog.info("start consume processor")
+        pollDelay = 5
+        showMsg   = 1
+        showRow   = 1
+        self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
+
+        # wait for data ready
+        prepareEnvThread.join()        
+        prepareEnvThread2.join()
+        
+        tdLog.info("insert process end, and start to check consume result")
+        expectRows = 2
+        resultList = self.selectConsumeResult(expectRows)
+        totalConsumeRows = 0
+        for i in range(expectRows):
+            totalConsumeRows += resultList[i]
+        
+        if totalConsumeRows != expectrowcnt:
+            tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
+            tdLog.exit("tmq consume rows error!")
+
+        tdSql.query("drop topic %s"%topicName1)
+        tdSql.query("drop topic %s"%topicName2)
+
+        tdLog.printNoPrefix("======== test case 7 end ...... ")
+
     def run(self):
         tdSql.prepare()
 
@@ -386,9 +708,14 @@ class TDTestCase:
         cfgPath = buildPath + "/../sim/psim/cfg"
         tdLog.info("cfgPath: %s" % cfgPath)
 
-        #self.tmqCase1(cfgPath, buildPath)
+        self.tmqCase1(cfgPath, buildPath)
         self.tmqCase2(cfgPath, buildPath) 
-        #self.tmqCase3(cfgPath, buildPath)
+        self.tmqCase3(cfgPath, buildPath)
+        self.tmqCase4(cfgPath, buildPath)
+        self.tmqCase5(cfgPath, buildPath)
+        self.tmqCase6(cfgPath, buildPath)
+        self.tmqCase7(cfgPath, buildPath)
+
 
     def stop(self):
         tdSql.close()

tests/system-test/fulltest.sh

@@ -54,5 +54,8 @@ python3 ./test.py -f 2-query/arcsin.py
 python3 ./test.py -f 2-query/arccos.py
 python3 ./test.py -f 2-query/arctan.py
 python3 ./test.py -f 2-query/query_cols_tags_and_or.py
+python3 ./test.py -f 2-query/nestedQuery.py
 
 python3 ./test.py -f 7-tmq/basic5.py
+python3 ./test.py -f 7-tmq/subscribeDb.py
+