Merge branch '3.0' of https://github.com/taosdata/TDengine into feat/row_refact

include/client/taos.h

@@ -194,6 +194,7 @@ DLL_EXPORT int         taos_errno(TAOS_RES *tres);
 
 DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param);
 DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param);
+DLL_EXPORT void taos_fetch_raw_block_a(TAOS_RES* res, __taos_async_fn_t fp, void* param);
 
 // Shuduo: temporary enable for app build
 #if 1
@@ -209,12 +210,12 @@ DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLi
 
 /* --------------------------TMQ INTERFACE------------------------------- */
 
-enum tmq_resp_err_t {
+enum {
   TMQ_RESP_ERR__FAIL = -1,
   TMQ_RESP_ERR__SUCCESS = 0,
 };
 
-typedef enum tmq_resp_err_t tmq_resp_err_t;
+typedef int32_t tmq_resp_err_t;
 
 typedef struct tmq_t                   tmq_t;
 typedef struct tmq_topic_vgroup_t      tmq_topic_vgroup_t;

include/common/tdatablock.h

@@ -206,7 +206,7 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3
 size_t blockDataGetSize(const SSDataBlock* pBlock);
 size_t blockDataGetRowSize(SSDataBlock* pBlock);
 double blockDataGetSerialRowSize(const SSDataBlock* pBlock);
-size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock);
+size_t blockDataGetSerialMetaSize(uint32_t numOfCols);
 
 int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo);
 int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst);
@@ -238,7 +238,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool
                             const char* stbFullName, int32_t vgId);
 
 static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) {
-  return blockDataGetSerialMetaSize(pBlock) + blockDataGetSize(pBlock);
+  return blockDataGetSerialMetaSize(pBlock->info.numOfCols) + blockDataGetSize(pBlock);
 }
 
 static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data,

include/common/tmsg.h

@@ -2500,6 +2500,7 @@ typedef struct {
   int64_t   sliding;
   int64_t   dstTbUid;  
   int32_t   dstVgId;  // for stream  
+  SEpSet    epSet;
   char*     expr;
 } STableIndexInfo;
 

include/libs/catalog/catalog.h

@@ -71,16 +71,21 @@ typedef struct SCatalogReq {
   bool    forceUpdate;    
 } SCatalogReq;
 
+typedef struct SMetaRes {
+  int32_t code;
+  void*   pRes;
+} SMetaRes;
+
 typedef struct SMetaData {
-  SArray    *pDbVgroup;   // SArray<SArray<SVgroupInfo>*>
+  SArray    *pDbVgroup;   // pRes = SArray<SVgroupInfo>*
-  SArray    *pDbCfg;      // SArray<SDbCfgInfo>
+  SArray    *pDbCfg;      // pRes = SDbCfgInfo*
-  SArray    *pDbInfo;     // SArray<SDbInfo>
+  SArray    *pDbInfo;     // pRes = SDbInfo*
-  SArray    *pTableMeta;  // SArray<STableMeta*>
+  SArray    *pTableMeta;  // pRes = STableMeta*
-  SArray    *pTableHash;  // SArray<SVgroupInfo>
+  SArray    *pTableHash;  // pRes = SVgroupInfo*
-  SArray    *pUdfList;    // SArray<SFuncInfo>
+  SArray    *pUdfList;    // pRes = SFuncInfo*
-  SArray    *pIndex;      // SArray<SIndexInfo>
+  SArray    *pIndex;      // pRes = SIndexInfo*
-  SArray    *pUser;       // SArray<bool>
+  SArray    *pUser;       // pRes = bool*
-  SArray    *pQnodeList;  // SArray<SQueryNodeAddr>
+  SArray    *pQnodeList;  // pRes = SQueryNodeAddr*
 } SMetaData;
 
 typedef struct SCatalogCfg {

include/libs/function/function.h

@@ -197,6 +197,7 @@ typedef struct SAggFunctionInfo {
 struct SScalarParam {
   SColumnInfoData *columnData;
   SHashObj        *pHashFilter;
+  int32_t          hashValueType;
   void            *param;  // other parameter, such as meta handle from vnode, to extract table name/tag value
   int32_t          numOfRows;
 };

include/libs/function/functionMgt.h

@@ -127,6 +127,8 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_APERCENTILE_MERGE,
   FUNCTION_TYPE_SPREAD_PARTIAL,
   FUNCTION_TYPE_SPREAD_MERGE,
+  FUNCTION_TYPE_HISTOGRAM_PARTIAL,
+  FUNCTION_TYPE_HISTOGRAM_MERGE,
 
   // user defined funcion
   FUNCTION_TYPE_UDF = 10000

include/libs/qcom/query.h

@@ -210,8 +210,8 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
 #define NEED_CLIENT_RM_TBLMETA_ERROR(_code)                                                   \
   ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST ||       \
    (_code) == TSDB_CODE_PAR_INVALID_COLUMNS_NUM || (_code) == TSDB_CODE_PAR_INVALID_COLUMN || \
-   (_code) == TSDB_CODE_PAR_TAGS_NOT_MATCHED || (_code == TSDB_CODE_PAR_VALUE_TOO_LONG) || \
+   (_code) == TSDB_CODE_PAR_TAGS_NOT_MATCHED || (_code) == TSDB_CODE_PAR_VALUE_TOO_LONG || \
-   (_code == TSDB_CODE_PAR_INVALID_DROP_COL))
+   (_code) == TSDB_CODE_PAR_INVALID_DROP_COL || ((_code) == TSDB_CODE_TDB_INVALID_TABLE_ID))
 #define NEED_CLIENT_REFRESH_VG_ERROR(_code) \
   ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID)
 #define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_TABLE_RECREATED)

include/libs/stream/tstream.h

@@ -339,6 +339,7 @@ typedef struct {
   int32_t sourceTaskId;
   int32_t sourceVg;
   int32_t sourceChildId;
+  int32_t upstreamNodeId;
 #if 0
   int64_t sourceVer;
 #endif

include/libs/sync/syncTools.h

@@ -33,7 +33,7 @@ SSyncNode* syncNodeAcquire(int64_t rid);
 void       syncNodeRelease(SSyncNode* pNode);
 
 int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg);
-int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg);
+int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcHandleInfo* pInfo);
 void    syncSetMsgCb(int64_t rid, const SMsgCb* msgcb);
 char*   sync2SimpleStr(int64_t rid);
 

include/libs/transport/trpc.h

@@ -33,7 +33,10 @@ extern int32_t tsRpcHeadSize;
 typedef struct {
   uint32_t clientIp;
   uint16_t clientPort;
+  union {
     char    user[TSDB_USER_LEN];
+    int64_t applyIndex;
+  };
 } SRpcConnInfo;
 
 typedef struct SRpcHandleInfo {

include/util/taoserror.h

@@ -339,9 +339,9 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE       TAOS_DEF_ERROR_CODE(0, 0x060D)
 #define TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE    TAOS_DEF_ERROR_CODE(0, 0x060E)
 #define TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP       TAOS_DEF_ERROR_CODE(0, 0x060F)
-#define TSDB_CODE_TDB_INVALID_ACTION            TAOS_DEF_ERROR_CODE(0, 0x0600)
+#define TSDB_CODE_TDB_INVALID_ACTION            TAOS_DEF_ERROR_CODE(0, 0x0610)
-#define TSDB_CODE_TDB_INVALID_CREATE_TB_MSG     TAOS_DEF_ERROR_CODE(0, 0x0601)
+#define TSDB_CODE_TDB_INVALID_CREATE_TB_MSG     TAOS_DEF_ERROR_CODE(0, 0x0611)
-#define TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM      TAOS_DEF_ERROR_CODE(0, 0x0602)
+#define TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM      TAOS_DEF_ERROR_CODE(0, 0x0612)
 #define TSDB_CODE_TDB_FILE_ALREADY_EXISTS       TAOS_DEF_ERROR_CODE(0, 0x0613)
 #define TSDB_CODE_TDB_TABLE_RECONFIGURE         TAOS_DEF_ERROR_CODE(0, 0x0614)
 #define TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO     TAOS_DEF_ERROR_CODE(0, 0x0615)

include/util/tdef.h

@@ -356,8 +356,8 @@ typedef enum ELogicConditionType {
 
 #define TSDB_DEFAULT_EXPLAIN_VERBOSE false
 
-#define TSDB_EXPLAIN_RESULT_ROW_SIZE    1024
+#define TSDB_EXPLAIN_RESULT_ROW_SIZE    512
-#define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY PLAN"
+#define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY_PLAN"
 
 #define TSDB_MAX_FIELD_LEN             16384
 #define TSDB_MAX_BINARY_LEN            (TSDB_MAX_FIELD_LEN - TSDB_KEYSIZE)  // keep 16384

source/client/inc/clientInt.h

@@ -45,7 +45,7 @@ extern "C" {
 
 #define ERROR_MSG_BUF_DEFAULT_SIZE 512
 #define HEARTBEAT_INTERVAL         1500  // ms
-#define SYNC_ON_TOP_OF_ASYNC       0
+#define SYNC_ON_TOP_OF_ASYNC       1
 
 enum {
   RES_TYPE__QUERY = 1,
@@ -144,7 +144,7 @@ typedef struct STscObj {
   int32_t       numOfReqs;  // number of sqlObj bound to this connection
   SAppInstInfo* pAppInfo;
   SHashObj*     pRequests;
-  int8_t        schemalessType;
+  int8_t        schemalessType;  // todo remove it, this attribute should be move to request
 } STscObj;
 
 typedef struct SResultColumn {
@@ -171,6 +171,7 @@ typedef struct SReqResultInfo {
   uint32_t       current;
   bool           completed;
   int32_t        precision;
+  bool           convertUcs4;
   int32_t        payloadLen;
 } SReqResultInfo;
 
@@ -222,7 +223,7 @@ typedef struct SSyncQueryParam {
   SRequestObj* pRequest;
 } SSyncQueryParam;
 
-void*   doAsyncFetchRow(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
+void*   doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
 void*   doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
 
 void    doSetOneRowPtr(SReqResultInfo* pResultInfo);

source/client/src/clientEnv.c

@@ -191,6 +191,8 @@ void *createRequest(STscObj *pObj, int32_t type) {
   pRequest->requestId = generateRequestId();
   pRequest->metric.start = taosGetTimestampUs();
 
+  pRequest->body.resInfo.convertUcs4 = true;  // convert ucs4 by default
+
   pRequest->type = type;
   pRequest->pTscObj = pObj;
   pRequest->msgBuf = taosMemoryCalloc(1, ERROR_MSG_BUF_DEFAULT_SIZE);

source/client/src/clientImpl.c

@@ -580,7 +580,8 @@ void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) {
 
   STscObj* pTscObj = pRequest->pTscObj;
   if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code)) {
-    tscDebug("0x%"PRIx64" client retry to handle the error, code:%s, reqId:0x%"PRIx64, pRequest->self, tstrerror(code), pRequest->requestId);
+    tscDebug("0x%"PRIx64" client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%"PRIx64, pRequest->self, code, tstrerror(code),
+        pRequest->retry, pRequest->requestId);
     pRequest->prevCode = code;
     doAsyncQuery(pRequest, true);
     return;
@@ -592,6 +593,7 @@ void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) {
     pRequest->code = code;
   }
 
+  tscDebug("schedulerExecCb request type %s", TMSG_INFO(pRequest->type));
   if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type)) {
     removeMeta(pTscObj, pRequest->tableList);
   }
@@ -695,6 +697,8 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) {
       if (TSDB_CODE_SUCCESS == code) {
         schedulerAsyncExecJob(pAppInfo->pTransporter, pNodeList, pRequest->body.pDag, &pRequest->body.queryJob,
                               pRequest->sqlstr, pRequest->metric.start, schedulerExecCb, pRequest);
+      } else {
+        pRequest->body.queryFp(pRequest->body.param, pRequest, code);
       }
 
       //todo not to be released here
@@ -703,6 +707,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) {
     }
     case QUERY_EXEC_MODE_EMPTY_RESULT:
       pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT;      
+      pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
       break;
     default:
       break;
@@ -1109,7 +1114,7 @@ static void syncFetchFn(void* param, TAOS_RES* res, int32_t numOfRows) {
   tsem_post(&pParam->sem);
 }
 
-void* doAsyncFetchRow(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) {
+void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) {
   assert(pRequest != NULL);
 
   SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
@@ -1121,6 +1126,10 @@ void* doAsyncFetchRow(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUc
     }
 
     SSyncQueryParam* pParam = pRequest->body.param;
+
+    // convert ucs4 to native multi-bytes string
+    pResultInfo->convertUcs4 = convertUcs4;
+
     taos_fetch_rows_a(pRequest, syncFetchFn, pParam);
     tsem_wait(&pParam->sem);
   }
@@ -1339,6 +1348,17 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
   uint64_t groupId = *(uint64_t*)p;
   p += sizeof(uint64_t);
 
+  // check fields
+  for(int32_t i = 0; i < numOfCols; ++i) {
+    int16_t type = *(int16_t*) p;
+    p += sizeof(int16_t);
+
+    int32_t bytes = *(int32_t*) p;
+    p += sizeof(int32_t);
+
+//    ASSERT(type == pFields[i].type && bytes == pFields[i].bytes);
+  }
+
   int32_t* colLength = (int32_t*)p;
   p += sizeof(int32_t) * numOfCols;
 

source/client/src/clientMain.c

@@ -219,14 +219,13 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
 
   if (TD_RES_QUERY(res)) {
     SRequestObj *pRequest = (SRequestObj *)res;
+#if SYNC_ON_TOP_OF_ASYNC
+    return doAsyncFetchRows(pRequest, true, true);
+#else
     if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT ||
         pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) {
       return NULL;
     }
-
-#if SYNC_ON_TOP_OF_ASYNC
-    return doAsyncFetchRow(pRequest, true, true);
-#else
     return doFetchRows(pRequest, true, true);
 #endif
 
@@ -489,6 +488,7 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
   if (res == NULL) {
     return 0;
   }
+
   if (TD_RES_QUERY(res)) {
     SRequestObj *pRequest = (SRequestObj *)res;
 
@@ -500,7 +500,11 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
       return 0;
     }
 
-    doFetchRows(pRequest, false, true);
+#if SYNC_ON_TOP_OF_ASYNC
+    doAsyncFetchRows(pRequest, false, true);
+#else
+    doFetchRows(pRequest, true, true);
+#endif
 
     // TODO refactor
     SReqResultInfo *pResultInfo = &pRequest->body.resInfo;
@@ -548,7 +552,11 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
     return 0;
   }
 
+#if SYNC_ON_TOP_OF_ASYNC
+  doAsyncFetchRows(pRequest, false, false);
+#else
   doFetchRows(pRequest, false, false);
+#endif
 
   SReqResultInfo *pResultInfo = &pRequest->body.resInfo;
 
@@ -625,8 +633,10 @@ void retrieveMetaCallback(SMetaData* pResultMeta, void* param, int32_t code) {
     taosMemoryFree(pWrapper);
     launchAsyncQuery(pRequest, pQuery);
   } else {
+    tscDebug("error happens, code:%d", code);
     if (NEED_CLIENT_HANDLE_ERROR(code)) {
-      tscDebug("0x%"PRIx64" client retry to handle the error, code:%s, reqId:0x%"PRIx64, pRequest->self, tstrerror(code), pRequest->requestId);
+      tscDebug("0x%"PRIx64" client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%"PRIx64, pRequest->self, code, tstrerror(code),
+               pRequest->retry, pRequest->requestId);
       pRequest->prevCode = code;
       doAsyncQuery(pRequest, true);
       return;
@@ -691,6 +701,7 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext** pCxt) {
                        .pTransporter = pTscObj->pAppInfo->pTransporter,
                        .pStmtCb = NULL,
                        .pUser = pTscObj->user,
+                       .schemalessType = pTscObj->schemalessType,
                        .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)),
                        .async = true,};
   return TSDB_CODE_SUCCESS;
@@ -699,13 +710,14 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext** pCxt) {
 void doAsyncQuery(SRequestObj* pRequest, bool updateMetaForce) {
   SParseContext* pCxt = NULL;
   STscObj *pTscObj = pRequest->pTscObj;
+  int32_t code = 0;
 
   if (pRequest->retry++ > REQUEST_TOTAL_EXEC_TIMES) {
-    pRequest->code = pRequest->prevCode;
+    code = pRequest->prevCode;
     goto _error;
   }
 
-  int32_t code = createParseContext(pRequest, &pCxt);
+  code = createParseContext(pRequest, &pCxt);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -742,7 +754,7 @@ void doAsyncQuery(SRequestObj* pRequest, bool updateMetaForce) {
   }
 
   _error:
-  tscError("0x%"PRIx64" error happens, code:%s, reqId:0x%"PRIx64, pRequest->self, tstrerror(code), pRequest->requestId);
+  tscError("0x%"PRIx64" error happens, code:%d - %s, reqId:0x%"PRIx64, pRequest->self, code, tstrerror(code), pRequest->requestId);
   terrno = code;
   pRequest->code = code;
   pRequest->body.queryFp(pRequest->body.param, pRequest, code);
@@ -763,11 +775,11 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
   }
 
   if (pRequest->code != TSDB_CODE_SUCCESS) {
-    pRequest->code = code;
     pRequest->body.fetchFp(pRequest->body.param, pRequest, 0);
+    return;
   }
 
-  pRequest->code = setQueryResultFromRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pResultInfo->pData, true, false);
+  pRequest->code = setQueryResultFromRsp(pResultInfo, (SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4, false);
   if (pRequest->code != TSDB_CODE_SUCCESS) {
     pResultInfo->numOfRows = 0;
     pRequest->code = code;
@@ -807,6 +819,13 @@ void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
   schedulerAsyncFetchRows(pRequest->body.queryJob, fetchCallback, pRequest);
 }
 
+void taos_fetch_raw_block_a(TAOS_RES* res, __taos_async_fn_t fp, void* param) {
+  ASSERT(res != NULL && fp != NULL);
+  SRequestObj *pRequest = res;
+  pRequest->body.resInfo.convertUcs4 = false;
+  taos_fetch_rows_a(res, fp, param);
+}
+
 TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char *topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp,
                          void *param, int interval) {
   // TODO

source/client/src/tmq.c

@@ -323,7 +323,7 @@ static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) {
 
 int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) {
   SMqCommitCbParam* pParam = (SMqCommitCbParam*)param;
-  pParam->rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL;
+  pParam->rspErr = code;
   if (pParam->async) {
     if (pParam->automatic && pParam->tmq->commitCb) {
       pParam->tmq->commitCb(pParam->tmq, pParam->rspErr, (tmq_topic_vgroup_list_t*)pParam->offsets,
@@ -432,12 +432,13 @@ int32_t tmqCommitInner(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int8_
     code = pParam->rspErr;
     tsem_destroy(&pParam->rspSem);
     taosMemoryFree(pParam);
+  } else {
+    code = 0;
   }
 
   // avoid double free if msg is sent
   buf = NULL;
 
-  code = 0;
 END:
   if (buf) taosMemoryFree(buf);
   /*if (pParam) taosMemoryFree(pParam);*/
@@ -445,9 +446,9 @@ END:
 
   if (code != 0 && async) {
     if (automatic) {
-      tmq->commitCb(tmq, TMQ_RESP_ERR__FAIL, (tmq_topic_vgroup_list_t*)pOffsets, tmq->commitCbUserParam);
+      tmq->commitCb(tmq, code, (tmq_topic_vgroup_list_t*)pOffsets, tmq->commitCbUserParam);
     } else {
-      userCb(tmq, TMQ_RESP_ERR__FAIL, (tmq_topic_vgroup_list_t*)pOffsets, userParam);
+      userCb(tmq, code, (tmq_topic_vgroup_list_t*)pOffsets, userParam);
     }
   }
 
@@ -493,7 +494,6 @@ int32_t tmqHandleAllDelayedTask(tmq_t* tmq) {
       tmqAskEp(tmq, true);
       taosTmrReset(tmqAssignDelayedHbTask, 1000, tmq, tmqMgmt.timer, &tmq->hbTimer);
     } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) {
-      /*tmq_commit(tmq, NULL, true);*/
       tmqCommitInner(tmq, NULL, 1, 1, tmq->commitCb, tmq->commitCbUserParam);
       taosTmrReset(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, tmq, tmqMgmt.timer, &tmq->commitTimer);
     } else if (*pTaskType == TMQ_DELAYED_TASK__REPORT) {
@@ -666,94 +666,6 @@ FAIL:
 
 tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async) {
   return tmqCommitInner(tmq, offsets, 0, async, tmq->commitCb, tmq->commitCbUserParam);
-#if 0
-  // TODO: add read write lock
-  SRequestObj*   pRequest = NULL;
-  tmq_resp_err_t resp = TMQ_RESP_ERR__SUCCESS;
-  // build msg
-  // send to mnode
-  SMqCMCommitOffsetReq req;
-  SArray*              pOffsets = NULL;
-
-  if (offsets == NULL) {
-    pOffsets = taosArrayInit(0, sizeof(SMqOffset));
-    for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
-      SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
-      for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
-        SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
-        SMqOffset    offset;
-        strcpy(offset.topicName, pTopic->topicName);
-        strcpy(offset.cgroup, tmq->groupId);
-        offset.vgId = pVg->vgId;
-        offset.offset = pVg->currentOffset;
-        taosArrayPush(pOffsets, &offset);
-      }
-    }
-    req.num = pOffsets->size;
-    req.offsets = pOffsets->pData;
-  } else {
-    req.num = taosArrayGetSize(&offsets->container);
-    req.offsets = (SMqOffset*)offsets->container.pData;
-  }
-
-  SEncoder encoder;
-
-  tEncoderInit(&encoder, NULL, 0);
-  tEncodeSMqCMCommitOffsetReq(&encoder, &req);
-  int32_t tlen = encoder.pos;
-  void*   buf = taosMemoryMalloc(tlen);
-  if (buf == NULL) {
-    tEncoderClear(&encoder);
-    return -1;
-  }
-  tEncoderClear(&encoder);
-
-  tEncoderInit(&encoder, buf, tlen);
-  tEncodeSMqCMCommitOffsetReq(&encoder, &req);
-  tEncoderClear(&encoder);
-
-  pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_MQ_COMMIT_OFFSET);
-  if (pRequest == NULL) {
-    tscError("failed to malloc request");
-  }
-
-  SMqCommitCbParam* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam));
-  if (pParam == NULL) {
-    return -1;
-  }
-  pParam->tmq = tmq;
-  tsem_init(&pParam->rspSem, 0, 0);
-  pParam->async = async;
-  pParam->offsets = pOffsets;
-
-  pRequest->body.requestMsg = (SDataBuf){
-      .pData = buf,
-      .len = tlen,
-      .handle = NULL,
-  };
-
-  SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
-  sendInfo->requestObjRefId = 0;
-  sendInfo->param = pParam;
-  sendInfo->fp = tmqCommitCb;
-  SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
-
-  int64_t transporterId = 0;
-  asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
-
-  if (!async) {
-    tsem_wait(&pParam->rspSem);
-    resp = pParam->rspErr;
-    tsem_destroy(&pParam->rspSem);
-    taosMemoryFree(pParam);
-
-    if (pOffsets) {
-      taosArrayDestroy(pOffsets);
-    }
-  }
-
-  return resp;
-#endif
 }
 
 tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
@@ -858,93 +770,6 @@ void tmq_conf_set_auto_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* para
   conf->commitCbUserParam = param;
 }
 
-#if 0
-TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbName, const char* sql) {
-  STscObj*     pTscObj = (STscObj*)taos;
-  SRequestObj* pRequest = NULL;
-  SQuery*      pQueryNode = NULL;
-  char*        astStr = NULL;
-  int32_t      sqlLen;
-
-  terrno = TSDB_CODE_SUCCESS;
-  if (taos == NULL || streamName == NULL || sql == NULL) {
-    tscError("invalid parameters for creating stream, connObj:%p, stream name:%s, sql:%s", taos, streamName, sql);
-    terrno = TSDB_CODE_TSC_INVALID_INPUT;
-    goto _return;
-  }
-  sqlLen = strlen(sql);
-
-  if (strlen(tbName) >= TSDB_TABLE_NAME_LEN) {
-    tscError("output tb name too long, max length:%d", TSDB_TABLE_NAME_LEN - 1);
-    terrno = TSDB_CODE_TSC_INVALID_INPUT;
-    goto _return;
-  }
-
-  if (sqlLen > TSDB_MAX_ALLOWED_SQL_LEN) {
-    tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN);
-    terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT;
-    goto _return;
-  }
-
-  tscDebug("start to create stream: %s", streamName);
-
-  int32_t code = 0;
-  CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
-  CHECK_CODE_GOTO(parseSql(pRequest, false, &pQueryNode, NULL), _return);
-  CHECK_CODE_GOTO(nodesNodeToString(pQueryNode->pRoot, false, &astStr, NULL), _return);
-
-  /*printf("%s\n", pStr);*/
-
-  SName name = {.acctId = pTscObj->acctId, .type = TSDB_TABLE_NAME_T};
-  strcpy(name.dbname, pRequest->pDb);
-  strcpy(name.tname, streamName);
-
-  SCMCreateStreamReq req = {
-      .igExists = 1,
-      .ast = astStr,
-      .sql = (char*)sql,
-  };
-  tNameExtractFullName(&name, req.name);
-  strcpy(req.targetStbFullName, tbName);
-
-  int   tlen = tSerializeSCMCreateStreamReq(NULL, 0, &req);
-  void* buf = taosMemoryMalloc(tlen);
-  if (buf == NULL) {
-    goto _return;
-  }
-
-  tSerializeSCMCreateStreamReq(buf, tlen, &req);
-
-  pRequest->body.requestMsg = (SDataBuf){
-      .pData = buf,
-      .len = tlen,
-      .handle = NULL,
-  };
-  pRequest->type = TDMT_MND_CREATE_STREAM;
-
-  SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
-  SEpSet        epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
-
-  int64_t transporterId = 0;
-  asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
-
-  tsem_wait(&pRequest->body.rspSem);
-
-_return:
-  taosMemoryFreeClear(astStr);
-  qDestroyQuery(pQueryNode);
-  /*if (sendInfo != NULL) {*/
-  /*destroySendMsgInfo(sendInfo);*/
-  /*}*/
-
-  if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) {
-    pRequest->code = terrno;
-  }
-
-  return pRequest;
-}
-#endif
-
 #if 0
 int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) {
   if (tmq_message == NULL) return 0;
@@ -1474,16 +1299,16 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
 tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) {
   if (tmq->status == TMQ_CONSUMER_STATUS__READY) {
     tmq_resp_err_t rsp = tmq_commit_sync(tmq, NULL);
-    if (rsp == TMQ_RESP_ERR__FAIL) {
+    if (rsp != TMQ_RESP_ERR__SUCCESS) {
-      return TMQ_RESP_ERR__FAIL;
+      return rsp;
     }
 
     tmq_list_t* lst = tmq_list_new();
     rsp = tmq_subscribe(tmq, lst);
     tmq_list_destroy(lst);
 
-    if (rsp == TMQ_RESP_ERR__FAIL) {
+    if (rsp != TMQ_RESP_ERR__SUCCESS) {
-      return TMQ_RESP_ERR__FAIL;
+      return rsp;
     }
   }
   // TODO: free resources
@@ -1493,8 +1318,11 @@ tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) {
 const char* tmq_err2str(tmq_resp_err_t err) {
   if (err == TMQ_RESP_ERR__SUCCESS) {
     return "success";
-  }
+  } else if (err == TMQ_RESP_ERR__FAIL) {
     return "fail";
+  } else {
+    return tstrerror(err);
+  }
 }
 
 const char* tmq_get_topic_name(TAOS_RES* res) {
@@ -1536,10 +1364,11 @@ const char* tmq_get_table_name(TAOS_RES* res) {
   }
   return NULL;
 }
-DLL_EXPORT void tmq_commit_async(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, tmq_commit_cb* cb, void* param) {
+
+void tmq_commit_async(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, tmq_commit_cb* cb, void* param) {
   tmqCommitInner(tmq, offsets, 0, 1, cb, param);
 }
 
-DLL_EXPORT tmq_resp_err_t tmq_commit_sync(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets) {
+tmq_resp_err_t tmq_commit_sync(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets) {
   return tmqCommitInner(tmq, offsets, 0, 0, NULL, NULL);
 }

source/client/test/clientTests.cpp

@@ -780,31 +780,28 @@ TEST(testCase, async_api_test) {
 
   taos_query(pConn, "use test");
 
-  TAOS_RES* pRes = taos_query(pConn, "select * from t1");
+  TAOS_RES* pRes = taos_query(pConn, "desc abc1.tu");
-
-  taos_query(pConn, "alter table t1 add column b int");
-  pRes = taos_query(pConn, "insert into t1 values(now, 1, 2)");
   if (taos_errno(pRes) != 0) {
     printf("failed, reason:%s\n", taos_errstr(pRes));
   }
 
-//  int32_t n = 0;
+  int32_t n = 0;
-//  TAOS_ROW    pRow = NULL;
+  TAOS_ROW    pRow = NULL;
-//  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
+  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
-//  int32_t     numOfFields = taos_num_fields(pRes);
+  int32_t     numOfFields = taos_num_fields(pRes);
-//
+
-//  char str[512] = {0};
+  char str[512] = {0};
-//  while ((pRow = taos_fetch_row(pRes)) != NULL) {
+  while ((pRow = taos_fetch_row(pRes)) != NULL) {
-//    int32_t* length = taos_fetch_lengths(pRes);
+    int32_t* length = taos_fetch_lengths(pRes);
-//    for(int32_t i = 0; i < numOfFields; ++i) {
+    for(int32_t i = 0; i < numOfFields; ++i) {
-//      printf("(%d):%d " , i, length[i]);
+      printf("(%d):%d " , i, length[i]);
-//    }
+    }
-//    printf("\n");
+    printf("\n");
-//
+
-//    int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
+    int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
-//    printf("%s\n", str);
+    printf("%s\n", str);
-//    memset(str, 0, sizeof(str));
+    memset(str, 0, sizeof(str));
-//  }
+  }
 
   taos_query_a(pConn, "alter table test.m1 comment 'abcde' ", queryCallback, pConn);
   getchar();

source/common/src/tdatablock.c

@@ -682,9 +682,10 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) {
  * @param pBlock
  * @return
  */
-size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock) {
+size_t blockDataGetSerialMetaSize(uint32_t numOfCols) {
-  // | total rows/total length | block group id | each column length |
+  // | total rows/total length | block group id | column schema | each column length |
-  return sizeof(int32_t) + sizeof(uint64_t) + pBlock->info.numOfCols * sizeof(int32_t);
+  return sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)) +
+         numOfCols * sizeof(int32_t);
 }
 
 double blockDataGetSerialRowSize(const SSDataBlock* pBlock) {
@@ -1217,6 +1218,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
   pBlock->info.numOfCols = numOfCols;
   pBlock->info.hasVarCol = pDataBlock->info.hasVarCol;
   pBlock->info.rowSize = pDataBlock->info.rowSize;
+  pBlock->info.groupId = pDataBlock->info.groupId;
 
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData  colInfo = {0};
@@ -1246,7 +1248,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
 }
 
 size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) {
-  int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(pBlock);
+  int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(pBlock->info.numOfCols);
 
   int32_t rowSize = pBlock->info.rowSize;
 
@@ -1885,33 +1887,43 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
 
 void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols,
                          int8_t needCompress) {
+  // todo extract method
   int32_t* actualLen = (int32_t*)data;
   data += sizeof(int32_t);
 
   uint64_t* groupId = (uint64_t*)data;
   data += sizeof(uint64_t);
 
+  for (int32_t i = 0; i < numOfCols; ++i) {
+    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
+    *((int16_t*)data) = pColInfoData->info.type;
+    data += sizeof(int16_t);
+
+    *((int32_t*)data) = pColInfoData->info.bytes;
+    data += sizeof(int32_t);
+  }
+
   int32_t* colSizes = (int32_t*)data;
   data += numOfCols * sizeof(int32_t);
 
-  *dataLen = (numOfCols * sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t));
+  *dataLen = blockDataGetSerialMetaSize(numOfCols);
 
   int32_t numOfRows = pBlock->info.rows;
   for (int32_t col = 0; col < numOfCols; ++col) {
     SColumnInfoData* pColRes = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, col);
 
     // copy the null bitmap
+    size_t metaSize = 0;
     if (IS_VAR_DATA_TYPE(pColRes->info.type)) {
-      size_t metaSize = numOfRows * sizeof(int32_t);
+      metaSize = numOfRows * sizeof(int32_t);
       memcpy(data, pColRes->varmeta.offset, metaSize);
+    } else {
+      metaSize = BitmapLen(numOfRows);
+      memcpy(data, pColRes->nullbitmap, metaSize);
+    }
+
     data += metaSize;
     (*dataLen) += metaSize;
-    } else {
-      int32_t len = BitmapLen(numOfRows);
-      memcpy(data, pColRes->nullbitmap, len);
-      data += len;
-      (*dataLen) += len;
-    }
 
     if (needCompress) {
       colSizes[col] = blockCompressColData(pColRes, numOfRows, data, needCompress);
@@ -1933,6 +1945,8 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen
 
 const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) {
   blockDataEnsureCapacity(pBlock, numOfRows);
+  pBlock->info.rows = numOfRows;
+
   const char* pStart = pData;
 
   int32_t dataLen = *(int32_t*)pStart;
@@ -1941,6 +1955,29 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t
   pBlock->info.groupId = *(uint64_t*)pStart;
   pStart += sizeof(uint64_t);
 
+  if (pBlock->pDataBlock == NULL) {
+    pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
+    taosArraySetSize(pBlock->pDataBlock, numOfCols);
+  }
+
+  pBlock->info.numOfCols = taosArrayGetSize(pBlock->pDataBlock);
+  ASSERT(pBlock->pDataBlock->size >= numOfCols);
+
+  for (int32_t i = 0; i < numOfCols; ++i) {
+    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
+    pColInfoData->info.type = *(int16_t*)pStart;
+    pStart += sizeof(int16_t);
+
+    pColInfoData->info.bytes = *(int32_t*)pStart;
+    pStart += sizeof(int32_t);
+
+    if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
+      pBlock->info.hasVarCol = true;
+    }
+  }
+
+  blockDataEnsureCapacity(pBlock, numOfRows);
+
   int32_t* colLen = (int32_t*)pStart;
   pStart += sizeof(int32_t) * numOfCols;
 
@@ -1961,11 +1998,17 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t
         pColInfoData->pData = taosMemoryMalloc(colLen[i]);
       }
     } else {
+      if (pColInfoData->nullbitmap == NULL) {
+        pColInfoData->nullbitmap = taosMemoryCalloc(1, BitmapLen(numOfRows));
+      }
       memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows));
       pStart += BitmapLen(numOfRows);
     }
 
     if (colLen[i] > 0) {
+      if (pColInfoData->pData == NULL) {
+        pColInfoData->pData = taosMemoryCalloc(1, colLen[i]);
+      }
       memcpy(pColInfoData->pData, pStart, colLen[i]);
     }
 

source/common/src/tdataformat.c

@@ -208,7 +208,7 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S
     if (iColumn == 0) {
       ASSERT(pColVal->cid == pTColumn->colId);
       ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP);
-      ASSERT(pTColumn->colId == 0);
+      ASSERT(pTColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID);
 
       iColVal++;
     } else {
@@ -244,7 +244,7 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S
     }
   }
 
-  ASSERT(flags);
+  // ASSERT(flags); // only 1 column(ts)
 
   // decide
   uint32_t nData = 0;
@@ -268,7 +268,8 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S
         nDataT = BIT2_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntv;
         break;
       default:
-        ASSERT(0);
+        break; // only ts column
+        // ASSERT(0);
     }
 
     uint8_t tflags = 0;
@@ -283,7 +284,7 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S
       tflags |= TSROW_KV_BIG;
     }
 
-    if (nDataT < nDataK) {
+    if (nDataT <= nDataK) {
       nData = nDataT;
     } else {
       nData = nDataK;
@@ -352,7 +353,7 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S
     ntv = 0;
     iColVal = 1;
 
-    if (flags & 0xf0 == 0) {
+    if ((flags & 0xf0) == 0) {
       switch (flags & 0xf) {
         case TSROW_HAS_VAL:
           pf = (*ppRow)->pData;
@@ -373,7 +374,8 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S
           ptv = pf + pTSchema->flen;
           break;
         default:
-          ASSERT(0);
+          // ASSERT(0);
+          break;
       }
     } else {
       pTSKVRow = (STSKVRow *)(*ppRow)->pData;
@@ -417,13 +419,13 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S
       }
 
     _set_none:
-      if (flags & 0xf0 == 0) {
+      if ((flags & 0xf0) == 0) {
         setBitMap(pb, 0, iColumn - 1, flags);
       }
       continue;
 
     _set_null:
-      if (flags & 0xf0 == 0) {
+      if ((flags & 0xf0) == 0) {
         setBitMap(pb, 1, iColumn - 1, flags);
       } else {
         SET_IDX(pidx, pTSKVRow->nCols, nkv, flags);
@@ -433,7 +435,7 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S
       continue;
 
     _set_value:
-      if (flags & 0xf0 == 0) {
+      if ((flags & 0xf0) == 0) {
         setBitMap(pb, 2, iColumn - 1, flags);
 
         if (IS_VAR_DATA_TYPE(pTColumn->type)) {
@@ -489,13 +491,13 @@ void tTSRowFree(STSRow2 *pRow) {
 }
 
 void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) {
-  uint8_t   isTuple = (pRow->flags & 0xf0 == 0) ? 1 : 0;
+  uint8_t   isTuple = ((pRow->flags & 0xf0) == 0) ? 1 : 0;
   STColumn *pTColumn = &pTSchema->columns[iCol];
   uint8_t   flags = pRow->flags & (uint8_t)0xf;
   SValue    value;
 
   ASSERT(iCol < pTSchema->numOfCols);
-  ASSERT(flags);
+  // ASSERT(flags); // only 1 ts column
   ASSERT(pRow->sver == pTSchema->version);
 
   if (iCol == 0) {
@@ -505,7 +507,7 @@ void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal
 
   if (flags == TSROW_HAS_NONE) {
     goto _return_none;
-  } else if (flags == TSROW_HAS_NONE) {
+  } else if (flags == TSROW_HAS_NULL) {
     goto _return_null;
   }
 

source/common/src/tmsg.c

@@ -2427,6 +2427,7 @@ int32_t tSerializeSTableIndexInfo(SEncoder *pEncoder, STableIndexInfo* pInfo) {
   if (tEncodeI64(pEncoder, pInfo->sliding) < 0) return -1;
   if (tEncodeI64(pEncoder, pInfo->dstTbUid) < 0) return -1;
   if (tEncodeI32(pEncoder, pInfo->dstVgId) < 0) return -1;
+  if (tEncodeSEpSet(pEncoder, &pInfo->epSet) < 0) return -1;
   if (tEncodeCStr(pEncoder, pInfo->expr) < 0) return -1;
   return 0;
 }
@@ -2459,6 +2460,7 @@ int32_t tDeserializeSTableIndexInfo(SDecoder *pDecoder, STableIndexInfo *pInfo)
   if (tDecodeI64(pDecoder, &pInfo->sliding) < 0) return -1;
   if (tDecodeI64(pDecoder, &pInfo->dstTbUid) < 0) return -1;
   if (tDecodeI32(pDecoder, &pInfo->dstVgId) < 0) return -1;
+  if (tDecodeSEpSet(pDecoder, &pInfo->epSet) < 0) return -1;
   if (tDecodeCStrAlloc(pDecoder, &pInfo->expr) < 0) return -1;
 
   return 0;

source/common/test/CMakeLists.txt

@@ -5,7 +5,12 @@ MESSAGE(STATUS "build parser unit test")
 SET(CMAKE_CXX_STANDARD 11)
 AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
-ADD_EXECUTABLE(commonTest ${SOURCE_LIST})
+ADD_EXECUTABLE(commonTest "")
+TARGET_SOURCES(
+    commonTest
+    PRIVATE
+    "commonTests.cpp"
+)
 TARGET_LINK_LIBRARIES(
         commonTest
         PUBLIC os util common gtest
@@ -24,7 +29,12 @@ target_sources(
     PRIVATE
     "dataformatTest.cpp"
 )
-target_link_libraries(dataformatTest gtest gtest_main util)
+target_link_libraries(dataformatTest gtest gtest_main util common)
+target_include_directories(
+        dataformatTest
+        PUBLIC "${TD_SOURCE_DIR}/include/common"
+        PUBLIC "${TD_SOURCE_DIR}/include/util"
+)
 
 # tmsg test
 # add_executable(tmsgTest "")

source/common/test/dataformatTest.cpp

@@ -1 +1,481 @@
-#include "gtest/gtest.h"
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <gtest/gtest.h>
+
+#include <taoserror.h>
+#include <tdataformat.h>
+#include <tglobal.h>
+#include <tmsg.h>
+#include <iostream>
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wwrite-strings"
+#pragma GCC diagnostic ignored "-Wunused-function"
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#pragma GCC diagnostic ignored "-Wsign-compare"
+
+#define NONE_CSTR "no"
+#define NULL_CSTR "nu"
+#define NONE_LEN  2
+#define NULL_LEN  2
+const static int16_t MAX_COLS = 14;
+
+int main(int argc, char **argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+// ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint, c9 bool
+STSchema *genSTSchema(int16_t nCols) {
+  EXPECT_LE(nCols, MAX_COLS);
+  SSchema *pSchema = (SSchema *)taosMemoryCalloc(nCols, sizeof(SSchema));
+  EXPECT_NE(pSchema, nullptr);
+
+  for (int16_t i = 0; i < nCols; ++i) {
+    pSchema[i].colId = PRIMARYKEY_TIMESTAMP_COL_ID + i;
+    char colName[TSDB_COL_NAME_LEN] = {0};
+    snprintf(colName, TSDB_COL_NAME_LEN, "c%" PRIi16, i);
+    strncpy(pSchema[i].name, colName, TSDB_COL_NAME_LEN);
+
+    switch (i) {
+      case 0: {
+        pSchema[0].type = TSDB_DATA_TYPE_TIMESTAMP;
+        pSchema[0].bytes = TYPE_BYTES[pSchema[0].type];
+      } break;
+      case 1: {
+        pSchema[1].type = TSDB_DATA_TYPE_INT;
+        pSchema[1].bytes = TYPE_BYTES[pSchema[1].type];
+        ;
+      } break;
+      case 2: {
+        pSchema[2].type = TSDB_DATA_TYPE_BIGINT;
+        pSchema[2].bytes = TYPE_BYTES[pSchema[2].type];
+      } break;
+      case 3: {
+        pSchema[3].type = TSDB_DATA_TYPE_FLOAT;
+        pSchema[3].bytes = TYPE_BYTES[pSchema[3].type];
+      } break;
+      case 4: {
+        pSchema[4].type = TSDB_DATA_TYPE_DOUBLE;
+        pSchema[4].bytes = TYPE_BYTES[pSchema[4].type];
+      } break;
+      case 5: {
+        pSchema[5].type = TSDB_DATA_TYPE_BINARY;
+        pSchema[5].bytes = 12;
+      } break;
+      case 6: {
+        pSchema[6].type = TSDB_DATA_TYPE_NCHAR;
+        pSchema[6].bytes = 42;
+      } break;
+      case 7: {
+        pSchema[7].type = TSDB_DATA_TYPE_TINYINT;
+        pSchema[7].bytes = TYPE_BYTES[pSchema[7].type];
+      } break;
+      case 8: {
+        pSchema[8].type = TSDB_DATA_TYPE_SMALLINT;
+        pSchema[8].bytes = TYPE_BYTES[pSchema[8].type];
+      } break;
+      case 9: {
+        pSchema[9].type = TSDB_DATA_TYPE_BOOL;
+        pSchema[9].bytes = TYPE_BYTES[pSchema[9].type];
+      } break;
+      case 10: {
+        pSchema[10].type = TSDB_DATA_TYPE_UTINYINT;
+        pSchema[10].bytes = TYPE_BYTES[pSchema[10].type];
+      } break;
+      case 11: {
+        pSchema[11].type = TSDB_DATA_TYPE_USMALLINT;
+        pSchema[11].bytes = TYPE_BYTES[pSchema[11].type];
+      } break;
+      case 12: {
+        pSchema[12].type = TSDB_DATA_TYPE_UINT;
+        pSchema[12].bytes = TYPE_BYTES[pSchema[12].type];
+      } break;
+      case 13: {
+        pSchema[13].type = TSDB_DATA_TYPE_UBIGINT;
+        pSchema[13].bytes = TYPE_BYTES[pSchema[13].type];
+      } break;
+
+      default:
+        ASSERT(0);
+        break;
+    }
+  }
+
+  STSchema *pResult = NULL;
+  pResult = tdGetSTSChemaFromSSChema(&pSchema, nCols);
+
+  taosMemoryFree(pSchema);
+  return pResult;
+}
+
+// ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint, c9 bool
+static int32_t genTestData(const char **data, int16_t nCols, SArray **pArray) {
+  if (!(*pArray)) {
+    *pArray = taosArrayInit(nCols, sizeof(SColVal));
+    if (!(*pArray)) return -1;
+  }
+
+  for (int16_t i = 0; i < nCols; ++i) {
+    SColVal colVal = {0};
+    colVal.cid = PRIMARYKEY_TIMESTAMP_COL_ID + i;
+    if (strncasecmp(data[i], NONE_CSTR, NONE_LEN) == 0) {
+      colVal.isNone = 1;
+      taosArrayPush(*pArray, &colVal);
+      continue;
+    } else if (strncasecmp(data[i], NULL_CSTR, NULL_LEN) == 0) {
+      colVal.isNull = 1;
+      taosArrayPush(*pArray, &colVal);
+      continue;
+    }
+
+    switch (i) {
+      case 0:
+        sscanf(data[i], "%" PRIi64, &colVal.value.ts);
+        break;
+      case 1: {
+        sscanf(data[i], "%" PRIi32, &colVal.value.i32);
+      } break;
+      case 2:
+        sscanf(data[i], "%" PRIi64, &colVal.value.i64);
+        break;
+      case 3:
+        sscanf(data[i], "%f", &colVal.value.f);
+        break;
+      case 4:
+        sscanf(data[i], "%lf", &colVal.value.d);
+        break;
+      case 5: {
+        int16_t dataLen = strlen(data[i]) + 1;
+        colVal.value.nData = dataLen < 10 ? dataLen : 10;
+        colVal.value.pData = (uint8_t *)data[i];
+      } break;
+      case 6: {
+        int16_t dataLen = strlen(data[i]) + 1;
+        colVal.value.nData = dataLen < 40 ? dataLen : 40;
+        colVal.value.pData = (uint8_t *)data[i];  // just for test, not real nchar
+      } break;
+      case 7:
+      case 9: {
+        int32_t d8;
+        sscanf(data[i], "%" PRId32, &d8);
+        colVal.value.i8 = (int8_t)d8;
+      } break;
+      case 8: {
+        int32_t d16;
+        sscanf(data[i], "%" PRId32, &d16);
+        colVal.value.i16 = (int16_t)d16;
+      } break;
+      case 10: {
+        uint32_t u8;
+        sscanf(data[i], "%" PRId32, &u8);
+        colVal.value.u8 = (uint8_t)u8;
+      } break;
+      case 11: {
+        uint32_t u16;
+        sscanf(data[i], "%" PRId32, &u16);
+        colVal.value.u16 = (uint16_t)u16;
+      } break;
+      case 12: {
+        sscanf(data[i], "%" PRIu32, &colVal.value.u32);
+      } break;
+      case 13: {
+        sscanf(data[i], "%" PRIu64, &colVal.value.u64);
+      } break;
+      default:
+        ASSERT(0);
+    }
+    taosArrayPush(*pArray, &colVal);
+  }
+  return 0;
+}
+
+int32_t debugPrintSColVal(SColVal *cv, int8_t type) {
+  if (cv->isNone) {
+    printf("None ");
+    return 0;
+  }
+  if (cv->isNull) {
+    printf("Null ");
+    return 0;
+  }
+  switch (type) {
+    case TSDB_DATA_TYPE_BOOL:
+      printf("%s ", cv->value.i8 == 0 ? "false" : "true");
+      break;
+    case TSDB_DATA_TYPE_TINYINT:
+      printf("%" PRIi8 " ", cv->value.i8);
+      break;
+    case TSDB_DATA_TYPE_SMALLINT:
+      printf("%" PRIi16 " ", cv->value.i16);
+      break;
+    case TSDB_DATA_TYPE_INT:
+      printf("%" PRIi32 " ", cv->value.i32);
+      break;
+    case TSDB_DATA_TYPE_BIGINT:
+      printf("%" PRIi64 " ", cv->value.i64);
+      break;
+    case TSDB_DATA_TYPE_FLOAT:
+      printf("%f ", cv->value.f);
+      break;
+    case TSDB_DATA_TYPE_DOUBLE:
+      printf("%lf ", cv->value.d);
+      break;
+    case TSDB_DATA_TYPE_VARCHAR: {
+      char tv[15] = {0};
+      snprintf(tv, 15, "%s", cv->value.pData);
+      printf("%s ", tv);
+    } break;
+    case TSDB_DATA_TYPE_TIMESTAMP:
+      printf("%" PRIi64 " ", cv->value.i64);
+      break;
+    case TSDB_DATA_TYPE_NCHAR: {
+      char tv[15] = {0};
+      snprintf(tv, 15, "%s", cv->value.pData);
+      printf("%s ", tv);
+    } break;
+    case TSDB_DATA_TYPE_UTINYINT:
+      printf("%" PRIu8 " ", cv->value.u8);
+      break;
+    case TSDB_DATA_TYPE_USMALLINT:
+      printf("%" PRIu16 " ", cv->value.u16);
+      break;
+    case TSDB_DATA_TYPE_UINT:
+      printf("%" PRIu32 " ", cv->value.u32);
+      break;
+    case TSDB_DATA_TYPE_UBIGINT:
+      printf("%" PRIu64 " ", cv->value.u64);
+      break;
+    case TSDB_DATA_TYPE_JSON:
+      printf("JSON ");
+      break;
+    case TSDB_DATA_TYPE_VARBINARY:
+      printf("VARBIN ");
+      break;
+    case TSDB_DATA_TYPE_DECIMAL:
+      printf("DECIMAL ");
+      break;
+    case TSDB_DATA_TYPE_BLOB:
+      printf("BLOB ");
+      break;
+    case TSDB_DATA_TYPE_MEDIUMBLOB:
+      printf("MedBLOB ");
+      break;
+    // case TSDB_DATA_TYPE_BINARY:
+    //   printf("BINARY ");
+    //   break;
+    case TSDB_DATA_TYPE_MAX:
+      printf("UNDEF ");
+      break;
+    default:
+      printf("UNDEF ");
+      break;
+  }
+  return 0;
+}
+
+void debugPrintTSRow(STSRow2 *row, STSchema *pTSchema, const char *tags, int32_t ln) {
+  printf("%s:%d %s:v%d:%d ", tags, ln, (row->flags & 0xf0) ? "KV" : "TP", row->sver, row->nData);
+  for (int16_t i = 0; i < schemaNCols(pTSchema); ++i) {
+    SColVal cv = {0};
+    tTSRowGet(row, pTSchema, i, &cv);
+    debugPrintSColVal(&cv, pTSchema->columns[i].type);
+  }
+  printf("\n");
+  fflush(stdout);
+}
+
+static int32_t checkSColVal(const char *rawVal, SColVal *cv, int8_t type) {
+  ASSERT(rawVal);
+
+  if (cv->isNone) {
+    EXPECT_STRCASEEQ(rawVal, NONE_CSTR);
+    return 0;
+  }
+  if (cv->isNull) {
+    EXPECT_STRCASEEQ(rawVal, NULL_CSTR);
+    return 0;
+  }
+
+  SValue rawSVal = {0};
+  switch (type) {
+    case TSDB_DATA_TYPE_BOOL:
+    case TSDB_DATA_TYPE_TINYINT: {
+      int32_t d8;
+      sscanf(rawVal, "%" PRId32, &d8);
+      EXPECT_EQ(cv->value.i8, (int8_t)d8);
+    } break;
+    case TSDB_DATA_TYPE_SMALLINT: {
+      int32_t d16;
+      sscanf(rawVal, "%" PRId32, &d16);
+      EXPECT_EQ(cv->value.i16, (int16_t)d16);
+    } break;
+    case TSDB_DATA_TYPE_INT: {
+      sscanf(rawVal, "%" PRId32, &rawSVal.i32);
+      EXPECT_EQ(cv->value.i32, rawSVal.i32);
+    } break;
+    case TSDB_DATA_TYPE_BIGINT: {
+      sscanf(rawVal, "%" PRIi64, &rawSVal.i64);
+      EXPECT_EQ(cv->value.i64, rawSVal.i64);
+    } break;
+    case TSDB_DATA_TYPE_FLOAT: {
+      sscanf(rawVal, "%f", &rawSVal.f);
+      EXPECT_FLOAT_EQ(cv->value.f, rawSVal.f);
+    } break;
+    case TSDB_DATA_TYPE_DOUBLE: {
+      sscanf(rawVal, "%lf", &rawSVal.d);
+      EXPECT_DOUBLE_EQ(cv->value.d, rawSVal.d);
+    } break;
+    case TSDB_DATA_TYPE_VARCHAR: {
+      EXPECT_STRCASEEQ(rawVal, (const char *)cv->value.pData);
+    } break;
+    case TSDB_DATA_TYPE_TIMESTAMP: {
+      sscanf(rawVal, "%" PRIi64, &rawSVal.ts);
+      EXPECT_DOUBLE_EQ(cv->value.ts, rawSVal.ts);
+    } break;
+    case TSDB_DATA_TYPE_NCHAR: {
+      EXPECT_STRCASEEQ(rawVal, (const char *)cv->value.pData);  // informal nchar comparsion
+    } break;
+    case TSDB_DATA_TYPE_UTINYINT: {
+      uint32_t u8;
+      sscanf(rawVal, "%" PRIu32, &u8);
+      EXPECT_EQ(cv->value.u8, (uint8_t)u8);
+    } break;
+    case TSDB_DATA_TYPE_USMALLINT: {
+      uint32_t u16;
+      sscanf(rawVal, "%" PRIu32, &u16);
+      EXPECT_EQ(cv->value.u16, (uint16_t)u16);
+    } break;
+    case TSDB_DATA_TYPE_UINT: {
+      sscanf(rawVal, "%" PRIu32, &rawSVal.u32);
+      EXPECT_EQ(cv->value.u32, rawSVal.u32);
+    } break;
+    case TSDB_DATA_TYPE_UBIGINT: {
+      sscanf(rawVal, "%" PRIu64, &rawSVal.u64);
+      EXPECT_EQ(cv->value.u64, rawSVal.u64);
+    } break;
+    case TSDB_DATA_TYPE_JSON:
+      printf("JSON ");
+      break;
+    case TSDB_DATA_TYPE_VARBINARY:
+      printf("VARBIN ");
+      break;
+    case TSDB_DATA_TYPE_DECIMAL:
+      printf("DECIMAL ");
+      break;
+    case TSDB_DATA_TYPE_BLOB:
+      printf("BLOB ");
+      break;
+    case TSDB_DATA_TYPE_MEDIUMBLOB:
+      printf("MedBLOB ");
+      break;
+    // case TSDB_DATA_TYPE_BINARY:
+    //   printf("BINARY ");
+    //   break;
+    case TSDB_DATA_TYPE_MAX:
+      printf("UNDEF ");
+      break;
+    default:
+      printf("UNDEF ");
+      break;
+  }
+  return 0;
+}
+
+static void checkTSRow(const char **data, STSRow2 *row, STSchema *pTSchema) {
+  for (int16_t i = 0; i < schemaNCols(pTSchema); ++i) {
+    SColVal cv = {0};
+    tTSRowGet(row, pTSchema, i, &cv);
+    checkSColVal(data[i], &cv, pTSchema->columns[i].type);
+  }
+}
+
+TEST(testCase, AllNormTest) {
+  int16_t  nCols = 1;
+  STSRow2 *row = nullptr;
+  SArray  *pArray = taosArrayInit(nCols, sizeof(SColVal));
+  EXPECT_NE(pArray, nullptr);
+
+  STSchema *pTSchema = genSTSchema(nCols);
+  EXPECT_NE(pTSchema, nullptr);
+
+  // ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint,
+  // c9 bool
+  char *data[10] = {"1653694220000", "10", "20", "10.1", "10.1", "binary10", "nchar10", "10", "10", "1"};
+
+  genTestData((const char **)&data, nCols, &pArray);
+
+  tTSRowNew(NULL, pArray, pTSchema, &row);
+
+  debugPrintTSRow(row, pTSchema, __func__, __LINE__);
+  checkTSRow((const char **)&data, row, pTSchema);
+
+  taosArrayDestroy(pArray);
+  taosMemoryFree(pTSchema);
+}
+
+#if 1
+TEST(testCase, NoneTest) {
+  const static int nCols = 14;
+  const static int nRows = 20;
+  STSRow2         *row = nullptr;
+  SArray          *pArray = taosArrayInit(nCols, sizeof(SColVal));
+  EXPECT_NE(pArray, nullptr);
+
+  STSchema *pTSchema = genSTSchema(nCols);
+  EXPECT_NE(pTSchema, nullptr);
+
+  // ts timestamp,  c1 int,  c2 bigint,  c3 float,  c4 double,  c5 binary(10),  c6 nchar(10),  c7 tinyint,  c8 smallint,
+  // c9 bool c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned
+  const char *data[nRows][nCols] = {
+      {"1653694220000", "no", "20", "10.1", "10.1", "binary10", "no", "10", "10", "nu", "10", "20", "30", "40"},
+      {"1653694220001", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"},
+      {"1653694220002", "no", "no", "no", "no", "no", "nu", "no", "no", "no", "no", "no", "no", "nu"},
+      {"1653694220003", "nu", "no", "no", "no", "no", "nu", "no", "no", "no", "no", "no", "no", "no"},
+      {"1653694220004", "no", "20", "no", "no", "no", "nchar10", "no", "no", "no", "no", "no", "no", "no"},
+      {"1653694220005", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"},
+      {"1653694220006", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"},
+      {"1653694220007", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no"},
+      {"1653694220008", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no"},
+      {"1653694220009", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"},
+      {"1653694220010", "-1", "-1", "-1", "-1", "binary10", "nu", "-1", "0", "0", "0", "0", "0", "0"},
+      {"1653694220011", "-2147483648", "nu", "nu", "nu", "biy10", "nu", "nu", "32767", "no", "nu", "nu", "nu", "100"},
+      {"1653694220012", "2147483647", "nu", "nu", "nu", "ary10", "nu", "nu", "-32768", "no", "nu", "nu", "nu", "100"},
+      {"1653694220013", "no", "-9223372036854775818", "nu", "nu", "b1", "nu", "nu", "10", "no", "nu", "nu", "nu", "nu"},
+      {"1653694220014", "no", "nu", "nu", "nu", "b0", "nu", "nu", "10", "no", "nu", "nu", "nu", "9223372036854775808"},
+      {"1653694220015", "no", "nu", "nu", "nu", "binary30", "char4", "nu", "10", "no", "nu", "nu", "nu",
+       "18446744073709551615"},
+      {"1653694220016", "2147483647", "nu", "nu", "nu", "bin50", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"},
+      {"1653694220017", "2147483646", "0", "0", "0", "binary10", "0", "0", "0", "0", "255", "0", "0", "0"},
+      {"1653694220018", "no", "-9223372036854775808", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu",
+       "4294967295", "100"},
+      {"1653694220019", "no", "9223372036854775807", "nu", "nu", "bin10", "nu", "nu", "10", "no", "254", "nu", "nu",
+       "no"}};
+
+
+  for (int r = 0; r < nRows; ++r) {
+    genTestData((const char **)&data[r], nCols, &pArray);
+    tTSRowNew(NULL, pArray, pTSchema, &row);
+    debugPrintTSRow(row, pTSchema, __func__, __LINE__);  // debug print
+    checkTSRow((const char **)&data[r], row, pTSchema);  // check
+    taosMemoryFreeClear(row);
+    taosArrayClear(pArray);
+  }
+
+  taosArrayDestroy(pArray);
+  taosMemoryFree(pTSchema);
+}
+#endif

source/dnode/mgmt/mgmt_vnode/inc/vmInt.h

@@ -34,6 +34,7 @@ typedef struct SVnodeMgmt {
   SQWorkerPool   fetchPool;
   SWWorkerPool   syncPool;
   SWWorkerPool   writePool;
+  SWWorkerPool   applyPool;
   SWWorkerPool   mergePool;
   SSingleWorker  mgmtWorker;
   SSingleWorker  monitorWorker;
@@ -52,10 +53,9 @@ typedef struct {
 
 typedef struct {
   int32_t     vgId;
-  int32_t     refCount;
   int32_t     vgVersion;
+  int32_t     refCount;
   int8_t      dropped;
-  int8_t      accessState;
   char       *path;
   SVnode     *pImpl;
   STaosQueue *pWriteQ;

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -323,7 +323,6 @@ SArray *vmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_MON_VM_INFO, vmPutMsgToMonitorQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MON_VM_LOAD, vmPutMsgToMonitorQueue, 0) == NULL) goto _OVER;
 
-  // Requests handled by VNODE
   if (dmSetMgmtHandle(pArray, TDMT_VND_SUBMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_CONTINUE, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER;
@@ -353,7 +352,9 @@ SArray *vmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER_RSP, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
 
   if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

source/dnode/mgmt/mgmt_vnode/src/vmInt.c

@@ -49,10 +49,9 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) {
   }
 
   pVnode->vgId = pCfg->vgId;
-  pVnode->refCount = 0;
   pVnode->vgVersion = pCfg->vgVersion;
+  pVnode->refCount = 0;
   pVnode->dropped = 0;
-  pVnode->accessState = TSDB_VN_ALL_ACCCESS;
   pVnode->path = tstrdup(pCfg->path);
   pVnode->pImpl = pImpl;
 
@@ -96,7 +95,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
   dDebug("vgId:%d, vnode is closed", pVnode->vgId);
 
   if (pVnode->dropped) {
-    dDebug("vgId:%d, vnode is destroyed for dropped:%d", pVnode->vgId, pVnode->dropped);
+    dInfo("vgId:%d, vnode is destroyed, dropped:%d", pVnode->vgId, pVnode->dropped);
     snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pVnode->vgId);
     vnodeDestroy(path, pMgmt->pTfs);
   }

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -16,10 +16,8 @@
 #define _DEFAULT_SOURCE
 #include "vmInt.h"
 
-#include "sync.h"
-#include "syncTools.h"
-
 static inline void vmSendRsp(SRpcMsg *pMsg, int32_t code) {
+  if (pMsg->info.handle == NULL) return;
   SRpcMsg rsp = {
       .code = code,
       .pCont = pMsg->info.rsp,
@@ -53,9 +51,9 @@ static void vmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
   }
 
   if (IsReq(pMsg)) {
-    if (code != 0 && terrno != 0) {
+    if (code != 0) {
-      dError("msg:%p failed to process since %s", pMsg, terrstr());
+      if (terrno != 0) code = terrno;
-      code = terrno;
+      dError("msg:%p, failed to process since %s", pMsg, terrstr());
     }
     vmSendRsp(pMsg, code);
   }
@@ -97,110 +95,6 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
   taosFreeQitem(pMsg);
 }
 
-static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
-  int32_t    code = 0;
-  SRpcMsg   *pMsg = NULL;
-  SVnodeObj *pVnode = pInfo->ahandle;
-  int64_t    sync = vnodeGetSyncHandle(pVnode->pImpl);
-  SArray    *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg **));
-
-  for (int32_t m = 0; m < numOfMsgs; m++) {
-    if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
-    dTrace("vgId:%d, msg:%p get from vnode-write queue", pVnode->vgId, pMsg);
-
-    if (taosArrayPush(pArray, &pMsg) == NULL) {
-      dError("vgId:%d, failed to push msg:%p to vnode-write array", pVnode->vgId, pMsg);
-      vmSendRsp(pMsg, TSDB_CODE_OUT_OF_MEMORY);
-    }
-  }
-
-  for (int32_t m = 0; m < taosArrayGetSize(pArray); m++) {
-    pMsg = *(SRpcMsg **)taosArrayGet(pArray, m);
-    code = vnodePreprocessReq(pVnode->pImpl, pMsg);
-
-    if (code == TSDB_CODE_ACTION_IN_PROGRESS) {
-      dTrace("vgId:%d, msg:%p in progress and no rsp", pVnode->vgId, pMsg);
-      continue;
-    }
-
-    if (pMsg->msgType != TDMT_VND_ALTER_REPLICA) {
-      code = syncPropose(sync, pMsg, false);
-    }
-
-    if (code == TAOS_SYNC_PROPOSE_SUCCESS) {
-      dTrace("vgId:%d, msg:%p is proposed and no rsp", pVnode->vgId, pMsg);
-      continue;
-    } else if (code == TAOS_SYNC_PROPOSE_NOT_LEADER) {
-      SEpSet newEpSet = {0};
-      syncGetEpSet(sync, &newEpSet);
-      SEp *pEp = &newEpSet.eps[newEpSet.inUse];
-      if (pEp->port == tsServerPort && strcmp(pEp->fqdn, tsLocalFqdn) == 0) {
-        newEpSet.inUse = (newEpSet.inUse + 1) % newEpSet.numOfEps;
-      }
-
-      dTrace("vgId:%d, msg:%p is redirect since not leader, numOfEps:%d inUse:%d", pVnode->vgId, pMsg,
-             newEpSet.numOfEps, newEpSet.inUse);
-      for (int32_t i = 0; i < newEpSet.numOfEps; ++i) {
-        dTrace("vgId:%d, msg:%p ep:%s:%u", pVnode->vgId, pMsg, newEpSet.eps[i].fqdn, newEpSet.eps[i].port);
-      }
-
-      SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info};
-      tmsgSendRedirectRsp(&rsp, &newEpSet);
-    } else {
-      dError("vgId:%d, msg:%p failed to propose write since %s, code:0x%x", pVnode->vgId, pMsg, tstrerror(code), code);
-      vmSendRsp(pMsg, code);
-    }
-  }
-
-  for (int32_t i = 0; i < numOfMsgs; i++) {
-    pMsg = *(SRpcMsg **)taosArrayGet(pArray, i);
-    dTrace("vgId:%d, msg:%p is freed", pVnode->vgId, pMsg);
-    rpcFreeCont(pMsg->pCont);
-    taosFreeQitem(pMsg);
-  }
-
-  taosArrayDestroy(pArray);
-}
-
-static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
-  SVnodeObj *pVnode = pInfo->ahandle;
-  SRpcMsg   *pMsg = NULL;
-
-  for (int32_t i = 0; i < numOfMsgs; ++i) {
-    if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
-    dTrace("vgId:%d, msg:%p get from vnode-apply queue", pVnode->vgId, pMsg);
-
-    // init response rpc msg
-    SRpcMsg rsp = {0};
-
-    // get original rpc msg
-    assert(pMsg->msgType == TDMT_SYNC_APPLY_MSG);
-    SyncApplyMsg *pSyncApplyMsg = syncApplyMsgFromRpcMsg2(pMsg);
-    syncApplyMsgLog2("==vmProcessApplyQueue==", pSyncApplyMsg);
-    SRpcMsg originalRpcMsg;
-    syncApplyMsg2OriginalRpcMsg(pSyncApplyMsg, &originalRpcMsg);
-
-    // apply data into tsdb
-    if (vnodeProcessWriteReq(pVnode->pImpl, &originalRpcMsg, pSyncApplyMsg->fsmMeta.index, &rsp) < 0) {
-      rsp.code = terrno;
-      dError("vgId:%d, msg:%p failed to apply since %s", pVnode->vgId, pMsg, terrstr());
-    }
-
-    syncApplyMsgDestroy(pSyncApplyMsg);
-    rpcFreeCont(originalRpcMsg.pCont);
-
-    // if leader, send response
-    if (pMsg->info.handle != NULL) {
-      rsp.info = pMsg->info;
-      tmsgSendRsp(&rsp);
-    }
-
-    dTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, rsp.code);
-    rpcFreeCont(pMsg->pCont);
-    taosFreeQitem(pMsg);
-  }
-}
-
 static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
   SVnodeObj *pVnode = pInfo->ahandle;
   SRpcMsg   *pMsg = NULL;
@@ -211,12 +105,10 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf
 
     int32_t code = vnodeProcessSyncReq(pVnode->pImpl, pMsg, NULL);
     if (code != 0) {
-      dError("vgId:%d, msg:%p failed to sync since %s", pVnode->vgId, pMsg, terrstr());
-      if (pMsg->info.handle != NULL) {
       if (terrno != 0) code = terrno;
+      dError("vgId:%d, msg:%p failed to sync since %s", pVnode->vgId, pMsg, terrstr());
       vmSendRsp(pMsg, code);
     }
-    }
 
     dTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code);
     rpcFreeCont(pMsg->pCont);
@@ -234,8 +126,8 @@ static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
 
     int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo);
     if (code != 0) {
-      dError("vgId:%d, msg:%p failed to merge since %s", pVnode->vgId, pMsg, terrstr());
       if (terrno != 0) code = terrno;
+      dError("vgId:%d, msg:%p failed to merge since %s", pVnode->vgId, pMsg, terrstr());
       vmSendRsp(pMsg, code);
     }
 
@@ -254,7 +146,7 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
 
   SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
   if (pVnode == NULL) {
-    dError("vgId:%d, failed to put msg:%p into vnode queue since %s, type:%s", pHead->vgId, pMsg, terrstr(),
+    dError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s", pHead->vgId, pMsg, terrstr(),
            TMSG_INFO(pMsg->msgType));
     return terrno != 0 ? terrno : -1;
   }
@@ -322,7 +214,7 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
   if (pMsg == NULL) return -1;
 
   SMsgHead *pHead = pRpc->pCont;
-  dTrace("vgId:%d, msg:%p is created, type:%s", pHead->vgId, pMsg, TMSG_INFO(pMsg->msgType));
+  dTrace("vgId:%d, msg:%p is created, type:%s", pHead->vgId, pMsg, TMSG_INFO(pRpc->msgType));
 
   pHead->contLen = htonl(pHead->contLen);
   pHead->vgId = htonl(pHead->vgId);
@@ -362,9 +254,9 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) {
 }
 
 int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
-  pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessWriteQueue);
+  pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode->pImpl, (FItems)vnodeProposeMsg);
   pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)vmProcessSyncQueue);
-  pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessApplyQueue);
+  pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->applyPool, pVnode->pImpl, (FItems)vnodeApplyMsg);
   pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue);
   pVnode->pFetchQ = tQWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItem)vmProcessFetchQueue);
   pVnode->pMergeQ = tWWorkerAllocQueue(&pMgmt->mergePool, pVnode, (FItems)vmProcessMergeQueue);
@@ -381,8 +273,8 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
 
 void vmFreeQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
   tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ);
+  tWWorkerFreeQueue(&pMgmt->applyPool, pVnode->pApplyQ);
   tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ);
-  tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ);
   tQWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ);
   tQWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ);
   tWWorkerFreeQueue(&pMgmt->mergePool, pVnode->pMergeQ);
@@ -413,6 +305,11 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) {
   pWPool->max = tsNumOfVnodeWriteThreads;
   if (tWWorkerInit(pWPool) != 0) return -1;
 
+  SWWorkerPool *pAPool = &pMgmt->applyPool;
+  pAPool->name = "vnode-apply";
+  pAPool->max = tsNumOfVnodeWriteThreads;
+  if (tWWorkerInit(pAPool) != 0) return -1;
+
   SWWorkerPool *pSPool = &pMgmt->syncPool;
   pSPool->name = "vnode-sync";
   pSPool->max = tsNumOfVnodeSyncThreads;
@@ -449,6 +346,7 @@ void vmStopWorker(SVnodeMgmt *pMgmt) {
   tSingleWorkerCleanup(&pMgmt->monitorWorker);
   tSingleWorkerCleanup(&pMgmt->mgmtWorker);
   tWWorkerCleanup(&pMgmt->writePool);
+  tWWorkerCleanup(&pMgmt->applyPool);
   tWWorkerCleanup(&pMgmt->syncPool);
   tQWorkerCleanup(&pMgmt->queryPool);
   tQWorkerCleanup(&pMgmt->fetchPool);

source/dnode/mnode/impl/src/mndConsumer.c

@@ -92,6 +92,15 @@ static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg) {
   SMqConsumerObj     *pConsumer = mndAcquireConsumer(pMnode, pLostMsg->consumerId);
   ASSERT(pConsumer);
 
+
+  mInfo("receive consumer lost msg, consumer id %ld, status %s", pLostMsg->consumerId,
+        mndConsumerStatusName(pConsumer->status));
+
+  if (pConsumer->status != MQ_CONSUMER_STATUS__READY) {
+    mndReleaseConsumer(pMnode, pConsumer);
+    return -1;
+  }
+
   SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup);
   pConsumerNew->updateType = CONSUMER_UPDATE__LOST;
 

source/dnode/mnode/impl/src/mndShow.c

@@ -266,7 +266,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
   }
 
   size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pShow->pMeta->numOfColumns +
-         blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(pBlock);
+         blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(pBlock->info.numOfCols);
 
   SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size);
   if (pRsp == NULL) {

source/dnode/mnode/impl/src/mndSma.c

@@ -894,6 +894,15 @@ static int32_t mndGetTableSma(SMnode *pMnode, STableIndexReq *indexReq, STableIn
     info.sliding = pSma->sliding;
     info.dstTbUid = pSma->dstTbUid;
     info.dstVgId = pSma->dstVgId;
+
+    SVgObj* pVg = mndAcquireVgroup(pMnode, pSma->dstVgId);
+    if (pVg == NULL) {
+      code = -1;
+      sdbRelease(pSdb, pSma);
+      return code;
+    }
+    info.epSet = mndGetVgroupEpset(pMnode, pVg);
+    
     info.expr = taosMemoryMalloc(pSma->exprLen + 1);
     if (info.expr == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;

source/dnode/vnode/inc/vnode.h

@@ -71,6 +71,9 @@ int32_t vnodeSnapshotReaderClose(SVSnapshotReader *pReader);
 int32_t vnodeSnapshotRead(SVSnapshotReader *pReader, const void **ppData, uint32_t *nData);
 int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen);
 
+void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs);
+void vnodeApplyMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs);
+
 // meta
 typedef struct SMeta       SMeta;  // todo: remove
 typedef struct SMetaReader SMetaReader;

source/dnode/vnode/src/inc/vnd.h

@@ -84,7 +84,6 @@ int32_t vnodeAsyncCommit(SVnode* pVnode);
 int32_t vnodeSyncOpen(SVnode* pVnode, char* path);
 void    vnodeSyncStart(SVnode* pVnode);
 void    vnodeSyncClose(SVnode* pVnode);
-int32_t vnodeSyncAlter(SVnode* pVnode, SRpcMsg* pMsg);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/src/inc/vnodeInt.h

@@ -220,8 +220,10 @@ struct SVnode {
   SWal*      pWal;
   STQ*       pTq;
   SSink*     pSink;
-  int64_t    sync;
   tsem_t     canCommit;
+  int64_t    sync;
+  int32_t    syncCount;
+  sem_t      syncSem;
   SQHandle*  pQuery;
 };
 

source/dnode/vnode/src/tq/tq.c

@@ -448,7 +448,7 @@ int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg) {
 }
 
 int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) {
-  SStreamDispatchRsp* pRsp = pMsg->pCont;
+  SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
   int32_t             taskId = pRsp->taskId;
   SStreamTask*        pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t));
   streamProcessDispatchRsp(pTask, &pTq->pVnode->msgCb, pRsp);

source/dnode/vnode/src/tq/tqExec.c

@@ -39,6 +39,9 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataBlkRsp* pRs
 
 static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, int32_t workerId, SMqDataBlkRsp* pRsp) {
   SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pExecReader[workerId]->pSchemaWrapper);
+  if (pSW == NULL) {
+    return -1;
+  }
   taosArrayPush(pRsp->blockSchema, &pSW);
   return 0;
 }

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -81,7 +81,9 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
   pVnode->state.applied = info.state.committed;
   pVnode->pTfs = pTfs;
   pVnode->msgCb = msgCb;
+  pVnode->syncCount = 0;
 
+  tsem_init(&pVnode->syncSem, 0, 0);
   tsem_init(&(pVnode->canCommit), 0, 1);
 
   // open buffer pool
@@ -175,6 +177,7 @@ void vnodeClose(SVnode *pVnode) {
     vnodeCloseBufPool(pVnode);
     // destroy handle
     tsem_destroy(&(pVnode->canCommit));
+    tsem_destroy(&pVnode->syncSem);
     taosMemoryFree(pVnode);
   }
 }

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -91,9 +91,6 @@ int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg) {
       }
 
     } break;
-    case TDMT_VND_ALTER_REPLICA: {
-      code = vnodeSyncAlter(pVnode, pMsg);
-    } break;
     default:
       break;
   }
@@ -107,7 +104,7 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
   int32_t len;
   int32_t ret;
 
-  vTrace("vgId:%d, start to process write request %s, version %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType),
+  vTrace("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType),
          version);
 
   pVnode->state.applied = version;
@@ -173,7 +170,7 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
       break;
   }
 
-  vTrace("vgId:%d, process %s request success, version: %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version);
+  vTrace("vgId:%d, process %s request success, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version);
 
   if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) {
     vError("vgId:%d, failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno));

source/dnode/vnode/src/vnd/vnodeSync.c

@@ -16,13 +16,239 @@
 #define _DEFAULT_SOURCE
 #include "vnd.h"
 
-static int32_t   vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg);
+static inline bool vnodeIsMsgBlock(tmsg_t type) {
-static int32_t   vnodeSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg);
+  return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_CONFIRM) || (type == TDMT_VND_ALTER_REPLICA);
-static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode);
+}
-static void      vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
+
-static void      vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
+static inline bool vnodeIsMsgWeak(tmsg_t type) { return false; }
-static void      vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
+
-static int32_t   vnodeSyncGetSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot);
+static inline void vnodeAccumBlockMsg(SVnode *pVnode, tmsg_t type) {
+  if (!vnodeIsMsgBlock(type)) return;
+
+  int32_t count = atomic_add_fetch_32(&pVnode->syncCount, 1);
+  vTrace("vgId:%d, accum block, count:%d type:%s", pVnode->config.vgId, count, TMSG_INFO(type));
+}
+
+static inline void vnodeWaitBlockMsg(SVnode *pVnode) {
+  int32_t count = atomic_load_32(&pVnode->syncCount);
+  if (count <= 0) return;
+
+  vTrace("vgId:%d, wait block finish, count:%d", pVnode->config.vgId, count);
+  tsem_wait(&pVnode->syncSem);
+}
+
+static inline void vnodePostBlockMsg(SVnode *pVnode, tmsg_t type) {
+  if (!vnodeIsMsgBlock(type)) return;
+
+  int32_t count = atomic_load_32(&pVnode->syncCount);
+  if (count <= 0) return;
+
+  count = atomic_sub_fetch_32(&pVnode->syncCount, 1);
+  vTrace("vgId:%d, post block, count:%d type:%s", pVnode->config.vgId, count, TMSG_INFO(type));
+  if (count <= 0) {
+    tsem_post(&pVnode->syncSem);
+  }
+}
+
+static int32_t vnodeProcessSyncReconfigReq(SVnode *pVnode, SRpcMsg *pMsg) {
+  SAlterVnodeReq req = {0};
+  if (tDeserializeSAlterVnodeReq((char *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead), &req) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return TSDB_CODE_INVALID_MSG;
+  }
+
+  vInfo("vgId:%d, start to alter vnode replica to %d", TD_VID(pVnode), req.replica);
+  SSyncCfg cfg = {.replicaNum = req.replica, .myIndex = req.selfIndex};
+  for (int32_t r = 0; r < req.replica; ++r) {
+    SNodeInfo *pNode = &cfg.nodeInfo[r];
+    tstrncpy(pNode->nodeFqdn, req.replicas[r].fqdn, sizeof(pNode->nodeFqdn));
+    pNode->nodePort = req.replicas[r].port;
+    vInfo("vgId:%d, replica:%d %s:%u", TD_VID(pVnode), r, pNode->nodeFqdn, pNode->nodePort);
+  }
+
+  return syncReconfig(pVnode->sync, &cfg);
+}
+
+void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
+  SVnode  *pVnode = pInfo->ahandle;
+  int32_t  vgId = pVnode->config.vgId;
+  int32_t  code = 0;
+  SRpcMsg *pMsg = NULL;
+
+  for (int32_t m = 0; m < numOfMsgs; m++) {
+    if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
+    vTrace("vgId:%d, msg:%p get from vnode-write queue handle:%p", vgId, pMsg, pMsg->info.handle);
+
+    if (pMsg->msgType == TDMT_VND_ALTER_REPLICA) {
+      code = vnodeProcessSyncReconfigReq(pVnode, pMsg);
+    } else {
+      code = vnodePreprocessReq(pVnode, pMsg);
+      if (code != 0) {
+        vError("vgId:%d, failed to pre-process msg:%p since %s", vgId, pMsg, terrstr());
+      } else {
+        code = syncPropose(pVnode->sync, pMsg, vnodeIsMsgWeak(pMsg->msgType));
+      }
+    }
+
+    if (code == 0) {
+      vnodeAccumBlockMsg(pVnode, pMsg->msgType);
+      if (pMsg->msgType == TDMT_VND_ALTER_REPLICA) {
+        // todo refactor
+        SRpcMsg rsp = {.code = code, .info = pMsg->info};
+        tmsgSendRsp(&rsp);
+      }
+    } else if (code == TAOS_SYNC_PROPOSE_NOT_LEADER) {
+      SEpSet newEpSet = {0};
+      syncGetEpSet(pVnode->sync, &newEpSet);
+      SEp *pEp = &newEpSet.eps[newEpSet.inUse];
+      if (pEp->port == tsServerPort && strcmp(pEp->fqdn, tsLocalFqdn) == 0) {
+        newEpSet.inUse = (newEpSet.inUse + 1) % newEpSet.numOfEps;
+      }
+
+      vTrace("vgId:%d, msg:%p is redirect since not leader, numOfEps:%d inUse:%d", vgId, pMsg, newEpSet.numOfEps,
+             newEpSet.inUse);
+      for (int32_t i = 0; i < newEpSet.numOfEps; ++i) {
+        vTrace("vgId:%d, msg:%p redirect:%d ep:%s:%u", vgId, pMsg, i, newEpSet.eps[i].fqdn, newEpSet.eps[i].port);
+      }
+
+      SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info};
+      tmsgSendRedirectRsp(&rsp, &newEpSet);
+    } else {
+      if (terrno != 0) code = terrno;
+      vError("vgId:%d, msg:%p failed to propose since %s, code:0x%x", vgId, pMsg, tstrerror(code), code);
+      SRpcMsg rsp = {.code = code, .info = pMsg->info};
+      tmsgSendRsp(&rsp);
+    }
+
+    vTrace("vgId:%d, msg:%p is freed, code:0x%x", vgId, pMsg, code);
+    rpcFreeCont(pMsg->pCont);
+    taosFreeQitem(pMsg);
+  }
+
+  vnodeWaitBlockMsg(pVnode);
+}
+
+void vnodeApplyMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
+  SVnode  *pVnode = pInfo->ahandle;
+  int32_t  vgId = pVnode->config.vgId;
+  int32_t  code = 0;
+  SRpcMsg *pMsg = NULL;
+
+  for (int32_t i = 0; i < numOfMsgs; ++i) {
+    if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
+    vTrace("vgId:%d, msg:%p get from vnode-apply queue, handle:%p", vgId, pMsg, pMsg->info.handle);
+
+    SRpcMsg rsp = {.code = pMsg->code, .info = pMsg->info};
+    if (rsp.code == 0) {
+      if (vnodeProcessWriteReq(pVnode, pMsg, pMsg->conn.applyIndex, &rsp) < 0) {
+        rsp.code = terrno;
+        vError("vgId:%d, msg:%p failed to apply since %s", vgId, pMsg, terrstr());
+      }
+    }
+
+    vnodePostBlockMsg(pVnode, pMsg->msgType);
+    if (rsp.info.handle != NULL) {
+      tmsgSendRsp(&rsp);
+    }
+
+    vTrace("vgId:%d, msg:%p is freed, code:0x%x", vgId, pMsg, rsp.code);
+    rpcFreeCont(pMsg->pCont);
+    taosFreeQitem(pMsg);
+  }
+}
+
+static int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
+  int32_t code = tmsgPutToQueue(msgcb, SYNC_QUEUE, pMsg);
+  if (code != 0) {
+    rpcFreeCont(pMsg->pCont);
+    pMsg->pCont = NULL;
+  }
+  return code;
+}
+
+static int32_t vnodeSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) {
+  int32_t code = tmsgSendReq(pEpSet, pMsg);
+  if (code != 0) {
+    rpcFreeCont(pMsg->pCont);
+    pMsg->pCont = NULL;
+  }
+  return code;
+}
+
+static int32_t vnodeSyncGetSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) {
+  vnodeGetSnapshot(pFsm->data, pSnapshot);
+  return 0;
+}
+
+static void vnodeSyncReconfig(struct SSyncFSM *pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) {
+  SVnode *pVnode = pFsm->data;
+  vInfo("vgId:%d, sync reconfig is confirmed", TD_VID(pVnode));
+
+  // todo rpc response here
+  // build rpc msg
+  // put into apply queue
+}
+
+static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
+  SVnode   *pVnode = pFsm->data;
+  SSnapshot snapshot = {0};
+  SyncIndex beginIndex = SYNC_INDEX_INVALID;
+  char      logBuf[256] = {0};
+
+  if (pFsm->FpGetSnapshot != NULL) {
+    (*pFsm->FpGetSnapshot)(pFsm, &snapshot);
+    beginIndex = snapshot.lastApplyIndex;
+  }
+
+  if (cbMeta.index > beginIndex) {
+    snprintf(
+        logBuf, sizeof(logBuf),
+        "==callback== ==CommitCb== execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, beginIndex :%ld\n",
+        pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), beginIndex);
+    syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
+
+    SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen, .conn.applyIndex = cbMeta.index};
+    rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
+    memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen);
+    syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &rpcMsg.info);
+    tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg);
+  } else {
+    char logBuf[256] = {0};
+    snprintf(logBuf, sizeof(logBuf),
+             "==callback== ==CommitCb== do not execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, "
+             "beginIndex :%ld\n",
+             pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state),
+             beginIndex);
+    syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
+  }
+}
+
+static void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
+  char logBuf[256] = {0};
+  snprintf(logBuf, sizeof(logBuf),
+           "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index,
+           cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state));
+  syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
+}
+
+static void vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
+  char logBuf[256] = {0};
+  snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n",
+           pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state));
+  syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
+}
+
+static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {
+  SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM));
+  pFsm->data = pVnode;
+  pFsm->FpCommitCb = vnodeSyncCommitMsg;
+  pFsm->FpPreCommitCb = vnodeSyncPreCommitMsg;
+  pFsm->FpRollBackCb = vnodeSyncRollBackMsg;
+  pFsm->FpGetSnapshot = vnodeSyncGetSnapshot;
+  pFsm->FpRestoreFinishCb = NULL;
+  pFsm->FpReConfigCb = vnodeSyncReconfig;
+  return pFsm;
+}
 
 int32_t vnodeSyncOpen(SVnode *pVnode, char *path) {
   SSyncInfo syncInfo = {
@@ -50,33 +276,6 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path) {
   return 0;
 }
 
-int32_t vnodeSyncAlter(SVnode *pVnode, SRpcMsg *pMsg) {
-  SAlterVnodeReq req = {0};
-  if (tDeserializeSAlterVnodeReq((char *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead), &req) != 0) {
-    terrno = TSDB_CODE_INVALID_MSG;
-    return TSDB_CODE_INVALID_MSG;
-  }
-
-  vInfo("vgId:%d, start to alter vnode replica to %d", TD_VID(pVnode), req.replica);
-  SSyncCfg cfg = {.replicaNum = req.replica, .myIndex = req.selfIndex};
-  for (int32_t r = 0; r < req.replica; ++r) {
-    SNodeInfo *pNode = &cfg.nodeInfo[r];
-    tstrncpy(pNode->nodeFqdn, req.replicas[r].fqdn, sizeof(pNode->nodeFqdn));
-    pNode->nodePort = req.replicas[r].port;
-    vInfo("vgId:%d, replica:%d %s:%u", TD_VID(pVnode), r, pNode->nodeFqdn, pNode->nodePort);
-  }
-
-  int32_t code = syncReconfig(pVnode->sync, &cfg);
-  if (code == TAOS_SYNC_PROPOSE_SUCCESS) {
-    // todo refactor
-    SRpcMsg rsp = {.info = pMsg->info, .code = 0};
-    tmsgSendRsp(&rsp);
-    return TSDB_CODE_ACTION_IN_PROGRESS;
-  }
-
-  return code;
-}
-
 void vnodeSyncStart(SVnode *pVnode) {
   syncSetMsgCb(pVnode->sync, &pVnode->msgCb);
   if (pVnode->config.standby) {
@@ -87,107 +286,3 @@ void vnodeSyncStart(SVnode *pVnode) {
 }
 
 void vnodeSyncClose(SVnode *pVnode) { syncStop(pVnode->sync); }
-
-int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
-  int32_t code = tmsgPutToQueue(msgcb, SYNC_QUEUE, pMsg);
-  if (code != 0) {
-    rpcFreeCont(pMsg->pCont);
-    pMsg->pCont = NULL;
-  }
-  return code;
-}
-
-int32_t vnodeSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) {
-  int32_t code = tmsgSendReq(pEpSet, pMsg);
-  if (code != 0) {
-    rpcFreeCont(pMsg->pCont);
-    pMsg->pCont = NULL;
-  }
-  return code;
-}
-
-int32_t vnodeSyncGetSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) {
-  vnodeGetSnapshot(pFsm->data, pSnapshot);
-  return 0;
-}
-
-void vnodeSyncReconfig(struct SSyncFSM *pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) {
-  SVnode *pVnode = pFsm->data;
-  vInfo("vgId:%d, sync reconfig is confirmed", TD_VID(pVnode));
-
-  // todo rpc response here
-}
-
-void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
-  SyncIndex beginIndex = SYNC_INDEX_INVALID;
-  if (pFsm->FpGetSnapshot != NULL) {
-    SSnapshot snapshot = {0};
-    pFsm->FpGetSnapshot(pFsm, &snapshot);
-    beginIndex = snapshot.lastApplyIndex;
-  }
-
-  if (cbMeta.index > beginIndex) {
-    char logBuf[256] = {0};
-    snprintf(
-        logBuf, sizeof(logBuf),
-        "==callback== ==CommitCb== execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, beginIndex :%ld\n",
-        pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), beginIndex);
-    syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
-
-    SVnode       *pVnode = pFsm->data;
-    SyncApplyMsg *pSyncApplyMsg = syncApplyMsgBuild2(pMsg, pVnode->config.vgId, &cbMeta);
-    SRpcMsg       applyMsg;
-    syncApplyMsg2RpcMsg(pSyncApplyMsg, &applyMsg);
-    syncApplyMsgDestroy(pSyncApplyMsg);
-
-    // recover handle for response
-    SRpcMsg saveRpcMsg;
-    int32_t ret = syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &saveRpcMsg);
-    if (ret == 1 && cbMeta.state == TAOS_SYNC_STATE_LEADER) {
-      applyMsg.info = saveRpcMsg.info;
-    } else {
-      applyMsg.info.handle = NULL;
-      applyMsg.info.ahandle = NULL;
-    }
-
-    // put to applyQ
-    tmsgPutToQueue(&(pVnode->msgCb), APPLY_QUEUE, &applyMsg);
-
-  } else {
-    char logBuf[256] = {0};
-    snprintf(logBuf, sizeof(logBuf),
-             "==callback== ==CommitCb== do not execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, "
-             "beginIndex :%ld\n",
-             pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state),
-             beginIndex);
-    syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
-  }
-}
-
-void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
-  char logBuf[256] = {0};
-  snprintf(logBuf, sizeof(logBuf),
-           "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index,
-           cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state));
-  syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
-}
-
-void vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
-  char logBuf[256] = {0};
-  snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n",
-           pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state));
-  syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
-}
-
-SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {
-  SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM));
-  pFsm->data = pVnode;
-  pFsm->FpCommitCb = vnodeSyncCommitMsg;
-  pFsm->FpPreCommitCb = vnodeSyncPreCommitMsg;
-  pFsm->FpRollBackCb = vnodeSyncRollBackMsg;
-  pFsm->FpGetSnapshot = vnodeSyncGetSnapshot;
-  pFsm->FpRestoreFinishCb = NULL;
-  pFsm->FpReConfigCb = vnodeSyncReconfig;
-
-  return pFsm;
-}

source/libs/catalog/inc/catalogInt.h

@@ -173,7 +173,6 @@ typedef struct SCtgJob {
   SArray*          pTasks;
   int32_t          taskDone;
   SMetaData        jobRes;
-  int32_t          rspCode;
 
   uint64_t         queryId;
   SCatalog*        pCtg; 
@@ -205,6 +204,7 @@ typedef struct SCtgTask {
   SCtgJob*      pJob;
   void*         taskCtx;
   SCtgMsgCtx    msgCtx;
+  int32_t       code;
   void*         res;
 } SCtgTask;
 

source/libs/catalog/src/ctgAsync.c

@@ -437,13 +437,14 @@ _return:
 int32_t ctgDumpTbMetaRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pTableMeta) {
-    pJob->jobRes.pTableMeta = taosArrayInit(pJob->tbMetaNum, POINTER_BYTES);
+    pJob->jobRes.pTableMeta = taosArrayInit(pJob->tbMetaNum, sizeof(SMetaRes));
     if (NULL == pJob->jobRes.pTableMeta) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
   }
 
-  taosArrayPush(pJob->jobRes.pTableMeta, &pTask->res);
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pTableMeta, &res);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -451,14 +452,14 @@ int32_t ctgDumpTbMetaRes(SCtgTask* pTask) {
 int32_t ctgDumpDbVgRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pDbVgroup) {
-    pJob->jobRes.pDbVgroup = taosArrayInit(pJob->dbVgNum, POINTER_BYTES);
+    pJob->jobRes.pDbVgroup = taosArrayInit(pJob->dbVgNum, sizeof(SMetaRes));
     if (NULL == pJob->jobRes.pDbVgroup) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
   }
 
-  taosArrayPush(pJob->jobRes.pDbVgroup, &pTask->res);
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
-  pTask->res = NULL;
+  taosArrayPush(pJob->jobRes.pDbVgroup, &res);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -466,13 +467,14 @@ int32_t ctgDumpDbVgRes(SCtgTask* pTask) {
 int32_t ctgDumpTbHashRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pTableHash) {
-    pJob->jobRes.pTableHash = taosArrayInit(pJob->tbHashNum, sizeof(SVgroupInfo));
+    pJob->jobRes.pTableHash = taosArrayInit(pJob->tbHashNum, sizeof(SMetaRes));
     if (NULL == pJob->jobRes.pTableHash) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
   }
 
-  taosArrayPush(pJob->jobRes.pTableHash, pTask->res);
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pTableHash, &res);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -480,21 +482,29 @@ int32_t ctgDumpTbHashRes(SCtgTask* pTask) {
 int32_t ctgDumpIndexRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pIndex) {
-    pJob->jobRes.pIndex = taosArrayInit(pJob->indexNum, sizeof(SIndexInfo));
+    pJob->jobRes.pIndex = taosArrayInit(pJob->indexNum, sizeof(SMetaRes));
     if (NULL == pJob->jobRes.pIndex) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
   }
 
-  taosArrayPush(pJob->jobRes.pIndex, pTask->res);
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pIndex, &res);
 
   return TSDB_CODE_SUCCESS;
 }
 
 int32_t ctgDumpQnodeRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
+  if (NULL == pJob->jobRes.pQnodeList) {
+    pJob->jobRes.pQnodeList = taosArrayInit(1, sizeof(SMetaRes));
+    if (NULL == pJob->jobRes.pQnodeList) {
+      CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
+    }
+  }
 
-  TSWAP(pJob->jobRes.pQnodeList, pTask->res); 
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pQnodeList, &res);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -502,13 +512,14 @@ int32_t ctgDumpQnodeRes(SCtgTask* pTask) {
 int32_t ctgDumpDbCfgRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pDbCfg) {
-    pJob->jobRes.pDbCfg = taosArrayInit(pJob->dbCfgNum, sizeof(SDbCfgInfo));
+    pJob->jobRes.pDbCfg = taosArrayInit(pJob->dbCfgNum, sizeof(SMetaRes));
     if (NULL == pJob->jobRes.pDbCfg) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
   }
 
-  taosArrayPush(pJob->jobRes.pDbCfg, pTask->res);
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pDbCfg, &res);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -516,13 +527,14 @@ int32_t ctgDumpDbCfgRes(SCtgTask* pTask) {
 int32_t ctgDumpDbInfoRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pDbInfo) {
-    pJob->jobRes.pDbInfo = taosArrayInit(pJob->dbInfoNum, sizeof(SDbInfo));
+    pJob->jobRes.pDbInfo = taosArrayInit(pJob->dbInfoNum, sizeof(SMetaRes));
     if (NULL == pJob->jobRes.pDbInfo) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
   }
 
-  taosArrayPush(pJob->jobRes.pDbInfo, pTask->res);
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pDbInfo, &res);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -530,13 +542,14 @@ int32_t ctgDumpDbInfoRes(SCtgTask* pTask) {
 int32_t ctgDumpUdfRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pUdfList) {
-    pJob->jobRes.pUdfList = taosArrayInit(pJob->udfNum, sizeof(SFuncInfo));
+    pJob->jobRes.pUdfList = taosArrayInit(pJob->udfNum, sizeof(SMetaRes));
     if (NULL == pJob->jobRes.pUdfList) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
   }
 
-  taosArrayPush(pJob->jobRes.pUdfList, pTask->res);
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pUdfList, &res);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -544,13 +557,14 @@ int32_t ctgDumpUdfRes(SCtgTask* pTask) {
 int32_t ctgDumpUserRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pUser) {
-    pJob->jobRes.pUser = taosArrayInit(pJob->userNum, sizeof(bool));
+    pJob->jobRes.pUser = taosArrayInit(pJob->userNum, sizeof(SMetaRes));
     if (NULL == pJob->jobRes.pUser) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
   }
 
-  taosArrayPush(pJob->jobRes.pUser, pTask->res);
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pUser, &res);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -559,20 +573,13 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) {
   SCtgJob* pJob = pTask->pJob;
   int32_t code = 0;
 
-  qDebug("QID:%" PRIx64 " task %d end with rsp %s", pJob->queryId, pTask->taskId, tstrerror(rspCode));
+  qDebug("QID:0x%" PRIx64 " task %d end with rsp %s", pJob->queryId, pTask->taskId, tstrerror(rspCode));
 
-  if (rspCode) {
+  pTask->code = rspCode;
-    int32_t lastCode = atomic_val_compare_exchange_32(&pJob->rspCode, 0, rspCode);
-    if (0 == lastCode) {
-      CTG_ERR_JRET(rspCode);
-    }
-
-    return TSDB_CODE_SUCCESS;  
-  }
 
   int32_t taskDone = atomic_add_fetch_32(&pJob->taskDone, 1);
   if (taskDone < taosArrayGetSize(pJob->pTasks)) {
-    qDebug("task done: %d, total: %d", taskDone, (int32_t)taosArrayGetSize(pJob->pTasks));
+    qDebug("QID:0x%" PRIx64 " task done: %d, total: %d", pJob->queryId, taskDone, (int32_t)taosArrayGetSize(pJob->pTasks));
     return TSDB_CODE_SUCCESS;
   }
 

source/libs/catalog/src/ctgUtil.c

@@ -48,10 +48,12 @@ void ctgFreeSMetaData(SMetaData* pData) {
   taosArrayDestroy(pData->pTableMeta);
   pData->pTableMeta = NULL;
 
+/*  
   for (int32_t i = 0; i < taosArrayGetSize(pData->pDbVgroup); ++i) {
     SArray** pArray = taosArrayGet(pData->pDbVgroup, i);
     taosArrayDestroy(*pArray);
   }
+*/
   taosArrayDestroy(pData->pDbVgroup);
   pData->pDbVgroup = NULL;
   
@@ -61,10 +63,12 @@ void ctgFreeSMetaData(SMetaData* pData) {
   taosArrayDestroy(pData->pUdfList);
   pData->pUdfList = NULL;
 
+/*
   for (int32_t i = 0; i < taosArrayGetSize(pData->pDbCfg); ++i) {
     SDbCfgInfo* pInfo = taosArrayGet(pData->pDbCfg, i);
     taosArrayDestroy(pInfo->pRetensions);
   }
+*/  
   taosArrayDestroy(pData->pDbCfg);
   pData->pDbCfg = NULL;
 
@@ -321,7 +325,11 @@ void ctgFreeTask(SCtgTask* pTask) {
     }
     case CTG_TASK_GET_DB_CFG: {
       taosMemoryFreeClear(pTask->taskCtx);
+      if (pTask->res) {
+        SDbCfgInfo* pInfo = (SDbCfgInfo*)pTask->res;
+        taosArrayDestroy(pInfo->pRetensions);
         taosMemoryFreeClear(pTask->res);
+      }
       break;
     }
     case CTG_TASK_GET_DB_INFO: {

source/libs/command/inc/commandInt.h

@@ -115,9 +115,9 @@ typedef struct SExplainCtx {
 #define EXPLAIN_ROW_NEW(level, ...)                                                                               \
   do {                                                                                                            \
     if (isVerboseLine) {                                                                                          \
-      tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE, "%*s", (level) * 2 + 3, "");       \
+      tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE, "%*s", (level) * 3 + 3, "");       \
     } else {                                                                                                      \
-      tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE, "%*s%s", (level) * 2, "", "-> ");  \
+      tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE, "%*s%s", (level) * 3, "", "-> ");  \
     }                                                                                                             \
     tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE - tlen, __VA_ARGS__);         \
   } while (0)

source/libs/command/src/command.c

@@ -28,91 +28,94 @@ static int32_t getSchemaBytes(const SSchema* pSchema) {
   }
 }
 
-// todo : to convert data according to SSDatablock
+static SSDataBlock* buildDescResultDataBlock() {
-static void buildRspData(const STableMeta* pMeta, char* pData) {
+  SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
-  int32_t* payloadLen = (int32_t*) pData;
+  pBlock->info.numOfCols = DESCRIBE_RESULT_COLS;
-  uint64_t* groupId = (uint64_t*)(pData + sizeof(int32_t));
+  pBlock->info.hasVarCol = true;
 
-  int32_t* pColSizes = (int32_t*)(pData + sizeof(int32_t) + sizeof(uint64_t));
+  pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
-  pData = (char*) pColSizes + DESCRIBE_RESULT_COLS * sizeof(int32_t);
 
-  int32_t numOfRows = TABLE_TOTAL_COL_NUM(pMeta);
+  SColumnInfoData infoData = {0};
+  infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
+  infoData.info.bytes = DESCRIBE_RESULT_FIELD_LEN;
 
-  // Field
+  taosArrayPush(pBlock->pDataBlock, &infoData);
-  int32_t* pOffset = (int32_t*)pData;
+
-  pData += numOfRows * sizeof(int32_t);
+  infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
+  infoData.info.bytes = DESCRIBE_RESULT_TYPE_LEN;
+  taosArrayPush(pBlock->pDataBlock, &infoData);
+
+  infoData.info.type = TSDB_DATA_TYPE_INT;
+  infoData.info.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes;;
+  taosArrayPush(pBlock->pDataBlock, &infoData);
+
+  infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
+  infoData.info.bytes = DESCRIBE_RESULT_NOTE_LEN;
+  taosArrayPush(pBlock->pDataBlock, &infoData);
+
+  return pBlock;
+}
+
+static void setDescResultIntoDataBlock(SSDataBlock* pBlock, int32_t numOfRows, STableMeta* pMeta) {
+  blockDataEnsureCapacity(pBlock, numOfRows);
+  pBlock->info.rows = numOfRows;
+
+  // field
+  SColumnInfoData* pCol1 = taosArrayGet(pBlock->pDataBlock, 0);
+  char buf[DESCRIBE_RESULT_FIELD_LEN] = {0};
   for (int32_t i = 0; i < numOfRows; ++i) {
-    STR_TO_VARSTR(pData, pMeta->schema[i].name);
+    STR_TO_VARSTR(buf, pMeta->schema[i].name);
-    int16_t len = varDataTLen(pData);
+    colDataAppend(pCol1, i, buf, false);
-    pData += len;
-    *pOffset = pColSizes[0];
-    pOffset += 1;
-    pColSizes[0] += len;
   }
 
   // Type
-  pOffset = (int32_t*)pData;
+  SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1);
-  pData += numOfRows * sizeof(int32_t);
   for (int32_t i = 0; i < numOfRows; ++i) {
-    STR_TO_VARSTR(pData, tDataTypes[pMeta->schema[i].type].name);
+    STR_TO_VARSTR(buf, tDataTypes[pMeta->schema[i].type].name);
-    int16_t len = varDataTLen(pData);
+    colDataAppend(pCol2, i, buf, false);
-    pData += len;
-    *pOffset = pColSizes[1];
-    pOffset += 1;
-    pColSizes[1] += len;
   }
 
   // Length
-  pData += BitmapLen(numOfRows);
+  SColumnInfoData* pCol3 = taosArrayGet(pBlock->pDataBlock, 2);
   for (int32_t i = 0; i < numOfRows; ++i) {
-    *(int32_t*)pData = getSchemaBytes(pMeta->schema + i);
+    int32_t bytes = getSchemaBytes(pMeta->schema + i);
-    pData += sizeof(int32_t);
+    colDataAppend(pCol3, i, (const char*)&bytes, false);
   }
-  pColSizes[2] = sizeof(int32_t) * numOfRows;
 
   // Note
-  pOffset = (int32_t*)pData;
+  SColumnInfoData* pCol4 = taosArrayGet(pBlock->pDataBlock, 3);
-  pData += numOfRows * sizeof(int32_t);
   for (int32_t i = 0; i < numOfRows; ++i) {
-    STR_TO_VARSTR(pData, i >= pMeta->tableInfo.numOfColumns ? "TAG" : "");
+    STR_TO_VARSTR(buf, i >= pMeta->tableInfo.numOfColumns ? "TAG" : "");
-    int16_t len = varDataTLen(pData);
+    colDataAppend(pCol4, i, buf, false);
-    pData += len;
-    *pOffset = pColSizes[3];
-    pOffset += 1;
-    pColSizes[3] += len;
   }
-
-  for (int32_t i = 0; i < DESCRIBE_RESULT_COLS; ++i) {
-    pColSizes[i] = htonl(pColSizes[i]);
-  }
-
-
-  *payloadLen = (int32_t)(pData - (char*)payloadLen);
-}
-
-static int32_t calcRspSize(const STableMeta* pMeta) {
-  int32_t numOfRows = TABLE_TOTAL_COL_NUM(pMeta);
-  return sizeof(SRetrieveTableRsp) + 
-      (numOfRows * sizeof(int32_t) + numOfRows * DESCRIBE_RESULT_FIELD_LEN) + 
-      (numOfRows * sizeof(int32_t) + numOfRows * DESCRIBE_RESULT_TYPE_LEN) +
-      (BitmapLen(numOfRows) + numOfRows * sizeof(int32_t)) + 
-      (numOfRows * sizeof(int32_t) + numOfRows * DESCRIBE_RESULT_NOTE_LEN) +
-      sizeof(int32_t) + sizeof(uint64_t);
 }
 
 static int32_t execDescribe(SNode* pStmt, SRetrieveTableRsp** pRsp) {
   SDescribeStmt* pDesc = (SDescribeStmt*) pStmt;
-  *pRsp = taosMemoryCalloc(1, calcRspSize(pDesc->pMeta));
+  int32_t numOfRows = TABLE_TOTAL_COL_NUM(pDesc->pMeta);
+
+  SSDataBlock* pBlock = buildDescResultDataBlock();
+  setDescResultIntoDataBlock(pBlock, numOfRows, pDesc->pMeta);
+
+  size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
+  *pRsp = taosMemoryCalloc(1, rspSize);
   if (NULL == *pRsp) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
+
   (*pRsp)->useconds = 0;
   (*pRsp)->completed = 1;
   (*pRsp)->precision = 0;
   (*pRsp)->compressed = 0;
   (*pRsp)->compLen = 0;
-  (*pRsp)->numOfRows = htonl(TABLE_TOTAL_COL_NUM(pDesc->pMeta));
+  (*pRsp)->numOfRows = htonl(numOfRows);
-  buildRspData(pDesc->pMeta, (*pRsp)->data);
+  (*pRsp)->numOfCols = htonl(DESCRIBE_RESULT_COLS);
+
+  int32_t len = 0;
+  blockCompressEncode(pBlock, (*pRsp)->data, &len, DESCRIBE_RESULT_COLS, false);
+  ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
+
+  blockDataDestroy(pBlock);
   return TSDB_CODE_SUCCESS;
 }
 

source/libs/command/src/explain.c

@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "tdatablock.h"
 #include "commandInt.h"
 #include "plannodes.h"
 #include "query.h"
@@ -657,7 +658,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
 
       if (EXPLAIN_MODE_ANALYZE == ctx->mode) {
         // sort key
-        EXPLAIN_ROW_NEW(level, "Sort Key: ");
+        EXPLAIN_ROW_NEW(level + 1, "Sort Key: ");
         if (pResNode->pExecInfo) {
           for (int32_t i = 0; i < LIST_LENGTH(pSortNode->pSortKeys); ++i) {
             SOrderByExprNode *ptn = nodesListGetNode(pSortNode->pSortKeys, i);
@@ -669,7 +670,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
         QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
 
         // sort method
-        EXPLAIN_ROW_NEW(level, "Sort Method: ");
+        EXPLAIN_ROW_NEW(level + 1, "Sort Method: ");
 
         int32_t           nodeNum = taosArrayGetSize(pResNode->pExecInfo);
         SExplainExecInfo *execInfo = taosArrayGet(pResNode->pExecInfo, 0);
@@ -965,9 +966,32 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
     QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
   }
 
-  int32_t colNum = 1;
+  SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
-  int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum +
+  SColumnInfoData infoData = {0};
-                    sizeof(int32_t) * rowNum + pCtx->dataSize;
+  infoData.info.type  = TSDB_DATA_TYPE_VARCHAR;
+  infoData.info.bytes = TSDB_EXPLAIN_RESULT_ROW_SIZE;
+
+  pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
+  taosArrayPush(pBlock->pDataBlock, &infoData);
+
+  SColumnInfoData* pInfoData = taosArrayGet(pBlock->pDataBlock, 0);
+  pInfoData->hasNull = false;
+  colInfoDataEnsureCapacity(pInfoData, 0, rowNum);
+
+  char buf[1024] = {0};
+  for (int32_t i = 0; i < rowNum; ++i) {
+    SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i);
+    varDataCopy(buf, row->buf);
+    ASSERT(varDataTLen(row->buf) == row->len);
+    colDataAppend(pInfoData, i, buf, false);
+  }
+
+  pBlock->info.numOfCols = 1;
+  pBlock->info.rows = rowNum;
+  pBlock->info.hasVarCol = true;
+
+  int32_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
+
   SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize);
   if (NULL == rsp) {
     qError("malloc SRetrieveTableRsp failed, size:%d", rspSize);
@@ -977,34 +1001,13 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
   rsp->completed = 1;
   rsp->numOfRows = htonl(rowNum);
 
-  // payload length
+  int32_t len = 0;
-  *(int32_t *)rsp->data =
+  blockCompressEncode(pBlock, rsp->data, &len, pBlock->info.numOfCols, 0);
-      sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize;
+  ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
 
-  // group id
+  rsp->compLen = htonl(len);
-  *(uint64_t *)(rsp->data + sizeof(int32_t)) = 0;
 
-  // column length
+  blockDataDestroy(pBlock);
-  int32_t *colLength = (int32_t *)(rsp->data + sizeof(int32_t) + sizeof(uint64_t));
-
-  // varchar column offset segment
-  int32_t *offset = (int32_t *)((char *)colLength + sizeof(int32_t));
-
-  // varchar data real payload
-  char *data = (char *)(offset + rowNum);
-
-  char *start = data;
-  for (int32_t i = 0; i < rowNum; ++i) {
-    SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i);
-    offset[i] = data - start;
-
-    varDataCopy(data, row->buf);
-    ASSERT(varDataTLen(row->buf) == row->len);
-    data += row->len;
-  }
-
-  *colLength = htonl(data - start);
-  rsp->compLen = htonl(rspSize);
 
   *pRsp = rsp;
   return TSDB_CODE_SUCCESS;

source/libs/executor/inc/executorimpl.h

@@ -688,6 +688,10 @@ typedef struct SSortedMergeOperatorInfo {
   int32_t          numOfResPerPage;
   char**           groupVal;
   SArray          *groupInfo;
+
+  bool hasGroupId;
+  uint64_t groupId;
+  STupleHandle* prefetchedTuple;
 } SSortedMergeOperatorInfo;
 
 typedef struct SSortOperatorInfo {
@@ -700,6 +704,10 @@ typedef struct SSortOperatorInfo {
 
   int64_t      startTs;       // sort start time
   uint64_t     sortElapsed;   // sort elapsed time, time to flush to disk not included.
+
+  STupleHandle *prefetchedTuple;
+  bool  hasGroupId;
+  uint64_t groupId;
 } SSortOperatorInfo;
 
 typedef struct STagFilterOperatorInfo {
@@ -759,7 +767,7 @@ void    setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData
 SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode);
 SColumn extractColumnFromColumnNode(SColumnNode* pColNode);
 
-SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo);
+SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo, SSortOperatorInfo* pInfo);
 SSDataBlock* loadNextDataBlock(void* param);
 
 void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset);

source/libs/executor/src/dataDispatcher.c

@@ -68,10 +68,10 @@ static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) {
 }
 
 // data format:
-// +----------------+--------------+----------+--------------------------------------+-------------+-----------+-------------+-----------+
+// +----------------+--------------+----------+--------------------------------------------+--------------------------------------+-------------+-----------+-------------+-----------+
-// |SDataCacheEntry | total length | group id | column#1 length, column#2 length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | ....
+// |SDataCacheEntry | total length | group id | col1_schema | col2_schema | col3_schema ...| column#1 length, column#2 length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | ....
-// |                |  (4 bytes)   |(8 bytes) | sizeof(int32_t) * numOfCols          | actual size |           | actual size |           |
+// |                |  (4 bytes)   |(8 bytes) |(sizeof(int16_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols          | actual size |           | actual size |           |
-// +----------------+--------------+----------+--------------------------------------+-------------+-----------+-------------+-----------+
+// +----------------+--------------+----------+--------------------------------------------+--------------------------------------+-------------+-----------+-------------+-----------+
 // The length of bitmap is decided by number of rows of this data block, and the length of each column data is
 // recorded in the first segment, next to the struct header
 static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) {

source/libs/executor/src/executorimpl.c

@@ -2571,37 +2571,7 @@ int32_t setDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadIn
       }
     }
 
-    blockDataEnsureCapacity(pBlock, numOfRows);
+    blockCompressDecode(pBlock, numOfCols, numOfRows, pStart);
-
-    int32_t  dataLen = *(int32_t*)pStart;
-    uint64_t groupId = *(uint64_t*)(pStart + sizeof(int32_t));
-    pStart += sizeof(int32_t) + sizeof(uint64_t);
-
-    int32_t* colLen = (int32_t*)(pStart);
-    pStart += sizeof(int32_t) * numOfCols;
-
-    for (int32_t i = 0; i < numOfCols; ++i) {
-      colLen[i] = htonl(colLen[i]);
-      ASSERT(colLen[i] >= 0);
-
-      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
-      if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
-        pColInfoData->varmeta.length = colLen[i];
-        pColInfoData->varmeta.allocLen = colLen[i];
-
-        memcpy(pColInfoData->varmeta.offset, pStart, sizeof(int32_t) * numOfRows);
-        pStart += sizeof(int32_t) * numOfRows;
-
-        pColInfoData->pData = taosMemoryMalloc(colLen[i]);
-      } else {
-        memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows));
-        pStart += BitmapLen(numOfRows);
-      }
-
-      memcpy(pColInfoData->pData, pStart, colLen[i]);
-      pStart += colLen[i];
-    }
-
     // data from mnode
     relocateColumnData(pRes, pColList, pBlock->pDataBlock);
     taosArrayDestroy(pBlock->pDataBlock);
@@ -3133,6 +3103,68 @@ static SSDataBlock* doMerge(SOperatorInfo* pOperator) {
   return (pInfo->binfo.pRes->info.rows > 0) ? pInfo->binfo.pRes : NULL;
 }
 
+SSDataBlock* getSortedMergeBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo,
+                                SSortedMergeOperatorInfo *pInfo) {
+  blockDataCleanup(pDataBlock);
+
+  SSDataBlock* p = tsortGetSortedDataBlock(pHandle);
+  if (p == NULL) {
+    return NULL;
+  }
+
+  blockDataEnsureCapacity(p, capacity);
+
+  while (1) {
+    STupleHandle* pTupleHandle = NULL;
+    if (pInfo->prefetchedTuple == NULL) {
+      pTupleHandle = tsortNextTuple(pHandle);
+    } else {
+      pTupleHandle = pInfo->prefetchedTuple;
+      pInfo->groupId = tsortGetGroupId(pTupleHandle);
+      pInfo->prefetchedTuple = NULL;
+    }
+
+    if (pTupleHandle == NULL) {
+      break;
+    }
+
+    uint64_t tupleGroupId = tsortGetGroupId(pTupleHandle);
+    if (!pInfo->hasGroupId) {
+      pInfo->groupId = tupleGroupId;
+      pInfo->hasGroupId = true;
+      appendOneRowToDataBlock(p, pTupleHandle);
+    } else if (pInfo->groupId == tupleGroupId) {
+      appendOneRowToDataBlock(p, pTupleHandle);
+    } else {
+      pInfo->prefetchedTuple = pTupleHandle;
+      break;
+    }
+
+    if (p->info.rows >= capacity) {
+      break;
+    }
+  }
+
+  if (p->info.rows > 0) {
+    int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
+    for (int32_t i = 0; i < numOfCols; ++i) {
+      SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i);
+      ASSERT(pmInfo->matchType == COL_MATCH_FROM_SLOT_ID);
+
+      SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
+      SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId);
+      colDataAssign(pDst, pSrc, p->info.rows);
+    }
+
+    pDataBlock->info.rows = p->info.rows;
+    pDataBlock->info.capacity = p->info.rows;
+    pDataBlock->info.groupId = pInfo->groupId;
+  }
+
+  blockDataDestroy(p);
+  return (pDataBlock->info.rows > 0) ? pDataBlock : NULL;
+}
+
 static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator) {
   if (pOperator->status == OP_EXEC_DONE) {
     return NULL;
@@ -3141,7 +3173,7 @@ static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator) {
   SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
   SSortedMergeOperatorInfo* pInfo = pOperator->info;
   if (pOperator->status == OP_RES_TO_RETURN) {
-    return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, NULL);
+    return getSortedMergeBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, NULL, pInfo);
   }
 
   int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;

source/libs/executor/src/scanoperator.c

@@ -846,7 +846,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
   size_t total = taosArrayGetSize(pInfo->pBlockLists);
   if (pInfo->blockType == STREAM_DATA_TYPE_SSDATA_BLOCK) {
     if (pInfo->validBlockIndex >= total) {
-      doClearBufferedBlocks(pInfo);
+      /*doClearBufferedBlocks(pInfo);*/
       pOperator->status = OP_EXEC_DONE;
       return NULL;
     }

source/libs/executor/src/sortoperator.c

@@ -42,6 +42,8 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR
 
   pInfo->pSortInfo = pSortInfo;
   pInfo->pColMatchInfo = pColMatchColInfo;
+  pInfo->hasGroupId = false;
+  pInfo->prefetchedTuple = NULL;
   pOperator->name = "SortOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT;
   pOperator->blocking = true;
@@ -81,8 +83,8 @@ void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
   pBlock->info.rows += 1;
 }
 
-SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity,
+SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo,
-                                SArray* pColMatchInfo) {
+                                SSortOperatorInfo* pInfo) {
   blockDataCleanup(pDataBlock);
 
   SSDataBlock* p = tsortGetSortedDataBlock(pHandle);
@@ -93,14 +95,33 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i
   blockDataEnsureCapacity(p, capacity);
 
   while (1) {
-    STupleHandle* pTupleHandle = tsortNextTuple(pHandle);
+    STupleHandle* pTupleHandle = NULL;
+    if (pInfo->prefetchedTuple == NULL) {
+      pTupleHandle = tsortNextTuple(pHandle);
+    } else {
+      pTupleHandle = pInfo->prefetchedTuple;
+      pInfo->groupId = tsortGetGroupId(pTupleHandle);
+      pInfo->prefetchedTuple = NULL;
+    }
+
     if (pTupleHandle == NULL) {
       break;
     }
 
+    uint64_t tupleGroupId = tsortGetGroupId(pTupleHandle);
+    if (!pInfo->hasGroupId) {
+      pInfo->groupId = tupleGroupId;
+      pInfo->hasGroupId = true;
       appendOneRowToDataBlock(p, pTupleHandle);
+    } else if (pInfo->groupId == tupleGroupId) {
+      appendOneRowToDataBlock(p, pTupleHandle);
+    } else {
+      pInfo->prefetchedTuple = pTupleHandle;
+      break;
+    }
+
     if (p->info.rows >= capacity) {
-      return pDataBlock;
+      break;
     }
   }
 
@@ -117,6 +138,7 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i
 
     pDataBlock->info.rows = p->info.rows;
     pDataBlock->info.capacity = p->info.rows;
+    pDataBlock->info.groupId = pInfo->groupId;
   }
 
   blockDataDestroy(p);
@@ -188,8 +210,8 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) {
     longjmp(pTaskInfo->env, code);
   }
 
-  SSDataBlock* pBlock =
+  SSDataBlock* pBlock = getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity,
-      getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo);
+                                           pInfo->pColMatchInfo, pInfo);
 
   if (pBlock != NULL) {
     pOperator->resultInfo.totalRows += pBlock->info.rows;
@@ -285,12 +307,12 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
   blockDataEnsureCapacity(p, capacity);
 
   while (1) {
-
     STupleHandle* pTupleHandle = NULL;
     if (pInfo->prefetchedTuple == NULL) {
       pTupleHandle = tsortNextTuple(pHandle);
     } else {
       pTupleHandle = pInfo->prefetchedTuple;
+      pInfo->groupId = tsortGetGroupId(pTupleHandle);
       pInfo->prefetchedTuple = NULL;
     }
 
@@ -307,14 +329,12 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
       appendOneRowToDataBlock(p, pTupleHandle);
     } else {
       pInfo->prefetchedTuple = pTupleHandle;
-      pInfo->groupId = tupleGroupId;
       break;
     }
 
     if (p->info.rows >= capacity) {
       break;
     }
-
   }
 
   if (p->info.rows > 0) {
@@ -330,13 +350,13 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
 
     pDataBlock->info.rows = p->info.rows;
     pDataBlock->info.capacity = p->info.rows;
+    pDataBlock->info.groupId = pInfo->groupId;
   }
 
   blockDataDestroy(p);
   return (pDataBlock->info.rows > 0) ? pDataBlock : NULL;
 }
 
-
 SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
   if (pOperator->status == OP_EXEC_DONE) {
     return NULL;
@@ -350,12 +370,8 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
     longjmp(pTaskInfo->env, code);
   }
 
-  SSDataBlock* pBlock =
+  SSDataBlock* pBlock = getMultiwaySortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes,
-      getMultiwaySortedBlockData(pInfo->pSortHandle,
+                                                   pOperator->resultInfo.capacity, pInfo->pColMatchInfo, pInfo);
-                         pInfo->binfo.pRes,
-                         pOperator->resultInfo.capacity,
-                         pInfo->pColMatchInfo,
-                                 pInfo);
 
   if (pBlock != NULL) {
     pOperator->resultInfo.totalRows += pBlock->info.rows;
@@ -386,9 +402,9 @@ int32_t getMultiwaySortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrEx
   return TSDB_CODE_SUCCESS;
 }
 
-SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams, SSDataBlock* pInputBlock,
+SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams,
-                                                   SSDataBlock* pResBlock, SArray* pSortInfo, SArray* pColMatchColInfo,
+                                                   SSDataBlock* pInputBlock, SSDataBlock* pResBlock, SArray* pSortInfo,
-                                                   SExecTaskInfo* pTaskInfo) {
+                                                   SArray* pColMatchColInfo, SExecTaskInfo* pTaskInfo) {
   SMultiwaySortMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMultiwaySortMergeOperatorInfo));
   SOperatorInfo*                  pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   int32_t                         rowSize = pResBlock->info.rowSize;
@@ -412,7 +428,8 @@ SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams,
 
   pInfo->bufPageSize = rowSize < 1024 ? 1024 : rowSize * 2;
   pInfo->sortBufSize = pInfo->bufPageSize * 16;
-
+  pInfo->hasGroupId = false;
+  pInfo->prefetchedTuple = NULL;
   pOperator->pTaskInfo = pTaskInfo;
   pOperator->fpSet =
       createOperatorFpSet(doOpenMultiwaySortMergeOperator, doMultiwaySortMerge, NULL, NULL,

source/libs/executor/src/timewindowoperator.c

@@ -1227,7 +1227,10 @@ void doClearWindow(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, char* pData, int
   SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId);
   SResultRowPosition* p1 =
       (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
-  ASSERT(p1);
+  if (!p1) {
+    // window has been closed
+    return;
+  }
   doClearWindowImpl(p1, pSup->pResultBuf, pBinfo, numOfOutput);
 }
 
@@ -2202,12 +2205,12 @@ void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) {
   }
 }
 
-int32_t initBiasicInfo(SOptrBasicInfo* pBasicInfo, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock,
+int32_t initBiasicInfo(SOptrBasicInfo* pBasicInfo, SExprInfo* pExprInfo,
-                       SDiskbasedBuf* pResultBuf) {
+    int32_t numOfCols, SSDataBlock* pResultBlock) {
   pBasicInfo->pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pBasicInfo->rowCellInfoOffset);
   pBasicInfo->pRes = pResultBlock;
   for (int32_t i = 0; i < numOfCols; ++i) {
-    pBasicInfo->pCtx[i].pBuf = pResultBuf;
+    pBasicInfo->pCtx[i].pBuf = NULL;
   }
   return TSDB_CODE_SUCCESS;
 }
@@ -2237,16 +2240,15 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SEx
 
   initResultSizeInfo(pOperator, 4096);
 
-  code = initSessionAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo");
+  code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock);
-  if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
-  }
-
-  code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock, pInfo->streamAggSup.pResultBuf);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
   pInfo->streamAggSup.resultRowSize = getResultRowSize(pInfo->binfo.pCtx, numOfCols);
+  code = initSessionAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo");
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
 
   pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
   if (pInfo->pDummyCtx == NULL) {
@@ -3101,6 +3103,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
   SSDataBlock*                 pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
   int32_t                      tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;
   SColumnNode*                 pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr;
+  int32_t                      code = TSDB_CODE_OUT_OF_MEMORY;
 
   SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo));
   SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
@@ -3121,17 +3124,18 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
       .winMap = NULL,
   };
   initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
-  int32_t code = initStateAggSupporter(&pInfo->streamAggSup, "StreamStateAggOperatorInfo");
-  if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
-  }
 
-  code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock, pInfo->streamAggSup.pResultBuf);
+  code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
 
   pInfo->streamAggSup.resultRowSize = getResultRowSize(pInfo->binfo.pCtx, numOfCols);
+  code = initStateAggSupporter(&pInfo->streamAggSup, "StreamStateAggOperatorInfo");
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
+
   pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
   if (pInfo->pDummyCtx == NULL) {
     goto _error;

source/libs/executor/src/tsort.c

@@ -144,7 +144,9 @@ static int32_t doAddNewExternalMemSource(SDiskbasedBuf *pBuf, SArray* pAllSource
   (*sourceId) += 1;
 
   int32_t rowSize = blockDataGetSerialRowSize(pSource->src.pBlock);
-  int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(pBlock))/rowSize;  // The value of numOfRows must be greater than 0, which is guaranteed by the previous memory allocation
+
+  // The value of numOfRows must be greater than 0, which is guaranteed by the previous memory allocation
+  int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(pBlock->info.numOfCols))/rowSize;
   ASSERT(numOfRows > 0);
   return blockDataEnsureCapacity(pSource->src.pBlock, numOfRows);
 }
@@ -359,6 +361,11 @@ int32_t msortComparFn(const void *pLeft, const void *pRight, void *param) {
   SSDataBlock* pLeftBlock = pLeftSource->src.pBlock;
   SSDataBlock* pRightBlock = pRightSource->src.pBlock;
 
+  // first sort by block groupId
+  if (pLeftBlock->info.groupId != pRightBlock->info.groupId) {
+    return pLeftBlock->info.groupId < pRightBlock->info.groupId ? -1 : 1;
+  }
+
   for(int32_t i = 0; i < pInfo->size; ++i) {
     SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i);
     SColumnInfoData* pLeftColInfoData = TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->slotId);
@@ -522,15 +529,23 @@ static int32_t createInitialSources(SSortHandle* pHandle) {
     SSortSource* source = taosArrayGetP(pHandle->pOrderedSource, 0);
     taosArrayClear(pHandle->pOrderedSource);
 
+    bool hasGroupId = false;
+    SSDataBlock* prefetchedDataBlock = NULL;
+
     while (1) {
-      SSDataBlock* pBlock = pHandle->fetchfp(source->param);
+      SSDataBlock* pBlock = NULL;
+      if (prefetchedDataBlock == NULL) {
+        pBlock = pHandle->fetchfp(source->param);
+      } else {
+        pBlock = prefetchedDataBlock;
+        prefetchedDataBlock = NULL;
+      }
+
       if (pBlock == NULL) {
         break;
       }
 
-      if (pHandle->pDataBlock == NULL) {
+      if (!hasGroupId) {
-        pHandle->pDataBlock = createOneDataBlock(pBlock, false);
-
         // calculate the buffer pages according to the total available buffers.
         int32_t rowSize = blockDataGetRowSize(pBlock);
         if (rowSize * 4 > 4096) {
@@ -542,13 +557,16 @@ static int32_t createInitialSources(SSortHandle* pHandle) {
         // todo!!
         pHandle->numOfPages = 1024;
         sortBufSize = pHandle->numOfPages * pHandle->pageSize;
+
+        hasGroupId = true;
+        pHandle->pDataBlock = createOneDataBlock(pBlock, false);
       }
 
+      if (pHandle->pDataBlock->info.groupId == pBlock->info.groupId) {
         // perform the scalar function calculation before apply the sort
         if (pHandle->beforeFp != NULL) {
           pHandle->beforeFp(pBlock, pHandle->param);
         }
-
         // todo relocate the columns
         int32_t code = blockDataMerge(pHandle->pDataBlock, pBlock);
         if (code != 0) {
@@ -566,6 +584,10 @@ static int32_t createInitialSources(SSortHandle* pHandle) {
 
           doAddToBuf(pHandle->pDataBlock, pHandle);
         }
+      } else {
+        prefetchedDataBlock = pBlock;
+        pHandle->pDataBlock = createOneDataBlock(pBlock, false);
+      }
     }
 
     if (pHandle->pDataBlock != NULL && pHandle->pDataBlock->info.rows > 0) {

source/libs/function/inc/builtinsimpl.h

@@ -78,13 +78,14 @@ bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultI
 int32_t percentileFunction(SqlFunctionCtx *pCtx);
 int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 
-int32_t getApercentileMaxSize();
 bool getApercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool apercentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t apercentileFunction(SqlFunctionCtx *pCtx);
 int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx);
 int32_t apercentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 int32_t apercentilePartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
+int32_t getApercentileMaxSize();
 
 bool getDiffFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool diffFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo);
@@ -102,13 +103,13 @@ int32_t topFunction(SqlFunctionCtx *pCtx);
 int32_t bottomFunction(SqlFunctionCtx *pCtx);
 int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 
-int32_t getSpreadInfoSize();
 bool getSpreadFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool spreadFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t spreadFunction(SqlFunctionCtx* pCtx);
 int32_t spreadFunctionMerge(SqlFunctionCtx* pCtx);
 int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 int32_t spreadPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+int32_t getSpreadInfoSize();
 
 bool getElapsedFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool elapsedFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
@@ -118,7 +119,10 @@ int32_t elapsedFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 bool getHistogramFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t histogramFunction(SqlFunctionCtx* pCtx);
+int32_t histogramFunctionMerge(SqlFunctionCtx* pCtx);
 int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+int32_t histogramPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+int32_t getHistogramInfoSize();
 
 bool getHLLFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 int32_t hllFunction(SqlFunctionCtx* pCtx);

source/libs/function/src/builtins.c

@@ -503,6 +503,58 @@ static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t l
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t translateHistogramImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) {
+  int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
+  if (isPartial) {
+    if (4 != numOfParams) {
+      return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+
+    uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+    if (!IS_NUMERIC_TYPE(colType)) {
+      return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+
+    // param1 ~ param3
+    for (int32_t i = 1; i < numOfParams; ++i) {
+      SNode* pParamNode = nodesListGetNode(pFunc->pParameterList, i);
+      if (QUERY_NODE_VALUE != nodeType(pParamNode)) {
+        return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+      }
+
+      SValueNode* pValue = (SValueNode*)pParamNode;
+
+      pValue->notReserved = true;
+    }
+
+    if (((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type != TSDB_DATA_TYPE_BINARY ||
+        ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type != TSDB_DATA_TYPE_BINARY ||
+        ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 3))->resType.type != TSDB_DATA_TYPE_BIGINT) {
+      return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+
+    pFunc->node.resType = (SDataType){.bytes = getHistogramInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
+  } else {
+    if (1 != numOfParams) {
+      return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+
+    if (((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type != TSDB_DATA_TYPE_BINARY) {
+      return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+
+    pFunc->node.resType = (SDataType){.bytes = 512, .type = TSDB_DATA_TYPE_BINARY};
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t translateHistogramPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  return translateHistogramImpl(pFunc, pErrBuf, len, true);
+}
+static int32_t translateHistogramMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  return translateHistogramImpl(pFunc, pErrBuf, len, false);
+}
+
 static int32_t translateHLL(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   if (1 != LIST_LENGTH(pFunc->pParameterList)) {
     return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
@@ -1241,7 +1293,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getPercentileFuncEnv,
     .initFunc     = percentileFunctionSetup,
     .processFunc  = percentileFunction,
-    .finalizeFunc = percentileFinalize
+    .finalizeFunc = percentileFinalize,
+    .invertFunc   = NULL,
+    .combineFunc  = NULL,
   },
   {
     .name = "apercentile",
@@ -1252,6 +1306,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .initFunc     = apercentileFunctionSetup,
     .processFunc  = apercentileFunction,
     .finalizeFunc = apercentileFinalize,
+    .combineFunc  = apercentileCombine,
     .pPartialFunc = "_apercentile_partial",
     .pMergeFunc   = "_apercentile_merge"
   },
@@ -1391,6 +1446,28 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getHistogramFuncEnv,
     .initFunc     = histogramFunctionSetup,
     .processFunc  = histogramFunction,
+    .finalizeFunc = histogramFinalize,
+    .pPartialFunc = "_histogram_partial",
+    .pMergeFunc   = "_histogram_merge"
+  },
+  {
+    .name = "_histogram_partial",
+    .type = FUNCTION_TYPE_HISTOGRAM_PARTIAL,
+    .classification = FUNC_MGT_AGG_FUNC,
+    .translateFunc = translateHistogramPartial,
+    .getEnvFunc   = getHistogramFuncEnv,
+    .initFunc     = histogramFunctionSetup,
+    .processFunc  = histogramFunction,
+    .finalizeFunc = histogramPartialFinalize
+  },
+  {
+    .name = "_histogram_merge",
+    .type = FUNCTION_TYPE_HISTOGRAM_MERGE,
+    .classification = FUNC_MGT_AGG_FUNC,
+    .translateFunc = translateHistogramMerge,
+    .getEnvFunc   = getHistogramFuncEnv,
+    .initFunc     = functionSetup,
+    .processFunc  = histogramFunctionMerge,
     .finalizeFunc = histogramFinalize
   },
   {

source/libs/function/src/builtinsimpl.c

@@ -2187,9 +2187,7 @@ int32_t apercentilePartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
   SAPercentileInfo*       pInfo = (SAPercentileInfo*)GET_ROWCELL_INTERBUF(pResInfo);
 
-  int32_t bytesHist   = (int32_t)(sizeof(SAPercentileInfo) + sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1));
+  int32_t resultBytes = getApercentileMaxSize();
-  int32_t bytesDigest = (int32_t)(sizeof(SAPercentileInfo) + TDIGEST_SIZE(COMPRESSION));
-  int32_t resultBytes = TMAX(bytesHist, bytesDigest);
   char *res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char));
 
   if (pInfo->algo == APERCT_ALGO_TDIGEST) {
@@ -2217,6 +2215,26 @@ int32_t apercentilePartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   return pResInfo->numOfRes;
 }
 
+int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) {
+  SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx);
+  SAPercentileInfo*    pDBuf = GET_ROWCELL_INTERBUF(pDResInfo);
+  int32_t type = pDestCtx->input.pData[0]->info.type;
+
+  SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx);
+  SAPercentileInfo*    pSBuf = GET_ROWCELL_INTERBUF(pSResInfo);
+  ASSERT(pDBuf->algo == pSBuf->algo);
+  if (pDBuf->algo == APERCT_ALGO_TDIGEST) {
+    tdigestMerge(pDBuf->pTDigest, pSBuf->pTDigest);
+  } else {
+    SHistogramInfo* pTmp = tHistogramMerge(pDBuf->pHisto, pSBuf->pHisto, MAX_HISTOGRAM_BIN);
+    memcpy(pDBuf->pHisto, pTmp, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1));
+    pDBuf->pHisto->elems = (SHistBin*) ((char *)pDBuf->pHisto + sizeof(SHistogramInfo));
+    tHistogramDestroy(&pTmp);
+  }
+  pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes);
+  return TSDB_CODE_SUCCESS;
+}
+
 bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
   SColumnNode* pNode = nodesListGetNode(pFunc->pParameterList, 0);
   pEnv->calcMemSize = pNode->node.resType.bytes + sizeof(int64_t);
@@ -3118,6 +3136,10 @@ int32_t elapsedFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   return functionFinalize(pCtx, pBlock);
 }
 
+int32_t getHistogramInfoSize() {
+  return (int32_t)sizeof(SHistoFuncInfo) + HISTOGRAM_MAX_BINS_NUM * sizeof(SHistoFuncBin);
+}
+
 bool getHistogramFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
   pEnv->calcMemSize = sizeof(SHistoFuncInfo) + HISTOGRAM_MAX_BINS_NUM * sizeof(SHistoFuncBin);
   return true;
@@ -3328,6 +3350,30 @@ int32_t histogramFunction(SqlFunctionCtx *pCtx) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t histogramFunctionMerge(SqlFunctionCtx *pCtx) {
+  SInputColumnInfoData* pInput = &pCtx->input;
+  SColumnInfoData* pCol = pInput->pData[0];
+  ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY);
+
+  SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+
+  int32_t start = pInput->startRowIndex;
+  char* data = colDataGetData(pCol, start);
+  SHistoFuncInfo* pInputInfo = (SHistoFuncInfo *)varDataVal(data);
+
+  pInfo->normalized = pInputInfo->normalized;
+  pInfo->numOfBins  = pInputInfo->numOfBins;
+  pInfo->totalCount += pInputInfo->totalCount;
+  for (int32_t k = 0; k < pInfo->numOfBins; ++k) {
+    pInfo->bins[k].lower = pInputInfo->bins[k].lower;
+    pInfo->bins[k].upper = pInputInfo->bins[k].upper;
+    pInfo->bins[k].count += pInputInfo->bins[k].count;
+  }
+
+  SET_VAL(GET_RES_INFO(pCtx), pInfo->numOfBins, pInfo->numOfBins);
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
   SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
@@ -3364,6 +3410,24 @@ int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   return pResInfo->numOfRes;
 }
 
+int32_t histogramPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
+  SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
+  SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+  int32_t resultBytes = getHistogramInfoSize();
+  char *res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char));
+
+  memcpy(varDataVal(res), pInfo, resultBytes);
+  varDataSetLen(res, resultBytes);
+
+  int32_t          slotId = pCtx->pExpr->base.resSchema.slotId;
+  SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
+
+  colDataAppend(pCol, pBlock->info.rows, res, false);
+
+  taosMemoryFree(res);
+  return 1;
+}
+
 bool getHLLFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
   pEnv->calcMemSize = sizeof(SHLLInfo);
   return true;

source/libs/index/inc/indexCache.h

@@ -70,7 +70,7 @@ void indexCacheBroadcast(void* cache);
 void indexCacheWait(void* cache);
 
 Iterate* indexCacheIteratorCreate(IndexCache* cache);
-void     indexCacheIteratorDestroy(Iterate* iiter);
+void     idxCacheIteratorDestroy(Iterate* iiter);
 
 int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid);
 
@@ -82,7 +82,7 @@ void indexCacheUnRef(IndexCache* cache);
 
 void indexCacheDebug(IndexCache* cache);
 
-void indexCacheDestroyImm(IndexCache* cache);
+void idxCacheDestroyImm(IndexCache* cache);
 #ifdef __cplusplus
 }
 #endif

source/libs/index/inc/indexComm.h

@@ -26,9 +26,9 @@ extern "C" {
 extern char JSON_COLUMN[];
 extern char JSON_VALUE_DELIM;
 
-char* indexPackJsonData(SIndexTerm* itm);
+char* idxPackJsonData(SIndexTerm* itm);
-char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip);
+char* idxPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip);
-char* indexPackJsonDataPrefixNoType(SIndexTerm* itm, int32_t* skip);
+char* idxPackJsonDataPrefixNoType(SIndexTerm* itm, int32_t* skip);
 
 typedef enum { MATCH, CONTINUE, BREAK } TExeCond;
 
@@ -40,13 +40,14 @@ TExeCond      tDoCompare(__compar_fn_t func, int8_t cmpType, void* a, void* b);
 
 _cache_range_compare indexGetCompare(RangeType ty);
 
-int32_t indexConvertData(void* src, int8_t type, void** dst);
+int32_t idxConvertData(void* src, int8_t type, void** dst);
-int32_t indexConvertDataToStr(void* src, int8_t type, void** dst);
+int32_t idxConvertDataToStr(void* src, int8_t type, void** dst);
 
-int32_t indexGetDataByteLen(int8_t type);
+int32_t idxGetDataByteLen(int8_t type);
 
-char* indexInt2str(int64_t val, char* dst, int radix);
+char* idxInt2str(int64_t val, char* dst, int radix);
 
+int idxUidCompare(const void* a, const void* b);
 #ifdef __cplusplus
 }
 #endif

source/libs/index/inc/indexFst.h

@@ -31,7 +31,7 @@ extern "C" {
 
 typedef struct Fst     Fst;
 typedef struct FstNode FstNode;
-typedef struct StreamWithState StreamWithState;
+typedef struct FStmSt  FStmSt;
 
 typedef enum { Included, Excluded, Unbounded } FstBound;
 
@@ -40,12 +40,12 @@ typedef struct FstBoundWithData {
   FstBound type;
 } FstBoundWithData;
 
-typedef struct FstStreamBuilder {
+typedef struct FStmBuilder {
   Fst*              fst;
-  AutomationCtx*    aut;
+  FAutoCtx*         aut;
   FstBoundWithData* min;
   FstBoundWithData* max;
-} FstStreamBuilder, FstStreamWithStateBuilder;
+} FStmBuilder, FStmStBuilder;
 
 typedef struct FstRange {
   uint64_t start;
@@ -273,11 +273,11 @@ FstNode*          fstGetRoot(Fst* fst);
 FstType      fstGetType(Fst* fst);
 CompiledAddr fstGetRootAddr(Fst* fst);
 Output       fstEmptyFinalOutput(Fst* fst, bool* null);
-FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx);
+FStmBuilder* fstSearch(Fst* fst, FAutoCtx* ctx);
 
-FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx);
+FStmStBuilder* fstSearchWithState(Fst* fst, FAutoCtx* ctx);
 // into stream to expand later
-StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb);
+FStmSt* stmBuilderIntoStm(FStmBuilder* sb);
 
 bool fstVerify(Fst* fst);
 
@@ -293,41 +293,40 @@ typedef struct StreamState {
 
 void streamStateDestroy(void* s);
 
-typedef struct StreamWithState {
+typedef struct FStmSt {
   Fst*              fst;
-  AutomationCtx*    aut;
+  FAutoCtx*         aut;
   SArray*           inp;
   FstOutput         emptyOutput;
   SArray*           stack;  // <StreamState>
   FstBoundWithData* endAt;
-} StreamWithState;
+} FStmSt;
 
-typedef struct StreamWithStateResult {
+typedef struct FStmStRslt {
   FstSlice  data;
   FstOutput out;
   void*     state;
-} StreamWithStateResult;
+} FStmStRslt;
 
-StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state);
+FStmStRslt* swsResultCreate(FstSlice* data, FstOutput fOut, void* state);
-void                   swsResultDestroy(StreamWithStateResult* result);
+void        swsResultDestroy(FStmStRslt* result);
 
 typedef void* (*StreamCallback)(void*);
-StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min,
+FStmSt* stmStCreate(Fst* fst, FAutoCtx* automation, FstBoundWithData* min, FstBoundWithData* max);
-                                       FstBoundWithData* max);
 
-void streamWithStateDestroy(StreamWithState* sws);
+void stmStDestroy(FStmSt* sws);
 
-bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min);
+bool stmStSeekMin(FStmSt* sws, FstBoundWithData* min);
 
-StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallback callback);
+FStmStRslt* stmStNextWith(FStmSt* sws, StreamCallback callback);
 
-FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut);
+FStmBuilder* stmBuilderCreate(Fst* fst, FAutoCtx* aut);
 
-void fstStreamBuilderDestroy(FstStreamBuilder* b);
+void stmBuilderDestroy(FStmBuilder* b);
 
 // set up bound range
 // refator later:  to simple code by marco
-void fstStreamBuilderSetRange(FstStreamBuilder* b, FstSlice* val, RangeType type);
+void stmBuilderSetRange(FStmBuilder* b, FstSlice* val, RangeType type);
 
 #ifdef __cplusplus
 }

source/libs/index/inc/indexFstAutomation.h

@@ -22,24 +22,24 @@ extern "C" {
 #include "indexFstUtil.h"
 #include "indexInt.h"
 
-typedef struct AutomationCtx AutomationCtx;
+typedef struct FAutoCtx FAutoCtx;
 
 typedef enum AutomationType { AUTOMATION_ALWAYS, AUTOMATION_PREFIX, AUTMMATION_MATCH } AutomationType;
 
 typedef struct StartWith {
-  AutomationCtx* autoSelf;
+  FAutoCtx* autoSelf;
 } StartWith;
 
 typedef struct Complement {
-  AutomationCtx* autoSelf;
+  FAutoCtx* autoSelf;
 } Complement;
 
 // automation
-typedef struct AutomationCtx {
+typedef struct FAutoCtx {
   AutomationType type;
   void*          stdata;
   char*          data;
-} AutomationCtx;
+} FAutoCtx;
 
 typedef enum ValueType { FST_INT, FST_CHAR, FST_ARRAY } ValueType;
 typedef enum StartWithStateKind { Done, Running } StartWithStateKind;
@@ -60,16 +60,16 @@ StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv);
 void                 startWithStateValueDestroy(void* sv);
 
 typedef struct AutomationFunc {
-  void* (*start)(AutomationCtx* ctx);
+  void* (*start)(FAutoCtx* ctx);
-  bool (*isMatch)(AutomationCtx* ctx, void*);
+  bool (*isMatch)(FAutoCtx* ctx, void*);
-  bool (*canMatch)(AutomationCtx* ctx, void* data);
+  bool (*canMatch)(FAutoCtx* ctx, void* data);
-  bool (*willAlwaysMatch)(AutomationCtx* ctx, void* state);
+  bool (*willAlwaysMatch)(FAutoCtx* ctx, void* state);
-  void* (*accept)(AutomationCtx* ctx, void* state, uint8_t byte);
+  void* (*accept)(FAutoCtx* ctx, void* state, uint8_t byte);
-  void* (*acceptEof)(AutomationCtx* ct, void* state);
+  void* (*acceptEof)(FAutoCtx* ct, void* state);
 } AutomationFunc;
 
-AutomationCtx* automCtxCreate(void* data, AutomationType atype);
+FAutoCtx* automCtxCreate(void* data, AutomationType atype);
-void           automCtxDestroy(AutomationCtx* ctx);
+void      automCtxDestroy(FAutoCtx* ctx);
 
 extern AutomationFunc automFuncs[];
 #ifdef __cplusplus

source/libs/index/src/index.c

@@ -77,15 +77,15 @@ typedef struct SIdxColInfo {
 
 static TdThreadOnce isInit = PTHREAD_ONCE_INIT;
 // static void           indexInit();
-static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* term, SArray** result);
+static int idxTermSearch(SIndex* sIdx, SIndexTermQuery* term, SArray** result);
 
-static void indexInterResultsDestroy(SArray* results);
+static void idxInterRsltDestroy(SArray* results);
-static int  indexMergeFinalResults(SArray* in, EIndexOperatorType oType, SArray* out);
+static int  idxMergeFinalResults(SArray* in, EIndexOperatorType oType, SArray* out);
 
-static int indexGenTFile(SIndex* index, IndexCache* cache, SArray* batch);
+static int idxGenTFile(SIndex* index, IndexCache* cache, SArray* batch);
 
 // merge cache and tfile by opera type
-static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv, SIdxTRslt* helper);
+static void idxMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv, SIdxTRslt* helper);
 
 // static int32_t indexSerialTermKey(SIndexTerm* itm, char* buf);
 // int32_t        indexSerialKey(ICacheKey* key, char* buf);
@@ -221,11 +221,11 @@ int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result
   for (size_t i = 0; i < nQuery; i++) {
     SIndexTermQuery* qterm = taosArrayGet(multiQuerys->query, i);
     SArray*          trslt = NULL;
-    indexTermSearch(index, qterm, &trslt);
+    idxTermSearch(index, qterm, &trslt);
     taosArrayPush(iRslts, (void*)&trslt);
   }
-  indexMergeFinalResults(iRslts, opera, result);
+  idxMergeFinalResults(iRslts, opera, result);
-  indexInterResultsDestroy(iRslts);
+  idxInterRsltDestroy(iRslts);
   return 0;
 }
 
@@ -289,7 +289,7 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy
   tm->nColName = nColName;
 
   char*   buf = NULL;
-  int32_t len = indexConvertDataToStr((void*)colVal, INDEX_TYPE_GET_TYPE(colType), (void**)&buf);
+  int32_t len = idxConvertDataToStr((void*)colVal, INDEX_TYPE_GET_TYPE(colType), (void**)&buf);
   assert(len != -1);
 
   tm->colVal = buf;
@@ -319,7 +319,7 @@ void indexMultiTermDestroy(SIndexMultiTerm* terms) {
   taosArrayDestroy(terms);
 }
 
-static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result) {
+static int idxTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result) {
   SIndexTerm* term = query->term;
   const char* colName = term->colName;
   int32_t     nColName = term->nColName;
@@ -374,7 +374,7 @@ END:
   idxTRsltDestroy(tr);
   return -1;
 }
-static void indexInterResultsDestroy(SArray* results) {
+static void idxInterRsltDestroy(SArray* results) {
   if (results == NULL) {
     return;
   }
@@ -387,7 +387,7 @@ static void indexInterResultsDestroy(SArray* results) {
   taosArrayDestroy(results);
 }
 
-static int indexMergeFinalResults(SArray* in, EIndexOperatorType oType, SArray* out) {
+static int idxMergeFinalResults(SArray* in, EIndexOperatorType oType, SArray* out) {
   // refactor, merge interResults into fResults by oType
   for (int i = 0; i < taosArrayGetSize(in); i--) {
     SArray* t = taosArrayGetP(in, i);
@@ -407,7 +407,7 @@ static int indexMergeFinalResults(SArray* in, EIndexOperatorType oType, SArray*
   return 0;
 }
 
-static void indexMayMergeTempToFinalResult(SArray* result, TFileValue* tfv, SIdxTRslt* tr) {
+static void idxMayMergeTempToFinalRslt(SArray* result, TFileValue* tfv, SIdxTRslt* tr) {
   int32_t sz = taosArrayGetSize(result);
   if (sz > 0) {
     TFileValue* lv = taosArrayGetP(result, sz - 1);
@@ -427,11 +427,11 @@ static void indexMayMergeTempToFinalResult(SArray* result, TFileValue* tfv, SIdx
     taosArrayPush(result, &tfv);
   }
 }
-static void indexMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateValue* tv, SIdxTRslt* tr) {
+static void idxMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateValue* tv, SIdxTRslt* tr) {
   char*       colVal = (cv != NULL) ? cv->colVal : tv->colVal;
   TFileValue* tfv = tfileValueCreate(colVal);
 
-  indexMayMergeTempToFinalResult(result, tfv, tr);
+  idxMayMergeTempToFinalRslt(result, tfv, tr);
 
   if (cv != NULL) {
     uint64_t id = *(uint64_t*)taosArrayGet(cv->val, 0);
@@ -446,7 +446,7 @@ static void indexMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateVal
     taosArrayAddAll(tr->total, tv->val);
   }
 }
-static void indexDestroyFinalResult(SArray* result) {
+static void idxDestroyFinalRslt(SArray* result) {
   int32_t sz = result ? taosArrayGetSize(result) : 0;
   for (size_t i = 0; i < sz; i++) {
     TFileValue* tv = taosArrayGetP(result, i);
@@ -475,7 +475,7 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) {
   Iterate* cacheIter = indexCacheIteratorCreate(pCache);
   if (cacheIter == NULL) {
     indexError("%p immtable is empty, ignore merge opera", pCache);
-    indexCacheDestroyImm(pCache);
+    idxCacheDestroyImm(pCache);
     tfileReaderUnRef(pReader);
     atomic_store_32(&pCache->merging, 0);
     if (quit) {
@@ -509,26 +509,26 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) {
       comp = 1;
     }
     if (comp == 0) {
-      indexMergeCacheAndTFile(result, cv, tv, tr);
+      idxMergeCacheAndTFile(result, cv, tv, tr);
       cn = cacheIter->next(cacheIter);
       tn = tfileIter->next(tfileIter);
     } else if (comp < 0) {
-      indexMergeCacheAndTFile(result, cv, NULL, tr);
+      idxMergeCacheAndTFile(result, cv, NULL, tr);
       cn = cacheIter->next(cacheIter);
     } else {
-      indexMergeCacheAndTFile(result, NULL, tv, tr);
+      idxMergeCacheAndTFile(result, NULL, tv, tr);
       tn = tfileIter->next(tfileIter);
     }
   }
-  indexMayMergeTempToFinalResult(result, NULL, tr);
+  idxMayMergeTempToFinalRslt(result, NULL, tr);
   idxTRsltDestroy(tr);
 
-  int ret = indexGenTFile(sIdx, pCache, result);
+  int ret = idxGenTFile(sIdx, pCache, result);
-  indexDestroyFinalResult(result);
+  idxDestroyFinalRslt(result);
 
-  indexCacheDestroyImm(pCache);
+  idxCacheDestroyImm(pCache);
 
-  indexCacheIteratorDestroy(cacheIter);
+  idxCacheIteratorDestroy(cacheIter);
   tfileIteratorDestroy(tfileIter);
 
   tfileReaderUnRef(pReader);
@@ -578,7 +578,7 @@ static int64_t indexGetAvaialbleVer(SIndex* sIdx, IndexCache* cache) {
   tfileReaderUnRef(rd);
   return ver;
 }
-static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) {
+static int idxGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) {
   int64_t version = indexGetAvaialbleVer(sIdx, cache);
   indexInfo("file name version: %" PRId64 "", version);
   uint8_t colType = cache->type;
@@ -625,7 +625,7 @@ int32_t indexSerialCacheKey(ICacheKey* key, char* buf) {
 
   char* p = buf;
   char  tbuf[65] = {0};
-  indexInt2str((int64_t)key->suid, tbuf, 0);
+  idxInt2str((int64_t)key->suid, tbuf, 0);
 
   SERIALIZE_STR_VAR_TO_BUF(buf, tbuf, strlen(tbuf));
   SERIALIZE_VAR_TO_BUF(buf, '_', char);

source/libs/index/src/indexCache.c

@@ -26,15 +26,15 @@
 #define MEM_SIGNAL_QUIT    MEM_THRESHOLD * 20
 #define MEM_ESTIMATE_RADIO 1.5
 
-static void indexMemRef(MemTable* tbl);
+static void idxMemRef(MemTable* tbl);
-static void indexMemUnRef(MemTable* tbl);
+static void idxMemUnRef(MemTable* tbl);
 
-static void    indexCacheTermDestroy(CacheTerm* ct);
+static void    idxCacheTermDestroy(CacheTerm* ct);
-static int32_t indexCacheTermCompare(const void* l, const void* r);
+static int32_t idxCacheTermCompare(const void* l, const void* r);
-static int32_t indexCacheJsonTermCompare(const void* l, const void* r);
+static int32_t idxCacheJsonTermCompare(const void* l, const void* r);
-static char*   indexCacheTermGet(const void* pData);
+static char*   idxCacheTermGet(const void* pData);
 
-static MemTable* indexInternalCacheCreate(int8_t type);
+static MemTable* idxInternalCacheCreate(int8_t type);
 
 static int32_t cacheSearchTerm(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
 static int32_t cacheSearchPrefix(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
@@ -82,7 +82,7 @@ static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTRslt* tr, STe
   pCt->colVal = term->colVal;
   pCt->version = atomic_load_64(&pCache->version);
 
-  char* key = indexCacheTermGet(pCt);
+  char* key = idxCacheTermGet(pCt);
 
   SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
   while (tSkipListIterNext(iter)) {
@@ -134,7 +134,7 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTRslt*
   pCt->colType = term->colType;
   pCt->version = atomic_load_64(&pCache->version);
 
-  char* key = indexCacheTermGet(pCt);
+  char* key = idxCacheTermGet(pCt);
 
   SSkipListIterator* iter = tSkipListCreateIter(mem->mem);
   while (tSkipListIterNext(iter)) {
@@ -188,10 +188,10 @@ static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr
 
   char* exBuf = NULL;
   if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) {
-    exBuf = indexPackJsonData(term);
+    exBuf = idxPackJsonData(term);
     pCt->colVal = exBuf;
   }
-  char* key = indexCacheTermGet(pCt);
+  char* key = idxCacheTermGet(pCt);
 
   SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
   while (tSkipListIterNext(iter)) {
@@ -269,20 +269,19 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR
   int8_t dType = INDEX_TYPE_GET_TYPE(term->colType);
   int    skip = 0;
   char*  exBuf = NULL;
-
   if (type == CONTAINS) {
     SIndexTerm tm = {.suid = term->suid,
                      .operType = term->operType,
                      .colType = term->colType,
                      .colName = term->colVal,
                      .nColName = term->nColVal};
-    exBuf = indexPackJsonDataPrefixNoType(&tm, &skip);
+    exBuf = idxPackJsonDataPrefixNoType(&tm, &skip);
     pCt->colVal = exBuf;
   } else {
-    exBuf = indexPackJsonDataPrefix(term, &skip);
+    exBuf = idxPackJsonDataPrefix(term, &skip);
     pCt->colVal = exBuf;
   }
-  char* key = indexCacheTermGet(pCt);
+  char* key = idxCacheTermGet(pCt);
 
   SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
   while (tSkipListIterNext(iter)) {
@@ -341,7 +340,7 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in
     return NULL;
   };
 
-  cache->mem = indexInternalCacheCreate(type);
+  cache->mem = idxInternalCacheCreate(type);
   cache->mem->pCache = cache;
   cache->colName = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? tstrdup(JSON_COLUMN) : tstrdup(colName);
   cache->type = type;
@@ -364,7 +363,7 @@ void indexCacheDebug(IndexCache* cache) {
 
   taosThreadMutexLock(&cache->mtx);
   tbl = cache->mem;
-  indexMemRef(tbl);
+  idxMemRef(tbl);
   taosThreadMutexUnlock(&cache->mtx);
 
   {
@@ -380,13 +379,13 @@ void indexCacheDebug(IndexCache* cache) {
     }
     tSkipListDestroyIter(iter);
 
-    indexMemUnRef(tbl);
+    idxMemUnRef(tbl);
   }
 
   {
     taosThreadMutexLock(&cache->mtx);
     tbl = cache->imm;
-    indexMemRef(tbl);
+    idxMemRef(tbl);
     taosThreadMutexUnlock(&cache->mtx);
     if (tbl != NULL) {
       SSkipList*         slt = tbl->mem;
@@ -402,7 +401,7 @@ void indexCacheDebug(IndexCache* cache) {
       tSkipListDestroyIter(iter);
     }
 
-    indexMemUnRef(tbl);
+    idxMemUnRef(tbl);
   }
 }
 
@@ -427,7 +426,7 @@ void indexCacheWait(void* cache) {
   IndexCache* pCache = cache;
   taosThreadCondWait(&pCache->finished, &pCache->mtx);
 }
-void indexCacheDestroyImm(IndexCache* cache) {
+void idxCacheDestroyImm(IndexCache* cache) {
   if (cache == NULL) {
     return;
   }
@@ -440,8 +439,8 @@ void indexCacheDestroyImm(IndexCache* cache) {
 
   taosThreadMutexUnlock(&cache->mtx);
 
-  indexMemUnRef(tbl);
+  idxMemUnRef(tbl);
-  indexMemUnRef(tbl);
+  idxMemUnRef(tbl);
 }
 void indexCacheDestroy(void* cache) {
   IndexCache* pCache = cache;
@@ -449,8 +448,8 @@ void indexCacheDestroy(void* cache) {
     return;
   }
 
-  indexMemUnRef(pCache->mem);
+  idxMemUnRef(pCache->mem);
-  indexMemUnRef(pCache->imm);
+  idxMemUnRef(pCache->imm);
   taosMemoryFree(pCache->colName);
 
   taosThreadMutexDestroy(&pCache->mtx);
@@ -471,7 +470,7 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) {
   }
   taosThreadMutexLock(&cache->mtx);
 
-  indexMemRef(cache->imm);
+  idxMemRef(cache->imm);
 
   MemTable* tbl = cache->imm;
   iiter->val.val = taosArrayInit(1, sizeof(uint64_t));
@@ -484,7 +483,7 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) {
 
   return iiter;
 }
-void indexCacheIteratorDestroy(Iterate* iter) {
+void idxCacheIteratorDestroy(Iterate* iter) {
   if (iter == NULL) {
     return;
   }
@@ -518,7 +517,7 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) {
 
       indexCacheRef(cache);
       cache->imm = cache->mem;
-      cache->mem = indexInternalCacheCreate(cache->type);
+      cache->mem = idxInternalCacheCreate(cache->type);
       cache->mem->pCache = cache;
       cache->occupiedMem = 0;
       if (quit == false) {
@@ -546,7 +545,7 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) {
   // set up key
   ct->colType = term->colType;
   if (hasJson) {
-    ct->colVal = indexPackJsonData(term);
+    ct->colVal = idxPackJsonData(term);
   } else {
     ct->colVal = (char*)taosMemoryCalloc(1, sizeof(char) * (term->nColVal + 1));
     memcpy(ct->colVal, term->colVal, term->nColVal);
@@ -562,9 +561,9 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) {
   pCache->occupiedMem += estimate;
   indexCacheMakeRoomForWrite(pCache);
   MemTable* tbl = pCache->mem;
-  indexMemRef(tbl);
+  idxMemRef(tbl);
   tSkipListPut(tbl->mem, (char*)ct);
-  indexMemUnRef(tbl);
+  idxMemUnRef(tbl);
 
   taosThreadMutexUnlock(&pCache->mtx);
 
@@ -615,8 +614,8 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* result, STe
   taosThreadMutexLock(&pCache->mtx);
   mem = pCache->mem;
   imm = pCache->imm;
-  indexMemRef(mem);
+  idxMemRef(mem);
-  indexMemRef(imm);
+  idxMemRef(imm);
   taosThreadMutexUnlock(&pCache->mtx);
 
   int ret = (mem && mem->mem) ? indexQueryMem(mem, query, result, s) : 0;
@@ -625,8 +624,8 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* result, STe
     ret = (imm && imm->mem) ? indexQueryMem(imm, query, result, s) : 0;
   }
 
-  indexMemUnRef(mem);
+  idxMemUnRef(mem);
-  indexMemUnRef(imm);
+  idxMemUnRef(imm);
   indexInfo("cache search, time cost %" PRIu64 "us", taosGetTimestampUs() - st);
 
   return ret;
@@ -649,14 +648,14 @@ void indexCacheUnRef(IndexCache* cache) {
   }
 }
 
-void indexMemRef(MemTable* tbl) {
+void idxMemRef(MemTable* tbl) {
   if (tbl == NULL) {
     return;
   }
   int ref = T_REF_INC(tbl);
   UNUSED(ref);
 }
-void indexMemUnRef(MemTable* tbl) {
+void idxMemUnRef(MemTable* tbl) {
   if (tbl == NULL) {
     return;
   }
@@ -668,18 +667,18 @@ void indexMemUnRef(MemTable* tbl) {
   }
 }
 
-static void indexCacheTermDestroy(CacheTerm* ct) {
+static void idxCacheTermDestroy(CacheTerm* ct) {
   if (ct == NULL) {
     return;
   }
   taosMemoryFree(ct->colVal);
   taosMemoryFree(ct);
 }
-static char* indexCacheTermGet(const void* pData) {
+static char* idxCacheTermGet(const void* pData) {
   CacheTerm* p = (CacheTerm*)pData;
   return (char*)p;
 }
-static int32_t indexCacheTermCompare(const void* l, const void* r) {
+static int32_t idxCacheTermCompare(const void* l, const void* r) {
   CacheTerm* lt = (CacheTerm*)l;
   CacheTerm* rt = (CacheTerm*)r;
   // compare colVal
@@ -700,7 +699,7 @@ static int indexFindCh(char* a, char c) {
   }
   return p - a;
 }
-static int indexCacheJsonTermCompareImpl(char* a, char* b) {
+static int idxCacheJsonTermCompareImpl(char* a, char* b) {
   // int alen = indexFindCh(a, '&');
   // int blen = indexFindCh(b, '&');
 
@@ -720,7 +719,7 @@ static int indexCacheJsonTermCompareImpl(char* a, char* b) {
   //}
   return 0;
 }
-static int32_t indexCacheJsonTermCompare(const void* l, const void* r) {
+static int32_t idxCacheJsonTermCompare(const void* l, const void* r) {
   CacheTerm* lt = (CacheTerm*)l;
   CacheTerm* rt = (CacheTerm*)r;
   // compare colVal
@@ -730,16 +729,15 @@ static int32_t indexCacheJsonTermCompare(const void* l, const void* r) {
   }
   return cmp;
 }
-static MemTable* indexInternalCacheCreate(int8_t type) {
+static MemTable* idxInternalCacheCreate(int8_t type) {
   int ttype = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? TSDB_DATA_TYPE_BINARY : TSDB_DATA_TYPE_BINARY;
   int32_t (*cmpFn)(const void* l, const void* r) =
-      INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? indexCacheJsonTermCompare : indexCacheTermCompare;
+      INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? idxCacheJsonTermCompare : idxCacheTermCompare;
 
   MemTable* tbl = taosMemoryCalloc(1, sizeof(MemTable));
-  indexMemRef(tbl);
+  idxMemRef(tbl);
   if (ttype == TSDB_DATA_TYPE_BINARY || ttype == TSDB_DATA_TYPE_NCHAR) {
-    tbl->mem =
+    tbl->mem = tSkipListCreate(MAX_SKIP_LIST_LEVEL, ttype, MAX_INDEX_KEY_LEN, cmpFn, SL_ALLOW_DUP_KEY, idxCacheTermGet);
-        tSkipListCreate(MAX_SKIP_LIST_LEVEL, ttype, MAX_INDEX_KEY_LEN, cmpFn, SL_ALLOW_DUP_KEY, indexCacheTermGet);
   }
   return tbl;
 }

source/libs/index/src/indexComm.c

@@ -48,7 +48,7 @@
 char JSON_COLUMN[] = "JSON";
 char JSON_VALUE_DELIM = '&';
 
-char* indexInt2str(int64_t val, char* dst, int radix) {
+char* idxInt2str(int64_t val, char* dst, int radix) {
   char     buffer[65] = {0};
   char*    p;
   int64_t  new_val;
@@ -207,7 +207,7 @@ static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = {
 
 _cache_range_compare indexGetCompare(RangeType ty) { return rangeCompare[ty]; }
 
-char* indexPackJsonData(SIndexTerm* itm) {
+char* idxPackJsonData(SIndexTerm* itm) {
   /*
    * |<-----colname---->|<-----dataType---->|<--------colVal---------->|
    * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->|
@@ -235,7 +235,7 @@ char* indexPackJsonData(SIndexTerm* itm) {
   return buf;
 }
 
-char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip) {
+char* idxPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip) {
   /*
    * |<-----colname---->|<-----dataType---->|<--------colVal---------->|
    * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->|
@@ -262,7 +262,7 @@ char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip) {
 
   return buf;
 }
-char* indexPackJsonDataPrefixNoType(SIndexTerm* itm, int32_t* skip) {
+char* idxPackJsonDataPrefixNoType(SIndexTerm* itm, int32_t* skip) {
   /*
    * |<-----colname---->|<-----dataType---->|<--------colVal---------->|
    * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->|
@@ -283,7 +283,12 @@ char* indexPackJsonDataPrefixNoType(SIndexTerm* itm, int32_t* skip) {
   return buf;
 }
 
-int32_t indexConvertData(void* src, int8_t type, void** dst) {
+int idxUidCompare(const void* a, const void* b) {
+  uint64_t l = *(uint64_t*)a;
+  uint64_t r = *(uint64_t*)b;
+  return l - r;
+}
+int32_t idxConvertData(void* src, int8_t type, void** dst) {
   int tlen = -1;
   switch (type) {
     case TSDB_DATA_TYPE_TIMESTAMP:
@@ -368,44 +373,44 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) {
   // indexMayFillNumbericData(*dst, tlen);
   return tlen;
 }
-int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) {
+int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) {
   int     tlen = tDataTypes[type].bytes;
   int32_t bufSize = 64;
   switch (type) {
     case TSDB_DATA_TYPE_TIMESTAMP:
       *dst = taosMemoryCalloc(1, bufSize + 1);
-      indexInt2str(*(int64_t*)src, *dst, -1);
+      idxInt2str(*(int64_t*)src, *dst, -1);
       tlen = strlen(*dst);
       break;
     case TSDB_DATA_TYPE_BOOL:
     case TSDB_DATA_TYPE_UTINYINT:
       *dst = taosMemoryCalloc(1, bufSize + 1);
-      indexInt2str(*(uint8_t*)src, *dst, 1);
+      idxInt2str(*(uint8_t*)src, *dst, 1);
       tlen = strlen(*dst);
       break;
     case TSDB_DATA_TYPE_TINYINT:
       *dst = taosMemoryCalloc(1, bufSize + 1);
-      indexInt2str(*(int8_t*)src, *dst, 1);
+      idxInt2str(*(int8_t*)src, *dst, 1);
       tlen = strlen(*dst);
       break;
     case TSDB_DATA_TYPE_SMALLINT:
       *dst = taosMemoryCalloc(1, bufSize + 1);
-      indexInt2str(*(int16_t*)src, *dst, -1);
+      idxInt2str(*(int16_t*)src, *dst, -1);
       tlen = strlen(*dst);
       break;
     case TSDB_DATA_TYPE_USMALLINT:
       *dst = taosMemoryCalloc(1, bufSize + 1);
-      indexInt2str(*(uint16_t*)src, *dst, -1);
+      idxInt2str(*(uint16_t*)src, *dst, -1);
       tlen = strlen(*dst);
       break;
     case TSDB_DATA_TYPE_INT:
       *dst = taosMemoryCalloc(1, bufSize + 1);
-      indexInt2str(*(int32_t*)src, *dst, -1);
+      idxInt2str(*(int32_t*)src, *dst, -1);
       tlen = strlen(*dst);
       break;
     case TSDB_DATA_TYPE_UINT:
       *dst = taosMemoryCalloc(1, bufSize + 1);
-      indexInt2str(*(uint32_t*)src, *dst, 1);
+      idxInt2str(*(uint32_t*)src, *dst, 1);
       tlen = strlen(*dst);
       break;
     case TSDB_DATA_TYPE_BIGINT:
@@ -415,7 +420,7 @@ int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) {
       break;
     case TSDB_DATA_TYPE_UBIGINT:
       *dst = taosMemoryCalloc(1, bufSize + 1);
-      indexInt2str(*(uint64_t*)src, *dst, 1);
+      idxInt2str(*(uint64_t*)src, *dst, 1);
       tlen = strlen(*dst);
     case TSDB_DATA_TYPE_FLOAT:
       *dst = taosMemoryCalloc(1, bufSize + 1);

source/libs/index/src/indexFilter.c

@@ -547,6 +547,8 @@ static int32_t sifExecLogic(SLogicConditionNode *node, SIFCtx *ctx, SIFParam *ou
       } else if (node->condType == LOGIC_COND_TYPE_NOT) {
         // taosArrayAddAll(output->result, params[m].result);
       }
+      taosArraySort(output->result, idxUidCompare);
+      taosArrayRemoveDuplicate(output->result, idxUidCompare, NULL);
     }
   } else {
     for (int32_t m = 0; m < node->pParameterList->length; m++) {

source/libs/index/src/indexFst.c

@@ -1087,19 +1087,19 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) {
   *out = tOut;
   return true;
 }
-FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx) {
+FStmBuilder* fstSearch(Fst* fst, FAutoCtx* ctx) {
   // refactor later
-  return fstStreamBuilderCreate(fst, ctx);
+  return stmBuilderCreate(fst, ctx);
 }
-StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb) {
+FStmSt* stmBuilderIntoStm(FStmBuilder* sb) {
   if (sb == NULL) {
     return NULL;
   }
-  return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max);
+  return stmStCreate(sb->fst, sb->aut, sb->min, sb->max);
 }
-FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx) {
+FStmStBuilder* fstSearchWithState(Fst* fst, FAutoCtx* ctx) {
   // refactor later
-  return fstStreamBuilderCreate(fst, ctx);
+  return stmBuilderCreate(fst, ctx);
 }
 
 FstNode* fstGetRoot(Fst* fst) {
@@ -1176,9 +1176,8 @@ bool fstBoundWithDataIsIncluded(FstBoundWithData* bound) { return bound->type ==
 
 void fstBoundDestroy(FstBoundWithData* bound) { taosMemoryFree(bound); }
 
-StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min,
+FStmSt* stmStCreate(Fst* fst, FAutoCtx* automation, FstBoundWithData* min, FstBoundWithData* max) {
-                                       FstBoundWithData* max) {
+  FStmSt* sws = taosMemoryCalloc(1, sizeof(FStmSt));
-  StreamWithState* sws = taosMemoryCalloc(1, sizeof(StreamWithState));
   if (sws == NULL) {
     return NULL;
   }
@@ -1192,11 +1191,11 @@ StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstB
 
   sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState));
   sws->endAt = max;
-  streamWithStateSeekMin(sws, min);
+  stmStSeekMin(sws, min);
 
   return sws;
 }
-void streamWithStateDestroy(StreamWithState* sws) {
+void stmStDestroy(FStmSt* sws) {
   if (sws == NULL) {
     return;
   }
@@ -1207,8 +1206,8 @@ void streamWithStateDestroy(StreamWithState* sws) {
   taosMemoryFree(sws);
 }
 
-bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) {
+bool stmStSeekMin(FStmSt* sws, FstBoundWithData* min) {
-  AutomationCtx* aut = sws->aut;
+  FAutoCtx* aut = sws->aut;
   if (fstBoundWithDataIsEmpty(min)) {
     if (fstBoundWithDataIsIncluded(min)) {
       sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null));
@@ -1301,8 +1300,8 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) {
 
   return false;
 }
-StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallback callback) {
+FStmStRslt* stmStNextWith(FStmSt* sws, StreamCallback callback) {
-  AutomationCtx* aut = sws->aut;
+  FAutoCtx* aut = sws->aut;
   FstOutput output = sws->emptyOutput;
   if (output.null == false) {
     FstSlice emptySlice = fstSliceCreate(NULL, 0);
@@ -1368,7 +1367,7 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
     }
     if (FST_NODE_IS_FINAL(nextNode) && isMatch) {
       FstOutput   fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)};
-      StreamWithStateResult* result = swsResultCreate(&slice, fOutput, tState);
+      FStmStRslt* result = swsResultCreate(&slice, fOutput, tState);
       taosMemoryFreeClear(buf);
       fstSliceDestroy(&slice);
       taosArrayDestroy(nodes);
@@ -1382,8 +1381,8 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
   return NULL;
 }
 
-StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state) {
+FStmStRslt* swsResultCreate(FstSlice* data, FstOutput fOut, void* state) {
-  StreamWithStateResult* result = taosMemoryCalloc(1, sizeof(StreamWithStateResult));
+  FStmStRslt* result = taosMemoryCalloc(1, sizeof(FStmStRslt));
   if (result == NULL) {
     return NULL;
   }
@@ -1393,7 +1392,7 @@ StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* sta
   result->state = state;
   return result;
 }
-void swsResultDestroy(StreamWithStateResult* result) {
+void swsResultDestroy(FStmStRslt* result) {
   if (NULL == result) {
     return;
   }
@@ -1411,8 +1410,8 @@ void streamStateDestroy(void* s) {
   fstNodeDestroy(ss->node);
 }
 
-FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) {
+FStmBuilder* stmBuilderCreate(Fst* fst, FAutoCtx* aut) {
-  FstStreamBuilder* b = taosMemoryCalloc(1, sizeof(FstStreamBuilder));
+  FStmBuilder* b = taosMemoryCalloc(1, sizeof(FStmBuilder));
   if (NULL == b) {
     return NULL;
   }
@@ -1423,14 +1422,14 @@ FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) {
   b->max = fstBoundStateCreate(Unbounded, NULL);
   return b;
 }
-void fstStreamBuilderDestroy(FstStreamBuilder* b) {
+void stmBuilderDestroy(FStmBuilder* b) {
   fstSliceDestroy(&b->min->data);
   fstSliceDestroy(&b->max->data);
   taosMemoryFreeClear(b->min);
   taosMemoryFreeClear(b->max);
   taosMemoryFree(b);
 }
-void fstStreamBuilderSetRange(FstStreamBuilder* b, FstSlice* val, RangeType type) {
+void stmBuilderSetRange(FStmBuilder* b, FstSlice* val, RangeType type) {
   if (b == NULL) {
     return;
   }

source/libs/index/src/indexFstAutomation.c

@@ -71,19 +71,19 @@ StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) {
 }
 
 // iterate fst
-static void* alwaysMatchStart(AutomationCtx* ctx) { return NULL; }
+static void* alwaysMatchStart(FAutoCtx* ctx) { return NULL; }
-static bool  alwaysMatchIsMatch(AutomationCtx* ctx, void* state) { return true; }
+static bool  alwaysMatchIsMatch(FAutoCtx* ctx, void* state) { return true; }
-static bool  alwaysMatchCanMatch(AutomationCtx* ctx, void* state) { return true; }
+static bool  alwaysMatchCanMatch(FAutoCtx* ctx, void* state) { return true; }
-static bool  alwaysMatchWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; }
+static bool  alwaysMatchWillAlwaysMatch(FAutoCtx* ctx, void* state) { return true; }
-static void* alwaysMatchAccpet(AutomationCtx* ctx, void* state, uint8_t byte) { return NULL; }
+static void* alwaysMatchAccpet(FAutoCtx* ctx, void* state, uint8_t byte) { return NULL; }
-static void* alwaysMatchAccpetEof(AutomationCtx* ctx, void* state) { return NULL; }
+static void* alwaysMatchAccpetEof(FAutoCtx* ctx, void* state) { return NULL; }
 // prefix query, impl later
 
-static void* prefixStart(AutomationCtx* ctx) {
+static void* prefixStart(FAutoCtx* ctx) {
   StartWithStateValue* data = (StartWithStateValue*)(ctx->stdata);
   return startWithStateValueDump(data);
 };
-static bool prefixIsMatch(AutomationCtx* ctx, void* sv) {
+static bool prefixIsMatch(FAutoCtx* ctx, void* sv) {
   StartWithStateValue* ssv = (StartWithStateValue*)sv;
   if (ssv == NULL) {
     return false;
@@ -94,15 +94,15 @@ static bool prefixIsMatch(AutomationCtx* ctx, void* sv) {
     return false;
   }
 }
-static bool prefixCanMatch(AutomationCtx* ctx, void* sv) {
+static bool prefixCanMatch(FAutoCtx* ctx, void* sv) {
   StartWithStateValue* ssv = (StartWithStateValue*)sv;
   if (ssv == NULL) {
     return false;
   }
   return ssv->val >= 0;
 }
-static bool  prefixWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; }
+static bool  prefixWillAlwaysMatch(FAutoCtx* ctx, void* state) { return true; }
-static void* prefixAccept(AutomationCtx* ctx, void* state, uint8_t byte) {
+static void* prefixAccept(FAutoCtx* ctx, void* state, uint8_t byte) {
   StartWithStateValue* ssv = (StartWithStateValue*)state;
   if (ssv == NULL || ctx == NULL) {
     return NULL;
@@ -125,18 +125,18 @@ static void* prefixAccept(AutomationCtx* ctx, void* state, uint8_t byte) {
   }
   return NULL;
 }
-static void* prefixAcceptEof(AutomationCtx* ctx, void* state) { return NULL; }
+static void* prefixAcceptEof(FAutoCtx* ctx, void* state) { return NULL; }
 
 // pattern query, impl later
 
-static void* patternStart(AutomationCtx* ctx) { return NULL; }
+static void* patternStart(FAutoCtx* ctx) { return NULL; }
-static bool  patternIsMatch(AutomationCtx* ctx, void* data) { return true; }
+static bool  patternIsMatch(FAutoCtx* ctx, void* data) { return true; }
-static bool  patternCanMatch(AutomationCtx* ctx, void* data) { return true; }
+static bool  patternCanMatch(FAutoCtx* ctx, void* data) { return true; }
-static bool  patternWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; }
+static bool  patternWillAlwaysMatch(FAutoCtx* ctx, void* state) { return true; }
 
-static void* patternAccept(AutomationCtx* ctx, void* state, uint8_t byte) { return NULL; }
+static void* patternAccept(FAutoCtx* ctx, void* state, uint8_t byte) { return NULL; }
 
-static void* patternAcceptEof(AutomationCtx* ctx, void* state) { return NULL; }
+static void* patternAcceptEof(FAutoCtx* ctx, void* state) { return NULL; }
 
 AutomationFunc automFuncs[] = {
     {alwaysMatchStart, alwaysMatchIsMatch, alwaysMatchCanMatch, alwaysMatchWillAlwaysMatch, alwaysMatchAccpet,
@@ -146,8 +146,8 @@ AutomationFunc automFuncs[] = {
     // add more search type
 };
 
-AutomationCtx* automCtxCreate(void* data, AutomationType atype) {
+FAutoCtx* automCtxCreate(void* data, AutomationType atype) {
-  AutomationCtx* ctx = taosMemoryCalloc(1, sizeof(AutomationCtx));
+  FAutoCtx* ctx = taosMemoryCalloc(1, sizeof(FAutoCtx));
   if (ctx == NULL) {
     return NULL;
   }
@@ -169,7 +169,7 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) {
   ctx->stdata = (void*)sv;
   return ctx;
 }
-void automCtxDestroy(AutomationCtx* ctx) {
+void automCtxDestroy(FAutoCtx* ctx) {
   startWithStateValueDestroy(ctx->stdata);
   taosMemoryFree(ctx->data);
   taosMemoryFree(ctx);

source/libs/index/src/indexTfile.c

@@ -26,15 +26,14 @@
 const static uint64_t tfileMagicNumber = 0xdb4775248b80fb57ull;
 
 typedef struct TFileFstIter {
-  FstStreamBuilder* fb;
+  FStmBuilder* fb;
-  StreamWithState*  st;
+  FStmSt*      st;
-  AutomationCtx*    ctx;
+  FAutoCtx*    ctx;
   TFileReader* rdr;
 } TFileFstIter;
 
 #define TF_TABLE_TATOAL_SIZE(sz) (sizeof(sz) + sz * sizeof(uint64_t))
 
-static int  tfileUidCompare(const void* a, const void* b);
 static int  tfileStrCompare(const void* a, const void* b);
 static int  tfileValueCompare(const void* a, const void* b, const void* param);
 static void tfileSerialTableIdsToBuf(char* buf, SArray* tableIds);
@@ -245,26 +244,21 @@ static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
 }
 
 static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
-  bool     hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON);
   char*    p = tem->colVal;
   uint64_t sz = tem->nColVal;
-  if (hasJson) {
-    p = indexPackJsonData(tem);
-    sz = strlen(p);
-  }
 
   SArray* offsets = taosArrayInit(16, sizeof(uint64_t));
 
-  AutomationCtx*         ctx = automCtxCreate((void*)p, AUTOMATION_PREFIX);
+  FAutoCtx*    ctx = automCtxCreate((void*)p, AUTOMATION_PREFIX);
-  FstStreamBuilder*      sb = fstSearch(((TFileReader*)reader)->fst, ctx);
+  FStmBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx);
-  StreamWithState*       st = streamBuilderIntoStream(sb);
+  FStmSt*      st = stmBuilderIntoStm(sb);
-  StreamWithStateResult* rt = NULL;
+  FStmStRslt*  rt = NULL;
-  while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
+  while ((rt = stmStNextWith(st, NULL)) != NULL) {
     taosArrayPush(offsets, &(rt->out.out));
     swsResultDestroy(rt);
   }
-  streamWithStateDestroy(st);
+  stmStDestroy(st);
-  fstStreamBuilderDestroy(sb);
+  stmBuilderDestroy(sb);
 
   int32_t ret = 0;
   for (int i = 0; i < taosArrayGetSize(offsets); i++) {
@@ -275,27 +269,14 @@ static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
       return TSDB_CODE_TDB_FILE_CORRUPTED;
     }
   }
-  if (hasJson) {
-    taosMemoryFree(p);
-  }
   return 0;
 }
 static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
-  bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON);
-
   int      ret = 0;
   char*    p = tem->colVal;
   uint64_t sz = tem->nColVal;
-  if (hasJson) {
-    p = indexPackJsonData(tem);
-    sz = strlen(p);
-  }
   int64_t  st = taosGetTimestampUs();
   FstSlice key = fstSliceCreate(p, sz);
-  /*impl later*/
-  if (hasJson) {
-    taosMemoryFree(p);
-  }
   fstSliceDestroy(&key);
   return 0;
 }
@@ -306,7 +287,7 @@ static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   char*    p = tem->colVal;
   uint64_t sz = tem->nColVal;
   if (hasJson) {
-    p = indexPackJsonData(tem);
+    p = idxPackJsonData(tem);
     sz = strlen(p);
   }
   int64_t  st = taosGetTimestampUs();
@@ -328,16 +309,16 @@ static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTRslt* tr,
 
   SArray* offsets = taosArrayInit(16, sizeof(uint64_t));
 
-  AutomationCtx*    ctx = automCtxCreate((void*)p, AUTOMATION_ALWAYS);
+  FAutoCtx*    ctx = automCtxCreate((void*)p, AUTOMATION_ALWAYS);
-  FstStreamBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx);
+  FStmBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx);
 
   FstSlice h = fstSliceCreate((uint8_t*)p, skip);
-  fstStreamBuilderSetRange(sb, &h, type);
+  stmBuilderSetRange(sb, &h, type);
   fstSliceDestroy(&h);
 
-  StreamWithState*       st = streamBuilderIntoStream(sb);
+  FStmSt*     st = stmBuilderIntoStm(sb);
-  StreamWithStateResult* rt = NULL;
+  FStmStRslt* rt = NULL;
-  while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
+  while ((rt = stmStNextWith(st, NULL)) != NULL) {
     FstSlice* s = &rt->data;
     char*     ch = (char*)fstSliceData(s, NULL);
     // if (0 != strncmp(ch, tem->colName, tem->nColName)) {
@@ -355,8 +336,8 @@ static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTRslt* tr,
     }
     swsResultDestroy(rt);
   }
-  streamWithStateDestroy(st);
+  stmStDestroy(st);
-  fstStreamBuilderDestroy(sb);
+  stmBuilderDestroy(sb);
   return TSDB_CODE_SUCCESS;
 }
 static int32_t tfSearchLessThan(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
@@ -372,37 +353,17 @@ static int32_t tfSearchGreaterEqual(void* reader, SIndexTerm* tem, SIdxTRslt* tr
   return tfSearchCompareFunc(reader, tem, tr, GE);
 }
 static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
-  bool     hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON);
   int      ret = 0;
   char*    p = tem->colVal;
   uint64_t sz = tem->nColVal;
-  if (hasJson) {
-    p = indexPackJsonData(tem);
-    sz = strlen(p);
-  }
   int64_t  st = taosGetTimestampUs();
   FstSlice key = fstSliceCreate(p, sz);
-  // uint64_t offset;
-  // if (fstGet(((TFileReader*)reader)->fst, &key, &offset)) {
-  //  int64_t et = taosGetTimestampUs();
-  //  int64_t cost = et - st;
-  //  indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex, time cost: %" PRIu64 "us",
-  //            tem->suid, tem->colName, tem->colVal, cost);
-
-  //  ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total);
-  //  cost = taosGetTimestampUs() - et;
-  //  indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, time cost: %" PRIu64 "us", tem->suid,
-  //            tem->colName, tem->colVal, cost);
-  //}
-  if (hasJson) {
-    taosMemoryFree(p);
-  }
   fstSliceDestroy(&key);
   return 0;
 }
 static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   int   ret = 0;
-  char* p = indexPackJsonData(tem);
+  char* p = idxPackJsonData(tem);
   int   sz = strlen(p);
 
   int64_t  st = taosGetTimestampUs();
@@ -422,8 +383,6 @@ static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   }
   fstSliceDestroy(&key);
   return 0;
-  // deprecate api
-  return TSDB_CODE_SUCCESS;
 }
 static int32_t tfSearchEqual_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   return tfSearchCompareFunc_JSON(reader, tem, tr, EQ);
@@ -467,21 +426,21 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt
                      .colType = tem->colType,
                      .colName = tem->colVal,
                      .nColName = tem->nColVal};
-    p = indexPackJsonDataPrefixNoType(&tm, &skip);
+    p = idxPackJsonDataPrefixNoType(&tm, &skip);
   } else {
-    p = indexPackJsonDataPrefix(tem, &skip);
+    p = idxPackJsonDataPrefix(tem, &skip);
   }
 
   _cache_range_compare cmpFn = indexGetCompare(ctype);
 
   SArray* offsets = taosArrayInit(16, sizeof(uint64_t));
 
-  AutomationCtx*    ctx = automCtxCreate((void*)p, AUTOMATION_PREFIX);
+  FAutoCtx*    ctx = automCtxCreate((void*)p, AUTOMATION_PREFIX);
-  FstStreamBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx);
+  FStmBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx);
 
-  StreamWithState*       st = streamBuilderIntoStream(sb);
+  FStmSt*     st = stmBuilderIntoStm(sb);
-  StreamWithStateResult* rt = NULL;
+  FStmStRslt* rt = NULL;
-  while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
+  while ((rt = stmStNextWith(st, NULL)) != NULL) {
     FstSlice* s = &rt->data;
 
     int32_t  sz = 0;
@@ -509,8 +468,8 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt
     }
     swsResultDestroy(rt);
   }
-  streamWithStateDestroy(st);
+  stmStDestroy(st);
-  fstStreamBuilderDestroy(sb);
+  stmBuilderDestroy(sb);
   return TSDB_CODE_SUCCESS;
 }
 int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTRslt* tr) {
@@ -595,8 +554,8 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) {
   // ugly code, refactor later
   for (size_t i = 0; i < sz; i++) {
     TFileValue* v = taosArrayGetP((SArray*)data, i);
-    taosArraySort(v->tableId, tfileUidCompare);
+    taosArraySort(v->tableId, idxUidCompare);
-    taosArrayRemoveDuplicate(v->tableId, tfileUidCompare, NULL);
+    taosArrayRemoveDuplicate(v->tableId, idxUidCompare, NULL);
     int32_t tbsz = taosArrayGetSize(v->tableId);
     fstOffset += TF_TABLE_TATOAL_SIZE(tbsz);
   }
@@ -722,7 +681,7 @@ static bool tfileIteratorNext(Iterate* iiter) {
   uint64_t offset = 0;
 
   TFileFstIter* tIter = iiter->iter;
-  StreamWithStateResult* rt = streamWithStateNextWith(tIter->st, NULL);
+  FStmStRslt*   rt = stmStNextWith(tIter->st, NULL);
   if (rt == NULL) {
     return false;
   }
@@ -756,7 +715,7 @@ static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) {
 
   iter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS);
   iter->fb = fstSearch(reader->fst, iter->ctx);
-  iter->st = streamBuilderIntoStream(iter->fb);
+  iter->st = stmBuilderIntoStm(iter->fb);
   iter->rdr = reader;
   return iter;
 }
@@ -787,8 +746,8 @@ void tfileIteratorDestroy(Iterate* iter) {
   iterateValueDestroy(iv, true);
 
   TFileFstIter* tIter = iter->iter;
-  streamWithStateDestroy(tIter->st);
+  stmStDestroy(tIter->st);
-  fstStreamBuilderDestroy(tIter->fb);
+  stmBuilderDestroy(tIter->fb);
   automCtxDestroy(tIter->ctx);
   taosMemoryFree(tIter);
 
@@ -808,11 +767,6 @@ TFileReader* tfileGetReaderByCol(IndexTFile* tf, uint64_t suid, char* colName) {
   return rd;
 }
 
-static int tfileUidCompare(const void* a, const void* b) {
-  uint64_t l = *(uint64_t*)a;
-  uint64_t r = *(uint64_t*)b;
-  return l - r;
-}
 static int tfileStrCompare(const void* a, const void* b) {
   int ret = strcmp((char*)a, (char*)b);
   if (ret == 0) {

source/libs/index/test/fstTest.cc

@@ -85,11 +85,11 @@ class FstReadMemory {
     return ok;
   }
   // add later
-  bool Search(AutomationCtx* ctx, std::vector<uint64_t>& result) {
+  bool Search(FAutoCtx* ctx, std::vector<uint64_t>& result) {
-    FstStreamBuilder*      sb = fstSearch(_fst, ctx);
+    FStmBuilder* sb = fstSearch(_fst, ctx);
-    StreamWithState*       st = streamBuilderIntoStream(sb);
+    FStmSt*      st = stmBuilderIntoStm(sb);
-    StreamWithStateResult* rt = NULL;
+    FStmStRslt*  rt = NULL;
-    while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
+    while ((rt = stmStNextWith(st, NULL)) != NULL) {
       // result.push_back((uint64_t)(rt->out.out));
       FstSlice*   s = &rt->data;
       int32_t     sz = 0;
@@ -99,27 +99,27 @@ class FstReadMemory {
       result.push_back(rt->out.out);
       swsResultDestroy(rt);
     }
-    streamWithStateDestroy(st);
+    stmStDestroy(st);
-    fstStreamBuilderDestroy(sb);
+    stmBuilderDestroy(sb);
     return true;
   }
-  bool SearchRange(AutomationCtx* ctx, const std::string& low, RangeType lowType, const std::string& high,
+  bool SearchRange(FAutoCtx* ctx, const std::string& low, RangeType lowType, const std::string& high,
                    RangeType highType, std::vector<uint64_t>& result) {
-    FstStreamBuilder* sb = fstSearch(_fst, ctx);
+    FStmBuilder* sb = fstSearch(_fst, ctx);
 
     FstSlice l = fstSliceCreate((uint8_t*)low.c_str(), low.size());
     FstSlice h = fstSliceCreate((uint8_t*)high.c_str(), high.size());
 
     // range [low, high);
-    fstStreamBuilderSetRange(sb, &l, lowType);
+    stmBuilderSetRange(sb, &l, lowType);
-    fstStreamBuilderSetRange(sb, &h, highType);
+    stmBuilderSetRange(sb, &h, highType);
 
     fstSliceDestroy(&l);
     fstSliceDestroy(&h);
 
-    StreamWithState*       st = streamBuilderIntoStream(sb);
+    FStmSt*     st = stmBuilderIntoStm(sb);
-    StreamWithStateResult* rt = NULL;
+    FStmStRslt* rt = NULL;
-    while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
+    while ((rt = stmStNextWith(st, NULL)) != NULL) {
       // result.push_back((uint64_t)(rt->out.out));
       FstSlice*   s = &rt->data;
       int32_t     sz = 0;
@@ -129,11 +129,11 @@ class FstReadMemory {
       result.push_back(rt->out.out);
       swsResultDestroy(rt);
     }
-    streamWithStateDestroy(st);
+    stmStDestroy(st);
-    fstStreamBuilderDestroy(sb);
+    stmBuilderDestroy(sb);
     return true;
   }
-  bool SearchWithTimeCostUs(AutomationCtx* ctx, std::vector<uint64_t>& result) {
+  bool SearchWithTimeCostUs(FAutoCtx* ctx, std::vector<uint64_t>& result) {
     int64_t s = taosGetTimestampUs();
     bool    ok = this->Search(ctx, result);
     int64_t e = taosGetTimestampUs();
@@ -253,7 +253,7 @@ void checkFstLongTerm() {
   // prefix search
   // std::vector<uint64_t> result;
 
-  // AutomationCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_ALWAYS);
+  // FAutoCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_ALWAYS);
   // m->Search(ctx, result);
   // std::cout << "size: " << result.size() << std::endl;
   // assert(result.size() == count);
@@ -286,7 +286,7 @@ void checkFstCheckIterator1() {
   // prefix search
   std::vector<uint64_t> result;
 
-  AutomationCtx* ctx = automCtxCreate((void*)"He", AUTOMATION_ALWAYS);
+  FAutoCtx* ctx = automCtxCreate((void*)"He", AUTOMATION_ALWAYS);
   m->Search(ctx, result);
   std::cout << "size: " << result.size() << std::endl;
   // assert(result.size() == count);
@@ -321,7 +321,7 @@ void checkFstCheckIterator2() {
   // prefix search
   std::vector<uint64_t> result;
 
-  AutomationCtx* ctx = automCtxCreate((void*)"He", AUTOMATION_ALWAYS);
+  FAutoCtx* ctx = automCtxCreate((void*)"He", AUTOMATION_ALWAYS);
   m->Search(ctx, result);
   std::cout << "size: " << result.size() << std::endl;
   // assert(result.size() == count);
@@ -361,7 +361,7 @@ void checkFstCheckIteratorPrefix() {
     // prefix search
     std::vector<uint64_t> result;
 
-    AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_PREFIX);
+    FAutoCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_PREFIX);
     m->Search(ctx, result);
     assert(result.size() == 1);
     automCtxDestroy(ctx);
@@ -370,7 +370,7 @@ void checkFstCheckIteratorPrefix() {
     // prefix search
     std::vector<uint64_t> result;
 
-    AutomationCtx* ctx = automCtxCreate((void*)"Hello", AUTOMATION_PREFIX);
+    FAutoCtx* ctx = automCtxCreate((void*)"Hello", AUTOMATION_PREFIX);
     m->Search(ctx, result);
     assert(result.size() == 2);
     automCtxDestroy(ctx);
@@ -378,7 +378,7 @@ void checkFstCheckIteratorPrefix() {
   {
     std::vector<uint64_t> result;
 
-    AutomationCtx* ctx = automCtxCreate((void*)"jddd", AUTOMATION_PREFIX);
+    FAutoCtx* ctx = automCtxCreate((void*)"jddd", AUTOMATION_PREFIX);
     m->Search(ctx, result);
     assert(result.size() == 1);
     automCtxDestroy(ctx);
@@ -412,7 +412,7 @@ void checkFstCheckIteratorRange1() {
   {
     // prefix search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "b", GE, "e", LT, result);
     assert(result.size() == 3);
@@ -421,7 +421,7 @@ void checkFstCheckIteratorRange1() {
   {
     // prefix search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "b", GT, "e", LT, result);
     assert(result.size() == 2);
@@ -430,7 +430,7 @@ void checkFstCheckIteratorRange1() {
   {
     // prefix search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "b", GT, "e", LE, result);
     assert(result.size() == 3);
@@ -439,7 +439,7 @@ void checkFstCheckIteratorRange1() {
   {
     // prefix search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "b", GE, "e", LE, result);
     assert(result.size() == 4);
@@ -473,7 +473,7 @@ void checkFstCheckIteratorRange2() {
   {
     // range  search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "b", GE, "ed", LT, result);
     assert(result.size() == 4);
@@ -482,7 +482,7 @@ void checkFstCheckIteratorRange2() {
   {
     // range  search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "bb", GE, "ed", LT, result);
     assert(result.size() == 3);
@@ -491,7 +491,7 @@ void checkFstCheckIteratorRange2() {
   {
     // range  search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "b", GE, "ed", LE, result);
     assert(result.size() == 5);
@@ -501,7 +501,7 @@ void checkFstCheckIteratorRange2() {
   {
     // range  search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "b", GT, "ed", LE, result);
     assert(result.size() == 4);
@@ -510,7 +510,7 @@ void checkFstCheckIteratorRange2() {
   {
     // range  search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "b", GT, "ed", LT, result);
     assert(result.size() == 3);
@@ -544,7 +544,7 @@ void checkFstCheckIteratorRange3() {
   {
     // range  search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "b", GE, "", (RangeType)10, result);
     assert(result.size() == 5);
@@ -553,7 +553,7 @@ void checkFstCheckIteratorRange3() {
   {
     // range  search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "", (RangeType)20, "ab", LE, result);
     assert(result.size() == 1);
@@ -563,7 +563,7 @@ void checkFstCheckIteratorRange3() {
   {
     // range  search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "", (RangeType)30, "ab", LT, result);
     assert(result.size() == 0);
@@ -572,7 +572,7 @@ void checkFstCheckIteratorRange3() {
   {
     // range  search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "ed", GT, "ed", (RangeType)40, result);
     assert(result.size() == 0);

source/libs/index/test/fstUT.cc

@@ -106,11 +106,11 @@ class FstReadMemory {
     return ok;
   }
   // add later
-  bool Search(AutomationCtx* ctx, std::vector<uint64_t>& result) {
+  bool Search(FAutoCtx* ctx, std::vector<uint64_t>& result) {
-    FstStreamBuilder*      sb = fstSearch(_fst, ctx);
+    FStmBuilder* sb = fstSearch(_fst, ctx);
-    StreamWithState*       st = streamBuilderIntoStream(sb);
+    FStmSt*      st = stmBuilderIntoStm(sb);
-    StreamWithStateResult* rt = NULL;
+    FStmStRslt*  rt = NULL;
-    while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
+    while ((rt = stmStNextWith(st, NULL)) != NULL) {
       // result.push_back((uint64_t)(rt->out.out));
       FstSlice*   s = &rt->data;
       int32_t     sz = 0;
@@ -122,7 +122,7 @@ class FstReadMemory {
     std::cout << std::endl;
     return true;
   }
-  bool SearchWithTimeCostUs(AutomationCtx* ctx, std::vector<uint64_t>& result) {
+  bool SearchWithTimeCostUs(FAutoCtx* ctx, std::vector<uint64_t>& result) {
     int64_t s = taosGetTimestampUs();
     bool    ok = this->Search(ctx, result);
     int64_t e = taosGetTimestampUs();
@@ -192,7 +192,7 @@ class TFst {
     }
     return fr->Get(k, v);
   }
-  bool Search(AutomationCtx* ctx, std::vector<uint64_t>& result) {
+  bool Search(FAutoCtx* ctx, std::vector<uint64_t>& result) {
     // add more
     return fr->Search(ctx, result);
   }
@@ -231,7 +231,7 @@ TEST_F(FstEnv, writeNormal) {
   assert(val == 0);
 
   std::vector<uint64_t> rlt;
-  AutomationCtx*        ctx = automCtxCreate((void*)"ab", AUTOMATION_ALWAYS);
+  FAutoCtx*             ctx = automCtxCreate((void*)"ab", AUTOMATION_ALWAYS);
   assert(fst->Search(ctx, rlt) == true);
 }
 TEST_F(FstEnv, WriteMillonrRecord) {}

source/libs/index/test/indexTests.cc

@@ -106,17 +106,17 @@ class FstReadMemory {
     return ok;
   }
   // add later
-  bool Search(AutomationCtx* ctx, std::vector<uint64_t>& result) {
+  bool Search(FAutoCtx* ctx, std::vector<uint64_t>& result) {
-    FstStreamBuilder*      sb = fstSearch(_fst, ctx);
+    FStmBuilder* sb = fstSearch(_fst, ctx);
-    StreamWithState*       st = streamBuilderIntoStream(sb);
+    FStmSt*      st = stmBuilderIntoStm(sb);
-    StreamWithStateResult* rt = NULL;
+    FStmStRslt*  rt = NULL;
 
-    while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
+    while ((rt = stmStNextWith(st, NULL)) != NULL) {
       result.push_back((uint64_t)(rt->out.out));
     }
     return true;
   }
-  bool SearchWithTimeCostUs(AutomationCtx* ctx, std::vector<uint64_t>& result) {
+  bool SearchWithTimeCostUs(FAutoCtx* ctx, std::vector<uint64_t>& result) {
     int64_t s = taosGetTimestampUs();
     bool    ok = this->Search(ctx, result);
     int64_t e = taosGetTimestampUs();
@@ -220,7 +220,7 @@ void checkFstPrefixSearch() {
   // prefix search
   std::vector<uint64_t> result;
 
-  AutomationCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_PREFIX);
+  FAutoCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_PREFIX);
   m->Search(ctx, result);
   assert(result.size() == count);
   for (int i = 0; i < result.size(); i++) {

source/libs/index/test/utilUT.cc

@@ -327,13 +327,13 @@ TEST_F(UtilEnv, testFill) {
   for (int i = 0; i < 1000000; i++) {
     int64_t val = i;
     char    buf[65] = {0};
-    indexInt2str(val, buf, 1);
+    idxInt2str(val, buf, 1);
     EXPECT_EQ(val, taosStr2int64(buf));
   }
   for (int i = 0; i < 1000000; i++) {
     int64_t val = 0 - i;
     char    buf[65] = {0};
-    indexInt2str(val, buf, -1);
+    idxInt2str(val, buf, -1);
     EXPECT_EQ(val, taosStr2int64(buf));
   }
 }

source/libs/parser/src/parAstParser.c

@@ -108,9 +108,10 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt);
 
 static EDealRes collectMetaKeyFromFunction(SCollectMetaKeyFromExprCxt* pCxt, SFunctionNode* pFunc) {
   if (fmIsBuiltinFunc(pFunc->functionName)) {
-    return TSDB_CODE_SUCCESS;
+    return DEAL_RES_CONTINUE;
   }
-  return reserveUdfInCache(pFunc->functionName, pCxt->pComCxt->pMetaCache);
+  pCxt->errCode = reserveUdfInCache(pFunc->functionName, pCxt->pComCxt->pMetaCache);
+  return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR;
 }
 
 static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, SRealTableNode* pRealTable,
@@ -179,6 +180,10 @@ static int32_t collectMetaKeyFromSelect(SCollectMetaKeyCxt* pCxt, SSelectStmt* p
   return cxt.errCode;
 }
 
+static int32_t collectMetaKeyFromAlterDatabase(SCollectMetaKeyCxt* pCxt, SAlterDatabaseStmt* pStmt) {
+  return reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache);
+}
+
 static int32_t collectMetaKeyFromCreateTable(SCollectMetaKeyCxt* pCxt, SCreateTableStmt* pStmt) {
   int32_t code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache);
   if (TSDB_CODE_SUCCESS == code && NULL == pStmt->pTags) {
@@ -376,6 +381,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
       return collectMetaKeyFromSetOperator(pCxt, (SSetOperator*)pStmt);
     case QUERY_NODE_SELECT_STMT:
       return collectMetaKeyFromSelect(pCxt, (SSelectStmt*)pStmt);
+    case QUERY_NODE_ALTER_DATABASE_STMT:
+      return collectMetaKeyFromAlterDatabase(pCxt, (SAlterDatabaseStmt*)pStmt);
     case QUERY_NODE_CREATE_TABLE_STMT:
       return collectMetaKeyFromCreateTable(pCxt, (SCreateTableStmt*)pStmt);
     case QUERY_NODE_CREATE_MULTI_TABLE_STMT:

source/libs/parser/src/parInsert.c

@@ -1531,9 +1531,12 @@ typedef struct SInsertParseSyntaxCxt {
 
 static int32_t skipParentheses(SInsertParseSyntaxCxt* pCxt) {
   SToken  sToken;
+  int32_t expectRightParenthesis = 1;
   while (1) {
     NEXT_TOKEN(pCxt->pSql, sToken);
-    if (TK_NK_RP == sToken.type) {
+    if (TK_NK_LP == sToken.type) {
+      ++expectRightParenthesis;
+    } else if (TK_NK_RP == sToken.type && 0 == --expectRightParenthesis) {
       break;
     }
     if (0 == sToken.n) {
@@ -2040,6 +2043,10 @@ int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TA
     return TSDB_CODE_QRY_APP_ERROR;
   }
 
+  if (pDataBlock->pTableMeta->tableType != TSDB_SUPER_TABLE && pDataBlock->pTableMeta->tableType != TSDB_CHILD_TABLE) {
+    return TSDB_CODE_TSC_STMT_API_ERROR;
+  }
+
   SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta);
   if (tags->numOfBound <= 0) {
     *fieldNum = 0;

source/libs/parser/src/parTranslater.c

@@ -103,15 +103,14 @@ static int32_t collectUseTable(const SName* pName, SHashObj* pDbs) {
 
 static int32_t getTableMetaImpl(STranslateContext* pCxt, const SName* pName, STableMeta** pMeta) {
   SParseContext* pParCxt = pCxt->pParseCxt;
-  int32_t        code = TSDB_CODE_SUCCESS;
+  int32_t        code = collectUseDatabase(pName, pCxt->pDbs);
-  if (pParCxt->async) {
-    code = getTableMetaFromCache(pCxt->pMetaCache, pName, pMeta);
-  } else {
-    code = collectUseDatabase(pName, pCxt->pDbs);
   if (TSDB_CODE_SUCCESS == code) {
     code = collectUseTable(pName, pCxt->pTables);
   }
   if (TSDB_CODE_SUCCESS == code) {
+    if (pParCxt->async) {
+      code = getTableMetaFromCache(pCxt->pMetaCache, pName, pMeta);
+    } else {
       code = catalogGetTableMeta(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, pName, pMeta);
     }
   }
@@ -150,12 +149,11 @@ static int32_t getDBVgInfoImpl(STranslateContext* pCxt, const SName* pName, SArr
   SParseContext* pParCxt = pCxt->pParseCxt;
   char           fullDbName[TSDB_DB_FNAME_LEN];
   tNameGetFullDbName(pName, fullDbName);
-  int32_t code = TSDB_CODE_SUCCESS;
+  int32_t code = collectUseDatabaseImpl(fullDbName, pCxt->pDbs);
+  if (TSDB_CODE_SUCCESS == code) {
     if (pParCxt->async) {
       code = getDbVgInfoFromCache(pCxt->pMetaCache, fullDbName, pVgInfo);
     } else {
-    code = collectUseDatabaseImpl(fullDbName, pCxt->pDbs);
-    if (TSDB_CODE_SUCCESS == code) {
       code = catalogGetDBVgInfo(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, fullDbName, pVgInfo);
     }
   }
@@ -175,15 +173,14 @@ static int32_t getDBVgInfo(STranslateContext* pCxt, const char* pDbName, SArray*
 
 static int32_t getTableHashVgroupImpl(STranslateContext* pCxt, const SName* pName, SVgroupInfo* pInfo) {
   SParseContext* pParCxt = pCxt->pParseCxt;
-  int32_t        code = TSDB_CODE_SUCCESS;
+  int32_t        code = collectUseDatabase(pName, pCxt->pDbs);
-  if (pParCxt->async) {
-    code = getTableVgroupFromCache(pCxt->pMetaCache, pName, pInfo);
-  } else {
-    code = collectUseDatabase(pName, pCxt->pDbs);
   if (TSDB_CODE_SUCCESS == code) {
     code = collectUseTable(pName, pCxt->pTables);
   }
   if (TSDB_CODE_SUCCESS == code) {
+    if (pParCxt->async) {
+      code = getTableVgroupFromCache(pCxt->pMetaCache, pName, pInfo);
+    } else {
       code = catalogGetTableHashVgroup(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, pName, pInfo);
     }
   }
@@ -203,12 +200,11 @@ static int32_t getTableHashVgroup(STranslateContext* pCxt, const char* pDbName,
 static int32_t getDBVgVersion(STranslateContext* pCxt, const char* pDbFName, int32_t* pVersion, int64_t* pDbId,
                               int32_t* pTableNum) {
   SParseContext* pParCxt = pCxt->pParseCxt;
-  int32_t        code = TSDB_CODE_SUCCESS;
+  int32_t        code = collectUseDatabaseImpl(pDbFName, pCxt->pDbs);
+  if (TSDB_CODE_SUCCESS == code) {
     if (pParCxt->async) {
       code = getDbVgVersionFromCache(pCxt->pMetaCache, pDbFName, pVersion, pDbId, pTableNum);
     } else {
-    code = collectUseDatabaseImpl(pDbFName, pCxt->pDbs);
-    if (TSDB_CODE_SUCCESS == code) {
       code = catalogGetDBVgVersion(pParCxt->pCatalog, pDbFName, pVersion, pDbId, pTableNum);
     }
   }
@@ -224,12 +220,11 @@ static int32_t getDBCfg(STranslateContext* pCxt, const char* pDbName, SDbCfgInfo
   tNameSetDbName(&name, pCxt->pParseCxt->acctId, pDbName, strlen(pDbName));
   char dbFname[TSDB_DB_FNAME_LEN] = {0};
   tNameGetFullDbName(&name, dbFname);
-  int32_t code = TSDB_CODE_SUCCESS;
+  int32_t code = collectUseDatabaseImpl(dbFname, pCxt->pDbs);
+  if (TSDB_CODE_SUCCESS == code) {
     if (pParCxt->async) {
       code = getDbCfgFromCache(pCxt->pMetaCache, dbFname, pInfo);
     } else {
-    code = collectUseDatabaseImpl(dbFname, pCxt->pDbs);
-    if (TSDB_CODE_SUCCESS == code) {
       code = catalogGetDBCfg(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, dbFname, pInfo);
     }
   }
@@ -817,6 +812,10 @@ static EDealRes translateArithmeticOperator(STranslateContext* pCxt, SOperatorNo
   return DEAL_RES_CONTINUE;
 }
 
+static bool dataTypeEqual(const SDataType* l, const SDataType* r) {
+  return (l->type == r->type && l->bytes == r->bytes && l->precision == r->precision && l->scale == r->scale);
+}
+
 static EDealRes translateComparisonOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
   SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType;
   SDataType rdt = ((SExprNode*)(pOp->pRight))->resType;
@@ -824,7 +823,24 @@ static EDealRes translateComparisonOperator(STranslateContext* pCxt, SOperatorNo
     return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
   }
   if (OP_TYPE_IN == pOp->opType || OP_TYPE_NOT_IN == pOp->opType) {
-    ((SExprNode*)pOp->pRight)->resType = ((SExprNode*)pOp->pLeft)->resType;
+    SNodeListNode* pRight = (SNodeListNode*)pOp->pRight;
+    bool first = true;
+    SDataType targetDt = {0};
+    SNode* pNode = NULL;
+    FOREACH(pNode, pRight->pNodeList) {
+      SDataType dt = ((SExprNode*)pNode)->resType;
+      if (first) {
+        targetDt = dt;
+        if (targetDt.type != TSDB_DATA_TYPE_NULL) {
+          first = false;
+        }
+      } else if (dt.type != targetDt.type && dt.type != TSDB_DATA_TYPE_NULL) {
+        return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)pNode)->aliasName);
+      } else if (dt.bytes > targetDt.bytes) {
+        targetDt.bytes = dt.bytes;
+      }
+    }
+    pRight->dataType = targetDt;
   }
   if (nodesIsRegularOp(pOp)) {
     if (!IS_VAR_DATA_TYPE(((SExprNode*)(pOp->pLeft))->resType.type)) {
@@ -2015,10 +2031,6 @@ static SNode* createSetOperProject(const char* pTableAlias, SNode* pNode) {
   return (SNode*)pCol;
 }
 
-static bool dataTypeEqual(const SDataType* l, const SDataType* r) {
-  return (l->type == r->type && l->bytes == r->bytes && l->precision == r->precision && l->scale == r->scale);
-}
-
 static int32_t createCastFunc(STranslateContext* pCxt, SNode* pExpr, SDataType dt, SNode** pCast) {
   SFunctionNode* pFunc = nodesMakeNode(QUERY_NODE_FUNCTION);
   if (NULL == pFunc) {

source/libs/parser/src/parUtil.c

@@ -484,20 +484,6 @@ static int32_t buildDbReq(SHashObj* pDbsHash, SArray** pDbs) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t buildTableMetaReq(SHashObj* pTableMetaHash, SArray** pTableMeta) {
-  return buildTableReq(pTableMetaHash, pTableMeta);
-}
-
-static int32_t buildDbVgroupReq(SHashObj* pDbVgroupHash, SArray** pDbVgroup) {
-  return buildDbReq(pDbVgroupHash, pDbVgroup);
-}
-
-static int32_t buildTableVgroupReq(SHashObj* pTableVgroupHash, SArray** pTableVgroup) {
-  return buildTableReq(pTableVgroupHash, pTableVgroup);
-}
-
-static int32_t buildDbCfgReq(SHashObj* pDbCfgHash, SArray** pDbCfg) { return buildDbReq(pDbCfgHash, pDbCfg); }
-
 static int32_t buildUserAuthReq(SHashObj* pUserAuthHash, SArray** pUserAuth) {
   if (NULL != pUserAuthHash) {
     *pUserAuth = taosArrayInit(taosHashGetSize(pUserAuthHash), sizeof(SUserAuthInfo));
@@ -537,15 +523,18 @@ static int32_t buildUdfReq(SHashObj* pUdfHash, SArray** pUdf) {
 }
 
 int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq) {
-  int32_t code = buildTableMetaReq(pMetaCache->pTableMeta, &pCatalogReq->pTableMeta);
+  int32_t code = buildTableReq(pMetaCache->pTableMeta, &pCatalogReq->pTableMeta);
   if (TSDB_CODE_SUCCESS == code) {
-    code = buildDbVgroupReq(pMetaCache->pDbVgroup, &pCatalogReq->pDbVgroup);
+    code = buildDbReq(pMetaCache->pDbVgroup, &pCatalogReq->pDbVgroup);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = buildTableVgroupReq(pMetaCache->pTableVgroup, &pCatalogReq->pTableHash);
+    code = buildTableReq(pMetaCache->pTableVgroup, &pCatalogReq->pTableHash);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = buildDbCfgReq(pMetaCache->pDbCfg, &pCatalogReq->pDbCfg);
+    code = buildDbReq(pMetaCache->pDbCfg, &pCatalogReq->pDbCfg);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = buildDbReq(pMetaCache->pDbInfo, &pCatalogReq->pDbInfo);
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = buildUserAuthReq(pMetaCache->pUserAuth, &pCatalogReq->pUser);
@@ -556,51 +545,39 @@ int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalog
   return code;
 }
 
-static int32_t putTableMetaToCache(const SArray* pTableMetaReq, const SArray* pTableMetaData, SHashObj* pTableMeta) {
+static int32_t putMetaDataToHash(const char* pKey, int32_t len, const SArray* pData, int32_t index, SHashObj* pHash) {
-  int32_t ntables = taosArrayGetSize(pTableMetaReq);
+  SMetaRes* pRes = taosArrayGet(pData, index);
+  return taosHashPut(pHash, pKey, len, &pRes, POINTER_BYTES);
+}
+
+static int32_t getMetaDataFromHash(const char* pKey, int32_t len, SHashObj* pHash, void** pOutput) {
+  SMetaRes** pRes = taosHashGet(pHash, pKey, len);
+  if (NULL == pRes || NULL == *pRes) {
+    return TSDB_CODE_PAR_INTERNAL_ERROR;
+  }
+  if (TSDB_CODE_SUCCESS == (*pRes)->code) {
+    *pOutput = (*pRes)->pRes;
+  }
+  return (*pRes)->code;
+}
+
+static int32_t putTableDataToCache(const SArray* pTableReq, const SArray* pTableData, SHashObj* pTable) {
+  int32_t ntables = taosArrayGetSize(pTableReq);
   for (int32_t i = 0; i < ntables; ++i) {
     char fullName[TSDB_TABLE_FNAME_LEN];
-    tNameExtractFullName(taosArrayGet(pTableMetaReq, i), fullName);
+    tNameExtractFullName(taosArrayGet(pTableReq, i), fullName);
-    if (TSDB_CODE_SUCCESS !=
+    if (TSDB_CODE_SUCCESS != putMetaDataToHash(fullName, strlen(fullName), pTableData, i, pTable)) {
-        taosHashPut(pTableMeta, fullName, strlen(fullName), taosArrayGet(pTableMetaData, i), POINTER_BYTES)) {
       return TSDB_CODE_OUT_OF_MEMORY;
     }
   }
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t putDbVgroupToCache(const SArray* pDbVgroupReq, const SArray* pDbVgroupData, SHashObj* pDbVgroup) {
+static int32_t putDbDataToCache(const SArray* pDbReq, const SArray* pDbData, SHashObj* pDb) {
-  int32_t nvgs = taosArrayGetSize(pDbVgroupReq);
+  int32_t nvgs = taosArrayGetSize(pDbReq);
   for (int32_t i = 0; i < nvgs; ++i) {
-    char* pDbFName = taosArrayGet(pDbVgroupReq, i);
+    char* pDbFName = taosArrayGet(pDbReq, i);
-    if (TSDB_CODE_SUCCESS !=
+    if (TSDB_CODE_SUCCESS != putMetaDataToHash(pDbFName, strlen(pDbFName), pDbData, i, pDb)) {
-        taosHashPut(pDbVgroup, pDbFName, strlen(pDbFName), taosArrayGet(pDbVgroupData, i), POINTER_BYTES)) {
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-  }
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t putTableVgroupToCache(const SArray* pTableVgroupReq, const SArray* pTableVgroupData,
-                                     SHashObj* pTableVgroup) {
-  int32_t ntables = taosArrayGetSize(pTableVgroupReq);
-  for (int32_t i = 0; i < ntables; ++i) {
-    char fullName[TSDB_TABLE_FNAME_LEN];
-    tNameExtractFullName(taosArrayGet(pTableVgroupReq, i), fullName);
-    SVgroupInfo* pInfo = taosArrayGet(pTableVgroupData, i);
-    if (TSDB_CODE_SUCCESS != taosHashPut(pTableVgroup, fullName, strlen(fullName), &pInfo, POINTER_BYTES)) {
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-  }
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t putDbCfgToCache(const SArray* pDbCfgReq, const SArray* pDbCfgData, SHashObj* pDbCfg) {
-  int32_t nvgs = taosArrayGetSize(pDbCfgReq);
-  for (int32_t i = 0; i < nvgs; ++i) {
-    char*       pDbFName = taosArrayGet(pDbCfgReq, i);
-    SDbCfgInfo* pInfo = taosArrayGet(pDbCfgData, i);
-    if (TSDB_CODE_SUCCESS != taosHashPut(pDbCfg, pDbFName, strlen(pDbFName), &pInfo, POINTER_BYTES)) {
       return TSDB_CODE_OUT_OF_MEMORY;
     }
   }
@@ -613,7 +590,7 @@ static int32_t putUserAuthToCache(const SArray* pUserAuthReq, const SArray* pUse
     SUserAuthInfo* pUser = taosArrayGet(pUserAuthReq, i);
     char           key[USER_AUTH_KEY_MAX_LEN] = {0};
     int32_t        len = userAuthToStringExt(pUser->user, pUser->dbFName, pUser->type, key);
-    if (TSDB_CODE_SUCCESS != taosHashPut(pUserAuth, key, len, taosArrayGet(pUserAuthData, i), sizeof(bool))) {
+    if (TSDB_CODE_SUCCESS != putMetaDataToHash(key, len, pUserAuthData, i, pUserAuth)) {
       return TSDB_CODE_OUT_OF_MEMORY;
     }
   }
@@ -624,8 +601,7 @@ static int32_t putUdfToCache(const SArray* pUdfReq, const SArray* pUdfData, SHas
   int32_t num = taosArrayGetSize(pUdfReq);
   for (int32_t i = 0; i < num; ++i) {
     char* pFunc = taosArrayGet(pUdfReq, i);
-    SFuncInfo* pInfo = taosArrayGet(pUdfData, i);
+    if (TSDB_CODE_SUCCESS != putMetaDataToHash(pFunc, strlen(pFunc), pUdfData, i, pUdf)) {
-    if (TSDB_CODE_SUCCESS != taosHashPut(pUdf, pFunc, strlen(pFunc), &pInfo, POINTER_BYTES)) {
       return TSDB_CODE_OUT_OF_MEMORY;
     }
   }
@@ -633,15 +609,18 @@ static int32_t putUdfToCache(const SArray* pUdfReq, const SArray* pUdfData, SHas
 }
 
 int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache) {
-  int32_t code = putTableMetaToCache(pCatalogReq->pTableMeta, pMetaData->pTableMeta, pMetaCache->pTableMeta);
+  int32_t code = putTableDataToCache(pCatalogReq->pTableMeta, pMetaData->pTableMeta, pMetaCache->pTableMeta);
   if (TSDB_CODE_SUCCESS == code) {
-    code = putDbVgroupToCache(pCatalogReq->pDbVgroup, pMetaData->pDbVgroup, pMetaCache->pDbVgroup);
+    code = putDbDataToCache(pCatalogReq->pDbVgroup, pMetaData->pDbVgroup, pMetaCache->pDbVgroup);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = putTableVgroupToCache(pCatalogReq->pTableHash, pMetaData->pTableHash, pMetaCache->pTableVgroup);
+    code = putTableDataToCache(pCatalogReq->pTableHash, pMetaData->pTableHash, pMetaCache->pTableVgroup);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = putDbCfgToCache(pCatalogReq->pDbCfg, pMetaData->pDbCfg, pMetaCache->pDbCfg);
+    code = putDbDataToCache(pCatalogReq->pDbCfg, pMetaData->pDbCfg, pMetaCache->pDbCfg);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = putDbDataToCache(pCatalogReq->pDbInfo, pMetaData->pDbInfo, pMetaCache->pDbInfo);
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = putUserAuthToCache(pCatalogReq->pUser, pMetaData->pUser, pMetaCache->pUserAuth);
@@ -681,16 +660,15 @@ int32_t reserveTableMetaInCacheExt(const SName* pName, SParseMetaCache* pMetaCac
 int32_t getTableMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta) {
   char fullName[TSDB_TABLE_FNAME_LEN];
   tNameExtractFullName(pName, fullName);
-  STableMeta** pRes = taosHashGet(pMetaCache->pTableMeta, fullName, strlen(fullName));
+  STableMeta* pTableMeta = NULL;
-  if (NULL == pRes || NULL == *pRes) {
+  int32_t     code = getMetaDataFromHash(fullName, strlen(fullName), pMetaCache->pTableMeta, (void**)&pTableMeta);
-    parserError("getTableMetaFromCache error: %s", fullName);
+  if (TSDB_CODE_SUCCESS == code) {
-    return TSDB_CODE_PAR_INTERNAL_ERROR;
+    *pMeta = tableMetaDup(pTableMeta);
-  }
-  *pMeta = tableMetaDup(*pRes);
     if (NULL == *pMeta) {
-    return TSDB_CODE_OUT_OF_MEMORY;
+      code = TSDB_CODE_OUT_OF_MEMORY;
     }
-  return TSDB_CODE_SUCCESS;
+  }
+  return code;
 }
 
 static int32_t reserveDbReqInCache(int32_t acctId, const char* pDb, SHashObj** pDbs) {
@@ -710,19 +688,16 @@ int32_t reserveDbVgInfoInCache(int32_t acctId, const char* pDb, SParseMetaCache*
 }
 
 int32_t getDbVgInfoFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SArray** pVgInfo) {
-  SArray** pRes = taosHashGet(pMetaCache->pDbVgroup, pDbFName, strlen(pDbFName));
+  SArray* pVgList = NULL;
-  if (NULL == pRes) {
+  int32_t code = getMetaDataFromHash(pDbFName, strlen(pDbFName), pMetaCache->pDbVgroup, (void**)&pVgList);
-    parserError("getDbVgInfoFromCache error: %s", pDbFName);
+  // pVgList is null, which is a legal value, indicating that the user DB has not been created
-    return TSDB_CODE_PAR_INTERNAL_ERROR;
+  if (TSDB_CODE_SUCCESS == code && NULL != pVgList) {
-  }
+    *pVgInfo = taosArrayDup(pVgList);
-  // *pRes is null, which is a legal value, indicating that the user DB has not been created
-  if (NULL != *pRes) {
-    *pVgInfo = taosArrayDup(*pRes);
     if (NULL == *pVgInfo) {
-      return TSDB_CODE_OUT_OF_MEMORY;
+      code = TSDB_CODE_OUT_OF_MEMORY;
     }
   }
-  return TSDB_CODE_SUCCESS;
+  return code;
 }
 
 int32_t reserveTableVgroupInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache) {
@@ -738,30 +713,28 @@ int32_t reserveTableVgroupInCacheExt(const SName* pName, SParseMetaCache* pMetaC
 int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, SVgroupInfo* pVgroup) {
   char fullName[TSDB_TABLE_FNAME_LEN];
   tNameExtractFullName(pName, fullName);
-  SVgroupInfo** pRes = taosHashGet(pMetaCache->pTableVgroup, fullName, strlen(fullName));
+  SVgroupInfo* pVg = NULL;
-  if (NULL == pRes || NULL == *pRes) {
+  int32_t      code = getMetaDataFromHash(fullName, strlen(fullName), pMetaCache->pTableVgroup, (void**)&pVg);
-    parserError("getTableVgroupFromCache error: %s", fullName);
+  if (TSDB_CODE_SUCCESS == code) {
-    return TSDB_CODE_PAR_INTERNAL_ERROR;
+    memcpy(pVgroup, pVg, sizeof(SVgroupInfo));
   }
-  memcpy(pVgroup, *pRes, sizeof(SVgroupInfo));
+  return code;
-  return TSDB_CODE_SUCCESS;
 }
 
 int32_t reserveDbVgVersionInCache(int32_t acctId, const char* pDb, SParseMetaCache* pMetaCache) {
-  return reserveDbReqInCache(acctId, pDb, &pMetaCache->pDbCfg);
+  return reserveDbReqInCache(acctId, pDb, &pMetaCache->pDbInfo);
 }
 
 int32_t getDbVgVersionFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, int32_t* pVersion, int64_t* pDbId,
                                 int32_t* pTableNum) {
-  SDbInfo** pRes = taosHashGet(pMetaCache->pDbCfg, pDbFName, strlen(pDbFName));
+  SDbInfo* pDbInfo = NULL;
-  if (NULL == pRes || NULL == *pRes) {
+  int32_t  code = getMetaDataFromHash(pDbFName, strlen(pDbFName), pMetaCache->pDbInfo, (void**)&pDbInfo);
-    parserError("getDbVgVersionFromCache error: %s", pDbFName);
+  if (TSDB_CODE_SUCCESS == code) {
-    return TSDB_CODE_PAR_INTERNAL_ERROR;
+    *pVersion = pDbInfo->vgVer;
+    *pDbId = pDbInfo->dbId;
+    *pTableNum = pDbInfo->tbNum;
   }
-  *pVersion = (*pRes)->vgVer;
+  return code;
-  *pDbId = (*pRes)->dbId;
-  *pTableNum = (*pRes)->tbNum;
-  return TSDB_CODE_SUCCESS;
 }
 
 int32_t reserveDbCfgInCache(int32_t acctId, const char* pDb, SParseMetaCache* pMetaCache) {
@@ -769,13 +742,12 @@ int32_t reserveDbCfgInCache(int32_t acctId, const char* pDb, SParseMetaCache* pM
 }
 
 int32_t getDbCfgFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SDbCfgInfo* pInfo) {
-  SDbCfgInfo** pRes = taosHashGet(pMetaCache->pDbCfg, pDbFName, strlen(pDbFName));
+  SDbCfgInfo* pDbCfg = NULL;
-  if (NULL == pRes || NULL == *pRes) {
+  int32_t     code = getMetaDataFromHash(pDbFName, strlen(pDbFName), pMetaCache->pDbCfg, (void**)&pDbCfg);
-    parserError("getDbCfgFromCache error: %s", pDbFName);
+  if (TSDB_CODE_SUCCESS == code) {
-    return TSDB_CODE_PAR_INTERNAL_ERROR;
+    memcpy(pInfo, pDbCfg, sizeof(SDbCfgInfo));
   }
-  memcpy(pInfo, *pRes, sizeof(SDbCfgInfo));
+  return code;
-  return TSDB_CODE_SUCCESS;
 }
 
 static int32_t reserveUserAuthInCacheImpl(const char* pKey, int32_t len, SParseMetaCache* pMetaCache) {
@@ -808,13 +780,12 @@ int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, const char* pUser, con
                              bool* pPass) {
   char    key[USER_AUTH_KEY_MAX_LEN] = {0};
   int32_t len = userAuthToStringExt(pUser, pDbFName, type, key);
-  bool*   pRes = taosHashGet(pMetaCache->pUserAuth, key, len);
+  bool*   pRes = NULL;
-  if (NULL == pRes) {
+  int32_t code = getMetaDataFromHash(key, len, pMetaCache->pUserAuth, (void**)&pRes);
-    parserError("getUserAuthFromCache error: %s, %s, %d", pUser, pDbFName, type);
+  if (TSDB_CODE_SUCCESS == code) {
-    return TSDB_CODE_PAR_INTERNAL_ERROR;
-  }
     *pPass = *pRes;
-  return TSDB_CODE_SUCCESS;
+  }
+  return code;
 }
 
 int32_t reserveUdfInCache(const char* pFunc, SParseMetaCache* pMetaCache) {
@@ -828,11 +799,10 @@ int32_t reserveUdfInCache(const char* pFunc, SParseMetaCache* pMetaCache) {
 }
 
 int32_t getUdfInfoFromCache(SParseMetaCache* pMetaCache, const char* pFunc, SFuncInfo* pInfo) {
-  SFuncInfo** pRes = taosHashGet(pMetaCache->pUdf, pFunc, strlen(pFunc));
+  SFuncInfo* pFuncInfo = NULL;
-  if (NULL == pRes || NULL == *pRes) {
+  int32_t    code = getMetaDataFromHash(pFunc, strlen(pFunc), pMetaCache->pUdf, (void**)&pFuncInfo);
-    parserError("getUdfInfoFromCache error: %s", pFunc);
+  if (TSDB_CODE_SUCCESS == code) {
-    return TSDB_CODE_PAR_INTERNAL_ERROR;
+    memcpy(pInfo, pFuncInfo, sizeof(SFuncInfo));
   }
-  memcpy(pInfo, *pRes, sizeof(SFuncInfo));
+  return code;
-  return TSDB_CODE_SUCCESS;
 }

source/libs/parser/test/mockCatalogService.cpp

@@ -367,49 +367,40 @@ class MockCatalogServiceImpl {
   }
 
   int32_t getAllTableMeta(SArray* pTableMetaReq, SArray** pTableMetaData) const {
-    int32_t code = TSDB_CODE_SUCCESS;
     if (NULL != pTableMetaReq) {
       int32_t ntables = taosArrayGetSize(pTableMetaReq);
-      *pTableMetaData = taosArrayInit(ntables, POINTER_BYTES);
+      *pTableMetaData = taosArrayInit(ntables, sizeof(SMetaRes));
       for (int32_t i = 0; i < ntables; ++i) {
-        STableMeta* pMeta = NULL;
+        SMetaRes res = {0};
-        code = catalogGetTableMeta((const SName*)taosArrayGet(pTableMetaReq, i), &pMeta);
+        res.code = catalogGetTableMeta((const SName*)taosArrayGet(pTableMetaReq, i), (STableMeta**)&res.pRes);
-        if (TSDB_CODE_SUCCESS == code) {
+        taosArrayPush(*pTableMetaData, &res);
-          taosArrayPush(*pTableMetaData, &pMeta);
-        } else {
-          break;
       }
     }
-    }
+    return TSDB_CODE_SUCCESS;
-    return code;
   }
 
   int32_t getAllTableVgroup(SArray* pTableVgroupReq, SArray** pTableVgroupData) const {
-    int32_t code = TSDB_CODE_SUCCESS;
     if (NULL != pTableVgroupReq) {
       int32_t ntables = taosArrayGetSize(pTableVgroupReq);
-      *pTableVgroupData = taosArrayInit(ntables, sizeof(SVgroupInfo));
+      *pTableVgroupData = taosArrayInit(ntables, sizeof(SMetaRes));
       for (int32_t i = 0; i < ntables; ++i) {
-        SVgroupInfo vgInfo = {0};
+        SMetaRes res = {0};
-        code = catalogGetTableHashVgroup((const SName*)taosArrayGet(pTableVgroupReq, i), &vgInfo);
+        res.pRes = taosMemoryCalloc(1, sizeof(SVgroupInfo));
-        if (TSDB_CODE_SUCCESS == code) {
+        res.code = catalogGetTableHashVgroup((const SName*)taosArrayGet(pTableVgroupReq, i), (SVgroupInfo*)res.pRes);
-          taosArrayPush(*pTableVgroupData, &vgInfo);
+        taosArrayPush(*pTableVgroupData, &res);
-        } else {
-          break;
       }
     }
-    }
+    return TSDB_CODE_SUCCESS;
-    return code;
   }
 
   int32_t getAllDbVgroup(SArray* pDbVgroupReq, SArray** pDbVgroupData) const {
     int32_t code = TSDB_CODE_SUCCESS;
     if (NULL != pDbVgroupReq) {
       int32_t ndbs = taosArrayGetSize(pDbVgroupReq);
-      *pDbVgroupData = taosArrayInit(ndbs, POINTER_BYTES);
+      *pDbVgroupData = taosArrayInit(ndbs, sizeof(SMetaRes));
       for (int32_t i = 0; i < ndbs; ++i) {
-        int64_t zeroVg = 0;
+        SMetaRes res = {0};
-        taosArrayPush(*pDbVgroupData, &zeroVg);
+        taosArrayPush(*pDbVgroupData, &res);
       }
     }
     return code;
@@ -419,10 +410,11 @@ class MockCatalogServiceImpl {
     int32_t code = TSDB_CODE_SUCCESS;
     if (NULL != pDbCfgReq) {
       int32_t ndbs = taosArrayGetSize(pDbCfgReq);
-      *pDbCfgData = taosArrayInit(ndbs, sizeof(SDbCfgInfo));
+      *pDbCfgData = taosArrayInit(ndbs, sizeof(SMetaRes));
       for (int32_t i = 0; i < ndbs; ++i) {
-        SDbCfgInfo dbCfg = {0};
+        SMetaRes res = {0};
-        taosArrayPush(*pDbCfgData, &dbCfg);
+        res.pRes = taosMemoryCalloc(1, sizeof(SDbCfgInfo));
+        taosArrayPush(*pDbCfgData, &res);
       }
     }
     return code;
@@ -432,10 +424,11 @@ class MockCatalogServiceImpl {
     int32_t code = TSDB_CODE_SUCCESS;
     if (NULL != pDbInfoReq) {
       int32_t ndbs = taosArrayGetSize(pDbInfoReq);
-      *pDbInfoData = taosArrayInit(ndbs, sizeof(SDbCfgInfo));
+      *pDbInfoData = taosArrayInit(ndbs, sizeof(SMetaRes));
       for (int32_t i = 0; i < ndbs; ++i) {
-        SDbInfo dbInfo = {0};
+        SMetaRes res = {0};
-        taosArrayPush(*pDbInfoData, &dbInfo);
+        res.pRes = taosMemoryCalloc(1, sizeof(SDbInfo));
+        taosArrayPush(*pDbInfoData, &res);
       }
     }
     return code;
@@ -445,31 +438,29 @@ class MockCatalogServiceImpl {
     int32_t code = TSDB_CODE_SUCCESS;
     if (NULL != pUserAuthReq) {
       int32_t num = taosArrayGetSize(pUserAuthReq);
-      *pUserAuthData = taosArrayInit(num, sizeof(bool));
+      *pUserAuthData = taosArrayInit(num, sizeof(SMetaRes));
       for (int32_t i = 0; i < num; ++i) {
-        bool pass = true;
+        SMetaRes res = {0};
-        taosArrayPush(*pUserAuthData, &pass);
+        res.pRes = taosMemoryCalloc(1, sizeof(bool));
+        *(bool*)(res.pRes) = true;
+        taosArrayPush(*pUserAuthData, &res);
       }
     }
     return code;
   }
 
   int32_t getAllUdf(SArray* pUdfReq, SArray** pUdfData) const {
-    int32_t code = TSDB_CODE_SUCCESS;
     if (NULL != pUdfReq) {
       int32_t num = taosArrayGetSize(pUdfReq);
-      *pUdfData = taosArrayInit(num, sizeof(SFuncInfo));
+      *pUdfData = taosArrayInit(num, sizeof(SMetaRes));
       for (int32_t i = 0; i < num; ++i) {
-        SFuncInfo info = {0};
+        SMetaRes res = {0};
-        code = catalogGetUdfInfo((char*)taosArrayGet(pUdfReq, i), &info);
+        res.pRes = taosMemoryCalloc(1, sizeof(SFuncInfo));
-        if (TSDB_CODE_SUCCESS == code) {
+        res.code = catalogGetUdfInfo((char*)taosArrayGet(pUdfReq, i), (SFuncInfo*)res.pRes);
-          taosArrayPush(*pUdfData, &info);
+        taosArrayPush(*pUdfData, &res);
-        } else {
-          break;
       }
     }
-    }
+    return TSDB_CODE_SUCCESS;
-    return code;
   }
 
   uint64_t                      id_;

source/libs/parser/test/parInitialATest.cpp

@@ -39,6 +39,8 @@ TEST_F(ParserInitialATest, alterDatabase) {
   useDb("root", "test");
 
   run("ALTER DATABASE wxy_db CACHELAST 1 FSYNC 200 WAL 1");
+
+  run("ALTER DATABASE wxy_db KEEP 2400");
 }
 
 // todo ALTER local

source/libs/qcom/src/querymsg.c

@@ -312,10 +312,6 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isStb, STableMeta *
   pTableMeta->sversion = msg->sversion;
   pTableMeta->tversion = msg->tversion;
 
-  if (isStb) {
-    qDebug("stable %s meta returned, suid:%" PRIx64, msg->stbName, pTableMeta->suid);
-  }
-
   pTableMeta->tableInfo.numOfTags = msg->numOfTags;
   pTableMeta->tableInfo.precision = msg->precision;
   pTableMeta->tableInfo.numOfColumns = msg->numOfColumns;
@@ -326,6 +322,12 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isStb, STableMeta *
     pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes;
   }
 
+  qDebug("table %s uid %" PRIx64 " meta returned, type %d vgId %d db %s stb %s suid %" PRIx64 " sver %d tver %d" PRIx64 
+    " tagNum %d colNum %d precision %d rowSize %d", 
+    msg->tbName, pTableMeta->uid, pTableMeta->tableType, pTableMeta->vgId, msg->dbFName, msg->stbName, pTableMeta->suid,
+    pTableMeta->sversion, pTableMeta->tversion, pTableMeta->tableInfo.numOfTags, pTableMeta->tableInfo.numOfColumns,
+    pTableMeta->tableInfo.precision, pTableMeta->tableInfo.rowSize);
+
   *pMeta = pTableMeta;
   return TSDB_CODE_SUCCESS;
 }

source/libs/qworker/src/qwDbg.c

@@ -9,7 +9,7 @@
 #include "tmsg.h"
 #include "tname.h"
 
-SQWDebug     gQWDebug = {.statusEnable = true, .dumpEnable = true};
+SQWDebug     gQWDebug = {.statusEnable = true, .dumpEnable = false};
 
 int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, bool *ignore) {
   if (!gQWDebug.statusEnable) {
@@ -97,7 +97,11 @@ _return:
   QW_RET(code);
 }
 
-void qwDbgDumpSchInfo(SQWSchStatus *sch, int32_t i) {}
+void qwDbgDumpSchInfo(SQWorker *mgmt, SQWSchStatus *sch, int32_t i) {
+  QW_LOCK(QW_READ, &sch->tasksLock);
+  QW_DLOG("the %dth scheduler status, hbBrokenTs:%" PRId64 ",taskNum:%d", i, sch->hbBrokenTs, taosHashGetSize(sch->tasksHash));
+  QW_UNLOCK(QW_READ, &sch->tasksLock);
+}
 
 void qwDbgDumpMgmtInfo(SQWorker *mgmt) {
   if (!gQWDebug.dumpEnable) {
@@ -106,7 +110,7 @@ void qwDbgDumpMgmtInfo(SQWorker *mgmt) {
 
   QW_LOCK(QW_READ, &mgmt->schLock);
 
-  /*QW_DUMP("total remain schduler num:%d", taosHashGetSize(mgmt->schHash));*/
+  QW_DUMP("total remain scheduler num %d", taosHashGetSize(mgmt->schHash));
 
   void         *key = NULL;
   size_t        keyLen = 0;
@@ -116,14 +120,14 @@ void qwDbgDumpMgmtInfo(SQWorker *mgmt) {
   void *pIter = taosHashIterate(mgmt->schHash, NULL);
   while (pIter) {
     sch = (SQWSchStatus *)pIter;
-    qwDbgDumpSchInfo(sch, i);
+    qwDbgDumpSchInfo(mgmt, sch, i);
     ++i;
     pIter = taosHashIterate(mgmt->schHash, pIter);
   }
 
   QW_UNLOCK(QW_READ, &mgmt->schLock);
 
-  /*QW_DUMP("total remain ctx num:%d", taosHashGetSize(mgmt->ctxHash));*/
+  QW_DUMP("total remain ctx num %d", taosHashGetSize(mgmt->ctxHash));
 }
 
 

source/libs/scalar/inc/sclInt.h

@@ -22,11 +22,18 @@ extern "C" {
 #include "thash.h"
 #include "query.h"
 
+typedef struct SOperatorValueType {
+  int32_t opResType;
+  int32_t selfType;
+  int32_t peerType;
+} SOperatorValueType;
+
 typedef struct SScalarCtx {
   int32_t      code;
   SArray      *pBlockList;  /* element is SSDataBlock* */
   SHashObj    *pRes;  /* element is SScalarParam */
   void        *param;      // additional parameter (meta actually) for acquire value such as tbname/tags values
+  SOperatorValueType type;
 } SScalarCtx;
 
 
@@ -53,7 +60,7 @@ int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out);
 SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows);
 int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode);
 
-#define GET_PARAM_TYPE(_c)      ((_c)->columnData->info.type)
+#define GET_PARAM_TYPE(_c)      ((_c)->columnData ? (_c)->columnData->info.type : (_c)->hashValueType)
 #define GET_PARAM_BYTES(_c)     ((_c)->columnData->info.bytes)
 #define GET_PARAM_PRECISON(_c)  ((_c)->columnData->info.precision)
 

source/libs/scalar/src/filter.c

@@ -3669,6 +3669,14 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
           stat->scalarMode = true;
           return DEAL_RES_CONTINUE;
         }
+      } else {
+        SColumnNode *refNode = (SColumnNode *)node->pLeft;
+        SNodeListNode *listNode = (SNodeListNode *)node->pRight;
+        int32_t type = vectorGetConvertType(refNode->node.resType.type, listNode->dataType.type);
+        if (0 != type && type != refNode->node.resType.type) {
+          stat->scalarMode = true;
+          return DEAL_RES_CONTINUE;
+        }
       }
     }
 

source/libs/scalar/src/scalar.c

@@ -208,7 +208,13 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
         SCL_RET(TSDB_CODE_QRY_INVALID_INPUT);
       }
 
-      SCL_ERR_RET(scalarGenerateSetFromList((void **)&param->pHashFilter, node, nodeList->dataType.type));
+      int32_t type = vectorGetConvertType(ctx->type.selfType, ctx->type.peerType);
+      if (type == 0) {
+        type = nodeList->dataType.type;
+      }
+      
+      SCL_ERR_RET(scalarGenerateSetFromList((void **)&param->pHashFilter, node, type));
+      param->hashValueType = type;
       if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) {
         taosHashCleanup(param->pHashFilter);
         sclError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param));
@@ -334,6 +340,46 @@ _return:
   SCL_RET(code);
 }
 
+int32_t sclGetNodeType(SNode *pNode, SScalarCtx *ctx) {
+  if (NULL == pNode) {
+    return -1;
+  }
+  
+  switch (nodeType(pNode)) {
+    case QUERY_NODE_VALUE: {
+      SValueNode *valueNode = (SValueNode *)pNode;
+      return valueNode->node.resType.type;
+    }
+    case QUERY_NODE_NODE_LIST: {
+      SNodeListNode *nodeList = (SNodeListNode *)pNode;
+      return nodeList->dataType.type;
+    }
+    case QUERY_NODE_COLUMN: {
+      SColumnNode *colNode = (SColumnNode *)pNode;
+      return colNode->node.resType.type;
+    }
+    case QUERY_NODE_FUNCTION:
+    case QUERY_NODE_OPERATOR:
+    case QUERY_NODE_LOGIC_CONDITION: {
+      SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, &pNode, POINTER_BYTES);
+      if (NULL == res) {
+        sclError("no result for node, type:%d, node:%p", nodeType(pNode), pNode);
+        return -1;
+      }
+      return res->columnData->info.type;
+    }
+  }
+
+  return -1;
+}
+
+
+void sclSetOperatorValueType(SOperatorNode *node, SScalarCtx *ctx) {
+  ctx->type.opResType = node->node.resType.type;
+  ctx->type.selfType = sclGetNodeType(node->pLeft, ctx);
+  ctx->type.peerType = sclGetNodeType(node->pRight, ctx);
+}
+
 int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScalarCtx *ctx, int32_t *rowNum) {
   int32_t code = 0;
   int32_t paramNum = scalarGetOperatorParamNum(node->opType);
@@ -348,8 +394,11 @@ int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScal
     SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
   }
 
+  sclSetOperatorValueType(node, ctx);
+
   SCL_ERR_JRET(sclInitParam(node->pLeft, &paramList[0], ctx, rowNum));
   if (paramNum > 1) {
+    TSWAP(ctx->type.selfType, ctx->type.peerType);
     SCL_ERR_JRET(sclInitParam(node->pRight, &paramList[1], ctx, rowNum));
   }
 

source/libs/scalar/src/sclfunc.c

@@ -901,13 +901,23 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *
 
     if (hasFraction) {
       int32_t fracLen = (int32_t)strlen(fraction) + 1;
-      char *tzInfo = strchr(buf, '+');
+
+      char *tzInfo;
+      if (buf[len - 1] == 'z' || buf[len - 1] == 'Z') {
+        tzInfo = &buf[len - 1];
+        memmove(tzInfo + fracLen, tzInfo, strlen(tzInfo));
+      } else {
+        tzInfo = strchr(buf, '+');
         if (tzInfo) {
           memmove(tzInfo + fracLen, tzInfo, strlen(tzInfo));
         } else {
-        tzInfo = strchr(buf, '-');
+          //search '-' backwards
+          tzInfo = strrchr(buf, '-');
+          if (tzInfo) {
             memmove(tzInfo + fracLen, tzInfo, strlen(tzInfo));
           }
+        }
+      }
 
       char tmp[32] = {0};
       sprintf(tmp, ".%s", fraction);

source/libs/scalar/src/sclvector.c

@@ -627,6 +627,11 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) {
   SColumnInfoData* pInputCol  = pIn->columnData;
   SColumnInfoData* pOutputCol = pOut->columnData;
 
+  if (NULL == pInputCol) {
+    sclError("input column is NULL, hashFilter %p", pIn->pHashFilter);
+    return TSDB_CODE_APP_ERROR;
+  }
+  
   int16_t inType   = pInputCol->info.type;
   int16_t outType  = pOutputCol->info.type;
 
@@ -827,11 +832,26 @@ int32_t vectorGetConvertType(int32_t type1, int32_t type2) {
   return gConvertTypes[type2][type1];
 }
 
-int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* pLeftOut, SScalarParam* pRightOut) {
+int32_t vectorConvertScalarParam(SScalarParam *input, SScalarParam *output, int32_t type) {
-  if (pLeft->pHashFilter != NULL || pRight->pHashFilter != NULL) {
+  int32_t code = 0;
+  SDataType t = {.type = type, .bytes = tDataTypes[type].bytes};
+  output->numOfRows = input->numOfRows;
+
+  output->columnData = createColumnInfoData(&t, input->numOfRows);
+  if (output->columnData == NULL) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  code = vectorConvertImpl(input, output);
+  if (code) {
+//      taosMemoryFreeClear(paramOut1->data);
+    return code;
+  }
+
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* pLeftOut, SScalarParam* pRightOut) {
   int32_t leftType  = GET_PARAM_TYPE(pLeft);
   int32_t rightType = GET_PARAM_TYPE(pRight);
   if (leftType == rightType) {
@@ -860,31 +880,14 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p
   }
 
   if (type != GET_PARAM_TYPE(param1)) {
-    SDataType t = {.type = type, .bytes = tDataTypes[type].bytes};
+    code = vectorConvertScalarParam(param1, paramOut1, type);
-    paramOut1->numOfRows = param1->numOfRows;
-
-    paramOut1->columnData = createColumnInfoData(&t, param1->numOfRows);
-    if (paramOut1->columnData == NULL) {
-      return terrno;
-    }
-
-    code = vectorConvertImpl(param1, paramOut1);
     if (code) {
-//      taosMemoryFreeClear(paramOut1->data);
       return code;
     }
   }
   
   if (type != GET_PARAM_TYPE(param2)) {
-    SDataType t = {.type = type, .bytes = tDataTypes[type].bytes};
+    code = vectorConvertScalarParam(param2, paramOut2, type);
-    paramOut2->numOfRows = param2->numOfRows;
-
-    paramOut2->columnData = createColumnInfoData(&t, param2->numOfRows);
-    if (paramOut2->columnData == NULL) {
-      return terrno;
-    }
-
-    code = vectorConvertImpl(param2, paramOut2);
     if (code) {
       return code;
     }

source/libs/scheduler/src/schJob.c

@@ -545,7 +545,7 @@ int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) {
         SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
       }
 
-      SCH_TASK_DLOG("set %dth condidate addr, id %d, fqdn:%s, port:%d", i, naddr->nodeId, SCH_GET_CUR_EP(naddr)->fqdn, SCH_GET_CUR_EP(naddr)->port);
+      SCH_TASK_DLOG("set %dth candidate addr, id %d, fqdn:%s, port:%d", i, naddr->nodeId, SCH_GET_CUR_EP(naddr)->fqdn, SCH_GET_CUR_EP(naddr)->port);
 
       ++addNum;
     }
@@ -897,6 +897,7 @@ int32_t schProcessOnJobFailureImpl(SSchJob *pJob, int32_t status, int32_t errCod
     if (SCH_EXEC_CB == atomic_val_compare_exchange_32(&pJob->userCb, SCH_EXEC_CB, 0)) {
       schNotifyUserQueryRes(pJob);
     } else if (SCH_FETCH_CB == atomic_val_compare_exchange_32(&pJob->userCb, SCH_FETCH_CB, 0)) {
+      atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0);
       schNotifyUserFetchRes(pJob);
     }
   }
@@ -925,6 +926,7 @@ int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) {
   } else if (SCH_EXEC_CB == atomic_val_compare_exchange_32(&pJob->userCb, SCH_EXEC_CB, 0)) {
     schNotifyUserQueryRes(pJob);
   } else if (SCH_FETCH_CB == atomic_val_compare_exchange_32(&pJob->userCb, SCH_FETCH_CB, 0)) {
+    atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0);
     schNotifyUserFetchRes(pJob);
   }
 
@@ -945,6 +947,8 @@ void schProcessOnDataFetched(SSchJob *job) {
   if (job->attr.syncSchedule) {
     tsem_post(&job->rspSem);
   } else if (SCH_FETCH_CB == atomic_val_compare_exchange_32(&job->userCb, SCH_FETCH_CB, 0)) {
+    atomic_val_compare_exchange_8(&job->userFetch, 1, 0);
+    
     schNotifyUserFetchRes(job);
   }
 }
@@ -1680,9 +1684,9 @@ int32_t schAsyncFetchRows(SSchJob *pJob) {
   }
 
   if (pJob->attr.explainMode == EXPLAIN_MODE_STATIC) {
-    SCH_ERR_JRET(schNotifyUserFetchRes(pJob));
-    
     atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0);
+
+    SCH_ERR_JRET(schNotifyUserFetchRes(pJob));    
   } else {
     pJob->userCb = SCH_FETCH_CB;
     

source/libs/stream/src/stream.c

@@ -59,7 +59,7 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg*
 
   // rsp by input status
   void* buf = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamDispatchRsp));
-  ((SMsgHead*)buf)->vgId = htonl(pReq->sourceVg);
+  ((SMsgHead*)buf)->vgId = htonl(pReq->upstreamNodeId);
   SStreamDispatchRsp* pCont = POINTER_SHIFT(buf, sizeof(SMsgHead));
   pCont->inputStatus = status;
   pCont->streamId = pReq->streamId;
@@ -78,7 +78,18 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDisp
   // 2.1. idle: exec
   // 2.2. executing: return
   // 2.3. closing: keep trying
+  if (pTask->execType != TASK_EXEC__NONE) {
     streamExec(pTask, pMsgCb);
+  } else {
+    ASSERT(pTask->sinkType != TASK_SINK__NONE);
+    while (1) {
+      void* data = streamQueueNextItem(pTask->inputQueue);
+      if (data == NULL) return 0;
+      if (streamTaskOutput(pTask, data) < 0) {
+        ASSERT(0);
+      }
+    }
+  }
 
   // 3. handle output
   // 3.1 check and set status

source/libs/stream/src/streamDispatch.c

@@ -22,6 +22,7 @@ int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* p
   if (tEncodeI32(pEncoder, pReq->sourceTaskId) < 0) return -1;
   if (tEncodeI32(pEncoder, pReq->sourceVg) < 0) return -1;
   if (tEncodeI32(pEncoder, pReq->sourceChildId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pReq->upstreamNodeId) < 0) return -1;
   if (tEncodeI32(pEncoder, pReq->blockNum) < 0) return -1;
   ASSERT(taosArrayGetSize(pReq->data) == pReq->blockNum);
   ASSERT(taosArrayGetSize(pReq->dataLen) == pReq->blockNum);
@@ -42,6 +43,7 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) {
   if (tDecodeI32(pDecoder, &pReq->sourceTaskId) < 0) return -1;
   if (tDecodeI32(pDecoder, &pReq->sourceVg) < 0) return -1;
   if (tDecodeI32(pDecoder, &pReq->sourceChildId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pReq->upstreamNodeId) < 0) return -1;
   if (tDecodeI32(pDecoder, &pReq->blockNum) < 0) return -1;
   ASSERT(pReq->blockNum > 0);
   pReq->data = taosArrayInit(pReq->blockNum, sizeof(void*));
@@ -94,6 +96,7 @@ int32_t streamBuildDispatchMsg(SStreamTask* pTask, SStreamDataBlock* data, SRpcM
       .sourceTaskId = pTask->taskId,
       .sourceVg = data->sourceVg,
       .sourceChildId = pTask->childId,
+      .upstreamNodeId = pTask->nodeId,
       .blockNum = blockNum,
   };
 
@@ -184,13 +187,17 @@ int32_t streamDispatch(SStreamTask* pTask, SMsgCb* pMsgCb) {
 #endif
 
   SStreamDataBlock* pBlock = streamQueueNextItem(pTask->outputQueue);
-  if (pBlock == NULL) return 0;
+  if (pBlock == NULL) {
+    atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL);
+    return 0;
+  }
   ASSERT(pBlock->type == STREAM_DATA_TYPE_SSDATA_BLOCK);
 
   SRpcMsg dispatchMsg = {0};
   SEpSet* pEpSet = NULL;
   if (streamBuildDispatchMsg(pTask, pBlock, &dispatchMsg, &pEpSet) < 0) {
     ASSERT(0);
+    atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL);
     return -1;
   }
 

source/libs/stream/src/streamExec.c

@@ -65,6 +65,7 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) {
       }
       qRes->type = STREAM_INPUT__DATA_BLOCK;
       qRes->blocks = pRes;
+      /*qRes->sourceVg = pTask->nodeId;*/
       if (streamTaskOutput(pTask, qRes) < 0) {
         streamQueueProcessFail(pTask->inputQueue);
         taosArrayDestroy(pRes);

source/libs/sync/src/syncMain.c

@@ -281,7 +281,7 @@ int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg) {
   return ret;
 }
 
-int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg) {
+int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcHandleInfo* pInfo) {
   SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid);
   if (pSyncNode == NULL) {
     return TAOS_SYNC_STATE_ERROR;
@@ -291,7 +291,7 @@ int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg) {
   SRespStub stub;
   int32_t   ret = syncRespMgrGetAndDel(pSyncNode->pSyncRespMgr, index, &stub);
   if (ret == 1) {
-    memcpy(msg, &(stub.rpcMsg), sizeof(SRpcMsg));
+    *pInfo = stub.rpcMsg.info;
   }
 
   taosReleaseRef(tsNodeRefId, pSyncNode->rid);

tests/script/tsim/query/complex_having.sim

@@ -105,7 +105,7 @@ endi
 sql  select sum(c1) ,count(c1) from ct4 group by c1 having count(c7)  < 1  or  sum(c1) > 2 ;
 print ====> sql : select sum(c1) ,count(c1) from ct4 group by c1 having count(c7)  < 1  or  sum(c1) > 2 ;
 print ====> rows: $rows
-if $rows != 7 then
+if $rows != 8 then
     return -1
 endi
 
@@ -214,8 +214,8 @@ print =================== count all rows
 sql select count(c1) from stb1
 print ====> sql : select count(c1) from stb1
 print ====> rows: $data00
-if $data00 != 20 then
+if $data00 != 17 then
-  print expect 20, actual: $data00
+  print expect 17, actual: $data00
   return -1
 endi
 
@@ -246,7 +246,7 @@ print =================== count all rows
 sql select count(c1) from stb1
 print ====> sql : select count(c1) from stb1
 print ====> rows: $data00
-if $data00 != 20 then
+if $data00 != 17 then
   return -1
 endi
 
@@ -279,7 +279,7 @@ endi
 sql  select sum(c1) ,count(c1) from ct4 group by c1 having count(c7)  < 1  or  sum(c1) > 2 ;
 print ====> sql : select sum(c1) ,count(c1) from ct4 group by c1 having count(c7)  < 1  or  sum(c1) > 2 ;
 print ====> rows: $rows
-if $rows != 7 then
+if $rows != 8 then
     return -1
 endi
 

tests/script/tsim/query/crash_sql.sim

@@ -25,7 +25,7 @@ sql connect
 print =============== create database
 sql create database db
 sql show databases
-if $rows != 2 then
+if $rows != 3 then
   return -1
 endi
 
@@ -96,8 +96,6 @@ sql insert into ct4 values ( '2022-05-21 01:01:01.000', NULL, NULL, NULL, NULL,
 
 print ================ start query ======================
 print ================ SQL used to cause taosd or taos shell  crash
-sql select sum(c1) ,count(c1) from ct4 group by c1 having  sum(c10)  between 0 and 1 ;
+sql_error select sum(c1) ,count(c1) from ct4 group by c1 having  sum(c10)  between 0 and 1 ;
-
-
 
 #system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/stable/tag_filter.sim

@@ -31,6 +31,7 @@ if $rows != 1 then
   return -1
 endi
 
+
 sql select * from db.stb where t1 < 1 
 if $rows != 0 then 
   return -=1
@@ -56,4 +57,21 @@ if $rows != 5 then
   return -1
 endi
 
+sql select * from db.stb where t1 between 1 and 1 
+if $rows != 1 then 
+  return -1
+endi
+
+sql select * from db.stb where t1 between 1 and 6 
+if $rows != 6 then 
+  return -1
+endi
+
+sql select * from db.stb where t1 between 1 and 7 
+if $rows != 6 then 
+  return -1
+endi
+
+
+
 system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/system-test/2-query/query_cols_tags_and_or.py

@@ -1066,9 +1066,7 @@ class TDTestCase:
         tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
         # in
         query_sql = f'select {select_elm} from {tb_name} where c5 in (1, 6.6)'
-        tdSql.query(query_sql)
+        tdSql.error(query_sql)
-        tdSql.checkRows(1)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 6) if select_elm == "*" else False
         # not in
         query_sql = f'select {select_elm} from {tb_name} where c5 not in (2, 3)'
         tdSql.query(query_sql)
@@ -1076,14 +1074,10 @@ class TDTestCase:
         tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
         # and
         query_sql = f'select {select_elm} from {tb_name} where c5 > 0 and c5 >= 1 and c5 < 7 and c5 <= 6.6 and c5 != 2 and c5 <> 2 and c5 = 6.6 and c5 is not null and c5 between 2 and 6.6 and c5 not between 1 and 2 and c5 in (2,6.6) and c5 not in (1,2)'
-        tdSql.query(query_sql)
+        tdSql.error(query_sql)
-        tdSql.checkRows(1)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 6) if select_elm == "*" else False
         # or
         query_sql = f'select {select_elm} from {tb_name} where c5 > 6 or c5 >= 6.6 or c5 < 1 or c5 <= 0 or c5 != 1.1 or c5 <> 1.1 or c5 = 5 or c5 is null or c5 between 4 and 5 or c5 not between 1 and 3 or c5 in (4,5) or c5 not in (1.1,3)'
-        tdSql.query(query_sql)
+        tdSql.error(query_sql)
-        tdSql.checkRows(1)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 6) if select_elm == "*" else False
         # and or
         query_sql = f'select {select_elm} from {tb_name} where c5 > 0 and c5 >= 1 or c5 < 5 and c5 <= 6.6 and c5 != 2 and c5 <> 2 and c5 = 4 or c5 is not null and c5 between 2 and 4 and c5 not between 1 and 2 and c5 in (2,4) and c5 not in (1,2)'
         tdSql.query(query_sql)
@@ -1151,9 +1145,7 @@ class TDTestCase:
         tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
         # in
         query_sql = f'select {select_elm} from {tb_name} where c6 in (1, 7.7)'
-        tdSql.query(query_sql)
+        tdSql.error(query_sql)
-        tdSql.checkRows(1)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 7) if select_elm == "*" else False
         # not in
         query_sql = f'select {select_elm} from {tb_name} where c6 not in (2, 3)'
         tdSql.query(query_sql)
@@ -1161,14 +1153,10 @@ class TDTestCase:
         tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
         # and
         query_sql = f'select {select_elm} from {tb_name} where c6 > 0 and c6 >= 1 and c6 < 8 and c6 <= 7.7 and c6 != 2 and c6 <> 2 and c6 = 7.7 and c6 is not null and c6 between 2 and 7.7 and c6 not between 1 and 2 and c6 in (2,7.7) and c6 not in (1,2)'
-        tdSql.query(query_sql)
+        tdSql.error(query_sql)
-        tdSql.checkRows(1)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 7) if select_elm == "*" else False
         # or
         query_sql = f'select {select_elm} from {tb_name} where c6 > 7 or c6 >= 7.7 or c6 < 1 or c6 <= 0 or c6 != 1.1 or c6 <> 1.1 or c6 = 5 or c6 is null or c6 between 4 and 5 or c6 not between 1 and 3 or c6 in (4,5) or c6 not in (1.1,3)'
-        tdSql.query(query_sql)
+        tdSql.error(query_sql)
-        tdSql.checkRows(1)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 7) if select_elm == "*" else False
         # and or
         query_sql = f'select {select_elm} from {tb_name} where c6 > 0 and c6 >= 1 or c6 < 5 and c6 <= 7.7 and c6 != 2 and c6 <> 2 and c6 = 4 or c6 is not null and c6 between 2 and 4 and c6 not between 1 and 2 and c6 in (2,4) and c6 not in (1,2)'
         tdSql.query(query_sql)
@@ -1410,9 +1398,7 @@ class TDTestCase:
         tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
         # in
         query_sql = f'select {select_elm} from {tb_name} where c9 in ("binar", false)'
-        tdSql.query(query_sql)
+        tdSql.error(query_sql)
-        tdSql.checkRows(2)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
         # # not in
         query_sql = f'select {select_elm} from {tb_name} where c9 not in (true)'
         tdSql.query(query_sql)
@@ -1421,19 +1407,13 @@ class TDTestCase:
 
         # # and
         query_sql = f'select {select_elm} from {tb_name} where c9 = true and c9 != "false" and c9 <> "binary" and c9 is not null and c9 in ("binary", true) and c9 not in ("binary")'
-        tdSql.query(query_sql)
+        tdSql.error(query_sql)
-        tdSql.checkRows(9)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 9) if select_elm == "*" else False
         # # or
         query_sql = f'select {select_elm} from {tb_name} where c9 = true or c9 != "false" or c9 <> "binary" or c9 = "true" or c9 is not null or c9 in ("binary", true) or c9 not in ("binary")'
-        tdSql.query(query_sql)
+        tdSql.error(query_sql)
-        tdSql.checkRows(11)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
         # # and or
         query_sql = f'select {select_elm} from {tb_name} where c9 = true and c9 != "false" or c9 <> "binary" or c9 = "true" and c9 is not null or c9 in ("binary", true) or c9 not in ("binary")'
-        tdSql.query(query_sql)
+        tdSql.error(query_sql)
-        tdSql.checkRows(11)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
         query_sql = f'select c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13 from {tb_name} where c9 > "binary" and c9 >= "binary8" or c9 < "binary9" and c9 <= "binary" and c9 != 2 and c9 <> 2 and c9 = 4 or c9 is not null and c9 between 2 and 4 and c9 not between 1 and 2 and c9 in (2,4) and c9 not in (1,2)'
         tdSql.query(query_sql)
         tdSql.checkRows(9)

tests/test/c/tmqSim.c

@@ -514,14 +514,14 @@ void* consumeThreadFunc(void* param) {
   err = tmq_unsubscribe(pInfo->tmq);
   if (err) {
     pError("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err));
-    pInfo->consumeMsgCnt = -1;
+    /*pInfo->consumeMsgCnt = -1;*/
-    return NULL;
+    /*return NULL;*/
   }
 
   err = tmq_consumer_close(pInfo->tmq);
   if (err) {
     pError("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err));
-    exit(-1);
+    /*exit(-1);*/
   }
   pInfo->tmq = NULL;