merge from 3.0

2.0/src/query/inc/qExecutor.h

@@ -582,9 +582,9 @@ typedef struct SOrderOperatorInfo {
 
 void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream);
 
-SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime);
+SOperatorInfo* createTableScanOperatorInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime);
 SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime);
-SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv);
+SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv);
 
 SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
 SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
@@ -622,7 +622,7 @@ void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p);
 
 SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows);
 
-void* destroyOutputBuf(SSDataBlock* pBlock);
+void* blockDataDestroy(SSDataBlock* pBlock);
 void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols);
 
 void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order);

2.0/src/query/src/qExecutor.c

@@ -336,7 +336,7 @@ SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numO
   return res;
 }
 
-void* destroyOutputBuf(SSDataBlock* pBlock) {
+void* blockDataDestroy(SSDataBlock* pBlock) {
   if (pBlock == NULL) {
     return NULL;
   }
@@ -4835,11 +4835,11 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr
       break;
     }
     case OP_TableSeqScan: {
-      pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv);
+      pRuntimeEnv->proot = createTableSeqScanOperatorInfo(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv);
       break;
     }
     case OP_DataBlocksOptScan: {
-      pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0);
+      pRuntimeEnv->proot = createTableScanOperatorInfo(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0);
       break;
     }
     case OP_TableScan: {
@@ -5162,7 +5162,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv*
   return pOperator;
 }
 
-SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) {
+SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) {
   STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo));
 
   pInfo->pTsdbReadHandle     = pTsdbQueryHandle;
@@ -5267,7 +5267,7 @@ void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInf
   }
 }
 
-SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime) {
+SOperatorInfo* createTableScanOperatorInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime) {
   assert(repeatTime > 0);
 
   STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo));
@@ -5278,7 +5278,7 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntime
   pInfo->order          = pRuntimeEnv->pQueryAttr->order.order;
 
   SOperatorInfo* pOptr = calloc(1, sizeof(SOperatorInfo));
-  pOptr->name          = "DataBlocksOptimizedScanOperator";
+  pOptr->name          = "TableScanOperator";
   pOptr->operatorType  = OP_DataBlocksOptScan;
   pOptr->pRuntimeEnv   = pRuntimeEnv;
   pOptr->blockingOptr  = false;
@@ -5373,7 +5373,7 @@ static void destroyGlobalAggOperatorInfo(void* param, int32_t numOfOutput) {
 static void destroySlimitOperatorInfo(void* param, int32_t numOfOutput) {
   SSLimitOperatorInfo *pInfo = (SSLimitOperatorInfo*) param;
   taosArrayDestroy(pInfo->orderColumnList);
-  pInfo->pRes = destroyOutputBuf(pInfo->pRes);
+  pInfo->pRes = blockDataDestroy(pInfo->pRes);
   tfree(pInfo->prevRow);
 }
 
@@ -6566,7 +6566,7 @@ static void doDestroyBasicInfo(SOptrBasicInfo* pInfo, int32_t numOfOutput) {
   tfree(pInfo->rowCellInfoOffset);
 
   cleanupResultRowInfo(&pInfo->resultRowInfo);
-  pInfo->pRes = destroyOutputBuf(pInfo->pRes);
+  pInfo->pRes = blockDataDestroy(pInfo->pRes);
 }
 
 static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) {
@@ -6590,7 +6590,7 @@ static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) {
 static void destroySFillOperatorInfo(void* param, int32_t numOfOutput) {
   SFillOperatorInfo* pInfo = (SFillOperatorInfo*) param;
   pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo);
-  pInfo->pRes = destroyOutputBuf(pInfo->pRes);
+  pInfo->pRes = blockDataDestroy(pInfo->pRes);
   tfree(pInfo->p);
 }
 
@@ -6607,12 +6607,12 @@ static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) {
 
 static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) {
   STagScanInfo* pInfo = (STagScanInfo*) param;
-  pInfo->pRes = destroyOutputBuf(pInfo->pRes);
+  pInfo->pRes = blockDataDestroy(pInfo->pRes);
 }
 
 static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) {
   SOrderOperatorInfo* pInfo = (SOrderOperatorInfo*) param;
-  pInfo->pDataBlock = destroyOutputBuf(pInfo->pDataBlock);
+  pInfo->pDataBlock = blockDataDestroy(pInfo->pDataBlock);
 }
 
 static void destroyConditionOperatorInfo(void* param, int32_t numOfOutput) {
@@ -6625,7 +6625,7 @@ static void destroyDistinctOperatorInfo(void* param, int32_t numOfOutput) {
   taosHashCleanup(pInfo->pSet);
   tfree(pInfo->buf);
   taosArrayDestroy(pInfo->pDistinctDataInfo);
-  pInfo->pRes = destroyOutputBuf(pInfo->pRes);
+  pInfo->pRes = blockDataDestroy(pInfo->pRes);
 }
 
 SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) {

2.0/src/query/tests/resultBufferTest.cpp

@@ -13,7 +13,7 @@
 namespace {
 // simple test
 void simpleTest() {
-  SDiskbasedResultBuf* pResultBuf = NULL;
+  SDiskbasedBuf* pResultBuf = NULL;
   int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4096, 1);
   
   int32_t pageId = 0;
@@ -22,40 +22,40 @@ void simpleTest() {
   tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
   ASSERT_TRUE(pBufPage != NULL);
   
-  ASSERT_EQ(getResBufSize(pResultBuf), 1024);
+  ASSERT_EQ(getTotalBufSize(pResultBuf), 1024);
   
   SIDList list = getDataBufPagesIdList(pResultBuf, groupId);
   ASSERT_EQ(taosArrayGetSize(list), 1);
   ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1);
 
-  releaseResBufPage(pResultBuf, pBufPage);
+  releaseBufPage(pResultBuf, pBufPage);
 
   tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId);
 
-  tFilePage* t = getResBufPage(pResultBuf, pageId);
+  tFilePage* t = getBufPage(pResultBuf, pageId);
   ASSERT_TRUE(t == pBufPage1);
 
   tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId);
-  tFilePage* t1 = getResBufPage(pResultBuf, pageId);
+  tFilePage* t1 = getBufPage(pResultBuf, pageId);
   ASSERT_TRUE(t1 == pBufPage2);
 
   tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId);
-  tFilePage* t2 = getResBufPage(pResultBuf, pageId);
+  tFilePage* t2 = getBufPage(pResultBuf, pageId);
   ASSERT_TRUE(t2 == pBufPage3);
 
   tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId);
-  tFilePage* t3 = getResBufPage(pResultBuf, pageId);
+  tFilePage* t3 = getBufPage(pResultBuf, pageId);
   ASSERT_TRUE(t3 == pBufPage4);
 
   tFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId);
-  tFilePage* t4 = getResBufPage(pResultBuf, pageId);
+  tFilePage* t4 = getBufPage(pResultBuf, pageId);
   ASSERT_TRUE(t4 == pBufPage5);
 
   destroyResultBuf(pResultBuf);
 }
 
 void writeDownTest() {
-  SDiskbasedResultBuf* pResultBuf = NULL;
+  SDiskbasedBuf* pResultBuf = NULL;
   int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4*1024, 1);
 
   int32_t pageId = 0;
@@ -68,31 +68,31 @@ void writeDownTest() {
 
   *(int32_t*)(pBufPage->data) = nx;
   writePageId = pageId;
-  releaseResBufPage(pResultBuf, pBufPage);
+  releaseBufPage(pResultBuf, pBufPage);
 
   tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId);
-  tFilePage* t1 = getResBufPage(pResultBuf, pageId);
+  tFilePage* t1 = getBufPage(pResultBuf, pageId);
   ASSERT_TRUE(t1 == pBufPage1);
   ASSERT_TRUE(pageId == 1);
 
   tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId);
-  tFilePage* t2 = getResBufPage(pResultBuf, pageId);
+  tFilePage* t2 = getBufPage(pResultBuf, pageId);
   ASSERT_TRUE(t2 == pBufPage2);
   ASSERT_TRUE(pageId == 2);
 
   tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId);
-  tFilePage* t3 = getResBufPage(pResultBuf, pageId);
+  tFilePage* t3 = getBufPage(pResultBuf, pageId);
   ASSERT_TRUE(t3 == pBufPage3);
   ASSERT_TRUE(pageId == 3);
 
   tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId);
-  tFilePage* t4 = getResBufPage(pResultBuf, pageId);
+  tFilePage* t4 = getBufPage(pResultBuf, pageId);
   ASSERT_TRUE(t4 == pBufPage4);
   ASSERT_TRUE(pageId == 4);
-  releaseResBufPage(pResultBuf, t4);
+  releaseBufPage(pResultBuf, t4);
 
   // flush the written page to disk, and read it out again
-  tFilePage* pBufPagex = getResBufPage(pResultBuf, writePageId);
+  tFilePage* pBufPagex = getBufPage(pResultBuf, writePageId);
   ASSERT_EQ(*(int32_t*)pBufPagex->data, nx);
 
   SArray* pa = getDataBufPagesIdList(pResultBuf, groupId);
@@ -102,7 +102,7 @@ void writeDownTest() {
 }
 
 void recyclePageTest() {
-  SDiskbasedResultBuf* pResultBuf = NULL;
+  SDiskbasedBuf* pResultBuf = NULL;
   int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4*1024, 1);
 
   int32_t pageId = 0;
@@ -112,41 +112,41 @@ void recyclePageTest() {
 
   tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
   ASSERT_TRUE(pBufPage != NULL);
-  releaseResBufPage(pResultBuf, pBufPage);
+  releaseBufPage(pResultBuf, pBufPage);
 
   tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId);
-  tFilePage* t1 = getResBufPage(pResultBuf, pageId);
+  tFilePage* t1 = getBufPage(pResultBuf, pageId);
   ASSERT_TRUE(t1 == pBufPage1);
   ASSERT_TRUE(pageId == 1);
 
   tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId);
-  tFilePage* t2 = getResBufPage(pResultBuf, pageId);
+  tFilePage* t2 = getBufPage(pResultBuf, pageId);
   ASSERT_TRUE(t2 == pBufPage2);
   ASSERT_TRUE(pageId == 2);
 
   tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId);
-  tFilePage* t3 = getResBufPage(pResultBuf, pageId);
+  tFilePage* t3 = getBufPage(pResultBuf, pageId);
   ASSERT_TRUE(t3 == pBufPage3);
   ASSERT_TRUE(pageId == 3);
 
   tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId);
-  tFilePage* t4 = getResBufPage(pResultBuf, pageId);
+  tFilePage* t4 = getBufPage(pResultBuf, pageId);
   ASSERT_TRUE(t4 == pBufPage4);
   ASSERT_TRUE(pageId == 4);
-  releaseResBufPage(pResultBuf, t4);
+  releaseBufPage(pResultBuf, t4);
 
   tFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId);
-  tFilePage* t5 = getResBufPage(pResultBuf, pageId);
+  tFilePage* t5 = getBufPage(pResultBuf, pageId);
   ASSERT_TRUE(t5 == pBufPage5);
   ASSERT_TRUE(pageId == 5);
 
   // flush the written page to disk, and read it out again
-  tFilePage* pBufPagex = getResBufPage(pResultBuf, writePageId);
+  tFilePage* pBufPagex = getBufPage(pResultBuf, writePageId);
   *(int32_t*)(pBufPagex->data) = nx;
   writePageId = pageId;   // update the data
-  releaseResBufPage(pResultBuf, pBufPagex);
+  releaseBufPage(pResultBuf, pBufPagex);
 
-  tFilePage* pBufPagex1 = getResBufPage(pResultBuf, 1);
+  tFilePage* pBufPagex1 = getBufPage(pResultBuf, 1);
 
   SArray* pa = getDataBufPagesIdList(pResultBuf, groupId);
   ASSERT_EQ(taosArrayGetSize(pa), 6);

include/common/common.h

@@ -16,6 +16,11 @@
 #ifndef TDENGINE_COMMON_H
 #define TDENGINE_COMMON_H
 
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 #include "taosdef.h"
 #include "tarray.h"
 #include "tmsg.h"
@@ -80,12 +85,22 @@ typedef struct SSDataBlock {
   SDataBlockInfo  info;
 } SSDataBlock;
 
+typedef struct SVarColAttr {
+  int32_t *offset;    // start position for each entry in the list
+  uint32_t length;    // used buffer size that contain the valid data
+  uint32_t allocLen;  // allocated buffer size
+} SVarColAttr;
+
 // pBlockAgg->numOfNull == info.rows, all data are null
 // pBlockAgg->numOfNull == 0, no data are null.
 typedef struct SColumnInfoData {
   SColumnInfo info;   // TODO filter info needs to be removed
-  char*       nullbitmap;  //
+  bool        hasNull;// if current column data has null value.
   char       *pData;  // the corresponding block data in memory
+  union {
+    char       *nullbitmap;  // bitmap, one bit for each item in the list
+    SVarColAttr varmeta;
+  };
 } SColumnInfoData;
 
 static FORCE_INLINE int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) {
@@ -261,4 +276,8 @@ typedef struct SSessionWindow {
 
 #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP)
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif  // TDENGINE_COMMON_H

include/common/tep.h

@@ -7,12 +7,22 @@ extern "C" {
 
 #include "os.h"
 #include "tmsg.h"
+#include "common.h"
 
 typedef struct SCorEpSet {
   int32_t version;
   SEpSet  epSet;
 } SCorEpSet;
 
+typedef struct SBlockOrderInfo {
+  int32_t          order;
+  int32_t          colIndex;
+  SColumnInfoData *pColData;
+//  int32_t          type;
+//  int32_t          bytes;
+//  bool             hasNull;
+} SBlockOrderInfo;
+
 int  taosGetFqdnPortFromEp(const char *ep, SEp *pEp);
 void addEpIntoEpSet(SEpSet *pEpSet, const char *fqdn, uint16_t port);
 
@@ -21,6 +31,77 @@ bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2);
 void   updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet);
 SEpSet getEpSet_s(SCorEpSet *pEpSet);
 
+#define NBIT                     (3u)
+#define BitPos(_n)               ((_n) & ((1 << NBIT) - 1))
+#define BMCharPos(bm_, r_)       ((bm_)[(r_) >> NBIT])
+#define colDataIsNull_f(bm_, r_) ((BMCharPos(bm_, r_) & (1u << (7u - BitPos(r_)))) == (1u << (7u - BitPos(r_))))
+
+#define colDataSetNull_f(bm_, r_)                    \
+  do {                                               \
+    BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \
+  } while (0)
+
+static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, uint32_t totalRows, uint32_t row, SColumnDataAgg* pColAgg) {
+  if (!pColumnInfoData->hasNull) {
+    return false;
+  }
+
+  if (pColAgg != NULL) {
+    if (pColAgg->numOfNull == totalRows) {
+      ASSERT(pColumnInfoData->nullbitmap == NULL);
+      return true;
+    } else if (pColAgg->numOfNull == 0) {
+      ASSERT(pColumnInfoData->nullbitmap == NULL);
+      return false;
+    }
+  }
+
+  if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
+    return pColumnInfoData->varmeta.offset[row] == -1;
+  } else {
+    if (pColumnInfoData->nullbitmap == NULL) {
+      return false;
+    }
+
+    return colDataIsNull_f(pColumnInfoData->nullbitmap, row);
+  }
+}
+
+#define colDataGet(p1_, r_)                                                          \
+  ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? (p1_)->pData + (p1_)->varmeta.offset[(r_)] \
+                                        : (p1_)->pData + ((r_) * (p1_)->info.bytes));
+
+int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull);
+int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, uint32_t numOfRow2);
+int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock);
+
+int32_t colDataGetSize(const SColumnInfoData* pColumnInfoData, int32_t numOfRows);
+void colDataTrim(SColumnInfoData* pColumnInfoData);
+
+size_t colDataGetNumOfCols(const SSDataBlock* pBlock);
+size_t colDataGetNumOfRows(const SSDataBlock* pBlock);
+
+int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc);
+int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, int32_t pageSize);
+SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t rowCount);
+
+int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock);
+int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf);
+
+size_t blockDataGetSize(const SSDataBlock* pBlock);
+size_t blockDataGetRowSize(const SSDataBlock* pBlock);
+double blockDataGetSerialRowSize(const SSDataBlock* pBlock);
+size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock);
+
+size_t blockDataNumOfRowsForSerialize(const SSDataBlock* pBlock, int32_t blockSize);
+
+int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst);
+int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst);
+
+int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows);
+void    blockDataClearup(SSDataBlock* pDataBlock, bool hasVarCol);
+void   *blockDataDestroy(SSDataBlock *pBlock);
+
 #ifdef __cplusplus
 }
 #endif

include/common/tmsg.h

@@ -234,9 +234,9 @@ typedef struct {
   void*   pMsg;
 } SSubmitMsgIter;
 
-int     tInitSubmitMsgIter(SSubmitMsg* pMsg, SSubmitMsgIter* pIter);
+int32_t tInitSubmitMsgIter(SSubmitMsg* pMsg, SSubmitMsgIter* pIter);
-int     tGetSubmitMsgNext(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock);
+int32_t tGetSubmitMsgNext(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock);
-int     tInitSubmitBlkIter(SSubmitBlk* pBlock, SSubmitBlkIter* pIter);
+int32_t tInitSubmitBlkIter(SSubmitBlk* pBlock, SSubmitBlkIter* pIter);
 STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter);
 
 typedef struct {
@@ -272,8 +272,8 @@ typedef struct {
   char    comment[TSDB_STB_COMMENT_LEN];
 } SMCreateStbReq;
 
-int32_t tSerializeSMCreateStbReq(void** buf, SMCreateStbReq* pReq);
+int32_t tSerializeSMCreateStbReq(void* buf, int32_t bufLen, SMCreateStbReq* pReq);
-void*   tDeserializeSMCreateStbReq(void* buf, SMCreateStbReq* pReq);
+int32_t tDeserializeSMCreateStbReq(void* buf, int32_t bufLen, SMCreateStbReq* pReq);
 void    tFreeSMCreateStbReq(SMCreateStbReq* pReq);
 
 typedef struct {
@@ -281,8 +281,8 @@ typedef struct {
   int8_t igNotExists;
 } SMDropStbReq;
 
-int32_t tSerializeSMDropStbReq(void** buf, SMDropStbReq* pReq);
+int32_t tSerializeSMDropStbReq(void* buf, int32_t bufLen, SMDropStbReq* pReq);
-void*   tDeserializeSMDropStbReq(void* buf, SMDropStbReq* pReq);
+int32_t tDeserializeSMDropStbReq(void* buf, int32_t bufLen, SMDropStbReq* pReq);
 
 typedef struct {
   char    name[TSDB_TABLE_FNAME_LEN];
@@ -291,8 +291,20 @@ typedef struct {
   SArray* pFields;
 } SMAltertbReq;
 
-int32_t tSerializeSMAlterStbReq(void** buf, SMAltertbReq* pReq);
+int32_t tSerializeSMAlterStbReq(void* buf, int32_t bufLen, SMAltertbReq* pReq);
-void*   tDeserializeSMAlterStbReq(void* buf, SMAltertbReq* pReq);
+int32_t tDeserializeSMAlterStbReq(void* buf, int32_t bufLen, SMAltertbReq* pReq);
+void    tFreeSMAltertbReq(SMAltertbReq* pReq);
+
+typedef struct SEpSet {
+  int8_t inUse;
+  int8_t numOfEps;
+  SEp    eps[TSDB_MAX_REPLICA];
+} SEpSet;
+
+int32_t tEncodeSEpSet(SCoder* pEncoder, const SEpSet* pEp);
+int32_t tDecodeSEpSet(SCoder* pDecoder, SEpSet* pEp);
+int32_t taosEncodeSEpSet(void** buf, const SEpSet* pEp);
+void*   taosDecodeSEpSet(void* buf, SEpSet* pEp);
 
 typedef struct {
   int32_t pid;
@@ -301,62 +313,21 @@ typedef struct {
   int64_t startTime;
 } SConnectReq;
 
-typedef struct SEpSet {
+int32_t tSerializeSConnectReq(void* buf, int32_t bufLen, SConnectReq* pReq);
-  int8_t inUse;
+int32_t tDeserializeSConnectReq(void* buf, int32_t bufLen, SConnectReq* pReq);
-  int8_t numOfEps;
-  SEp    eps[TSDB_MAX_REPLICA];
-} SEpSet;
-
-static FORCE_INLINE int taosEncodeSEpSet(void** buf, const SEpSet* pEp) {
-  int tlen = 0;
-  tlen += taosEncodeFixedI8(buf, pEp->inUse);
-  tlen += taosEncodeFixedI8(buf, pEp->numOfEps);
-  for (int i = 0; i < TSDB_MAX_REPLICA; i++) {
-    tlen += taosEncodeFixedU16(buf, pEp->eps[i].port);
-    tlen += taosEncodeString(buf, pEp->eps[i].fqdn);
-  }
-  return tlen;
-}
-
-static FORCE_INLINE void* taosDecodeSEpSet(void* buf, SEpSet* pEp) {
-  buf = taosDecodeFixedI8(buf, &pEp->inUse);
-  buf = taosDecodeFixedI8(buf, &pEp->numOfEps);
-  for (int i = 0; i < TSDB_MAX_REPLICA; i++) {
-    buf = taosDecodeFixedU16(buf, &pEp->eps[i].port);
-    buf = taosDecodeStringTo(buf, pEp->eps[i].fqdn);
-  }
-  return buf;
-}
-static FORCE_INLINE int32_t tEncodeSEpSet(SCoder* pEncoder, const SEpSet* pEp) {
-  if (tEncodeI8(pEncoder, pEp->inUse) < 0) return -1;
-  if (tEncodeI8(pEncoder, pEp->numOfEps) < 0) return -1;
-  for (int i = 0; i < TSDB_MAX_REPLICA; i++) {
-    if (tEncodeU16(pEncoder, pEp->eps[i].port) < 0) return -1;
-    if (tEncodeCStr(pEncoder, pEp->eps[i].fqdn) < 0) return -1;
-  }
-  return 0;
-}
-
-static FORCE_INLINE int32_t tDecodeSEpSet(SCoder* pDecoder, SEpSet* pEp) {
-  if (tDecodeI8(pDecoder, &pEp->inUse) < 0) return -1;
-  if (tDecodeI8(pDecoder, &pEp->numOfEps) < 0) return -1;
-  for (int i = 0; i < TSDB_MAX_REPLICA; i++) {
-    if (tDecodeU16(pDecoder, &pEp->eps[i].port) < 0) return -1;
-    if (tDecodeCStrTo(pDecoder, pEp->eps[i].fqdn) < 0) return -1;
-  }
-  return 0;
-}
 
 typedef struct {
   int32_t acctId;
   int64_t clusterId;
   int32_t connId;
   int8_t  superUser;
-  int8_t  align[3];
   SEpSet  epSet;
   char    sVersion[128];
 } SConnectRsp;
 
+int32_t tSerializeSConnectRsp(void* buf, int32_t bufLen, SConnectRsp* pRsp);
+int32_t tDeserializeSConnectRsp(void* buf, int32_t bufLen, SConnectRsp* pRsp);
+
 typedef struct {
   char    user[TSDB_USER_LEN];
   char    pass[TSDB_PASSWORD_LEN];
@@ -803,6 +774,9 @@ typedef struct {
   char     tbName[TSDB_TABLE_NAME_LEN];
 } STableInfoReq;
 
+int32_t tSerializeSTableInfoReq(void* buf, int32_t bufLen, STableInfoReq* pReq);
+int32_t tDeserializeSTableInfoReq(void* buf, int32_t bufLen, STableInfoReq* pReq);
+
 typedef struct {
   int8_t  metaClone;  // create local clone of the cached table meta
   int32_t numOfVgroups;
@@ -839,9 +813,21 @@ typedef struct {
   uint64_t suid;
   uint64_t tuid;
   int32_t  vgId;
-  SSchema  pSchema[];
+  SSchema* pSchemas;
 } STableMetaRsp;
 
+int32_t tSerializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp);
+int32_t tDeserializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp);
+void    tFreeSTableMetaRsp(STableMetaRsp* pRsp);
+
+typedef struct {
+  SArray* pArray;  // Array of STableMetaRsp
+} STableMetaBatchRsp;
+
+int32_t tSerializeSTableMetaBatchRsp(void* buf, int32_t bufLen, STableMetaBatchRsp* pRsp);
+int32_t tDeserializeSTableMetaBatchRsp(void* buf, int32_t bufLen, STableMetaBatchRsp* pRsp);
+void    tFreeSTableMetaBatchRsp(STableMetaBatchRsp* pRsp);
+
 typedef struct {
   int32_t numOfTables;
   int32_t numOfVgroup;
@@ -875,18 +861,15 @@ int32_t tSerializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq);
 int32_t tDeserializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq);
 void    tFreeSShowReq(SShowReq* pReq);
 
-typedef struct {
-  char    db[TSDB_DB_FNAME_LEN];
-  int32_t numOfVgroup;
-  int32_t vgid[];
-} SCompactReq;
-
 typedef struct {
   int64_t       showId;
   STableMetaRsp tableMeta;
-} SShowRsp;
+} SShowRsp, SVShowTablesRsp;
+
+int32_t tSerializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp);
+int32_t tDeserializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp);
+void    tFreeSShowRsp(SShowRsp* pRsp);
 
-// todo: the show handle should be replaced with id
 typedef struct {
   int64_t showId;
   int8_t  free;
@@ -1103,41 +1086,17 @@ typedef struct {
   char*  sql;
   char*  physicalPlan;
   char*  logicalPlan;
-} SCMCreateTopicReq;
+} SMCreateTopicReq;
 
-static FORCE_INLINE int tSerializeSCMCreateTopicReq(void** buf, const SCMCreateTopicReq* pReq) {
+int32_t tSerializeMCreateTopicReq(void** buf, const SMCreateTopicReq* pReq);
-  int tlen = 0;
+void*   tDeserializeSMCreateTopicReq(void* buf, SMCreateTopicReq* pReq);
-  tlen += taosEncodeFixedI8(buf, pReq->igExists);
-  tlen += taosEncodeString(buf, pReq->name);
-  tlen += taosEncodeString(buf, pReq->sql);
-  tlen += taosEncodeString(buf, pReq->physicalPlan);
-  tlen += taosEncodeString(buf, pReq->logicalPlan);
-  return tlen;
-}
-
-static FORCE_INLINE void* tDeserializeSCMCreateTopicReq(void* buf, SCMCreateTopicReq* pReq) {
-  buf = taosDecodeFixedI8(buf, &(pReq->igExists));
-  buf = taosDecodeString(buf, &(pReq->name));
-  buf = taosDecodeString(buf, &(pReq->sql));
-  buf = taosDecodeString(buf, &(pReq->physicalPlan));
-  buf = taosDecodeString(buf, &(pReq->logicalPlan));
-  return buf;
-}
 
 typedef struct {
   int64_t topicId;
-} SCMCreateTopicRsp;
+} SMCreateTopicRsp;
 
-static FORCE_INLINE int tSerializeSCMCreateTopicRsp(void** buf, const SCMCreateTopicRsp* pRsp) {
+int32_t tSerializeSMCreateTopicRsp(void* buf, int32_t bufLen, const SMCreateTopicRsp* pRsp);
-  int tlen = 0;
+int32_t tDeserializeSMCreateTopicRsp(void* buf, int32_t bufLen, SMCreateTopicRsp* pRsp);
-  tlen += taosEncodeFixedI64(buf, pRsp->topicId);
-  return tlen;
-}
-
-static FORCE_INLINE void* tDeserializeSCMCreateTopicRsp(void* buf, SCMCreateTopicRsp* pRsp) {
-  buf = taosDecodeFixedI64(buf, &pRsp->topicId);
-  return buf;
-}
 
 typedef struct {
   int32_t topicNum;
@@ -1284,19 +1243,13 @@ typedef struct {
   int64_t status;
 } SMVSubscribeRsp;
 
-typedef struct {
-  char    name[TSDB_TOPIC_NAME_LEN];
-  int8_t  igExists;
-  int32_t execLen;
-  void*   executor;
-  int32_t sqlLen;
-  char*   sql;
-} SCreateTopicReq;
-
 typedef struct {
   char   name[TSDB_TABLE_FNAME_LEN];
   int8_t igNotExists;
-} SDropTopicReq;
+} SMDropTopicReq;
+
+int32_t tSerializeSMDropTopicReqq(void* buf, int32_t bufLen, SMDropTopicReq* pReq);
+int32_t tDeserializeSMDropTopicReq(void* buf, int32_t bufLen, SMDropTopicReq* pReq);
 
 typedef struct {
   char    name[TSDB_TABLE_FNAME_LEN];
@@ -1398,11 +1351,6 @@ typedef struct {
   SMsgHead head;
 } SVShowTablesReq;
 
-typedef struct {
-  int64_t       id;
-  STableMetaRsp metaInfo;
-} SVShowTablesRsp;
-
 typedef struct {
   SMsgHead head;
   int32_t  id;
@@ -1607,7 +1555,7 @@ int32_t tDeserializeSClientHbBatchRsp(void* buf, int32_t bufLen, SClientHbBatchR
 static FORCE_INLINE int32_t tEncodeSKv(SCoder* pEncoder, const SKv* pKv) {
   if (tEncodeI32(pEncoder, pKv->key) < 0) return -1;
   if (tEncodeI32(pEncoder, pKv->valueLen) < 0) return -1;
-  if (tEncodeCStrWithLen(pEncoder, (const char*)pKv->value, pKv->valueLen) < 0) return -1;
+  if (tEncodeBinary(pEncoder, (const char*)pKv->value, pKv->valueLen) < 0) return -1;
   return 0;
 }
 
@@ -1803,7 +1751,7 @@ typedef struct {
   SSchema* pSchema;
 } SSchemaWrapper;
 
-static FORCE_INLINE int32_t tEncodeSSchema(void** buf, const SSchema* pSchema) {
+static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema) {
   int32_t tlen = 0;
   tlen += taosEncodeFixedI8(buf, pSchema->type);
   tlen += taosEncodeFixedI32(buf, pSchema->bytes);
@@ -1812,7 +1760,7 @@ static FORCE_INLINE int32_t tEncodeSSchema(void** buf, const SSchema* pSchema) {
   return tlen;
 }
 
-static FORCE_INLINE void* tDecodeSSchema(void* buf, SSchema* pSchema) {
+static FORCE_INLINE void* taosDecodeSSchema(void* buf, SSchema* pSchema) {
   buf = taosDecodeFixedI8(buf, &pSchema->type);
   buf = taosDecodeFixedI32(buf, &pSchema->bytes);
   buf = taosDecodeFixedI32(buf, &pSchema->colId);
@@ -1820,11 +1768,27 @@ static FORCE_INLINE void* tDecodeSSchema(void* buf, SSchema* pSchema) {
   return buf;
 }
 
+static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSchema) {
+  if (tEncodeI8(pEncoder, pSchema->type) < 0) return -1;
+  if (tEncodeI32(pEncoder, pSchema->bytes) < 0) return -1;
+  if (tEncodeI32(pEncoder, pSchema->colId) < 0) return -1;
+  if (tEncodeCStr(pEncoder, pSchema->name) < 0) return -1;
+  return 0;
+}
+
+static FORCE_INLINE int32_t tDecodeSSchema(SCoder* pDecoder, SSchema* pSchema) {
+  if (tDecodeI8(pDecoder, &pSchema->type) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pSchema->bytes) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pSchema->colId) < 0) return -1;
+  if (tDecodeCStrTo(pDecoder, pSchema->name) < 0) return -1;
+  return 0;
+}
+
 static FORCE_INLINE int32_t tEncodeSSchemaWrapper(void** buf, const SSchemaWrapper* pSW) {
   int32_t tlen = 0;
   tlen += taosEncodeFixedU32(buf, pSW->nCols);
   for (int32_t i = 0; i < pSW->nCols; i++) {
-    tlen += tEncodeSSchema(buf, &pSW->pSchema[i]);
+    tlen += taosEncodeSSchema(buf, &pSW->pSchema[i]);
   }
   return tlen;
 }
@@ -1835,8 +1799,9 @@ static FORCE_INLINE void* tDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW)
   if (pSW->pSchema == NULL) {
     return NULL;
   }
+
   for (int32_t i = 0; i < pSW->nCols; i++) {
-    buf = tDecodeSSchema(buf, &pSW->pSchema[i]);
+    buf = taosDecodeSSchema(buf, &pSW->pSchema[i]);
   }
   return buf;
 }

include/common/tmsgdef.h

@@ -139,7 +139,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_TRANS, "mnode-trans", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_GRANT, "mnode-grant", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_AUTH, "mnode-auth", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TOPIC, "mnode-create-topic", SCMCreateTopicReq, SCMCreateTopicRsp)
+  TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TOPIC, "mnode-create-topic", SMCreateTopicReq, SMCreateTopicRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp)

include/libs/nodes/querynodes.h

@@ -113,13 +113,13 @@ typedef enum ELogicConditionType {
 } ELogicConditionType;
 
 typedef struct SLogicConditionNode {
-  ENodeType type; // QUERY_NODE_LOGIC_CONDITION
+  SExprNode node; // QUERY_NODE_LOGIC_CONDITION
   ELogicConditionType condType;
   SNodeList* pParameterList;
 } SLogicConditionNode;
 
 typedef struct SIsNullCondNode {
-  ENodeType type; // QUERY_NODE_IS_NULL_CONDITION
+  SExprNode node; // QUERY_NODE_IS_NULL_CONDITION
   SNode* pExpr;
   bool isNull;
 } SIsNullCondNode;

include/libs/planner/plannerOp.h

@@ -24,7 +24,7 @@
 #endif
 
 OP_ENUM_MACRO(StreamScan)
-OP_ENUM_MACRO(DataBlocksOptScan)
+OP_ENUM_MACRO(TableScan)
 OP_ENUM_MACRO(TableSeqScan)
 OP_ENUM_MACRO(TagScan)
 OP_ENUM_MACRO(SystemTableScan)

include/util/tlosertree.h

@@ -22,28 +22,31 @@ extern "C" {
 
 typedef int (*__merge_compare_fn_t)(const void *, const void *, void *param);
 
-typedef struct SLoserTreeNode {
+typedef struct STreeNode {
   int32_t index;
-  void   *pData;
+  void   *pData;  // TODO remove it?
-} SLoserTreeNode;
+} STreeNode;
 
-typedef struct SLoserTreeInfo {
+typedef struct SMultiwayMergeTreeInfo {
-  int32_t              numOfEntries;
+  int32_t              numOfSources;
-  int32_t              totalEntries;
+  int32_t              totalSources;
   __merge_compare_fn_t comparFn;
   void *               param;
-  SLoserTreeNode      *pNode;
+  struct STreeNode    *pNode;
-} SLoserTreeInfo;
+} SMultiwayMergeTreeInfo;
 
-uint32_t tLoserTreeCreate(SLoserTreeInfo **pTree, int32_t numOfEntries, void *param, __merge_compare_fn_t compareFn);
+#define tMergeTreeGetChosenIndex(t_) ((t_)->pNode[0].index)
+#define tMergeTreeGetAdjustIndex(t_) (tMergeTreeGetChosenIndex(t_) + (t_)->numOfSources)
 
-void tLoserTreeInit(SLoserTreeInfo *pTree);
+int32_t tMergeTreeCreate(SMultiwayMergeTreeInfo **pTree, uint32_t numOfEntries, void *param, __merge_compare_fn_t compareFn);
 
-void tLoserTreeAdjust(SLoserTreeInfo *pTree, int32_t idx);
+void tMergeTreeDestroy(SMultiwayMergeTreeInfo* pTree);
 
-void tLoserTreeRebuild(SLoserTreeInfo *pTree);
+void tMergeTreeAdjust(SMultiwayMergeTreeInfo *pTree, int32_t idx);
 
-void tLoserTreeDisplay(SLoserTreeInfo *pTree);
+void tMergeTreeRebuild(SMultiwayMergeTreeInfo *pTree);
+
+void tMergeTreePrint(const SMultiwayMergeTreeInfo *pTree);
 
 #ifdef __cplusplus
 }

include/util/tpagedbuf.h

@@ -0,0 +1,171 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TPAGEDBUF_H
+#define TDENGINE_TPAGEDBUF_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "tlist.h"
+#include "thash.h"
+#include "os.h"
+#include "tlockfree.h"
+
+typedef struct SArray* SIDList;
+typedef struct SPageInfo SPageInfo;
+typedef struct SDiskbasedBuf SDiskbasedBuf;
+
+#define DEFAULT_INTERN_BUF_PAGE_SIZE  (1024L)                          // in bytes
+#define DEFAULT_PAGE_SIZE             (16384L)
+
+typedef struct SFilePage {
+  int64_t num;
+  char    data[];
+} SFilePage;
+
+typedef struct SDiskbasedBufStatis {
+  int64_t flushBytes;
+  int64_t loadBytes;
+  int32_t loadPages;
+  int32_t getPages;
+  int32_t releasePages;
+  int32_t flushPages;
+} SDiskbasedBufStatis;
+
+/**
+ * create disk-based result buffer
+ * @param pBuf
+ * @param rowSize
+ * @param pagesize
+ * @param inMemPages
+ * @param handle
+ * @return
+ */
+int32_t createDiskbasedBuffer(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir);
+
+/**
+ *
+ * @param pBuf
+ * @param groupId
+ * @param pageId
+ * @return
+ */
+SFilePage* getNewDataBuf(SDiskbasedBuf* pBuf, int32_t groupId, int32_t* pageId);
+
+/**
+ *
+ * @param pBuf
+ * @param groupId
+ * @return
+ */
+SIDList getDataBufPagesIdList(SDiskbasedBuf* pBuf, int32_t groupId);
+
+/**
+ * get the specified buffer page by id
+ * @param pBuf
+ * @param id
+ * @return
+ */
+SFilePage* getBufPage(SDiskbasedBuf* pBuf, int32_t id);
+
+/**
+ * release the referenced buf pages
+ * @param pBuf
+ * @param page
+ */
+void releaseBufPage(SDiskbasedBuf* pBuf, void* page);
+
+/**
+ *
+ * @param pBuf
+ * @param pi
+ */
+void releaseBufPageInfo(SDiskbasedBuf* pBuf, struct SPageInfo* pi);
+
+/**
+ * get the total buffer size in the format of disk file
+ * @param pBuf
+ * @return
+ */
+size_t getTotalBufSize(const SDiskbasedBuf* pBuf);
+
+/**
+ * get the number of groups in the result buffer
+ * @param pBuf
+ * @return
+ */
+size_t getNumOfResultBufGroupId(const SDiskbasedBuf* pBuf);
+
+/**
+ * destroy result buffer
+ * @param pBuf
+ */
+void destroyResultBuf(SDiskbasedBuf* pBuf);
+
+/**
+ *
+ * @param pList
+ * @return
+ */
+SPageInfo* getLastPageInfo(SIDList pList);
+
+/**
+ *
+ * @param pPgInfo
+ * @return
+ */
+int32_t getPageId(const SPageInfo* pPgInfo);
+
+/**
+ * Return the buffer page size.
+ * @param pBuf
+ * @return
+ */
+int32_t getBufPageSize(const SDiskbasedBuf* pBuf);
+
+int32_t getNumOfInMemBufPages(const SDiskbasedBuf* pBuf);
+
+/**
+ *
+ * @param pBuf
+ * @return
+ */
+bool isAllDataInMemBuf(const SDiskbasedBuf* pBuf);
+
+/**
+ * Set the buffer page is dirty, and needs to be flushed to disk when swap out.
+ * @param pPageInfo
+ * @param dirty
+ */
+void setBufPageDirty(SFilePage* pPageInfo, bool dirty);
+
+/**
+ * Print the statistics when closing this buffer
+ * @param pBuf
+ */
+void printStatisBeforeClose(SDiskbasedBuf* pBuf);
+
+/**
+ * return buf statistics.
+ */
+SDiskbasedBufStatis getDBufStatis(const SDiskbasedBuf* pBuf);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_TPAGEDBUF_H

include/util/tpagedfile.h

@@ -1,169 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_TPAGEDFILE_H
-#define TDENGINE_TPAGEDFILE_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "tlist.h"
-#include "thash.h"
-#include "os.h"
-#include "tlockfree.h"
-
-typedef struct SArray* SIDList;
-
-typedef struct SPageDiskInfo {
-  int32_t offset;
-  int32_t length;
-} SPageDiskInfo;
-
-typedef struct SPageInfo {
-  SListNode*    pn;       // point to list node
-  int32_t       pageId;
-  SPageDiskInfo info;
-  void*         pData;
-  bool          used;     // set current page is in used
-} SPageInfo;
-
-typedef struct SFreeListItem {
-  int32_t offset;
-  int32_t len;
-} SFreeListItem;
-
-typedef struct SResultBufStatis {
-  int32_t flushBytes;
-  int32_t loadBytes;
-  int32_t getPages;
-  int32_t releasePages;
-  int32_t flushPages;
-} SResultBufStatis;
-
-typedef struct SDiskbasedResultBuf {
-  int32_t   numOfPages;
-  int64_t   totalBufSize;
-  int64_t   fileSize;            // disk file size
-  FILE*     file;
-  int32_t   allocateId;          // allocated page id
-  char*     path;                // file path
-  int32_t   pageSize;            // current used page size
-  int32_t   inMemPages;          // numOfPages that are allocated in memory
-  SHashObj* groupSet;            // id hash table
-  SHashObj* all;
-  SList*    lruList;
-  void*     emptyDummyIdList;    // dummy id list
-  void*     assistBuf;           // assistant buffer for compress/decompress data
-  SArray*   pFree;               // free area in file
-  bool      comp;                // compressed before flushed to disk
-  int32_t   nextPos;             // next page flush position
-
-  uint64_t  qId;                 // for debug purpose
-  SResultBufStatis statis;
-} SDiskbasedResultBuf;
-
-#define DEFAULT_INTERN_BUF_PAGE_SIZE  (1024L)                          // in bytes
-#define PAGE_INFO_INITIALIZER         (SPageDiskInfo){-1, -1}
-#define DEFAULT_PAGE_SIZE             (16384L)
-
-typedef struct SFilePage {
-  int64_t num;
-  char    data[];
-} SFilePage;
-
-/**
- * create disk-based result buffer
- * @param pResultBuf
- * @param rowSize
- * @param pagesize
- * @param inMemPages
- * @param handle
- * @return
- */
-int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir);
-
-/**
- *
- * @param pResultBuf
- * @param groupId
- * @param pageId
- * @return
- */
-SFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId);
-
-/**
- *
- * @param pResultBuf
- * @param groupId
- * @return
- */
-SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId);
-
-/**
- * get the specified buffer page by id
- * @param pResultBuf
- * @param id
- * @return
- */
-SFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id);
-
-/**
- * release the referenced buf pages
- * @param pResultBuf
- * @param page
- */
-void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page);
-
-/**
- *
- * @param pResultBuf
- * @param pi
- */
-void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi);
-
-
-/**
- * get the total buffer size in the format of disk file
- * @param pResultBuf
- * @return
- */
-size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf);
-
-/**
- * get the number of groups in the result buffer
- * @param pResultBuf
- * @return
- */
-size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf);
-
-/**
- * destroy result buffer
- * @param pResultBuf
- */
-void destroyResultBuf(SDiskbasedResultBuf* pResultBuf);
-
-/**
- *
- * @param pList
- * @return
- */
-SPageInfo* getLastPageInfo(SIDList pList);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  // TDENGINE_TPAGEDFILE_H

source/client/src/clientHb.c

@@ -70,62 +70,42 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
     }
   }
 
+  tFreeSUseDbBatchRsp(&batchUseRsp);
   return TSDB_CODE_SUCCESS;
 }
 
 static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) {
-  int32_t msgLen = 0;
   int32_t code = 0;
-  int32_t schemaNum = 0;
 
-  while (msgLen < valueLen) {
+  STableMetaBatchRsp batchMetaRsp = {0};
-    STableMetaRsp *rsp = (STableMetaRsp *)((char *)value + msgLen);
+  if (tDeserializeSTableMetaBatchRsp(value, valueLen, &batchMetaRsp) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
 
-    rsp->numOfColumns = ntohl(rsp->numOfColumns);
+  int32_t numOfBatchs = taosArrayGetSize(batchMetaRsp.pArray);
-    rsp->suid = be64toh(rsp->suid);
+  for (int32_t i = 0; i < numOfBatchs; ++i) {
-    rsp->dbId = be64toh(rsp->dbId);
+    STableMetaRsp *rsp = taosArrayGet(batchMetaRsp.pArray, i);
 
     if (rsp->numOfColumns < 0) {
-      schemaNum = 0;
-      
       tscDebug("hb remove stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName);
-
       catalogRemoveStbMeta(pCatalog, rsp->dbFName, rsp->dbId, rsp->stbName, rsp->suid);
     } else {
       tscDebug("hb update stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName);
-
+      if (rsp->pSchemas[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) {
-      rsp->numOfTags = ntohl(rsp->numOfTags);
+        tscError("invalid colId[%d] for the first column in table meta rsp msg", rsp->pSchemas[0].colId);
-      rsp->sversion = ntohl(rsp->sversion);
+        tFreeSTableMetaBatchRsp(&batchMetaRsp);
-      rsp->tversion = ntohl(rsp->tversion);
-      rsp->tuid = be64toh(rsp->tuid);
-      rsp->vgId = ntohl(rsp->vgId);
-
-      SSchema* pSchema = rsp->pSchema;
-      
-      schemaNum = rsp->numOfColumns + rsp->numOfTags;
-
-      for (int i = 0; i < schemaNum; ++i) {
-        pSchema->bytes = ntohl(pSchema->bytes);
-        pSchema->colId = ntohl(pSchema->colId);
-
-        pSchema++;
-      }
-
-      if (rsp->pSchema[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) {
-        tscError("invalid colId[%d] for the first column in table meta rsp msg", rsp->pSchema[0].colId);
         return TSDB_CODE_TSC_INVALID_VALUE;
       }
 
       catalogUpdateSTableMeta(pCatalog, rsp);
     }
-
-    msgLen += sizeof(STableMetaRsp) + schemaNum * sizeof(SSchema);
   }
 
+  tFreeSTableMetaBatchRsp(&batchMetaRsp);
   return TSDB_CODE_SUCCESS;
 }
 
-
 static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRsp) {
   SHbConnInfo * info = taosHashGet(pAppHbMgr->connInfo, &pRsp->connKey, sizeof(SClientHbKey));
   if (NULL == info) {

source/client/src/clientImpl.c

@@ -9,7 +9,7 @@
 #include "tglobal.h"
 #include "tmsgtype.h"
 #include "tnote.h"
-#include "tpagedfile.h"
+#include "tpagedbuf.h"
 #include "tref.h"
 
 static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet);
@@ -365,32 +365,30 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj *pRequest) {
   }
 
   pMsgSendInfo->msgType = TDMT_MND_CONNECT;
-  pMsgSendInfo->msgInfo.len     = sizeof(SConnectReq);
   pMsgSendInfo->requestObjRefId = pRequest->self;
   pMsgSendInfo->requestId = pRequest->requestId;
   pMsgSendInfo->fp = handleRequestRspFp[TMSG_INDEX(pMsgSendInfo->msgType)];
   pMsgSendInfo->param = pRequest;
 
-  SConnectReq *pConnect = calloc(1, sizeof(SConnectReq));
+  SConnectReq connectReq = {0};
-  if (pConnect == NULL) {
-    tfree(pMsgSendInfo);
-    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
-    return NULL;
-  }
-
   STscObj*    pObj = pRequest->pTscObj;
 
   char* db = getDbOfConnection(pObj);
   if (db != NULL) {
-    tstrncpy(pConnect->db, db, sizeof(pConnect->db));
+    tstrncpy(connectReq.db, db, sizeof(connectReq.db));
   }
   tfree(db);
 
-  pConnect->pid = htonl(appInfo.pid);
+  connectReq.pid = htonl(appInfo.pid);
-  pConnect->startTime = htobe64(appInfo.startTime);
+  connectReq.startTime = htobe64(appInfo.startTime);
-  tstrncpy(pConnect->app, appInfo.appName, tListLen(pConnect->app));
+  tstrncpy(connectReq.app, appInfo.appName, sizeof(connectReq.app));
 
-  pMsgSendInfo->msgInfo.pData = pConnect;
+  int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq);
+  void*   pReq = malloc(contLen);
+  tSerializeSConnectReq(pReq, contLen, &connectReq);
+
+  pMsgSendInfo->msgInfo.len = contLen;
+  pMsgSendInfo->msgInfo.pData = pReq;
   return pMsgSendInfo;
 }
 

source/client/src/clientMsgHandler.c

@@ -20,14 +20,14 @@
 #include "clientLog.h"
 #include "catalog.h"
 
-int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code);
+int32_t (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code);
 
 static void setErrno(SRequestObj* pRequest, int32_t code) {
   pRequest->code = code;
   terrno = code;
 }
 
-int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code) {
+int32_t genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code) {
   SRequestObj* pRequest = param;
   setErrno(pRequest, code);
 
@@ -36,7 +36,7 @@ int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code) {
   return code;
 }
 
-int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
+int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
   SRequestObj* pRequest = param;
   if (code != TSDB_CODE_SUCCESS) {
     free(pMsg->pData);
@@ -47,39 +47,33 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
 
   STscObj* pTscObj = pRequest->pTscObj;
 
-  SConnectRsp *pConnect = (SConnectRsp *)pMsg->pData;
+  SConnectRsp connectRsp = {0};
-  pConnect->acctId    = htonl(pConnect->acctId);
+  tDeserializeSConnectRsp(pMsg->pData, pMsg->len, &connectRsp);
-  pConnect->connId    = htonl(pConnect->connId);
+  assert(connectRsp.epSet.numOfEps > 0);
-  pConnect->clusterId = htobe64(pConnect->clusterId);
 
-  assert(pConnect->epSet.numOfEps > 0);
+  if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &connectRsp.epSet)) {
-  for(int32_t i = 0; i < pConnect->epSet.numOfEps; ++i) {
+    updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &connectRsp.epSet);
-    pConnect->epSet.eps[i].port = htons(pConnect->epSet.eps[i].port);
   }
 
-  if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &pConnect->epSet)) {
+  for (int32_t i = 0; i < connectRsp.epSet.numOfEps; ++i) {
-    updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &pConnect->epSet);
+    tscDebug("0x%" PRIx64 " epSet.fqdn[%d]:%s port:%d, connObj:0x%" PRIx64, pRequest->requestId, i,
+             connectRsp.epSet.eps[i].fqdn, connectRsp.epSet.eps[i].port, pTscObj->id);
   }
 
-  for (int i = 0; i < pConnect->epSet.numOfEps; ++i) {
+  pTscObj->connId = connectRsp.connId;
-    tscDebug("0x%" PRIx64 " epSet.fqdn[%d]:%s port:%d, connObj:0x%"PRIx64, pRequest->requestId, i, pConnect->epSet.eps[i].fqdn,
+  pTscObj->acctId = connectRsp.acctId;
-        pConnect->epSet.eps[i].port, pTscObj->id);
+  tstrncpy(pTscObj->ver, connectRsp.sVersion, tListLen(pTscObj->ver));
-  }
-
-  pTscObj->connId = pConnect->connId;
-  pTscObj->acctId = pConnect->acctId;
-  tstrncpy(pTscObj->ver, pConnect->sVersion, tListLen(pTscObj->ver));
 
   // update the appInstInfo
-  pTscObj->pAppInfo->clusterId = pConnect->clusterId;
+  pTscObj->pAppInfo->clusterId = connectRsp.clusterId;
   atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1);
 
   pTscObj->connType = HEARTBEAT_TYPE_QUERY;
 
-  hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pConnect->connId, pConnect->clusterId, HEARTBEAT_TYPE_QUERY);
+  hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, connectRsp.connId, connectRsp.clusterId, HEARTBEAT_TYPE_QUERY);
 
   //  pRequest->body.resInfo.pRspMsg = pMsg->pData;
-  tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, pConnect->clusterId,
+  tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, connectRsp.clusterId,
            pTscObj->pAppInfo->numOfConns);
 
   free(pMsg->pData);
@@ -135,39 +129,29 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) {
     return code;
   }
 
-  SShowRsp* pShow = (SShowRsp *)pMsg->pData;
+  SShowRsp showRsp = {0};
-  pShow->showId   = htobe64(pShow->showId);
+  tDeserializeSShowRsp(pMsg->pData, pMsg->len, &showRsp);
+  STableMetaRsp *pMetaMsg = &showRsp.tableMeta;
 
-  STableMetaRsp *pMetaMsg = &(pShow->tableMeta);
-  pMetaMsg->numOfColumns = htonl(pMetaMsg->numOfColumns);
-
-  SSchema* pSchema = pMetaMsg->pSchema;
-  pMetaMsg->tuid = htobe64(pMetaMsg->tuid);
-  for (int i = 0; i < pMetaMsg->numOfColumns; ++i) {
-    pSchema->bytes = htonl(pSchema->bytes);
-    pSchema->colId = htonl(pSchema->colId);
-    pSchema++;
-  }
-
-  pSchema = pMetaMsg->pSchema;
   tfree(pRequest->body.resInfo.pRspMsg);
-
   pRequest->body.resInfo.pRspMsg = pMsg->pData;
   SReqResultInfo* pResInfo = &pRequest->body.resInfo;
 
   if (pResInfo->fields == NULL) {
     TAOS_FIELD* pFields = calloc(pMetaMsg->numOfColumns, sizeof(TAOS_FIELD));
     for (int32_t i = 0; i < pMetaMsg->numOfColumns; ++i) {
-      tstrncpy(pFields[i].name, pSchema[i].name, tListLen(pFields[i].name));
+      SSchema* pSchema = &pMetaMsg->pSchemas[i];
-      pFields[i].type = pSchema[i].type;
+      tstrncpy(pFields[i].name, pSchema->name, tListLen(pFields[i].name));
-      pFields[i].bytes = pSchema[i].bytes;
+      pFields[i].type = pSchema->type;
+      pFields[i].bytes = pSchema->bytes;
     }
 
     pResInfo->fields = pFields;
   }
 
   pResInfo->numOfCols = pMetaMsg->numOfColumns;
-  pRequest->body.showInfo.execId = pShow->showId;
+  pRequest->body.showInfo.execId = showRsp.showId;
+  tFreeSShowRsp(&showRsp);
 
   // todo
   if (pRequest->type == TDMT_VND_SHOW_TABLES) {

source/client/src/tmq.c

@@ -25,7 +25,7 @@
 #include "tglobal.h"
 #include "tmsgtype.h"
 #include "tnote.h"
-#include "tpagedfile.h"
+#include "tpagedbuf.h"
 #include "tref.h"
 
 struct tmq_list_t {
@@ -381,7 +381,7 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i
   char topicFname[TSDB_TOPIC_FNAME_LEN] = {0};
   tNameExtractFullName(&name, topicFname);
 
-  SCMCreateTopicReq req = {
+  SMCreateTopicReq req = {
       .name = (char*)topicFname,
       .igExists = 1,
       .physicalPlan = (char*)pStr,
@@ -389,14 +389,14 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i
       .logicalPlan = (char*)"no logic plan",
   };
 
-  int   tlen = tSerializeSCMCreateTopicReq(NULL, &req);
+  int   tlen = tSerializeMCreateTopicReq(NULL, &req);
   void* buf = malloc(tlen);
   if (buf == NULL) {
     goto _return;
   }
 
   void* abuf = buf;
-  tSerializeSCMCreateTopicReq(&abuf, &req);
+  tSerializeMCreateTopicReq(&abuf, &req);
   /*printf("formatted: %s\n", dagStr);*/
 
   pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen};

source/common/src/tmsg.c

@@ -85,6 +85,47 @@ STSRow *tGetSubmitBlkNext(SSubmitBlkIter *pIter) {
   }
 }
 
+int32_t tEncodeSEpSet(SCoder *pEncoder, const SEpSet *pEp) {
+  if (tEncodeI8(pEncoder, pEp->inUse) < 0) return -1;
+  if (tEncodeI8(pEncoder, pEp->numOfEps) < 0) return -1;
+  for (int32_t i = 0; i < TSDB_MAX_REPLICA; i++) {
+    if (tEncodeU16(pEncoder, pEp->eps[i].port) < 0) return -1;
+    if (tEncodeCStr(pEncoder, pEp->eps[i].fqdn) < 0) return -1;
+  }
+  return 0;
+}
+
+int32_t tDecodeSEpSet(SCoder *pDecoder, SEpSet *pEp) {
+  if (tDecodeI8(pDecoder, &pEp->inUse) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pEp->numOfEps) < 0) return -1;
+  for (int32_t i = 0; i < TSDB_MAX_REPLICA; i++) {
+    if (tDecodeU16(pDecoder, &pEp->eps[i].port) < 0) return -1;
+    if (tDecodeCStrTo(pDecoder, pEp->eps[i].fqdn) < 0) return -1;
+  }
+  return 0;
+}
+
+int32_t taosEncodeSEpSet(void **buf, const SEpSet *pEp) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI8(buf, pEp->inUse);
+  tlen += taosEncodeFixedI8(buf, pEp->numOfEps);
+  for (int32_t i = 0; i < TSDB_MAX_REPLICA; i++) {
+    tlen += taosEncodeFixedU16(buf, pEp->eps[i].port);
+    tlen += taosEncodeString(buf, pEp->eps[i].fqdn);
+  }
+  return tlen;
+}
+
+void *taosDecodeSEpSet(void *buf, SEpSet *pEp) {
+  buf = taosDecodeFixedI8(buf, &pEp->inUse);
+  buf = taosDecodeFixedI8(buf, &pEp->numOfEps);
+  for (int32_t i = 0; i < TSDB_MAX_REPLICA; i++) {
+    buf = taosDecodeFixedU16(buf, &pEp->eps[i].port);
+    buf = taosDecodeStringTo(buf, pEp->eps[i].fqdn);
+  }
+  return buf;
+}
+
 static int32_t tSerializeSClientHbReq(SCoder *pEncoder, const SClientHbReq *pReq) {
   if (tEncodeSClientHbKey(pEncoder, &pReq->connKey) < 0) return -1;
 
@@ -375,132 +416,172 @@ void *tDeserializeSVDropTbReq(void *buf, SVDropTbReq *pReq) {
   return buf;
 }
 
-int32_t tSerializeSMCreateStbReq(void **buf, SMCreateStbReq *pReq) {
+int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq) {
-  int32_t tlen = 0;
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
 
-  tlen += taosEncodeString(buf, pReq->name);
+  if (tStartEncode(&encoder) < 0) return -1;
-  tlen += taosEncodeFixedI8(buf, pReq->igExists);
+  if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
-  tlen += taosEncodeFixedI32(buf, pReq->numOfColumns);
+  if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1;
-  tlen += taosEncodeFixedI32(buf, pReq->numOfTags);
+  if (tEncodeI32(&encoder, pReq->numOfColumns) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->numOfTags) < 0) return -1;
 
   for (int32_t i = 0; i < pReq->numOfColumns; ++i) {
     SField *pField = taosArrayGet(pReq->pColumns, i);
-    tlen += taosEncodeFixedU8(buf, pField->type);
+    if (tEncodeI8(&encoder, pField->type) < 0) return -1;
-    tlen += taosEncodeFixedI32(buf, pField->bytes);
+    if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
-    tlen += taosEncodeString(buf, pField->name);
+    if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
   }
 
   for (int32_t i = 0; i < pReq->numOfTags; ++i) {
     SField *pField = taosArrayGet(pReq->pTags, i);
-    tlen += taosEncodeFixedU8(buf, pField->type);
+    if (tEncodeI8(&encoder, pField->type) < 0) return -1;
-    tlen += taosEncodeFixedI32(buf, pField->bytes);
+    if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
-    tlen += taosEncodeString(buf, pField->name);
+    if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
   }
 
-  tlen += taosEncodeString(buf, pReq->comment);
+  if (tEncodeCStr(&encoder, pReq->comment) < 0) return -1;
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
   return tlen;
 }
 
-void *tDeserializeSMCreateStbReq(void *buf, SMCreateStbReq *pReq) {
+int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq) {
-  buf = taosDecodeStringTo(buf, pReq->name);
+  SCoder decoder = {0};
-  buf = taosDecodeFixedI8(buf, &pReq->igExists);
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
-  buf = taosDecodeFixedI32(buf, &pReq->numOfColumns);
+
-  buf = taosDecodeFixedI32(buf, &pReq->numOfTags);
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->numOfColumns) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->numOfTags) < 0) return -1;
 
   pReq->pColumns = taosArrayInit(pReq->numOfColumns, sizeof(SField));
   pReq->pTags = taosArrayInit(pReq->numOfTags, sizeof(SField));
   if (pReq->pColumns == NULL || pReq->pTags == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return NULL;
+    return -1;
   }
 
   for (int32_t i = 0; i < pReq->numOfColumns; ++i) {
     SField field = {0};
-    buf = taosDecodeFixedU8(buf, &field.type);
+    if (tDecodeI8(&decoder, &field.type) < 0) return -1;
-    buf = taosDecodeFixedI32(buf, &field.bytes);
+    if (tDecodeI32(&decoder, &field.bytes) < 0) return -1;
-    buf = taosDecodeStringTo(buf, field.name);
+    if (tDecodeCStrTo(&decoder, field.name) < 0) return -1;
     if (taosArrayPush(pReq->pColumns, &field) == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
-      return NULL;
+      return -1;
     }
   }
 
   for (int32_t i = 0; i < pReq->numOfTags; ++i) {
     SField field = {0};
-    buf = taosDecodeFixedU8(buf, &field.type);
+    if (tDecodeI8(&decoder, &field.type) < 0) return -1;
-    buf = taosDecodeFixedI32(buf, &field.bytes);
+    if (tDecodeI32(&decoder, &field.bytes) < 0) return -1;
-    buf = taosDecodeStringTo(buf, field.name);
+    if (tDecodeCStrTo(&decoder, field.name) < 0) return -1;
     if (taosArrayPush(pReq->pTags, &field) == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
-      return NULL;
+      return -1;
     }
   }
 
-  buf = taosDecodeStringTo(buf, pReq->comment);
+  if (tDecodeCStrTo(&decoder, pReq->comment) < 0) return -1;
-  return buf;
+  tEndDecode(&decoder);
+
+  tCoderClear(&decoder);
+  return 0;
 }
 
 void tFreeSMCreateStbReq(SMCreateStbReq *pReq) {
   taosArrayDestroy(pReq->pColumns);
   taosArrayDestroy(pReq->pTags);
+  pReq->pColumns = NULL;
+  pReq->pTags = NULL;
 }
 
-int32_t tSerializeSMDropStbReq(void **buf, SMDropStbReq *pReq) {
+int32_t tSerializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq) {
-  int32_t tlen = 0;
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
 
-  tlen += taosEncodeString(buf, pReq->name);
+  if (tStartEncode(&encoder) < 0) return -1;
-  tlen += taosEncodeFixedI8(buf, pReq->igNotExists);
+  if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
+  tEndEncode(&encoder);
 
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
   return tlen;
 }
 
-void *tDeserializeSMDropStbReq(void *buf, SMDropStbReq *pReq) {
+int32_t tDeserializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq) {
-  buf = taosDecodeStringTo(buf, pReq->name);
+  SCoder decoder = {0};
-  buf = taosDecodeFixedI8(buf, &pReq->igNotExists);
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
 
-  return buf;
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
+  tEndDecode(&decoder);
+
+  tCoderClear(&decoder);
+  return 0;
 }
 
-int32_t tSerializeSMAlterStbReq(void **buf, SMAltertbReq *pReq) {
+int32_t tSerializeSMAlterStbReq(void *buf, int32_t bufLen, SMAltertbReq *pReq) {
-  int32_t tlen = 0;
+  SCoder encoder = {0};
-
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
-  tlen += taosEncodeString(buf, pReq->name);
-  tlen += taosEncodeFixedI8(buf, pReq->alterType);
-  tlen += taosEncodeFixedI32(buf, pReq->numOfFields);
 
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->alterType) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->numOfFields) < 0) return -1;
   for (int32_t i = 0; i < pReq->numOfFields; ++i) {
     SField *pField = taosArrayGet(pReq->pFields, i);
-    tlen += taosEncodeFixedU8(buf, pField->type);
+    if (tEncodeI8(&encoder, pField->type) < 0) return -1;
-    tlen += taosEncodeFixedI32(buf, pField->bytes);
+    if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
-    tlen += taosEncodeString(buf, pField->name);
+    if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
   }
+  tEndEncode(&encoder);
 
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
   return tlen;
 }
 
-void *tDeserializeSMAlterStbReq(void *buf, SMAltertbReq *pReq) {
+int32_t tDeserializeSMAlterStbReq(void *buf, int32_t bufLen, SMAltertbReq *pReq) {
-  buf = taosDecodeStringTo(buf, pReq->name);
+  SCoder decoder = {0};
-  buf = taosDecodeFixedI8(buf, &pReq->alterType);
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
-  buf = taosDecodeFixedI32(buf, &pReq->numOfFields);
 
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->alterType) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->numOfFields) < 0) return -1;
   pReq->pFields = taosArrayInit(pReq->numOfFields, sizeof(SField));
   if (pReq->pFields == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return NULL;
+    return -1;
   }
 
   for (int32_t i = 0; i < pReq->numOfFields; ++i) {
     SField field = {0};
-    buf = taosDecodeFixedU8(buf, &field.type);
+    if (tDecodeI8(&decoder, &field.type) < 0) return -1;
-    buf = taosDecodeFixedI32(buf, &field.bytes);
+    if (tDecodeI32(&decoder, &field.bytes) < 0) return -1;
-    buf = taosDecodeStringTo(buf, field.name);
+    if (tDecodeCStrTo(&decoder, field.name) < 0) return -1;
     if (taosArrayPush(pReq->pFields, &field) == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
-      return NULL;
+      return -1;
     }
   }
 
-  return buf;
+  tEndDecode(&decoder);
+  tCoderClear(&decoder);
+  return 0;
+}
+
+void tFreeSMAltertbReq(SMAltertbReq *pReq) {
+  taosArrayDestroy(pReq->pFields);
+  pReq->pFields = NULL;
 }
 
 int32_t tSerializeSStatusReq(void **buf, SStatusReq *pReq) {
@@ -1485,7 +1566,7 @@ int32_t tSerializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) {
   if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->payloadLen) < 0) return -1;
   if (pReq->payloadLen > 0) {
-    if (tEncodeCStr(&encoder, pReq->payload) < 0) return -1;
+    if (tEncodeBinary(&encoder, pReq->payload, pReq->payloadLen) < 0) return -1;
   }
   tEndEncode(&encoder);
 
@@ -1513,7 +1594,7 @@ int32_t tDeserializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) {
   return 0;
 }
 
-void tFreeSShowReq(SShowReq *pReq) { free(pReq->payload); }
+void tFreeSShowReq(SShowReq *pReq) { tfree(pReq->payload); }
 
 int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq *pReq) {
   SCoder encoder = {0};
@@ -1541,3 +1622,348 @@ int32_t tDeserializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableR
   tCoderClear(&decoder);
   return 0;
 }
+
+static int32_t tEncodeSTableMetaRsp(SCoder *pEncoder, STableMetaRsp *pRsp) {
+  if (tEncodeCStr(pEncoder, pRsp->tbName) < 0) return -1;
+  if (tEncodeCStr(pEncoder, pRsp->stbName) < 0) return -1;
+  if (tEncodeCStr(pEncoder, pRsp->dbFName) < 0) return -1;
+  if (tEncodeU64(pEncoder, pRsp->dbId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pRsp->numOfTags) < 0) return -1;
+  if (tEncodeI32(pEncoder, pRsp->numOfColumns) < 0) return -1;
+  if (tEncodeI8(pEncoder, pRsp->precision) < 0) return -1;
+  if (tEncodeI8(pEncoder, pRsp->tableType) < 0) return -1;
+  if (tEncodeI8(pEncoder, pRsp->update) < 0) return -1;
+  if (tEncodeI32(pEncoder, pRsp->sversion) < 0) return -1;
+  if (tEncodeI32(pEncoder, pRsp->tversion) < 0) return -1;
+  if (tEncodeU64(pEncoder, pRsp->suid) < 0) return -1;
+  if (tEncodeU64(pEncoder, pRsp->tuid) < 0) return -1;
+  if (tEncodeI32(pEncoder, pRsp->vgId) < 0) return -1;
+  for (int32_t i = 0; i < pRsp->numOfColumns + pRsp->numOfTags; ++i) {
+    SSchema *pSchema = &pRsp->pSchemas[i];
+    if (tEncodeSSchema(pEncoder, pSchema) < 0) return -1;
+  }
+
+  return 0;
+}
+
+static int32_t tDecodeSTableMetaRsp(SCoder *pDecoder, STableMetaRsp *pRsp) {
+  if (tDecodeCStrTo(pDecoder, pRsp->tbName) < 0) return -1;
+  if (tDecodeCStrTo(pDecoder, pRsp->stbName) < 0) return -1;
+  if (tDecodeCStrTo(pDecoder, pRsp->dbFName) < 0) return -1;
+  if (tDecodeU64(pDecoder, &pRsp->dbId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pRsp->numOfTags) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pRsp->numOfColumns) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pRsp->precision) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pRsp->tableType) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pRsp->update) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pRsp->sversion) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pRsp->tversion) < 0) return -1;
+  if (tDecodeU64(pDecoder, &pRsp->suid) < 0) return -1;
+  if (tDecodeU64(pDecoder, &pRsp->tuid) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pRsp->vgId) < 0) return -1;
+
+  int32_t totalCols = pRsp->numOfTags + pRsp->numOfColumns;
+  pRsp->pSchemas = malloc(sizeof(SSchema) * totalCols);
+  if (pRsp->pSchemas == NULL) return -1;
+
+  for (int32_t i = 0; i < totalCols; ++i) {
+    SSchema *pSchema = &pRsp->pSchemas[i];
+    if (tDecodeSSchema(pDecoder, pSchema) < 0) return -1;
+  }
+
+  return 0;
+}
+
+int32_t tSerializeSTableMetaRsp(void *buf, int32_t bufLen, STableMetaRsp *pRsp) {
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeSTableMetaRsp(&encoder, pRsp) < 0) return -1;
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tSerializeSTableMetaBatchRsp(void *buf, int32_t bufLen, STableMetaBatchRsp *pRsp) {
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+
+  int32_t numOfBatch = taosArrayGetSize(pRsp->pArray);
+  if (tEncodeI32(&encoder, numOfBatch) < 0) return -1;
+  for (int32_t i = 0; i < numOfBatch; ++i) {
+    STableMetaRsp *pMetaRsp = taosArrayGet(pRsp->pArray, i);
+    if (tEncodeSTableMetaRsp(&encoder, pMetaRsp) < 0) return -1;
+  }
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSTableMetaRsp(void *buf, int32_t bufLen, STableMetaRsp *pRsp) {
+  SCoder decoder = {0};
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeSTableMetaRsp(&decoder, pRsp) < 0) return -1;
+
+  tEndDecode(&decoder);
+  tCoderClear(&decoder);
+  return 0;
+}
+
+int32_t tDeserializeSTableMetaBatchRsp(void *buf, int32_t bufLen, STableMetaBatchRsp *pRsp) {
+  SCoder decoder = {0};
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+
+  int32_t numOfBatch = taosArrayGetSize(pRsp->pArray);
+  if (tDecodeI32(&decoder, &numOfBatch) < 0) return -1;
+
+  pRsp->pArray = taosArrayInit(numOfBatch, sizeof(STableMetaRsp));
+  if (pRsp->pArray == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  for (int32_t i = 0; i < numOfBatch; ++i) {
+    STableMetaRsp tableMetaRsp = {0};
+    if (tDecodeSTableMetaRsp(&decoder, &tableMetaRsp) < 0) return -1;
+    taosArrayPush(pRsp->pArray, &tableMetaRsp);
+  }
+  tEndDecode(&decoder);
+
+  tCoderClear(&decoder);
+  return 0;
+}
+
+void tFreeSTableMetaRsp(STableMetaRsp *pRsp) { tfree(pRsp->pSchemas); }
+
+void tFreeSTableMetaBatchRsp(STableMetaBatchRsp *pRsp) {
+  int32_t numOfBatch = taosArrayGetSize(pRsp->pArray);
+  for (int32_t i = 0; i < numOfBatch; ++i) {
+    STableMetaRsp *pMetaRsp = taosArrayGet(pRsp->pArray, i);
+    tFreeSTableMetaRsp(pMetaRsp);
+  }
+
+  taosArrayDestroy(pRsp->pArray);
+}
+
+int32_t tSerializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) {
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeI64(&encoder, pRsp->showId) < 0) return -1;
+  if (tEncodeSTableMetaRsp(&encoder, &pRsp->tableMeta) < 0) return -1;
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) {
+  SCoder decoder = {0};
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeI64(&decoder, &pRsp->showId) < 0) return -1;
+  if (tDecodeSTableMetaRsp(&decoder, &pRsp->tableMeta) < 0) return -1;
+
+  tEndDecode(&decoder);
+  tCoderClear(&decoder);
+  return 0;
+}
+
+void tFreeSShowRsp(SShowRsp *pRsp) { tFreeSTableMetaRsp(&pRsp->tableMeta); }
+
+int32_t tSerializeSTableInfoReq(void *buf, int32_t bufLen, STableInfoReq *pReq) {
+  int32_t headLen = sizeof(SMsgHead);
+  if (buf != NULL) {
+    buf = (char *)buf + headLen;
+    bufLen -= headLen;
+  }
+
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->dbFName) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->tbName) < 0) return -1;
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+
+  if (buf != NULL) {
+    SMsgHead *pHead = (SMsgHead *)((char *)buf - headLen);
+    pHead->vgId = htonl(pReq->header.vgId);
+    pHead->contLen = htonl(tlen + headLen);
+  }
+
+  return tlen + headLen;
+}
+
+int32_t tDeserializeSTableInfoReq(void *buf, int32_t bufLen, STableInfoReq *pReq) {
+  int32_t headLen = sizeof(SMsgHead);
+
+  SMsgHead *pHead = buf;
+  pHead->vgId = pReq->header.vgId;
+  pHead->contLen = pReq->header.contLen;
+
+  SCoder decoder = {0};
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, (char *)buf + headLen, bufLen - headLen, TD_DECODER);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->dbFName) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->tbName) < 0) return -1;
+
+  tEndDecode(&decoder);
+  tCoderClear(&decoder);
+  return 0;
+}
+
+int32_t tSerializeSMDropTopicReqq(void *buf, int32_t bufLen, SMDropTopicReq *pReq) {
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSMDropTopicReq(void *buf, int32_t bufLen, SMDropTopicReq *pReq) {
+  SCoder decoder = {0};
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
+  tEndDecode(&decoder);
+
+  tCoderClear(&decoder);
+  return 0;
+}
+
+int32_t tSerializeMCreateTopicReq(void **buf, const SMCreateTopicReq *pReq) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI8(buf, pReq->igExists);
+  tlen += taosEncodeString(buf, pReq->name);
+  tlen += taosEncodeString(buf, pReq->sql);
+  tlen += taosEncodeString(buf, pReq->physicalPlan);
+  tlen += taosEncodeString(buf, pReq->logicalPlan);
+  return tlen;
+}
+
+void *tDeserializeSMCreateTopicReq(void *buf, SMCreateTopicReq *pReq) {
+  buf = taosDecodeFixedI8(buf, &(pReq->igExists));
+  buf = taosDecodeString(buf, &(pReq->name));
+  buf = taosDecodeString(buf, &(pReq->sql));
+  buf = taosDecodeString(buf, &(pReq->physicalPlan));
+  buf = taosDecodeString(buf, &(pReq->logicalPlan));
+  return buf;
+}
+
+int32_t tSerializeSMCreateTopicRsp(void *buf, int32_t bufLen, const SMCreateTopicRsp *pRsp) {
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeI64(&encoder, pRsp->topicId) < 0) return -1;
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSMCreateTopicRsp(void *buf, int32_t bufLen, SMCreateTopicRsp *pRsp) {
+  SCoder decoder = {0};
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeI64(&decoder, &pRsp->topicId) < 0) return -1;
+  tEndDecode(&decoder);
+
+  tCoderClear(&decoder);
+  return 0;
+}
+
+int32_t tSerializeSConnectReq(void *buf, int32_t bufLen, SConnectReq *pReq) {
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->pid) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->app) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
+  if (tEncodeI64(&encoder, pReq->startTime) < 0) return -1;
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSConnectReq(void *buf, int32_t bufLen, SConnectReq *pReq) {
+  SCoder decoder = {0};
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->pid) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->app) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
+  if (tDecodeI64(&decoder, &pReq->startTime) < 0) return -1;
+  tEndDecode(&decoder);
+
+  tCoderClear(&decoder);
+  return 0;
+}
+
+int32_t tSerializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) {
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeI32(&encoder, pRsp->acctId) < 0) return -1;
+  if (tEncodeI64(&encoder, pRsp->clusterId) < 0) return -1;
+  if (tEncodeI32(&encoder, pRsp->connId) < 0) return -1;
+  if (tEncodeI8(&encoder, pRsp->superUser) < 0) return -1;
+  if (tEncodeSEpSet(&encoder, &pRsp->epSet) < 0) return -1;
+  if (tEncodeCStr(&encoder, pRsp->sVersion) < 0) return -1;
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) {
+  SCoder decoder = {0};
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeI32(&decoder, &pRsp->acctId) < 0) return -1;
+  if (tDecodeI64(&decoder, &pRsp->clusterId) < 0) return -1;
+  if (tDecodeI32(&decoder, &pRsp->connId) < 0) return -1;
+  if (tDecodeI8(&decoder, &pRsp->superUser) < 0) return -1;
+  if (tDecodeSEpSet(&decoder, &pRsp->epSet) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pRsp->sVersion) < 0) return -1;
+  tEndDecode(&decoder);
+
+  tCoderClear(&decoder);
+  return 0;
+}

source/common/src/tname.c

@@ -1,3 +1,4 @@
+#include <common.h>
 #include "os.h"
 #include "tutil.h"
 
@@ -269,3 +270,4 @@ SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* nam
   tstrncpy(s.name, name, tListLen(s.name));
   return s;
 }
+

source/common/src/ttszip.c

@@ -344,8 +344,8 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
     UNUSED(ret);
   }
 
-  fread(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f);
+  int32_t ret = fread(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f);
-  fread(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f);
+  ret = fread(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f);
 
   // NOTE: mix types tags are not supported
   size_t sz = 0;

source/common/test/commonTests.cpp

@@ -1,11 +1,12 @@
+#include <common.h>
 #include <gtest/gtest.h>
+#include <tep.h>
 #include <iostream>
 
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wwrite-strings"
 #pragma GCC diagnostic ignored "-Wunused-function"
 #pragma GCC diagnostic ignored "-Wunused-variable"
-#pragma GCC diagnostic ignored "-Wunused-but-set-variable"
 #pragma GCC diagnostic ignored "-Wsign-compare"
 #include "os.h"
 
@@ -96,4 +97,199 @@ TEST(testCase, toInteger_test) {
   ASSERT_EQ(ret, -1);
 }
 
+TEST(testCase, Datablock_test) {
+  SSDataBlock* b = static_cast<SSDataBlock*>(calloc(1, sizeof(SSDataBlock)));
+  b->info.numOfCols = 2;
+  b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
+
+  SColumnInfoData infoData = {0};
+  infoData.info.bytes = 4;
+  infoData.info.type = TSDB_DATA_TYPE_INT;
+  infoData.info.colId = 1;
+
+  infoData.pData = (char*) calloc(40, infoData.info.bytes);
+  infoData.nullbitmap = (char*) calloc(1, sizeof(char) * (40/8));
+  taosArrayPush(b->pDataBlock, &infoData);
+
+  SColumnInfoData infoData1 = {0};
+  infoData1.info.bytes = 40;
+  infoData1.info.type = TSDB_DATA_TYPE_BINARY;
+  infoData1.info.colId = 2;
+
+  infoData1.varmeta.offset = (int32_t*) calloc(40, sizeof(uint32_t));
+  taosArrayPush(b->pDataBlock, &infoData1);
+
+  char* str = "the value of: %d";
+  char buf[128] = {0};
+  char varbuf[128] = {0};
+
+  for(int32_t i = 0; i < 40; ++i) {
+    SColumnInfoData* p0 = (SColumnInfoData *) taosArrayGet(b->pDataBlock, 0);
+    SColumnInfoData* p1 = (SColumnInfoData *) taosArrayGet(b->pDataBlock, 1);
+
+    if (i&0x01) {
+      int32_t len = sprintf(buf, str, i);
+      STR_TO_VARSTR(varbuf, buf)
+      colDataAppend(p0, i, (const char*) &i, false);
+      colDataAppend(p1, i, (const char*) varbuf, false);
+
+      memset(varbuf, 0, sizeof(varbuf));
+      memset(buf, 0, sizeof(buf));
+    } else {
+      colDataAppend(p0, i, (const char*) &i, true);
+      colDataAppend(p1, i, (const char*) varbuf, true);
+    }
+
+    b->info.rows++;
+  }
+
+  SColumnInfoData* p0 = (SColumnInfoData *) taosArrayGet(b->pDataBlock, 0);
+  SColumnInfoData* p1 = (SColumnInfoData *) taosArrayGet(b->pDataBlock, 1);
+  for(int32_t i = 0; i < 40; ++i) {
+    if (i & 0x01) {
+      ASSERT_EQ(colDataIsNull_f(p0->nullbitmap, i), false);
+      ASSERT_EQ(colDataIsNull(p1, b->info.rows, i, nullptr), false);
+    } else {
+      ASSERT_EQ(colDataIsNull_f(p0->nullbitmap, i), true);
+
+      ASSERT_EQ(colDataIsNull(p0, b->info.rows, i, nullptr), true);
+      ASSERT_EQ(colDataIsNull(p1, b->info.rows, i, nullptr), true);
+    }
+  }
+
+  printf("binary column length:%d\n", *(int32_t*) p1->pData);
+
+  ASSERT_EQ(colDataGetNumOfCols(b), 2);
+  ASSERT_EQ(colDataGetNumOfRows(b), 40);
+
+  char* pData = colDataGet(p1, 3);
+  printf("the second row of binary:%s, length:%d\n", (char*)varDataVal(pData), varDataLen(pData));
+
+  SArray* pOrderInfo = taosArrayInit(3, sizeof(SBlockOrderInfo));
+  SBlockOrderInfo order = {.order = TSDB_ORDER_ASC, .colIndex = 0};
+  taosArrayPush(pOrderInfo, &order);
+
+  blockDataSort(b, pOrderInfo, true);
+  blockDataDestroy(b);
+
+  taosArrayDestroy(pOrderInfo);
+}
+
+#if 0
+TEST(testCase, non_var_dataBlock_split_test) {
+  SSDataBlock* b = static_cast<SSDataBlock*>(calloc(1, sizeof(SSDataBlock)));
+  b->info.numOfCols = 2;
+  b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
+
+  SColumnInfoData infoData = {0};
+  infoData.info.bytes = 4;
+  infoData.info.type = TSDB_DATA_TYPE_INT;
+  infoData.info.colId = 1;
+
+  int32_t numOfRows = 1000000;
+
+  infoData.pData = (char*) calloc(numOfRows, infoData.info.bytes);
+  infoData.nullbitmap = (char*) calloc(1, sizeof(char) * (numOfRows/8));
+  taosArrayPush(b->pDataBlock, &infoData);
+
+  SColumnInfoData infoData1 = {0};
+  infoData1.info.bytes = 1;
+  infoData1.info.type = TSDB_DATA_TYPE_TINYINT;
+  infoData1.info.colId = 2;
+
+  infoData1.pData = (char*) calloc(numOfRows, infoData.info.bytes);
+  infoData1.nullbitmap = (char*) calloc(1, sizeof(char) * (numOfRows/8));
+  taosArrayPush(b->pDataBlock, &infoData1);
+
+  for(int32_t i = 0; i < numOfRows; ++i) {
+    SColumnInfoData* p0 = (SColumnInfoData*)taosArrayGet(b->pDataBlock, 0);
+    SColumnInfoData* p1 = (SColumnInfoData*)taosArrayGet(b->pDataBlock, 1);
+
+    int8_t v = i;
+    colDataAppend(p0, i, (const char*)&i, false);
+    colDataAppend(p1, i, (const char*)&v, false);
+    b->info.rows++;
+  }
+
+  int32_t pageSize = 64 * 1024;
+
+  int32_t startIndex= 0;
+  int32_t stopIndex = 0;
+  int32_t count = 1;
+  while(1) {
+    blockDataSplitRows(b, false, startIndex, &stopIndex, pageSize);
+    printf("the %d split, from: %d to %d\n", count++, startIndex, stopIndex);
+
+    if (stopIndex == numOfRows - 1) {
+      break;
+    }
+
+    startIndex = stopIndex + 1;
+  }
+
+}
+
+#endif
+
+TEST(testCase, var_dataBlock_split_test) {
+  SSDataBlock* b = static_cast<SSDataBlock*>(calloc(1, sizeof(SSDataBlock)));
+  b->info.numOfCols = 2;
+  b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
+
+  int32_t numOfRows = 1000000;
+
+  SColumnInfoData infoData = {0};
+  infoData.info.bytes = 4;
+  infoData.info.type = TSDB_DATA_TYPE_INT;
+  infoData.info.colId = 1;
+
+  infoData.pData = (char*) calloc(numOfRows, infoData.info.bytes);
+  infoData.nullbitmap = (char*) calloc(1, sizeof(char) * (numOfRows/8));
+  taosArrayPush(b->pDataBlock, &infoData);
+
+  SColumnInfoData infoData1 = {0};
+  infoData1.info.bytes = 40;
+  infoData1.info.type = TSDB_DATA_TYPE_BINARY;
+  infoData1.info.colId = 2;
+
+  infoData1.varmeta.offset = (int32_t*) calloc(numOfRows, sizeof(uint32_t));
+  taosArrayPush(b->pDataBlock, &infoData1);
+
+  char buf[41] = {0};
+  char buf1[100] = {0};
+
+  for(int32_t i = 0; i < numOfRows; ++i) {
+    SColumnInfoData* p0 = (SColumnInfoData*)taosArrayGet(b->pDataBlock, 0);
+    SColumnInfoData* p1 = (SColumnInfoData*)taosArrayGet(b->pDataBlock, 1);
+
+    int8_t v = i;
+    colDataAppend(p0, i, (const char*)&i, false);
+
+    sprintf(buf, "the number of row:%d", i);
+    int32_t len = sprintf(buf1, buf, i);
+    STR_TO_VARSTR(buf1, buf)
+    colDataAppend(p1, i, buf1, false);
+    b->info.rows++;
+
+    memset(buf, 0, sizeof(buf));
+    memset(buf1, 0, sizeof(buf1));
+  }
+
+  int32_t pageSize = 64 * 1024;
+
+  int32_t startIndex= 0;
+  int32_t stopIndex = 0;
+  int32_t count = 1;
+  while(1) {
+    blockDataSplitRows(b, true, startIndex, &stopIndex, pageSize);
+    printf("the %d split, from: %d to %d\n", count++, startIndex, stopIndex);
+
+    if (stopIndex == numOfRows - 1) {
+      break;
+    }
+
+    startIndex = stopIndex + 1;
+  }
+}
+
 #pragma GCC diagnostic pop

source/dnode/mgmt/impl/test/sut/inc/sut.h

@@ -74,7 +74,7 @@ class Testbase {
 
  private:
   int64_t            showId;
-  STableMetaRsp*     pMeta;
+  STableMetaRsp      metaRsp;
   SRetrieveTableRsp* pRetrieveRsp;
   char*              pData;
   int32_t            pos;

source/dnode/mgmt/impl/test/sut/src/sut.cpp

@@ -56,9 +56,16 @@ void Testbase::Init(const char* path, int16_t port) {
   server.Start(path, fqdn, port, firstEp);
   client.Init("root", "taosdata", fqdn, port);
   taosMsleep(1100);
+
+  tFreeSTableMetaRsp(&metaRsp);
+  showId = 0;
+  pData = 0;
+  pos = 0;
+  pRetrieveRsp = NULL;
 }
 
 void Testbase::Cleanup() {
+  tFreeSTableMetaRsp(&metaRsp);
   server.Stop();
   client.Cleanup();
   dndCleanup();
@@ -85,51 +92,43 @@ void Testbase::SendShowMetaReq(int8_t showType, const char* db) {
   strcpy(showReq.db, db);
 
   int32_t contLen = tSerializeSShowReq(NULL, 0, &showReq);
-  char*   pReq = (char*)rpcMallocCont(contLen);
+  void*   pReq = rpcMallocCont(contLen);
   tSerializeSShowReq(pReq, contLen, &showReq);
   tFreeSShowReq(&showReq);
 
   SRpcMsg* pRsp = SendReq(TDMT_MND_SHOW, pReq, contLen);
-  SShowRsp* pShowRsp = (SShowRsp*)pRsp->pCont;
+  ASSERT(pRsp->pCont != nullptr);
 
-  ASSERT(pShowRsp != nullptr);
+  SShowRsp showRsp = {0};
-  pShowRsp->showId = htobe64(pShowRsp->showId);
+  tDeserializeSShowRsp(pRsp->pCont, pRsp->contLen, &showRsp);
-  pMeta = &pShowRsp->tableMeta;
+  tFreeSTableMetaRsp(&metaRsp);
-  pMeta->numOfTags = htonl(pMeta->numOfTags);
+  metaRsp = showRsp.tableMeta;
-  pMeta->numOfColumns = htonl(pMeta->numOfColumns);
+  showId = showRsp.showId;
-  pMeta->sversion = htonl(pMeta->sversion);
-  pMeta->tversion = htonl(pMeta->tversion);
-  pMeta->tuid = htobe64(pMeta->tuid);
-  pMeta->suid = htobe64(pMeta->suid);
-
-  showId = pShowRsp->showId;
 }
 
 int32_t Testbase::GetMetaColId(int32_t index) {
-  SSchema* pSchema = &pMeta->pSchema[index];
+  SSchema* pSchema = &metaRsp.pSchemas[index];
-  pSchema->colId = htonl(pSchema->colId);
   return pSchema->colId;
 }
 
 int8_t Testbase::GetMetaType(int32_t index) {
-  SSchema* pSchema = &pMeta->pSchema[index];
+  SSchema* pSchema = &metaRsp.pSchemas[index];
   return pSchema->type;
 }
 
 int32_t Testbase::GetMetaBytes(int32_t index) {
-  SSchema* pSchema = &pMeta->pSchema[index];
+  SSchema* pSchema = &metaRsp.pSchemas[index];
-  pSchema->bytes = htonl(pSchema->bytes);
   return pSchema->bytes;
 }
 
 const char* Testbase::GetMetaName(int32_t index) {
-  SSchema* pSchema = &pMeta->pSchema[index];
+  SSchema* pSchema = &metaRsp.pSchemas[index];
   return pSchema->name;
 }
 
-int32_t Testbase::GetMetaNum() { return pMeta->numOfColumns; }
+int32_t Testbase::GetMetaNum() { return metaRsp.numOfColumns; }
 
-const char* Testbase::GetMetaTbName() { return pMeta->tbName; }
+const char* Testbase::GetMetaTbName() { return metaRsp.tbName; }
 
 void Testbase::SendShowRetrieveReq() {
   SRetrieveTableReq retrieveReq = {0};
@@ -150,7 +149,7 @@ void Testbase::SendShowRetrieveReq() {
   pos = 0;
 }
 
-const char* Testbase::GetShowName() { return pMeta->tbName; }
+const char* Testbase::GetShowName() { return metaRsp.tbName; }
 
 int8_t Testbase::GetShowInt8() {
   int8_t data = *((int8_t*)(pData + pos));
@@ -191,6 +190,6 @@ const char* Testbase::GetShowBinary(int32_t len) {
 
 int32_t Testbase::GetShowRows() { return pRetrieveRsp->numOfRows; }
 
-STableMetaRsp* Testbase::GetShowMeta() { return pMeta; }
+STableMetaRsp* Testbase::GetShowMeta() { return &metaRsp; }
 
 SRetrieveTableRsp* Testbase::GetRetrieveRsp() { return pRetrieveRsp; }

source/dnode/mnode/impl/inc/mndAuth.h

@@ -36,6 +36,9 @@ int32_t mndCheckCreateDbAuth(SUserObj *pOperUser);
 int32_t mndCheckAlterDropCompactSyncDbAuth(SUserObj *pOperUser, SDbObj *pDb);
 int32_t mndCheckUseDbAuth(SUserObj *pOperUser, SDbObj *pDb);
 
+int32_t mndCheckWriteAuth(SUserObj *pOperUser, SDbObj *pDb);
+int32_t mndCheckReadAuth(SUserObj *pOperUser, SDbObj *pDb);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/mnode/impl/inc/mndDb.h

@@ -24,7 +24,7 @@ extern "C" {
 
 int32_t mndInitDb(SMnode *pMnode);
 void    mndCleanupDb(SMnode *pMnode);
-SDbObj *mndAcquireDb(SMnode *pMnode, char *db);
+SDbObj *mndAcquireDb(SMnode *pMnode, const char *db);
 void    mndReleaseDb(SMnode *pMnode, SDbObj *pDb);
 int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, void **ppRsp, int32_t *pRspLen);
 

source/dnode/mnode/impl/inc/mndStb.h

@@ -24,12 +24,11 @@ extern "C" {
 
 int32_t  mndInitStb(SMnode *pMnode);
 void     mndCleanupStb(SMnode *pMnode);
-
 SStbObj *mndAcquireStb(SMnode *pMnode, char *stbName);
 void     mndReleaseStb(SMnode *pMnode, SStbObj *pStb);
-
+SSdbRaw *mndStbActionEncode(SStbObj *pStb);
-int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *stbs, int32_t num, void **rsp, int32_t *rspLen);
+int32_t  mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbs, int32_t numOfStbs, void **ppRsp,
-
+                            int32_t *pRspLen);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/src/mndAuth.c

@@ -141,3 +141,29 @@ int32_t mndCheckAlterDropCompactSyncDbAuth(SUserObj *pOperUser, SDbObj *pDb) {
 }
 
 int32_t mndCheckUseDbAuth(SUserObj *pOperUser, SDbObj *pDb) { return 0; }
+
+int32_t mndCheckWriteAuth(SUserObj *pOperUser, SDbObj *pDb) {
+  if (pOperUser->superUser || strcmp(pOperUser->user, pDb->createUser) == 0) {
+    return 0;
+  }
+
+  if (taosHashGet(pOperUser->writeDbs, pDb->name, strlen(pDb->name) + 1) != NULL) {
+    return 0;
+  }
+
+  terrno = TSDB_CODE_MND_NO_RIGHTS;
+  return -1;
+}
+
+int32_t mndCheckReadAuth(SUserObj *pOperUser, SDbObj *pDb) {
+  if (pOperUser->superUser || strcmp(pOperUser->user, pDb->createUser) == 0) {
+    return 0;
+  }
+
+  if (taosHashGet(pOperUser->readDbs, pDb->name, strlen(pDb->name) + 1) != NULL) {
+    return 0;
+  }
+
+  terrno = TSDB_CODE_MND_NO_RIGHTS;
+  return -1;
+}

source/dnode/mnode/impl/src/mndBnode.c

@@ -433,27 +433,27 @@ static int32_t mndGetBnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *
   SSdb   *pSdb = pMnode->pSdb;
 
   int32_t  cols = 0;
-  SSchema *pSchema = pMeta->pSchema;
+  SSchema *pSchema = pMeta->pSchemas;
 
   pShow->bytes[cols] = 2;
   pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
   strcpy(pSchema[cols].name, "id");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "endpoint");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
   strcpy(pSchema[cols].name, "create_time");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
-  pMeta->numOfColumns = htonl(cols);
+  pMeta->numOfColumns = cols;
   pShow->numOfColumns = cols;
 
   pShow->offset[0] = 0;

source/dnode/mnode/impl/src/mndCluster.c

@@ -165,27 +165,27 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) {
 
 static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta) {
   int32_t  cols = 0;
-  SSchema *pSchema = pMeta->pSchema;
+  SSchema *pSchema = pMeta->pSchemas;
 
   pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_BIGINT;
   strcpy(pSchema[cols].name, "id");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "name");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
   strcpy(pSchema[cols].name, "create_time");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
-  pMeta->numOfColumns = htonl(cols);
+  pMeta->numOfColumns = cols;
   strcpy(pMeta->tbName, mndShowStr(pShow->type));
   pShow->numOfColumns = cols;
 

source/dnode/mnode/impl/src/mndConsumer.c

@@ -14,8 +14,8 @@
  */
 
 #define _DEFAULT_SOURCE
-
 #include "mndConsumer.h"
+#include "mndAuth.h"
 #include "mndDb.h"
 #include "mndDnode.h"
 #include "mndMnode.h"
@@ -54,12 +54,13 @@ int32_t mndInitConsumer(SMnode *pMnode) {
 void mndCleanupConsumer(SMnode *pMnode) {}
 
 SMqConsumerObj *mndCreateConsumer(int64_t consumerId, const char *cgroup) {
-  SMqConsumerObj* pConsumer = malloc(sizeof(SMqConsumerObj));
+  SMqConsumerObj *pConsumer = calloc(1, sizeof(SMqConsumerObj));
   if (pConsumer == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
-  pConsumer->recentRemovedTopics = taosArrayInit(0, sizeof(char*));
+
+  pConsumer->recentRemovedTopics = taosArrayInit(1, sizeof(char *));
   pConsumer->epoch = 1;
   pConsumer->consumerId = consumerId;
   atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__INIT);
@@ -70,6 +71,7 @@ SMqConsumerObj* mndCreateConsumer(int64_t consumerId, const char* cgroup) {
 
 SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) {
   terrno = TSDB_CODE_OUT_OF_MEMORY;
+
   void   *buf = NULL;
   int32_t tlen = tEncodeSMqConsumerObj(NULL, pConsumer);
   int32_t size = sizeof(int32_t) + tlen + MND_CONSUMER_RESERVE_SIZE;

source/dnode/mnode/impl/src/mndDb.c

@@ -18,6 +18,7 @@
 #include "mndAuth.h"
 #include "mndDnode.h"
 #include "mndShow.h"
+#include "mndStb.h"
 #include "mndTrans.h"
 #include "mndUser.h"
 #include "mndVgroup.h"
@@ -194,7 +195,7 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) {
   return 0;
 }
 
-SDbObj *mndAcquireDb(SMnode *pMnode, char *db) {
+SDbObj *mndAcquireDb(SMnode *pMnode, const char *db) {
   SSdb   *pSdb = pMnode->pSdb;
   SDbObj *pDb = sdbAcquire(pSdb, SDB_DB, db);
   if (pDb == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
@@ -722,6 +723,24 @@ static int32_t mndSetDropDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD
     sdbRelease(pSdb, pVgroup);
   }
 
+  while (1) {
+    SStbObj *pStb = NULL;
+    pIter = sdbFetch(pSdb, SDB_STB, pIter, (void **)&pStb);
+    if (pIter == NULL) break;
+
+    if (pStb->dbUid == pDb->uid) {
+      SSdbRaw *pStbRaw = mndStbActionEncode(pStb);
+      if (pStbRaw == NULL || mndTransAppendCommitlog(pTrans, pStbRaw) != 0) {
+        sdbCancelFetch(pSdb, pIter);
+        sdbRelease(pSdb, pStbRaw);
+        return -1;
+      }
+      sdbSetRawStatus(pStbRaw, SDB_STATUS_DROPPED);
+    }
+
+    sdbRelease(pSdb, pStb);
+  }
+
   return 0;
 }
 
@@ -1111,117 +1130,117 @@ static int32_t mndGetDbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMe
   SSdb   *pSdb = pMnode->pSdb;
 
   int32_t  cols = 0;
-  SSchema *pSchema = pMeta->pSchema;
+  SSchema *pSchema = pMeta->pSchemas;
 
   pShow->bytes[cols] = (TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "name");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
   strcpy(pSchema[cols].name, "create_time");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 2;
   pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
   strcpy(pSchema[cols].name, "vgroups");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 4;
   pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "ntables");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 2;
   pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
   strcpy(pSchema[cols].name, "replica");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 2;
   pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
   strcpy(pSchema[cols].name, "quorum");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 2;
   pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
   strcpy(pSchema[cols].name, "days");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 24 + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "keep0,keep1,keep2");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 4;
   pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "cache");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 4;
   pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "blocks");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 4;
   pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "minrows");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 4;
   pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "maxrows");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 1;
   pSchema[cols].type = TSDB_DATA_TYPE_TINYINT;
   strcpy(pSchema[cols].name, "wallevel");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 4;
   pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "fsync");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 1;
   pSchema[cols].type = TSDB_DATA_TYPE_TINYINT;
   strcpy(pSchema[cols].name, "comp");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 1;
   pSchema[cols].type = TSDB_DATA_TYPE_TINYINT;
   strcpy(pSchema[cols].name, "cachelast");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 3 + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "precision");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 1;
   pSchema[cols].type = TSDB_DATA_TYPE_TINYINT;
   strcpy(pSchema[cols].name, "update");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
-  pMeta->numOfColumns = htonl(cols);
+  pMeta->numOfColumns = cols;
   pShow->numOfColumns = cols;
 
   pShow->offset[0] = 0;

source/dnode/mnode/impl/src/mndDnode.c

@@ -628,21 +628,21 @@ static int32_t mndProcessConfigDnodeRsp(SMnodeMsg *pRsp) {
 
 static int32_t mndGetConfigMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) {
   int32_t  cols = 0;
-  SSchema *pSchema = pMeta->pSchema;
+  SSchema *pSchema = pMeta->pSchemas;
 
   pShow->bytes[cols] = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   tstrncpy(pSchema[cols].name, "name", sizeof(pSchema[cols].name));
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = TSDB_CONIIG_VALUE_LEN + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   tstrncpy(pSchema[cols].name, "value", sizeof(pSchema[cols].name));
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
-  pMeta->numOfColumns = htonl(cols);
+  pMeta->numOfColumns = cols;
   pShow->numOfColumns = cols;
 
   pShow->offset[0] = 0;
@@ -705,51 +705,51 @@ static int32_t mndGetDnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *
   SSdb   *pSdb = pMnode->pSdb;
 
   int32_t  cols = 0;
-  SSchema *pSchema = pMeta->pSchema;
+  SSchema *pSchema = pMeta->pSchemas;
 
   pShow->bytes[cols] = 2;
   pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
   strcpy(pSchema[cols].name, "id");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "endpoint");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 2;
   pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
   strcpy(pSchema[cols].name, "vnodes");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 2;
   pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
   strcpy(pSchema[cols].name, "support_vnodes");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "status");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
   strcpy(pSchema[cols].name, "create_time");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 24 + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "offline_reason");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
-  pMeta->numOfColumns = htonl(cols);
+  pMeta->numOfColumns = cols;
   pShow->numOfColumns = cols;
 
   pShow->offset[0] = 0;

source/dnode/mnode/impl/src/mndFunc.c

@@ -468,51 +468,51 @@ static int32_t mndGetFuncMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *p
   SSdb   *pSdb = pMnode->pSdb;
 
   int32_t  cols = 0;
-  SSchema *pSchema = pMeta->pSchema;
+  SSchema *pSchema = pMeta->pSchemas;
 
   pShow->bytes[cols] = TSDB_FUNC_NAME_LEN + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "name");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = PATH_MAX + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "comment");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 4;
   pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "aggregate");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = TSDB_TYPE_STR_MAX_LEN + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "outputtype");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
   strcpy(pSchema[cols].name, "create_time");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 4;
   pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "code_len");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 4;
   pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "bufsize");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
-  pMeta->numOfColumns = htonl(cols);
+  pMeta->numOfColumns = cols;
   pShow->numOfColumns = cols;
 
   pShow->offset[0] = 0;

source/dnode/mnode/impl/src/mndMnode.c

@@ -243,7 +243,7 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {
       pEpSet->inUse = pEpSet->numOfEps;
     }
 
-    addEpIntoEpSet(pEpSet, pObj->pDnode->fqdn, htons(pObj->pDnode->port));
+    addEpIntoEpSet(pEpSet, pObj->pDnode->fqdn, pObj->pDnode->port);
     sdbRelease(pSdb, pObj);
   }
 }
@@ -620,39 +620,39 @@ static int32_t mndGetMnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *
   SSdb   *pSdb = pMnode->pSdb;
 
   int32_t  cols = 0;
-  SSchema *pSchema = pMeta->pSchema;
+  SSchema *pSchema = pMeta->pSchemas;
 
   pShow->bytes[cols] = 2;
   pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
   strcpy(pSchema[cols].name, "id");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "endpoint");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 12 + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "role");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
   strcpy(pSchema[cols].name, "create_time");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
   strcpy(pSchema[cols].name, "role_time");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
-  pMeta->numOfColumns = htonl(cols);
+  pMeta->numOfColumns = cols;
   pShow->numOfColumns = cols;
 
   pShow->offset[0] = 0;

source/dnode/mnode/impl/src/mndProfile.c

@@ -189,10 +189,12 @@ static int32_t mndProcessConnectReq(SMnodeMsg *pReq) {
   SDbObj     *pDb = NULL;
   SConnObj   *pConn = NULL;
   int32_t     code = -1;
+  SConnectReq connReq = {0};
 
-  SConnectReq *pConnReq = pReq->rpcMsg.pCont;
+  if (tDeserializeSConnectReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &connReq) != 0) {
-  pConnReq->pid = htonl(pConnReq->pid);
+    terrno = TSDB_CODE_INVALID_MSG;
-  pConnReq->startTime = htobe64(pConnReq->startTime);
+    goto CONN_OVER;
+  }
 
   SRpcConnInfo info = {0};
   if (rpcGetConnInfo(pReq->rpcMsg.handle, &info) != 0) {
@@ -209,41 +211,42 @@ static int32_t mndProcessConnectReq(SMnodeMsg *pReq) {
     goto CONN_OVER;
   }
 
-  if (pConnReq->db[0]) {
+  if (connReq.db[0]) {
-    snprintf(pReq->db, TSDB_DB_FNAME_LEN, "%d%s%s", pUser->acctId, TS_PATH_DELIMITER, pConnReq->db);
+    snprintf(pReq->db, TSDB_DB_FNAME_LEN, "%d%s%s", pUser->acctId, TS_PATH_DELIMITER, connReq.db);
     pDb = mndAcquireDb(pMnode, pReq->db);
     if (pDb == NULL) {
       terrno = TSDB_CODE_MND_INVALID_DB;
-      mError("user:%s, failed to login from %s while use db:%s since %s", pReq->user, ip, pConnReq->db, terrstr());
+      mError("user:%s, failed to login from %s while use db:%s since %s", pReq->user, ip, connReq.db, terrstr());
       goto CONN_OVER;
     }
   }
 
-  pConn = mndCreateConn(pMnode, &info, pConnReq->pid, pConnReq->app, pConnReq->startTime);
+  pConn = mndCreateConn(pMnode, &info, connReq.pid, connReq.app, connReq.startTime);
   if (pConn == NULL) {
     mError("user:%s, failed to login from %s while create connection since %s", pReq->user, ip, terrstr());
     goto CONN_OVER;
   }
 
-  SConnectRsp *pRsp = rpcMallocCont(sizeof(SConnectRsp));
+  SConnectRsp connectRsp = {0};
-  if (pRsp == NULL) {
+  connectRsp.acctId = pUser->acctId;
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
+  connectRsp.superUser = pUser->superUser;
-    mError("user:%s, failed to login from %s while create rsp since %s", pReq->user, ip, terrstr());
+  connectRsp.clusterId = pMnode->clusterId;
-    goto CONN_OVER;
+  connectRsp.connId = pConn->id;
-  }
 
-  pRsp->acctId    = htonl(pUser->acctId);
+  snprintf(connectRsp.sVersion, sizeof(connectRsp.sVersion), "ver:%s\nbuild:%s\ngitinfo:%s", version, buildinfo,
-  pRsp->superUser = pUser->superUser;
+           gitinfo);
-  pRsp->clusterId = htobe64(pMnode->clusterId);
+  mndGetMnodeEpSet(pMnode, &connectRsp.epSet);
-  pRsp->connId    = htonl(pConn->id);
 
-  snprintf(pRsp->sVersion, tListLen(pRsp->sVersion), "ver:%s\nbuild:%s\ngitinfo:%s", version, buildinfo, gitinfo);
+  int32_t contLen = tSerializeSConnectRsp(NULL, 0, &connectRsp);
-  mndGetMnodeEpSet(pMnode, &pRsp->epSet);
+  if (contLen < 0) goto CONN_OVER;
+  void *pRsp = rpcMallocCont(contLen);
+  if (pRsp == NULL) goto CONN_OVER;
+  tSerializeSConnectRsp(pRsp, contLen, &connectRsp);
 
-  pReq->contLen = sizeof(SConnectRsp);
+  pReq->contLen = contLen;
   pReq->pCont = pRsp;
 
-  mDebug("user:%s, login from %s, conn:%d, app:%s", info.user, ip, pConn->id, pConnReq->app);
+  mDebug("user:%s, login from %s, conn:%d, app:%s", info.user, ip, pConn->id, connReq.app);
 
   code = 0;
 
@@ -578,53 +581,53 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *
   mndReleaseUser(pMnode, pUser);
 
   int32_t  cols = 0;
-  SSchema *pSchema = pMeta->pSchema;
+  SSchema *pSchema = pMeta->pSchemas;
 
   pShow->bytes[cols] = 4;
   pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "connId");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "user");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   // app name
   pShow->bytes[cols] = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "program");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   // app pid
   pShow->bytes[cols] = 4;
   pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "pid");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "ip:port");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
   strcpy(pSchema[cols].name, "login_time");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
   strcpy(pSchema[cols].name, "last_access");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
-  pMeta->numOfColumns = htonl(cols);
+  pMeta->numOfColumns = cols;
   pShow->numOfColumns = cols;
 
   pShow->offset[0] = 0;
@@ -707,93 +710,93 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *
   mndReleaseUser(pMnode, pUser);
 
   int32_t  cols = 0;
-  SSchema *pSchema = pMeta->pSchema;
+  SSchema *pSchema = pMeta->pSchemas;
 
   pShow->bytes[cols] = 4;
   pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "queryId");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 4;
   pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "connId");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "user");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "ip:port");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 22 + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "qid");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
   strcpy(pSchema[cols].name, "created_time");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_BIGINT;
   strcpy(pSchema[cols].name, "time");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = QUERY_OBJ_ID_SIZE + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "sql_obj_id");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 4;
   pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "pid");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "ep");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 1;
   pSchema[cols].type = TSDB_DATA_TYPE_BOOL;
   strcpy(pSchema[cols].name, "stable_query");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 4;
   pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "sub_queries");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "sub_query_info");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "sql");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
-  pMeta->numOfColumns = htonl(cols);
+  pMeta->numOfColumns = cols;
   pShow->numOfColumns = cols;
 
   pShow->offset[0] = 0;

source/dnode/mnode/impl/src/mndQnode.c

@@ -433,27 +433,27 @@ static int32_t mndGetQnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *
   SSdb   *pSdb = pMnode->pSdb;
 
   int32_t  cols = 0;
-  SSchema *pSchema = pMeta->pSchema;
+  SSchema *pSchema = pMeta->pSchemas;
 
   pShow->bytes[cols] = 2;
   pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
   strcpy(pSchema[cols].name, "id");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "endpoint");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
   strcpy(pSchema[cols].name, "create_time");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
-  pMeta->numOfColumns = htonl(cols);
+  pMeta->numOfColumns = cols;
   pShow->numOfColumns = cols;
 
   pShow->offset[0] = 0;

source/dnode/mnode/impl/src/mndShow.c

@@ -120,6 +120,7 @@ static int32_t mndProcessShowReq(SMnodeMsg *pReq) {
   SShowMgmt *pMgmt = &pMnode->showMgmt;
   int32_t    code = -1;
   SShowReq   showReq = {0};
+  SShowRsp   showRsp = {0};
 
   if (tDeserializeSShowReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &showReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
@@ -142,25 +143,26 @@ static int32_t mndProcessShowReq(SMnodeMsg *pReq) {
     goto SHOW_OVER;
   }
 
-  int32_t   size = sizeof(SShowRsp) + sizeof(SSchema) * TSDB_MAX_COLUMNS + TSDB_EXTRA_PAYLOAD_SIZE;
+  showRsp.showId = pShow->id;
-  SShowRsp *pRsp = rpcMallocCont(size);
+  showRsp.tableMeta.pSchemas = calloc(TSDB_MAX_COLUMNS, sizeof(SSchema));
-  if (pRsp == NULL) {
+  if (showRsp.tableMeta.pSchemas == NULL) {
     mndReleaseShowObj(pShow, true);
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     goto SHOW_OVER;
   }
 
-  code = (*metaFp)(pReq, pShow, &pRsp->tableMeta);
+  code = (*metaFp)(pReq, pShow, &showRsp.tableMeta);
   mDebug("show:0x%" PRIx64 ", get meta finished, numOfRows:%d cols:%d showReq.type:%s, result:%s", pShow->id,
          pShow->numOfRows, pShow->numOfColumns, mndShowStr(showReq.type), tstrerror(code));
 
-  if (code == TSDB_CODE_SUCCESS) {
+  if (code == 0) {
-    pReq->contLen = sizeof(SShowRsp) + sizeof(SSchema) * pShow->numOfColumns;
+    int32_t bufLen = tSerializeSShowRsp(NULL, 0, &showRsp);
-    pReq->pCont = pRsp;
+    void   *pBuf = rpcMallocCont(bufLen);
-    pRsp->showId = htobe64(pShow->id);
+    tSerializeSShowRsp(pBuf, bufLen, &showRsp);
+    pReq->contLen = bufLen;
+    pReq->pCont = pBuf;
     mndReleaseShowObj(pShow, false);
   } else {
-    rpcFreeCont(pRsp);
     mndReleaseShowObj(pShow, true);
   }
 
@@ -170,6 +172,7 @@ SHOW_OVER:
   }
 
   tFreeSShowReq(&showReq);
+  tFreeSShowRsp(&showRsp);
   return code;
 }
 

source/dnode/mnode/impl/src/mndSnode.c

@@ -436,27 +436,27 @@ static int32_t mndGetSnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *
   SSdb   *pSdb = pMnode->pSdb;
 
   int32_t  cols = 0;
-  SSchema *pSchema = pMeta->pSchema;
+  SSchema *pSchema = pMeta->pSchemas;
 
   pShow->bytes[cols] = 2;
   pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
   strcpy(pSchema[cols].name, "id");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "endpoint");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
   strcpy(pSchema[cols].name, "create_time");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
-  pMeta->numOfColumns = htonl(cols);
+  pMeta->numOfColumns = cols;
   pShow->numOfColumns = cols;
 
   pShow->offset[0] = 0;

source/dnode/mnode/impl/src/mndStb.c

@@ -15,6 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "mndStb.h"
+#include "mndAuth.h"
 #include "mndDb.h"
 #include "mndDnode.h"
 #include "mndMnode.h"
@@ -27,7 +28,6 @@
 #define TSDB_STB_VER_NUMBER 1
 #define TSDB_STB_RESERVE_SIZE 64
 
-static SSdbRaw *mndStbActionEncode(SStbObj *pStb);
 static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw);
 static int32_t  mndStbActionInsert(SSdb *pSdb, SStbObj *pStb);
 static int32_t  mndStbActionDelete(SSdb *pSdb, SStbObj *pStb);
@@ -69,7 +69,7 @@ int32_t mndInitStb(SMnode *pMnode) {
 
 void mndCleanupStb(SMnode *pMnode) {}
 
-static SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
+SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
   terrno = TSDB_CODE_OUT_OF_MEMORY;
 
   int32_t  size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + TSDB_STB_RESERVE_SIZE;
@@ -549,12 +549,19 @@ static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq) {
   SStbObj       *pTopicStb = NULL;
   SStbObj       *pStb = NULL;
   SDbObj        *pDb = NULL;
+  SUserObj      *pUser = NULL;
   SMCreateStbReq createReq = {0};
 
-  if (tDeserializeSMCreateStbReq(pReq->rpcMsg.pCont, &createReq) == NULL) goto CREATE_STB_OVER;
+  if (tDeserializeSMCreateStbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    goto CREATE_STB_OVER;
+  }
 
   mDebug("stb:%s, start to create", createReq.name);
-  if (mndCheckCreateStbReq(&createReq) != 0) goto CREATE_STB_OVER;
+  if (mndCheckCreateStbReq(&createReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    goto CREATE_STB_OVER;
+  }
 
   pStb = mndAcquireStb(pMnode, createReq.name);
   if (pStb != NULL) {
@@ -582,6 +589,15 @@ static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq) {
     goto CREATE_STB_OVER;
   }
 
+  pUser = mndAcquireUser(pMnode, pReq->user);
+  if (pUser == NULL) {
+    goto CREATE_STB_OVER;
+  }
+
+  if (mndCheckWriteAuth(pUser, pDb) != 0) {
+    goto CREATE_STB_OVER;
+  }
+
   code = mndCreateStb(pMnode, pReq, &createReq, pDb);
   if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
 
@@ -593,8 +609,8 @@ CREATE_STB_OVER:
   mndReleaseStb(pMnode, pStb);
   mndReleaseStb(pMnode, pTopicStb);
   mndReleaseDb(pMnode, pDb);
-  taosArrayDestroy(createReq.pColumns);
+  mndReleaseUser(pMnode, pUser);
-  taosArrayDestroy(createReq.pTags);
+  tFreeSMCreateStbReq(&createReq);
 
   return code;
 }
@@ -1020,9 +1036,13 @@ static int32_t mndProcessMAlterStbReq(SMnodeMsg *pReq) {
   int32_t      code = -1;
   SDbObj      *pDb = NULL;
   SStbObj     *pStb = NULL;
+  SUserObj    *pUser = NULL;
   SMAltertbReq alterReq = {0};
 
-  if (tDeserializeSMAlterStbReq(pReq->rpcMsg.pCont, &alterReq) == NULL) goto ALTER_STB_OVER;
+  if (tDeserializeSMAlterStbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    goto ALTER_STB_OVER;
+  }
 
   mDebug("stb:%s, start to alter", alterReq.name);
   if (mndCheckAlterStbReq(&alterReq) != 0) goto ALTER_STB_OVER;
@@ -1039,6 +1059,15 @@ static int32_t mndProcessMAlterStbReq(SMnodeMsg *pReq) {
     goto ALTER_STB_OVER;
   }
 
+  pUser = mndAcquireUser(pMnode, pReq->user);
+  if (pUser == NULL) {
+    goto ALTER_STB_OVER;
+  }
+
+  if (mndCheckWriteAuth(pUser, pDb) != 0) {
+    goto ALTER_STB_OVER;
+  }
+
   code = mndAlterStb(pMnode, pReq, &alterReq, pDb, pStb);
   if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
 
@@ -1049,6 +1078,7 @@ ALTER_STB_OVER:
 
   mndReleaseStb(pMnode, pStb);
   mndReleaseDb(pMnode, pDb);
+  mndReleaseUser(pMnode, pUser);
   taosArrayDestroy(alterReq.pFields);
 
   return code;
@@ -1136,42 +1166,59 @@ DROP_STB_OVER:
 
 static int32_t mndProcessMDropStbReq(SMnodeMsg *pReq) {
   SMnode      *pMnode = pReq->pMnode;
-
+  int32_t      code = -1;
+  SUserObj    *pUser = NULL;
+  SDbObj      *pDb = NULL;
+  SStbObj     *pStb = NULL;
   SMDropStbReq dropReq = {0};
-  tDeserializeSMDropStbReq(pReq->rpcMsg.pCont, &dropReq);
+
+  if (tDeserializeSMDropStbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    goto DROP_STB_OVER;
+  }
 
   mDebug("stb:%s, start to drop", dropReq.name);
 
-  SStbObj *pStb = mndAcquireStb(pMnode, dropReq.name);
+  pStb = mndAcquireStb(pMnode, dropReq.name);
   if (pStb == NULL) {
     if (dropReq.igNotExists) {
       mDebug("stb:%s, not exist, ignore not exist is set", dropReq.name);
-      return 0;
+      code = 0;
+      goto DROP_STB_OVER;
     } else {
       terrno = TSDB_CODE_MND_STB_NOT_EXIST;
-      mError("stb:%s, failed to drop since %s", dropReq.name, terrstr());
+      goto DROP_STB_OVER;
-      return -1;
     }
   }
 
-  SDbObj *pDb = mndAcquireDbByStb(pMnode, dropReq.name);
+  pDb = mndAcquireDbByStb(pMnode, dropReq.name);
   if (pDb == NULL) {
-    mndReleaseStb(pMnode, pStb);
     terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
-    mError("stb:%s, failed to drop since %s", dropReq.name, terrstr());
+    goto DROP_STB_OVER;
-    return -1;
+  }
+
+  pUser = mndAcquireUser(pMnode, pReq->user);
+  if (pUser == NULL) {
+    goto DROP_STB_OVER;
+  }
+
+  if (mndCheckWriteAuth(pUser, pDb) != 0) {
+    goto DROP_STB_OVER;
+  }
+
+  code = mndDropStb(pMnode, pReq, pDb, pStb);
+  if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
+
+DROP_STB_OVER:
+  if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
+    mError("stb:%s, failed to drop since %s", dropReq.name, terrstr());
   }
 
-  int32_t code = mndDropStb(pMnode, pReq, pDb, pStb);
   mndReleaseDb(pMnode, pDb);
   mndReleaseStb(pMnode, pStb);
+  mndReleaseUser(pMnode, pUser);
 
-  if (code != 0) {
+  return code;
-    mError("stb:%s, failed to drop since %s", dropReq.name, terrstr());
-    return -1;
-  }
-
-  return TSDB_CODE_MND_ACTION_IN_PROGRESS;
 }
 
 static int32_t mndProcessVDropStbRsp(SMnodeMsg *pRsp) {
@@ -1179,19 +1226,59 @@ static int32_t mndProcessVDropStbRsp(SMnodeMsg *pRsp) {
   return 0;
 }
 
-static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) {
+static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, STableMetaRsp *pRsp) {
-  SMnode        *pMnode = pReq->pMnode;
+  taosRLockLatch(&pStb->lock);
-  STableInfoReq *pInfo = pReq->rpcMsg.pCont;
 
+  int32_t totalCols = pStb->numOfColumns + pStb->numOfTags;
+  pRsp->pSchemas = calloc(totalCols, sizeof(SSchema));
+  if (pRsp->pSchemas == NULL) {
+    taosRUnLockLatch(&pStb->lock);
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  strcpy(pRsp->dbFName, pStb->db);
+  strcpy(pRsp->tbName, tbName);
+  strcpy(pRsp->stbName, tbName);
+  pRsp->dbId = pDb->uid;
+  pRsp->numOfTags = pStb->numOfTags;
+  pRsp->numOfColumns = pStb->numOfColumns;
+  pRsp->precision = pDb->cfg.precision;
+  pRsp->tableType = TSDB_SUPER_TABLE;
+  pRsp->update = pDb->cfg.update;
+  pRsp->sversion = pStb->version;
+  pRsp->suid = pStb->uid;
+  pRsp->tuid = pStb->uid;
+
+  for (int32_t i = 0; i < pStb->numOfColumns; ++i) {
+    SSchema *pSchema = &pRsp->pSchemas[i];
+    SSchema *pSrcSchema = &pStb->pColumns[i];
+    memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN);
+    pSchema->type = pSrcSchema->type;
+    pSchema->colId = pSrcSchema->colId;
+    pSchema->bytes = pSrcSchema->bytes;
+  }
+
+  for (int32_t i = 0; i < pStb->numOfTags; ++i) {
+    SSchema *pSchema = &pRsp->pSchemas[i + pStb->numOfColumns];
+    SSchema *pSrcSchema = &pStb->pTags[i];
+    memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN);
+    pSchema->type = pSrcSchema->type;
+    pSchema->colId = pSrcSchema->colId;
+    pSchema->bytes = pSrcSchema->bytes;
+  }
+
+  taosRUnLockLatch(&pStb->lock);
+  return 0;
+}
+
+static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp) {
   char tbFName[TSDB_TABLE_FNAME_LEN] = {0};
-  snprintf(tbFName, sizeof(tbFName), "%s.%s", pInfo->dbFName, pInfo->tbName);
+  snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, tbName);
 
-  mDebug("stb:%s, start to retrieve meta", tbFName);
+  SDbObj *pDb = mndAcquireDb(pMnode, dbFName);
-
-  SDbObj *pDb = mndAcquireDb(pMnode, pInfo->dbFName);
   if (pDb == NULL) {
     terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
-    mError("stb:%s, failed to retrieve meta since %s", tbFName, terrstr());
     return -1;
   }
 
@@ -1199,175 +1286,103 @@ static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) {
   if (pStb == NULL) {
     mndReleaseDb(pMnode, pDb);
     terrno = TSDB_CODE_MND_INVALID_STB;
-    mError("stb:%s, failed to get meta since %s", tbFName, terrstr());
     return -1;
   }
 
-  taosRLockLatch(&pStb->lock);
+  int32_t code = mndBuildStbSchemaImp(pDb, pStb, tbName, pRsp);
-  int32_t totalCols = pStb->numOfColumns + pStb->numOfTags;
-  int32_t contLen = sizeof(STableMetaRsp) + totalCols * sizeof(SSchema);
-
-  STableMetaRsp *pMeta = rpcMallocCont(contLen);
-  if (pMeta == NULL) {
-    taosRUnLockLatch(&pStb->lock);
   mndReleaseDb(pMnode, pDb);
   mndReleaseStb(pMnode, pStb);
+  return code;
+}
+
+static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) {
+  SMnode       *pMnode = pReq->pMnode;
+  int32_t       code = -1;
+  STableInfoReq infoReq = {0};
+  STableMetaRsp metaRsp = {0};
+
+  if (tDeserializeSTableInfoReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &infoReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    goto RETRIEVE_META_OVER;
+  }
+
+  mDebug("stb:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName);
+  if (mndBuildStbSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp) != 0) {
+    goto RETRIEVE_META_OVER;
+  }
+
+  int32_t rspLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp);
+  if (rspLen < 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    goto RETRIEVE_META_OVER;
+  }
+
+  void *pRsp = rpcMallocCont(rspLen);
+  if (pRsp == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    goto RETRIEVE_META_OVER;
+  }
+
+  tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp);
+  pReq->pCont = pRsp;
+  pReq->contLen = rspLen;
+  code = 0;
+
+  mDebug("stb:%s.%s, meta is retrieved", infoReq.dbFName, infoReq.tbName);
+
+RETRIEVE_META_OVER:
+  if (code != 0) {
+    mError("stb:%s.%s, failed to retrieve meta since %s", infoReq.dbFName, infoReq.tbName, terrstr());
+  }
+
+  tFreeSTableMetaRsp(&metaRsp);
+  return code;
+}
+
+int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbVersions, int32_t numOfStbs, void **ppRsp,
+                           int32_t *pRspLen) {
+  STableMetaBatchRsp batchMetaRsp = {0};
+  batchMetaRsp.pArray = taosArrayInit(numOfStbs, sizeof(STableMetaRsp));
+  if (batchMetaRsp.pArray == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
-    mError("stb:%s, failed to get meta since %s", tbFName, terrstr());
     return -1;
   }
 
-  strcpy(pMeta->dbFName, pStb->db);
+  for (int32_t i = 0; i < numOfStbs; ++i) {
-  strcpy(pMeta->tbName, pInfo->tbName);
+    SSTableMetaVersion *pStbVersion = &pStbVersions[i];
-  strcpy(pMeta->stbName, pInfo->tbName);
+    pStbVersion->suid = be64toh(pStbVersion->suid);
-  pMeta->dbId = htobe64(pDb->uid);
+    pStbVersion->sversion = ntohs(pStbVersion->sversion);
-  pMeta->numOfTags = htonl(pStb->numOfTags);
+    pStbVersion->tversion = ntohs(pStbVersion->tversion);
-  pMeta->numOfColumns = htonl(pStb->numOfColumns);
-  pMeta->precision = pDb->cfg.precision;
-  pMeta->tableType = TSDB_SUPER_TABLE;
-  pMeta->update = pDb->cfg.update;
-  pMeta->sversion = htonl(pStb->version);
-  pMeta->suid = htobe64(pStb->uid);
-  pMeta->tuid = htobe64(pStb->uid);
 
-  for (int32_t i = 0; i < pStb->numOfColumns; ++i) {
+    STableMetaRsp metaRsp = {0};
-    SSchema *pSchema = &pMeta->pSchema[i];
+    mDebug("stb:%s.%s, start to retrieve meta", pStbVersion->dbFName, pStbVersion->stbName);
-    SSchema *pSrcSchema = &pStb->pColumns[i];
+    if (mndBuildStbSchema(pMnode, pStbVersion->dbFName, pStbVersion->stbName, &metaRsp) != 0) {
-    memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN);
+      metaRsp.numOfColumns = -1;
-    pSchema->type = pSrcSchema->type;
+      metaRsp.suid = pStbVersion->suid;
-    pSchema->colId = htonl(pSrcSchema->colId);
-    pSchema->bytes = htonl(pSrcSchema->bytes);
     }
 
-  for (int32_t i = 0; i < pStb->numOfTags; ++i) {
+    if (pStbVersion->sversion != metaRsp.sversion) {
-    SSchema *pSchema = &pMeta->pSchema[i + pStb->numOfColumns];
+      taosArrayPush(batchMetaRsp.pArray, &metaRsp);
-    SSchema *pSrcSchema = &pStb->pTags[i];
+    }
-    memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN);
-    pSchema->type = pSrcSchema->type;
-    pSchema->colId = htonl(pSrcSchema->colId);
-    pSchema->bytes = htonl(pSrcSchema->bytes);
   }
 
-  taosRUnLockLatch(&pStb->lock);
+  int32_t rspLen = tSerializeSTableMetaBatchRsp(NULL, 0, &batchMetaRsp);
-  mndReleaseDb(pMnode, pDb);
+  if (rspLen < 0) {
-  mndReleaseStb(pMnode, pStb);
+    tFreeSTableMetaBatchRsp(&batchMetaRsp);
-
+    terrno = TSDB_CODE_INVALID_MSG;
-  pReq->pCont = pMeta;
+    return -1;
-  pReq->contLen = contLen;
-
-  mDebug("stb:%s, meta is retrieved, cols:%d tags:%d", tbFName, pStb->numOfColumns, pStb->numOfTags);
-  return 0;
   }
 
-int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *stbs, int32_t num, void **rsp, int32_t *rspLen) {
+  void *pRsp = malloc(rspLen);
-  SSdb          *pSdb = pMnode->pSdb;
+  if (pRsp == NULL) {
-  int32_t        bufSize = num * (sizeof(STableMetaRsp) + 4 * sizeof(SSchema));
+    tFreeSTableMetaBatchRsp(&batchMetaRsp);
-  void          *buf = malloc(bufSize);
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
-  int32_t        len = 0;
+    return -1;
-  int32_t        contLen = 0;
-  STableMetaRsp *pRsp = NULL;
-
-  for (int32_t i = 0; i < num; ++i) {
-    SSTableMetaVersion *stb = &stbs[i];
-    stb->suid = be64toh(stb->suid);
-    stb->sversion = ntohs(stb->sversion);
-    stb->tversion = ntohs(stb->tversion);
-
-    if ((contLen + sizeof(STableMetaRsp)) > bufSize) {
-      bufSize = contLen + (num - i) * (sizeof(STableMetaRsp) + 4 * sizeof(SSchema));
-      buf = realloc(buf, bufSize);
-    }
-
-    pRsp = (STableMetaRsp *)((char *)buf + contLen);
-
-    strcpy(pRsp->dbFName, stb->dbFName);
-    strcpy(pRsp->tbName, stb->stbName);
-    strcpy(pRsp->stbName, stb->stbName);
-
-    mDebug("start to retrieve meta, db:%s, stb:%s", stb->dbFName, stb->stbName);
-
-    SDbObj *pDb = mndAcquireDb(pMnode, stb->dbFName);
-    if (pDb == NULL) {
-      pRsp->numOfColumns = -1;
-      pRsp->suid = htobe64(stb->suid);
-      contLen += sizeof(STableMetaRsp);
-      mWarn("db:%s, failed to require db since %s", stb->dbFName, terrstr());
-      continue;
-    }
-
-    char tbFName[TSDB_TABLE_FNAME_LEN] = {0};
-    snprintf(tbFName, sizeof(tbFName), "%s.%s", stb->dbFName, stb->stbName);
-
-    SStbObj *pStb = mndAcquireStb(pMnode, tbFName);
-    if (pStb == NULL) {
-      mndReleaseDb(pMnode, pDb);
-      pRsp->numOfColumns = -1;
-      pRsp->suid = htobe64(stb->suid);
-      contLen += sizeof(STableMetaRsp);
-      mWarn("stb:%s, failed to get meta since %s", tbFName, terrstr());
-      continue;
-    }
-
-    taosRLockLatch(&pStb->lock);
-
-    if (stb->suid == pStb->uid && stb->sversion == pStb->version) {
-      taosRUnLockLatch(&pStb->lock);
-      mndReleaseDb(pMnode, pDb);
-      mndReleaseStb(pMnode, pStb);
-      continue;
-    }
-
-    int32_t totalCols = pStb->numOfColumns + pStb->numOfTags;
-    int32_t len = totalCols * sizeof(SSchema);
-
-    contLen += sizeof(STableMetaRsp) + len;
-
-    if (contLen > bufSize) {
-      bufSize = contLen + (num - i - 1) * (sizeof(STableMetaRsp) + 4 * sizeof(SSchema));
-      buf = realloc(buf, bufSize);
-    }
-
-    pRsp->numOfTags = htonl(pStb->numOfTags);
-    pRsp->numOfColumns = htonl(pStb->numOfColumns);
-    pRsp->precision = pDb->cfg.precision;
-    pRsp->tableType = TSDB_SUPER_TABLE;
-    pRsp->update = pDb->cfg.update;
-    pRsp->sversion = htonl(pStb->version);
-    pRsp->suid = htobe64(pStb->uid);
-    pRsp->tuid = htobe64(pStb->uid);
-
-    for (int32_t i = 0; i < pStb->numOfColumns; ++i) {
-      SSchema *pSchema = &pRsp->pSchema[i];
-      SSchema *pSrcSchema = &pStb->pColumns[i];
-      memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN);
-      pSchema->type = pSrcSchema->type;
-      pSchema->colId = htonl(pSrcSchema->colId);
-      pSchema->bytes = htonl(pSrcSchema->bytes);
-    }
-
-    for (int32_t i = 0; i < pStb->numOfTags; ++i) {
-      SSchema *pSchema = &pRsp->pSchema[i + pStb->numOfColumns];
-      SSchema *pSrcSchema = &pStb->pTags[i];
-      memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN);
-      pSchema->type = pSrcSchema->type;
-      pSchema->colId = htonl(pSrcSchema->colId);
-      pSchema->bytes = htonl(pSrcSchema->bytes);
-    }
-
-    taosRUnLockLatch(&pStb->lock);
-    mndReleaseDb(pMnode, pDb);
-    mndReleaseStb(pMnode, pStb);
-  }
-
-  if (contLen > 0) {
-    *rsp = buf;
-    *rspLen = contLen;
-  } else {
-    *rsp = NULL;
-    tfree(buf);
-    *rspLen = 0;
   }
 
+  tSerializeSTableMetaBatchRsp(pRsp, rspLen, &batchMetaRsp);
+  *ppRsp = pRsp;
+  *pRspLen = rspLen;
   return 0;
 }
 
@@ -1407,33 +1422,33 @@ static int32_t mndGetStbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pM
   }
 
   int32_t  cols = 0;
-  SSchema *pSchema = pMeta->pSchema;
+  SSchema *pSchema = pMeta->pSchemas;
 
   pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "name");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
   strcpy(pSchema[cols].name, "create_time");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 4;
   pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "columns");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 4;
   pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "tags");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
-  pMeta->numOfColumns = htonl(cols);
+  pMeta->numOfColumns = cols;
   pShow->numOfColumns = cols;
 
   pShow->offset[0] = 0;
@@ -1482,7 +1497,7 @@ static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int3
     if (pShow->pIter == NULL) break;
 
     if (pStb->dbUid != pDb->uid) {
-      if (strncmp(pStb->db, pDb->name, tListLen(pStb->db)) == 0) {
+      if (strncmp(pStb->db, pDb->name, prefixLen) == 0) {
         mError("Inconsistent table data, name:%s, db:%s, dbUid:%" PRIu64, pStb->name, pDb->name, pDb->uid);
       }
 

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -12,8 +12,8 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#define _DEFAULT_SOURCE
 
+#define _DEFAULT_SOURCE
 #include "mndSubscribe.h"
 #include "mndConsumer.h"
 #include "mndDb.h"
@@ -54,12 +54,12 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg);
 static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg);
 static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg);
 
-static int mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup,
+static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup,
                                       const SMqConsumerEp *pConsumerEp);
 
 static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp);
 
-static int mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSubscribeObj *pSub);
+static int32_t mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSubscribeObj *pSub);
 
 int32_t mndInitSubscribe(SMnode *pMnode) {
   SSdbTable table = {.sdbType = SDB_SUBSCRIBE,
@@ -232,22 +232,22 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) {
   if (epoch != rsp.epoch) {
     mInfo("send new assignment to consumer, consumer epoch %d, server epoch %d", epoch, rsp.epoch);
     SArray *pTopics = pConsumer->currentTopics;
-    int     sz = taosArrayGetSize(pTopics);
+    int32_t sz = taosArrayGetSize(pTopics);
     rsp.topics = taosArrayInit(sz, sizeof(SMqSubTopicEp));
-    for (int i = 0; i < sz; i++) {
+    for (int32_t i = 0; i < sz; i++) {
       char            *topicName = taosArrayGetP(pTopics, i);
       SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topicName);
       ASSERT(pSub);
-      int csz = taosArrayGetSize(pSub->consumers);
+      int32_t csz = taosArrayGetSize(pSub->consumers);
       // TODO: change to bsearch
-      for (int j = 0; j < csz; j++) {
+      for (int32_t j = 0; j < csz; j++) {
         SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, j);
         if (consumerId == pSubConsumer->consumerId) {
-          int           vgsz = taosArrayGetSize(pSubConsumer->vgInfo);
+          int32_t       vgsz = taosArrayGetSize(pSubConsumer->vgInfo);
           SMqSubTopicEp topicEp;
           strcpy(topicEp.topic, topicName);
           topicEp.vgs = taosArrayInit(vgsz, sizeof(SMqSubVgEp));
-          for (int k = 0; k < vgsz; k++) {
+          for (int32_t k = 0; k < vgsz; k++) {
             SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, k);
 
             SMqSubVgEp vgEp = {.epSet = pConsumerEp->epSet, .vgId = pConsumerEp->vgId};
@@ -276,7 +276,7 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) {
 }
 
 static int32_t mndSplitSubscribeKey(char *key, char **topic, char **cgroup) {
-  int i = 0;
+  int32_t i = 0;
   while (key[i] != ':') {
     i++;
   }
@@ -317,8 +317,8 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) {
           atomic_val_compare_exchange_32(&pConsumer->status, MQ_CONSUMER_STATUS__ACTIVE, MQ_CONSUMER_STATUS__LOST);
       if (old == MQ_CONSUMER_STATUS__ACTIVE) {
         // get all topics of that topic
-        int sz = taosArrayGetSize(pConsumer->currentTopics);
+        int32_t sz = taosArrayGetSize(pConsumer->currentTopics);
-        for (int i = 0; i < sz; i++) {
+        for (int32_t i = 0; i < sz; i++) {
           char            *topic = taosArrayGetP(pConsumer->currentTopics, i);
           char            *key = mndMakeSubscribeKey(pConsumer->cgroup, topic);
           SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key);
@@ -334,8 +334,8 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) {
       } else {
         rebSubs = pConsumer->recentRemovedTopics;
       }
-      int sz = taosArrayGetSize(rebSubs);
+      int32_t sz = taosArrayGetSize(rebSubs);
-      for (int i = 0; i < sz; i++) {
+      for (int32_t i = 0; i < sz; i++) {
         char            *topic = taosArrayGetP(rebSubs, i);
         char            *key = mndMakeSubscribeKey(pConsumer->cgroup, topic);
         SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key);
@@ -375,12 +375,12 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) {
     mInfo("mq rebalance subscription: %s", pSub->key);
 
     // remove lost consumer
-    for (int i = 0; i < taosArrayGetSize(pRebSub->lostConsumers); i++) {
+    for (int32_t i = 0; i < taosArrayGetSize(pRebSub->lostConsumers); i++) {
       int64_t lostConsumerId = *(int64_t *)taosArrayGet(pRebSub->lostConsumers, i);
 
       mInfo("mq remove lost consumer %ld", lostConsumerId);
 
-      for (int j = 0; j < taosArrayGetSize(pSub->consumers); j++) {
+      for (int32_t j = 0; j < taosArrayGetSize(pSub->consumers); j++) {
         SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, j);
         if (pSubConsumer->consumerId == lostConsumerId) {
           taosArrayAddAll(pSub->unassignedVg, pSubConsumer->vgInfo);
@@ -400,10 +400,10 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) {
       int32_t imbalanceSolved = 0;
 
       // iterate all consumers, set unassignedVgStash
-      for (int i = 0; i < consumerNum; i++) {
+      for (int32_t i = 0; i < consumerNum; i++) {
         SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, i);
-        int             vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo);
+        int32_t         vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo);
-        int             vgThisConsumerAfterRb;
+        int32_t         vgThisConsumerAfterRb;
         if (i < imbalanceVg)
           vgThisConsumerAfterRb = vgEachConsumer + 1;
         else
@@ -442,10 +442,10 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) {
 
       // assign to vgroup
       if (taosArrayGetSize(pSub->unassignedVg) != 0) {
-        for (int i = 0; i < consumerNum; i++) {
+        for (int32_t i = 0; i < consumerNum; i++) {
           SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, i);
-          int             vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo);
+          int32_t         vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo);
-          int             vgThisConsumerAfterRb;
+          int32_t         vgThisConsumerAfterRb;
           if (i < imbalanceVg)
             vgThisConsumerAfterRb = vgEachConsumer + 1;
           else
@@ -602,7 +602,7 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) {
 
 #if 0
     //update consumer status for the subscribption
-    for (int i = 0; i < taosArrayGetSize(pSub->assigned); i++) {
+    for (int32_t i = 0; i < taosArrayGetSize(pSub->assigned); i++) {
       SMqConsumerEp *pCEp = taosArrayGet(pSub->assigned, i);
       int64_t        consumerId = pCEp->consumerId;
       if (pCEp->status != -1) {
@@ -619,7 +619,7 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) {
         // TODO: swap with last one, reduce size and reset i
         taosArrayRemove(pSub->assigned, i);
         // remove from available consumer
-        for (int j = 0; j < taosArrayGetSize(pSub->availConsumer); j++) {
+        for (int32_t j = 0; j < taosArrayGetSize(pSub->availConsumer); j++) {
           if (*(int64_t *)taosArrayGet(pSub->availConsumer, i) == pCEp->consumerId) {
             taosArrayRemove(pSub->availConsumer, j);
             break;
@@ -699,7 +699,7 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) {
 }
 #endif
 
-static int mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSubscribeObj *pSub) {
+static int32_t mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSubscribeObj *pSub) {
   SSdb      *pSdb = pMnode->pSdb;
   SVgObj    *pVgroup = NULL;
   SQueryDag *pDag = qStringToDag(pTopic->physicalPlan);
@@ -742,7 +742,7 @@ static int mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSub
   return 0;
 }
 
-static int mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup,
+static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup,
                                       const SMqConsumerEp *pConsumerEp) {
   ASSERT(pConsumerEp->oldConsumerId == -1);
   int32_t vgId = pConsumerEp->vgId;
@@ -890,7 +890,7 @@ static char *mndMakeSubscribeKey(const char *cgroup, const char *topicName) {
   if (key == NULL) {
     return NULL;
   }
-  int tlen = strlen(cgroup);
+  int32_t tlen = strlen(cgroup);
   memcpy(key, cgroup, tlen);
   key[tlen] = ':';
   strcpy(key + tlen + 1, topicName);
@@ -931,12 +931,12 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) {
   char   *cgroup = subscribe.consumerGroup;
 
   SArray *newSub = subscribe.topicNames;
-  int     newTopicNum = subscribe.topicNum;
+  int32_t newTopicNum = subscribe.topicNum;
 
   taosArraySortString(newSub, taosArrayCompareString);
 
   SArray *oldSub = NULL;
-  int     oldTopicNum = 0;
+  int32_t oldTopicNum = 0;
   bool    createConsumer = false;
   // create consumer if not exist
   SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId);
@@ -960,7 +960,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) {
     return -1;
   }
 
-  int i = 0, j = 0;
+  int32_t i = 0, j = 0;
   while (i < newTopicNum || j < oldTopicNum) {
     char *newTopicName = NULL;
     char *oldTopicName = NULL;
@@ -975,7 +975,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) {
       newTopicName = taosArrayGetP(newSub, i);
       oldTopicName = taosArrayGetP(oldSub, j);
 
-      int comp = compareLenPrefixedStr(newTopicName, oldTopicName);
+      int32_t comp = compareLenPrefixedStr(newTopicName, oldTopicName);
       if (comp == 0) {
         // do nothing
         oldTopicName = newTopicName = NULL;
@@ -997,12 +997,12 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) {
       // cancel subscribe of old topic
       SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, cgroup, oldTopicName);
       ASSERT(pSub);
-      int csz = taosArrayGetSize(pSub->consumers);
+      int32_t csz = taosArrayGetSize(pSub->consumers);
-      for (int ci = 0; ci < csz; ci++) {
+      for (int32_t ci = 0; ci < csz; ci++) {
         SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, ci);
         if (pSubConsumer->consumerId == consumerId) {
-          int vgsz = taosArrayGetSize(pSubConsumer->vgInfo);
+          int32_t vgsz = taosArrayGetSize(pSubConsumer->vgInfo);
-          for (int vgi = 0; vgi < vgsz; vgi++) {
+          for (int32_t vgi = 0; vgi < vgsz; vgi++) {
             SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, vgi);
             mndPersistCancelConnReq(pMnode, pTrans, pConsumerEp);
             taosArrayPush(pSub->unassignedVg, pConsumerEp);

source/dnode/mnode/impl/src/mndTopic.c

@@ -31,12 +31,12 @@
 static int32_t mndTopicActionInsert(SSdb *pSdb, SMqTopicObj *pTopic);
 static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic);
 static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pTopic, SMqTopicObj *pNewTopic);
-static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg);
+static int32_t mndProcessCreateTopicReq(SMnodeMsg *pReq);
-static int32_t mndProcessDropTopicMsg(SMnodeMsg *pMsg);
+static int32_t mndProcessDropTopicReq(SMnodeMsg *pReq);
-static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pMsg);
+static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pRsp);
-static int32_t mndProcessTopicMetaMsg(SMnodeMsg *pMsg);
+static int32_t mndProcessTopicMetaReq(SMnodeMsg *pReq);
-static int32_t mndGetTopicMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta);
+static int32_t mndGetTopicMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta);
-static int32_t mndRetrieveTopic(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows);
+static int32_t mndRetrieveTopic(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows);
 static void    mndCancelGetNextTopic(SMnode *pMnode, void *pIter);
 
 int32_t mndInitTopic(SMnode *pMnode) {
@@ -48,8 +48,8 @@ int32_t mndInitTopic(SMnode *pMnode) {
                      .updateFp = (SdbUpdateFp)mndTopicActionUpdate,
                      .deleteFp = (SdbDeleteFp)mndTopicActionDelete};
 
-  mndSetMsgHandle(pMnode, TDMT_MND_CREATE_TOPIC, mndProcessCreateTopicMsg);
+  mndSetMsgHandle(pMnode, TDMT_MND_CREATE_TOPIC, mndProcessCreateTopicReq);
-  mndSetMsgHandle(pMnode, TDMT_MND_DROP_TOPIC, mndProcessDropTopicMsg);
+  mndSetMsgHandle(pMnode, TDMT_MND_DROP_TOPIC, mndProcessDropTopicReq);
   mndSetMsgHandle(pMnode, TDMT_VND_DROP_TOPIC_RSP, mndProcessDropTopicInRsp);
 
   return sdbSetTable(pMnode->pSdb, table);
@@ -225,12 +225,12 @@ static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMq
   return pDrop;
 }
 
-static int32_t mndCheckCreateTopicMsg(SCMCreateTopicReq *creattopReq) {
+static int32_t mndCheckCreateTopicMsg(SMCreateTopicReq *creattopReq) {
   // deserialize and other stuff
   return 0;
 }
 
-static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq *pCreate, SDbObj *pDb) {
+static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SMCreateTopicReq *pCreate, SDbObj *pDb) {
   mDebug("topic:%s to create", pCreate->name);
   SMqTopicObj topicObj = {0};
   tstrncpy(topicObj.name, pCreate->name, TSDB_TOPIC_FNAME_LEN);
@@ -248,7 +248,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq
   SSdbRaw *pTopicRaw = mndTopicActionEncode(&topicObj);
   if (pTopicRaw == NULL) return -1;
   if (sdbSetRawStatus(pTopicRaw, SDB_STATUS_READY) != 0) return -1;
-  /*STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg);*/
+  /*STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pReq->rpcMsg);*/
   /*mndTransAppendRedolog(pTrans, pTopicRaw);*/
   /*if (mndTransPrepare(pMnode, pTrans) != 0) {*/
   /*mError("mq-createTopic-trans:%d, failed to prepare since %s", pTrans->id, terrstr());*/
@@ -260,12 +260,12 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq
   return sdbWrite(pMnode->pSdb, pTopicRaw);
 }
 
-static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg) {
+static int32_t mndProcessCreateTopicReq(SMnodeMsg *pReq) {
-  SMnode *pMnode = pMsg->pMnode;
+  SMnode *pMnode = pReq->pMnode;
-  char   *msgStr = pMsg->rpcMsg.pCont;
+  char   *msgStr = pReq->rpcMsg.pCont;
 
-  SCMCreateTopicReq createTopicReq = {0};
+  SMCreateTopicReq createTopicReq = {0};
-  tDeserializeSCMCreateTopicReq(msgStr, &createTopicReq);
+  tDeserializeSMCreateTopicReq(msgStr, &createTopicReq);
 
   mDebug("topic:%s, start to create, sql:%s", createTopicReq.name, createTopicReq.sql);
 
@@ -294,7 +294,7 @@ static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg) {
     return -1;
   }
 
-  int32_t code = mndCreateTopic(pMnode, pMsg, &createTopicReq, pDb);
+  int32_t code = mndCreateTopic(pMnode, pReq, &createTopicReq, pDb);
   mndReleaseDb(pMnode, pDb);
 
   if (code != TSDB_CODE_SUCCESS) {
@@ -306,48 +306,58 @@ static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg) {
   return 0;
 }
 
-static int32_t mndDropTopic(SMnode *pMnode, SMnodeMsg *pMsg, SMqTopicObj *pTopic) { return 0; }
+static int32_t mndDropTopic(SMnode *pMnode, SMnodeMsg *pReq, SMqTopicObj *pTopic) { return 0; }
 
-static int32_t mndProcessDropTopicMsg(SMnodeMsg *pMsg) {
+static int32_t mndProcessDropTopicReq(SMnodeMsg *pReq) {
-  SMnode        *pMnode = pMsg->pMnode;
+  SMnode        *pMnode = pReq->pMnode;
-  SDropTopicReq *pDrop = pMsg->rpcMsg.pCont;
+  SMDropTopicReq dropReq = {0};
 
-  mDebug("topic:%s, start to drop", pDrop->name);
+  if (tDeserializeSMDropTopicReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
 
-  SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pDrop->name);
+  mDebug("topic:%s, start to drop", dropReq.name);
+
+  SMqTopicObj *pTopic = mndAcquireTopic(pMnode, dropReq.name);
   if (pTopic == NULL) {
-    if (pDrop->igNotExists) {
+    if (dropReq.igNotExists) {
-      mDebug("topic:%s, not exist, ignore not exist is set", pDrop->name);
+      mDebug("topic:%s, not exist, ignore not exist is set", dropReq.name);
       return 0;
     } else {
       terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST;
-      mError("topic:%s, failed to drop since %s", pDrop->name, terrstr());
+      mError("topic:%s, failed to drop since %s", dropReq.name, terrstr());
       return -1;
     }
   }
 
-  int32_t code = mndDropTopic(pMnode, pMsg, pTopic);
+  int32_t code = mndDropTopic(pMnode, pReq, pTopic);
   mndReleaseTopic(pMnode, pTopic);
 
   if (code != 0) {
     terrno = code;
-    mError("topic:%s, failed to drop since %s", pDrop->name, terrstr());
+    mError("topic:%s, failed to drop since %s", dropReq.name, terrstr());
     return -1;
   }
 
   return TSDB_CODE_MND_ACTION_IN_PROGRESS;
 }
 
-static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pMsg) {
+static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pRsp) {
-  mndTransProcessRsp(pMsg);
+  mndTransProcessRsp(pRsp);
   return 0;
 }
 
-static int32_t mndProcessTopicMetaMsg(SMnodeMsg *pMsg) {
+static int32_t mndProcessTopicMetaReq(SMnodeMsg *pReq) {
-  SMnode        *pMnode = pMsg->pMnode;
+  SMnode       *pMnode = pReq->pMnode;
-  STableInfoReq *pInfo = pMsg->rpcMsg.pCont;
+  STableInfoReq infoReq = {0};
 
-  mDebug("topic:%s, start to retrieve meta", pInfo->tbName);
+  if (tSerializeSTableInfoReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &infoReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  mDebug("topic:%s, start to retrieve meta", infoReq.tbName);
 
 #if 0
   SDbObj *pDb = mndAcquireDbByTopic(pMnode, pInfo->tableFname);
@@ -389,7 +399,7 @@ static int32_t mndProcessTopicMetaMsg(SMnodeMsg *pMsg) {
   pMeta->tuid = htonl(pTopic->uid);
 
   for (int32_t i = 0; i < totalCols; ++i) {
-    SSchema *pSchema = &pMeta->pSchema[i];
+    SSchema *pSchema = &pMeta->pSchemas[i];
     SSchema *pSrcSchema = &pTopic->pSchema[i];
     memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN);
     pSchema->type = pSrcSchema->type;
@@ -400,8 +410,8 @@ static int32_t mndProcessTopicMetaMsg(SMnodeMsg *pMsg) {
   mndReleaseDb(pMnode, pDb);
   mndReleaseTopic(pMnode, pTopic);
 
-  pMsg->pCont = pMeta;
+  pReq->pCont = pMeta;
-  pMsg->contLen = contLen;
+  pReq->contLen = contLen;
 
   mDebug("topic:%s, meta is retrieved, cols:%d tags:%d", pInfo->tableFname, pTopic->numOfColumns, pTopic->numOfTags);
 #endif
@@ -433,8 +443,8 @@ static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTo
   return 0;
 }
 
-static int32_t mndGetTopicMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta) {
+static int32_t mndGetTopicMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) {
-  SMnode *pMnode = pMsg->pMnode;
+  SMnode *pMnode = pReq->pMnode;
   SSdb   *pSdb = pMnode->pSdb;
 
   if (mndGetNumOfTopics(pMnode, pShow->db, &pShow->numOfRows) != 0) {
@@ -442,33 +452,21 @@ static int32_t mndGetTopicMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *
   }
 
   int32_t  cols = 0;
-  SSchema *pSchema = pMeta->pSchema;
+  SSchema *pSchema = pMeta->pSchemas;
 
   pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "name");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
   strcpy(pSchema[cols].name, "create_time");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
-  pShow->bytes[cols] = 4;
+  pMeta->numOfColumns = cols;
-  pSchema[cols].type = TSDB_DATA_TYPE_INT;
-  strcpy(pSchema[cols].name, "columns");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_INT;
-  strcpy(pSchema[cols].name, "tags");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
-  cols++;
-
-  pMeta->numOfColumns = htonl(cols);
   pShow->numOfColumns = cols;
 
   pShow->offset[0] = 0;
@@ -483,29 +481,19 @@ static int32_t mndGetTopicMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *
   return 0;
 }
 
-static void mndExtractTableName(char *tableId, char *name) {
+static int32_t mndRetrieveTopic(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) {
-  int32_t pos = -1;
+  SMnode      *pMnode = pReq->pMnode;
-  int32_t num = 0;
-  for (pos = 0; tableId[pos] != 0; ++pos) {
-    if (tableId[pos] == '.') num++;
-    if (num == 2) break;
-  }
-
-  if (num == 2) {
-    strcpy(name, tableId + pos + 1);
-  }
-}
-
-static int32_t mndRetrieveTopic(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) {
-  SMnode      *pMnode = pMsg->pMnode;
   SSdb        *pSdb = pMnode->pSdb;
   int32_t      numOfRows = 0;
   SMqTopicObj *pTopic = NULL;
   int32_t      cols = 0;
   char        *pWrite;
-  char         prefix[64] = {0};
+  char         prefix[TSDB_DB_FNAME_LEN] = {0};
 
-  tstrncpy(prefix, pShow->db, 64);
+  SDbObj *pDb = mndAcquireDb(pMnode, pShow->db);
+  if (pDb == NULL) return 0;
+
+  tstrncpy(prefix, pShow->db, TSDB_DB_FNAME_LEN);
   strcat(prefix, TS_PATH_DELIMITER);
   int32_t prefixLen = (int32_t)strlen(prefix);
 
@@ -513,7 +501,11 @@ static int32_t mndRetrieveTopic(SMnodeMsg *pMsg, SShowObj *pShow, char *data, in
     pShow->pIter = sdbFetch(pSdb, SDB_TOPIC, pShow->pIter, (void **)&pTopic);
     if (pShow->pIter == NULL) break;
 
+    if (pTopic->dbUid != pDb->uid) {
       if (strncmp(pTopic->name, prefix, prefixLen) != 0) {
+        mError("Inconsistent table data, name:%s, db:%s, dbUid:%" PRIu64, pTopic->name, pDb->name, pDb->uid);
+      }
+
       sdbRelease(pSdb, pTopic);
       continue;
     }
@@ -530,18 +522,11 @@ static int32_t mndRetrieveTopic(SMnodeMsg *pMsg, SShowObj *pShow, char *data, in
     *(int64_t *)pWrite = pTopic->createTime;
     cols++;
 
-    /*pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;*/
-    /**(int32_t *)pWrite = pTopic->numOfColumns;*/
-    /*cols++;*/
-
-    /*pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;*/
-    /**(int32_t *)pWrite = pTopic->numOfTags;*/
-    /*cols++;*/
-
     numOfRows++;
     sdbRelease(pSdb, pTopic);
   }
 
+  mndReleaseDb(pMnode, pDb);
   pShow->numOfReads += numOfRows;
   mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
   return numOfRows;

source/dnode/mnode/impl/src/mndUser.c

@@ -302,7 +302,10 @@ static int32_t mndProcessCreateUserReq(SMnodeMsg *pReq) {
   SUserObj      *pOperUser = NULL;
   SCreateUserReq createReq = {0};
 
-  if (tDeserializeSCreateUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) goto CREATE_USER_OVER;
+  if (tDeserializeSCreateUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    goto CREATE_USER_OVER;
+  }
 
   mDebug("user:%s, start to create", createReq.user);
 
@@ -402,7 +405,10 @@ static int32_t mndProcessAlterUserReq(SMnodeMsg *pReq) {
   SUserObj      newUser = {0};
   SAlterUserReq alterReq = {0};
 
-  if (tDeserializeSAlterUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) goto ALTER_USER_OVER;
+  if (tDeserializeSAlterUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    goto ALTER_USER_OVER;
+  }
 
   mDebug("user:%s, start to alter", alterReq.user);
 
@@ -537,7 +543,10 @@ static int32_t mndProcessDropUserReq(SMnodeMsg *pReq) {
   SUserObj    *pOperUser = NULL;
   SDropUserReq dropReq = {0};
 
-  if (tDeserializeSDropUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) goto DROP_USER_OVER;
+  if (tDeserializeSDropUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    goto DROP_USER_OVER;
+  }
 
   mDebug("user:%s, start to drop", dropReq.user);
 
@@ -583,7 +592,10 @@ static int32_t mndProcessGetUserAuthReq(SMnodeMsg *pReq) {
   SGetUserAuthReq authReq = {0};
   SGetUserAuthRsp authRsp = {0};
 
-  if (tDeserializeSGetUserAuthReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &authReq) != 0) goto GET_AUTH_OVER;
+  if (tDeserializeSGetUserAuthReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &authReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    goto GET_AUTH_OVER;
+  }
 
   mTrace("user:%s, start to get auth", authReq.user);
 
@@ -640,33 +652,33 @@ static int32_t mndGetUserMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *p
   SSdb   *pSdb = pMnode->pSdb;
 
   int32_t  cols = 0;
-  SSchema *pSchema = pMeta->pSchema;
+  SSchema *pSchema = pMeta->pSchemas;
 
   pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "name");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "privilege");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
   strcpy(pSchema[cols].name, "create_time");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "account");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
-  pMeta->numOfColumns = htonl(cols);
+  pMeta->numOfColumns = cols;
   pShow->numOfColumns = cols;
 
   pShow->offset[0] = 0;

source/dnode/mnode/impl/src/mndVgroup.c

@@ -488,35 +488,35 @@ static int32_t mndGetVgroupMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp
   }
 
   int32_t  cols = 0;
-  SSchema *pSchema = pMeta->pSchema;
+  SSchema *pSchema = pMeta->pSchemas;
 
   pShow->bytes[cols] = 4;
   pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "vgId");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 4;
   pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "tables");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   for (int32_t i = 0; i < pShow->replica; ++i) {
     pShow->bytes[cols] = 2;
     pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
     snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_dnode", i + 1);
-    pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+    pSchema[cols].bytes = pShow->bytes[cols];
     cols++;
 
     pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE;
     pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
     snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_status", i + 1);
-    pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+    pSchema[cols].bytes = pShow->bytes[cols];
     cols++;
   }
 
-  pMeta->numOfColumns = htonl(cols);
+  pMeta->numOfColumns = cols;
   pShow->numOfColumns = cols;
 
   pShow->offset[0] = 0;
@@ -608,21 +608,21 @@ static int32_t mndGetVnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *
   SSdb   *pSdb = pMnode->pSdb;
 
   int32_t  cols = 0;
-  SSchema *pSchema = pMeta->pSchema;
+  SSchema *pSchema = pMeta->pSchemas;
 
   pShow->bytes[cols] = 4;
   pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "vgId");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
   pShow->bytes[cols] = 12 + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "status");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
+  pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
-  pMeta->numOfColumns = htonl(cols);
+  pMeta->numOfColumns = cols;
   pShow->numOfColumns = cols;
 
   pShow->offset[0] = 0;

source/dnode/mnode/impl/test/profile/profile.cpp

@@ -28,44 +28,44 @@ Testbase MndTestProfile::test;
 int32_t  MndTestProfile::connId;
 
 TEST_F(MndTestProfile, 01_ConnectMsg) {
-  int32_t contLen = sizeof(SConnectReq);
+  SConnectReq connectReq = {0};
+  connectReq.pid = 1234;
+  strcpy(connectReq.app, "mnode_test_profile");
+  strcpy(connectReq.db, "");
 
-  SConnectReq* pReq = (SConnectReq*)rpcMallocCont(contLen);
+  int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq);
-  pReq->pid = htonl(1234);
+  void*   pReq = rpcMallocCont(contLen);
-  strcpy(pReq->app, "mnode_test_profile");
+  tSerializeSConnectReq(pReq, contLen, &connectReq);
-  strcpy(pReq->db, "");
 
   SRpcMsg* pMsg = test.SendReq(TDMT_MND_CONNECT, pReq, contLen);
   ASSERT_NE(pMsg, nullptr);
   ASSERT_EQ(pMsg->code, 0);
 
-  SConnectRsp* pRsp = (SConnectRsp*)pMsg->pCont;
+  SConnectRsp connectRsp = {0};
-  ASSERT_NE(pRsp, nullptr);
+  tDeserializeSConnectRsp(pMsg->pCont, pMsg->contLen, &connectRsp);
-  pRsp->acctId = htonl(pRsp->acctId);
-  pRsp->clusterId = htobe64(pRsp->clusterId);
-  pRsp->connId = htonl(pRsp->connId);
-  pRsp->epSet.eps[0].port = htons(pRsp->epSet.eps[0].port);
 
-  EXPECT_EQ(pRsp->acctId, 1);
+  EXPECT_EQ(connectRsp.acctId, 1);
-  EXPECT_GT(pRsp->clusterId, 0);
+  EXPECT_GT(connectRsp.clusterId, 0);
-  EXPECT_EQ(pRsp->connId, 1);
+  EXPECT_EQ(connectRsp.connId, 1);
-  EXPECT_EQ(pRsp->superUser, 1);
+  EXPECT_EQ(connectRsp.superUser, 1);
 
-  EXPECT_EQ(pRsp->epSet.inUse, 0);
+  EXPECT_EQ(connectRsp.epSet.inUse, 0);
-  EXPECT_EQ(pRsp->epSet.numOfEps, 1);
+  EXPECT_EQ(connectRsp.epSet.numOfEps, 1);
-  EXPECT_EQ(pRsp->epSet.eps[0].port, 9031);
+  EXPECT_EQ(connectRsp.epSet.eps[0].port, 9031);
-  EXPECT_STREQ(pRsp->epSet.eps[0].fqdn, "localhost");
+  EXPECT_STREQ(connectRsp.epSet.eps[0].fqdn, "localhost");
 
-  connId = pRsp->connId;
+  connId = connectRsp.connId;
 }
 
 TEST_F(MndTestProfile, 02_ConnectMsg_InvalidDB) {
-  int32_t contLen = sizeof(SConnectReq);
+  SConnectReq connectReq = {0};
+  connectReq.pid = 1234;
+  strcpy(connectReq.app, "mnode_test_profile");
+  strcpy(connectReq.db, "invalid_db");
 
-  SConnectReq* pReq = (SConnectReq*)rpcMallocCont(contLen);
+  int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq);
-  pReq->pid = htonl(1234);
+  void*   pReq = rpcMallocCont(contLen);
-  strcpy(pReq->app, "mnode_test_profile");
+  tSerializeSConnectReq(pReq, contLen, &connectReq);
-  strcpy(pReq->db, "invalid_db");
 
   SRpcMsg* pRsp = test.SendReq(TDMT_MND_CONNECT, pReq, contLen);
   ASSERT_NE(pRsp, nullptr);
@@ -194,35 +194,33 @@ TEST_F(MndTestProfile, 05_KillConnMsg) {
   }
 
   {
-    int32_t contLen = sizeof(SConnectReq);
+    SConnectReq connectReq = {0};
+    connectReq.pid = 1234;
+    strcpy(connectReq.app, "mnode_test_profile");
+    strcpy(connectReq.db, "invalid_db");
 
-    SConnectReq* pReq = (SConnectReq*)rpcMallocCont(contLen);
+    int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq);
-    pReq->pid = htonl(1234);
+    void*   pReq = rpcMallocCont(contLen);
-    strcpy(pReq->app, "mnode_test_profile");
+    tSerializeSConnectReq(pReq, contLen, &connectReq);
-    strcpy(pReq->db, "");
 
     SRpcMsg* pMsg = test.SendReq(TDMT_MND_CONNECT, pReq, contLen);
     ASSERT_NE(pMsg, nullptr);
     ASSERT_EQ(pMsg->code, 0);
 
-    SConnectRsp* pRsp = (SConnectRsp*)pMsg->pCont;
+    SConnectRsp connectRsp = {0};
-    ASSERT_NE(pRsp, nullptr);
+    tDeserializeSConnectRsp(pMsg->pCont, pMsg->contLen, &connectRsp);
-    pRsp->acctId = htonl(pRsp->acctId);
-    pRsp->clusterId = htobe64(pRsp->clusterId);
-    pRsp->connId = htonl(pRsp->connId);
-    pRsp->epSet.port[0] = htons(pRsp->epSet.port[0]);
 
-    EXPECT_EQ(pRsp->acctId, 1);
+    EXPECT_EQ(connectRsp.acctId, 1);
-    EXPECT_GT(pRsp->clusterId, 0);
+    EXPECT_GT(connectRsp.clusterId, 0);
-    EXPECT_GT(pRsp->connId, connId);
+    EXPECT_GT(connectRsp.connId, connId);
-    EXPECT_EQ(pRsp->superUser, 1);
+    EXPECT_EQ(connectRsp.superUser, 1);
 
-    EXPECT_EQ(pRsp->epSet.inUse, 0);
+    EXPECT_EQ(connectRsp.epSet.inUse, 0);
-    EXPECT_EQ(pRsp->epSet.numOfEps, 1);
+    EXPECT_EQ(connectRsp.epSet.numOfEps, 1);
-    EXPECT_EQ(pRsp->epSet.port[0], 9031);
+    EXPECT_EQ(connectRsp.epSet.port[0], 9031);
-    EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost");
+    EXPECT_STREQ(connectRsp.epSet.fqdn[0], "localhost");
 
-    connId = pRsp->connId;
+    connId = connectRsp.connId;
   }
 #endif
 }

source/dnode/mnode/impl/test/show/show.cpp

@@ -54,12 +54,14 @@ TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) {
 }
 
 TEST_F(MndTestShow, 03_ShowMsg_Conn) {
-  int32_t contLen = sizeof(SConnectReq);
+  SConnectReq connectReq = {0};
+  connectReq.pid = 1234;
+  strcpy(connectReq.app, "mnode_test_show");
+  strcpy(connectReq.db, "");
 
-  SConnectReq* pReq = (SConnectReq*)rpcMallocCont(contLen);
+  int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq);
-  pReq->pid = htonl(1234);
+  void*   pReq = rpcMallocCont(contLen);
-  strcpy(pReq->app, "mnode_test_show");
+  tSerializeSConnectReq(pReq, contLen, &connectReq);
-  strcpy(pReq->db, "");
 
   SRpcMsg* pRsp = test.SendReq(TDMT_MND_CONNECT, pReq, contLen);
   ASSERT_NE(pRsp, nullptr);

source/dnode/mnode/impl/test/stb/stb.cpp

@@ -129,11 +129,10 @@ void* MndTestStb::BuildCreateStbReq(const char* stbname, int32_t* pContLen) {
     taosArrayPush(createReq.pTags, &field);
   }
 
-  int32_t tlen = tSerializeSMCreateStbReq(NULL, &createReq);
+  int32_t tlen = tSerializeSMCreateStbReq(NULL, 0, &createReq);
   void*   pHead = rpcMallocCont(tlen);
-
+  tSerializeSMCreateStbReq(pHead, tlen, &createReq);
-  void* pBuf = pHead;
+  tFreeSMCreateStbReq(&createReq);
-  tSerializeSMCreateStbReq(&pBuf, &createReq);
   *pContLen = tlen;
   return pHead;
 }
@@ -151,10 +150,9 @@ void* MndTestStb::BuildAlterStbAddTagReq(const char* stbname, const char* tagnam
   strcpy(field.name, tagname);
   taosArrayPush(req.pFields, &field);
 
-  int32_t contLen = tSerializeSMAlterStbReq(NULL, &req);
+  int32_t contLen = tSerializeSMAlterStbReq(NULL, 0, &req);
   void*   pHead = rpcMallocCont(contLen);
-  void*   pBuf = pHead;
+  tSerializeSMAlterStbReq(pHead, contLen, &req);
-  tSerializeSMAlterStbReq(&pBuf, &req);
 
   *pContLen = contLen;
   return pHead;
@@ -173,10 +171,9 @@ void* MndTestStb::BuildAlterStbDropTagReq(const char* stbname, const char* tagna
   strcpy(field.name, tagname);
   taosArrayPush(req.pFields, &field);
 
-  int32_t contLen = tSerializeSMAlterStbReq(NULL, &req);
+  int32_t contLen = tSerializeSMAlterStbReq(NULL, 0, &req);
   void*   pHead = rpcMallocCont(contLen);
-  void*   pBuf = pHead;
+  tSerializeSMAlterStbReq(pHead, contLen, &req);
-  tSerializeSMAlterStbReq(&pBuf, &req);
 
   *pContLen = contLen;
   return pHead;
@@ -202,10 +199,9 @@ void* MndTestStb::BuildAlterStbUpdateTagNameReq(const char* stbname, const char*
   strcpy(field2.name, newtagname);
   taosArrayPush(req.pFields, &field2);
 
-  int32_t contLen = tSerializeSMAlterStbReq(NULL, &req);
+  int32_t contLen = tSerializeSMAlterStbReq(NULL, 0, &req);
   void*   pHead = rpcMallocCont(contLen);
-  void*   pBuf = pHead;
+  tSerializeSMAlterStbReq(pHead, contLen, &req);
-  tSerializeSMAlterStbReq(&pBuf, &req);
 
   *pContLen = contLen;
   return pHead;
@@ -225,10 +221,9 @@ void* MndTestStb::BuildAlterStbUpdateTagBytesReq(const char* stbname, const char
   strcpy(field.name, tagname);
   taosArrayPush(req.pFields, &field);
 
-  int32_t contLen = tSerializeSMAlterStbReq(NULL, &req);
+  int32_t contLen = tSerializeSMAlterStbReq(NULL, 0, &req);
   void*   pHead = rpcMallocCont(contLen);
-  void*   pBuf = pHead;
+  tSerializeSMAlterStbReq(pHead, contLen, &req);
-  tSerializeSMAlterStbReq(&pBuf, &req);
 
   *pContLen = contLen;
   return pHead;
@@ -247,10 +242,9 @@ void* MndTestStb::BuildAlterStbAddColumnReq(const char* stbname, const char* col
   strcpy(field.name, colname);
   taosArrayPush(req.pFields, &field);
 
-  int32_t contLen = tSerializeSMAlterStbReq(NULL, &req);
+  int32_t contLen = tSerializeSMAlterStbReq(NULL, 0, &req);
   void*   pHead = rpcMallocCont(contLen);
-  void*   pBuf = pHead;
+  tSerializeSMAlterStbReq(pHead, contLen, &req);
-  tSerializeSMAlterStbReq(&pBuf, &req);
 
   *pContLen = contLen;
   return pHead;
@@ -269,10 +263,9 @@ void* MndTestStb::BuildAlterStbDropColumnReq(const char* stbname, const char* co
   strcpy(field.name, colname);
   taosArrayPush(req.pFields, &field);
 
-  int32_t contLen = tSerializeSMAlterStbReq(NULL, &req);
+  int32_t contLen = tSerializeSMAlterStbReq(NULL, 0, &req);
   void*   pHead = rpcMallocCont(contLen);
-  void*   pBuf = pHead;
+  tSerializeSMAlterStbReq(pHead, contLen, &req);
-  tSerializeSMAlterStbReq(&pBuf, &req);
 
   *pContLen = contLen;
   return pHead;
@@ -292,10 +285,9 @@ void* MndTestStb::BuildAlterStbUpdateColumnBytesReq(const char* stbname, const c
   strcpy(field.name, colname);
   taosArrayPush(req.pFields, &field);
 
-  int32_t contLen = tSerializeSMAlterStbReq(NULL, &req);
+  int32_t contLen = tSerializeSMAlterStbReq(NULL, 0, &req);
   void*   pHead = rpcMallocCont(contLen);
-  void*   pBuf = pHead;
+  tSerializeSMAlterStbReq(pHead, contLen, &req);
-  tSerializeSMAlterStbReq(&pBuf, &req);
 
   *pContLen = contLen;
   return pHead;
@@ -339,45 +331,37 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
 
   // ----- meta ------
   {
-    int32_t        contLen = sizeof(STableInfoReq);
+    STableInfoReq infoReq = {0};
-    STableInfoReq* pReq = (STableInfoReq*)rpcMallocCont(contLen);
+    strcpy(infoReq.dbFName, dbname);
-    strcpy(pReq->dbFName, dbname);
+    strcpy(infoReq.tbName, "stb");
-    strcpy(pReq->tbName, "stb");
+
+    int32_t contLen = tSerializeSTableInfoReq(NULL, 0, &infoReq);
+    void*   pReq = rpcMallocCont(contLen);
+    tSerializeSTableInfoReq(pReq, contLen, &infoReq);
 
     SRpcMsg* pMsg = test.SendReq(TDMT_MND_STB_META, pReq, contLen);
     ASSERT_NE(pMsg, nullptr);
     ASSERT_EQ(pMsg->code, 0);
 
-    STableMetaRsp* pRsp = (STableMetaRsp*)pMsg->pCont;
+    STableMetaRsp metaRsp = {0};
-    pRsp->numOfTags = htonl(pRsp->numOfTags);
+    tDeserializeSTableMetaRsp(pMsg->pCont, pMsg->contLen, &metaRsp);
-    pRsp->numOfColumns = htonl(pRsp->numOfColumns);
-    pRsp->sversion = htonl(pRsp->sversion);
-    pRsp->tversion = htonl(pRsp->tversion);
-    pRsp->suid = be64toh(pRsp->suid);
-    pRsp->tuid = be64toh(pRsp->tuid);
-    pRsp->vgId = be64toh(pRsp->vgId);
-    for (int32_t i = 0; i < pRsp->numOfTags + pRsp->numOfColumns; ++i) {
-      SSchema* pSchema = &pRsp->pSchema[i];
-      pSchema->colId = htonl(pSchema->colId);
-      pSchema->bytes = htonl(pSchema->bytes);
-    }
 
-    EXPECT_STREQ(pRsp->dbFName, dbname);
+    EXPECT_STREQ(metaRsp.dbFName, dbname);
-    EXPECT_STREQ(pRsp->tbName, "stb");
+    EXPECT_STREQ(metaRsp.tbName, "stb");
-    EXPECT_STREQ(pRsp->stbName, "stb");
+    EXPECT_STREQ(metaRsp.stbName, "stb");
-    EXPECT_EQ(pRsp->numOfColumns, 2);
+    EXPECT_EQ(metaRsp.numOfColumns, 2);
-    EXPECT_EQ(pRsp->numOfTags, 3);
+    EXPECT_EQ(metaRsp.numOfTags, 3);
-    EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI);
+    EXPECT_EQ(metaRsp.precision, TSDB_TIME_PRECISION_MILLI);
-    EXPECT_EQ(pRsp->tableType, TSDB_SUPER_TABLE);
+    EXPECT_EQ(metaRsp.tableType, TSDB_SUPER_TABLE);
-    EXPECT_EQ(pRsp->update, 0);
+    EXPECT_EQ(metaRsp.update, 0);
-    EXPECT_EQ(pRsp->sversion, 1);
+    EXPECT_EQ(metaRsp.sversion, 1);
-    EXPECT_EQ(pRsp->tversion, 0);
+    EXPECT_EQ(metaRsp.tversion, 0);
-    EXPECT_GT(pRsp->suid, 0);
+    EXPECT_GT(metaRsp.suid, 0);
-    EXPECT_GT(pRsp->tuid, 0);
+    EXPECT_GT(metaRsp.tuid, 0);
-    EXPECT_EQ(pRsp->vgId, 0);
+    EXPECT_EQ(metaRsp.vgId, 0);
 
     {
-      SSchema* pSchema = &pRsp->pSchema[0];
+      SSchema* pSchema = &metaRsp.pSchemas[0];
       EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP);
       EXPECT_EQ(pSchema->colId, 1);
       EXPECT_EQ(pSchema->bytes, 8);
@@ -385,7 +369,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
     }
 
     {
-      SSchema* pSchema = &pRsp->pSchema[1];
+      SSchema* pSchema = &metaRsp.pSchemas[1];
       EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY);
       EXPECT_EQ(pSchema->colId, 2);
       EXPECT_EQ(pSchema->bytes, 12);
@@ -393,7 +377,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
     }
 
     {
-      SSchema* pSchema = &pRsp->pSchema[2];
+      SSchema* pSchema = &metaRsp.pSchemas[2];
       EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TINYINT);
       EXPECT_EQ(pSchema->colId, 3);
       EXPECT_EQ(pSchema->bytes, 2);
@@ -401,7 +385,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
     }
 
     {
-      SSchema* pSchema = &pRsp->pSchema[3];
+      SSchema* pSchema = &metaRsp.pSchemas[3];
       EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BIGINT);
       EXPECT_EQ(pSchema->colId, 4);
       EXPECT_EQ(pSchema->bytes, 8);
@@ -409,12 +393,14 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
     }
 
     {
-      SSchema* pSchema = &pRsp->pSchema[4];
+      SSchema* pSchema = &metaRsp.pSchemas[4];
       EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY);
       EXPECT_EQ(pSchema->colId, 5);
       EXPECT_EQ(pSchema->bytes, 16);
       EXPECT_STREQ(pSchema->name, "tag3");
     }
+
+    tFreeSTableMetaRsp(&metaRsp);
   }
 
   // restart
@@ -436,10 +422,9 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
     SMDropStbReq dropReq = {0};
     strcpy(dropReq.name, stbname);
 
-    int32_t contLen = tSerializeSMDropStbReq(NULL, &dropReq);
+    int32_t contLen = tSerializeSMDropStbReq(NULL, 0, &dropReq);
     void*   pHead = rpcMallocCont(contLen);
-    void*   pBuf = pHead;
+    tSerializeSMDropStbReq(pHead, contLen, &dropReq);
-    tSerializeSMDropStbReq(&pBuf, &dropReq);
 
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_STB, pHead, contLen);
     ASSERT_NE(pRsp, nullptr);

source/dnode/vnode/src/tq/tq.c

@@ -228,7 +228,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
 
   if (pReq->reqType == TMQ_REQ_TYPE_COMMIT_ONLY) {
     pTopic->committedOffset = pReq->offset;
-    printf("offset %ld committed\n", pTopic->committedOffset);
+    /*printf("offset %ld committed\n", pTopic->committedOffset);*/
     pMsg->pCont = NULL;
     pMsg->contLen = 0;
     pMsg->code = 0;
@@ -239,7 +239,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
   if (pReq->reqType == TMQ_REQ_TYPE_CONSUME_AND_COMMIT) {
     if (pTopic->committedOffset < pReq->offset - 1) {
       pTopic->committedOffset = pReq->offset - 1;
-      printf("offset %ld committed\n", pTopic->committedOffset);
+      /*printf("offset %ld committed\n", pTopic->committedOffset);*/
     }
   }
 

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -2174,8 +2174,8 @@ static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t nu
   assert(cnt <= numOfBlocks && numOfQualTables <= numOfTables);  // the pTableQueryInfo[j]->numOfBlocks may be 0
   sup.numOfTables = numOfQualTables;
 
-  SLoserTreeInfo* pTree = NULL;
+  SMultiwayMergeTreeInfo* pTree = NULL;
-  uint8_t ret = tLoserTreeCreate(&pTree, sup.numOfTables, &sup, dataBlockOrderCompar);
+  uint8_t ret = tMergeTreeCreate(&pTree, sup.numOfTables, &sup, dataBlockOrderCompar);
   if (ret != TSDB_CODE_SUCCESS) {
     cleanBlockOrderSupporter(&sup, numOfTables);
     return TSDB_CODE_TDB_OUT_OF_MEMORY;
@@ -2184,7 +2184,7 @@ static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t nu
   int32_t numOfTotal = 0;
 
   while (numOfTotal < cnt) {
-    int32_t pos = pTree->pNode[0].index;
+    int32_t pos = tMergeTreeGetChosenIndex(pTree);
     int32_t index = sup.blockIndexArray[pos]++;
 
     STableBlockInfo* pBlocksInfo = sup.pDataBlockInfo[pos];
@@ -2195,7 +2195,7 @@ static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t nu
       sup.blockIndexArray[pos] = sup.numOfBlocksPerTable[pos] + 1;
     }
 
-    tLoserTreeAdjust(pTree, pos + sup.numOfTables);
+    tMergeTreeAdjust(pTree, tMergeTreeGetAdjustIndex(pTree));
   }
 
   /*
@@ -3643,13 +3643,13 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch
                                  SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId, uint64_t taskId) {
   STbCfg* pTbCfg = metaGetTbInfoByUid(pMeta, uid);
   if (pTbCfg == NULL) {
-//    tsdbError("%p failed to get stable, uid:%"PRIu64", TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, uid, taskId, reqId);
+    tsdbError("%p failed to get stable, uid:%"PRIu64", TID:0x%"PRIx64" QID:0x%"PRIx64, pMeta, uid, taskId, reqId);
     terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
     goto _error;
   }
 
   if (pTbCfg->type != META_SUPER_TABLE) {
-//    tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, uid, taskId, reqId);
+    tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", TID:0x%"PRIx64" QID:0x%"PRIx64, pMeta, uid, taskId, reqId);
     terrno = TSDB_CODE_OPS_NOT_SUPPORT; //basically, this error is caused by invalid sql issued by client
     goto _error;
   }
@@ -3668,8 +3668,8 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch
     pGroupInfo->numOfTables = (uint32_t) taosArrayGetSize(res);
     pGroupInfo->pGroupList  = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey);
 
-//    tsdbDebug("%p no table name/tag condition, all tables qualified, numOfTables:%u, group:%zu, TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb,
+    tsdbDebug("%p no table name/tag condition, all tables qualified, numOfTables:%u, group:%zu, TID:0x%"PRIx64" QID:0x%"PRIx64, pMeta,
-//              pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList), taskId, reqId);
+              pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList), taskId, reqId);
 
     taosArrayDestroy(res);
     return ret;

source/dnode/vnode/src/vnd/vnodeQuery.c

@@ -71,7 +71,6 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg) {
 }
 
 static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) {
-  STableInfoReq * pReq = (STableInfoReq *)(pMsg->pCont);
   STbCfg *        pTbCfg = NULL;
   STbCfg *        pStbCfg = NULL;
   tb_uid_t        uid;
@@ -79,12 +78,19 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) {
   int32_t         nTagCols;
   SSchemaWrapper *pSW = NULL;
   STableMetaRsp  *pTbMetaMsg = NULL;
+  STableMetaRsp   metaRsp = {0};
   SSchema        *pTagSchema;
   SRpcMsg         rpcMsg;
   int             msgLen = 0;
   int32_t         code = TSDB_CODE_VND_APP_ERROR;
 
-  pTbCfg = metaGetTbInfoByName(pVnode->pMeta, pReq->tbName, &uid);
+  STableInfoReq infoReq = {0};
+  if (tDeserializeSTableInfoReq(pMsg->pCont, pMsg->contLen, &infoReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    goto _exit;
+  }
+
+  pTbCfg = metaGetTbInfoByName(pVnode->pMeta, infoReq.tbName, &uid);
   if (pTbCfg == NULL) {
     code = TSDB_CODE_VND_TB_NOT_EXIST;
     goto _exit;
@@ -114,44 +120,51 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) {
     pTagSchema = NULL;
   }
 
-  msgLen = sizeof(STableMetaRsp) + sizeof(SSchema) * (nCols + nTagCols);
+  metaRsp.pSchemas = calloc(nCols + nTagCols, sizeof(SSchema));
-  pTbMetaMsg = (STableMetaRsp *)rpcMallocCont(msgLen);
+  if (metaRsp.pSchemas == NULL) {
-  if (pTbMetaMsg == NULL) {
     code = TSDB_CODE_VND_OUT_OF_MEMORY;
     goto _exit;
   }
 
-  pTbMetaMsg->dbId = htobe64(pVnode->config.dbId);
+  metaRsp.dbId = htobe64(pVnode->config.dbId);
-  memcpy(pTbMetaMsg->dbFName, pReq->dbFName, sizeof(pTbMetaMsg->dbFName));
+  memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName));
-  strcpy(pTbMetaMsg->tbName, pReq->tbName);
+  strcpy(metaRsp.tbName, infoReq.tbName);
   if (pTbCfg->type == META_CHILD_TABLE) {
-    strcpy(pTbMetaMsg->stbName, pStbCfg->name);
+    strcpy(metaRsp.stbName, pStbCfg->name);
-    pTbMetaMsg->suid = htobe64(pTbCfg->ctbCfg.suid);
+    metaRsp.suid = pTbCfg->ctbCfg.suid;
   } else if (pTbCfg->type == META_SUPER_TABLE) {
-    strcpy(pTbMetaMsg->stbName, pTbCfg->name);
+    strcpy(metaRsp.stbName, pTbCfg->name);
-    pTbMetaMsg->suid = htobe64(uid);
+    metaRsp.suid = uid;
   }
-  pTbMetaMsg->numOfTags = htonl(nTagCols);
+  metaRsp.numOfTags = nTagCols;
-  pTbMetaMsg->numOfColumns = htonl(nCols);
+  metaRsp.numOfColumns = nCols;
-  pTbMetaMsg->tableType = pTbCfg->type;
+  metaRsp.tableType = pTbCfg->type;
-  pTbMetaMsg->tuid = htobe64(uid);
+  metaRsp.tuid = uid;
-  pTbMetaMsg->vgId = htonl(pVnode->vgId);
+  metaRsp.vgId = pVnode->vgId;
 
-  memcpy(pTbMetaMsg->pSchema, pSW->pSchema, sizeof(SSchema) * pSW->nCols);
+  memcpy(metaRsp.pSchemas, pSW->pSchema, sizeof(SSchema) * pSW->nCols);
   if (nTagCols) {
-    memcpy(POINTER_SHIFT(pTbMetaMsg->pSchema, sizeof(SSchema) * pSW->nCols), pTagSchema, sizeof(SSchema) * nTagCols);
+    memcpy(POINTER_SHIFT(metaRsp.pSchemas, sizeof(SSchema) * pSW->nCols), pTagSchema, sizeof(SSchema) * nTagCols);
   }
 
-  for (int i = 0; i < nCols + nTagCols; i++) {
+  int32_t rspLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp);
-    SSchema *pSch = pTbMetaMsg->pSchema + i;
+  if (rspLen < 0) {
-    pSch->colId = htonl(pSch->colId);
+    code = TSDB_CODE_INVALID_MSG;
-    pSch->bytes = htonl(pSch->bytes);
+    goto _exit;
   }
 
+  void *pRsp = rpcMallocCont(rspLen);
+  if (pRsp == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _exit;
+  }
+  tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp);
+
   code = 0;
 
 _exit:
 
+  tFreeSTableMetaRsp(&metaRsp);
   if (pSW != NULL) {
     tfree(pSW->pSchema);
     tfree(pSW);
@@ -170,13 +183,13 @@ _exit:
 
   rpcMsg.handle = pMsg->handle;
   rpcMsg.ahandle = pMsg->ahandle;
-  rpcMsg.pCont = pTbMetaMsg;
+  rpcMsg.pCont = pRsp;
-  rpcMsg.contLen = msgLen;
+  rpcMsg.contLen = rspLen;
   rpcMsg.code = code;
 
   rpcSendResponse(&rpcMsg);
 
-  return 0;
+  return code;
 }
 
 static void freeItemHelper(void *pItem) {

source/libs/catalog/src/catalog.c

@@ -1440,7 +1440,7 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgm
 
   SCtgUpdateTblMsg *msg = NULL;
   STableMetaOutput  moutput = {0};
-  STableMetaOutput *output = malloc(sizeof(STableMetaOutput));
+  STableMetaOutput *output = calloc(1, sizeof(STableMetaOutput));
   if (NULL == output) {
     ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput));
     CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);

source/libs/catalog/test/catalogTests.cpp

@@ -222,7 +222,7 @@ void ctgTestBuildDBVgroup(SDBVgInfo **pdbVgroup) {
     for (int32_t n = 0; n < vgInfo.epset.numOfEps; ++n) {
       SEp *addr = &vgInfo.epset.eps[n];
       strcpy(addr->fqdn, "a0");
-      addr->port = htons(n + 22);
+      addr->port = n + 22;
     }
 
     taosHashPut(dbVgroup->vgHash, &vgInfo.vgId, sizeof(vgInfo.vgId), &vgInfo, sizeof(vgInfo));
@@ -247,19 +247,19 @@ void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) {
   rspMsg->vgId = 1;
 
   SSchema *s = NULL;
-  s = &rspMsg->pSchema[0];
+  s = &rspMsg->pSchemas[0];
   s->type = TSDB_DATA_TYPE_TIMESTAMP;
   s->colId = 1;
   s->bytes = 8;
   strcpy(s->name, "ts");
 
-  s = &rspMsg->pSchema[1];
+  s = &rspMsg->pSchemas[1];
   s->type = TSDB_DATA_TYPE_INT;
   s->colId = 2;
   s->bytes = 4;
   strcpy(s->name, "col1s");
 
-  s = &rspMsg->pSchema[2];
+  s = &rspMsg->pSchemas[2];
   s->type = TSDB_DATA_TYPE_BINARY;
   s->colId = 3;
   s->bytes = 12 + 1;
@@ -309,173 +309,189 @@ void ctgTestRspDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *
 }
 
 void ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
-  STableMetaRsp *rspMsg = NULL;  // todo
+  STableMetaRsp metaRsp = {0};
-
+  strcpy(metaRsp.dbFName, ctgTestDbname);
-  pRsp->code = 0;
+  strcpy(metaRsp.tbName, ctgTestTablename);
-  pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema);
+  metaRsp.numOfTags = 0;
-  pRsp->pCont = calloc(1, pRsp->contLen);
+  metaRsp.numOfColumns = ctgTestColNum;
-  rspMsg = (STableMetaRsp *)pRsp->pCont;
+  metaRsp.precision = 1;
-  strcpy(rspMsg->dbFName, ctgTestDbname);
+  metaRsp.tableType = TSDB_NORMAL_TABLE;
-  strcpy(rspMsg->tbName, ctgTestTablename);
+  metaRsp.update = 1;
-  rspMsg->numOfTags = 0;
+  metaRsp.sversion = ctgTestSVersion;
-  rspMsg->numOfColumns = htonl(ctgTestColNum);
+  metaRsp.tversion = ctgTestTVersion;
-  rspMsg->precision = 1;
+  metaRsp.suid = 0;
-  rspMsg->tableType = TSDB_NORMAL_TABLE;
+  metaRsp.tuid = 0x0000000000000001;
-  rspMsg->update = 1;
+  metaRsp.vgId = 8;
-  rspMsg->sversion = htonl(ctgTestSVersion);
+  metaRsp.pSchemas = (SSchema *)malloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema));
-  rspMsg->tversion = htonl(ctgTestTVersion);
-  rspMsg->suid = 0;
-  rspMsg->tuid = htobe64(0x0000000000000001);
-  rspMsg->vgId = htonl(8);
 
   SSchema *s = NULL;
-  s = &rspMsg->pSchema[0];
+  s = &metaRsp.pSchemas[0];
   s->type = TSDB_DATA_TYPE_TIMESTAMP;
-  s->colId = htonl(1);
+  s->colId = 1;
-  s->bytes = htonl(8);
+  s->bytes = 8;
   strcpy(s->name, "ts");
 
-  s = &rspMsg->pSchema[1];
+  s = &metaRsp.pSchemas[1];
   s->type = TSDB_DATA_TYPE_INT;
-  s->colId = htonl(2);
+  s->colId = 2;
-  s->bytes = htonl(4);
+  s->bytes = 4;
   strcpy(s->name, "col1");
 
-  return;
+  int32_t contLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp);
+  void   *pReq = rpcMallocCont(contLen);
+  tSerializeSTableMetaRsp(pReq, contLen, &metaRsp);
+
+  pRsp->code = 0;
+  pRsp->contLen = contLen;
+  pRsp->pCont = pReq;
+
+  tFreeSTableMetaRsp(&metaRsp);
 }
 
 void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
-  STableMetaRsp *rspMsg = NULL;  // todo
+  STableMetaRsp metaRsp = {0};
-
+  strcpy(metaRsp.dbFName, ctgTestDbname);
-  pRsp->code = 0;
+  strcpy(metaRsp.tbName, ctgTestCTablename);
-  pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema);
+  strcpy(metaRsp.stbName, ctgTestSTablename);
-  pRsp->pCont = calloc(1, pRsp->contLen);
+  metaRsp.numOfTags = ctgTestTagNum;
-  rspMsg = (STableMetaRsp *)pRsp->pCont;
+  metaRsp.numOfColumns = ctgTestColNum;
-  strcpy(rspMsg->dbFName, ctgTestDbname);
+  metaRsp.precision = 1;
-  strcpy(rspMsg->tbName, ctgTestCTablename);
+  metaRsp.tableType = TSDB_CHILD_TABLE;
-  strcpy(rspMsg->stbName, ctgTestSTablename);
+  metaRsp.update = 1;
-  rspMsg->numOfTags = htonl(ctgTestTagNum);
+  metaRsp.sversion = ctgTestSVersion;
-  rspMsg->numOfColumns = htonl(ctgTestColNum);
+  metaRsp.tversion = ctgTestTVersion;
-  rspMsg->precision = 1;
+  metaRsp.suid = 0x0000000000000002;
-  rspMsg->tableType = TSDB_CHILD_TABLE;
+  metaRsp.tuid = 0x0000000000000003;
-  rspMsg->update = 1;
+  metaRsp.vgId = 9;
-  rspMsg->sversion = htonl(ctgTestSVersion);
+  metaRsp.pSchemas = (SSchema *)malloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema));
-  rspMsg->tversion = htonl(ctgTestTVersion);
-  rspMsg->suid = htobe64(0x0000000000000002);
-  rspMsg->tuid = htobe64(0x0000000000000003);
-  rspMsg->vgId = htonl(9);
 
   SSchema *s = NULL;
-  s = &rspMsg->pSchema[0];
+  s = &metaRsp.pSchemas[0];
   s->type = TSDB_DATA_TYPE_TIMESTAMP;
-  s->colId = htonl(1);
+  s->colId = 1;
-  s->bytes = htonl(8);
+  s->bytes = 8;
   strcpy(s->name, "ts");
 
-  s = &rspMsg->pSchema[1];
+  s = &metaRsp.pSchemas[1];
   s->type = TSDB_DATA_TYPE_INT;
-  s->colId = htonl(2);
+  s->colId = 2;
-  s->bytes = htonl(4);
+  s->bytes = 4;
   strcpy(s->name, "col1s");
 
-  s = &rspMsg->pSchema[2];
+  s = &metaRsp.pSchemas[2];
   s->type = TSDB_DATA_TYPE_BINARY;
-  s->colId = htonl(3);
+  s->colId = 3;
-  s->bytes = htonl(12);
+  s->bytes = 12;
   strcpy(s->name, "tag1s");
 
-  return;
+  int32_t contLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp);
+  void   *pReq = rpcMallocCont(contLen);
+  tSerializeSTableMetaRsp(pReq, contLen, &metaRsp);
+
+  pRsp->code = 0;
+  pRsp->contLen = contLen;
+  pRsp->pCont = pReq;
+
+  tFreeSTableMetaRsp(&metaRsp);
 }
 
 void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
-  STableMetaRsp *rspMsg = NULL;  // todo
+  STableMetaRsp metaRsp = {0};
-
+  strcpy(metaRsp.dbFName, ctgTestDbname);
-  pRsp->code = 0;
+  strcpy(metaRsp.tbName, ctgTestSTablename);
-  pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema);
+  strcpy(metaRsp.stbName, ctgTestSTablename);
-  pRsp->pCont = calloc(1, pRsp->contLen);
+  metaRsp.numOfTags = ctgTestTagNum;
-  rspMsg = (STableMetaRsp *)pRsp->pCont;
+  metaRsp.numOfColumns = ctgTestColNum;
-  strcpy(rspMsg->dbFName, ctgTestDbname);
+  metaRsp.precision = 1;
-  strcpy(rspMsg->tbName, ctgTestSTablename);
+  metaRsp.tableType = TSDB_SUPER_TABLE;
-  strcpy(rspMsg->stbName, ctgTestSTablename);
+  metaRsp.update = 1;
-  rspMsg->numOfTags = htonl(ctgTestTagNum);
+  metaRsp.sversion = ctgTestSVersion;
-  rspMsg->numOfColumns = htonl(ctgTestColNum);
+  metaRsp.tversion = ctgTestTVersion;
-  rspMsg->precision = 1;
+  metaRsp.suid = ctgTestSuid;
-  rspMsg->tableType = TSDB_SUPER_TABLE;
+  metaRsp.tuid = ctgTestSuid;
-  rspMsg->update = 1;
+  metaRsp.vgId = 0;
-  rspMsg->sversion = htonl(ctgTestSVersion);
+  metaRsp.pSchemas = (SSchema *)malloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema));
-  rspMsg->tversion = htonl(ctgTestTVersion);
-  rspMsg->suid = htobe64(ctgTestSuid);
-  rspMsg->tuid = htobe64(ctgTestSuid);
-  rspMsg->vgId = 0;
 
   SSchema *s = NULL;
-  s = &rspMsg->pSchema[0];
+  s = &metaRsp.pSchemas[0];
   s->type = TSDB_DATA_TYPE_TIMESTAMP;
-  s->colId = htonl(1);
+  s->colId = 1;
-  s->bytes = htonl(8);
+  s->bytes = 8;
   strcpy(s->name, "ts");
 
-  s = &rspMsg->pSchema[1];
+  s = &metaRsp.pSchemas[1];
   s->type = TSDB_DATA_TYPE_INT;
-  s->colId = htonl(2);
+  s->colId = 2;
-  s->bytes = htonl(4);
+  s->bytes = 4;
   strcpy(s->name, "col1s");
 
-  s = &rspMsg->pSchema[2];
+  s = &metaRsp.pSchemas[2];
   s->type = TSDB_DATA_TYPE_BINARY;
-  s->colId = htonl(3);
+  s->colId = 3;
-  s->bytes = htonl(12);
+  s->bytes = 12;
   strcpy(s->name, "tag1s");
 
-  return;
+  int32_t contLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp);
+  void   *pReq = rpcMallocCont(contLen);
+  tSerializeSTableMetaRsp(pReq, contLen, &metaRsp);
+
+  pRsp->code = 0;
+  pRsp->contLen = contLen;
+  pRsp->pCont = pReq;
+
+  tFreeSTableMetaRsp(&metaRsp);
 }
 
 void ctgTestRspMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
-  STableMetaRsp *rspMsg = NULL;  // todo
   static int32_t idx = 1;
 
-  pRsp->code = 0;
+  STableMetaRsp metaRsp = {0};
-  pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema);
+  strcpy(metaRsp.dbFName, ctgTestDbname);
-  pRsp->pCont = calloc(1, pRsp->contLen);
+  sprintf(metaRsp.tbName, "%s_%d", ctgTestSTablename, idx);
-  rspMsg = (STableMetaRsp *)pRsp->pCont;
+  sprintf(metaRsp.stbName, "%s_%d", ctgTestSTablename, idx);
-  strcpy(rspMsg->dbFName, ctgTestDbname);
+  metaRsp.numOfTags = ctgTestTagNum;
-  sprintf(rspMsg->tbName, "%s_%d", ctgTestSTablename, idx);
+  metaRsp.numOfColumns = ctgTestColNum;
-  sprintf(rspMsg->stbName, "%s_%d", ctgTestSTablename, idx);
+  metaRsp.precision = 1;
-  rspMsg->numOfTags = htonl(ctgTestTagNum);
+  metaRsp.tableType = TSDB_SUPER_TABLE;
-  rspMsg->numOfColumns = htonl(ctgTestColNum);
+  metaRsp.update = 1;
-  rspMsg->precision = 1;
+  metaRsp.sversion = ctgTestSVersion;
-  rspMsg->tableType = TSDB_SUPER_TABLE;
+  metaRsp.tversion = ctgTestTVersion;
-  rspMsg->update = 1;
+  metaRsp.suid = ctgTestSuid + idx;
-  rspMsg->sversion = htonl(ctgTestSVersion);
+  metaRsp.tuid = ctgTestSuid + idx;
-  rspMsg->tversion = htonl(ctgTestTVersion);
+  metaRsp.vgId = 0;
-  rspMsg->suid = htobe64(ctgTestSuid + idx);
+  metaRsp.pSchemas = (SSchema *)malloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema));
-  rspMsg->tuid = htobe64(ctgTestSuid + idx);
-  rspMsg->vgId = 0;
 
   SSchema *s = NULL;
-  s = &rspMsg->pSchema[0];
+  s = &metaRsp.pSchemas[0];
   s->type = TSDB_DATA_TYPE_TIMESTAMP;
-  s->colId = htonl(1);
+  s->colId = 1;
-  s->bytes = htonl(8);
+  s->bytes = 8;
   strcpy(s->name, "ts");
 
-  s = &rspMsg->pSchema[1];
+  s = &metaRsp.pSchemas[1];
   s->type = TSDB_DATA_TYPE_INT;
-  s->colId = htonl(2);
+  s->colId = 2;
-  s->bytes = htonl(4);
+  s->bytes = 4;
   strcpy(s->name, "col1s");
 
-  s = &rspMsg->pSchema[2];
+  s = &metaRsp.pSchemas[2];
   s->type = TSDB_DATA_TYPE_BINARY;
-  s->colId = htonl(3);
+  s->colId = 3;
-  s->bytes = htonl(12);
+  s->bytes = 12;
   strcpy(s->name, "tag1s");
 
   ++idx;
 
-  return;
+  int32_t contLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp);
-}
+  void   *pReq = rpcMallocCont(contLen);
+  tSerializeSTableMetaRsp(pReq, contLen, &metaRsp);
 
+  pRsp->code = 0;
+  pRsp->contLen = contLen;
+  pRsp->pCont = pReq;
+
+  tFreeSTableMetaRsp(&metaRsp);
+}
 
 void ctgTestRspByIdx(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
   switch (ctgTestRspFunc[ctgTestRspIdx]) {
@@ -503,7 +519,6 @@ void ctgTestRspByIdx(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp
   return;
 }
 
-
 void ctgTestRspDbVgroupsAndNormalMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
   ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp);
 

source/libs/executor/inc/executil.h

@@ -16,8 +16,8 @@
 #define TDENGINE_QUERYUTIL_H
 
 #include "common.h"
-#include "tpagedfile.h"
 #include "tbuffer.h"
+#include "tpagedbuf.h"
 
 #define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid)     \
   do {                                               \
@@ -126,6 +126,13 @@ static FORCE_INLINE char* getPosInResultPage(struct STaskAttr* pQueryAttr, SFile
 //  return ((char *)page->data) + rowOffset + offset * numOfRows;
 }
 
+static FORCE_INLINE char* getPosInResultPage_rv(SFilePage* page, int32_t rowOffset, int32_t offset) {
+  assert(rowOffset >= 0);
+
+  int32_t numOfRows = 1;//(int32_t)getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery);
+  return ((char *)page->data) + rowOffset + offset * numOfRows;
+}
+
 //bool isNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type);
 //bool notNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type);
 

source/libs/executor/inc/executorimpl.h

@@ -15,21 +15,26 @@
 #ifndef TDENGINE_EXECUTORIMPL_H
 #define TDENGINE_EXECUTORIMPL_H
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 #include "os.h"
 #include "common.h"
+#include "tlosertree.h"
 #include "ttszip.h"
 #include "tvariant.h"
 
 #include "dataSinkMgt.h"
 #include "executil.h"
+#include "executor.h"
 #include "planner.h"
 #include "taosdef.h"
 #include "tarray.h"
 #include "tfilter.h"
 #include "thash.h"
 #include "tlockfree.h"
-#include "tpagedfile.h"
+#include "tpagedbuf.h"
-#include "executor.h"
 
 struct SColumnFilterElem;
 
@@ -92,13 +97,11 @@ typedef struct SSingleColumnFilterInfo {
 } SSingleColumnFilterInfo;
 
 typedef struct STableQueryInfo {
-  TSKEY       lastKey;
+  TSKEY          lastKey;     // last check ts
+  uint64_t       uid;         // table uid
   int32_t        groupIndex;  // group id in table list
-  SVariant    tag;
+//  SVariant       tag;
-  STimeWindow win;            // todo remove it later
+  SResultRowInfo resInfo;     // result info
-  STSCursor   cur;
-  void*       pTable;         // for retrieve the page id list
-  SResultRowInfo resInfo;
 } STableQueryInfo;
 
 typedef enum {
@@ -267,13 +270,14 @@ typedef struct STaskRuntimeEnv {
 
   int32_t              prevGroupId;  // previous executed group id
   bool                 enableGroupData;
-  SDiskbasedResultBuf*  pResultBuf;       // query result buffer based on blocked-wised disk file
+  SDiskbasedBuf* pResultBuf;           // query result buffer based on blocked-wised disk file
   SHashObj*            pResultRowHashTable;  // quick locate the window object for each result
   SHashObj*            pResultRowListSet;    // used to check if current ResultRowInfo has ResultRow object or not
   SArray*              pResultRowArrayList;  // The array list that contains the Result rows
   char*                keyBuf;               // window key buffer
-  SResultRowPool*       pool;             // The window result objects pool, all the resultRow Objects are allocated and managed by this object.
+  // The window result objects pool, all the resultRow Objects are allocated and managed by this object.
   char** prevRow;
+  SResultRowPool* pool;
 
   SArray*   prevResult;  // intermediate result, SArray<SInterResult>
   STSBuf*   pTsBuf;      // timestamp filter list
@@ -318,11 +322,6 @@ typedef struct SOperatorInfo {
   __optr_cleanup_fn_t    cleanupFn;
 } SOperatorInfo;
 
-enum {
-  QUERY_RESULT_NOT_READY = 1,
-  QUERY_RESULT_READY     = 2,
-};
-
 typedef struct {
   int32_t      numOfTags;
   int32_t      numOfCols;
@@ -370,15 +369,28 @@ typedef struct STaskParam {
   struct SUdfInfo* pUdfInfo;
 } STaskParam;
 
+enum {
+  DATA_NOT_READY = 0x1,
+  DATA_READY     = 0x2,
+  DATA_EXHAUSTED = 0x3,
+};
+
+typedef struct SSourceDataInfo {
+  struct SExchangeInfo *pEx;
+  int32_t            index;
+  SRetrieveTableRsp *pRsp;
+  uint64_t           totalRows;
+  int32_t            status;
+} SSourceDataInfo;
+
 typedef struct SExchangeInfo {
   SArray*            pSources;
+  SArray*            pSourceDataInfo;
   tsem_t             ready;
   void*              pTransporter;
-  SRetrieveTableRsp *pRsp;
   SSDataBlock*       pResult;
+  bool               seqLoadData;   // sequential load data or not, false by default
   int32_t            current;
-  uint64_t           rowsOfCurrentSource;
-
   uint64_t           totalSize;     // total load bytes from remote
   uint64_t           totalRows;     // total number of rows
   uint64_t           totalElapsed;  // total elapsed time
@@ -428,20 +440,24 @@ typedef struct SOptrBasicInfo {
   int32_t*        rowCellInfoOffset;  // offset value for each row result cell info
   SqlFunctionCtx* pCtx;
   SSDataBlock*    pRes;
+  uint32_t        resRowSize;
+  int32_t         capacity;
 } SOptrBasicInfo;
 
 typedef struct SOptrBasicInfo STableIntervalOperatorInfo;
 
 typedef struct SAggOperatorInfo {
   SOptrBasicInfo       binfo;
-  uint32_t              seed;
+  SDiskbasedBuf       *pResultBuf;           // query result buffer based on blocked-wised disk file
-  SDiskbasedResultBuf  *pResultBuf;       // query result buffer based on blocked-wised disk file
   SHashObj*            pResultRowHashTable;  // quick locate the window object for each result
   SHashObj*            pResultRowListSet;    // used to check if current ResultRowInfo has ResultRow object or not
   SArray*              pResultRowArrayList;  // The array list that contains the Result rows
   char*                keyBuf;               // window key buffer
   SResultRowPool      *pool;  // The window result objects pool, all the resultRow Objects are allocated and managed by this object.
   STableQueryInfo     *current;
+  uint32_t             groupId;
+  SGroupResInfo        groupResInfo;
+  STableQueryInfo     *pTableQueryInfo;
 } SAggOperatorInfo;
 
 typedef struct SProjectOperatorInfo {
@@ -555,43 +571,79 @@ typedef struct SMultiwayMergeInfo {
   bool    multiGroupResults;
 } SMultiwayMergeInfo;
 
-// todo support the disk-based sort
+typedef struct SMsortComparParam {
+  struct SExternalMemSource **pSources;
+  int32_t       numOfSources;
+  SArray       *orderInfo;   // SArray<SBlockOrderInfo>
+  bool          nullFirst;
+} SMsortComparParam;
+
 typedef struct SOrderOperatorInfo {
-  int32_t      colIndex;
+  int32_t                 sourceId;
-  int32_t      order;
+  uint32_t                sortBufSize;  // max buffer size for in-memory sort
   SSDataBlock            *pDataBlock;
+  bool                    hasVarCol;    // has variable length column, such as binary/varchar/nchar
+  int32_t                 numOfCompleted;
+  SDiskbasedBuf          *pSortInternalBuf;
+  SMultiwayMergeTreeInfo *pMergeTree;
+  SArray                 *pSources;     // SArray<SExternalMemSource*>
+  int32_t                 bufPageSize;
+  int32_t                 numOfRowsInRes;
+
+  SMsortComparParam       cmpParam;
+
+  int64_t                 startTs;       // sort start time
+  uint64_t                sortElapsed;   // sort elapsed time, time to flush to disk not included.
+  uint64_t                totalSize;     // total load bytes from remote
+  uint64_t                totalRows;     // total number of rows
+  uint64_t                totalElapsed;  // total elapsed time
 } SOrderOperatorInfo;
 
 SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* pSchema, SExecTaskInfo* pTaskInfo);
-
+SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput,
-SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo);
+                                           int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createTableSeqScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv);
+SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv);
-SOperatorInfo* createSubmitBlockScanOperatorInfo(void *pSubmitBlockReadHandle, int32_t numOfOutput, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
-
+SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
-SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr,
-SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
+                                         int32_t numOfOutput);
 SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream);
-SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
+SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr,
-SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
+                                              int32_t numOfOutput);
-SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
+SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream,
-SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult);
+                                                 SExprInfo* pExpr, int32_t numOfOutput);
-SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
+SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr,
-SOperatorInfo* createMultiTableAggOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
+                                         int32_t numOfOutput);
-SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
+SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr,
-SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
+                                      int32_t numOfOutput, bool multigroupResult);
+SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr,
+                                         int32_t numOfOutput);
+
+SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream,
+                                                        SExprInfo* pExpr, int32_t numOfOutput);
+SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream,
+                                                           SExprInfo* pExpr, int32_t numOfOutput);
 SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput);
-SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
+SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr,
+                                          int32_t numOfOutput);
 SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv);
 SOperatorInfo* createMultiwaySortOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput,
                                               int32_t numOfRows, void* merger);
-SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, bool groupResultMixedUp);
+SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream,
-SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
+                                                 SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo,
-SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, void* merger, bool multigroupResult);
+                                                 bool groupResultMixedUp);
+SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr,
+                                             int32_t numOfOutput);
+SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr,
+                                        int32_t numOfOutput, void* merger, bool multigroupResult);
 SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr,
                                         int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter);
 
-SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pdownstream, int32_t numOfDownstream, SSchema* pSchema, int32_t numOfOutput);
+SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pdownstream, int32_t numOfDownstream, SSchema* pSchema,
-SOperatorInfo* createOrderOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal);
+                                      int32_t numOfOutput);
+SOperatorInfo* createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SArray* pOrderVal);
+SOperatorInfo* createMergeSortOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput,
+                                           SOrder* pOrderVal);
 
 // SSDataBlock* doGlobalAggregate(void* param, bool* newgroup);
 // SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup);
@@ -604,18 +656,17 @@ void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p);
 
 SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows);
 
-void* destroyOutputBuf(SSDataBlock* pBlock);
 void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols);
 
 void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order);
-void finalizeQueryResult(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset);
+void finalizeQueryResult(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo,
+                         int32_t* rowCellInfoOffset);
 void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t* bufCapacity, int32_t numOfInputRows);
 void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t* bufCapacity);
 void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput);
 
-void freeParam(STaskParam *param);
+int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo, SSqlExpr** pExprMsg,
-int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo,
+                        SColumnInfo* pTagCols, int32_t queryType, void* pMsg, struct SUdfInfo* pUdfInfo);
-                        SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg, struct SUdfInfo* pUdfInfo);
 
 int32_t createIndirectQueryFuncExprFromMsg(SQueryTableReq* pQueryMsg, int32_t numOfOutput, SExprInfo** pExprInfo,
                                            SSqlExpr** pExpr, SExprInfo* prevExpr, struct SUdfInfo* pUdfInfo);
@@ -630,7 +681,7 @@ int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo*
 int32_t createFilterInfo(STaskAttr* pQueryAttr, uint64_t qId);
 void    freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters);
 
-STableQueryInfo *createTableQueryInfo(STaskAttr* pQueryAttr, void* pTable, bool groupbyColumn, STimeWindow win, void* buf);
+STableQueryInfo* createTableQueryInfo(void* buf, bool groupbyColumn, STimeWindow win);
 STableQueryInfo* createTmpTableQueryInfo(STimeWindow win);
 
 int32_t buildArithmeticExprFromMsg(SExprInfo* pArithExprInfo, void* pQueryMsg);
@@ -663,4 +714,8 @@ void doInvokeUdf(struct SUdfInfo* pUdfInfo, SqlFunctionCtx *pCtx, int32_t idx, i
 void    setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status);
 int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId);
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif  // TDENGINE_EXECUTORIMPL_H

source/libs/executor/src/executil.c

@@ -141,9 +141,9 @@ void clearResultRow(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16_
     return;
   }
 
-  // the result does not put into the SDiskbasedResultBuf, ignore it.
+  // the result does not put into the SDiskbasedBuf, ignore it.
   if (pResultRow->pageId >= 0) {
-    SFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResultRow->pageId);
+    SFilePage *page = getBufPage(pRuntimeEnv->pResultBuf, pResultRow->pageId);
 
     int16_t offset = 0;
     for (int32_t i = 0; i < pRuntimeEnv->pQueryAttr->numOfOutput; ++i) {
@@ -358,7 +358,6 @@ void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo)
 
   pGroupResInfo->pRows = taosArrayFromList(pResultInfo->pResult, pResultInfo->size, POINTER_BYTES);
   pGroupResInfo->index = 0;
-
   assert(pGroupResInfo->index <= getNumOfTotalRes(pGroupResInfo));
 }
 
@@ -533,7 +532,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv
   int32_t code = TSDB_CODE_SUCCESS;
 
   int32_t *posList = NULL;
-  SLoserTreeInfo *pTree = NULL;
+  SMultiwayMergeTreeInfo *pTree = NULL;
   STableQueryInfo **pTableQueryInfoList = NULL;
 
   size_t size = taosArrayGetSize(pTableList);
@@ -566,7 +565,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv
 
   SCompSupporter cs = {pTableQueryInfoList, posList, pRuntimeEnv->pQueryAttr->order.order};
 
-  int32_t ret = tLoserTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn);
+  int32_t ret = tMergeTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn);
   if (ret != TSDB_CODE_SUCCESS) {
     code = TSDB_CODE_QRY_OUT_OF_MEMORY;
     goto _end;
@@ -576,7 +575,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv
   int64_t startt = taosGetTimestampMs();
 
   while (1) {
-    int32_t tableIndex = pTree->pNode[0].index;
+    int32_t tableIndex = tMergeTreeGetChosenIndex(pTree);
 
     SResultRowInfo *pWindowResInfo = &pTableQueryInfoList[tableIndex]->resInfo;
     SResultRow  *pWindowRes = getResultRow(pWindowResInfo, cs.rowIndex[tableIndex]);
@@ -612,7 +611,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv
       }
     }
 
-    tLoserTreeAdjust(pTree, tableIndex + pTree->numOfEntries);
+    tMergeTreeAdjust(pTree, tMergeTreeGetAdjustIndex(pTree));
   }
 
   int64_t endt = taosGetTimestampMs();

source/libs/executor/test/executorTests.cpp

@@ -33,103 +33,95 @@
 #include "stub.h"
 #include "executor.h"
 
-/**
+namespace {
-{
-	"Id":	{
-		"QueryId":	1.3108161807422521e+19,
-		"TemplateId":	0,
-		"SubplanId":	0
-	},
-	"Node":	{
-		"Name":	"TableScan",
-		"Targets":	[{
-				"Base":	{
-					"Schema":	{
-						"Type":	9,
-						"ColId":	5000,
-						"Bytes":	8
-					},
-					"Columns":	[{
-							"TableId":	1,
-							"Flag":	0,
-							"Info":	{
-								"ColId":	1,
-								"Type":	9,
-								"Bytes":	8
-							}
-						}],
-					"InterBytes":	0
-				},
-				"Expr":	{
-					"Type":	4,
-					"Column":	{
-						"Type":	9,
-						"ColId":	1,
-						"Bytes":	8
-					}
-				}
-			}, {
-				"Base":	{
-					"Schema":	{
-						"Type":	4,
-						"ColId":	5001,
-						"Bytes":	4
-					},
-					"Columns":	[{
-							"TableId":	1,
-							"Flag":	0,
-							"Info":	{
-								"ColId":	2,
-								"Type":	4,
-								"Bytes":	4
-							}
-						}],
-					"InterBytes":	0
-				},
-				"Expr":	{
-					"Type":	4,
-					"Column":	{
-						"Type":	4,
-						"ColId":	2,
-						"Bytes":	4
-					}
-				}
-			}],
-		"InputSchema":	[{
-				"Type":	9,
-				"ColId":	5000,
-				"Bytes":	8
-			}, {
-				"Type":	4,
-				"ColId":	5001,
-				"Bytes":	4
-			}],
-		"TableScan":	{
-			"TableId":	1,
-			"TableType":	2,
-			"Flag":	0,
-			"Window":	{
-				"StartKey":	-9.2233720368547758e+18,
-				"EndKey":	9.2233720368547758e+18
-			}
-		}
-	},
-	"DataSink":	{
-		"Name":	"Dispatch",
-		"Dispatch":	{
-		}
-	}
-}
- */
 
+typedef struct SDummyInputInfo {
+  int32_t max;
+  int32_t current;
+  int32_t startVal;
+  SSDataBlock* pBlock;
+} SDummyInputInfo;
+
+SSDataBlock* getDummyBlock(void* param, bool* newgroup) {
+  SOperatorInfo* pOperator = static_cast<SOperatorInfo*>(param);
+  SDummyInputInfo* pInfo = static_cast<SDummyInputInfo*>(pOperator->info);
+  if (pInfo->current >= pInfo->max) {
+    return NULL;
+  }
+
+  int32_t numOfRows = 1000;
+
+  if (pInfo->pBlock == NULL) {
+    pInfo->pBlock = static_cast<SSDataBlock*>(calloc(1, sizeof(SSDataBlock)));
+
+    pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
+
+    SColumnInfoData colInfo = {0};
+    colInfo.info.type = TSDB_DATA_TYPE_INT;
+    colInfo.info.bytes = sizeof(int32_t);
+    colInfo.info.colId = 1;
+    colInfo.pData = static_cast<char*>(calloc(numOfRows, sizeof(int32_t)));
+    colInfo.nullbitmap = static_cast<char*>(calloc(1, (numOfRows + 7) / 8));
+
+    taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo);
+
+//    SColumnInfoData colInfo1 = {0};
+//    colInfo1.info.type = TSDB_DATA_TYPE_BINARY;
+//    colInfo1.info.bytes = 40;
+//    colInfo1.info.colId = 2;
+//
+//    colInfo1.varmeta.allocLen = 0;//numOfRows * sizeof(int32_t);
+//    colInfo1.varmeta.length = 0;
+//    colInfo1.varmeta.offset = static_cast<int32_t*>(calloc(1, numOfRows * sizeof(int32_t)));
+//
+//    taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo1);
+  } else {
+    blockDataClearup(pInfo->pBlock, true);
+  }
+
+  SSDataBlock* pBlock = pInfo->pBlock;
+
+  char buf[128] = {0};
+  char b1[128] = {0};
+  for(int32_t i = 0; i < numOfRows; ++i) {
+    SColumnInfoData* pColInfo = static_cast<SColumnInfoData*>(TARRAY_GET_ELEM(pBlock->pDataBlock, 0));
+
+    int32_t v = (--pInfo->startVal);
+    colDataAppend(pColInfo, i, reinterpret_cast<const char*>(&v), false);
+
+//    sprintf(buf, "this is %d row", i);
+//    STR_TO_VARSTR(b1, buf);
+//
+//    SColumnInfoData* pColInfo2 = static_cast<SColumnInfoData*>(TARRAY_GET_ELEM(pBlock->pDataBlock, 1));
+//    colDataAppend(pColInfo2, i, b1, false);
+  }
+
+  pBlock->info.rows = numOfRows;
+  pBlock->info.numOfCols = 1;
+
+  pInfo->current += 1;
+  return pBlock;
+}
+
+SOperatorInfo* createDummyOperator(int32_t numOfBlocks) {
+  SOperatorInfo* pOperator = static_cast<SOperatorInfo*>(calloc(1, sizeof(SOperatorInfo)));
+  pOperator->name = "dummyInputOpertor4Test";
+  pOperator->exec = getDummyBlock;
+
+  SDummyInputInfo *pInfo = (SDummyInputInfo*) calloc(1, sizeof(SDummyInputInfo));
+  pInfo->max = numOfBlocks;
+  pInfo->startVal = 1500000;
+
+  pOperator->info = pInfo;
+  return pOperator;
+}
+}
 int main(int argc, char** argv) {
   testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }
 
 TEST(testCase, build_executor_tree_Test) {
-
-
   const char* msg = "{\n"
                     "\t\"Id\":\t{\n"
                     "\t\t\"QueryId\":\t1.3108161807422521e+19,\n"
@@ -219,7 +211,134 @@ TEST(testCase, build_executor_tree_Test) {
 
   SExecTaskInfo* pTaskInfo = nullptr;
   DataSinkHandle sinkHandle = nullptr;
-  int32_t code = qCreateExecTask((SReadHandle*) 1, 2, 1, NULL, (void**) &pTaskInfo, &sinkHandle);
+  SReadHandle handle = {.reader = reinterpret_cast<void*>(0x1), .meta = reinterpret_cast<void*>(0x1)};
+
+//  int32_t code = qCreateExecTask(&handle, 2, 1, NULL, (void**) &pTaskInfo, &sinkHandle);
 }
 
+//TEST(testCase, inMem_sort_Test) {
+//  SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder));
+//  SOrder o = {.order = TSDB_ORDER_ASC};
+//  o.col.info.colId = 1;
+//  o.col.info.type = TSDB_DATA_TYPE_INT;
+//  taosArrayPush(pOrderVal, &o);
+//
+//  SArray* pExprInfo = taosArrayInit(4, sizeof(SExprInfo));
+//  SExprInfo *exp = static_cast<SExprInfo*>(calloc(1, sizeof(SExprInfo)));
+//  exp->base.resSchema = createSchema(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1, "res");
+//  taosArrayPush(pExprInfo, &exp);
+//
+//  SExprInfo *exp1 = static_cast<SExprInfo*>(calloc(1, sizeof(SExprInfo)));
+//  exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1");
+//  taosArrayPush(pExprInfo, &exp1);
+//
+//  SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(5), pExprInfo, pOrderVal);
+//
+//  bool newgroup = false;
+//  SSDataBlock* pRes = pOperator->exec(pOperator, &newgroup);
+//
+//  SColumnInfoData* pCol1 = static_cast<SColumnInfoData*>(taosArrayGet(pRes->pDataBlock, 0));
+//  SColumnInfoData* pCol2 = static_cast<SColumnInfoData*>(taosArrayGet(pRes->pDataBlock, 1));
+//  for(int32_t i = 0; i < pRes->info.rows; ++i) {
+//    char* p = colDataGet(pCol2, i);
+//    printf("%d: %d, %s\n", i, ((int32_t*)pCol1->pData)[i], (char*)varDataVal(p));
+//  }
+//}
+
+typedef struct su {
+  int32_t v;
+  char   *c;
+} su;
+
+int32_t cmp(const void* p1, const void* p2) {
+  su* v1 = (su*) p1;
+  su* v2 = (su*) p2;
+
+  int32_t x1 = *(int32_t*) v1->c;
+  int32_t x2 = *(int32_t*) v2->c;
+  if (x1 == x2) {
+    return 0;
+  } else {
+    return x1 < x2? -1:1;
+  }
+}
+
+TEST(testCase, external_sort_Test) {
+#if 0
+  su* v = static_cast<su*>(calloc(1000000, sizeof(su)));
+  for(int32_t i = 0; i < 1000000; ++i) {
+    v[i].v = rand();
+    v[i].c = static_cast<char*>(malloc(4));
+    *(int32_t*) v[i].c = i;
+  }
+
+  qsort(v, 1000000, sizeof(su), cmp);
+//  for(int32_t i = 0; i < 1000; ++i) {
+//    printf("%d ", v[i]);
+//  }
+//  printf("\n");
+  return;
+#endif
+
+  srand(time(NULL));
+
+  SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder));
+  SOrder o = {0};
+  o.order = TSDB_ORDER_ASC;
+  o.col.info.colId = 1;
+  o.col.info.type = TSDB_DATA_TYPE_INT;
+  taosArrayPush(pOrderVal, &o);
+
+  SArray* pExprInfo = taosArrayInit(4, sizeof(SExprInfo));
+  SExprInfo *exp = static_cast<SExprInfo*>(calloc(1, sizeof(SExprInfo)));
+  exp->base.resSchema = createSchema(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1, "res");
+  taosArrayPush(pExprInfo, &exp);
+
+  SExprInfo *exp1 = static_cast<SExprInfo*>(calloc(1, sizeof(SExprInfo)));
+  exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1");
+//  taosArrayPush(pExprInfo, &exp1);
+
+  SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(1500), pExprInfo, pOrderVal);
+
+  bool newgroup = false;
+  SSDataBlock* pRes = NULL;
+
+  int32_t total = 1;
+
+  int64_t s1 = taosGetTimestampUs();
+  int32_t t = 1;
+
+  while(1) {
+    int64_t s = taosGetTimestampUs();
+    pRes = pOperator->exec(pOperator, &newgroup);
+
+    int64_t e = taosGetTimestampUs();
+    if (t++ == 1) {
+      printf("---------------elapsed:%ld\n", e - s);
+    }
+
+    if (pRes == NULL) {
+      break;
+    }
+
+    SColumnInfoData* pCol1 = static_cast<SColumnInfoData*>(taosArrayGet(pRes->pDataBlock, 0));
+//    SColumnInfoData* pCol2 = static_cast<SColumnInfoData*>(taosArrayGet(pRes->pDataBlock, 1));
+    for (int32_t i = 0; i < pRes->info.rows; ++i) {
+//      char* p = colDataGet(pCol2, i);
+      printf("%d: %d\n", total++, ((int32_t*)pCol1->pData)[i]);
+//      printf("%d: %d, %s\n", total++, ((int32_t*)pCol1->pData)[i], (char*)varDataVal(p));
+    }
+  }
+
+  printStatisBeforeClose(((SOrderOperatorInfo*) pOperator->info)->pSortInternalBuf);
+
+  int64_t s2 = taosGetTimestampUs();
+  printf("total:%ld\n", s2 - s1);
+
+  pOperator->cleanupFn(pOperator->info, 2);
+  tfree(exp);
+  tfree(exp1);
+  taosArrayDestroy(pExprInfo);
+  taosArrayDestroy(pOrderVal);
+}
 #pragma GCC diagnostic pop

source/libs/function/inc/thistogram.h

@@ -49,7 +49,7 @@ typedef struct SHistogramInfo {
   SHistBin* elems;
 #else
   tSkipList*      pList;
-  SLoserTreeInfo* pLoserTree;
+  SMultiwayMergeTreeInfo* pLoserTree;
   int32_t         maxIndex;
   bool            ordered;
 #endif

source/libs/function/inc/tpercentile.h

@@ -20,7 +20,7 @@
 extern "C" {
 #endif
 
-#include "tpagedfile.h"
+#include "tpagedbuf.h"
 #include "ttszip.h"
 
 typedef struct MinMaxEntry {
@@ -63,7 +63,7 @@ typedef struct tMemBucket {
   __compar_fn_t comparFn;
 
   tMemBucketSlot *     pSlots;
-  SDiskbasedResultBuf *pBuffer;
+  SDiskbasedBuf *pBuffer;
   __perc_hash_func_t   hashFunc;
 } tMemBucket;
 

source/libs/function/src/thistogram.c

@@ -117,14 +117,14 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
 
       if ((*pHisto)->ordered) {
         int32_t         lastIndex = (*pHisto)->maxIndex;
-        SLoserTreeInfo* pTree = (*pHisto)->pLoserTree;
+        SMultiwayMergeTreeInfo* pTree = (*pHisto)->pLoserTree;
 
         (*pHisto)->pLoserTree->pNode[lastIndex + pTree->numOfEntries].pData = pResNode;
         pEntry1->index = (*pHisto)->pLoserTree->pNode[lastIndex + pTree->numOfEntries].index;
 
         // update the loser tree
         if ((*pHisto)->ordered) {
-          tLoserTreeAdjust(pTree, pEntry1->index + pTree->numOfEntries);
+          tMergeTreeAdjust(pTree, pEntry1->index + pTree->numOfEntries);
         }
 
         tSkipListKey kx =
@@ -142,10 +142,10 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
       SHistBin* pPrevEntry = (SHistBin*)pResNode->pBackward[0]->pData;
       pPrevEntry->delta = val - pPrevEntry->val;
 
-      SLoserTreeInfo* pTree = (*pHisto)->pLoserTree;
+      SMultiwayMergeTreeInfo* pTree = (*pHisto)->pLoserTree;
       if ((*pHisto)->ordered) {
-        tLoserTreeAdjust(pTree, pPrevEntry->index + pTree->numOfEntries);
+        tMergeTreeAdjust(pTree, pPrevEntry->index + pTree->numOfEntries);
-        tLoserTreeDisplay(pTree);
+        tMergeTreePrint(pTree);
       }
     }
 
@@ -155,7 +155,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
       if (!(*pHisto)->ordered) {
         SSkipListPrint((*pHisto)->pList, 1);
 
-        SLoserTreeInfo* pTree = (*pHisto)->pLoserTree;
+        SMultiwayMergeTreeInfo* pTree = (*pHisto)->pLoserTree;
         tSkipListNode*  pHead = (*pHisto)->pList->pHead.pForward[0];
 
         tSkipListNode* p1 = pHead;
@@ -183,13 +183,13 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
           pTree->pNode[i].index = -1;
         }
 
-        tLoserTreeDisplay(pTree);
+        tMergeTreePrint(pTree);
 
         for (int32_t i = pTree->totalEntries - 1; i >= pTree->numOfEntries; i--) {
-          tLoserTreeAdjust(pTree, i);
+          tMergeTreeAdjust(pTree, i);
         }
 
-        tLoserTreeDisplay(pTree);
+        tMergeTreePrint(pTree);
         (*pHisto)->ordered = true;
       }
 
@@ -219,7 +219,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
         pPrevEntry->delta = pEntry->val - pPrevEntry->val;
       }
 
-      SLoserTreeInfo* pTree = (*pHisto)->pLoserTree;
+      SMultiwayMergeTreeInfo* pTree = (*pHisto)->pLoserTree;
       if (pNextEntry->index != -1) {
         (*pHisto)->maxIndex = pNextEntry->index;
 
@@ -230,12 +230,12 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
         printf("disappear index is:%d\n", f);
       }
 
-      tLoserTreeAdjust(pTree, pEntry->index + pTree->numOfEntries);
+      tMergeTreeAdjust(pTree, pEntry->index + pTree->numOfEntries);
       // remove the next node in skiplist
       tSkipListRemoveNode((*pHisto)->pList, pNext);
       SSkipListPrint((*pHisto)->pList, 1);
 
-      tLoserTreeDisplay((*pHisto)->pLoserTree);
+      tMergeTreePrint((*pHisto)->pLoserTree);
     } else {  // add to heap
       if (pResNode->pForward[0] != NULL) {
         pEntry1->delta = ((SHistBin*)pResNode->pForward[0]->pData)->val - val;

source/libs/function/src/tpercentile.c

@@ -15,10 +15,10 @@
 #include <tglobal.h>
 #include "os.h"
 
-#include "tpercentile.h"
-#include "tpagedfile.h"
 #include "taosdef.h"
 #include "tcompare.h"
+#include "tpagedbuf.h"
+#include "tpercentile.h"
 #include "ttypes.h"
 
 #define DEFAULT_NUM_OF_SLOT 1024
@@ -35,9 +35,9 @@ static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx)
 
   int32_t offset = 0;
   for(int32_t i = 0; i < list->size; ++i) {
-    SPageInfo* pgInfo = *(SPageInfo**) taosArrayGet(list, i);
+    struct SPageInfo* pgInfo = *(struct SPageInfo**) taosArrayGet(list, i);
 
-    SFilePage* pg = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId);
+    SFilePage* pg = getBufPage(pMemBucket->pBuffer, getPageId(pgInfo));
     memcpy(buffer->data + offset, pg->data, (size_t)(pg->num * pMemBucket->bytes));
 
     offset += (int32_t)(pg->num * pMemBucket->bytes);
@@ -98,8 +98,8 @@ double findOnlyResult(tMemBucket *pMemBucket) {
     SIDList list = getDataBufPagesIdList(pMemBucket->pBuffer, groupId);
     assert(list->size == 1);
 
-    SPageInfo* pgInfo = (SPageInfo*) taosArrayGetP(list, 0);
+    struct SPageInfo* pgInfo = (struct SPageInfo*) taosArrayGetP(list, 0);
-    SFilePage* pPage = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId);
+    SFilePage* pPage = getBufPage(pMemBucket->pBuffer, getPageId(pgInfo));
     assert(pPage->num == 1);
 
     double v = 0;
@@ -254,7 +254,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval,
 
   resetSlotInfo(pBucket);
 
-  int32_t ret = createDiskbasedResultBuffer(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, 1, tsTempDir);
+  int32_t ret = createDiskbasedBuffer(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, 1, tsTempDir);
   if (ret != 0) {
     tMemBucketDestroy(pBucket);
     return NULL;
@@ -343,7 +343,7 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) {
         assert(pSlot->info.data->num >= pBucket->elemPerPage && pSlot->info.size > 0);
 
         // keep the pointer in memory
-        releaseResBufPage(pBucket->pBuffer, pSlot->info.data);
+        releaseBufPage(pBucket->pBuffer, pSlot->info.data);
         pSlot->info.data = NULL;
       }
 
@@ -471,10 +471,10 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction)
 
        for (int32_t f = 0; f < list->size; ++f) {
          SPageInfo *pgInfo = *(SPageInfo **)taosArrayGet(list, f);
-         SFilePage *pg = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId);
+         SFilePage *pg = getBufPage(pMemBucket->pBuffer, getPageId(pgInfo));
 
          tMemBucketPut(pMemBucket, pg->data, (int32_t)pg->num);
-         releaseResBufPageInfo(pMemBucket->pBuffer, pgInfo);
+         releaseBufPageInfo(pMemBucket->pBuffer, pgInfo);
        }
 
        return getPercentileImpl(pMemBucket, count - num, fraction);

source/libs/parser/inc/new_sql.y

@@ -55,6 +55,7 @@
 
 %left OR.
 %left AND.
+//%right NOT.
 %left UNION ALL MINUS EXCEPT INTERSECT.
 //%left BITAND BITOR LSHIFT RSHIFT.
 %left NK_PLUS NK_MINUS.
@@ -169,13 +170,41 @@ column_reference(A) ::= table_name(B) NK_DOT column_name(C).
 //pseudo_column(A) ::= NK_NOW.                                                      { PARSER_TRACE; A = createFunctionNode(pCxt, NULL, NULL); }
 
 /************************************************ predicate ***********************************************************/
-predicate(A) ::= expression(B) compare_op(C) expression(D).                       { PARSER_TRACE; A = createOperatorNode(pCxt, C, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D)); }
+predicate(A) ::= expression(B) compare_op(C) expression(D).                       {
+                                                                                    PARSER_TRACE;
+                                                                                    SToken s = getTokenFromRawExprNode(pCxt, B);
+                                                                                    SToken e = getTokenFromRawExprNode(pCxt, D);
+                                                                                    A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, C, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D)));
+                                                                                  }
 //predicate(A) ::= expression(B) compare_op sub_type expression(B).
-predicate(A) ::= expression(B) BETWEEN expression(C) AND expression(D).           { PARSER_TRACE; A = createBetweenAnd(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, D)); }
+predicate(A) ::= expression(B) BETWEEN expression(C) AND expression(D).           {
-predicate(A) ::= expression(B) NOT BETWEEN expression(C) AND expression(D).       { PARSER_TRACE; A = createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D)); }
+                                                                                    PARSER_TRACE;
-predicate(A) ::= expression(B) IS NULL.                                           { PARSER_TRACE; A = createIsNullCondNode(pCxt, releaseRawExprNode(pCxt, B), true); }
+                                                                                    SToken s = getTokenFromRawExprNode(pCxt, B);
-predicate(A) ::= expression(B) IS NOT NULL.                                       { PARSER_TRACE; A = createIsNullCondNode(pCxt, releaseRawExprNode(pCxt, B), false); }
+                                                                                    SToken e = getTokenFromRawExprNode(pCxt, D);
-predicate(A) ::= expression(B) in_op(C) in_predicate_value(D).                    { PARSER_TRACE; A = createOperatorNode(pCxt, C, releaseRawExprNode(pCxt, B), D); }
+                                                                                    A = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, D)));
+                                                                                  }
+predicate(A) ::= expression(B) NOT BETWEEN expression(C) AND expression(D).       {
+                                                                                    PARSER_TRACE;
+                                                                                    SToken s = getTokenFromRawExprNode(pCxt, B);
+                                                                                    SToken e = getTokenFromRawExprNode(pCxt, D);
+                                                                                    A = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D)));
+                                                                                  }
+predicate(A) ::= expression(B) IS NULL(C).                                        {
+                                                                                    PARSER_TRACE;
+                                                                                    SToken s = getTokenFromRawExprNode(pCxt, B);
+                                                                                    A = createRawExprNodeExt(pCxt, &s, &C, createIsNullCondNode(pCxt, releaseRawExprNode(pCxt, B), true));
+                                                                                  }
+predicate(A) ::= expression(B) IS NOT NULL(C).                                    {
+                                                                                    PARSER_TRACE;
+                                                                                    SToken s = getTokenFromRawExprNode(pCxt, B);
+                                                                                    A = createRawExprNodeExt(pCxt, &s, &C, createIsNullCondNode(pCxt, releaseRawExprNode(pCxt, B), false));
+                                                                                  }
+predicate(A) ::= expression(B) in_op(C) in_predicate_value(D).                    {
+                                                                                    PARSER_TRACE;
+                                                                                    SToken s = getTokenFromRawExprNode(pCxt, B);
+                                                                                    SToken e = getTokenFromRawExprNode(pCxt, D);
+                                                                                    A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, C, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D)));
+                                                                                  }
 
 %type compare_op                                                                  { EOperatorType }
 %destructor compare_op                                                            { PARSER_DESTRUCTOR_TRACE; }
@@ -195,18 +224,36 @@ compare_op(A) ::= NMATCH.
 in_op(A) ::= IN.                                                                  { PARSER_TRACE; A = OP_TYPE_IN; }
 in_op(A) ::= NOT IN.                                                              { PARSER_TRACE; A = OP_TYPE_NOT_IN; }
 
-in_predicate_value(A) ::= NK_LP expression_list(B) NK_RP.                         { PARSER_TRACE; A = createNodeListNode(pCxt, B); }
+in_predicate_value(A) ::= NK_LP(C) expression_list(B) NK_RP(D).                   { PARSER_TRACE; A = createRawExprNodeExt(pCxt, &C, &D, createNodeListNode(pCxt, B)); }
 
 /************************************************ boolean_value_expression ********************************************/
 boolean_value_expression(A) ::= boolean_primary(B).                               { PARSER_TRACE; A = B; }
-boolean_value_expression(A) ::= NOT boolean_primary(B).                           { PARSER_TRACE; A = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, B, NULL); }
+boolean_value_expression(A) ::= NOT(C) boolean_primary(B).                        {
+                                                                                    PARSER_TRACE;
+                                                                                    SToken e = getTokenFromRawExprNode(pCxt, B);
+                                                                                    A = createRawExprNodeExt(pCxt, &C, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, B), NULL));
+                                                                                  }
 boolean_value_expression(A) ::=
-  boolean_value_expression(B) OR boolean_value_expression(C).                     { PARSER_TRACE; A = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, B, C); }
+  boolean_value_expression(B) OR boolean_value_expression(C).                     {
+                                                                                    PARSER_TRACE;
+                                                                                    SToken s = getTokenFromRawExprNode(pCxt, B);
+                                                                                    SToken e = getTokenFromRawExprNode(pCxt, C);
+                                                                                    A = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C)));
+                                                                                  }
 boolean_value_expression(A) ::=
-  boolean_value_expression(B) AND boolean_value_expression(C).                    { PARSER_TRACE; A = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, B, C); }
+  boolean_value_expression(B) AND boolean_value_expression(C).                    {
+                                                                                    PARSER_TRACE;
+                                                                                    SToken s = getTokenFromRawExprNode(pCxt, B);
+                                                                                    SToken e = getTokenFromRawExprNode(pCxt, C);
+                                                                                    A = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C)));
+                                                                                  }
 
 boolean_primary(A) ::= predicate(B).                                              { PARSER_TRACE; A = B; }
-boolean_primary(A) ::= NK_LP boolean_value_expression(B) NK_RP.                   { PARSER_TRACE; A = B; }
+boolean_primary(A) ::= NK_LP(C) boolean_value_expression(B) NK_RP(D).             { PARSER_TRACE; A = createRawExprNodeExt(pCxt, &C, &D, releaseRawExprNode(pCxt, B)); }
+
+/************************************************ common_expression ********************************************/
+common_expression(A) ::= expression(B).                                           { A = B; }
+common_expression(A) ::= boolean_value_expression(B).                             { A = B; }
 
 /************************************************ from_clause *********************************************************/
 from_clause(A) ::= FROM table_reference_list(B).                                  { PARSER_TRACE; A = B; }
@@ -271,13 +318,13 @@ select_list(A) ::= select_sublist(B).
 select_sublist(A) ::= select_item(B).                                             { PARSER_TRACE; A = createNodeList(pCxt, B); }
 select_sublist(A) ::= select_sublist(B) NK_COMMA select_item(C).                  { PARSER_TRACE; A = addNodeToList(pCxt, B, C); }
 
-select_item(A) ::= expression(B).                                                 {
+select_item(A) ::= common_expression(B).                                          {
                                                                                     PARSER_TRACE;
                                                                                     SToken t = getTokenFromRawExprNode(pCxt, B);
                                                                                     A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &t);
                                                                                   }
-select_item(A) ::= expression(B) column_alias(C).                                 { PARSER_TRACE; A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); }
+select_item(A) ::= common_expression(B) column_alias(C).                          { PARSER_TRACE; A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); }
-select_item(A) ::= expression(B) AS column_alias(C).                              { PARSER_TRACE; A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); }
+select_item(A) ::= common_expression(B) AS column_alias(C).                       { PARSER_TRACE; A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); }
 select_item(A) ::= table_name(B) NK_DOT NK_STAR(C).                               { PARSER_TRACE; A = createColumnNode(pCxt, &B, &C); }
 
 where_clause_opt(A) ::= .                                                         { PARSER_TRACE; A = NULL; }
@@ -364,7 +411,7 @@ limit_clause_opt(A) ::= LIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B).
 subquery(A) ::= NK_LP(B) query_expression(C) NK_RP(D).                                  { PARSER_TRACE; A = createRawExprNodeExt(pCxt, &B, &D, C); }
 
 /************************************************ search_condition ****************************************************/
-search_condition(A) ::= boolean_value_expression(B).                              { PARSER_TRACE; A = B; }
+search_condition(A) ::= common_expression(B).                              { PARSER_TRACE; A = releaseRawExprNode(pCxt, B); }
 
 /************************************************ sort_specification_list *********************************************/
 %type sort_specification_list                                                     { SNodeList* }

source/libs/parser/src/astToMsg.c

@@ -404,15 +404,14 @@ char* buildCreateStbReq(SCreateTableSql* pCreateTableSql, int32_t* outputLen, SP
     return NULL;
   }
 
-  int32_t tlen = tSerializeSMCreateStbReq(NULL, &createReq);
+  int32_t tlen = tSerializeSMCreateStbReq(NULL, 0, &createReq);
   void*   pReq = malloc(tlen);
   if (pReq == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
 
-  void* pBuf = pReq;
+  tSerializeSMCreateStbReq(pReq, tlen, &createReq);
-  tSerializeSMCreateStbReq(&pBuf, &createReq);
   *outputLen = tlen;
   return pReq;
 }
@@ -433,15 +432,14 @@ char* buildDropStableReq(SSqlInfo* pInfo, int32_t* outputLen, SParseContext* pPa
   assert(code == TSDB_CODE_SUCCESS && name.type == TSDB_TABLE_NAME_T);
   dropReq.igNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0;
 
-  int32_t tlen = tSerializeSMDropStbReq(NULL, &dropReq);
+  int32_t tlen = tSerializeSMDropStbReq(NULL, 0, &dropReq);
   void*   pReq = malloc(tlen);
   if (pReq == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
 
-  void* pBuf = pReq;
+  tSerializeSMDropStbReq(pReq, tlen, &dropReq);
-  tSerializeSMDropStbReq(&pBuf, &dropReq);
   *outputLen = tlen;
   return pReq;
 }

source/libs/parser/src/parserImpl.c

@@ -628,6 +628,12 @@ static EDealRes translateExprSubquery(STranslateContext* pCxt, SNode* pNode) {
   return (TSDB_CODE_SUCCESS == translateSubquery(pCxt, pNode) ? DEAL_RES_CONTINUE : DEAL_RES_ERROR);
 }
 
+static EDealRes translateLogicCond(STranslateContext* pCxt, SLogicConditionNode* pCond) {
+  pCond->node.resType.type = TSDB_DATA_TYPE_BOOL;
+  pCond->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
+  return DEAL_RES_CONTINUE;
+}
+
 static EDealRes doTranslateExpr(SNode* pNode, void* pContext) {
   STranslateContext* pCxt = (STranslateContext*)pContext;
   switch (nodeType(pNode)) {
@@ -639,6 +645,8 @@ static EDealRes doTranslateExpr(SNode* pNode, void* pContext) {
       return translateOperator(pCxt, (SOperatorNode*)pNode);
     case QUERY_NODE_FUNCTION:
       return translateFunction(pCxt, (SFunctionNode*)pNode);
+    case QUERY_NODE_LOGIC_CONDITION:
+      return translateLogicCond(pCxt, (SLogicConditionNode*)pNode);
     case QUERY_NODE_TEMP_TABLE:
       return translateExprSubquery(pCxt, ((STempTableNode*)pNode)->pSubquery);
     default:

source/libs/parser/test/newParserTest.cpp

@@ -76,7 +76,8 @@ private:
     case QUERY_NODE_COLUMN:
     case QUERY_NODE_VALUE:
     case QUERY_NODE_OPERATOR:
-    case QUERY_NODE_FUNCTION: {
+    case QUERY_NODE_FUNCTION:
+    case QUERY_NODE_LOGIC_CONDITION: {
       SExprNode* pExpr = (SExprNode*)node;
       str.append(" [" + dataTypeToStr(pExpr->resType) + "]");
       if (isProject) {
@@ -215,6 +216,34 @@ private:
     str.append((NULL_ORDER_FIRST == pOrderBy->nullOrder ? " NULLS FIRST" : " NULLS LAST"));
   }
 
+  string logicConditionTypeToStr(ELogicConditionType type) {
+    switch (type) {
+      case LOGIC_COND_TYPE_AND:
+        return "AND";    
+      case LOGIC_COND_TYPE_OR:
+        return "OR";
+      case LOGIC_COND_TYPE_NOT:
+        return "NOT";
+      default:
+        break;
+    }
+    return "Unknown logic cond type";
+  }
+
+  void logicCondToStr(SLogicConditionNode* pCond, string& str) {
+    SNode* node = nullptr;
+    bool first = true;
+    FOREACH(node, pCond->pParameterList) {
+      if (first && LOGIC_COND_TYPE_NOT == pCond->condType) {
+        str.append(logicConditionTypeToStr(pCond->condType) + " ");
+      } else if (!first && LOGIC_COND_TYPE_NOT != pCond->condType) {
+        str.append(" " + logicConditionTypeToStr(pCond->condType) + " ");
+      }
+      first = false;
+      nodeToStr(node, str, false);
+    }
+  }
+
   void nodeToStr(const SNode* node, string& str, bool isProject) {
     if (nullptr == node) {
       return;
@@ -237,6 +266,10 @@ private:
         functionToStr((SFunctionNode*)node, str);
         break;
       }
+      case QUERY_NODE_LOGIC_CONDITION: {
+        logicCondToStr((SLogicConditionNode*)node, str);
+        break;
+      }
       case QUERY_NODE_GROUPING_SET: {
         groupingSetToStr((SGroupingSetNode*)node, str);
         break;
@@ -511,6 +544,12 @@ TEST_F(NewParserTest, selectExpression) {
 
   bind("SELECT ts + 10s, c1 + 10, concat(c2, 'abc') FROM t1");
   ASSERT_TRUE(run());
+
+  bind("SELECT ts > 0, c1 < 20 AND c2 = 'qaz' FROM t1");
+  ASSERT_TRUE(run());
+
+  bind("SELECT ts > 0, c1 BETWEEN 10 AND 20 AND c2 = 'qaz' FROM t1");
+  ASSERT_TRUE(run());
 }
 
 TEST_F(NewParserTest, selectClause) {

source/libs/planner/src/physicalPlan.c

@@ -216,7 +216,7 @@ static SPhyNode* createMultiTableScanNode(SQueryPlanNode* pPlanNode, SQueryTable
   } else if (needSeqScan(pPlanNode)) {
     return createUserTableScanNode(pPlanNode, pTable, OP_TableSeqScan);
   }
-  int32_t type = (pPlanNode->info.type == QNODE_TABLESCAN)? OP_DataBlocksOptScan:OP_StreamScan;
+  int32_t type = (pPlanNode->info.type == QNODE_TABLESCAN)? OP_TableScan:OP_StreamScan;
   return createUserTableScanNode(pPlanNode, pTable, type);
 }
 
@@ -288,7 +288,7 @@ static bool needMultiNodeScan(SQueryTableInfo* pTable) {
 static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTableInfo, SSubplan* subplan) {
   SVgroupsInfo* pVgroupsInfo = pTableInfo->pMeta->vgroupList;
   vgroupInfoToNodeAddr(&(pVgroupsInfo->vgroups[0]), &subplan->execNode);
-  int32_t type = (pPlanNode->info.type == QNODE_TABLESCAN)? OP_DataBlocksOptScan:OP_StreamScan;
+  int32_t type = (pPlanNode->info.type == QNODE_TABLESCAN)? OP_TableScan:OP_StreamScan;
   return createUserTableScanNode(pPlanNode, pTableInfo, type);
 }
 

source/libs/planner/src/physicalPlanJson.c

@@ -88,7 +88,7 @@ static const char* jkPnodeType = "Type";
 static int32_t getPnodeTypeSize(cJSON* json) {
   switch (getNumber(json, jkPnodeType)) {
     case OP_StreamScan:
-    case OP_DataBlocksOptScan:
+    case OP_TableScan:
     case OP_TableSeqScan:
       return sizeof(STableScanPhyNode);
     case OP_TagScan:
@@ -830,7 +830,7 @@ static bool specificPhyNodeToJson(const void* obj, cJSON* json) {
   const SPhyNode* phyNode = (const SPhyNode*)obj;
   switch (phyNode->info.type) {
     case OP_StreamScan:
-    case OP_DataBlocksOptScan:
+    case OP_TableScan:
     case OP_TableSeqScan:
       return tableScanNodeToJson(obj, json);
     case OP_TagScan:
@@ -868,7 +868,7 @@ static bool specificPhyNodeFromJson(const cJSON* json, void* obj) {
   SPhyNode* phyNode = (SPhyNode*)obj;
   switch (phyNode->info.type) {
     case OP_StreamScan:
-    case OP_DataBlocksOptScan:
+    case OP_TableScan:
     case OP_TableSeqScan:
       return tableScanNodeFromJson(json, obj);
     case OP_TagScan:

source/libs/qcom/src/querymsg.c

@@ -22,50 +22,41 @@
 #pragma GCC diagnostic ignored "-Wformat-truncation"
 
 int32_t (*queryBuildMsg[TDMT_MAX])(void *input, char **msg, int32_t msgSize, int32_t *msgLen) = {0};
-
 int32_t (*queryProcessMsgRsp[TDMT_MAX])(void *output, char *msg, int32_t msgSize) = {0};
 
 int32_t queryBuildTableMetaReqMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen) {
+  SBuildTableMetaInput *pInput = input;
   if (NULL == input || NULL == msg || NULL == msgLen) {
     return TSDB_CODE_TSC_INVALID_INPUT;
   }
 
-  SBuildTableMetaInput* bInput = (SBuildTableMetaInput *)input;
+  STableInfoReq infoReq = {0};
-
+  infoReq.header.vgId = pInput->vgId;
-  int32_t estimateSize = sizeof(STableInfoReq);
+  if (pInput->dbFName) {
-  if (NULL == *msg || msgSize < estimateSize) {
+    tstrncpy(infoReq.dbFName, pInput->dbFName, TSDB_DB_FNAME_LEN);
-    tfree(*msg);
-    *msg = rpcMallocCont(estimateSize);
-    if (NULL == *msg) {
-      return TSDB_CODE_TSC_OUT_OF_MEMORY;
-    }
   }
+  tstrncpy(infoReq.tbName, pInput->tbName, TSDB_TABLE_NAME_LEN);
 
-  STableInfoReq *bMsg = (STableInfoReq *)*msg;
+  int32_t bufLen = tSerializeSTableInfoReq(NULL, 0, &infoReq);
+  void   *pBuf = rpcMallocCont(bufLen);
+  tSerializeSTableInfoReq(pBuf, bufLen, &infoReq);
 
-  bMsg->header.vgId = htonl(bInput->vgId);
+  *msg = pBuf;
+  *msgLen = bufLen;
 
-  if (bInput->dbFName) {
-    tstrncpy(bMsg->dbFName, bInput->dbFName, tListLen(bMsg->dbFName));
-  }
-
-  tstrncpy(bMsg->tbName, bInput->tbName, tListLen(bMsg->tbName));
-
-  *msgLen = (int32_t)sizeof(*bMsg);
   return TSDB_CODE_SUCCESS;
 }
 
 int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen) {
-  if (NULL == input || NULL == msg || NULL == msgLen) {
+  SBuildUseDBInput *pInput = input;
+  if (NULL == pInput || NULL == msg || NULL == msgLen) {
     return TSDB_CODE_TSC_INVALID_INPUT;
   }
 
-  SBuildUseDBInput *bInput = input;
-
   SUseDbReq usedbReq = {0};
-  strncpy(usedbReq.db, bInput->db, sizeof(usedbReq.db));
+  strncpy(usedbReq.db, pInput->db, sizeof(usedbReq.db));
   usedbReq.db[sizeof(usedbReq.db) - 1] = 0;
-  usedbReq.vgVersion = bInput->vgVersion;
+  usedbReq.vgVersion = pInput->vgVersion;
 
   int32_t bufLen = tSerializeSUseDbReq(NULL, 0, &usedbReq);
   void   *pBuf = rpcMallocCont(bufLen);
@@ -78,75 +69,69 @@ int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *ms
 }
 
 int32_t queryProcessUseDBRsp(void *output, char *msg, int32_t msgSize) {
+  SUseDbOutput *pOut = output;
+  SUseDbRsp     usedbRsp = {0};
+  int32_t       code = -1;
+
   if (NULL == output || NULL == msg || msgSize <= 0) {
-    return TSDB_CODE_TSC_INVALID_INPUT;
+    code = TSDB_CODE_TSC_INVALID_INPUT;
+    goto PROCESS_USEDB_OVER;
   }
 
-  SUseDbOutput *pOut = (SUseDbOutput *)output;
-  int32_t       code = 0;
-
-  SUseDbRsp usedbRsp = {0};
   if (tDeserializeSUseDbRsp(msg, msgSize, &usedbRsp) != 0) {
     qError("invalid use db rsp msg, msgSize:%d", msgSize);
-    return TSDB_CODE_INVALID_MSG;
+    code = TSDB_CODE_INVALID_MSG;
+    goto PROCESS_USEDB_OVER;
   }
 
   if (usedbRsp.vgNum < 0) {
     qError("invalid db[%s] vgroup number[%d]", usedbRsp.db, usedbRsp.vgNum);
-    return TSDB_CODE_TSC_INVALID_VALUE;
+    code = TSDB_CODE_TSC_INVALID_VALUE;
+    goto PROCESS_USEDB_OVER;
   }
 
+  memcpy(pOut->db, usedbRsp.db, TSDB_DB_FNAME_LEN);
+  pOut->dbId = usedbRsp.uid;
   pOut->dbVgroup = calloc(1, sizeof(SDBVgInfo));
   if (NULL == pOut->dbVgroup) {
-    qError("calloc %d failed", (int32_t)sizeof(SDBVgInfo));
+    code = TSDB_CODE_TSC_OUT_OF_MEMORY;
-    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+    goto PROCESS_USEDB_OVER;
   }
 
-  pOut->dbId = usedbRsp.uid;
   pOut->dbVgroup->vgVersion = usedbRsp.vgVersion;
   pOut->dbVgroup->hashMethod = usedbRsp.hashMethod;
   pOut->dbVgroup->vgHash =
       taosHashInit(usedbRsp.vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
   if (NULL == pOut->dbVgroup->vgHash) {
-    qError("taosHashInit %d failed", usedbRsp.vgNum);
     tfree(pOut->dbVgroup);
-    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+    code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    goto PROCESS_USEDB_OVER;
   }
 
   for (int32_t i = 0; i < usedbRsp.vgNum; ++i) {
     SVgroupInfo *pVgInfo = taosArrayGet(usedbRsp.pVgroupInfos, i);
-
     if (0 != taosHashPut(pOut->dbVgroup->vgHash, &pVgInfo->vgId, sizeof(int32_t), pVgInfo, sizeof(SVgroupInfo))) {
-      qError("taosHashPut failed");
+      code = TSDB_CODE_TSC_OUT_OF_MEMORY;
-      goto _return;
+      goto PROCESS_USEDB_OVER;
     }
   }
 
-  memcpy(pOut->db, usedbRsp.db, TSDB_DB_FNAME_LEN);
+  code = 0;
-
-  return code;
-
-_return:
-  tFreeSUsedbRsp(&usedbRsp);
 
+PROCESS_USEDB_OVER:
+  if (code != 0) {
     if (pOut) {
-    taosHashCleanup(pOut->dbVgroup->vgHash);
+      if (pOut->dbVgroup) taosHashCleanup(pOut->dbVgroup->vgHash);
       tfree(pOut->dbVgroup);
     }
+    qError("failed to process usedb rsp since %s", terrstr());
+  }
 
+  tFreeSUsedbRsp(&usedbRsp);
   return code;
 }
 
 static int32_t queryConvertTableMetaMsg(STableMetaRsp *pMetaMsg) {
-  pMetaMsg->dbId = be64toh(pMetaMsg->dbId);
-  pMetaMsg->numOfTags = ntohl(pMetaMsg->numOfTags);
-  pMetaMsg->numOfColumns = ntohl(pMetaMsg->numOfColumns);
-  pMetaMsg->sversion = ntohl(pMetaMsg->sversion);
-  pMetaMsg->tversion = ntohl(pMetaMsg->tversion);
-  pMetaMsg->tuid = be64toh(pMetaMsg->tuid);
-  pMetaMsg->suid = be64toh(pMetaMsg->suid);
-  pMetaMsg->vgId = ntohl(pMetaMsg->vgId);
-
   if (pMetaMsg->numOfTags < 0 || pMetaMsg->numOfTags > TSDB_MAX_TAGS) {
     qError("invalid numOfTags[%d] in table meta rsp msg", pMetaMsg->numOfTags);
     return TSDB_CODE_TSC_INVALID_VALUE;
@@ -157,7 +142,8 @@ static int32_t queryConvertTableMetaMsg(STableMetaRsp* pMetaMsg) {
     return TSDB_CODE_TSC_INVALID_VALUE;
   }
 
-  if (pMetaMsg->tableType != TSDB_SUPER_TABLE && pMetaMsg->tableType != TSDB_CHILD_TABLE && pMetaMsg->tableType != TSDB_NORMAL_TABLE) {
+  if (pMetaMsg->tableType != TSDB_SUPER_TABLE && pMetaMsg->tableType != TSDB_CHILD_TABLE &&
+      pMetaMsg->tableType != TSDB_NORMAL_TABLE) {
     qError("invalid tableType[%d] in table meta rsp msg", pMetaMsg->tableType);
     return TSDB_CODE_TSC_INVALID_VALUE;
   }
@@ -172,18 +158,8 @@ static int32_t queryConvertTableMetaMsg(STableMetaRsp* pMetaMsg) {
     return TSDB_CODE_TSC_INVALID_VALUE;
   }
 
-  SSchema* pSchema = pMetaMsg->pSchema;
+  if (pMetaMsg->pSchemas[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) {
-
+    qError("invalid colId[%d] for the first column in table meta rsp msg", pMetaMsg->pSchemas[0].colId);
-  int32_t numOfTotalCols = pMetaMsg->numOfColumns + pMetaMsg->numOfTags;
-  for (int i = 0; i < numOfTotalCols; ++i) {
-    pSchema->bytes = ntohl(pSchema->bytes);
-    pSchema->colId = ntohl(pSchema->colId);
-
-    pSchema++;
-  }
-
-  if (pMetaMsg->pSchema[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) {
-    qError("invalid colId[%d] for the first column in table meta rsp msg", pMetaMsg->pSchema[0].colId);
     return TSDB_CODE_TSC_INVALID_VALUE;
   }
 
@@ -211,60 +187,71 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STabl
   pTableMeta->tableInfo.precision = msg->precision;
   pTableMeta->tableInfo.numOfColumns = msg->numOfColumns;
 
-  memcpy(pTableMeta->schema, msg->pSchema, sizeof(SSchema) * total);
+  memcpy(pTableMeta->schema, msg->pSchemas, sizeof(SSchema) * total);
 
   for (int32_t i = 0; i < msg->numOfColumns; ++i) {
     pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes;
   }
 
   *pMeta = pTableMeta;
-  
   return TSDB_CODE_SUCCESS;
 }
 
-
 int32_t queryProcessTableMetaRsp(void *output, char *msg, int32_t msgSize) {
-  STableMetaRsp *pMetaMsg = (STableMetaRsp *)msg;
+  int32_t       code = -1;
-  int32_t code = queryConvertTableMetaMsg(pMetaMsg);
+  STableMetaRsp metaRsp = {0};
+
+  if (NULL == output || NULL == msg || msgSize <= 0) {
+    code = TSDB_CODE_TSC_INVALID_INPUT;
+    goto PROCESS_META_OVER;
+  }
+
+  if (tDeserializeSTableMetaRsp(msg, msgSize, &metaRsp) != 0) {
+    code = TSDB_CODE_INVALID_MSG;
+    goto PROCESS_META_OVER;
+  }
+
+  code = queryConvertTableMetaMsg(&metaRsp);
   if (code != TSDB_CODE_SUCCESS) {
-    return code;
+    goto PROCESS_META_OVER;
   }
 
-  STableMetaOutput *pOut = (STableMetaOutput *)output;
+  if (!tIsValidSchema(metaRsp.pSchemas, metaRsp.numOfColumns, metaRsp.numOfTags)) {
-  
+    code = TSDB_CODE_TSC_INVALID_VALUE;
-  if (!tIsValidSchema(pMetaMsg->pSchema, pMetaMsg->numOfColumns, pMetaMsg->numOfTags)) {
+    goto PROCESS_META_OVER;
-    qError("validate table meta schema in rsp msg failed");
-    return TSDB_CODE_TSC_INVALID_VALUE;
   }
 
-  strcpy(pOut->dbFName, pMetaMsg->dbFName);
+  STableMetaOutput *pOut = output;
+  strcpy(pOut->dbFName, metaRsp.dbFName);
+  pOut->dbId = metaRsp.dbId;
 
-  pOut->dbId = pMetaMsg->dbId;
+  if (metaRsp.tableType == TSDB_CHILD_TABLE) {
-
-  if (pMetaMsg->tableType == TSDB_CHILD_TABLE) {
     SET_META_TYPE_BOTH_TABLE(pOut->metaType);
 
-    strcpy(pOut->ctbName, pMetaMsg->tbName);
+    strcpy(pOut->ctbName, metaRsp.tbName);
-    strcpy(pOut->tbName, pMetaMsg->stbName);
+    strcpy(pOut->tbName, metaRsp.stbName);
 
-    pOut->ctbMeta.vgId = pMetaMsg->vgId;
+    pOut->ctbMeta.vgId = metaRsp.vgId;
-    pOut->ctbMeta.tableType = pMetaMsg->tableType;
+    pOut->ctbMeta.tableType = metaRsp.tableType;
-    pOut->ctbMeta.uid = pMetaMsg->tuid;
+    pOut->ctbMeta.uid = metaRsp.tuid;
-    pOut->ctbMeta.suid = pMetaMsg->suid;
+    pOut->ctbMeta.suid = metaRsp.suid;
 
-    code = queryCreateTableMetaFromMsg(pMetaMsg, true, &pOut->tbMeta);
+    code = queryCreateTableMetaFromMsg(&metaRsp, true, &pOut->tbMeta);
   } else {
     SET_META_TYPE_TABLE(pOut->metaType);
-    
+    strcpy(pOut->tbName, metaRsp.tbName);
-    strcpy(pOut->tbName, pMetaMsg->tbName);
+    code = queryCreateTableMetaFromMsg(&metaRsp, (metaRsp.tableType == TSDB_SUPER_TABLE), &pOut->tbMeta);
-    
-    code = queryCreateTableMetaFromMsg(pMetaMsg, (pMetaMsg->tableType == TSDB_SUPER_TABLE), &pOut->tbMeta);
   }
 
+PROCESS_META_OVER:
+  if (code != 0) {
+    qError("failed to process table meta rsp since %s", terrstr());
+  }
+
+  tFreeSTableMetaRsp(&metaRsp);
   return code;
 }
 
-
 void initQueryModuleMsgHandle() {
   queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryBuildTableMetaReqMsg;
   queryBuildMsg[TMSG_INDEX(TDMT_MND_STB_META)] = queryBuildTableMetaReqMsg;

source/libs/qworker/src/qworkerMsg.c

@@ -173,43 +173,52 @@ int32_t qwBuildAndSendDropRsp(void *connection, int32_t code) {
 
 int32_t qwBuildAndSendShowRsp(SRpcMsg *pMsg, int32_t code) {
   int32_t         numOfCols = 6;
-  int32_t msgSize = sizeof(SVShowTablesRsp) + sizeof(SSchema) * numOfCols;
+  SVShowTablesRsp showRsp = {0};
 
-  SVShowTablesRsp *pRsp = (SVShowTablesRsp *)rpcMallocCont(msgSize);
+  // showRsp.showId = 1;
+  showRsp.tableMeta.pSchemas = calloc(numOfCols, sizeof(SSchema));
+  if (showRsp.tableMeta.pSchemas == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
 
   int32_t  cols = 0;
-  SSchema *pSchema = pRsp->metaInfo.pSchema;
+  SSchema *pSchema = showRsp.tableMeta.pSchemas;
 
   const SSchema *s = tGetTbnameColumnSchema();
-  *pSchema = createSchema(s->type, htonl(s->bytes), htonl(++cols), "name");
+  *pSchema = createSchema(s->type, s->bytes, ++cols, "name");
   pSchema++;
 
   int32_t type = TSDB_DATA_TYPE_TIMESTAMP;
-  *pSchema = createSchema(type, htonl(tDataTypes[type].bytes), htonl(++cols), "created");
+  *pSchema = createSchema(type, tDataTypes[type].bytes, ++cols, "created");
   pSchema++;
 
   type = TSDB_DATA_TYPE_SMALLINT;
-  *pSchema = createSchema(type, htonl(tDataTypes[type].bytes), htonl(++cols), "columns");
+  *pSchema = createSchema(type, tDataTypes[type].bytes, ++cols, "columns");
   pSchema++;
 
-  *pSchema = createSchema(s->type, htonl(s->bytes), htonl(++cols), "stable");
+  *pSchema = createSchema(s->type, s->bytes, ++cols, "stable");
   pSchema++;
 
   type = TSDB_DATA_TYPE_BIGINT;
-  *pSchema = createSchema(type, htonl(tDataTypes[type].bytes), htonl(++cols), "uid");
+  *pSchema = createSchema(type, tDataTypes[type].bytes, ++cols, "uid");
   pSchema++;
 
   type = TSDB_DATA_TYPE_INT;
-  *pSchema = createSchema(type, htonl(tDataTypes[type].bytes), htonl(++cols), "vgId");
+  *pSchema = createSchema(type, tDataTypes[type].bytes, ++cols, "vgId");
 
   assert(cols == numOfCols);
-  pRsp->metaInfo.numOfColumns = htonl(cols);
+  showRsp.tableMeta.numOfColumns = cols;
+
+  int32_t bufLen = tSerializeSShowRsp(NULL, 0, &showRsp);
+  void   *pBuf = rpcMallocCont(bufLen);
+  tSerializeSShowRsp(pBuf, bufLen, &showRsp);
 
   SRpcMsg rpcMsg = {
       .handle = pMsg->handle,
       .ahandle = pMsg->ahandle,
-      .pCont   = pRsp,
+      .pCont = pBuf,
-      .contLen = msgSize,
+      .contLen = bufLen,
       .code = code,
   };
 

source/libs/transport/src/transCli.c

@@ -17,6 +17,7 @@
 
 #include "transComm.h"
 
+#define CONN_HOST_THREAD_INDEX(conn ? ((SCliConn*)conn)->hThrdIdx : -1)
 #define CONN_PERSIST_TIME(para) (para * 1000 * 10)
 
 typedef struct SCliConn {
@@ -28,7 +29,8 @@ typedef struct SCliConn {
   void*        data;
   queue        conn;
   uint64_t     expireTime;
-  int8_t       notifyCount;  // timers already notify to client
+  int8_t       ctnRdCnt;  // timers already notify to client
+  int          hostThreadIndex;
 
   SRpcPush* push;
   int       persist;  //
@@ -131,11 +133,16 @@ static void clientHandleResp(SCliConn* conn) {
   rpcMsg.msgType = pHead->msgType;
   rpcMsg.ahandle = pCtx->ahandle;
 
+  if (rpcMsg.msgType == TDMT_VND_QUERY_RSP || rpcMsg.msgType == TDMT_VND_FETCH_RSP) {
+    rpcMsg.handle = conn;
+    conn->persist = 1;
+  }
+
   tDebug("client conn %p %s received from %s:%d, local info: %s:%d", conn, TMSG_INFO(pHead->msgType),
          inet_ntoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), inet_ntoa(conn->locaddr.sin_addr),
          ntohs(conn->locaddr.sin_port));
 
-  if (conn->push != NULL && conn->notifyCount != 0) {
+  if (conn->push != NULL && conn->ctnRdCnt != 0) {
     (*conn->push->callback)(conn->push->arg, &rpcMsg);
     conn->push = NULL;
   } else {
@@ -148,7 +155,7 @@ static void clientHandleResp(SCliConn* conn) {
       tsem_post(pCtx->pSem);
     }
   }
-  conn->notifyCount += 1;
+  conn->ctnRdCnt += 1;
   conn->secured = pHead->secured;
 
   // buf's mem alread translated to rpcMsg.pCont
@@ -157,13 +164,15 @@ static void clientHandleResp(SCliConn* conn) {
   uv_read_start((uv_stream_t*)conn->stream, clientAllocBufferCb, clientReadCb);
 
   SCliThrdObj* pThrd = conn->hostThrd;
+
   // user owns conn->persist = 1
-  if (conn->push == NULL) {
+  if (conn->push == NULL || conn->persist == 0) {
     addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn);
 
     destroyCmsg(conn->data);
     conn->data = NULL;
   }
+
   // start thread's timer of conn pool if not active
   if (!uv_is_active((uv_handle_t*)pThrd->timer) && pRpc->idleTime > 0) {
     uv_timer_start((uv_timer_t*)pThrd->timer, clientTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0);
@@ -178,13 +187,18 @@ static void clientHandleExcept(SCliConn* pConn) {
   tTrace("client conn %p start to destroy", pConn);
   SCliMsg* pMsg = pConn->data;
 
+  tmsg_t msgType = TDMT_MND_CONNECT;
+  if (pMsg != NULL) {
+    msgType = pMsg->msg.msgType;
+  }
   STransConnCtx* pCtx = pMsg->ctx;
 
   SRpcMsg rpcMsg = {0};
   rpcMsg.ahandle = pCtx->ahandle;
   rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
+  rpcMsg.msgType = msgType + 1;
 
-  if (pConn->push != NULL && pConn->notifyCount != 0) {
+  if (pConn->push != NULL && pConn->ctnRdCnt != 0) {
     (*pConn->push->callback)(pConn->push->arg, &rpcMsg);
     pConn->push = NULL;
   } else {
@@ -205,7 +219,7 @@ static void clientHandleExcept(SCliConn* pConn) {
   }
   // transDestroyConnCtx(pCtx);
   clientConnDestroy(pConn, true);
-  pConn->notifyCount += 1;
+  pConn->ctnRdCnt += 1;
 }
 
 static void clientTimeoutCb(uv_timer_t* handle) {
@@ -287,7 +301,7 @@ static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn) {
 
   conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime);
   SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key));
-  conn->notifyCount = 0;
+  conn->ctnRdCnt = 0;
   // list already create before
   assert(plist != NULL);
   QUEUE_PUSH(&plist->conn, &conn->conn);
@@ -418,6 +432,8 @@ static void clientWrite(SCliConn* pConn) {
   pHead->msgType = pMsg->msgType;
   pHead->msgLen = (int32_t)htonl((uint32_t)msgLen);
 
+  // if (pHead->msgType == TDMT_VND_QUERY || pHead->msgType == TDMT_VND_)
+
   uv_buf_t wb = uv_buf_init((char*)pHead, msgLen);
   tDebug("client conn %p %s is send to %s:%d, local info %s:%d", pConn, TMSG_INFO(pHead->msgType),
          inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr),
@@ -457,14 +473,25 @@ static void clientHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) {
 static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
   uint64_t et = taosGetTimestampUs();
   uint64_t el = et - pMsg->st;
-  tTrace("client msg tran time cost: %" PRIu64 "", el);
+  tTrace("client msg tran time cost: %" PRIu64 "us", el);
   et = taosGetTimestampUs();
 
   STransConnCtx* pCtx = pMsg->ctx;
-  SCliConn*      conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port);
+
+  SCliConn* conn = NULL;
+  if (pMsg->msg.handle == NULL) {
+    conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port);
+    if (conn != NULL) {
+      tTrace("client conn %d get from conn pool", conn);
+    }
+  } else {
+    conn = (SCliConn*)(pMsg->msg.handle);
+    if (conn != NULL) {
+      tTrace("client conn %d reused", conn);
+    }
+  }
+
   if (conn != NULL) {
-    // impl later
-    tTrace("client get conn %p from pool", conn);
     conn->data = pMsg;
     conn->writeReq->data = conn;
     transDestroyBuffer(&conn->readBuf);
@@ -475,8 +502,6 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
     }
     clientWrite(conn);
 
-    conn->push = pMsg->msg.push;
-
   } else {
     SCliConn* conn = calloc(1, sizeof(SCliConn));
     conn->ref++;
@@ -495,13 +520,18 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
     conn->data = pMsg;
     conn->hostThrd = pThrd;
 
-    conn->push = pMsg->msg.push;
+    // conn->push = pMsg->msg.push;
+    // conn->ctnRdCnt = 0;
 
     struct sockaddr_in addr;
     uv_ip4_addr(pMsg->ctx->ip, pMsg->ctx->port, &addr);
     // handle error in callback if fail to connect
     uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, clientConnCb);
   }
+
+  conn->push = pMsg->msg.push;
+  conn->ctnRdCnt = 0;
+  conn->hThrdIdx = pCtx->hThrdIdx;
 }
 static void clientAsyncCb(uv_async_t* handle) {
   SAsyncItem*  item = handle->data;
@@ -639,6 +669,13 @@ void taosCloseClient(void* arg) {
   free(cli->pThreadObj);
   free(cli);
 }
+static int clientRBChoseIdx(SRpcInfo* pRpc) {
+  int64_t index = pRpc->index;
+  if (pRpc->index++ >= pRpc->numOfThreads) {
+    pRpc->index = 0;
+  }
+  return index % pRpc->numOfThreads;
+}
 void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) {
   // impl later
   char*    ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn);
@@ -646,48 +683,45 @@ void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t*
 
   SRpcInfo* pRpc = (SRpcInfo*)shandle;
 
+  int index = CONN_HOST_THREAD_INDEX(pMsg.handle);
+  if (index == -1) {
+    index = clientRBChoseIdx(pRpc);
+  }
   int32_t flen = 0;
   if (transCompressMsg(pMsg->pCont, pMsg->contLen, &flen)) {
     // imp later
   }
-
   STransConnCtx* pCtx = calloc(1, sizeof(STransConnCtx));
-
   pCtx->pTransInst = (SRpcInfo*)shandle;
   pCtx->ahandle = pMsg->ahandle;
   pCtx->msgType = pMsg->msgType;
   pCtx->ip = strdup(ip);
   pCtx->port = port;
+  pCtx->hThrdIdx = index;
 
   assert(pRpc->connType == TAOS_CONN_CLIENT);
   // atomic or not
-  int64_t index = pRpc->index;
+
-  if (pRpc->index++ >= pRpc->numOfThreads) {
-    pRpc->index = 0;
-  }
   SCliMsg* cliMsg = malloc(sizeof(SCliMsg));
   cliMsg->ctx = pCtx;
   cliMsg->msg = *pMsg;
   cliMsg->st = taosGetTimestampUs();
 
-  SCliThrdObj* thrd = ((SClientObj*)pRpc->tcphandle)->pThreadObj[index % pRpc->numOfThreads];
+  SCliThrdObj* thrd = ((SClientObj*)pRpc->tcphandle)->pThreadObj[index];
-
-  // pthread_mutex_lock(&thrd->msgMtx);
-  // QUEUE_PUSH(&thrd->msg, &cliMsg->q);
-  // pthread_mutex_unlock(&thrd->msgMtx);
-
-  // int start = taosGetTimestampUs();
   transSendAsync(thrd->asyncPool, &(cliMsg->q));
-  // uv_async_send(thrd->cliAsync);
-  // int end = taosGetTimestampUs() - start;
-  // tError("client sent to rpc, time cost: %d", (int)end);
 }
+
 void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) {
   char*    ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn);
   uint32_t port = pEpSet->eps[pEpSet->inUse].port;
 
   SRpcInfo* pRpc = (SRpcInfo*)shandle;
 
+  int index = CONN_HOST_THREAD_INDEX(pMsg.handle);
+  if (index == -1) {
+    index = clientRBChoseIdx(pRpc);
+  }
+
   STransConnCtx* pCtx = calloc(1, sizeof(STransConnCtx));
   pCtx->pTransInst = (SRpcInfo*)shandle;
   pCtx->ahandle = pReq->ahandle;
@@ -698,24 +732,13 @@ void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) {
   pCtx->pRsp = pRsp;
   tsem_init(pCtx->pSem, 0, 0);
 
-  int64_t index = pRpc->index;
-  if (pRpc->index++ >= pRpc->numOfThreads) {
-    pRpc->index = 0;
-  }
   SCliMsg* cliMsg = malloc(sizeof(SCliMsg));
   cliMsg->ctx = pCtx;
   cliMsg->msg = *pReq;
   cliMsg->st = taosGetTimestampUs();
 
-  SCliThrdObj* thrd = ((SClientObj*)pRpc->tcphandle)->pThreadObj[index % pRpc->numOfThreads];
+  SCliThrdObj* thrd = ((SClientObj*)pRpc->tcphandle)->pThreadObj[index];
-
-  // pthread_mutex_lock(&thrd->msgMtx);
-  // QUEUE_PUSH(&thrd->msg, &cliMsg->q);
-  // pthread_mutex_unlock(&thrd->msgMtx);
-
-  // int start = taosGetTimestampUs();
   transSendAsync(thrd->asyncPool, &(cliMsg->q));
-
   tsem_t* pSem = pCtx->pSem;
   tsem_wait(pSem);
   tsem_destroy(pSem);

source/util/src/tlosertree.c

@@ -14,82 +14,85 @@
  */
 
 #include "os.h"
-#include "tlosertree.h"
 #include "ulog.h"
+#include "tlosertree.h"
+#include "taoserror.h"
 
-// set initial value for loser tree
-void tLoserTreeInit(SLoserTreeInfo* pTree) {
-  assert((pTree->totalEntries & 0x01) == 0 && (pTree->numOfEntries << 1 == pTree->totalEntries));
 
-  for (int32_t i = 0; i < pTree->totalEntries; ++i) {
+
-    if (i < pTree->numOfEntries) {
+// Set the initial value of the multiway merge tree.
+static void tMergeTreeInit(SMultiwayMergeTreeInfo* pTree) {
+  assert((pTree->totalSources & 0x01) == 0 && (pTree->numOfSources << 1 == pTree->totalSources));
+
+  for (int32_t i = 0; i < pTree->totalSources; ++i) {
+    if (i < pTree->numOfSources) {
       pTree->pNode[i].index = -1;
     } else {
-      pTree->pNode[i].index = i - pTree->numOfEntries;
+      pTree->pNode[i].index = i - pTree->numOfSources;
     }
   }
 }
 
-/*
+int32_t tMergeTreeCreate(SMultiwayMergeTreeInfo** pTree, uint32_t numOfSources, void* param, __merge_compare_fn_t compareFn) {
- * display whole loser tree on screen for debug purpose only.
+  int32_t totalEntries = numOfSources << 1u;
- */
-void tLoserTreeDisplay(SLoserTreeInfo* pTree) {
-  printf("the value of loser tree:\t");
-  for (int32_t i = 0; i < pTree->totalEntries; ++i) printf("%d\t", pTree->pNode[i].index);
-  printf("\n");
-}
 
-uint32_t tLoserTreeCreate(SLoserTreeInfo** pTree, int32_t numOfEntries, void* param, __merge_compare_fn_t compareFn) {
+  SMultiwayMergeTreeInfo* pTreeInfo = (SMultiwayMergeTreeInfo*)calloc(1, sizeof(SMultiwayMergeTreeInfo) + sizeof(STreeNode) * totalEntries);
-  int32_t totalEntries = numOfEntries << 1;
+  if (pTreeInfo == NULL) {
-
-  *pTree = (SLoserTreeInfo*)calloc(1, sizeof(SLoserTreeInfo) + sizeof(SLoserTreeNode) * totalEntries);
-  if ((*pTree) == NULL) {
     uError("allocate memory for loser-tree failed. reason:%s", strerror(errno));
-    return -1;
+    return TAOS_SYSTEM_ERROR(errno);
   }
 
-  (*pTree)->pNode = (SLoserTreeNode*)(((char*)(*pTree)) + sizeof(SLoserTreeInfo));
+  pTreeInfo->pNode = (STreeNode*)(((char*)pTreeInfo) + sizeof(SMultiwayMergeTreeInfo));
 
-  (*pTree)->numOfEntries = numOfEntries;
+  pTreeInfo->numOfSources = numOfSources;
-  (*pTree)->totalEntries = totalEntries;
+  pTreeInfo->totalSources = totalEntries;
-  (*pTree)->param = param;
+  pTreeInfo->param = param;
-  (*pTree)->comparFn = compareFn;
+  pTreeInfo->comparFn = compareFn;
 
   // set initial value for loser tree
-  tLoserTreeInit(*pTree);
+  tMergeTreeInit(pTreeInfo);
 
 #ifdef _DEBUG_VIEW
   printf("the initial value of loser tree:\n");
-  tLoserTreeDisplay(*pTree);
+  tLoserTreeDisplaypTreeInfo;
 #endif
 
-  for (int32_t i = totalEntries - 1; i >= numOfEntries; i--) {
+  for (int32_t i = totalEntries - 1; i >= numOfSources; i--) {
-    tLoserTreeAdjust(*pTree, i);
+    tMergeTreeAdjust(pTreeInfo, i);
   }
 
 #if defined(_DEBUG_VIEW)
   printf("after adjust:\n");
-  tLoserTreeDisplay(*pTree);
+  tLoserTreeDisplaypTreeInfo;
   printf("initialize local reducer completed!\n");
 #endif
 
+  *pTree = pTreeInfo;
   return 0;
 }
 
-void tLoserTreeAdjust(SLoserTreeInfo* pTree, int32_t idx) {
+void tMergeTreeDestroy(SMultiwayMergeTreeInfo* pTree) {
-  assert(idx <= pTree->totalEntries - 1 && idx >= pTree->numOfEntries && pTree->totalEntries >= 2);
+  if (pTree == NULL) {
+    return;
+  }
 
-  if (pTree->totalEntries == 2) {
+  tfree(pTree);
+}
+
+void tMergeTreeAdjust(SMultiwayMergeTreeInfo* pTree, int32_t idx) {
+  assert(idx <= pTree->totalSources - 1 && idx >= pTree->numOfSources && pTree->totalSources >= 2);
+
+  if (pTree->totalSources == 2) {
     pTree->pNode[0].index = 0;
     pTree->pNode[1].index = 0;
     return;
   }
 
   int32_t        parentId = idx >> 1;
-  SLoserTreeNode kLeaf = pTree->pNode[idx];
+  STreeNode kLeaf = pTree->pNode[idx];
 
   while (parentId > 0) {
-    SLoserTreeNode* pCur = &pTree->pNode[parentId];
+    STreeNode* pCur = &pTree->pNode[parentId];
     if (pCur->index == -1) {
       pTree->pNode[parentId] = kLeaf;
       return;
@@ -97,7 +100,7 @@ void tLoserTreeAdjust(SLoserTreeInfo* pTree, int32_t idx) {
 
     int32_t ret = pTree->comparFn(pCur, &kLeaf, pTree->param);
     if (ret < 0) {
-      SLoserTreeNode t = pTree->pNode[parentId];
+      STreeNode t = pTree->pNode[parentId];
       pTree->pNode[parentId] = kLeaf;
       kLeaf = t;
     }
@@ -111,11 +114,23 @@ void tLoserTreeAdjust(SLoserTreeInfo* pTree, int32_t idx) {
   }
 }
 
-void tLoserTreeRebuild(SLoserTreeInfo* pTree) {
+void tMergeTreeRebuild(SMultiwayMergeTreeInfo* pTree) {
-  assert((pTree->totalEntries & 0x1) == 0);
+  assert((pTree->totalSources & 0x1) == 0);
 
-  tLoserTreeInit(pTree);
+  tMergeTreeInit(pTree);
-  for (int32_t i = pTree->totalEntries - 1; i >= pTree->numOfEntries; i--) {
+  for (int32_t i = pTree->totalSources - 1; i >= pTree->numOfSources; i--) {
-    tLoserTreeAdjust(pTree, i);
+    tMergeTreeAdjust(pTree, i);
   }
 }
+
+/*
+ * display whole loser tree on screen for debug purpose only.
+ */
+void tMergeTreePrint(const SMultiwayMergeTreeInfo* pTree) {
+  printf("the value of loser tree:\t");
+  for (int32_t i = 0; i < pTree->totalSources; ++i) {
+    printf("%d\t", pTree->pNode[i].index);
+  }
+
+  printf("\n");
+}

source/util/src/tpagedbuf.c

@@ -0,0 +1,597 @@
+#include "os.h"
+#include "ulog.h"
+#include "tpagedbuf.h"
+#include "taoserror.h"
+#include "tcompression.h"
+#include "thash.h"
+
+#define GET_DATA_PAYLOAD(_p) ((char *)(_p)->pData + POINTER_BYTES)
+#define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages)
+
+typedef struct SFreeListItem {
+  int32_t offset;
+  int32_t len;
+} SFreeListItem;
+
+typedef struct SPageDiskInfo {
+  int64_t  offset;
+  int32_t  length;
+} SPageDiskInfo;
+
+typedef struct SPageInfo {
+  SListNode*    pn;       // point to list node
+  void*         pData;
+  int64_t       offset;
+  int32_t       pageId;
+  int32_t       length:30;
+  bool          used:1;     // set current page is in used
+  bool          dirty:1;    // set current buffer page is dirty or not
+} SPageInfo;
+
+typedef struct SDiskbasedBuf {
+  int32_t   numOfPages;
+  int64_t   totalBufSize;
+  uint64_t  fileSize;            // disk file size
+  FILE*     file;
+  int32_t   allocateId;          // allocated page id
+  char*     path;                // file path
+  int32_t   pageSize;            // current used page size
+  int32_t   inMemPages;          // numOfPages that are allocated in memory
+  SHashObj* groupSet;            // id hash table
+  SHashObj* all;
+  SList*    lruList;
+  void*     emptyDummyIdList;    // dummy id list
+  void*     assistBuf;           // assistant buffer for compress/decompress data
+  SArray*   pFree;               // free area in file
+  bool      comp;                // compressed before flushed to disk
+  uint64_t  nextPos;             // next page flush position
+
+  uint64_t  qId;                 // for debug purpose
+  bool      printStatis;         // Print statistics info when closing this buffer.
+  SDiskbasedBufStatis statis;
+} SDiskbasedBuf;
+
+static void printStatisData(const SDiskbasedBuf* pBuf);
+
+  int32_t createDiskbasedBuffer(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir) {
+  *pBuf = calloc(1, sizeof(SDiskbasedBuf));
+
+  SDiskbasedBuf* pResBuf = *pBuf;
+  if (pResBuf == NULL) {
+    return TSDB_CODE_OUT_OF_MEMORY;  
+  }
+
+  pResBuf->pageSize     = pagesize;
+  pResBuf->numOfPages   = 0;                        // all pages are in buffer in the first place
+  pResBuf->totalBufSize = 0;
+  pResBuf->inMemPages   = inMemBufSize/pagesize;    // maximum allowed pages, it is a soft limit.
+  pResBuf->allocateId   = -1;
+  pResBuf->comp         = true;
+  pResBuf->file         = NULL;
+  pResBuf->qId          = qId;
+  pResBuf->fileSize     = 0;
+
+  // at least more than 2 pages must be in memory
+  assert(inMemBufSize >= pagesize * 2);
+
+  pResBuf->lruList = tdListNew(POINTER_BYTES);
+
+  // init id hash table
+  pResBuf->groupSet  = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false);
+  pResBuf->assistBuf = malloc(pResBuf->pageSize + 2); // EXTRA BYTES
+  pResBuf->all = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false);
+
+  char path[PATH_MAX] = {0};
+  taosGetTmpfilePath(dir, "qbuf", path);
+  pResBuf->path = strdup(path);
+
+  pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t));
+
+//  qDebug("QInfo:0x%"PRIx64" create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", qId, pResBuf->pageSize,
+//         pResBuf->inMemPages, pResBuf->path);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t createDiskFile(SDiskbasedBuf* pBuf) {
+  pBuf->file = fopen(pBuf->path, "wb+");
+  if (pBuf->file == NULL) {
+//    qError("failed to create tmp file: %s on disk. %s", pBuf->path, strerror(errno));
+    return TAOS_SYSTEM_ERROR(errno);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static char* doCompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedBuf* pBuf) { // do nothing
+  if (!pBuf->comp) {
+    *dst = srcSize;
+    return data;
+  }
+
+  *dst = tsCompressString(data, srcSize, 1, pBuf->assistBuf, srcSize, ONE_STAGE_COMP, NULL, 0);
+
+  memcpy(data, pBuf->assistBuf, *dst);
+  return data;
+}
+
+static char* doDecompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedBuf* pBuf) { // do nothing
+  if (!pBuf->comp) {
+    *dst = srcSize;
+    return data;
+  }
+
+  *dst = tsDecompressString(data, srcSize, 1, pBuf->assistBuf, pBuf->pageSize+sizeof(SFilePage), ONE_STAGE_COMP, NULL, 0);
+  if (*dst > 0) {
+    memcpy(data, pBuf->assistBuf, *dst);
+  }
+  return data;
+}
+
+static uint64_t allocatePositionInFile(SDiskbasedBuf* pBuf, size_t size) {
+  if (pBuf->pFree == NULL) {
+    return pBuf->nextPos;
+  } else {
+    int32_t offset = -1;
+
+    size_t num = taosArrayGetSize(pBuf->pFree);
+    for(int32_t i = 0; i < num; ++i) {
+      SFreeListItem* pi = taosArrayGet(pBuf->pFree, i);
+      if (pi->len >= size) {
+        offset = pi->offset;
+        pi->offset += (int32_t)size;
+        pi->len -= (int32_t)size;
+
+        return offset;
+      }
+    }
+
+    // no available recycle space, allocate new area in file
+    return pBuf->nextPos;
+  }
+}
+
+static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
+  assert(!pg->used && pg->pData != NULL);
+
+  int32_t size = -1;
+  char*   t = NULL;
+  if (pg->offset == -1 || pg->dirty) {
+    SFilePage* pPage = (SFilePage*) GET_DATA_PAYLOAD(pg);
+    t = doCompressData(pPage->data, pBuf->pageSize, &size, pBuf);
+  }
+
+  // this page is flushed to disk for the first time
+  if (pg->offset == -1) {
+    assert(pg->dirty == true);
+
+    pg->offset = allocatePositionInFile(pBuf, size);
+    pBuf->nextPos += size;
+
+    int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET);
+    if (ret != 0) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      return NULL;
+    }
+
+    ret = (int32_t) fwrite(t, 1, size, pBuf->file);
+    if (ret != size) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      return NULL;
+    }
+
+    if (pBuf->fileSize < pg->offset + size) {
+      pBuf->fileSize = pg->offset + size;
+    }
+
+    pBuf->statis.flushBytes += size;
+    pBuf->statis.flushPages += 1;
+  } else if (pg->dirty) {
+    // length becomes greater, current space is not enough, allocate new place, otherwise, do nothing
+    if (pg->length < size) {
+      // 1. add current space to free list
+      SPageDiskInfo dinfo = {.length = pg->length, .offset = pg->offset};
+      taosArrayPush(pBuf->pFree, &dinfo);
+
+      // 2. allocate new position, and update the info
+      pg->offset = allocatePositionInFile(pBuf, size);
+      pBuf->nextPos += size;
+    }
+
+    // 3. write to disk.
+    int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET);
+    if (ret != 0) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      return NULL;
+    }
+
+    ret = (int32_t)fwrite(t, 1, size, pBuf->file);
+    if (ret != size) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      return NULL;
+    }
+
+    if (pBuf->fileSize < pg->offset + size) {
+      pBuf->fileSize = pg->offset + size;
+    }
+
+    pBuf->statis.flushBytes += size;
+    pBuf->statis.flushPages += 1;
+  }
+
+  char* pDataBuf = pg->pData;
+  memset(pDataBuf, 0, pBuf->pageSize + sizeof(SFilePage));
+
+  pg->pData  = NULL;  // this means the data is not in buffer
+  pg->length = size;
+  pg->dirty  = false;
+
+  return pDataBuf;
+}
+
+static char* flushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
+  int32_t ret = TSDB_CODE_SUCCESS;
+  assert(((int64_t) pBuf->numOfPages * pBuf->pageSize) == pBuf->totalBufSize && pBuf->numOfPages >= pBuf->inMemPages);
+
+  if (pBuf->file == NULL) {
+    if ((ret = createDiskFile(pBuf)) != TSDB_CODE_SUCCESS) {
+      terrno = ret;
+      return NULL;
+    }
+  }
+
+  return doFlushPageToDisk(pBuf, pg);
+}
+
+// load file block data in disk
+static int32_t loadPageFromDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
+  int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET);
+  if (ret != 0) {
+    ret = TAOS_SYSTEM_ERROR(errno);
+    return ret;
+  }
+
+  SFilePage* pPage = (SFilePage*) GET_DATA_PAYLOAD(pg);
+  ret = (int32_t)fread(pPage->data, 1, pg->length, pBuf->file);
+  if (ret != pg->length) {
+    ret = TAOS_SYSTEM_ERROR(errno);
+    return ret;
+  }
+
+  pBuf->statis.loadBytes += pg->length;
+  pBuf->statis.loadPages += 1;
+
+  int32_t fullSize = 0;
+  doDecompressData(pPage->data, pg->length, &fullSize, pBuf);
+  return 0;
+}
+
+static SIDList addNewGroup(SDiskbasedBuf* pBuf, int32_t groupId) {
+  assert(taosHashGet(pBuf->groupSet, (const char*) &groupId, sizeof(int32_t)) == NULL);
+
+  SArray* pa = taosArrayInit(1, POINTER_BYTES);
+  int32_t ret = taosHashPut(pBuf->groupSet, (const char*)&groupId, sizeof(int32_t), &pa, POINTER_BYTES);
+  assert(ret == 0);
+
+  return pa;
+}
+
+static SPageInfo* registerPage(SDiskbasedBuf* pBuf, int32_t groupId, int32_t pageId) {
+  SIDList list = NULL;
+
+  char** p = taosHashGet(pBuf->groupSet, (const char*)&groupId, sizeof(int32_t));
+  if (p == NULL) {  // it is a new group id
+    list = addNewGroup(pBuf, groupId);
+  } else {
+    list = (SIDList) (*p);
+  }
+
+  pBuf->numOfPages += 1;
+
+  SPageInfo* ppi = malloc(sizeof(SPageInfo));//{ .info = PAGE_INFO_INITIALIZER, .pageId = pageId, .pn = NULL};
+
+  ppi->pageId = pageId;
+  ppi->pData  = NULL;
+  ppi->offset = -1;
+  ppi->length = -1;
+  ppi->used   = true;
+  ppi->pn     = NULL;
+
+  return *(SPageInfo**) taosArrayPush(list, &ppi);
+}
+
+static SListNode* getEldestUnrefedPage(SDiskbasedBuf* pBuf) {
+  SListIter iter = {0};
+  tdListInitIter(pBuf->lruList, &iter, TD_LIST_BACKWARD);
+
+  SListNode* pn = NULL;
+  while((pn = tdListNext(&iter)) != NULL) {
+    assert(pn != NULL);
+
+    SPageInfo* pageInfo = *(SPageInfo**) pn->data;
+    assert(pageInfo->pageId >= 0 && pageInfo->pn == pn);
+
+    if (!pageInfo->used) {
+      break;
+    }
+  }
+
+  return pn;
+}
+
+static char* evacOneDataPage(SDiskbasedBuf* pBuf) {
+  char* bufPage = NULL;
+  SListNode* pn = getEldestUnrefedPage(pBuf);
+
+  // all pages are referenced by user, try to allocate new space
+  if (pn == NULL) {
+    assert(0);
+    int32_t prev = pBuf->inMemPages;
+
+    // increase by 50% of previous mem pages
+    pBuf->inMemPages = (int32_t)(pBuf->inMemPages * 1.5f);
+
+//    qWarn("%p in memory buf page not sufficient, expand from %d to %d, page size:%d", pBuf, prev,
+//          pBuf->inMemPages, pBuf->pageSize);
+  } else {
+    tdListPopNode(pBuf->lruList, pn);
+
+    SPageInfo* d = *(SPageInfo**) pn->data;
+    assert(d->pn == pn);
+
+    d->pn = NULL;
+    tfree(pn);
+
+    bufPage = flushPageToDisk(pBuf, d);
+  }
+
+  return bufPage;
+}
+
+static void lruListPushFront(SList *pList, SPageInfo* pi) {
+  tdListPrepend(pList, &pi);
+  SListNode* front = tdListGetHead(pList);
+  pi->pn = front;
+}
+
+static void lruListMoveToFront(SList *pList, SPageInfo* pi) {
+  tdListPopNode(pList, pi->pn);
+  tdListPrependNode(pList, pi->pn);
+}
+
+static FORCE_INLINE size_t getAllocPageSize(int32_t pageSize) {
+  return pageSize + POINTER_BYTES + 2 + sizeof(SFilePage);
+}
+
+SFilePage* getNewDataBuf(SDiskbasedBuf* pBuf, int32_t groupId, int32_t* pageId) {
+  pBuf->statis.getPages += 1;
+
+  char* availablePage = NULL;
+  if (NO_IN_MEM_AVAILABLE_PAGES(pBuf)) {
+    availablePage = evacOneDataPage(pBuf);
+
+    // Failed to allocate a new buffer page, and there is an error occurs.
+    if (availablePage == NULL) {
+      return NULL;
+    }
+  }
+
+  // register new id in this group
+  *pageId = (++pBuf->allocateId);
+
+  // register page id info
+  SPageInfo* pi = registerPage(pBuf, groupId, *pageId);
+
+  // add to LRU list
+  assert(listNEles(pBuf->lruList) < pBuf->inMemPages && pBuf->inMemPages > 0);
+  lruListPushFront(pBuf->lruList, pi);
+
+  // add to hash map
+  taosHashPut(pBuf->all, pageId, sizeof(int32_t), &pi, POINTER_BYTES);
+
+  // allocate buf
+  if (availablePage == NULL) {
+    pi->pData = calloc(1, getAllocPageSize(pBuf->pageSize));  // add extract bytes in case of zipped buffer increased.
+  } else {
+    pi->pData = availablePage;
+  }
+
+  pBuf->totalBufSize += pBuf->pageSize;
+
+  ((void**)pi->pData)[0] = pi;
+  pi->used = true;
+
+  return (void *)(GET_DATA_PAYLOAD(pi));
+}
+
+SFilePage* getBufPage(SDiskbasedBuf* pBuf, int32_t id) {
+  assert(pBuf != NULL && id >= 0);
+  pBuf->statis.getPages += 1;
+
+  SPageInfo** pi = taosHashGet(pBuf->all, &id, sizeof(int32_t));
+  assert(pi != NULL && *pi != NULL);
+
+  if ((*pi)->pData != NULL) { // it is in memory
+    // no need to update the LRU list if only one page exists
+    if (pBuf->numOfPages == 1) {
+      (*pi)->used = true;
+      return (void *)(GET_DATA_PAYLOAD(*pi));
+    }
+
+    SPageInfo** pInfo = (SPageInfo**) ((*pi)->pn->data);
+    assert(*pInfo == *pi);
+
+    lruListMoveToFront(pBuf->lruList, (*pi));
+    (*pi)->used = true;
+
+    return (void *)(GET_DATA_PAYLOAD(*pi));
+
+  } else { // not in memory
+    assert((*pi)->pData == NULL && (*pi)->pn == NULL && (*pi)->length >= 0 && (*pi)->offset >= 0);
+
+    char* availablePage = NULL;
+    if (NO_IN_MEM_AVAILABLE_PAGES(pBuf)) {
+      availablePage = evacOneDataPage(pBuf);
+      if (availablePage == NULL) {
+        return NULL;
+      }
+    }
+
+    if (availablePage == NULL) {
+      (*pi)->pData = calloc(1, getAllocPageSize(pBuf->pageSize));
+    } else {
+      (*pi)->pData = availablePage;
+    }
+
+    ((void**)((*pi)->pData))[0] = (*pi);
+
+    lruListPushFront(pBuf->lruList, *pi);
+    (*pi)->used = true;
+
+    int32_t code = loadPageFromDisk(pBuf, *pi);
+    if (code != 0) {
+      return NULL;
+    }
+
+    return (void *)(GET_DATA_PAYLOAD(*pi));
+  }
+}
+
+void releaseBufPage(SDiskbasedBuf* pBuf, void* page) {
+  assert(pBuf != NULL && page != NULL);
+  int32_t offset = offsetof(SPageInfo, pData);
+  char* p = page - offset;
+
+  SPageInfo* ppi = ((SPageInfo**) p)[0];
+  releaseBufPageInfo(pBuf, ppi);
+}
+
+void releaseBufPageInfo(SDiskbasedBuf* pBuf, SPageInfo* pi) {
+  assert(pi->pData != NULL && pi->used);
+
+  pi->used = false;
+  pBuf->statis.releasePages += 1;
+}
+
+size_t getNumOfResultBufGroupId(const SDiskbasedBuf* pBuf) { return taosHashGetSize(pBuf->groupSet); }
+
+size_t getTotalBufSize(const SDiskbasedBuf* pBuf) { return (size_t)pBuf->totalBufSize; }
+
+SIDList getDataBufPagesIdList(SDiskbasedBuf* pBuf, int32_t groupId) {
+  assert(pBuf != NULL);
+
+  char** p = taosHashGet(pBuf->groupSet, (const char*)&groupId, sizeof(int32_t));
+  if (p == NULL) {  // it is a new group id
+    return pBuf->emptyDummyIdList;
+  } else {
+    return (SArray*) (*p);
+  }
+}
+
+void destroyResultBuf(SDiskbasedBuf* pBuf) {
+  if (pBuf == NULL) {
+    return;
+  }
+
+  printStatisData(pBuf);
+
+  if (pBuf->file != NULL) {
+  uDebug("Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page size:%.2f Kb, %"PRIx64"\n",
+      pBuf->totalBufSize/1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0,
+      listNEles(pBuf->lruList), pBuf->fileSize/1024.0, pBuf->pageSize/1024.0f, pBuf->qId);
+
+  fclose(pBuf->file);
+  } else {
+    uDebug("Paged buffer closed, total:%.2f Kb, no file created, %"PRIx64, pBuf->totalBufSize/1024.0, pBuf->qId);
+  }
+
+  // print the statistics information
+  {
+    SDiskbasedBufStatis *ps = &pBuf->statis;
+    uDebug("Get/Release pages:%d/%d, flushToDisk:%.2f Kb (%d Pages), loadFromDisk:%.2f Kb (%d Pages), avgPageSize:%.2f Kb\n"
+        , ps->getPages, ps->releasePages, ps->flushBytes/1024.0f, ps->flushPages, ps->loadBytes/1024.0f, ps->loadPages
+        , ps->loadBytes/(1024.0 * ps->loadPages));
+  }
+
+  remove(pBuf->path);
+  tfree(pBuf->path);
+
+  SArray** p = taosHashIterate(pBuf->groupSet, NULL);
+  while(p) {
+    size_t n = taosArrayGetSize(*p);
+    for(int32_t i = 0; i < n; ++i) {
+      SPageInfo* pi = taosArrayGetP(*p, i);
+      tfree(pi->pData);
+      tfree(pi);
+    }
+
+    taosArrayDestroy(*p);
+    p = taosHashIterate(pBuf->groupSet, p);
+  }
+
+  tdListFree(pBuf->lruList);
+  taosArrayDestroy(pBuf->emptyDummyIdList);
+  taosHashCleanup(pBuf->groupSet);
+  taosHashCleanup(pBuf->all);
+
+  tfree(pBuf->assistBuf);
+  tfree(pBuf);
+}
+
+SPageInfo* getLastPageInfo(SIDList pList) {
+  size_t size = taosArrayGetSize(pList);
+  SPageInfo* pPgInfo = taosArrayGetP(pList, size - 1);
+  return pPgInfo;
+}
+
+int32_t getPageId(const SPageInfo* pPgInfo) {
+  ASSERT(pPgInfo != NULL);
+  return pPgInfo->pageId;
+}
+
+int32_t getBufPageSize(const SDiskbasedBuf* pBuf) {
+  return pBuf->pageSize;
+}
+
+int32_t getNumOfInMemBufPages(const SDiskbasedBuf* pBuf) {
+  return pBuf->inMemPages;
+}
+
+bool isAllDataInMemBuf(const SDiskbasedBuf* pBuf) {
+  return pBuf->fileSize == 0;
+}
+
+void setBufPageDirty(SFilePage* pPage, bool dirty) {
+  int32_t offset = offsetof(SPageInfo, pData);  // todo extract method
+  char* p = (char*)pPage - offset;
+
+  SPageInfo* ppi = ((SPageInfo**) p)[0];
+  ppi->dirty = dirty;
+}
+
+void printStatisBeforeClose(SDiskbasedBuf* pBuf) {
+  pBuf->printStatis = true;
+}
+
+SDiskbasedBufStatis getDBufStatis(const SDiskbasedBuf* pBuf) {
+  return pBuf->statis;
+}
+
+void printStatisData(const SDiskbasedBuf* pBuf) {
+  if (!pBuf->printStatis) {
+    return;
+  }
+
+  const SDiskbasedBufStatis* ps = &pBuf->statis;
+
+  printf(
+      "Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page size:%.2f "
+      "Kb, %" PRIx64 "\n",
+      pBuf->totalBufSize / 1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0,
+      listNEles(pBuf->lruList), pBuf->fileSize / 1024.0, pBuf->pageSize / 1024.0f, pBuf->qId);
+
+  printf(
+      "Get/Release pages:%d/%d, flushToDisk:%.2f Kb (%d Pages), loadFromDisk:%.2f Kb (%d Pages), avgPageSize:%.2f Kb\n",
+      ps->getPages, ps->releasePages, ps->flushBytes / 1024.0f, ps->flushPages, ps->loadBytes / 1024.0f, ps->loadPages,
+      ps->loadBytes / (1024.0 * ps->loadPages));
+}

source/util/src/tpagedfile.c

@@ -1,468 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-
-#include "tpagedfile.h"
-#include "thash.h"
-#include "stddef.h"
-#include "taoserror.h"
-#include "tcompression.h"
-
-#define GET_DATA_PAYLOAD(_p) ((char *)(_p)->pData + POINTER_BYTES)
-#define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages)
-
-int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir) {
-  *pResultBuf = calloc(1, sizeof(SDiskbasedResultBuf));
-
-  SDiskbasedResultBuf* pResBuf = *pResultBuf;
-  if (pResBuf == NULL) {
-    return TSDB_CODE_OUT_OF_MEMORY;  
-  }
-
-  pResBuf->pageSize     = pagesize;
-  pResBuf->numOfPages   = 0;                        // all pages are in buffer in the first place
-  pResBuf->totalBufSize = 0;
-  pResBuf->inMemPages   = inMemBufSize/pagesize;    // maximum allowed pages, it is a soft limit.
-  pResBuf->allocateId   = -1;
-  pResBuf->comp         = true;
-  pResBuf->file         = NULL;
-  pResBuf->qId          = qId;
-  pResBuf->fileSize     = 0;
-
-  // at least more than 2 pages must be in memory
-  assert(inMemBufSize >= pagesize * 2);
-
-  pResBuf->lruList = tdListNew(POINTER_BYTES);
-
-  // init id hash table
-  pResBuf->groupSet  = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false);
-  pResBuf->assistBuf = malloc(pResBuf->pageSize + 2); // EXTRA BYTES
-  pResBuf->all = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false);
-
-  char path[PATH_MAX] = {0};
-  taosGetTmpfilePath(dir, "qbuf", path);
-  pResBuf->path = strdup(path);
-
-  pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t));
-
-//  qDebug("QInfo:0x%"PRIx64" create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", qId, pResBuf->pageSize,
-//         pResBuf->inMemPages, pResBuf->path);
-
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t createDiskFile(SDiskbasedResultBuf* pResultBuf) {
-  pResultBuf->file = fopen(pResultBuf->path, "wb+");
-  if (pResultBuf->file == NULL) {
-//    qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno));
-    return TAOS_SYSTEM_ERROR(errno);
-  }
-
-  return TSDB_CODE_SUCCESS;
-}
-
-static char* doCompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedResultBuf* pResultBuf) { // do nothing
-  if (!pResultBuf->comp) {
-    *dst = srcSize;
-    return data;
-  }
-
-  *dst = tsCompressString(data, srcSize, 1, pResultBuf->assistBuf, srcSize, ONE_STAGE_COMP, NULL, 0);
-
-  memcpy(data, pResultBuf->assistBuf, *dst);
-  return data;
-}
-
-static char* doDecompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedResultBuf* pResultBuf) { // do nothing
-  if (!pResultBuf->comp) {
-    *dst = srcSize;
-    return data;
-  }
-
-  *dst = tsDecompressString(data, srcSize, 1, pResultBuf->assistBuf, pResultBuf->pageSize, ONE_STAGE_COMP, NULL, 0);
-  if (*dst > 0) {
-    memcpy(data, pResultBuf->assistBuf, *dst);
-  }
-  return data;
-}
-
-static int32_t allocatePositionInFile(SDiskbasedResultBuf* pResultBuf, size_t size) {
-  if (pResultBuf->pFree == NULL) {
-    return pResultBuf->nextPos;
-  } else {
-    int32_t offset = -1;
-
-    size_t num = taosArrayGetSize(pResultBuf->pFree);
-    for(int32_t i = 0; i < num; ++i) {
-      SFreeListItem* pi = taosArrayGet(pResultBuf->pFree, i);
-      if (pi->len >= size) {
-        offset = pi->offset;
-        pi->offset += (int32_t)size;
-        pi->len -= (int32_t)size;
-
-        return offset;
-      }
-    }
-
-    // no available recycle space, allocate new area in file
-    return pResultBuf->nextPos;
-  }
-}
-
-static char* doFlushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
-  assert(!pg->used && pg->pData != NULL);
-
-  int32_t size = -1;
-  char* t = doCompressData(GET_DATA_PAYLOAD(pg), pResultBuf->pageSize, &size, pResultBuf);
-
-  // this page is flushed to disk for the first time
-  if (pg->info.offset == -1) {
-    pg->info.offset = allocatePositionInFile(pResultBuf, size);
-    pResultBuf->nextPos += size;
-
-    int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
-    assert(ret == 0);
-
-    ret = (int32_t) fwrite(t, 1, size, pResultBuf->file);
-    assert(ret == size);
-
-    if (pResultBuf->fileSize < pg->info.offset + pg->info.length) {
-      pResultBuf->fileSize = pg->info.offset + pg->info.length;
-    }
-  } else {
-    // length becomes greater, current space is not enough, allocate new place, otherwise, do nothing
-    if (pg->info.length < size) {
-      // 1. add current space to free list
-      taosArrayPush(pResultBuf->pFree, &pg->info);
-
-      // 2. allocate new position, and update the info
-      pg->info.offset = allocatePositionInFile(pResultBuf, size);
-      pResultBuf->nextPos += size;
-    }
-
-    //3. write to disk.
-    int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
-    if (ret != 0) {  // todo handle the error case
-
-    }
-
-    ret = (int32_t)fwrite(t, size, 1, pResultBuf->file);
-    if (ret != size) {  // todo handle the error case
-
-    }
-
-    if (pResultBuf->fileSize < pg->info.offset + pg->info.length) {
-      pResultBuf->fileSize = pg->info.offset + pg->info.length;
-    }
-  }
-
-  char* ret = pg->pData;
-  memset(ret, 0, pResultBuf->pageSize);
-
-  pg->pData = NULL;
-  pg->info.length = size;
-
-  pResultBuf->statis.flushBytes += pg->info.length;
-
-  return ret;
-}
-
-static char* flushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
-  int32_t ret = TSDB_CODE_SUCCESS;
-  assert(((int64_t) pResultBuf->numOfPages * pResultBuf->pageSize) == pResultBuf->totalBufSize && pResultBuf->numOfPages >= pResultBuf->inMemPages);
-
-  if (pResultBuf->file == NULL) {
-    if ((ret = createDiskFile(pResultBuf)) != TSDB_CODE_SUCCESS) {
-      terrno = ret;
-      return NULL;
-    }
-  }
-
-  return doFlushPageToDisk(pResultBuf, pg);
-}
-
-// load file block data in disk
-static char* loadPageFromDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
-  int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
-  ret = (int32_t)fread(GET_DATA_PAYLOAD(pg), 1, pg->info.length, pResultBuf->file);
-  if (ret != pg->info.length) {
-    terrno = errno;
-    return NULL;
-  }
-
-  pResultBuf->statis.loadBytes += pg->info.length;
-
-  int32_t fullSize = 0;
-  doDecompressData(GET_DATA_PAYLOAD(pg), pg->info.length, &fullSize, pResultBuf);
-
-  return (char*)GET_DATA_PAYLOAD(pg);
-}
-
-static SIDList addNewGroup(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
-  assert(taosHashGet(pResultBuf->groupSet, (const char*) &groupId, sizeof(int32_t)) == NULL);
-
-  SArray* pa = taosArrayInit(1, POINTER_BYTES);
-  int32_t ret = taosHashPut(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t), &pa, POINTER_BYTES);
-  assert(ret == 0);
-
-  return pa;
-}
-
-static SPageInfo* registerPage(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) {
-  SIDList list = NULL;
-
-  char** p = taosHashGet(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t));
-  if (p == NULL) {  // it is a new group id
-    list = addNewGroup(pResultBuf, groupId);
-  } else {
-    list = (SIDList) (*p);
-  }
-
-  pResultBuf->numOfPages += 1;
-
-  SPageInfo* ppi = malloc(sizeof(SPageInfo));//{ .info = PAGE_INFO_INITIALIZER, .pageId = pageId, .pn = NULL};
-
-  ppi->pageId = pageId;
-  ppi->pData  = NULL;
-  ppi->info   = PAGE_INFO_INITIALIZER;
-  ppi->used   = true;
-  ppi->pn     = NULL;
-
-  return *(SPageInfo**) taosArrayPush(list, &ppi);
-}
-
-static SListNode* getEldestUnrefedPage(SDiskbasedResultBuf* pResultBuf) {
-  SListIter iter = {0};
-  tdListInitIter(pResultBuf->lruList, &iter, TD_LIST_BACKWARD);
-
-  SListNode* pn = NULL;
-  while((pn = tdListNext(&iter)) != NULL) {
-    assert(pn != NULL);
-
-    SPageInfo* pageInfo = *(SPageInfo**) pn->data;
-    assert(pageInfo->pageId >= 0 && pageInfo->pn == pn);
-
-    if (!pageInfo->used) {
-      break;
-    }
-  }
-
-  return pn;
-}
-
-static char* evicOneDataPage(SDiskbasedResultBuf* pResultBuf) {
-  char* bufPage = NULL;
-  SListNode* pn = getEldestUnrefedPage(pResultBuf);
-
-  // all pages are referenced by user, try to allocate new space
-  if (pn == NULL) {
-    int32_t prev = pResultBuf->inMemPages;
-
-    // increase by 50% of previous mem pages
-    pResultBuf->inMemPages = (int32_t)(pResultBuf->inMemPages * 1.5f);
-
-//    qWarn("%p in memory buf page not sufficient, expand from %d to %d, page size:%d", pResultBuf, prev,
-//          pResultBuf->inMemPages, pResultBuf->pageSize);
-  } else {
-    pResultBuf->statis.flushPages += 1;
-    tdListPopNode(pResultBuf->lruList, pn);
-
-    SPageInfo* d = *(SPageInfo**) pn->data;
-    assert(d->pn == pn);
-
-    d->pn = NULL;
-    tfree(pn);
-
-    bufPage = flushPageToDisk(pResultBuf, d);
-  }
-
-  return bufPage;
-}
-
-static void lruListPushFront(SList *pList, SPageInfo* pi) {
-  tdListPrepend(pList, &pi);
-  SListNode* front = tdListGetHead(pList);
-  pi->pn = front;
-}
-
-static void lruListMoveToFront(SList *pList, SPageInfo* pi) {
-  tdListPopNode(pList, pi->pn);
-  tdListPrependNode(pList, pi->pn);
-}
-
-static FORCE_INLINE size_t getAllocPageSize(int32_t pageSize) {
-  return pageSize + POINTER_BYTES + 2 + sizeof(SFilePage);
-}
-
-SFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) {
-  pResultBuf->statis.getPages += 1;
-
-  char* availablePage = NULL;
-  if (NO_IN_MEM_AVAILABLE_PAGES(pResultBuf)) {
-    availablePage = evicOneDataPage(pResultBuf);
-  }
-
-  // register new id in this group
-  *pageId = (++pResultBuf->allocateId);
-
-  // register page id info
-  SPageInfo* pi = registerPage(pResultBuf, groupId, *pageId);
-
-  // add to LRU list
-  assert(listNEles(pResultBuf->lruList) < pResultBuf->inMemPages && pResultBuf->inMemPages > 0);
-
-  lruListPushFront(pResultBuf->lruList, pi);
-
-  // add to hash map
-  taosHashPut(pResultBuf->all, pageId, sizeof(int32_t), &pi, POINTER_BYTES);
-
-  // allocate buf
-  if (availablePage == NULL) {
-    pi->pData = calloc(1, getAllocPageSize(pResultBuf->pageSize));  // add extract bytes in case of zipped buffer increased.
-  } else {
-    pi->pData = availablePage;
-  }
-
-  pResultBuf->totalBufSize += pResultBuf->pageSize;
-
-  ((void**)pi->pData)[0] = pi;
-  pi->used = true;
-
-  return (void *)(GET_DATA_PAYLOAD(pi));
-}
-
-SFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) {
-  assert(pResultBuf != NULL && id >= 0);
-  pResultBuf->statis.getPages += 1;
-
-  SPageInfo** pi = taosHashGet(pResultBuf->all, &id, sizeof(int32_t));
-  assert(pi != NULL && *pi != NULL);
-
-  if ((*pi)->pData != NULL) { // it is in memory
-    // no need to update the LRU list if only one page exists
-    if (pResultBuf->numOfPages == 1) {
-      (*pi)->used = true;
-      return (void *)(GET_DATA_PAYLOAD(*pi));
-    }
-
-    SPageInfo** pInfo = (SPageInfo**) ((*pi)->pn->data);
-    assert(*pInfo == *pi);
-
-    lruListMoveToFront(pResultBuf->lruList, (*pi));
-    (*pi)->used = true;
-
-    return (void *)(GET_DATA_PAYLOAD(*pi));
-
-  } else { // not in memory
-    assert((*pi)->pData == NULL && (*pi)->pn == NULL && (*pi)->info.length >= 0 && (*pi)->info.offset >= 0);
-
-    char* availablePage = NULL;
-    if (NO_IN_MEM_AVAILABLE_PAGES(pResultBuf)) {
-      availablePage = evicOneDataPage(pResultBuf);
-    }
-
-    if (availablePage == NULL) {
-      (*pi)->pData = calloc(1, getAllocPageSize(pResultBuf->pageSize));
-    } else {
-      (*pi)->pData = availablePage;
-    }
-
-    ((void**)((*pi)->pData))[0] = (*pi);
-
-    lruListPushFront(pResultBuf->lruList, *pi);
-    (*pi)->used = true;
-
-    loadPageFromDisk(pResultBuf, *pi);
-    return (void *)(GET_DATA_PAYLOAD(*pi));
-  }
-}
-
-void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page) {
-  assert(pResultBuf != NULL && page != NULL);
-  char* p = (char*) page - POINTER_BYTES;
-
-  SPageInfo* ppi = ((SPageInfo**) p)[0];
-  releaseResBufPageInfo(pResultBuf, ppi);
-}
-
-void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi) {
-  assert(pi->pData != NULL && pi->used);
-
-  pi->used = false;
-  pResultBuf->statis.releasePages += 1;
-}
-
-size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->groupSet); }
-
-size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf) { return (size_t)pResultBuf->totalBufSize; }
-
-SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
-  assert(pResultBuf != NULL);
-
-  char** p = taosHashGet(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t));
-  if (p == NULL) {  // it is a new group id
-    return pResultBuf->emptyDummyIdList;
-  } else {
-    return (SArray*) (*p);
-  }
-}
-
-void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) {
-  if (pResultBuf == NULL) {
-    return;
-  }
-
-  if (pResultBuf->file != NULL) {
-//    qDebug("QInfo:0x%"PRIx64" res output buffer closed, total:%.2f Kb, inmem size:%.2f Kb, file size:%.2f Kb",
-//        pResultBuf->qId, pResultBuf->totalBufSize/1024.0, listNEles(pResultBuf->lruList) * pResultBuf->pageSize / 1024.0,
-//        pResultBuf->fileSize/1024.0);
-
-    fclose(pResultBuf->file);
-  } else {
-//    qDebug("QInfo:0x%"PRIx64" res output buffer closed, total:%.2f Kb, no file created", pResultBuf->qId,
-//           pResultBuf->totalBufSize/1024.0);
-  }
-
-  remove(pResultBuf->path);
-  tfree(pResultBuf->path);
-
-  SArray** p = taosHashIterate(pResultBuf->groupSet, NULL);
-  while(p) {
-    size_t n = taosArrayGetSize(*p);
-    for(int32_t i = 0; i < n; ++i) {
-      SPageInfo* pi = taosArrayGetP(*p, i);
-      tfree(pi->pData);
-      tfree(pi);
-    }
-
-    taosArrayDestroy(*p);
-    p = taosHashIterate(pResultBuf->groupSet, p);
-  }
-
-  tdListFree(pResultBuf->lruList);
-  taosArrayDestroy(pResultBuf->emptyDummyIdList);
-  taosHashCleanup(pResultBuf->groupSet);
-  taosHashCleanup(pResultBuf->all);
-
-  tfree(pResultBuf->assistBuf);
-  tfree(pResultBuf);
-}
-
-SPageInfo* getLastPageInfo(SIDList pList) {
-  size_t size = taosArrayGetSize(pList);
-  return (SPageInfo*) taosArrayGetP(pList, size - 1);
-}
-

source/util/test/pageBufferTest.cpp

@@ -0,0 +1,165 @@
+#include <gtest/gtest.h>
+#include <cassert>
+#include <iostream>
+
+#include "taos.h"
+#include "tpagedbuf.h"
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-function"
+#pragma GCC diagnostic ignored "-Wunused-variable"
+
+namespace {
+// simple test
+void simpleTest() {
+  SDiskbasedBuf* pResultBuf = NULL;
+  int32_t ret = createDiskbasedBuffer(&pResultBuf, 1024, 4096, 1, "/tmp/");
+  
+  int32_t pageId = 0;
+  int32_t groupId = 0;
+  
+  SFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
+  ASSERT_TRUE(pBufPage != NULL);
+  
+  ASSERT_EQ(getTotalBufSize(pResultBuf), 1024);
+  
+  SIDList list = getDataBufPagesIdList(pResultBuf, groupId);
+  ASSERT_EQ(taosArrayGetSize(list), 1);
+  ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1);
+
+  releaseBufPage(pResultBuf, pBufPage);
+
+  SFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId);
+
+  SFilePage* t = getBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t == pBufPage1);
+
+  SFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  SFilePage* t1 = getBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t1 == pBufPage2);
+
+  SFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  SFilePage* t2 = getBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t2 == pBufPage3);
+
+  SFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  SFilePage* t3 = getBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t3 == pBufPage4);
+
+  SFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  SFilePage* t4 = getBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t4 == pBufPage5);
+
+  destroyResultBuf(pResultBuf);
+}
+
+void writeDownTest() {
+  SDiskbasedBuf* pResultBuf = NULL;
+  int32_t ret = createDiskbasedBuffer(&pResultBuf, 1024, 4*1024, 1, "/tmp/");
+
+  int32_t pageId = 0;
+  int32_t writePageId = 0;
+  int32_t groupId = 0;
+  int32_t nx = 12345;
+
+  SFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
+  ASSERT_TRUE(pBufPage != NULL);
+
+  *(int32_t*)(pBufPage->data) = nx;
+  writePageId = pageId;
+  releaseBufPage(pResultBuf, pBufPage);
+
+  SFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  SFilePage* t1 = getBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t1 == pBufPage1);
+  ASSERT_TRUE(pageId == 1);
+
+  SFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  SFilePage* t2 = getBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t2 == pBufPage2);
+  ASSERT_TRUE(pageId == 2);
+
+  SFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  SFilePage* t3 = getBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t3 == pBufPage3);
+  ASSERT_TRUE(pageId == 3);
+
+  SFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  SFilePage* t4 = getBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t4 == pBufPage4);
+  ASSERT_TRUE(pageId == 4);
+  releaseBufPage(pResultBuf, t4);
+
+  // flush the written page to disk, and read it out again
+  SFilePage* pBufPagex = getBufPage(pResultBuf, writePageId);
+  ASSERT_EQ(*(int32_t*)pBufPagex->data, nx);
+
+  SArray* pa = getDataBufPagesIdList(pResultBuf, groupId);
+  ASSERT_EQ(taosArrayGetSize(pa), 5);
+
+  destroyResultBuf(pResultBuf);
+}
+
+void recyclePageTest() {
+  SDiskbasedBuf* pResultBuf = NULL;
+  int32_t ret = createDiskbasedBuffer(&pResultBuf, 1024, 4*1024, 1, "/tmp/");
+
+  int32_t pageId = 0;
+  int32_t writePageId = 0;
+  int32_t groupId = 0;
+  int32_t nx = 12345;
+
+  SFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
+  ASSERT_TRUE(pBufPage != NULL);
+  releaseBufPage(pResultBuf, pBufPage);
+
+  SFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  SFilePage* t1 = getBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t1 == pBufPage1);
+  ASSERT_TRUE(pageId == 1);
+
+  SFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  SFilePage* t2 = getBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t2 == pBufPage2);
+  ASSERT_TRUE(pageId == 2);
+
+  SFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  SFilePage* t3 = getBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t3 == pBufPage3);
+  ASSERT_TRUE(pageId == 3);
+
+  SFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  SFilePage* t4 = getBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t4 == pBufPage4);
+  ASSERT_TRUE(pageId == 4);
+  releaseBufPage(pResultBuf, t4);
+
+  SFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  SFilePage* t5 = getBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t5 == pBufPage5);
+  ASSERT_TRUE(pageId == 5);
+
+  // flush the written page to disk, and read it out again
+  SFilePage* pBufPagex = getBufPage(pResultBuf, writePageId);
+  *(int32_t*)(pBufPagex->data) = nx;
+  writePageId = pageId;   // update the data
+  releaseBufPage(pResultBuf, pBufPagex);
+
+  SFilePage* pBufPagex1 = getBufPage(pResultBuf, 1);
+
+  SArray* pa = getDataBufPagesIdList(pResultBuf, groupId);
+  ASSERT_EQ(taosArrayGetSize(pa), 6);
+
+  destroyResultBuf(pResultBuf);
+}
+} // namespace
+
+
+TEST(testCase, resultBufferTest) {
+  srand(time(NULL));
+  simpleTest();
+  writeDownTest();
+  recyclePageTest();
+}
+
+#pragma GCC diagnostic pop