enh: refactor unit test of parser and planner

example/src/tstream.c

@@ -78,11 +78,14 @@ int32_t create_stream() {
   taos_free_result(pRes);
 
   /*const char* sql = "select min(k), max(k), sum(k) from tu1";*/
-  const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";
+  /*const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";*/
   /*const char* sql = "select sum(k) from tu1 interval(10m)";*/
-  pRes = tmq_create_stream(pConn, "stream1", "out1", sql);
+  /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/
+  pRes = taos_query(
+      pConn,
+      "create stream stream1 trigger window_close as select min(k), max(k), sum(k) as sum_of_k from tu1 interval(10m)");
   if (taos_errno(pRes) != 0) {
-    printf("failed to create stream out1, reason:%s\n", taos_errstr(pRes));
+    printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes));
     return -1;
   }
   taos_free_result(pRes);

include/client/taos.h

@@ -86,9 +86,9 @@ typedef struct taosField {
 } TAOS_FIELD;
 
 #ifdef WINDOWS
-  #define DLL_EXPORT    __declspec(dllexport)
+#define DLL_EXPORT __declspec(dllexport)
 #else
-  #define DLL_EXPORT 
+#define DLL_EXPORT
 #endif
 
 typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code);
@@ -271,10 +271,8 @@ DLL_EXPORT int64_t     tmq_get_response_offset(tmq_message_t *message);
 /* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */
 #if 0
 DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen);
-#endif
-
 DLL_EXPORT TAOS_RES *tmq_create_stream(TAOS *taos, const char *streamName, const char *tbName, const char *sql);
-
+#endif
 /* ------------------------------ TMQ END -------------------------------- */
 #if 1  // Shuduo: temporary enable for app build
 typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code);

include/common/taosdef.h

@@ -82,6 +82,8 @@ extern char *qtypeStr[];
 
 #define TSDB_PORT_HTTP      11
 
+#undef TD_DEBUG_PRINT_ROW
+
 #ifdef __cplusplus
 }
 #endif

include/common/tmsg.h

@@ -531,27 +531,24 @@ int32_t tDeserializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pR
 typedef struct {
   char    db[TSDB_DB_FNAME_LEN];
   int32_t numOfVgroups;
-  int32_t cacheBlockSize;  // MB
+  int32_t numOfStables;  // single_stable
-  int32_t totalBlocks;
+  int32_t buffer;        // MB
+  int32_t pageSize;
+  int32_t pages;
   int32_t daysPerFile;
   int32_t daysToKeep0;
   int32_t daysToKeep1;
   int32_t daysToKeep2;
   int32_t minRows;
   int32_t maxRows;
-  int32_t commitTime;
   int32_t fsyncPeriod;
-  int32_t ttl;
   int8_t  walLevel;
   int8_t  precision;  // time resolution
   int8_t  compression;
   int8_t  replications;
   int8_t  strict;
-  int8_t  update;
   int8_t  cacheLastRow;
   int8_t  ignoreExist;
-  int8_t  streamMode;
-  int8_t  singleSTable;
   int32_t numOfRetensions;
   SArray* pRetensions;  // SRetention
 } SCreateDbReq;
@@ -562,7 +559,10 @@ void    tFreeSCreateDbReq(SCreateDbReq* pReq);
 
 typedef struct {
   char    db[TSDB_DB_FNAME_LEN];
-  int32_t totalBlocks;
+  int32_t buffer;
+  int32_t pageSize;
+  int32_t pages;
+  int32_t daysPerFile;
   int32_t daysToKeep0;
   int32_t daysToKeep1;
   int32_t daysToKeep2;
@@ -626,26 +626,23 @@ int32_t tDeserializeSDbCfgReq(void* buf, int32_t bufLen, SDbCfgReq* pReq);
 
 typedef struct {
   int32_t numOfVgroups;
-  int32_t cacheBlockSize;
+  int32_t numOfStables;
-  int32_t totalBlocks;
+  int32_t buffer;
+  int32_t pageSize;
+  int32_t pages;
   int32_t daysPerFile;
   int32_t daysToKeep0;
   int32_t daysToKeep1;
   int32_t daysToKeep2;
   int32_t minRows;
   int32_t maxRows;
-  int32_t commitTime;
   int32_t fsyncPeriod;
-  int32_t ttl;
   int8_t  walLevel;
   int8_t  precision;
   int8_t  compression;
   int8_t  replications;
   int8_t  strict;
-  int8_t  update;
   int8_t  cacheLastRow;
-  int8_t  streamMode;
-  int8_t  singleSTable;
   int32_t numOfRetensions;
   SArray* pRetensions;
 } SDbCfgRsp;
@@ -845,15 +842,16 @@ typedef struct {
   char     db[TSDB_DB_FNAME_LEN];
   int64_t  dbUid;
   int32_t  vgVersion;
-  int32_t  cacheBlockSize;
+   int32_t numOfStables;
-  int32_t  totalBlocks;
+  int32_t  buffer;
+  int32_t  pageSize;
+  int32_t  pages;
   int32_t  daysPerFile;
   int32_t  daysToKeep0;
   int32_t  daysToKeep1;
   int32_t  daysToKeep2;
   int32_t  minRows;
   int32_t  maxRows;
-  int32_t  commitTime;
   int32_t  fsyncPeriod;
   uint32_t hashBegin;
   uint32_t hashEnd;
@@ -862,11 +860,9 @@ typedef struct {
   int8_t   precision;
   int8_t   compression;
   int8_t   strict;
-  int8_t   update;
   int8_t   cacheLastRow;
   int8_t   replica;
   int8_t   selfIndex;
-  int8_t   streamMode;
   SReplica replicas[TSDB_MAX_REPLICA];
   int32_t  numOfRetensions;
   SArray*  pRetensions;  // SRetention
@@ -896,10 +892,14 @@ int32_t tDeserializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq
 
 typedef struct {
   int32_t  vgVersion;
-  int32_t  totalBlocks;
+  int32_t  buffer;
+  int32_t  pageSize;
+  int32_t  pages;
+  int32_t  daysPerFile;
   int32_t  daysToKeep0;
   int32_t  daysToKeep1;
   int32_t  daysToKeep2;
+  int32_t  fsyncPeriod;
   int8_t   walLevel;
   int8_t   strict;
   int8_t   cacheLastRow;
@@ -954,7 +954,6 @@ typedef struct {
   int32_t  numOfColumns;
   int8_t   precision;
   int8_t   tableType;
-  int8_t   update;
   int32_t  sversion;
   int32_t  tversion;
   uint64_t suid;
@@ -1281,8 +1280,9 @@ typedef struct {
 #define STREAM_TRIGGER_WINDOW_CLOSE 2
 
 typedef struct {
-  char    name[TSDB_TOPIC_FNAME_LEN];
+  char    name[TSDB_TABLE_FNAME_LEN];
-  char    outputSTbName[TSDB_TABLE_FNAME_LEN];
+  char    sourceDB[TSDB_DB_FNAME_LEN];
+  char    targetStbFullName[TSDB_TABLE_FNAME_LEN];
   int8_t  igExists;
   char*   sql;
   char*   ast;
@@ -1473,7 +1473,6 @@ _err:
 // this message is sent from mnode to mnode(read thread to write thread), so there is no need for serialization or
 // deserialization
 typedef struct {
-  int8_t*   mqInReb;
   SHashObj* rebSubHash;  // SHashObj<key, SMqRebSubscribe>
 } SMqDoRebalanceMsg;
 

include/common/trow.h

@@ -214,6 +214,7 @@ STSRow *tdRowDup(STSRow *row);
 static FORCE_INLINE SKvRowIdx *tdKvRowColIdxAt(STSRow *pRow, col_id_t idx) {
   return (SKvRowIdx *)TD_ROW_COL_IDX(pRow) + idx;
 }
+
 static FORCE_INLINE int16_t tdKvRowColIdAt(STSRow *pRow, col_id_t idx) {
   ASSERT(idx >= 0);
   if (idx == 0) {
@@ -222,6 +223,7 @@ static FORCE_INLINE int16_t tdKvRowColIdAt(STSRow *pRow, col_id_t idx) {
 
   return ((SKvRowIdx *)TD_ROW_COL_IDX(pRow) + idx - 1)->colId;
 }
+
 static FORCE_INLINE void *tdKVRowColVal(STSRow *pRow, SKvRowIdx *pIdx) { return POINTER_SHIFT(pRow, pIdx->offset); }
 
 #define TD_ROW_OFFSET(p) ((p)->toffset);  // During ParseInsert when without STSchema, how to get the offset for STpRow?
@@ -593,6 +595,34 @@ static FORCE_INLINE int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols,
   return TSDB_CODE_SUCCESS;
 }
 
+/**
+ * @brief
+ *
+ * @param pBuilder
+ * @param nCols
+ * @param nBoundCols use -1 if not available
+ * @param flen
+ * @return FORCE_INLINE
+ */
+static FORCE_INLINE int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen) {
+  pBuilder->flen = flen;
+  pBuilder->nCols = nCols;
+  if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) {
+    TASSERT(0);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+#ifdef TD_SUPPORT_BITMAP
+  // the primary TS key is stored separatedly
+  pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1);
+#else
+  pBuilder->nBitmaps = 0;
+  pBuilder->nBoundBitmaps = 0;
+#endif
+  return TSDB_CODE_SUCCESS;
+}
+
+
 /**
  * @brief To judge row type: STpRow/SKvRow
  *
@@ -1117,7 +1147,7 @@ static FORCE_INLINE bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId,
   }
 
   if (!colFound) {
-    if(colId <= pIter->maxColId) {
+    if (colId <= pIter->maxColId) {
       pVal->valType = TD_VTYPE_NONE;
       return true;
     } else {
@@ -1367,14 +1397,14 @@ static void tdSCellValPrint(SCellVal *pVal, int8_t colType) {
   }
 }
 
-static void tdSRowPrint(STSRow *row, STSchema *pSchema) {
+static void tdSRowPrint(STSRow *row, STSchema *pSchema, const char* tag) {
   STSRowIter iter = {0};
   tdSTSRowIterInit(&iter, pSchema);
   tdSTSRowIterReset(&iter, row);
-  printf(">>>");
+  printf("%s >>>", tag);
   for (int i = 0; i < pSchema->numOfCols; ++i) {
     STColumn *stCol = pSchema->columns + i;
-    SCellVal  sVal = { 255, NULL};
+    SCellVal  sVal = {.valType = 255, .val = NULL};
     if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) {
       break;
     }
@@ -1383,7 +1413,6 @@ static void tdSRowPrint(STSRow *row, STSchema *pSchema) {
   }
   printf("\n");
 }
-
 #ifdef TROW_ORIGIN_HZ
 typedef struct {
   uint32_t nRows;

include/dnode/mnode/sdb/sdb.h

@@ -331,7 +331,6 @@ int32_t  sdbGetRawSoftVer(SSdbRaw *pRaw, int8_t *sver);
 int32_t  sdbGetRawTotalSize(SSdbRaw *pRaw);
 
 SSdbRow *sdbAllocRow(int32_t objSize);
-void     sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc);
 void    *sdbGetRowObj(SSdbRow *pRow);
 
 typedef struct SSdb {

include/libs/function/function.h

@@ -206,6 +206,8 @@ typedef struct SqlFunctionCtx {
   struct SDiskbasedBuf  *pBuf;
   struct SSDataBlock    *pSrcBlock;
   int32_t                curBufPage;
+
+  char                   udfName[TSDB_FUNC_NAME_LEN];
 } SqlFunctionCtx;
 
 enum {
@@ -334,8 +336,6 @@ int32_t udfcOpen();
  */
 int32_t udfcClose();
 
-typedef void *UdfcFuncHandle;
-
 #ifdef __cplusplus
 }
 #endif

include/libs/function/functionMgt.h

@@ -162,6 +162,7 @@ EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWin
 
 int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet);
 int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet);
+int32_t fmGetUdafExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet);
 
 #ifdef __cplusplus
 }

include/libs/function/tudf.h

@@ -42,8 +42,7 @@ enum {
   UDFC_CODE_INVALID_STATE = -5
 };
 
-
+typedef void *UdfcFuncHandle;
-
 
 /**
  * setup udf
@@ -95,6 +94,7 @@ typedef struct SUdfDataBlock {
 typedef struct SUdfInterBuf {
   int32_t bufLen;
   char* buf;
+  int8_t numOfResult; //zero or one
 } SUdfInterBuf;
 
 // output: interBuf
@@ -118,6 +118,10 @@ int32_t callUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t nu
  */
 int32_t teardownUdf(UdfcFuncHandle handle);
 
+bool udfAggGetEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
+bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
+int32_t udfAggProcess(struct SqlFunctionCtx *pCtx);
+int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock);
 // end API to taosd and qworker
 //=============================================================================================================================
 // begin API to UDF writer.
@@ -133,11 +137,11 @@ typedef int32_t (*TUdfTeardownFunc)();
 //typedef int32_t addVariableLengthColumnData(SColumnData *columnData, int rowIndex, bool isNull, int32_t dataLen, char * data);
 
 typedef int32_t (*TUdfFreeUdfColumnFunc)(SUdfColumn* column);
+typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock* block, SUdfColumn *resultCol);
 
-typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock block, SUdfColumn *resultCol);
+typedef int32_t (*TUdfAggStartFunc)(SUdfInterBuf *buf);
-typedef int32_t (*TUdfAggInitFunc)(SUdfInterBuf *buf);
+typedef int32_t (*TUdfAggProcessFunc)(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf);
-typedef int32_t (*TUdfAggProcessFunc)(SUdfDataBlock block, SUdfInterBuf *interBuf);
+typedef int32_t (*TUdfAggFinishFunc)(SUdfInterBuf* buf, SUdfInterBuf *resultData);
-typedef int32_t (*TUdfAggFinalizeFunc)(SUdfInterBuf buf, SUdfInterBuf *resultData);
 
 
 // end API to UDF writer

include/libs/nodes/plannodes.h

@@ -340,7 +340,7 @@ typedef struct SQueryPlan {
   int32_t      numOfSubplans;
   SNodeList*   pSubplans;  // Element is SNodeListNode. The execution level of subplan, starting from 0.
   SExplainInfo explainInfo;
-  SNodeList*   pPlaceholderValues;
+  SArray* pPlaceholderValues;
 } SQueryPlan;
 
 void nodesWalkPhysiPlan(SNode* pNode, FNodeWalker walker, void* pContext);

include/libs/parser/parser.h

@@ -73,6 +73,7 @@ typedef struct SQuery {
   SArray* pDbList;
   SArray* pTableList;
   bool showRewrite;
+  int32_t placeholderNum;
 } SQuery;
 
 int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery);

include/libs/planner/planner.h

@@ -34,7 +34,7 @@ typedef struct SPlanContext {
   bool showRewrite;
   int8_t triggerType;
   int64_t watermark;
-  bool isStmtQuery;
+  int32_t placeholderNum;
   void* pTransporter;
   struct SCatalog* pCatalog;
   char* pMsg;
@@ -50,7 +50,7 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo
 // @pSource one execution location of this group of datasource subplans 
 int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstreamSourceNode* pSource);
 
-int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx);
+int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId);
 
 // Convert to subplan to string for the scheduler to send to the executor
 int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen);

include/libs/stream/tstream.h

@@ -35,19 +35,6 @@ enum {
   STREAM_CREATED_BY__SMA,
 };
 
-#if 0
-// pipe  -> fetch/pipe queue
-// merge -> merge      queue
-// write -> write      queue
-enum {
-  TASK_DISPATCH_MSG__SND_PIPE = 1,
-  TASK_DISPATCH_MSG__SND_MERGE,
-  TASK_DISPATCH_MSG__VND_PIPE,
-  TASK_DISPATCH_MSG__VND_MERGE,
-  TASK_DISPATCH_MSG__VND_WRITE,
-};
-#endif
-
 typedef struct {
   int32_t nodeId;  // 0 for snode
   SEpSet  epSet;
@@ -100,10 +87,6 @@ typedef struct {
   int8_t reserved;
 } STaskSinkFetch;
 
-typedef struct {
-  int8_t reserved;
-} STaskSinkShow;
-
 enum {
   TASK_SOURCE__SCAN = 1,
   TASK_SOURCE__PIPE,
@@ -128,7 +111,6 @@ enum {
   TASK_SINK__TABLE,
   TASK_SINK__SMA,
   TASK_SINK__FETCH,
-  TASK_SINK__SHOW,
 };
 
 typedef struct {
@@ -155,7 +137,6 @@ typedef struct {
     STaskSinkTb    tbSink;
     STaskSinkSma   smaSink;
     STaskSinkFetch fetchSink;
-    STaskSinkShow  showSink;
   };
 
   // dispatch

include/libs/stream/tstreamUpdate.h

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef _TSTREAMUPDATE_H_
+#define _TSTREAMUPDATE_H_
+
+#include "taosdef.h"
+#include "tarray.h"
+#include "tmsg.h"
+#include "tscalablebf.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SUpdateInfo {
+  SArray *pTsBuckets;
+  uint64_t numBuckets;
+  SArray *pTsSBFs;
+  uint64_t numSBFs;
+  int64_t interval;
+  int64_t watermark;
+  TSKEY minTS;
+} SUpdateInfo;
+
+SUpdateInfo *updateInfoInitP(SInterval* pInterval, int64_t watermark);
+SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark);
+bool isUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts);
+void updateInfoDestroy(SUpdateInfo *pInfo);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* ifndef _TSTREAMUPDATE_H_ */

include/util/taoserror.h

@@ -244,9 +244,10 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_TOO_MANY_COLUMNS          TAOS_DEF_ERROR_CODE(0, 0x03AC)
 #define TSDB_CODE_MND_COLUMN_ALREADY_EXIST      TAOS_DEF_ERROR_CODE(0, 0x03AD)
 #define TSDB_CODE_MND_COLUMN_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03AE)
+#define TSDB_CODE_MND_SINGLE_STB_MODE_DB        TAOS_DEF_ERROR_CODE(0, 0x03B0)
 
 // mnode-infoSchema
-#define TSDB_CODE_MND_INVALID_SYS_TABLENAME         TAOS_DEF_ERROR_CODE(0, 0x03B0)
+#define TSDB_CODE_MND_INVALID_SYS_TABLENAME         TAOS_DEF_ERROR_CODE(0, 0x03BA)
 
 // mnode-func
 #define TSDB_CODE_MND_FUNC_ALREADY_EXIST        TAOS_DEF_ERROR_CODE(0, 0x03C0)

include/util/tarray.h

@@ -218,6 +218,13 @@ void taosArrayClear(SArray* pArray);
  */
 void taosArrayClearEx(SArray* pArray, void (*fp)(void*));
 
+/**
+ * clear the array (remove all element)
+ * @param pArray
+ * @param fp
+ */
+void taosArrayClearP(SArray* pArray, FDelete fp);
+
 void* taosArrayDestroy(SArray* pArray);
 void  taosArrayDestroyP(SArray* pArray, FDelete fp);
 void  taosArrayDestroyEx(SArray* pArray, FDelete fp);

include/util/tbloomfilter.h

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_UTIL_BLOOMFILTER_H_
+#define _TD_UTIL_BLOOMFILTER_H_
+
+#include "os.h"
+#include "thash.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SBloomFilter {
+  uint32_t hashFunctions;
+  uint64_t expectedEntries;
+  uint64_t numUnits;
+  uint64_t numBits;
+  uint64_t size;
+  _hash_fn_t hashFn1;
+  _hash_fn_t hashFn2;
+  void *buffer;
+  double errorRate;
+} SBloomFilter;
+
+SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate);
+int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len);
+int32_t tBloomFilterNoContain(const SBloomFilter *pBF, const void *keyBuf,
+                              uint32_t len);
+void tBloomFilterDestroy(SBloomFilter *pBF);
+void tBloomFilterDump(const SBloomFilter *pBF);
+bool tBloomFilterIsFull(const SBloomFilter *pBF);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_UTIL_BLOOMFILTER_H_*/

include/util/tdef.h

@@ -132,6 +132,7 @@ extern const int32_t TYPE_BYTES[15];
 #define TSDB_PERFS_TABLE_CONSUMERS     "consumers"
 #define TSDB_PERFS_TABLE_SUBSCRIPTIONS "subscriptions"
 #define TSDB_PERFS_TABLE_OFFSETS       "offsets"
+#define TSDB_PERFS_TABLE_STREAMS       "streams"
 
 #define TSDB_INDEX_TYPE_SMA      "SMA"
 #define TSDB_INDEX_TYPE_FULLTEXT "FULLTEXT"
@@ -326,15 +327,21 @@ typedef enum ELogicConditionType {
 #define TSDB_MIN_VNODES_PER_DB          1
 #define TSDB_MAX_VNODES_PER_DB          4096
 #define TSDB_DEFAULT_VN_PER_DB          2
-#define TSDB_MIN_CACHE_BLOCK_SIZE       1
+#define TSDB_MIN_BUFFER_PER_VNODE       3      // unit MB
-#define TSDB_MAX_CACHE_BLOCK_SIZE       128  // 128MB for each vnode
+#define TSDB_MAX_BUFFER_PER_VNODE       16384  // unit MB
-#define TSDB_DEFAULT_CACHE_BLOCK_SIZE   16
+#define TSDB_DEFAULT_BUFFER_PER_VNODE   96
-#define TSDB_MIN_TOTAL_BLOCKS           3
+#define TSDB_MIN_PAGES_PER_VNODE        64
-#define TSDB_MAX_TOTAL_BLOCKS           10000
+#define TSDB_MAX_PAGES_PER_VNODE        16384
-#define TSDB_DEFAULT_TOTAL_BLOCKS       6
+#define TSDB_DEFAULT_PAGES_PER_VNODE    256
+#define TSDB_MIN_PAGESIZE_PER_VNODE     1  // unit KB
+#define TSDB_MAX_PAGESIZE_PER_VNODE     16384
+#define TSDB_DEFAULT_PAGESIZE_PER_VNODE 4
 #define TSDB_MIN_DAYS_PER_FILE          60  // unit minute
 #define TSDB_MAX_DAYS_PER_FILE          (3650 * 1440)
 #define TSDB_DEFAULT_DAYS_PER_FILE      (10 * 1440)
+#define TSDB_MIN_DURATION_PER_FILE      60  // unit minute
+#define TSDB_MAX_DURATION_PER_FILE      (3650 * 1440)
+#define TSDB_DEFAULT_DURATION_PER_FILE  (10 * 1440)
 #define TSDB_MIN_KEEP                   (1 * 1440)       // data in db to be reserved. unit minute
 #define TSDB_MAX_KEEP                   (365000 * 1440)  // data in db to be reserved.
 #define TSDB_DEFAULT_KEEP               (3650 * 1440)    // ten years
@@ -344,9 +351,6 @@ typedef enum ELogicConditionType {
 #define TSDB_MIN_MAXROWS_FBLOCK         200
 #define TSDB_MAX_MAXROWS_FBLOCK         10000
 #define TSDB_DEFAULT_MAXROWS_FBLOCK     4096
-#define TSDB_MIN_COMMIT_TIME            30
-#define TSDB_MAX_COMMIT_TIME            40960
-#define TSDB_DEFAULT_COMMIT_TIME        3600
 #define TSDB_MIN_FSYNC_PERIOD           0
 #define TSDB_MAX_FSYNC_PERIOD           180000  // millisecond
 #define TSDB_DEFAULT_FSYNC_PERIOD       3000    // three second
@@ -365,9 +369,6 @@ typedef enum ELogicConditionType {
 #define TSDB_DB_STRICT_OFF              0
 #define TSDB_DB_STRICT_ON               1
 #define TSDB_DEFAULT_DB_STRICT          0
-#define TSDB_MIN_DB_UPDATE              0
-#define TSDB_MAX_DB_UPDATE              2
-#define TSDB_DEFAULT_DB_UPDATE          0
 #define TSDB_MIN_DB_CACHE_LAST_ROW      0
 #define TSDB_MAX_DB_CACHE_LAST_ROW      3
 #define TSDB_DEFAULT_CACHE_LAST_ROW     0
@@ -377,13 +378,6 @@ typedef enum ELogicConditionType {
 #define TSDB_DB_SINGLE_STABLE_ON        0
 #define TSDB_DB_SINGLE_STABLE_OFF       1
 #define TSDB_DEFAULT_DB_SINGLE_STABLE   0
-#define TSDB_MIN_BUFFER_PER_VNODE       3  // unit MB
-#define TSDB_DEFAULT_BUFFER_PER_VNODE   96
-#define TSDB_MIN_PAGES_PER_VNODE        64
-#define TSDB_DEFAULT_PAGES_PER_VNODE    256
-#define TSDB_MIN_PAGESIZE_PER_VNODE     1  // unit KB
-#define TSDB_MAX_PAGESIZE_PER_VNODE     16384
-#define TSDB_DEFAULT_PAGESIZE_PER_VNODE 4
 
 #define TSDB_MIN_ROLLUP_FILE_FACTOR     0
 #define TSDB_MAX_ROLLUP_FILE_FACTOR     1

include/util/tscalablebf.h

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_UTIL_SCALABLEBF_H_
+#define _TD_UTIL_SCALABLEBF_H_
+
+#include "tbloomfilter.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SScalableBf {
+  SArray *bfArray; // array of bloom filters
+  uint32_t growth;
+  uint64_t numBits;
+} SScalableBf;
+
+SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate);
+int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len);
+int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf,
+                             uint32_t len);
+void tScalableBfDestroy(SScalableBf *pSBf);
+void tScalableBfDump(const SScalableBf *pSBf);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_UTIL_SCALABLEBF_H_*/

source/client/inc/clientInt.h

@@ -188,7 +188,6 @@ typedef struct SRequestSendRecvBody {
 
 typedef struct {
   int8_t         resType;
-  int32_t        code;
   char           topic[TSDB_TOPIC_FNAME_LEN];
   int32_t        vgId;
   SSchemaWrapper schema;

source/client/inc/clientStmt.h

@@ -46,6 +46,12 @@ typedef struct SStmtTableCache {
   void*             boundTags;
 } SStmtTableCache;
 
+typedef struct SQueryFields {
+  TAOS_FIELD*    fields;
+  TAOS_FIELD*    userFields;
+  uint32_t       numOfCols;
+} SQueryFields;
+
 typedef struct SStmtBindInfo {
   bool         needParse;
   uint64_t     tbUid;
@@ -76,6 +82,7 @@ typedef struct SStmtSQLInfo {
   int32_t       sqlLen;
   SArray*       nodeList;
   SQueryPlan*   pQueryPlan;
+  SQueryFields  fields;
 } SStmtSQLInfo;
 
 typedef struct STscStmt {

source/client/src/clientImpl.c

@@ -234,7 +234,8 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra
                       .showRewrite = pQuery->showRewrite,
                       .pTransporter = pRequest->pTscObj->pAppInfo->pTransporter,
                       .pMsg = pRequest->msgBuf,
-                      .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE};
+                      .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE,
+                      .placeholderNum = pQuery->placeholderNum};
   int32_t      code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &cxt.pCatalog);
   if (TSDB_CODE_SUCCESS == code) {
     code = qCreateQueryPlan(&cxt, pPlan, pNodeList);

source/client/src/clientStmt.c

@@ -73,6 +73,22 @@ int32_t stmtGetTbName(TAOS_STMT *stmt, char **tbName) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t stmtBackupQueryFields(STscStmt* pStmt) {
+  SQueryFields *pFields = &pStmt->sql.fields;
+  int32_t size = pFields->numOfCols * sizeof(TAOS_FIELD);
+  
+  pFields->numOfCols = pStmt->exec.pRequest->body.resInfo.numOfCols;
+  pFields->fields = taosMemoryMalloc(size);
+  pFields->userFields = taosMemoryMalloc(size);
+  if (NULL == pFields->fields || NULL == pFields->userFields) {
+    STMT_ERR_RET(TSDB_CODE_TSC_OUT_OF_MEMORY);
+  }
+  memcpy(pFields->fields, pStmt->exec.pRequest->body.resInfo.fields, size);
+  memcpy(pFields->userFields, pStmt->exec.pRequest->body.resInfo.userFields, size);
+
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t stmtSetBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
@@ -258,37 +274,42 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
   STableMeta *pTableMeta = NULL;
   SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp);
   STMT_ERR_RET(catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta));
+  uint64_t uid = pTableMeta->uid;
+  uint64_t suid = pTableMeta->suid;
+  int8_t tableType = pTableMeta->tableType;
+  taosMemoryFree(pTableMeta);
   
-  if (pTableMeta->uid == pStmt->bInfo.tbUid) {
+  if (uid == pStmt->bInfo.tbUid) {
     pStmt->bInfo.needParse = false;
 
     return TSDB_CODE_SUCCESS;
   }
 
-  if (taosHashGet(pStmt->exec.pBlockHash, &pTableMeta->uid, sizeof(pTableMeta->uid))) {
+  if (taosHashGet(pStmt->exec.pBlockHash, &uid, sizeof(uid))) {
-    SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pTableMeta->uid, sizeof(pTableMeta->uid));
+    SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &uid, sizeof(uid));
     if (NULL == pCache) {
-      tscError("table uid %" PRIx64 "found in exec blockHash, but not in sql blockHash", pTableMeta->uid);
+      tscError("table uid %" PRIx64 "found in exec blockHash, but not in sql blockHash", uid);
+      
       STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR);
     }
     
     pStmt->bInfo.needParse = false;
     
-    pStmt->bInfo.tbUid = pTableMeta->uid;
+    pStmt->bInfo.tbUid = uid;
-    pStmt->bInfo.tbSuid = pTableMeta->suid;
+    pStmt->bInfo.tbSuid = suid;
-    pStmt->bInfo.tbType = pTableMeta->tableType;
+    pStmt->bInfo.tbType = tableType;
     pStmt->bInfo.boundTags = pCache->boundTags;
 
     return TSDB_CODE_SUCCESS;
   }
 
-  SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pTableMeta->uid, sizeof(pTableMeta->uid));
+  SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &uid, sizeof(uid));
   if (pCache) {
     pStmt->bInfo.needParse = false;
 
-    pStmt->bInfo.tbUid = pTableMeta->uid;
+    pStmt->bInfo.tbUid = uid;
-    pStmt->bInfo.tbSuid = pTableMeta->suid;
+    pStmt->bInfo.tbSuid = suid;
-    pStmt->bInfo.tbType = pTableMeta->tableType;
+    pStmt->bInfo.tbType = tableType;
     pStmt->bInfo.boundTags = pCache->boundTags;
 
     STableDataBlocks* pNewBlock = NULL;
@@ -475,9 +496,10 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx) {
       STMT_ERR_RET(getQueryPlan(pStmt->exec.pRequest, pStmt->sql.pQuery, &pStmt->sql.nodeList));
       pStmt->sql.pQueryPlan = pStmt->exec.pRequest->body.pDag;
       pStmt->exec.pRequest->body.pDag = NULL;
+      STMT_ERR_RET(stmtBackupQueryFields(pStmt));
     }
     
-    STMT_RET(qStmtBindParam(pStmt->sql.pQueryPlan, bind, colIdx));
+    STMT_RET(qStmtBindParam(pStmt->sql.pQueryPlan, bind, colIdx, pStmt->exec.pRequest->requestId));
   }
   
   STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid));
@@ -549,6 +571,8 @@ int stmtClose(TAOS_STMT *stmt) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
   STMT_RET(stmtCleanSQLInfo(pStmt));
+
+  taosMemoryFree(stmt);
 }
 
 const char *stmtErrstr(TAOS_STMT *stmt) {
@@ -601,7 +625,7 @@ int stmtGetParamNum(TAOS_STMT *stmt, int *nums) {
       pStmt->exec.pRequest->body.pDag = NULL;
     }
 
-    *nums = (pStmt->sql.pQueryPlan->pPlaceholderValues) ? pStmt->sql.pQueryPlan->pPlaceholderValues->length : 0;
+    *nums = taosArrayGetSize(pStmt->sql.pQueryPlan->pPlaceholderValues);
   } else {
     STMT_ERR_RET(stmtFetchColFields(stmt, nums, NULL));
   }

source/client/src/tmq.c

@@ -667,7 +667,7 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
   if (code != 0) goto FAIL;
 
   while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, false)) {
-    tscDebug("not ready, retry");
+    tscDebug("consumer not ready, retry");
     taosMsleep(500);
   }
 
@@ -693,6 +693,7 @@ void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) {
   conf->commitCb = cb;
 }
 
+#if 0
 TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbName, const char* sql) {
   STscObj*     pTscObj = (STscObj*)taos;
   SRequestObj* pRequest = NULL;
@@ -739,7 +740,7 @@ TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbNa
       .sql = (char*)sql,
   };
   tNameExtractFullName(&name, req.name);
-  strcpy(req.outputSTbName, tbName);
+  strcpy(req.targetStbFullName, tbName);
 
   int   tlen = tSerializeSCMCreateStreamReq(NULL, 0, &req);
   void* buf = taosMemoryMalloc(tlen);
@@ -777,6 +778,7 @@ _return:
 
   return pRequest;
 }
+#endif
 
 #if 0
 int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) {

source/common/src/tmsg.c

@@ -1676,27 +1676,24 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) {
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->numOfVgroups) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->cacheBlockSize) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->numOfStables) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->totalBlocks) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->buffer) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->pageSize) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->pages) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->daysPerFile) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->daysToKeep2) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->minRows) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->maxRows) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->commitTime) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->precision) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->compression) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->replications) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->strict) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->update) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->ignoreExist) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->streamMode) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->singleSTable) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->numOfRetensions) < 0) return -1;
   for (int32_t i = 0; i < pReq->numOfRetensions; ++i) {
     SRetention *pRetension = taosArrayGet(pReq->pRetensions, i);
@@ -1719,27 +1716,24 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq)
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->numOfVgroups) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->cacheBlockSize) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->numOfStables) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->totalBlocks) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->buffer) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->pageSize) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->pages) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->daysPerFile) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->daysToKeep2) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->minRows) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->maxRows) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->commitTime) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->ttl) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->precision) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->compression) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->replications) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->update) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->ignoreExist) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->streamMode) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->singleSTable) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->numOfRetensions) < 0) return -1;
   pReq->pRetensions = taosArrayInit(pReq->numOfRetensions, sizeof(SRetention));
   if (pReq->pRetensions == NULL) {
@@ -1776,7 +1770,10 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
 
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->totalBlocks) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->buffer) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->pageSize) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->pages) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->daysPerFile) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->daysToKeep2) < 0) return -1;
@@ -1798,7 +1795,10 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
 
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->totalBlocks) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->buffer) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->pageSize) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->pages) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->daysPerFile) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->daysToKeep2) < 0) return -1;
@@ -2153,24 +2153,23 @@ int32_t tSerializeSDbCfgRsp(void *buf, int32_t bufLen, const SDbCfgRsp *pRsp) {
 
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeI32(&encoder, pRsp->numOfVgroups) < 0) return -1;
-  if (tEncodeI32(&encoder, pRsp->cacheBlockSize) < 0) return -1;
+  if (tEncodeI32(&encoder, pRsp->numOfStables) < 0) return -1;
-  if (tEncodeI32(&encoder, pRsp->totalBlocks) < 0) return -1;
+  if (tEncodeI32(&encoder, pRsp->buffer) < 0) return -1;
+  if (tEncodeI32(&encoder, pRsp->pageSize) < 0) return -1;
+  if (tEncodeI32(&encoder, pRsp->pages) < 0) return -1;
   if (tEncodeI32(&encoder, pRsp->daysPerFile) < 0) return -1;
   if (tEncodeI32(&encoder, pRsp->daysToKeep0) < 0) return -1;
   if (tEncodeI32(&encoder, pRsp->daysToKeep1) < 0) return -1;
   if (tEncodeI32(&encoder, pRsp->daysToKeep2) < 0) return -1;
   if (tEncodeI32(&encoder, pRsp->minRows) < 0) return -1;
   if (tEncodeI32(&encoder, pRsp->maxRows) < 0) return -1;
-  if (tEncodeI32(&encoder, pRsp->commitTime) < 0) return -1;
   if (tEncodeI32(&encoder, pRsp->fsyncPeriod) < 0) return -1;
   if (tEncodeI8(&encoder, pRsp->walLevel) < 0) return -1;
   if (tEncodeI8(&encoder, pRsp->precision) < 0) return -1;
   if (tEncodeI8(&encoder, pRsp->compression) < 0) return -1;
   if (tEncodeI8(&encoder, pRsp->replications) < 0) return -1;
   if (tEncodeI8(&encoder, pRsp->strict) < 0) return -1;
-  if (tEncodeI8(&encoder, pRsp->update) < 0) return -1;
   if (tEncodeI8(&encoder, pRsp->cacheLastRow) < 0) return -1;
-  if (tEncodeI8(&encoder, pRsp->streamMode) < 0) return -1;
   if (tEncodeI32(&encoder, pRsp->numOfRetensions) < 0) return -1;
   for (int32_t i = 0; i < pRsp->numOfRetensions; ++i) {
     SRetention *pRetension = taosArrayGet(pRsp->pRetensions, i);
@@ -2192,24 +2191,23 @@ int32_t tDeserializeSDbCfgRsp(void *buf, int32_t bufLen, SDbCfgRsp *pRsp) {
 
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeI32(&decoder, &pRsp->numOfVgroups) < 0) return -1;
-  if (tDecodeI32(&decoder, &pRsp->cacheBlockSize) < 0) return -1;
+  if (tDecodeI32(&decoder, &pRsp->numOfStables) < 0) return -1;
-  if (tDecodeI32(&decoder, &pRsp->totalBlocks) < 0) return -1;
+  if (tDecodeI32(&decoder, &pRsp->buffer) < 0) return -1;
+  if (tDecodeI32(&decoder, &pRsp->pageSize) < 0) return -1;
+  if (tDecodeI32(&decoder, &pRsp->pages) < 0) return -1;
   if (tDecodeI32(&decoder, &pRsp->daysPerFile) < 0) return -1;
   if (tDecodeI32(&decoder, &pRsp->daysToKeep0) < 0) return -1;
   if (tDecodeI32(&decoder, &pRsp->daysToKeep1) < 0) return -1;
   if (tDecodeI32(&decoder, &pRsp->daysToKeep2) < 0) return -1;
   if (tDecodeI32(&decoder, &pRsp->minRows) < 0) return -1;
   if (tDecodeI32(&decoder, &pRsp->maxRows) < 0) return -1;
-  if (tDecodeI32(&decoder, &pRsp->commitTime) < 0) return -1;
   if (tDecodeI32(&decoder, &pRsp->fsyncPeriod) < 0) return -1;
   if (tDecodeI8(&decoder, &pRsp->walLevel) < 0) return -1;
   if (tDecodeI8(&decoder, &pRsp->precision) < 0) return -1;
   if (tDecodeI8(&decoder, &pRsp->compression) < 0) return -1;
   if (tDecodeI8(&decoder, &pRsp->replications) < 0) return -1;
   if (tDecodeI8(&decoder, &pRsp->strict) < 0) return -1;
-  if (tDecodeI8(&decoder, &pRsp->update) < 0) return -1;
   if (tDecodeI8(&decoder, &pRsp->cacheLastRow) < 0) return -1;
-  if (tDecodeI8(&decoder, &pRsp->streamMode) < 0) return -1;
   if (tDecodeI32(&decoder, &pRsp->numOfRetensions) < 0) return -1;
   pRsp->pRetensions = taosArrayInit(pRsp->numOfRetensions, sizeof(SRetention));
   if (pRsp->pRetensions == NULL) {
@@ -2368,7 +2366,6 @@ static int32_t tEncodeSTableMetaRsp(SCoder *pEncoder, STableMetaRsp *pRsp) {
   if (tEncodeI32(pEncoder, pRsp->numOfColumns) < 0) return -1;
   if (tEncodeI8(pEncoder, pRsp->precision) < 0) return -1;
   if (tEncodeI8(pEncoder, pRsp->tableType) < 0) return -1;
-  if (tEncodeI8(pEncoder, pRsp->update) < 0) return -1;
   if (tEncodeI32(pEncoder, pRsp->sversion) < 0) return -1;
   if (tEncodeI32(pEncoder, pRsp->tversion) < 0) return -1;
   if (tEncodeU64(pEncoder, pRsp->suid) < 0) return -1;
@@ -2391,7 +2388,6 @@ static int32_t tDecodeSTableMetaRsp(SCoder *pDecoder, STableMetaRsp *pRsp) {
   if (tDecodeI32(pDecoder, &pRsp->numOfColumns) < 0) return -1;
   if (tDecodeI8(pDecoder, &pRsp->precision) < 0) return -1;
   if (tDecodeI8(pDecoder, &pRsp->tableType) < 0) return -1;
-  if (tDecodeI8(pDecoder, &pRsp->update) < 0) return -1;
   if (tDecodeI32(pDecoder, &pRsp->sversion) < 0) return -1;
   if (tDecodeI32(pDecoder, &pRsp->tversion) < 0) return -1;
   if (tDecodeU64(pDecoder, &pRsp->suid) < 0) return -1;
@@ -2811,15 +2807,16 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR
   if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
   if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->cacheBlockSize) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->numOfStables) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->totalBlocks) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->buffer) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->pageSize) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->pages) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->daysPerFile) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->daysToKeep2) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->minRows) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->maxRows) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->commitTime) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1;
   if (tEncodeU32(&encoder, pReq->hashBegin) < 0) return -1;
   if (tEncodeU32(&encoder, pReq->hashEnd) < 0) return -1;
@@ -2828,11 +2825,9 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR
   if (tEncodeI8(&encoder, pReq->precision) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->compression) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->strict) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->update) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->replica) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->selfIndex) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->streamMode) < 0) return -1;
   for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) {
     SReplica *pReplica = &pReq->replicas[i];
     if (tEncodeSReplica(&encoder, pReplica) < 0) return -1;
@@ -2862,15 +2857,16 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
   if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
   if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->cacheBlockSize) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->numOfStables) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->totalBlocks) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->buffer) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->pageSize) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->pages) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->daysPerFile) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->daysToKeep2) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->minRows) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->maxRows) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->commitTime) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1;
   if (tDecodeU32(&decoder, &pReq->hashBegin) < 0) return -1;
   if (tDecodeU32(&decoder, &pReq->hashEnd) < 0) return -1;
@@ -2879,11 +2875,9 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
   if (tDecodeI8(&decoder, &pReq->precision) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->compression) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->update) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->replica) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->selfIndex) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->streamMode) < 0) return -1;
   for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) {
     SReplica *pReplica = &pReq->replicas[i];
     if (tDecodeSReplica(&decoder, pReplica) < 0) return -1;
@@ -2983,10 +2977,14 @@ int32_t tSerializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pReq
 
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->totalBlocks) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->buffer) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->pageSize) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->pages) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->daysPerFile) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->daysToKeep2) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->strict) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1;
@@ -3010,10 +3008,14 @@ int32_t tDeserializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pR
 
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->totalBlocks) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->buffer) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->pageSize) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->pages) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->daysPerFile) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->daysToKeep2) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1;
@@ -3569,10 +3571,12 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
 
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
-  if (tEncodeCStr(&encoder, pReq->outputSTbName) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->targetStbFullName) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1;
   if (tEncodeI32(&encoder, sqlLen) < 0) return -1;
   if (tEncodeI32(&encoder, astLen) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->triggerType) < 0) return -1;
+  if (tEncodeI64(&encoder, pReq->watermark) < 0) return -1;
   if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
   if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1;
 
@@ -3592,7 +3596,7 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea
 
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
-  if (tDecodeCStrTo(&decoder, pReq->outputSTbName) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->targetStbFullName) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1;
   if (tDecodeI32(&decoder, &sqlLen) < 0) return -1;
   if (tDecodeI32(&decoder, &astLen) < 0) return -1;

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -107,13 +107,11 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
 
   pCfg->vgId = pCreate->vgId;
   strcpy(pCfg->dbname, pCreate->db);
-  // pCfg->szBuf = pCreate->cacheBlockSize * 1024 * 1024;
-  pCfg->streamMode = pCreate->streamMode;
   pCfg->isWeak = true;
   pCfg->tsdbCfg.days = 10;
-  pCfg->tsdbCfg.keep2 = 3650;  // pCreate->daysToKeep0;
+  pCfg->tsdbCfg.keep2 = 3650;
-  pCfg->tsdbCfg.keep0 = 3650;  // pCreate->daysToKeep2;
+  pCfg->tsdbCfg.keep0 = 3650;
-  pCfg->tsdbCfg.keep1 = 3650;  // pCreate->daysToKeep0;
+  pCfg->tsdbCfg.keep1 = 3650;
   pCfg->tsdbCfg.retentions = pCreate->pRetensions;
   pCfg->walCfg.vgId = pCreate->vgId;
   pCfg->hashBegin = pCreate->hashBegin;

source/dnode/mgmt/test/vnode/vnode.cpp

@@ -33,22 +33,18 @@ TEST_F(DndTestVnode, 01_Create_Vnode) {
     strcpy(createReq.db, "1.d1");
     createReq.dbUid = 9527;
     createReq.vgVersion = 1;
-    createReq.cacheBlockSize = 16;
-    createReq.totalBlocks = 10;
     createReq.daysPerFile = 10;
     createReq.daysToKeep0 = 3650;
     createReq.daysToKeep1 = 3650;
     createReq.daysToKeep2 = 3650;
     createReq.minRows = 100;
     createReq.minRows = 4096;
-    createReq.commitTime = 3600;
     createReq.fsyncPeriod = 3000;
     createReq.walLevel = 1;
     createReq.precision = 0;
     createReq.compression = 2;
     createReq.replica = 1;
     createReq.strict = 1;
-    createReq.update = 0;
     createReq.cacheLastRow = 0;
     createReq.selfIndex = 0;
     for (int r = 0; r < createReq.replica; ++r) {
@@ -75,27 +71,15 @@ TEST_F(DndTestVnode, 01_Create_Vnode) {
 TEST_F(DndTestVnode, 02_Alter_Vnode) {
   for (int i = 0; i < 3; ++i) {
     SAlterVnodeReq alterReq = {0};
-    alterReq.vgId = 2;
-    alterReq.dnodeId = 1;
-    strcpy(alterReq.db, "1.d1");
-    alterReq.dbUid = 9527;
     alterReq.vgVersion = 2;
-    alterReq.cacheBlockSize = 16;
-    alterReq.totalBlocks = 10;
     alterReq.daysPerFile = 10;
     alterReq.daysToKeep0 = 3650;
     alterReq.daysToKeep1 = 3650;
     alterReq.daysToKeep2 = 3650;
-    alterReq.minRows = 100;
-    alterReq.minRows = 4096;
-    alterReq.commitTime = 3600;
     alterReq.fsyncPeriod = 3000;
     alterReq.walLevel = 1;
-    alterReq.precision = 0;
-    alterReq.compression = 2;
     alterReq.replica = 1;
     alterReq.strict = 1;
-    alterReq.update = 0;
     alterReq.cacheLastRow = 0;
     alterReq.selfIndex = 0;
     for (int r = 0; r < alterReq.replica; ++r) {

source/dnode/mnode/impl/inc/mndConsumer.h

@@ -44,6 +44,11 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw);
 
 int32_t mndSetConsumerCommitLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer);
 
+bool mndRebTryStart();
+void mndRebEnd();
+void mndRebCntInc();
+void mndRebCntDec();
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/mnode/impl/inc/mndDef.h

@@ -257,26 +257,23 @@ typedef struct {
 
 typedef struct {
   int32_t numOfVgroups;
-  int32_t cacheBlockSize;
+  int32_t numOfStables;
-  int32_t totalBlocks;
+  int32_t buffer;
+  int32_t pageSize;
+  int32_t pages;
   int32_t daysPerFile;
   int32_t daysToKeep0;
   int32_t daysToKeep1;
   int32_t daysToKeep2;
   int32_t minRows;
   int32_t maxRows;
-  int32_t commitTime;
   int32_t fsyncPeriod;
-  int32_t ttl;
   int8_t  walLevel;
   int8_t  precision;
   int8_t  compression;
   int8_t  replications;
   int8_t  strict;
-  int8_t  update;
   int8_t  cacheLastRow;
-  int8_t  streamMode;
-  int8_t  singleSTable;
   int8_t  hashMethod;  // default is 1
   int32_t numOfRetensions;
   SArray* pRetensions;
@@ -316,7 +313,6 @@ typedef struct {
   int64_t   pointsWritten;
   int8_t    compact;
   int8_t    replica;
-  int8_t    streamMode;
   SVnodeGid vnodeGid[TSDB_MAX_REPLICA];
 } SVgObj;
 
@@ -582,8 +578,9 @@ typedef struct {
 
 typedef struct {
   char     name[TSDB_TOPIC_FNAME_LEN];
-  char     db[TSDB_DB_FNAME_LEN];
+  char     sourceDb[TSDB_DB_FNAME_LEN];
-  char     outputSTbName[TSDB_TABLE_FNAME_LEN];
+  char     targetDb[TSDB_DB_FNAME_LEN];
+  char     targetSTbName[TSDB_TABLE_FNAME_LEN];
   int64_t  createTime;
   int64_t  updateTime;
   int64_t  uid;

source/dnode/mnode/impl/inc/mndStb.h

@@ -29,6 +29,7 @@ void     mndReleaseStb(SMnode *pMnode, SStbObj *pStb);
 SSdbRaw *mndStbActionEncode(SStbObj *pStb);
 int32_t  mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbs, int32_t numOfStbs, void **ppRsp,
                             int32_t *pRspLen);
+int32_t  mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/inc/mndTrans.h

@@ -36,8 +36,8 @@ typedef struct {
 typedef enum {
   TEST_TRANS_START_FUNC = 1,
   TEST_TRANS_STOP_FUNC = 2,
-  CONSUME_TRANS_START_FUNC = 3,
+  MQ_REB_TRANS_START_FUNC = 3,
-  CONSUME_TRANS_STOP_FUNC = 4,
+  MQ_REB_TRANS_STOP_FUNC = 4,
 } ETrnFuncType;
 
 typedef void (*TransCbFp)(SMnode *pMnode, void *param, int32_t paramLen);

source/dnode/mnode/impl/src/mndConsumer.c

@@ -35,7 +35,7 @@
 
 #define MND_CONSUMER_LOST_HB_CNT 3
 
-static int8_t mqInRebFlag = 0;
+static int8_t mqRebLock = 0;
 
 static const char *mndConsumerStatusName(int status);
 
@@ -75,6 +75,17 @@ int32_t mndInitConsumer(SMnode *pMnode) {
 
 void mndCleanupConsumer(SMnode *pMnode) {}
 
+bool mndRebTryStart() {
+  int8_t old = atomic_val_compare_exchange_8(&mqRebLock, 0, 1);
+  return old == 0;
+}
+
+void mndRebEnd() { atomic_sub_fetch_8(&mqRebLock, 1); }
+
+void mndRebCntInc() { atomic_add_fetch_8(&mqRebLock, 1); }
+
+void mndRebCntDec() { atomic_sub_fetch_8(&mqRebLock, 1); }
+
 static int32_t mndProcessConsumerLostMsg(SNodeMsg *pMsg) {
   SMnode             *pMnode = pMsg->pNode;
   SMqConsumerLostMsg *pLostMsg = pMsg->rpcMsg.pCont;
@@ -143,8 +154,7 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) {
   void           *pIter = NULL;
 
   // rebalance cannot be parallel
-  int8_t old = atomic_val_compare_exchange_8(&mqInRebFlag, 0, 1);
+  if (!mndRebTryStart()) {
-  if (old != 0) {
     mInfo("mq rebalance already in progress, do nothing");
     return 0;
   }
@@ -152,7 +162,6 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) {
   SMqDoRebalanceMsg *pRebMsg = rpcMallocCont(sizeof(SMqDoRebalanceMsg));
   pRebMsg->rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK);
   // TODO set cleanfp
-  pRebMsg->mqInReb = &mqInRebFlag;
 
   // iterate all consumers, find all modification
   while (1) {
@@ -223,7 +232,7 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) {
     taosHashCleanup(pRebMsg->rebSubHash);
     rpcFreeCont(pRebMsg);
     mTrace("mq rebalance finished, no modification");
-    atomic_store_8(&mqInRebFlag, 0);
+    mndRebEnd();
   }
   return 0;
 }
@@ -812,6 +821,7 @@ static int32_t mndRetrieveConsumer(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock
     colDataAppend(pColInfo, numOfRows, (const char *)status, false);
 
     // subscribed topics
+    // TODO: split into multiple rows
     char  topics[TSDB_SHOW_LIST_LEN + VARSTR_HEADER_SIZE] = {0};
     char *showStr = taosShowStrArray(pConsumer->assignedTopics);
     tstrncpy(varDataVal(topics), showStr, TSDB_SHOW_LIST_LEN);

source/dnode/mnode/impl/src/mndDb.c

@@ -84,26 +84,23 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
   SDB_SET_INT32(pRaw, dataPos, pDb->cfgVersion, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pDb->vgVersion, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfVgroups, _OVER)
-  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.cacheBlockSize, _OVER)
+  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfStables, _OVER)
-  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.totalBlocks, _OVER)
+  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.buffer, _OVER)
+  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.pageSize, _OVER)
+  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.pages, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysPerFile, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep0, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep1, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep2, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfg.minRows, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfg.maxRows, _OVER)
-  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.commitTime, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfg.fsyncPeriod, _OVER)
-  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.ttl, _OVER)
   SDB_SET_INT8(pRaw, dataPos, pDb->cfg.walLevel, _OVER)
   SDB_SET_INT8(pRaw, dataPos, pDb->cfg.precision, _OVER)
   SDB_SET_INT8(pRaw, dataPos, pDb->cfg.compression, _OVER)
   SDB_SET_INT8(pRaw, dataPos, pDb->cfg.replications, _OVER)
   SDB_SET_INT8(pRaw, dataPos, pDb->cfg.strict, _OVER)
-  SDB_SET_INT8(pRaw, dataPos, pDb->cfg.update, _OVER)
   SDB_SET_INT8(pRaw, dataPos, pDb->cfg.cacheLastRow, _OVER)
-  SDB_SET_INT8(pRaw, dataPos, pDb->cfg.streamMode, _OVER)
-  SDB_SET_INT8(pRaw, dataPos, pDb->cfg.singleSTable, _OVER)
   SDB_SET_INT8(pRaw, dataPos, pDb->cfg.hashMethod, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfRetensions, _OVER)
   for (int32_t i = 0; i < pDb->cfg.numOfRetensions; ++i) {
@@ -158,26 +155,23 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
   SDB_GET_INT32(pRaw, dataPos, &pDb->cfgVersion, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pDb->vgVersion, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.numOfVgroups, _OVER)
-  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.cacheBlockSize, _OVER)
+  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.numOfStables, _OVER)
-  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.totalBlocks, _OVER)
+  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.buffer, _OVER)
+  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.pageSize, _OVER)
+  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.pages, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysPerFile, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysToKeep0, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysToKeep1, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysToKeep2, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.minRows, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.maxRows, _OVER)
-  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.commitTime, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.fsyncPeriod, _OVER)
-  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.ttl, _OVER)
   SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.walLevel, _OVER)
   SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.precision, _OVER)
   SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.compression, _OVER)
   SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.replications, _OVER)
   SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.strict, _OVER)
-  SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.update, _OVER)
   SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.cacheLastRow, _OVER)
-  SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.streamMode, _OVER)
-  SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.singleSTable, _OVER)
   SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.hashMethod, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.numOfRetensions, _OVER)
   if (pDb->cfg.numOfRetensions > 0) {
@@ -268,10 +262,10 @@ static int32_t mndCheckDbName(const char *dbName, SUserObj *pUser) {
 
 static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
   if (pCfg->numOfVgroups < TSDB_MIN_VNODES_PER_DB || pCfg->numOfVgroups > TSDB_MAX_VNODES_PER_DB) return -1;
-  /*
+  if (pCfg->numOfStables < TSDB_DB_STREAM_MODE_OFF || pCfg->numOfStables > TSDB_DB_STREAM_MODE_ON) return -1;
-  if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) return -1;
+  if (pCfg->buffer < TSDB_MIN_BUFFER_PER_VNODE || pCfg->buffer > TSDB_MAX_BUFFER_PER_VNODE) return -1;
-  if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) return -1;
+  if (pCfg->pageSize < TSDB_MIN_PAGESIZE_PER_VNODE || pCfg->pageSize > TSDB_MAX_PAGESIZE_PER_VNODE) return -1;
-  */
+  if (pCfg->pages < TSDB_MIN_PAGES_PER_VNODE || pCfg->pages > TSDB_MAX_PAGES_PER_VNODE) return -1;
   if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) return -1;
   if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP) return -1;
   if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP) return -1;
@@ -282,9 +276,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
   if (pCfg->minRows < TSDB_MIN_MINROWS_FBLOCK || pCfg->minRows > TSDB_MAX_MINROWS_FBLOCK) return -1;
   if (pCfg->maxRows < TSDB_MIN_MAXROWS_FBLOCK || pCfg->maxRows > TSDB_MAX_MAXROWS_FBLOCK) return -1;
   if (pCfg->minRows > pCfg->maxRows) return -1;
-  if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME || pCfg->commitTime > TSDB_MAX_COMMIT_TIME) return -1;
   if (pCfg->fsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->fsyncPeriod > TSDB_MAX_FSYNC_PERIOD) return -1;
-  // if (pCfg->ttl < TSDB_MIN_TABLE_TTL) return -1;
   if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) return -1;
   if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) return -1;
   if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) return -1;
@@ -292,36 +284,30 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
   if (pCfg->replications > mndGetDnodeSize(pMnode)) return -1;
   if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1;
   if (pCfg->strict > pCfg->replications) return -1;
-  if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) return -1;
   if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) return -1;
-  if (pCfg->streamMode < TSDB_DB_STREAM_MODE_OFF || pCfg->streamMode > TSDB_DB_STREAM_MODE_ON) return -1;
-  if (pCfg->singleSTable < TSDB_DB_SINGLE_STABLE_ON || pCfg->streamMode > TSDB_DB_SINGLE_STABLE_OFF) return -1;
   if (pCfg->hashMethod != 1) return -1;
   return TSDB_CODE_SUCCESS;
 }
 
 static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
   if (pCfg->numOfVgroups < 0) pCfg->numOfVgroups = TSDB_DEFAULT_VN_PER_DB;
-  if (pCfg->cacheBlockSize < 0) pCfg->cacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
+  if (pCfg->numOfStables < 0) pCfg->numOfStables = TSDB_DEFAULT_DB_SINGLE_STABLE;
-  if (pCfg->totalBlocks < 0) pCfg->totalBlocks = TSDB_DEFAULT_TOTAL_BLOCKS;
+  if (pCfg->buffer < 0) pCfg->buffer = TSDB_DEFAULT_BUFFER_PER_VNODE;
-  if (pCfg->daysPerFile < 0) pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE;
+  if (pCfg->pageSize < 0) pCfg->pageSize = TSDB_DEFAULT_PAGES_PER_VNODE;
+  if (pCfg->pages < 0) pCfg->pages = TSDB_MAX_PAGESIZE_PER_VNODE;
+  if (pCfg->daysPerFile < 0) pCfg->daysPerFile = TSDB_DEFAULT_DURATION_PER_FILE;
   if (pCfg->daysToKeep0 < 0) pCfg->daysToKeep0 = TSDB_DEFAULT_KEEP;
   if (pCfg->daysToKeep1 < 0) pCfg->daysToKeep1 = pCfg->daysToKeep0;
   if (pCfg->daysToKeep2 < 0) pCfg->daysToKeep2 = pCfg->daysToKeep1;
   if (pCfg->minRows < 0) pCfg->minRows = TSDB_DEFAULT_MINROWS_FBLOCK;
   if (pCfg->maxRows < 0) pCfg->maxRows = TSDB_DEFAULT_MAXROWS_FBLOCK;
-  if (pCfg->commitTime < 0) pCfg->commitTime = TSDB_DEFAULT_COMMIT_TIME;
   if (pCfg->fsyncPeriod < 0) pCfg->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD;
-  if (pCfg->ttl < 0) pCfg->ttl = TSDB_DEFAULT_TABLE_TTL;
   if (pCfg->walLevel < 0) pCfg->walLevel = TSDB_DEFAULT_WAL_LEVEL;
   if (pCfg->precision < 0) pCfg->precision = TSDB_DEFAULT_PRECISION;
   if (pCfg->compression < 0) pCfg->compression = TSDB_DEFAULT_COMP_LEVEL;
   if (pCfg->replications < 0) pCfg->replications = TSDB_DEFAULT_DB_REPLICA;
   if (pCfg->strict < 0) pCfg->strict = TSDB_DEFAULT_DB_STRICT;
-  if (pCfg->update < 0) pCfg->update = TSDB_DEFAULT_DB_UPDATE;
   if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW;
-  if (pCfg->streamMode < 0) pCfg->streamMode = TSDB_DEFAULT_DB_STREAM_MODE;
-  if (pCfg->singleSTable < 0) pCfg->singleSTable = TSDB_DEFAULT_DB_SINGLE_STABLE;
   if (pCfg->numOfRetensions < 0) pCfg->numOfRetensions = 0;
 }
 
@@ -447,26 +433,23 @@ static int32_t mndCreateDb(SMnode *pMnode, SNodeMsg *pReq, SCreateDbReq *pCreate
   memcpy(dbObj.createUser, pUser->user, TSDB_USER_LEN);
   dbObj.cfg = (SDbCfg){
       .numOfVgroups = pCreate->numOfVgroups,
-      .cacheBlockSize = pCreate->cacheBlockSize,
+      .numOfStables = pCreate->numOfStables,
-      .totalBlocks = pCreate->totalBlocks,
+      .buffer = pCreate->buffer,
+      .pageSize = pCreate->pageSize,
+      .pages = pCreate->pages,
       .daysPerFile = pCreate->daysPerFile,
       .daysToKeep0 = pCreate->daysToKeep0,
       .daysToKeep1 = pCreate->daysToKeep1,
       .daysToKeep2 = pCreate->daysToKeep2,
       .minRows = pCreate->minRows,
       .maxRows = pCreate->maxRows,
-      .commitTime = pCreate->commitTime,
       .fsyncPeriod = pCreate->fsyncPeriod,
-      .ttl = pCreate->ttl,
       .walLevel = pCreate->walLevel,
       .precision = pCreate->precision,
       .compression = pCreate->compression,
       .replications = pCreate->replications,
       .strict = pCreate->strict,
-      .update = pCreate->update,
       .cacheLastRow = pCreate->cacheLastRow,
-      .streamMode = pCreate->streamMode,
-      .singleSTable = pCreate->singleSTable,
       .hashMethod = 1,
   };
 
@@ -570,8 +553,23 @@ _OVER:
 static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
   terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED;
 
-  if (pAlter->totalBlocks >= 0 && pAlter->totalBlocks != pDb->cfg.totalBlocks) {
+  if (pAlter->buffer >= 0 && pAlter->buffer != pDb->cfg.buffer) {
-    pDb->cfg.totalBlocks = pAlter->totalBlocks;
+    pDb->cfg.buffer = pAlter->buffer;
+    terrno = 0;
+  }
+
+  if (pAlter->pages >= 0 && pAlter->pages != pDb->cfg.pages) {
+    pDb->cfg.pages = pAlter->pages;
+    terrno = 0;
+  }
+
+  if (pAlter->pageSize >= 0 && pAlter->pageSize != pDb->cfg.pageSize) {
+    pDb->cfg.pageSize = pAlter->pageSize;
+    terrno = 0;
+  }
+
+  if (pAlter->daysPerFile >= 0 && pAlter->daysPerFile != pDb->cfg.daysPerFile) {
+    pDb->cfg.daysPerFile = pAlter->daysPerFile;
     terrno = 0;
   }
 
@@ -639,10 +637,14 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p
 void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) {
   SAlterVnodeReq alterReq = {0};
   alterReq.vgVersion = pVgroup->version;
-  alterReq.totalBlocks = pDb->cfg.totalBlocks;
+  alterReq.buffer = pDb->cfg.buffer;
+  alterReq.pages = pDb->cfg.pages;
+  alterReq.pageSize = pDb->cfg.pageSize;
+  alterReq.daysPerFile = pDb->cfg.daysPerFile;
   alterReq.daysToKeep0 = pDb->cfg.daysToKeep0;
   alterReq.daysToKeep1 = pDb->cfg.daysToKeep1;
   alterReq.daysToKeep2 = pDb->cfg.daysToKeep2;
+  alterReq.fsyncPeriod = pDb->cfg.fsyncPeriod;
   alterReq.walLevel = pDb->cfg.walLevel;
   alterReq.strict = pDb->cfg.strict;
   alterReq.cacheLastRow = pDb->cfg.cacheLastRow;
@@ -835,26 +837,23 @@ static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq) {
   }
 
   cfgRsp.numOfVgroups = pDb->cfg.numOfVgroups;
-  cfgRsp.cacheBlockSize = pDb->cfg.cacheBlockSize;
+  cfgRsp.numOfStables = pDb->cfg.numOfStables;
-  cfgRsp.totalBlocks = pDb->cfg.totalBlocks;
+  cfgRsp.buffer = pDb->cfg.buffer;
+  cfgRsp.pageSize = pDb->cfg.pageSize;
+  cfgRsp.pages = pDb->cfg.pages;
   cfgRsp.daysPerFile = pDb->cfg.daysPerFile;
   cfgRsp.daysToKeep0 = pDb->cfg.daysToKeep0;
   cfgRsp.daysToKeep1 = pDb->cfg.daysToKeep1;
   cfgRsp.daysToKeep2 = pDb->cfg.daysToKeep2;
   cfgRsp.minRows = pDb->cfg.minRows;
   cfgRsp.maxRows = pDb->cfg.maxRows;
-  cfgRsp.commitTime = pDb->cfg.commitTime;
   cfgRsp.fsyncPeriod = pDb->cfg.fsyncPeriod;
-  cfgRsp.ttl = pDb->cfg.ttl;
   cfgRsp.walLevel = pDb->cfg.walLevel;
   cfgRsp.precision = pDb->cfg.precision;
   cfgRsp.compression = pDb->cfg.compression;
   cfgRsp.replications = pDb->cfg.replications;
   cfgRsp.strict = pDb->cfg.strict;
-  cfgRsp.update = pDb->cfg.update;
   cfgRsp.cacheLastRow = pDb->cfg.cacheLastRow;
-  cfgRsp.streamMode = pDb->cfg.streamMode;
-  cfgRsp.singleSTable = pDb->cfg.singleSTable;
   cfgRsp.numOfRetensions = pDb->cfg.numOfRetensions;
   cfgRsp.pRetensions = pDb->cfg.pRetensions;
 
@@ -1453,10 +1452,13 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in
     colDataAppend(pColInfo, rows, (const char *)tmp, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.cacheBlockSize, false);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.buffer, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.totalBlocks, false);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.pageSize, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.pages, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.minRows, false);
@@ -1498,13 +1500,7 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in
     colDataAppend(pColInfo, rows, (const char *)t, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.ttl, false);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.numOfStables, false);
-
-    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.singleSTable, false);
-
-    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.streamMode, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
     colDataAppend(pColInfo, rows, (const char *)b, false);
@@ -1522,7 +1518,6 @@ static void setInformationSchemaDbCfg(SDbObj *pDbObj) {
   pDbObj->cfg.numOfVgroups = 0;
   pDbObj->cfg.strict = 1;
   pDbObj->cfg.replications = 1;
-  pDbObj->cfg.update = 1;
   pDbObj->cfg.precision = TSDB_TIME_PRECISION_MILLI;
 }
 
@@ -1534,7 +1529,6 @@ static void setPerfSchemaDbCfg(SDbObj *pDbObj) {
   pDbObj->cfg.numOfVgroups = 0;
   pDbObj->cfg.strict = 1;
   pDbObj->cfg.replications = 1;
-  pDbObj->cfg.update = 1;
   pDbObj->cfg.precision = TSDB_TIME_PRECISION_MILLI;
 }
 

source/dnode/mnode/impl/src/mndDef.c

@@ -410,7 +410,7 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) {
   int32_t sz = 0;
   /*int32_t outputNameSz = 0;*/
   if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1;
-  if (tEncodeCStr(pEncoder, pObj->db) < 0) return -1;
+  if (tEncodeCStr(pEncoder, pObj->sourceDb) < 0) return -1;
   if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1;
   if (tEncodeI64(pEncoder, pObj->updateTime) < 0) return -1;
   if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1;
@@ -418,6 +418,9 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) {
   if (tEncodeI32(pEncoder, pObj->version) < 0) return -1;
   if (tEncodeI8(pEncoder, pObj->status) < 0) return -1;
   if (tEncodeI8(pEncoder, pObj->createdBy) < 0) return -1;
+  if (tEncodeI8(pEncoder, pObj->trigger) < 0) return -1;
+  if (tEncodeI32(pEncoder, pObj->triggerParam) < 0) return -1;
+  if (tEncodeI64(pEncoder, pObj->waterMark) < 0) return -1;
   if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1;
   if (tEncodeI64(pEncoder, pObj->smaId) < 0) return -1;
   if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1;
@@ -456,7 +459,7 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) {
 
 int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) {
   if (tDecodeCStrTo(pDecoder, pObj->name) < 0) return -1;
-  if (tDecodeCStrTo(pDecoder, pObj->db) < 0) return -1;
+  if (tDecodeCStrTo(pDecoder, pObj->sourceDb) < 0) return -1;
   if (tDecodeI64(pDecoder, &pObj->createTime) < 0) return -1;
   if (tDecodeI64(pDecoder, &pObj->updateTime) < 0) return -1;
   if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1;
@@ -464,6 +467,9 @@ int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) {
   if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1;
   if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1;
   if (tDecodeI8(pDecoder, &pObj->createdBy) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pObj->trigger) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pObj->triggerParam) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pObj->waterMark) < 0) return -1;
   if (tDecodeI32(pDecoder, &pObj->fixedSinkVgId) < 0) return -1;
   if (tDecodeI64(pDecoder, &pObj->smaId) < 0) return -1;
   if (tDecodeCStrAlloc(pDecoder, &pObj->sql) < 0) return -1;

source/dnode/mnode/impl/src/mndInfoSchema.c

@@ -76,10 +76,11 @@ static const SInfosTableSchema userDBSchema[] = {
     {.name = "ntables", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
     {.name = "replica", .bytes = 2, .type = TSDB_DATA_TYPE_TINYINT},
     {.name = "strict", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "days", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "duration", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "keep", .bytes = 24 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "cache", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "buffer", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "pagesize", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "pages", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "minrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "maxrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "wal", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
@@ -87,9 +88,7 @@ static const SInfosTableSchema userDBSchema[] = {
     {.name = "comp", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
     {.name = "cachelast", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
     {.name = "precision", .bytes = 2 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "ttl", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "single_stable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
-    {.name = "stream_mode", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
     {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
     // {.name = "update", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},  // disable update
 };
@@ -124,14 +123,6 @@ static const SInfosTableSchema userStbsSchema[] = {
     {.name = "table_comment", .bytes = 1024 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
 };
 
-static const SInfosTableSchema userStreamsSchema[] = {
-    {.name = "stream_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "user_name", .bytes = 23, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "dest_table", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-    {.name = "sql", .bytes = 1024, .type = TSDB_DATA_TYPE_VARCHAR},
-};
-
 static const SInfosTableSchema userTblsSchema[] = {
     {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
@@ -259,7 +250,6 @@ static const SInfosTableMeta infosMeta[] = {
     {TSDB_INS_TABLE_USER_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema)},
     {TSDB_INS_TABLE_USER_INDEXES, userIdxSchema, tListLen(userIdxSchema)},
     {TSDB_INS_TABLE_USER_STABLES, userStbsSchema, tListLen(userStbsSchema)},
-    {TSDB_INS_TABLE_USER_STREAMS, userStreamsSchema, tListLen(userStreamsSchema)},
     {TSDB_INS_TABLE_USER_TABLES, userTblsSchema, tListLen(userTblsSchema)},
     {TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)},
     {TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)},

source/dnode/mnode/impl/src/mndPerfSchema.c

@@ -76,6 +76,18 @@ static const SPerfsTableSchema offsetSchema[] = {
     {.name = "skip_log_cnt", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
 };
 
+static const SPerfsTableSchema streamSchema[] = {
+    {.name = "stream_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+    {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "source_db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "target_db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "target_table", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "watermark", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
+    {.name = "trigger", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+};
+
 static const SPerfsTableMeta perfsMeta[] = {
     {TSDB_PERFS_TABLE_CONNECTIONS, connectionsSchema, tListLen(connectionsSchema)},
     {TSDB_PERFS_TABLE_QUERIES, queriesSchema, tListLen(queriesSchema)},
@@ -83,6 +95,7 @@ static const SPerfsTableMeta perfsMeta[] = {
     {TSDB_PERFS_TABLE_CONSUMERS, consumerSchema, tListLen(consumerSchema)},
     {TSDB_PERFS_TABLE_SUBSCRIPTIONS, subscriptionSchema, tListLen(subscriptionSchema)},
     {TSDB_PERFS_TABLE_OFFSETS, offsetSchema, tListLen(offsetSchema)},
+    {TSDB_PERFS_TABLE_STREAMS, streamSchema, tListLen(streamSchema)},
 };
 
 // connection/application/

source/dnode/mnode/impl/src/mndScheduler.c

@@ -308,8 +308,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
         // sink part
         if (level == 0) {
           // only for inplace
-          pTask->sinkType = TASK_SINK__SHOW;
+          pTask->sinkType = TASK_SINK__NONE;
-          pTask->showSink.reserved = 0;
           if (!hasExtraSink) {
 #if 1
             if (pStream->createdBy == STREAM_CREATED_BY__SMA) {
@@ -368,8 +367,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
       pTask->sourceType = TASK_SOURCE__PIPE;
 
       // sink part
-      pTask->sinkType = TASK_SINK__SHOW;
+      pTask->sinkType = TASK_SINK__NONE;
-      /*pTask->sinkType = TASK_SINK__NONE;*/
 
       // dispatch part
       ASSERT(hasExtraSink);
@@ -382,7 +380,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
           pTask->dispatchType = TASK_DISPATCH__SHUFFLE;
 
           pTask->dispatchMsgType = TDMT_VND_TASK_WRITE_EXEC;
-          SDbObj* pDb = mndAcquireDb(pMnode, pStream->db);
+          SDbObj* pDb = mndAcquireDb(pMnode, pStream->sourceDb);
           ASSERT(pDb);
           if (mndExtractDbInfo(pMnode, pDb, &pTask->shuffleDispatcher.dbInfo, NULL) < 0) {
             sdbRelease(pSdb, pDb);
@@ -456,7 +454,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
       pTask->sourceType = TASK_SOURCE__MERGE;
 
       // sink part
-      pTask->sinkType = TASK_SINK__SHOW;
+      pTask->sinkType = TASK_SINK__NONE;
 
       // dispatch part
       pTask->dispatchType = TASK_DISPATCH__NONE;

source/dnode/mnode/impl/src/mndShow.c

@@ -47,7 +47,7 @@ void mndCleanupShow(SMnode *pMnode) {
   }
 }
 
-static int32_t convertToRetrieveType(char* name, int32_t len) {
+static int32_t convertToRetrieveType(char *name, int32_t len) {
   int32_t type = -1;
 
   if (strncasecmp(name, TSDB_INS_TABLE_DNODES, len) == 0) {
@@ -72,8 +72,6 @@ static int32_t convertToRetrieveType(char* name, int32_t len) {
     //    type = TSDB_MGMT_TABLE_INDEX;
   } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, len) == 0) {
     type = TSDB_MGMT_TABLE_STB;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STREAMS, len) == 0) {
-    type = TSDB_MGMT_TABLE_STREAMS;
   } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, len) == 0) {
     type = TSDB_MGMT_TABLE_TABLE;
   } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, len) == 0) {
@@ -102,8 +100,10 @@ static int32_t convertToRetrieveType(char* name, int32_t len) {
     type = TSDB_MGMT_TABLE_VNODES;
   } else if (strncasecmp(name, TSDB_PERFS_TABLE_TOPICS, len) == 0) {
     type = TSDB_MGMT_TABLE_TOPICS;
+  } else if (strncasecmp(name, TSDB_PERFS_TABLE_STREAMS, len) == 0) {
+    type = TSDB_MGMT_TABLE_STREAMS;
   } else {
-//    ASSERT(0);
+    //    ASSERT(0);
   }
 
   return type;

source/dnode/mnode/impl/src/mndSma.c

@@ -40,7 +40,7 @@ static int32_t  mndProcessMCreateSmaReq(SNodeMsg *pReq);
 static int32_t  mndProcessMDropSmaReq(SNodeMsg *pReq);
 static int32_t  mndProcessVCreateSmaRsp(SNodeMsg *pRsp);
 static int32_t  mndProcessVDropSmaRsp(SNodeMsg *pRsp);
-static int32_t  mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows);
+static int32_t  mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void     mndCancelGetNextSma(SMnode *pMnode, void *pIter);
 
 int32_t mndInitSma(SMnode *pMnode) {
@@ -406,7 +406,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SNodeMsg *pReq, SMCreateSmaReq *pCre
 
   SStreamObj streamObj = {0};
   tstrncpy(streamObj.name, pCreate->name, TSDB_STREAM_FNAME_LEN);
-  tstrncpy(streamObj.db, pDb->name, TSDB_DB_FNAME_LEN);
+  tstrncpy(streamObj.sourceDb, pDb->name, TSDB_DB_FNAME_LEN);
   streamObj.createTime = taosGetTimestampMs();
   streamObj.updateTime = streamObj.createTime;
   streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name));
@@ -707,7 +707,8 @@ int32_t mndProcessGetSmaReq(SMnode      *pMnode, SUserIndexReq *indexReq, SUserI
     SNode *node = NULL;
     FOREACH(node, pList) {
       SFunctionNode *pFunc = (SFunctionNode *)node;
-      extOffset += snprintf(rsp->indexExts + extOffset, sizeof(rsp->indexExts) - extOffset - 1, "%s%s", (extOffset ? ",":""), pFunc->functionName);
+      extOffset += snprintf(rsp->indexExts + extOffset, sizeof(rsp->indexExts) - extOffset - 1, "%s%s",
+                            (extOffset ? "," : ""), pFunc->functionName);
     }
 
     *exist = true;
@@ -718,13 +719,12 @@ int32_t mndProcessGetSmaReq(SMnode      *pMnode, SUserIndexReq *indexReq, SUserI
   return code;
 }
 
-
 static int32_t mndProcessVDropSmaRsp(SNodeMsg *pRsp) {
   mndTransProcessRsp(pRsp);
   return 0;
 }
 
-static int32_t mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) {
+static int32_t mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
   SMnode  *pMnode = pReq->pNode;
   SSdb    *pSdb = pMnode->pSdb;
   int32_t  numOfRows = 0;
@@ -758,8 +758,8 @@ static int32_t mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlo
     char n1[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_TO_VARSTR(n1, (char *)tNameGetTableName(&stbName));
 
-    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char*) n, false);
+    colDataAppend(pColInfo, numOfRows, (const char *)n, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, numOfRows, (const char *)&pSma->createdTime, false);

source/dnode/mnode/impl/src/mndStb.c

@@ -774,6 +774,13 @@ static int32_t mndProcessMCreateStbReq(SNodeMsg *pReq) {
     goto _OVER;
   }
 
+  int32_t numOfStbs = -1;
+  mndGetNumOfStbs(pMnode, pDb->name, &numOfStbs);
+  if (pDb->cfg.numOfStables == 1 && numOfStbs != 0 ) {
+    terrno = TSDB_CODE_MND_SINGLE_STB_MODE_DB;
+    goto _OVER;
+  }
+
   code = mndCreateStb(pMnode, pReq, &createReq, pDb);
   if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
 
@@ -1427,7 +1434,6 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa
   pRsp->numOfColumns = pStb->numOfColumns;
   pRsp->precision = pDb->cfg.precision;
   pRsp->tableType = TSDB_SUPER_TABLE;
-  pRsp->update = pDb->cfg.update;
   pRsp->sversion = pStb->version;
   pRsp->suid = pStb->uid;
   pRsp->tuid = pStb->uid;
@@ -1580,7 +1586,7 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbVersions, int
   return 0;
 }
 
-static int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs) {
+int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs) {
   SSdb   *pSdb = pMnode->pSdb;
   SDbObj *pDb = mndAcquireDb(pMnode, dbName);
   if (pDb == NULL) {

source/dnode/mnode/impl/src/mndStream.c

@@ -40,7 +40,7 @@ static int32_t mndProcessTaskDeployInternalRsp(SNodeMsg *pRsp);
 /*static int32_t mndProcessDropStreamInRsp(SNodeMsg *pRsp);*/
 static int32_t mndProcessStreamMetaReq(SNodeMsg *pReq);
 static int32_t mndGetStreamMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta);
-static int32_t mndRetrieveStream(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows);
+static int32_t mndRetrieveStream(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void    mndCancelGetNextStream(SMnode *pMnode, void *pIter);
 
 int32_t mndInitStream(SMnode *pMnode) {
@@ -58,8 +58,8 @@ int32_t mndInitStream(SMnode *pMnode) {
   /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM, mndProcessDropStreamReq);*/
   /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM_RSP, mndProcessDropStreamInRsp);*/
 
-  //  mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndRetrieveStream);
+  mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndRetrieveStream);
-  /*mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextStream);*/
+  mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndCancelGetNextStream);
 
   return sdbSetTable(pMnode->pSdb, table);
 }
@@ -294,8 +294,8 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe
   mDebug("stream:%s to create", pCreate->name);
   SStreamObj streamObj = {0};
   tstrncpy(streamObj.name, pCreate->name, TSDB_STREAM_FNAME_LEN);
-  tstrncpy(streamObj.db, pDb->name, TSDB_DB_FNAME_LEN);
+  tstrncpy(streamObj.sourceDb, pDb->name, TSDB_DB_FNAME_LEN);
-  tstrncpy(streamObj.outputSTbName, pCreate->outputSTbName, TSDB_TABLE_FNAME_LEN);
+  tstrncpy(streamObj.targetSTbName, pCreate->targetStbFullName, TSDB_TABLE_FNAME_LEN);
   streamObj.createTime = taosGetTimestampMs();
   streamObj.updateTime = streamObj.createTime;
   streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name));
@@ -308,6 +308,8 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe
   streamObj.smaId = 0;
   /*streamObj.physicalPlan = "";*/
   streamObj.logicalPlan = "not implemented";
+  streamObj.trigger = pCreate->triggerType;
+  streamObj.waterMark = pCreate->watermark;
 
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg);
   if (pTrans == NULL) {
@@ -424,56 +426,58 @@ static int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfS
   return 0;
 }
 
-static int32_t mndRetrieveStream(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) {
+static int32_t mndRetrieveStream(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
   SMnode     *pMnode = pReq->pNode;
   SSdb       *pSdb = pMnode->pSdb;
   int32_t     numOfRows = 0;
   SStreamObj *pStream = NULL;
-  int32_t     cols = 0;
-  char       *pWrite;
-  char        prefix[TSDB_DB_FNAME_LEN] = {0};
-
-  SDbObj *pDb = mndAcquireDb(pMnode, pShow->db);
-  if (pDb == NULL) return 0;
-
-  tstrncpy(prefix, pShow->db, TSDB_DB_FNAME_LEN);
-  strcat(prefix, TS_PATH_DELIMITER);
-  int32_t prefixLen = (int32_t)strlen(prefix);
 
   while (numOfRows < rows) {
     pShow->pIter = sdbFetch(pSdb, SDB_STREAM, pShow->pIter, (void **)&pStream);
     if (pShow->pIter == NULL) break;
 
-    if (pStream->dbUid != pDb->uid) {
+    SColumnInfoData *pColInfo;
-      if (strncmp(pStream->name, prefix, prefixLen) != 0) {
+    SName            n;
-        mError("Inconsistent stream data, name:%s, db:%s, dbUid:%" PRIu64, pStream->name, pDb->name, pDb->uid);
+    int32_t          cols = 0;
-      }
 
-      sdbRelease(pSdb, pStream);
+    char streamName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
-      continue;
+    tNameFromString(&n, pStream->name, T_NAME_ACCT | T_NAME_DB);
-    }
+    tNameGetDbName(&n, varDataVal(streamName));
+    varDataSetLen(streamName, strlen(varDataVal(streamName)));
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)streamName, false);
 
-    cols = 0;
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pStream->createTime, false);
 
-    char streamName[TSDB_TABLE_NAME_LEN] = {0};
+    char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0};
-    tstrncpy(streamName, pStream->name + prefixLen, TSDB_TABLE_NAME_LEN);
+    tstrncpy(&sql[VARSTR_HEADER_SIZE], pStream->sql, TSDB_SHOW_SQL_LEN);
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    varDataSetLen(sql, strlen(&sql[VARSTR_HEADER_SIZE]));
-    STR_TO_VARSTR(pWrite, streamName);
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    cols++;
+    colDataAppend(pColInfo, numOfRows, (const char *)sql, false);
 
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    *(int64_t *)pWrite = pStream->createTime;
+    colDataAppend(pColInfo, numOfRows, (const char *)&pStream->status, true);
-    cols++;
 
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pStream->sql, pShow->bytes[cols]);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pStream->sourceDb, true);
-    cols++;
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pStream->targetDb, true);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pStream->targetSTbName, true);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pStream->waterMark, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pStream->trigger, false);
 
     numOfRows++;
     sdbRelease(pSdb, pStream);
   }
 
-  mndReleaseDb(pMnode, pDb);
   pShow->numOfRows += numOfRows;
   return numOfRows;
 }

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -309,9 +309,6 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
       newConsumerEp.consumerId = consumerId;
       newConsumerEp.vgs = taosArrayInit(0, sizeof(void *));
       taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp));
-      /*SMqConsumer* pTestNew = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));*/
-      /*ASSERT(pTestNew->consumerId == consumerId);*/
-      /*ASSERT(pTestNew->vgs == newConsumerEp.vgs);*/
       taosArrayPush(pOutput->newConsumers, &consumerId);
     }
   }
@@ -369,7 +366,13 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
     }
   }
 
-  // 8. generate logs
+  // 8. TODO generate logs
+  mInfo("rebalance calculation completed, rebalanced vg:");
+  for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) {
+    SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i);
+    mInfo("vg: %d moved from consumer %ld to consumer %ld", pOutputRebVg->pVgEp->vgId, pOutputRebVg->oldConsumerId,
+          pOutputRebVg->newConsumerId);
+  }
 
   // 9. clear
   taosHashCleanup(pHash);
@@ -447,7 +450,12 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SNodeMsg *pMsg, const SMqRebO
       goto REB_FAIL;
     }
   }
-  // 4. commit log: modification log
+  // 4. TODO commit log: modification log
+
+  // 5. set cb
+  mndTransSetCb(pTrans, MQ_REB_TRANS_START_FUNC, MQ_REB_TRANS_STOP_FUNC, NULL, 0);
+
+  // 6. execution
   if (mndTransPrepare(pMnode, pTrans) != 0) goto REB_FAIL;
 
   mndTransDrop(pTrans);
@@ -513,9 +521,9 @@ static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) {
   }
 
   // reset flag
-  atomic_store_8(pReq->mqInReb, 0);
   mInfo("mq rebalance completed successfully");
   taosHashCleanup(pReq->rebSubHash);
+  mndRebEnd();
 
   return 0;
 }

source/dnode/mnode/impl/src/mndTrans.c

@@ -16,6 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "mndTrans.h"
 #include "mndAuth.h"
+#include "mndConsumer.h"
 #include "mndDb.h"
 #include "mndShow.h"
 #include "mndSync.h"
@@ -442,6 +443,10 @@ static TransCbFp mndTransGetCbFp(ETrnFuncType ftype) {
       return mndTransTestStartFunc;
     case TEST_TRANS_STOP_FUNC:
       return mndTransTestStopFunc;
+    case MQ_REB_TRANS_START_FUNC:
+      return mndRebCntInc;
+    case MQ_REB_TRANS_STOP_FUNC:
+      return mndRebCntDec;
     default:
       return NULL;
   }

source/dnode/mnode/impl/src/mndUser.c

@@ -39,14 +39,16 @@ static int32_t  mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *p
 static void     mndCancelGetNextUser(SMnode *pMnode, void *pIter);
 
 int32_t mndInitUser(SMnode *pMnode) {
-  SSdbTable table = {.sdbType = SDB_USER,
+  SSdbTable table = {
+      .sdbType = SDB_USER,
       .keyType = SDB_KEY_BINARY,
       .deployFp = (SdbDeployFp)mndCreateDefaultUsers,
       .encodeFp = (SdbEncodeFp)mndUserActionEncode,
       .decodeFp = (SdbDecodeFp)mndUserActionDecode,
       .insertFp = (SdbInsertFp)mndUserActionInsert,
       .updateFp = (SdbUpdateFp)mndUserActionUpdate,
-                     .deleteFp = (SdbDeleteFp)mndUserActionDelete};
+      .deleteFp = (SdbDeleteFp)mndUserActionDelete,
+  };
 
   mndSetMsgHandle(pMnode, TDMT_MND_CREATE_USER, mndProcessCreateUserReq);
   mndSetMsgHandle(pMnode, TDMT_MND_ALTER_USER, mndProcessAlterUserReq);

source/dnode/mnode/impl/src/mndVgroup.c

@@ -190,25 +190,24 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg
   memcpy(createReq.db, pDb->name, TSDB_DB_FNAME_LEN);
   createReq.dbUid = pDb->uid;
   createReq.vgVersion = pVgroup->version;
-  createReq.cacheBlockSize = pDb->cfg.cacheBlockSize;
+  createReq.numOfStables = pDb->cfg.numOfStables;
-  createReq.totalBlocks = pDb->cfg.totalBlocks;
+  createReq.buffer = pDb->cfg.buffer;
+  createReq.pageSize = pDb->cfg.pageSize;
+  createReq.pages = pDb->cfg.pages;
   createReq.daysPerFile = pDb->cfg.daysPerFile;
   createReq.daysToKeep0 = pDb->cfg.daysToKeep0;
   createReq.daysToKeep1 = pDb->cfg.daysToKeep1;
   createReq.daysToKeep2 = pDb->cfg.daysToKeep2;
   createReq.minRows = pDb->cfg.minRows;
   createReq.maxRows = pDb->cfg.maxRows;
-  createReq.commitTime = pDb->cfg.commitTime;
   createReq.fsyncPeriod = pDb->cfg.fsyncPeriod;
   createReq.walLevel = pDb->cfg.walLevel;
   createReq.precision = pDb->cfg.precision;
   createReq.compression = pDb->cfg.compression;
   createReq.strict = pDb->cfg.strict;
-  createReq.update = pDb->cfg.update;
   createReq.cacheLastRow = pDb->cfg.cacheLastRow;
   createReq.replica = pVgroup->replica;
   createReq.selfIndex = -1;
-  createReq.streamMode = pVgroup->streamMode;
   createReq.hashBegin = pVgroup->hashBegin;
   createReq.hashEnd = pVgroup->hashEnd;
   createReq.hashMethod = pDb->cfg.hashMethod;
@@ -398,7 +397,6 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
     pVgroup->createdTime = taosGetTimestampMs();
     pVgroup->updateTime = pVgroups->createdTime;
     pVgroup->version = 1;
-    pVgroup->streamMode = pDb->cfg.streamMode;
     pVgroup->hashBegin = hashMin + hashInterval * v;
     if (v == pDb->cfg.numOfVgroups - 1) {
       pVgroup->hashEnd = hashMax;

source/dnode/mnode/impl/test/db/db.cpp

@@ -35,27 +35,24 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) {
     SCreateDbReq createReq = {0};
     strcpy(createReq.db, "1.d1");
     createReq.numOfVgroups = 2;
-    createReq.cacheBlockSize = 16;
+    createReq.buffer = -1;
-    createReq.totalBlocks = 10;
+    createReq.pageSize = -1;
+    createReq.pages = -1;
     createReq.daysPerFile = 1000;
     createReq.daysToKeep0 = 3650;
     createReq.daysToKeep1 = 3650;
     createReq.daysToKeep2 = 3650;
     createReq.minRows = 100;
     createReq.maxRows = 4096;
-    createReq.commitTime = 3600;
     createReq.fsyncPeriod = 3000;
-    createReq.ttl = 1;
     createReq.walLevel = 1;
     createReq.precision = 0;
     createReq.compression = 2;
     createReq.replications = 1;
     createReq.strict = 1;
-    createReq.update = 0;
     createReq.cacheLastRow = 0;
     createReq.ignoreExist = 1;
-    createReq.streamMode = 0;
+    createReq.numOfStables = 0;
-    createReq.singleSTable = 0;
     createReq.numOfRetensions = 0;
 
     int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq);
@@ -76,7 +73,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) {
   {
     SAlterDbReq alterdbReq = {0};
     strcpy(alterdbReq.db, "1.d1");
-    alterdbReq.totalBlocks = 12;
+    alterdbReq.buffer = 12;
     alterdbReq.daysToKeep0 = 300;
     alterdbReq.daysToKeep1 = 400;
     alterdbReq.daysToKeep2 = 500;
@@ -129,27 +126,24 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) {
     SCreateDbReq createReq = {0};
     strcpy(createReq.db, "1.d2");
     createReq.numOfVgroups = 2;
-    createReq.cacheBlockSize = 16;
+    createReq.buffer = -1;
-    createReq.totalBlocks = 10;
+    createReq.pageSize = -1;
+    createReq.pages = -1;
     createReq.daysPerFile = 1000;
     createReq.daysToKeep0 = 3650;
     createReq.daysToKeep1 = 3650;
     createReq.daysToKeep2 = 3650;
     createReq.minRows = 100;
     createReq.maxRows = 4096;
-    createReq.commitTime = 3600;
     createReq.fsyncPeriod = 3000;
-    createReq.ttl = 1;
     createReq.walLevel = 1;
     createReq.precision = 0;
     createReq.compression = 2;
     createReq.replications = 1;
     createReq.strict = 1;
-    createReq.update = 0;
     createReq.cacheLastRow = 0;
     createReq.ignoreExist = 1;
-    createReq.streamMode = 0;
+    createReq.numOfStables = 0;
-    createReq.singleSTable = 0;
     createReq.numOfRetensions = 0;
 
     int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq);

source/dnode/mnode/impl/test/sdb/sdbTest.cpp

@@ -9,75 +9,314 @@
  *
  */
 
-#include "sut.h"
+#include <gtest/gtest.h>
 
-class MndTestShow : public ::testing::Test {
+#include "sdb.h"
+
+class MndTestSdb : public ::testing::Test {
  protected:
-  static void SetUpTestSuite() { test.Init("/tmp/mnode_test_show", 9021); }
+  static void SetUpTestSuite() {}
-  static void TearDownTestSuite() { test.Cleanup(); }
+  static void TearDownTestSuite() {}
-
-  static Testbase test;
 
  public:
   void SetUp() override {}
   void TearDown() override {}
 };
 
-Testbase MndTestShow::test;
+typedef struct SMnode {
+  int32_t v100;
+  int32_t v200;
+  SSdb   *pSdb;
+} SMnode;
 
-TEST_F(MndTestShow, 01_ShowMsg_InvalidMsgMax) {
+typedef struct SStrObj {
-  SShowReq showReq = {0};
+  char    key[24];
-  showReq.type = TSDB_MGMT_TABLE_MAX;
+  int8_t  v8;
+  int16_t v16;
+  int32_t v32;
+  int64_t v64;
+  char    vstr[32];
+  char    unused[48];
+} SStrObj;
 
-  int32_t contLen = tSerializeSShowReq(NULL, 0, &showReq);
+typedef struct SI32Obj {
-  void*   pReq = rpcMallocCont(contLen);
+  int32_t key;
-  tSerializeSShowReq(pReq, contLen, &showReq);
+  int8_t  v8;
-  tFreeSShowReq(&showReq);
+  int16_t v16;
+  int32_t v32;
+  int64_t v64;
+  char    vstr[32];
+  char    unused[48];
+} SI32Obj;
 
-  SRpcMsg* pRsp = test.SendReq(TDMT_MND_SYSTABLE_RETRIEVE, pReq, contLen);
+typedef struct SI64Obj {
-  ASSERT_NE(pRsp, nullptr);
+  int64_t key;
-  ASSERT_NE(pRsp->code, 0);
+  int8_t  v8;
+  int16_t v16;
+  int32_t v32;
+  int64_t v64;
+  char    vstr[32];
+  char    unused[48];
+} SI64Obj;
+
+SSdbRaw *strEncode(SStrObj *pObj) {
+  int32_t  dataPos = 0;
+  SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, 1, sizeof(SStrObj));
+
+  sdbSetRawBinary(pRaw, dataPos, pObj->key, sizeof(pObj->key));
+  dataPos += sizeof(pObj->key);
+  sdbSetRawInt8(pRaw, dataPos, pObj->v8);
+  dataPos += sizeof(pObj->v8);
+  sdbSetRawInt16(pRaw, dataPos, pObj->v16);
+  dataPos += sizeof(pObj->v16);
+  sdbSetRawInt32(pRaw, dataPos, pObj->v32);
+  dataPos += sizeof(pObj->v32);
+  sdbSetRawInt64(pRaw, dataPos, pObj->v64);
+  dataPos += sizeof(pObj->v64);
+  sdbSetRawBinary(pRaw, dataPos, pObj->vstr, sizeof(pObj->vstr));
+  dataPos += sizeof(pObj->vstr);
+  sdbSetRawDataLen(pRaw, dataPos);
+
+  return pRaw;
 }
 
-TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) {
+SSdbRow *strDecode(SSdbRaw *pRaw) {
-  SShowReq showReq = {0};
+  int8_t sver = 0;
-  showReq.type = TSDB_MGMT_TABLE_START;
+  if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
+  if (sver != 1) return NULL;
 
-  int32_t contLen = tSerializeSShowReq(NULL, 0, &showReq);
+  SSdbRow *pRow = sdbAllocRow(sizeof(SStrObj));
-  void*   pReq = rpcMallocCont(contLen);
+  if (pRow == NULL) return NULL;
-  tSerializeSShowReq(pReq, contLen, &showReq);
-  tFreeSShowReq(&showReq);
 
-  SRpcMsg* pRsp = test.SendReq(TDMT_MND_SYSTABLE_RETRIEVE, pReq, contLen);
+  SStrObj *pObj = (SStrObj *)sdbGetRowObj(pRow);
-  ASSERT_NE(pRsp, nullptr);
+  if (pObj == NULL) return NULL;
-  ASSERT_NE(pRsp->code, 0);
+
+  int32_t dataPos = 0;
+  sdbGetRawBinary(pRaw, dataPos, pObj->key, sizeof(pObj->key));
+  dataPos += sizeof(pObj->key);
+  sdbGetRawInt8(pRaw, dataPos, &pObj->v8);
+  dataPos += sizeof(pObj->v8);
+  sdbGetRawInt16(pRaw, dataPos, &pObj->v16);
+  dataPos += sizeof(pObj->v16);
+  sdbGetRawInt32(pRaw, dataPos, &pObj->v32);
+  dataPos += sizeof(pObj->v32);
+  sdbGetRawInt64(pRaw, dataPos, &pObj->v64);
+  dataPos += sizeof(pObj->v64);
+  sdbGetRawBinary(pRaw, dataPos, pObj->vstr, sizeof(pObj->vstr));
+  dataPos += sizeof(pObj->vstr);
+
+  return pRow;
 }
 
-TEST_F(MndTestShow, 03_ShowMsg_Conn) {
+int32_t strInsert(SSdb *pSdb, SStrObj *pObj) { return 0; }
-  char passwd[] = "taosdata";
-  char secretEncrypt[TSDB_PASSWORD_LEN] = {0};
-  taosEncryptPass_c((uint8_t*)passwd, strlen(passwd), secretEncrypt);
 
-  SConnectReq connectReq = {0};
+int32_t strDelete(SSdb *pSdb, SStrObj *pObj, bool callFunc) { return 0; }
-  connectReq.pid = 1234;
-  strcpy(connectReq.app, "mnode_test_show");
-  strcpy(connectReq.db, "");
-  strcpy(connectReq.user, "root");
-  strcpy(connectReq.passwd, secretEncrypt);
 
-  int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq);
+int32_t strUpdate(SSdb *pSdb, SStrObj *pOld, SStrObj *pNew) {
-  void*   pReq = rpcMallocCont(contLen);
+  pOld->v8 = pNew->v8;
-  tSerializeSConnectReq(pReq, contLen, &connectReq);
+  pOld->v16 = pNew->v16;
-
+  pOld->v32 = pNew->v32;
-  SRpcMsg* pRsp = test.SendReq(TDMT_MND_CONNECT, pReq, contLen);
+  pOld->v64 = pNew->v64;
-  ASSERT_NE(pRsp, nullptr);
+  strcpy(pOld->vstr, pNew->vstr);
-  ASSERT_EQ(pRsp->code, 0);
+  return 0;
-
-  test.SendShowReq(TSDB_MGMT_TABLE_CONNS, "connections", "");
-  // EXPECT_EQ(test.GetShowRows(), 1);
 }
 
-TEST_F(MndTestShow, 04_ShowMsg_Cluster) {
+void strSetDefault(SStrObj *pObj, int32_t index) {
-  test.SendShowReq(TSDB_MGMT_TABLE_CLUSTER, "cluster", "");
+  memset(pObj, 0, sizeof(SStrObj));
-  EXPECT_EQ(test.GetShowRows(), 1);
+  snprintf(pObj->key, sizeof(pObj->key), "k%d", index * 1000);
+  pObj->v8 = index;
+  pObj->v16 = index;
+  pObj->v32 = index * 1000;
+  pObj->v64 = index * 1000;
+  snprintf(pObj->vstr, sizeof(pObj->vstr), "v%d", index * 1000);
+}
+
+int32_t strDefault(SMnode *pMnode) {
+  SStrObj  strObj;
+  SSdbRaw *pRaw = NULL;
+
+  strSetDefault(&strObj, 1);
+  pRaw = strEncode(&strObj);
+  sdbSetRawStatus(pRaw, SDB_STATUS_READY);
+  if (sdbWrite(pMnode->pSdb, pRaw) != 0) return -1;
+
+  strSetDefault(&strObj, 2);
+  pRaw = strEncode(&strObj);
+  sdbSetRawStatus(pRaw, SDB_STATUS_READY);
+  if (sdbWriteWithoutFree(pMnode->pSdb, pRaw) != 0) return -1;
+  sdbFreeRaw(pRaw);
+
+  return 0;
+}
+
+bool sdbTraverseSucc1(SMnode *pMnode, SStrObj *pObj, int32_t *p1, int32_t *p2, int32_t *p3) {
+  if (pObj->v8 == 1) {
+    *p1 = *p2 + *p3 + pObj->v8;
+  }
+  return true;
+}
+
+bool sdbTraverseSucc2(SMnode *pMnode, SStrObj *pObj, int32_t *p1, int32_t *p2, int32_t *p3) {
+  *p1 = *p2 + *p3 + pObj->v8;
+  return true;
+}
+
+bool sdbTraverseFail(SMnode *pMnode, SStrObj *pObj, int32_t *p1, int32_t *p2, int32_t *p3) {
+  *p1 = *p2 + *p3;
+  return false;
+}
+
+TEST_F(MndTestSdb, 01_Write) {
+  void    *pIter;
+  int32_t  num;
+  SStrObj *pObj;
+  SMnode   mnode;
+  SSdb    *pSdb;
+  SSdbOpt  opt = {0};
+  int32_t  p1 = 0;
+  int32_t  p2 = 111;
+  int32_t  p3 = 222;
+
+  mnode.v100 = 100;
+  mnode.v200 = 200;
+  opt.pMnode = &mnode;
+  opt.path = "/tmp/mnode_test_sdb";
+  taosRemoveDir(opt.path);
+
+  SSdbTable strTable = {
+      .sdbType = SDB_USER,
+      .keyType = SDB_KEY_BINARY,
+      .deployFp = (SdbDeployFp)strDefault,
+      .encodeFp = (SdbEncodeFp)strEncode,
+      .decodeFp = (SdbDecodeFp)strDecode,
+      .insertFp = (SdbInsertFp)strInsert,
+      .updateFp = (SdbUpdateFp)strUpdate,
+      .deleteFp = (SdbDeleteFp)strDelete,
+  };
+
+  pSdb = sdbInit(&opt);
+  mnode.pSdb = pSdb;
+
+  ASSERT_NE(pSdb, nullptr);
+  ASSERT_EQ(sdbSetTable(pSdb, strTable), 0);
+  ASSERT_EQ(sdbDeploy(pSdb), 0);
+#if 0
+  pObj = (SStrObj *)sdbAcquire(pSdb, SDB_USER, "k1000");
+  ASSERT_NE(pObj, nullptr);
+  EXPECT_STREQ(pObj->key, "k1000");
+  EXPECT_STREQ(pObj->vstr, "v1000");
+  EXPECT_EQ(pObj->v8, 1);
+  EXPECT_EQ(pObj->v16, 1);
+  EXPECT_EQ(pObj->v32, 1000);
+  EXPECT_EQ(pObj->v64, 1000);
+  sdbRelease(pSdb, pObj);
+
+  pObj = (SStrObj *)sdbAcquire(pSdb, SDB_USER, "k2000");
+  ASSERT_NE(pObj, nullptr);
+  EXPECT_STREQ(pObj->key, "k2000");
+  EXPECT_STREQ(pObj->vstr, "v2000");
+  EXPECT_EQ(pObj->v8, 2);
+  EXPECT_EQ(pObj->v16, 2);
+  EXPECT_EQ(pObj->v32, 2000);
+  EXPECT_EQ(pObj->v64, 2000);
+  sdbRelease(pSdb, pObj);
+
+  pObj = (SStrObj *)sdbAcquire(pSdb, SDB_USER, "k200");
+  ASSERT_EQ(pObj, nullptr);
+
+  pIter = NULL;
+  num = 0;
+  do {
+    pIter = sdbFetch(pSdb, SDB_USER, pIter, (void **)&pObj);
+    if (pIter == NULL) break;
+    ASSERT_NE(pObj, nullptr);
+    num++;
+    sdbRelease(pSdb, pObj);
+  } while (1);
+  EXPECT_EQ(num, 2);
+
+  do {
+    pIter = sdbFetch(pSdb, SDB_USER, pIter, (void **)&pObj);
+    if (pIter == NULL) break;
+    if (strcmp(pObj->key, "k1000") == 0) {
+      sdbCancelFetch(pSdb, pIter);
+      break;
+    }
+  } while (1);
+  EXPECT_STREQ(pObj->key, "k1000");
+
+  p1 = 0;
+  p2 = 111;
+  p3 = 222;
+  sdbTraverse(pSdb, SDB_USER, (sdbTraverseFp)sdbTraverseSucc2, &p1, &p2, &p3);
+  EXPECT_EQ(p1, 334);
+
+  p1 = 0;
+  p2 = 111;
+  p3 = 222;
+  sdbTraverse(pSdb, SDB_USER, (sdbTraverseFp)sdbTraverseSucc2, &p1, &p2, &p3);
+  EXPECT_EQ(p1, 669);
+
+  p1 = 0;
+  p2 = 111;
+  p3 = 222;
+  sdbTraverse(pSdb, SDB_USER, (sdbTraverseFp)sdbTraverseFail, &p1, &p2, &p3);
+  EXPECT_EQ(p1, 333);
+
+  EXPECT_EQ(sdbGetSize(pSdb, SDB_USER), 2);
+  EXPECT_EQ(sdbGetMaxId(pSdb, SDB_USER), -1);
+  EXPECT_EQ(sdbGetTableVer(pSdb, SDB_USER), 2);
+  EXPECT_EQ(sdbUpdateVer(pSdb, 0), 2);
+  EXPECT_EQ(sdbUpdateVer(pSdb, 1), 3);
+  EXPECT_EQ(sdbUpdateVer(pSdb, -1), 2);
+
+  // insert, call func
+
+  // update, call func
+
+  // delete, call func 2
+
+  // write version
+
+  // sdb Write ver
+
+  // sdbRead
+#endif
+  ASSERT_EQ(sdbWriteFile(pSdb), 0);
+  sdbCleanup(pSdb);
+}
+
+TEST_F(MndTestSdb, 01_Read) {
+  void    *pIter;
+  int32_t  num;
+  SStrObj *pObj;
+  SMnode   mnode;
+  SSdb    *pSdb;
+  SSdbOpt  opt = {0};
+  int32_t  p1 = 0;
+  int32_t  p2 = 111;
+  int32_t  p3 = 222;
+
+  mnode.v100 = 100;
+  mnode.v200 = 200;
+  opt.pMnode = &mnode;
+  opt.path = "/tmp/mnode_test_sdb";
+  taosRemoveDir(opt.path);
+
+  SSdbTable strTable = {
+      .sdbType = SDB_USER,
+      .keyType = SDB_KEY_BINARY,
+      .deployFp = (SdbDeployFp)strDefault,
+      .encodeFp = (SdbEncodeFp)strEncode,
+      .decodeFp = (SdbDecodeFp)strDecode,
+      .insertFp = (SdbInsertFp)strInsert,
+      .updateFp = (SdbUpdateFp)strDelete,
+      .deleteFp = (SdbDeleteFp)strUpdate,
+  };
+
+  pSdb = sdbInit(&opt);
+  mnode.pSdb = pSdb;
+
+  ASSERT_EQ(sdbReadFile(pSdb), 0);
+  sdbCleanup(pSdb);
 }

source/dnode/mnode/impl/test/sma/sma.cpp

@@ -40,24 +40,22 @@ void* MndTestSma::BuildCreateDbReq(const char* dbname, int32_t* pContLen) {
   SCreateDbReq createReq = {0};
   strcpy(createReq.db, dbname);
   createReq.numOfVgroups = 2;
-  createReq.cacheBlockSize = 16;
+  createReq.buffer = -1;
-  createReq.totalBlocks = 10;
+  createReq.pageSize = -1;
+  createReq.pages = -1;
   createReq.daysPerFile = 10 * 1440;
   createReq.daysToKeep0 = 3650 * 1440;
   createReq.daysToKeep1 = 3650 * 1440;
   createReq.daysToKeep2 = 3650 * 1440;
   createReq.minRows = 100;
   createReq.maxRows = 4096;
-  createReq.commitTime = 3600;
   createReq.fsyncPeriod = 3000;
   createReq.walLevel = 1;
   createReq.precision = 0;
   createReq.compression = 2;
   createReq.replications = 1;
   createReq.strict = 1;
-  createReq.update = 0;
   createReq.cacheLastRow = 0;
-  createReq.ttl = 1;
   createReq.ignoreExist = 1;
 
   int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq);

source/dnode/mnode/impl/test/stb/stb.cpp

@@ -41,24 +41,22 @@ void* MndTestStb::BuildCreateDbReq(const char* dbname, int32_t* pContLen) {
   SCreateDbReq createReq = {0};
   strcpy(createReq.db, dbname);
   createReq.numOfVgroups = 2;
-  createReq.cacheBlockSize = 16;
+  createReq.buffer = -1;
-  createReq.totalBlocks = 10;
+  createReq.pageSize = -1;
+  createReq.pages = -1;
   createReq.daysPerFile = 1000;
   createReq.daysToKeep0 = 3650;
   createReq.daysToKeep1 = 3650;
   createReq.daysToKeep2 = 3650;
   createReq.minRows = 100;
   createReq.maxRows = 4096;
-  createReq.commitTime = 3600;
   createReq.fsyncPeriod = 3000;
   createReq.walLevel = 1;
   createReq.precision = 0;
   createReq.compression = 2;
   createReq.replications = 1;
   createReq.strict = 1;
-  createReq.update = 0;
   createReq.cacheLastRow = 0;
-  createReq.ttl = 1;
   createReq.ignoreExist = 1;
 
   int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq);
@@ -343,7 +341,6 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
     EXPECT_EQ(metaRsp.numOfTags, 3);
     EXPECT_EQ(metaRsp.precision, TSDB_TIME_PRECISION_MILLI);
     EXPECT_EQ(metaRsp.tableType, TSDB_SUPER_TABLE);
-    EXPECT_EQ(metaRsp.update, 0);
     EXPECT_EQ(metaRsp.sversion, 1);
     EXPECT_EQ(metaRsp.tversion, 0);
     EXPECT_GT(metaRsp.suid, 0);

source/dnode/mnode/impl/test/topic/topic.cpp

@@ -33,24 +33,22 @@ void* MndTestTopic::BuildCreateDbReq(const char* dbname, int32_t* pContLen) {
   SCreateDbReq createReq = {0};
   strcpy(createReq.db, dbname);
   createReq.numOfVgroups = 2;
-  createReq.cacheBlockSize = 16;
+  createReq.buffer = -1;
-  createReq.totalBlocks = 10;
+  createReq.pageSize = -1;
+  createReq.pages = -1;
   createReq.daysPerFile = 10 * 1440;
   createReq.daysToKeep0 = 3650 * 1440;
   createReq.daysToKeep1 = 3650 * 1440;
   createReq.daysToKeep2 = 3650 * 1440;
   createReq.minRows = 100;
   createReq.maxRows = 4096;
-  createReq.commitTime = 3600;
   createReq.fsyncPeriod = 3000;
   createReq.walLevel = 1;
   createReq.precision = 0;
   createReq.compression = 2;
   createReq.replications = 1;
   createReq.strict = 1;
-  createReq.update = 0;
   createReq.cacheLastRow = 0;
-  createReq.ttl = 1;
   createReq.ignoreExist = 1;
 
   int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq);

source/dnode/mnode/impl/test/user/user.cpp

@@ -286,24 +286,22 @@ TEST_F(MndTestUser, 03_Alter_User) {
     SCreateDbReq createReq = {0};
     strcpy(createReq.db, "1.d2");
     createReq.numOfVgroups = 2;
-    createReq.cacheBlockSize = 16;
+    createReq.buffer = -1;
-    createReq.totalBlocks = 10;
+    createReq.pageSize = -1;
+    createReq.pages = -1;
     createReq.daysPerFile = 10 * 1440;
     createReq.daysToKeep0 = 3650 * 1440;
     createReq.daysToKeep1 = 3650 * 1440;
     createReq.daysToKeep2 = 3650 * 1440;
     createReq.minRows = 100;
     createReq.maxRows = 4096;
-    createReq.commitTime = 3600;
     createReq.fsyncPeriod = 3000;
     createReq.walLevel = 1;
     createReq.precision = 0;
     createReq.compression = 2;
     createReq.replications = 1;
     createReq.strict = 1;
-    createReq.update = 0;
     createReq.cacheLastRow = 0;
-    createReq.ttl = 1;
     createReq.ignoreExist = 1;
 
     int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq);

source/dnode/mnode/sdb/CMakeLists.txt

@@ -6,5 +6,5 @@ target_include_directories(
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 target_link_libraries(
-    sdb os common util
+    sdb os common util wal
 )

source/dnode/mnode/sdb/inc/sdbInt.h

@@ -52,6 +52,8 @@ typedef struct SSdbRow {
 const char *sdbTableName(ESdbType type);
 void        sdbPrintOper(SSdb *pSdb, SSdbRow *pRow, const char *oper);
 
+void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/mnode/sdb/src/sdb.c

@@ -141,7 +141,7 @@ int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) {
 }
 
 static int32_t sdbCreateDir(SSdb *pSdb) {
-  if (taosMkDir(pSdb->currDir) != 0) {
+  if (taosMulMkDir(pSdb->currDir) != 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     mError("failed to create dir:%s since %s", pSdb->currDir, terrstr());
     return -1;

source/dnode/mnode/sdb/src/sdbFile.c

@@ -16,6 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "sdbInt.h"
 #include "tchecksum.h"
+#include "wal.h"
 
 #define SDB_TABLE_SIZE 24
 #define SDB_RESERVE_SIZE 512
@@ -137,7 +138,7 @@ int32_t sdbReadFile(SSdb *pSdb) {
   int32_t readLen = 0;
   int64_t ret = 0;
 
-  SSdbRaw *pRaw = taosMemoryMalloc(SDB_MAX_SIZE);
+  SSdbRaw *pRaw = taosMemoryMalloc(WAL_MAX_SIZE + 100);
   if (pRaw == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     mError("failed read file since %s", terrstr());

source/dnode/qnode/src/qnode.c

@@ -17,7 +17,7 @@
 #include "qndInt.h"
 #include "query.h"
 #include "qworker.h"
-//#include "tudf.h"
+#include "libs/function/function.h"
 
 SQnode *qndOpen(const SQnodeOpt *pOption) {
   SQnode *pQnode = taosMemoryCalloc(1, sizeof(SQnode));
@@ -26,7 +26,9 @@ SQnode *qndOpen(const SQnodeOpt *pOption) {
     return NULL;
   }
 
-  //udfcOpen();
+  if (udfcOpen() != 0) {
+    qError("qnode can not open udfc");
+  }
 
   if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, NULL, (void **)&pQnode->pQuery, &pOption->msgCb)) {
     taosMemoryFreeClear(pQnode);
@@ -40,7 +42,7 @@ SQnode *qndOpen(const SQnodeOpt *pOption) {
 void qndClose(SQnode *pQnode) {
   qWorkerDestroy((void **)&pQnode->pQuery);
 
-  //udfcClose();
+  udfcClose();
 
   taosMemoryFree(pQnode);
 }

source/dnode/vnode/inc/vnode.h

@@ -157,7 +157,6 @@ struct SVnodeCfg {
   int32_t  szCache;
   uint64_t szBuf;
   bool     isHeap;
-  int8_t   streamMode;
   bool     isWeak;
   STsdbCfg tsdbCfg;
   SWalCfg  walCfg;

source/dnode/vnode/src/meta/metaQuery.c

@@ -159,7 +159,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo
 
   // decode
   pBuf = pVal;
-  pSW = taosMemoryMalloc(sizeof(pSW));
+  pSW = taosMemoryMalloc(sizeof(SSchemaWrapper));
 
   tCoderInit(&coder, TD_LITTLE_ENDIAN, pVal, vLen, TD_DECODER);
   tDecodeSSchemaWrapper(&coder, pSW);

source/dnode/vnode/src/tq/tq.c

@@ -864,7 +864,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) {
 }
 
 int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) {
-  SStreamTask* pTask = taosMemoryMalloc(sizeof(SStreamTask));
+  SStreamTask* pTask = taosMemoryCalloc(1, sizeof(SStreamTask));
   if (pTask == NULL) {
     return -1;
   }

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -1490,6 +1490,10 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
     pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row1));
   }
 
+#ifdef TD_DEBUG_PRINT_ROW
+  tdSRowPrint(row1, pSchema1, __func__);
+#endif
+
   if (isRow1DataRow) {
     numOfColsOfRow1 = schemaNCols(pSchema1);
   } else {

source/dnode/vnode/src/vnd/vnodeCfg.c

@@ -23,7 +23,6 @@ const SVnodeCfg vnodeCfgDefault = {
     .szCache = 256,
     .szBuf = 96 * 1024 * 1024,
     .isHeap = false,
-    .streamMode = 0,
     .isWeak = 0,
     .tsdbCfg = {.precision = TSDB_TIME_PRECISION_MILLI,
                 .update = 0,
@@ -56,7 +55,6 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
   if (tjsonAddIntegerToObject(pJson, "szCache", pCfg->szCache) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "szBuf", pCfg->szBuf) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "isHeap", pCfg->isHeap) < 0) return -1;
-  if (tjsonAddIntegerToObject(pJson, "streamMode", pCfg->streamMode) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "isWeak", pCfg->isWeak) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "update", pCfg->tsdbCfg.update) < 0) return -1;
@@ -104,7 +102,6 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
   if (tjsonGetNumberValue(pJson, "szCache", pCfg->szCache) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "szBuf", pCfg->szBuf) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "isHeap", pCfg->isHeap) < 0) return -1;
-  if (tjsonGetNumberValue(pJson, "streamMode", pCfg->streamMode) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "isWeak", pCfg->isWeak) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "update", pCfg->tsdbCfg.update) < 0) return -1;

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -61,7 +61,6 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
   pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
   len = pMsg->contLen - sizeof(SMsgHead);
 
-  // todo: change the interface here
   if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) {
     vError("vgId: %d failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno));
     return -1;

source/libs/catalog/test/catalogTests.cpp

@@ -97,24 +97,22 @@ void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) {
   SCreateDbReq createReq = {0};
   strcpy(createReq.db, "1.db1");
   createReq.numOfVgroups = 2;
-  createReq.cacheBlockSize = 16;
+  createReq.buffer = -1;
-  createReq.totalBlocks = 10;
+  createReq.pageSize = -1;
+  createReq.pages = -1;
   createReq.daysPerFile = 10;
   createReq.daysToKeep0 = 3650;
   createReq.daysToKeep1 = 3650;
   createReq.daysToKeep2 = 3650;
   createReq.minRows = 100;
   createReq.maxRows = 4096;
-  createReq.commitTime = 3600;
   createReq.fsyncPeriod = 3000;
   createReq.walLevel = 1;
   createReq.precision = 0;
   createReq.compression = 2;
   createReq.replications = 1;
   createReq.strict = 1;
-  createReq.update = 0;
   createReq.cacheLastRow = 0;
-  createReq.ttl = 1;
   createReq.ignoreExist = 1;
 
   int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq);
@@ -254,7 +252,6 @@ void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) {
   rspMsg->numOfColumns = ctgTestColNum;
   rspMsg->precision = 1 + 1;
   rspMsg->tableType = TSDB_SUPER_TABLE;
-  rspMsg->update = 1 + 1;
   rspMsg->sversion = ctgTestSVersion + 1;
   rspMsg->tversion = ctgTestTVersion + 1;
   rspMsg->suid = ctgTestSuid + 1;
@@ -333,7 +330,6 @@ void ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *
   metaRsp.numOfColumns = ctgTestColNum;
   metaRsp.precision = 1;
   metaRsp.tableType = TSDB_NORMAL_TABLE;
-  metaRsp.update = 1;
   metaRsp.sversion = ctgTestSVersion;
   metaRsp.tversion = ctgTestTVersion;
   metaRsp.suid = 0;
@@ -379,7 +375,6 @@ void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg
   metaRsp.numOfColumns = ctgTestColNum;
   metaRsp.precision = 1;
   metaRsp.tableType = TSDB_CHILD_TABLE;
-  metaRsp.update = 1;
   metaRsp.sversion = ctgTestSVersion;
   metaRsp.tversion = ctgTestTVersion;
   metaRsp.suid = 0x0000000000000002;
@@ -426,7 +421,6 @@ void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg
   metaRsp.numOfColumns = ctgTestColNum;
   metaRsp.precision = 1;
   metaRsp.tableType = TSDB_SUPER_TABLE;
-  metaRsp.update = 1;
   metaRsp.sversion = ctgTestSVersion;
   metaRsp.tversion = ctgTestTVersion;
   metaRsp.suid = ctgTestSuid;
@@ -475,7 +469,6 @@ void ctgTestRspMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRp
   metaRsp.numOfColumns = ctgTestColNum;
   metaRsp.precision = 1;
   metaRsp.tableType = TSDB_SUPER_TABLE;
-  metaRsp.update = 1;
   metaRsp.sversion = ctgTestSVersion;
   metaRsp.tversion = ctgTestTVersion;
   metaRsp.suid = ctgTestSuid + idx;

source/libs/executor/src/executorimpl.c

@@ -482,7 +482,8 @@ static SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowI
 
     // add a new result set for a new group
     SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset};
-    taosHashPut(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pos, sizeof(SResultRowPosition));
+    taosHashPut(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pos,
+                sizeof(SResultRowPosition));
   }
 
   // 2. set the new time window to be the new active time window
@@ -1034,8 +1035,8 @@ void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlo
   }
 }
 
-static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunctParam* pFuncParam,
+static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunctParam* pFuncParam, int32_t paramIndex,
-                                             int32_t paramIndex, int32_t numOfRows) {
+                                             int32_t numOfRows) {
   SColumnInfoData* pColInfo = NULL;
   if (pInput->pData[paramIndex] == NULL) {
     pColInfo = taosMemoryCalloc(1, sizeof(SColumnInfoData));
@@ -1066,9 +1067,9 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc
       colDataAppendDouble(pColInfo, i, &v);
     }
   } else if (type == TSDB_DATA_TYPE_VARCHAR) {
-    char *tmp = taosMemoryMalloc(pFuncParam->param.nLen + VARSTR_HEADER_SIZE);
+    char* tmp = taosMemoryMalloc(pFuncParam->param.nLen + VARSTR_HEADER_SIZE);
     STR_WITH_SIZE_TO_VARSTR(tmp, pFuncParam->param.pz, pFuncParam->param.nLen);
-    for(int32_t i = 0; i < numOfRows; ++i) {
+    for (int32_t i = 0; i < numOfRows; ++i) {
       colDataAppend(pColInfo, i, tmp, false);
     }
   }
@@ -1491,8 +1492,8 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
   if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
     SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
     pos->groupId = tableGroupId;
-    pos->pos     = (SResultRowPosition) {.pageId = pResult->pageId, .offset = pResult->offset};
+    pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset};
-    *(int64_t*) pos->key = pResult->win.skey;
+    *(int64_t*)pos->key = pResult->win.skey;
 
     taosArrayPush(pUpdated, &pos);
   }
@@ -1569,8 +1570,8 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
     if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
       SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
       pos->groupId = tableGroupId;
-      pos->pos     = (SResultRowPosition) {.pageId = pResult->pageId, .offset = pResult->offset};
+      pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset};
-      *(int64_t*) pos->key = pResult->win.skey;
+      *(int64_t*)pos->key = pResult->win.skey;
 
       taosArrayPush(pUpdated, &pos);
     }
@@ -1897,7 +1898,14 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
       pCtx->functionId = pExpr->pExpr->_function.pFunctNode->funcId;
 
       if (fmIsAggFunc(pCtx->functionId) || fmIsNonstandardSQLFunc(pCtx->functionId)) {
+        bool isUdaf = fmIsUserDefinedFunc(pCtx->functionId);
+        if (!isUdaf) {
           fmGetFuncExecFuncs(pCtx->functionId, &pCtx->fpSet);
+        } else {
+          char *udfName = pExpr->pExpr->_function.pFunctNode->functionName;
+          strncpy(pCtx->udfName, udfName, strlen(udfName));
+          fmGetUdafExecFuncs(pCtx->functionId, &pCtx->fpSet);
+        }
         pCtx->fpSet.getEnv(pExpr->pExpr->_function.pFunctNode, &env);
       } else {
         fmGetScalarFuncExecFuncs(pCtx->functionId, &pCtx->sfp);
@@ -2859,24 +2867,24 @@ void finalizeUpdatedResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbased
   size_t num = taosArrayGetSize(pUpdateList);
 
   for (int32_t i = 0; i < num; ++i) {
-    SResKeyPos * pPos = taosArrayGetP(pUpdateList, i);
+    SResKeyPos* pPos = taosArrayGetP(pUpdateList, i);
 
     SFilePage*  bufPage = getBufPage(pBuf, pPos->pos.pageId);
     SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->pos.offset);
-//
+    //
     for (int32_t j = 0; j < numOfOutput; ++j) {
       pCtx[j].resultInfo = getResultCell(pRow, j, rowCellInfoOffset);
-//
+      //
       struct SResultRowEntryInfo* pResInfo = pCtx[j].resultInfo;
-//      if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) {
+      //      if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) {
-//        continue;
+      //        continue;
-//      }
+      //      }
-//
+      //
-//      if (pCtx[j].fpSet.process) {  // TODO set the dummy function.
+      //      if (pCtx[j].fpSet.process) {  // TODO set the dummy function.
-////        pCtx[j].fpSet.finalize(&pCtx[j]);
+      ////        pCtx[j].fpSet.finalize(&pCtx[j]);
-//        pResInfo->initialized = true;
+      //        pResInfo->initialized = true;
-//      }
+      //      }
-//
+      //
       if (pRow->numOfRows < pResInfo->numOfRes) {
         pRow->numOfRows = pResInfo->numOfRes;
       }
@@ -2998,9 +3006,8 @@ void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, u
   SqlFunctionCtx* pCtx = pAggInfo->binfo.pCtx;
   int32_t*        rowCellInfoOffset = pAggInfo->binfo.rowCellInfoOffset;
 
-  SResultRow* pResultRow =
+  SResultRow* pResultRow = doSetResultOutBufByKey(pAggInfo->aggSup.pResultBuf, pResultRowInfo, uid, (char*)&groupId,
-      doSetResultOutBufByKey(pAggInfo->aggSup.pResultBuf, pResultRowInfo, uid, (char*)&groupId, sizeof(groupId),
+                                                  sizeof(groupId), true, groupId, pTaskInfo, false, &pAggInfo->aggSup);
-                                true, groupId, pTaskInfo, false, &pAggInfo->aggSup);
   assert(pResultRow != NULL);
 
   /*
@@ -3098,7 +3105,7 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased
   }
 
   for (int32_t i = start; (i < numOfRows) && (i >= 0); i += step) {
-    SResKeyPos *pPos = taosArrayGetP(pGroupResInfo->pRows, i);
+    SResKeyPos* pPos = taosArrayGetP(pGroupResInfo->pRows, i);
     SFilePage*  page = getBufPage(pBuf, pPos->pos.pageId);
 
     SResultRow* pRow = (SResultRow*)((char*)page + pPos->pos.offset);
@@ -3744,7 +3751,7 @@ static void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo,
   ASSERT(numOfSrcCols >= pBlock->info.numOfCols);
 
   int32_t i = 0, j = 0;
-  while(i < numOfSrcCols && j < taosArrayGetSize(pColMatchInfo)) {
+  while (i < numOfSrcCols && j < taosArrayGetSize(pColMatchInfo)) {
     SColumnInfoData* p = taosArrayGet(pCols, i);
     SColMatchInfo*   pmInfo = taosArrayGet(pColMatchInfo, j);
     if (!pmInfo->output) {
@@ -3770,10 +3777,10 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI
   blockDataEnsureCapacity(pRes, numOfRows);
 
   if (pColList == NULL) {  // data from other sources
-    int32_t dataLen = *(int32_t*) pData;
+    int32_t dataLen = *(int32_t*)pData;
     pData += sizeof(int32_t);
 
-    pRes->info.groupId = *(uint64_t*) pData;
+    pRes->info.groupId = *(uint64_t*)pData;
     pData += sizeof(uint64_t);
 
     int32_t* colLen = (int32_t*)pData;
@@ -3803,8 +3810,8 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI
         memcpy(pColInfoData->pData, pStart, colLen[i]);
       }
 
-      //TODO setting this flag to true temporarily so aggregate function on stable will
+      // TODO setting this flag to true temporarily so aggregate function on stable will
-      //examine NULL value for non-primary key column
+      // examine NULL value for non-primary key column
       pColInfoData->hasNull = true;
       pStart += colLen[i];
     }
@@ -3840,11 +3847,11 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI
 
     blockDataEnsureCapacity(&block, numOfRows);
 
-    int32_t dataLen = *(int32_t*) pStart;
+    int32_t  dataLen = *(int32_t*)pStart;
-    uint64_t groupId = *(uint64_t*) (pStart + sizeof(int32_t));
+    uint64_t groupId = *(uint64_t*)(pStart + sizeof(int32_t));
     pStart += sizeof(int32_t) + sizeof(uint64_t);
 
-    int32_t* colLen = (int32_t*) (pStart);
+    int32_t* colLen = (int32_t*)(pStart);
     pStart += sizeof(int32_t) * numOfCols;
 
     for (int32_t i = 0; i < numOfCols; ++i) {
@@ -4219,8 +4226,8 @@ SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock
   pOperator->numOfOutput = pBlock->info.numOfCols;
   pOperator->pTaskInfo = pTaskInfo;
 
-  pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL, destroyExchangeOperatorInfo,
+  pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL,
-                                         NULL, NULL, NULL);
+                                         destroyExchangeOperatorInfo, NULL, NULL, NULL);
 
 #if 1
   {  // todo refactor
@@ -4709,8 +4716,8 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR
   pOperator->info = pInfo;
 
   pOperator->pTaskInfo = pTaskInfo;
-  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSort, NULL, NULL, destroyOrderOperatorInfo,
+  pOperator->fpSet =
-                                         NULL, NULL, NULL);
+      createOperatorFpSet(operatorDummyOpenFn, doSort, NULL, NULL, destroyOrderOperatorInfo, NULL, NULL, NULL);
 
   int32_t code = appendDownstream(pOperator, &downstream, 1);
   return pOperator;
@@ -4926,8 +4933,8 @@ bool aggDecodeResultRow(SOperatorInfo* pOperator, SAggSupporter* pSup, SOptrBasi
     initResultRow(resultRow);
     prepareResultListBuffer(&pInfo->resultRowInfo, pOperator->pTaskInfo->env);
     //    pInfo->resultRowInfo.cur = pInfo->resultRowInfo.size;
-//    pInfo->resultRowInfo.pPosition[pInfo->resultRowInfo.size++] =
+    //    pInfo->resultRowInfo.pPosition[pInfo->resultRowInfo.size++] =
-//        (SResultRowPosition){.pageId = resultRow->pageId, .offset = resultRow->offset};
+    //        (SResultRowPosition){.pageId = resultRow->pageId, .offset = resultRow->offset};
     pInfo->resultRowInfo.cur = (SResultRowPosition){.pageId = resultRow->pageId, .offset = resultRow->offset};
   }
 
@@ -5008,7 +5015,8 @@ static int32_t handleLimitOffset(SOperatorInfo* pOperator, SSDataBlock* pBlock)
   // todo optimize performance
   // If there are slimit/soffset value exists, multi-round result can not be packed into one group, since the
   // they may not belong to the same group the limit/offset value is not valid in this case.
-  if (pRes->info.rows >= pOperator->resultInfo.threshold || pProjectInfo->slimit.offset != -1 || pProjectInfo->slimit.limit != -1) {
+  if (pRes->info.rows >= pOperator->resultInfo.threshold || pProjectInfo->slimit.offset != -1 ||
+      pProjectInfo->slimit.limit != -1) {
     return PROJECT_RETRIEVE_DONE;
   } else {  // not full enough, continue to accumulate the output data in the buffer.
     return PROJECT_RETRIEVE_CONTINUE;
@@ -5284,7 +5292,7 @@ static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator, bool* newgroup) {
   setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
   //  finalizeQueryResult(pSliceInfo->binfo.pCtx, pOperator->numOfOutput);
 
-//  initGroupedResultInfo(&pSliceInfo->groupResInfo, &pSliceInfo->binfo.resultRowInfo);
+  //  initGroupedResultInfo(&pSliceInfo->groupResInfo, &pSliceInfo->binfo.resultRowInfo);
   //  doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pSliceInfo->pRes);
 
   if (pSliceInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pSliceInfo->groupResInfo)) {
@@ -5335,7 +5343,7 @@ static SSDataBlock* doSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgroup
   finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf,
                                 &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset);
 
-//  initGroupedResultInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo);
+  //  initGroupedResultInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo);
   OPTR_SET_OPENED(pOperator);
 
   blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
@@ -5677,8 +5685,8 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n
   pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput);
   pAggSup->keyBuf = taosMemoryCalloc(1, keyBufSize + POINTER_BYTES + sizeof(int64_t));
   pAggSup->pResultRowHashTable = taosHashInit(10, hashFn, true, HASH_NO_LOCK);
-//  pAggSup->pResultRowListSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK);
+  //  pAggSup->pResultRowListSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK);
-//  pAggSup->pResultRowArrayList = taosArrayInit(10, sizeof(SResultRowCell));
+  //  pAggSup->pResultRowArrayList = taosArrayInit(10, sizeof(SResultRowCell));
 
   if (pAggSup->keyBuf == NULL /*|| pAggSup->pResultRowArrayList == NULL || pAggSup->pResultRowListSet == NULL*/ ||
       pAggSup->pResultRowHashTable == NULL) {
@@ -5696,8 +5704,8 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n
 static void cleanupAggSup(SAggSupporter* pAggSup) {
   taosMemoryFreeClear(pAggSup->keyBuf);
   taosHashCleanup(pAggSup->pResultRowHashTable);
-//  taosHashCleanup(pAggSup->pResultRowListSet);
+  //  taosHashCleanup(pAggSup->pResultRowListSet);
-//  taosArrayDestroy(pAggSup->pResultRowArrayList);
+  //  taosArrayDestroy(pAggSup->pResultRowArrayList);
   destroyDiskbasedBuf(pAggSup->pResultBuf);
 }
 
@@ -5708,7 +5716,7 @@ int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInf
 
   doInitAggInfoSup(pAggSup, pBasicInfo->pCtx, numOfCols, keyBufSize, pkey);
 
-  for(int32_t i = 0; i < numOfCols; ++i) {
+  for (int32_t i = 0; i < numOfCols; ++i) {
     pBasicInfo->pCtx[i].pBuf = pAggSup->pResultBuf;
   }
 
@@ -5725,6 +5733,10 @@ void initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows) {
 }
 
 static STableQueryInfo* initTableQueryInfo(const STableGroupInfo* pTableGroupInfo) {
+  if (pTableGroupInfo->numOfTables == 0) {
+    return NULL;
+  }
+
   STableQueryInfo* pTableQueryInfo = taosMemoryCalloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo));
   if (pTableQueryInfo == NULL) {
     return NULL;
@@ -5750,7 +5762,8 @@ static STableQueryInfo* initTableQueryInfo(const STableGroupInfo* pTableGroupInf
 
 SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
                                            SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo,
-                                           int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) {
+                                           int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo,
+                                           const STableGroupInfo* pTableGroupInfo) {
   SAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAggOperatorInfo));
   SOperatorInfo*    pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {
@@ -5764,7 +5777,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo*
   int32_t code =
       initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResultBlock, keyBufSize, pTaskInfo->id.str);
   pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo);
-  if (code != TSDB_CODE_SUCCESS || pInfo->pTableQueryInfo == NULL) {
+  if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
 
@@ -5919,8 +5932,8 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p
   pOperator->pExpr = pExprInfo;
   pOperator->numOfOutput = num;
 
-  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doProjectOperation, NULL, NULL, destroyProjectOperatorInfo,
+  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doProjectOperation, NULL, NULL,
-                                         NULL, NULL, NULL);
+                                         destroyProjectOperatorInfo, NULL, NULL, NULL);
 
   pOperator->pTaskInfo = pTaskInfo;
   int32_t code = appendDownstream(pOperator, &downstream, 1);
@@ -5947,7 +5960,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo*
 
   pInfo->order = TSDB_ORDER_ASC;
   pInfo->interval = *pInterval;
-//  pInfo->execModel      = OPTR_EXEC_MODEL_STREAM;
+  //  pInfo->execModel      = OPTR_EXEC_MODEL_STREAM;
   pInfo->execModel = pTaskInfo->execModel;
   pInfo->win = pTaskInfo->window;
   pInfo->twAggSup = *pTwAggSupp;
@@ -5977,8 +5990,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo*
   pOperator->numOfOutput = numOfCols;
   pOperator->info = pInfo;
 
-  pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, doStreamIntervalAgg, NULL, destroyIntervalOperatorInfo,
+  pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, doStreamIntervalAgg, NULL,
-                                         aggEncodeResultRow, aggDecodeResultRow, NULL);
+                                         destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
 
   code = appendDownstream(pOperator, &downstream, 1);
   if (code != TSDB_CODE_SUCCESS) {
@@ -5997,7 +6010,8 @@ _error:
 
 SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
                                                 SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
-                                                STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo) {
+                                                STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo,
+                                                SExecTaskInfo* pTaskInfo) {
   STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo));
   SOperatorInfo*              pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {
@@ -6035,8 +6049,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExpr
   pOperator->numOfOutput = numOfCols;
   pOperator->info = pInfo;
 
-  pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doStreamIntervalAgg, doStreamIntervalAgg, NULL, destroyIntervalOperatorInfo,
+  pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doStreamIntervalAgg, doStreamIntervalAgg, NULL,
-                                         aggEncodeResultRow, aggDecodeResultRow, NULL);
+                                         destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
 
   code = appendDownstream(pOperator, &downstream, 1);
   if (code != TSDB_CODE_SUCCESS) {
@@ -6045,13 +6059,12 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExpr
 
   return pOperator;
 
-  _error:
+_error:
   destroyIntervalOperatorInfo(pInfo, numOfCols);
   taosMemoryFreeClear(pInfo);
   taosMemoryFreeClear(pOperator);
   pTaskInfo->code = code;
   return NULL;
-
 }
 
 SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
@@ -6115,8 +6128,8 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInf
   pOperator->pTaskInfo = pTaskInfo;
   pOperator->info = pInfo;
 
-  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStateWindowAgg, NULL, NULL, destroyStateWindowOperatorInfo,
+  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStateWindowAgg, NULL, NULL,
-                                         aggEncodeResultRow, aggDecodeResultRow, NULL);
+                                         destroyStateWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
 
   int32_t code = appendDownstream(pOperator, &downstream, 1);
   return pOperator;
@@ -6161,8 +6174,8 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo
   pOperator->numOfOutput = numOfCols;
   pOperator->info = pInfo;
 
-  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, NULL, destroySWindowOperatorInfo,
+  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, NULL,
-                                         aggEncodeResultRow, aggDecodeResultRow, NULL);
+                                         destroySWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
   pOperator->pTaskInfo = pTaskInfo;
 
   code = appendDownstream(pOperator, &downstream, 1);
@@ -6250,8 +6263,8 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExp
   pOperator->numOfOutput = numOfCols;
   pOperator->info = pInfo;
 
-  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, NULL, destroySFillOperatorInfo,
+  pOperator->fpSet =
-                                         NULL, NULL, NULL);
+      createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, NULL, destroySFillOperatorInfo, NULL, NULL, NULL);
   pOperator->pTaskInfo = pTaskInfo;
   code = appendDownstream(pOperator, &downstream, 1);
   return pOperator;
@@ -6262,7 +6275,6 @@ _error:
   return NULL;
 }
 
-
 static int32_t getColumnIndexInSource(SQueriedTableInfo* pTableInfo, SExprBasicInfo* pExpr, SColumnInfo* pTagCols) {
   int32_t j = 0;
 
@@ -6496,8 +6508,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       }
 
       SInterval interval = extractIntervalInfo(pTableScanNode);
-      return createTableScanOperatorInfo(pDataReader, &cond, numOfCols, pTableScanNode->dataRequired, pTableScanNode->scanSeq, pColList,
+      return createTableScanOperatorInfo(pDataReader, &cond, numOfCols, pTableScanNode->dataRequired,
-                                         pResBlock, pScanPhyNode->node.pConditions, &interval, pTableScanNode->ratio, pTaskInfo);
+                                         pTableScanNode->scanSeq, pColList, pResBlock, pScanPhyNode->node.pConditions,
+                                         &interval, pTableScanNode->ratio, pTaskInfo);
     } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) {
       SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode;
       SSDataBlock*        pResBlock = createResDataBlock(pExchange->node.pOutputDataBlockDesc);
@@ -6505,14 +6518,16 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) {
       SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode;  // simple child table.
 
-      int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId);
+      int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo,
+                                        queryId, taskId);
       SArray* tableIdList = extractTableIdList(pTableGroupInfo);
 
       SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc);
 
       int32_t numOfCols = 0;
       SArray* pCols = extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols);
-      SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pResBlock, pCols, tableIdList, pTaskInfo);
+      SOperatorInfo* pOperator =
+          createStreamScanOperatorInfo(pHandle->reader, pResBlock, pCols, tableIdList, pTaskInfo);
       taosArrayDestroy(tableIdList);
       return pOperator;
     } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) {
@@ -6522,10 +6537,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       SSDataBlock* pResBlock = createResDataBlock(pScanNode->node.pOutputDataBlockDesc);
 
       int32_t numOfOutputCols = 0;
-      SArray* colList = extractColMatchInfo(pScanNode->pScanCols, pScanNode->node.pOutputDataBlockDesc, &numOfOutputCols);
+      SArray* colList =
+          extractColMatchInfo(pScanNode->pScanCols, pScanNode->node.pOutputDataBlockDesc, &numOfOutputCols);
       SOperatorInfo* pOperator = createSysTableScanOperatorInfo(
-          pHandle, pResBlock, &pScanNode->tableName, pScanNode->node.pConditions, pSysScanPhyNode->mgmtEpSet,
+          pHandle, pResBlock, &pScanNode->tableName, pScanNode->node.pConditions, pSysScanPhyNode->mgmtEpSet, colList,
-          colList, pTaskInfo, pSysScanPhyNode->showRewrite, pSysScanPhyNode->accountId);
+          pTaskInfo, pSysScanPhyNode->showRewrite, pSysScanPhyNode->accountId);
       return pOperator;
     } else {
       ASSERT(0);
@@ -6642,7 +6658,6 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
   return pOptr;
 }
 
-
 static int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode) {
   pCond->loadExternalRows = false;
 
@@ -6657,7 +6672,7 @@ static  int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableS
   pCond->twindow = pTableScanNode->scanRange;
 
 #if 1
-  //todo work around a problem, remove it later
+  // todo work around a problem, remove it later
   if ((pCond->order == TSDB_ORDER_ASC && pCond->twindow.skey > pCond->twindow.ekey) ||
       (pCond->order == TSDB_ORDER_DESC && pCond->twindow.skey < pCond->twindow.ekey)) {
     TSWAP(pCond->twindow.skey, pCond->twindow.ekey);
@@ -6710,7 +6725,7 @@ SArray* extractColumnInfo(SNodeList* pNodeList) {
 
       taosArrayPush(pList, &c);
     } else if (nodeType(pNode->pExpr) == QUERY_NODE_VALUE) {
-      SValueNode* pValNode = (SValueNode*) pNode->pExpr;
+      SValueNode* pValNode = (SValueNode*)pNode->pExpr;
       SColumn     c = {0};
       c.slotId = pNode->slotId;
       c.colId = pNode->slotId;
@@ -6918,7 +6933,8 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead
     goto _complete;
   }
 
-  (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoGroupInfo);
+  (*pTaskInfo)->pRoot =
+      createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoGroupInfo);
   if (NULL == (*pTaskInfo)->pRoot) {
     code = terrno;
     goto _complete;
@@ -7256,8 +7272,8 @@ SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOf
   pOperator->info = pInfo;
   pOperator->pTaskInfo = pTaskInfo;
 
-  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, NULL, destroyBasicOperatorInfo,
+  pOperator->fpSet =
-                                         NULL, NULL, NULL);
+      createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, NULL, destroyBasicOperatorInfo, NULL, NULL, NULL);
   int32_t code = appendDownstream(pOperator, pDownstream, numOfDownstream);
   return pOperator;
 

source/libs/function/CMakeLists.txt

@@ -51,6 +51,21 @@ target_link_libraries(
     udf1 PUBLIC os
 )
 
+add_library(udf2 MODULE test/udf2.c)
+target_include_directories(
+        udf2
+        PUBLIC
+            "${TD_SOURCE_DIR}/include/libs/function"
+            "${TD_SOURCE_DIR}/include/util"
+            "${TD_SOURCE_DIR}/include/common"
+            "${TD_SOURCE_DIR}/include/client"
+            "${TD_SOURCE_DIR}/include/os"
+        PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+)
+target_link_libraries(
+    udf2 PUBLIC os
+)
+
 #SET(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/build/bin)
 add_executable(udfd src/udfd.c)
 target_include_directories(

source/libs/function/inc/builtinsimpl.h

@@ -25,6 +25,7 @@ extern "C" {
 
 bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, char* finalResult);
 
 EFuncDataRequired countDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow);
 bool getCountFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);

source/libs/function/inc/tudfInt.h

@@ -43,6 +43,9 @@ typedef struct SUdfSetupRequest {
 
 typedef struct SUdfSetupResponse {
   int64_t udfHandle;
+  int8_t outputType;
+  int32_t outputLen;
+  int32_t bufSize;
 } SUdfSetupResponse;
 
 typedef struct SUdfCallRequest {

source/libs/function/src/builtinsimpl.c

@@ -139,6 +139,20 @@ int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   return pResInfo->numOfRes;
 }
 
+int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, char* finalResult) {
+  int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
+  SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
+
+  SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
+  pResInfo->isNullRes = (pResInfo->numOfRes == 0)? 1:0;
+  cleanupResultRowEntry(pResInfo);
+
+  char* in = finalResult;
+  colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes);
+
+  return pResInfo->numOfRes;
+}
+
 EFuncDataRequired countDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) {
   SNode* pParam = nodesListGetNode(pFunc->pParameterList, 0);
   if (QUERY_NODE_COLUMN == nodeType(pParam) && PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pParam)->colId) {

source/libs/function/src/functionMgt.c

@@ -21,6 +21,10 @@
 #include "taos.h"
 #include "taoserror.h"
 #include "thash.h"
+#include "builtins.h"
+#include "catalog.h"
+#include "tudf.h"
+
 
 typedef struct SFuncMgtService {
   SHashObj* pFuncNameHashTable;
@@ -120,6 +124,17 @@ int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t fmGetUdafExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet) {
+  if (!fmIsUserDefinedFunc(funcId)) {
+    return TSDB_CODE_FAILED;
+  }
+  pFpSet->getEnv = udfAggGetEnv;
+  pFpSet->init = udfAggInit;
+  pFpSet->process = udfAggProcess;
+  pFpSet->finalize = udfAggFinalize;
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet) {
   if (fmIsUserDefinedFunc(funcId) || funcId < 0 || funcId >= funcMgtBuiltinsNum) {
     return TSDB_CODE_FAILED;

source/libs/function/src/tudf.c

@@ -19,6 +19,9 @@
 #include "tudfInt.h"
 #include "tarray.h"
 #include "tdatablock.h"
+#include "querynodes.h"
+#include "builtinsimpl.h"
+#include "functionMgt.h"
 
 //TODO: network error processing.
 //TODO: add unit test
@@ -147,6 +150,10 @@ typedef struct SUdfUvSession {
   SUdfdProxy *udfc;
   int64_t severHandle;
   uv_pipe_t *udfSvcPipe;
+
+  int8_t  outputType;
+  int32_t outputLen;
+  int32_t bufSize;
 } SUdfUvSession;
 
 typedef struct SClientUvTaskNode {
@@ -235,12 +242,14 @@ void* decodeUdfSetupRequest(const void* buf, SUdfSetupRequest *request) {
 
 int32_t encodeUdfInterBuf(void **buf, const SUdfInterBuf* state) {
   int32_t len = 0;
+  len += taosEncodeFixedI8(buf, state->numOfResult);
   len += taosEncodeFixedI32(buf, state->bufLen);
   len += taosEncodeBinary(buf, state->buf, state->bufLen);
   return len;
 }
 
 void* decodeUdfInterBuf(const void* buf, SUdfInterBuf* state) {
+  buf = taosDecodeFixedI8(buf, &state->numOfResult);
   buf = taosDecodeFixedI32(buf, &state->bufLen);
   buf = taosDecodeBinary(buf, (void**)&state->buf, state->bufLen);
   return (void*)buf;
@@ -342,11 +351,17 @@ void* decodeUdfRequest(const void* buf, SUdfRequest* request) {
 int32_t encodeUdfSetupResponse(void **buf, const SUdfSetupResponse *setupRsp) {
   int32_t len = 0;
   len += taosEncodeFixedI64(buf, setupRsp->udfHandle);
+  len += taosEncodeFixedI8(buf, setupRsp->outputType);
+  len += taosEncodeFixedI32(buf, setupRsp->outputLen);
+  len += taosEncodeFixedI32(buf, setupRsp->bufSize);
   return len;
 }
 
 void* decodeUdfSetupResponse(const void* buf, SUdfSetupResponse* setupRsp) {
   buf = taosDecodeFixedI64(buf, &setupRsp->udfHandle);
+  buf = taosDecodeFixedI8(buf, &setupRsp->outputType);
+  buf = taosDecodeFixedI32(buf, &setupRsp->outputLen);
+  buf = taosDecodeFixedI32(buf, &setupRsp->bufSize);
   return (void*)buf;
 }
 
@@ -1049,6 +1064,9 @@ int32_t setupUdf(char udfName[], UdfcFuncHandle *funcHandle) {
 
   SUdfSetupResponse *rsp = &task->_setup.rsp;
   task->session->severHandle = rsp->udfHandle;
+  task->session->outputType = rsp->outputType;
+  task->session->outputLen = rsp->outputLen;
+  task->session->bufSize = rsp->bufSize;
   if (task->errCode != 0) {
     fnError("failed to setup udf. err: %d", task->errCode)
   } else {
@@ -1197,3 +1215,122 @@ int32_t teardownUdf(UdfcFuncHandle handle) {
 
   return err;
 }
+
+//memory layout |---SUdfAggRes----|-----final result-----|---inter result----|
+typedef struct SUdfAggRes {
+  SUdfUvSession *session;
+  int8_t finalResNum;
+  int8_t interResNum;
+  char* finalResBuf;
+  char* interResBuf;
+} SUdfAggRes;
+
+bool udfAggGetEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
+  if (fmIsScalarFunc(pFunc->funcId)) {
+    return false;
+  }
+  pEnv->calcMemSize = sizeof(SUdfAggRes) + pFunc->node.resType.bytes + pFunc->udfBufSize;
+  return true;
+}
+
+bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo) {
+  if (functionSetup(pCtx, pResultCellInfo) != true) {
+    return false;
+  }
+  UdfcFuncHandle handle;
+  if (setupUdf((char*)pCtx->udfName, &handle) != 0) {
+    return false;
+  }
+  SUdfUvSession *session = (SUdfUvSession *)handle;
+  SUdfAggRes *udfRes = (SUdfAggRes*)GET_ROWCELL_INTERBUF(pResultCellInfo);
+  udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes);
+  udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen;
+
+  int32_t envSize = sizeof(SUdfAggRes) + session->outputLen + session->bufSize;
+  memset(udfRes, 0, envSize);
+
+  udfRes->session = (SUdfUvSession *)handle;
+  SUdfInterBuf buf = {0};
+  if (callUdfAggInit(handle, &buf) != 0) {
+    return false;
+  }
+  udfRes->interResNum = buf.numOfResult;
+  memcpy(udfRes->interResBuf, buf.buf, buf.bufLen);
+  return true;
+}
+
+int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
+
+  SInputColumnInfoData* pInput = &pCtx->input;
+  int32_t numOfCols = pInput->numOfInputCols;
+
+  SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+  SUdfUvSession *session = udfRes->session;
+  udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes);
+  udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen;
+
+  int32_t start = pInput->startRowIndex;
+  int32_t numOfRows = pInput->numOfRows;
+
+
+  SSDataBlock tempBlock = {0};
+  tempBlock.info.numOfCols = numOfCols;
+  tempBlock.info.rows = numOfRows;
+  tempBlock.info.uid = pInput->uid;
+  bool hasVarCol = false;
+  tempBlock.pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
+
+  for (int32_t i = 0; i < numOfCols; ++i) {
+    SColumnInfoData *col = pInput->pData[i];
+    if (IS_VAR_DATA_TYPE(col->info.type)) {
+      hasVarCol = true;
+    }
+    taosArrayPush(tempBlock.pDataBlock, col);
+  }
+  tempBlock.info.hasVarCol = hasVarCol;
+
+  SSDataBlock *inputBlock = blockDataExtractBlock(&tempBlock, start, numOfRows);
+
+  SUdfInterBuf state = {.buf = udfRes->interResBuf,
+                        .bufLen = session->bufSize,
+                        .numOfResult = udfRes->interResNum};
+  SUdfInterBuf newState = {0};
+
+  callUdfAggProcess(session, inputBlock, &state, &newState);
+
+  udfRes->interResNum = newState.numOfResult;
+  memcpy(udfRes->interResBuf, newState.buf, newState.bufLen);
+
+  if (newState.numOfResult == 1 || state.numOfResult == 1) {
+    GET_RES_INFO(pCtx)->numOfRes = 1;
+  }
+
+  blockDataDestroy(inputBlock);
+
+  taosArrayDestroy(tempBlock.pDataBlock);
+
+  taosMemoryFree(newState.buf);
+  return 0;
+}
+
+int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) {
+  SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+  SUdfUvSession *session = udfRes->session;
+  udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes);
+  udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen;
+
+
+  SUdfInterBuf resultBuf = {.buf = udfRes->finalResBuf,
+                            .bufLen = session->outputLen,
+                            .numOfResult = udfRes->finalResNum};
+  SUdfInterBuf state = {.buf = udfRes->interResBuf,
+                        .bufLen = session->bufSize,
+                        .numOfResult = udfRes->interResNum};
+  callUdfAggFinalize(session, &state, &resultBuf);
+  teardownUdf(session);
+
+  if (resultBuf.numOfResult == 1) {
+    GET_RES_INFO(pCtx)->numOfRes = 1;
+  }
+  return functionFinalizeWithResultBuf(pCtx, pBlock, udfRes->finalResBuf);
+}

source/libs/function/src/udfd.c

@@ -77,6 +77,10 @@ typedef struct SUdf {
   uv_lib_t              lib;
   TUdfScalarProcFunc    scalarProcFunc;
   TUdfFreeUdfColumnFunc freeUdfColumn;
+
+  TUdfAggStartFunc      aggStartFunc;
+  TUdfAggProcessFunc    aggProcFunc;
+  TUdfAggFinishFunc     aggFinishFunc;
 } SUdf;
 
 // TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix
@@ -97,15 +101,32 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) {
     fnError("can not load library %s. error: %s", udf->path, uv_strerror(err));
     return UDFC_CODE_LOAD_UDF_FAILURE;
   }
-  // TODO: find all the functions
+  //TODO: init and destroy function
-  char normalFuncName[TSDB_FUNC_NAME_LEN] = {0};
+  if (udf->funcType == TSDB_FUNC_TYPE_SCALAR) {
-  strcpy(normalFuncName, udfName);
+    char processFuncName[TSDB_FUNC_NAME_LEN] = {0};
-  uv_dlsym(&udf->lib, normalFuncName, (void **)(&udf->scalarProcFunc));
+    strcpy(processFuncName, udfName);
-  char  freeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0};
+    uv_dlsym(&udf->lib, processFuncName, (void **)(&udf->scalarProcFunc));
+    char  freeFuncName[TSDB_FUNC_NAME_LEN + 5] = {0};
     char *freeSuffix = "_free";
-  strncpy(freeFuncName, normalFuncName, strlen(normalFuncName));
+    strncpy(freeFuncName, processFuncName, strlen(processFuncName));
     strncat(freeFuncName, freeSuffix, strlen(freeSuffix));
     uv_dlsym(&udf->lib, freeFuncName, (void **)(&udf->freeUdfColumn));
+  } else if (udf->funcType == TSDB_FUNC_TYPE_AGGREGATE) {
+    char processFuncName[TSDB_FUNC_NAME_LEN] = {0};
+    strcpy(processFuncName, udfName);
+    uv_dlsym(&udf->lib, processFuncName, (void **)(&udf->aggProcFunc));
+    char  startFuncName[TSDB_FUNC_NAME_LEN + 6] = {0};
+    char *startSuffix = "_start";
+    strncpy(startFuncName, processFuncName, strlen(processFuncName));
+    strncat(startFuncName, startSuffix, strlen(startSuffix));
+    uv_dlsym(&udf->lib, startFuncName, (void **)(&udf->aggStartFunc));
+    char  finishFuncName[TSDB_FUNC_NAME_LEN + 7] = {0};
+    char *finishSuffix = "_finish";
+    strncpy(finishFuncName, processFuncName, strlen(processFuncName));
+    strncat(finishFuncName, finishSuffix, strlen(finishSuffix));
+    uv_dlsym(&udf->lib, startFuncName, (void **)(&udf->aggFinishFunc));
+    //TODO: merge
+  }
   return 0;
 }
 
@@ -160,6 +181,9 @@ void udfdProcessRequest(uv_work_t *req) {
       rsp.type = request.type;
       rsp.code = 0;
       rsp.setupRsp.udfHandle = (int64_t)(handle);
+      rsp.setupRsp.outputType = udf->outputType;
+      rsp.setupRsp.outputLen = udf->outputLen;
+      rsp.setupRsp.bufSize = udf->bufSize;
       int32_t len = encodeUdfResponse(NULL, &rsp);
       rsp.msgLen = len;
       void *bufBegin = taosMemoryMalloc(len);
@@ -178,25 +202,57 @@ void udfdProcessRequest(uv_work_t *req) {
               call->udfHandle);
       SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(call->udfHandle);
       SUdf            *udf = handle->udf;
+      SUdfResponse  response = {0};
+      SUdfResponse *rsp = &response;
+      SUdfCallResponse *subRsp = &rsp->callRsp;
+
+      switch(call->callType) {
+        case TSDB_UDF_CALL_SCALA_PROC: {
+          SUdfColumn output = {0};
 
           SUdfDataBlock input = {0};
           convertDataBlockToUdfDataBlock(&call->block, &input);
-      SUdfColumn output = {0};
+          udf->scalarProcFunc(&input, &output);
-      // TODO: call different functions according to call type, for now just calar
+
-      if (call->callType == TSDB_UDF_CALL_SCALA_PROC) {
+          convertUdfColumnToDataBlock(&output, &response.callRsp.resultData);
-        udf->scalarProcFunc(input, &output);
+          udf->freeUdfColumn(&output);
+          break;
+        }
+        case TSDB_UDF_CALL_AGG_INIT: {
+          SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize),
+                                 .bufLen= udf->bufSize,
+                                 .numOfResult = 0};
+          udf->aggStartFunc(&outBuf);
+          subRsp->resultBuf = outBuf;
+          break;
+        }
+        case TSDB_UDF_CALL_AGG_PROC: {
+          SUdfDataBlock input = {0};
+          convertDataBlockToUdfDataBlock(&call->block, &input);
+          SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize),
+                                 .bufLen= udf->bufSize,
+                                 .numOfResult = 0};
+          udf->aggProcFunc(&input, &call->interBuf, &outBuf);
+          subRsp->resultBuf = outBuf;
+
+          break;
+        }
+        case TSDB_UDF_CALL_AGG_FIN: {
+          SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize),
+                                 .bufLen= udf->bufSize,
+                                 .numOfResult = 0};
+          udf->aggFinishFunc(&call->interBuf, &outBuf);
+          subRsp->resultBuf = outBuf;
+          break;
+        }
+        default:
+          break;
       }
 
-      SUdfResponse  response = {0};
-      SUdfResponse *rsp = &response;
-      if (call->callType == TSDB_UDF_CALL_SCALA_PROC) {
       rsp->seqNum = request.seqNum;
       rsp->type = request.type;
       rsp->code = 0;
-        SUdfCallResponse *subRsp = &rsp->callRsp;
       subRsp->callType = call->callType;
-        convertUdfColumnToDataBlock(&output, &subRsp->resultData);
-      }
 
       int32_t len = encodeUdfResponse(NULL, rsp);
       rsp->msgLen = len;
@@ -205,9 +261,6 @@ void udfdProcessRequest(uv_work_t *req) {
       encodeUdfResponse(&buf, rsp);
       uvUdf->output = uv_buf_init(bufBegin, len);
 
-      // TODO: free udf column
-      udf->freeUdfColumn(&output);
-
       taosMemoryFree(uvUdf->input.base);
       break;
     }

source/libs/function/test/udf1.c

@@ -9,18 +9,18 @@
 #undef free
 #define free free
 
-int32_t udf1_setup() {
+int32_t udf1_init() {
   return 0;
 }
 
-int32_t udf1_teardown() {
+int32_t udf1_destroy() {
   return 0;
 }
 
-int32_t udf1(SUdfDataBlock block, SUdfColumn *resultCol) {
+int32_t udf1(SUdfDataBlock* block, SUdfColumn *resultCol) {
   SUdfColumnData *resultData = &resultCol->colData;
-  resultData->numOfRows = block.numOfRows;
+  resultData->numOfRows = block->numOfRows;
-  SUdfColumnData *srcData = &block.udfCols[0]->colData;
+  SUdfColumnData *srcData = &block->udfCols[0]->colData;
   resultData->varLengthColumn = srcData->varLengthColumn;
 
   if (resultData->varLengthColumn) {

source/libs/function/test/udf2.c

@@ -0,0 +1,52 @@
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "tudf.h"
+
+#undef malloc
+#define malloc malloc
+#undef free
+#define free free
+
+int32_t udf2_init() {
+  return 0;
+}
+
+int32_t udf2_destroy() {
+  return 0;
+}
+
+int32_t udf2_start(SUdfInterBuf *buf) {
+  *(int64_t*)(buf->buf) = 0;
+  buf->bufLen = sizeof(int64_t);
+  buf->numOfResult = 0;
+  return 0;
+}
+
+int32_t udf2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) {
+  int64_t sumSquares = *(int64_t*)interBuf->buf;
+  for (int32_t i = 0; i < block->numOfCols; ++i) {
+    for (int32_t j = 0; j < block->numOfRows; ++i) {
+      SUdfColumn* col = block->udfCols[i];
+      //TODO: check the bitmap for null value
+      int32_t* rows = (int32_t*)col->colData.fixLenCol.data;
+      sumSquares += rows[j] * rows[j];
+    }
+  }
+
+  *(int64_t*)newInterBuf = sumSquares;
+  newInterBuf->bufLen = sizeof(int64_t);
+  //TODO: if all null value, numOfResult = 0;
+  newInterBuf->numOfResult = 1;
+  return 0;
+}
+
+int32_t udf2_finish(SUdfInterBuf* buf, SUdfInterBuf *resultData) {
+  //TODO: check numOfResults;
+  int64_t sumSquares = *(int64_t*)(buf->buf);
+  *(double*)(resultData->buf) = sqrt(sumSquares);
+  resultData->bufLen = sizeof(double);
+  resultData->numOfResult = 1;
+  return 0;
+}

source/libs/nodes/src/nodesTraverseFuncs.c

@@ -423,6 +423,9 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk
   EDealRes res = DEAL_RES_CONTINUE;
 
   switch (nodeType(pNode)) {
+    case QUERY_NODE_NODE_LIST:
+      res = walkPhysiPlans(((SNodeListNode*)pNode)->pNodeList, order, walker, pContext);
+      break;
     case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
       res = walkScanPhysi((SScanPhysiNode*)pNode, order, walker, pContext);
       break;
@@ -534,10 +537,7 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk
       break;
     case QUERY_NODE_PHYSICAL_SUBPLAN: {
       SSubplan* pSubplan = (SSubplan*)pNode;
-      res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext);
-      if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
       res = walkPhysiPlans(pSubplan->pChildren, order, walker, pContext);
-      }
       if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
         res = walkPhysiPlan((SNode*)pSubplan->pNode, order, walker, pContext);
       }

source/libs/parser/src/parAstCreater.c

@@ -44,7 +44,7 @@ void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt) {
   pCxt->notSupport = false;
   pCxt->valid = true;
   pCxt->pRootNode = NULL;
-  pCxt->placeholderNo = 1;
+  pCxt->placeholderNo = 0;
   pCxt->errCode = TSDB_CODE_SUCCESS;
 }
 
@@ -316,7 +316,7 @@ SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLitera
   CHECK_OUT_OF_MEM(val);
   val->literal = strndup(pLiteral->z, pLiteral->n);
   CHECK_OUT_OF_MEM(val->literal);
-  val->placeholderNo = pCxt->placeholderNo++;
+  val->placeholderNo = ++pCxt->placeholderNo;
   return (SNode*)val;
 }
 
@@ -381,6 +381,11 @@ SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType d
   CHECK_OUT_OF_MEM(func);
   strcpy(func->functionName, "cast");
   func->node.resType = dt;
+  if (TSDB_DATA_TYPE_BINARY == dt.type) {
+    func->node.resType.bytes += 2;
+  } else if (TSDB_DATA_TYPE_NCHAR == dt.type) {
+    func->node.resType.bytes = func->node.resType.bytes * TSDB_NCHAR_SIZE + 2;
+  }
   nodesListMakeAppend(&func->pParameterList, pExpr);
   return (SNode*)func;
 }

source/libs/parser/src/parAstParser.c

@@ -80,6 +80,7 @@ abort_parse:
       return TSDB_CODE_OUT_OF_MEMORY;
     }
     (*pQuery)->pRoot = cxt.pRootNode;
+    (*pQuery)->placeholderNum = cxt.placeholderNo;
   }
   return cxt.errCode;
 }

source/libs/parser/src/parInsert.c

@@ -937,6 +937,11 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks,
     }
 
     *gotRow = true;
+#ifdef TD_DEBUG_PRINT_ROW
+    STSchema* pSTSchema = tdGetSTSChemaFromSSChema(&schema, spd->numOfCols);
+    tdSRowPrint(row, pSTSchema, __func__);
+    taosMemoryFree(pSTSchema);
+#endif
   }
 
   // *len = pBuilder->extendedRowSize;
@@ -1328,10 +1333,6 @@ int32_t qBindStmtColsValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, in
     for (int c = 0; c < spd->numOfBound; ++c) {
       SSchema* pColSchema = &pSchema[spd->boundColumns[c] - 1];
 
-      if (bind[c].buffer_type != pColSchema->type) {
-        return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type");
-      }
-
       if (bind[c].num != rowNum) {
         return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same");
       }
@@ -1346,6 +1347,10 @@ int32_t qBindStmtColsValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, in
 
         CHECK_CODE(MemRowAppend(&pBuf, NULL, 0, &param));
       } else {
+        if (bind[c].buffer_type != pColSchema->type) {
+          return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type");
+        }
+
         int32_t colLen = pColSchema->bytes;
         if (IS_VAR_DATA_TYPE(pColSchema->type)) {
           colLen = bind[c].length[r];
@@ -1359,7 +1364,6 @@ int32_t qBindStmtColsValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, in
         checkTimestamp(pDataBlock, (const char*)&tsKey);
       }
     }
-
     // set the null value for the columns that do not assign values
     if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) {
       for (int32_t i = 0; i < spd->numOfCols; ++i) {
@@ -1369,6 +1373,11 @@ int32_t qBindStmtColsValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, in
         }
       }
     }
+#ifdef TD_DEBUG_PRINT_ROW
+    STSchema* pSTSchema = tdGetSTSChemaFromSSChema(&pSchema, spd->numOfCols);
+    tdSRowPrint(row, pSTSchema, __func__);
+    taosMemoryFree(pSTSchema);
+#endif
 
     pDataBlock->size += extendedRowSize;
   }
@@ -1447,6 +1456,14 @@ int32_t qBindStmtSingleColValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBu
         }
       }
     }
+
+#ifdef TD_DEBUG_PRINT_ROW
+    if(rowEnd) {
+      STSchema* pSTSchema = tdGetSTSChemaFromSSChema(&pSchema, spd->numOfCols);
+      tdSRowPrint(row, pSTSchema, __func__);
+      taosMemoryFree(pSTSchema);
+    }
+#endif
   }
 
   if (rowEnd) {

source/libs/parser/src/parTokenizer.c

@@ -47,6 +47,7 @@ static SKeyword keywordTable[] = {
     {"BNODE", TK_BNODE},
     {"BNODES", TK_BNODES},
     {"BOOL", TK_BOOL},
+    {"BUFFER", TK_BUFFER},
     {"BUFSIZE", TK_BUFSIZE},
     {"BY", TK_BY},
     {"CACHE", TK_CACHE},
@@ -132,6 +133,8 @@ static SKeyword keywordTable[] = {
     {"OUTPUTTYPE", TK_OUTPUTTYPE},
     {"PARTITION", TK_PARTITION},
     {"PASS", TK_PASS},
+    {"PAGES", TK_PAGES},
+    {"PAGESIZE", TK_PAGESIZE},
     {"PORT", TK_PORT},
     {"PPS", TK_PPS},
     {"PRECISION", TK_PRECISION},

source/libs/parser/src/parTranslater.c

@@ -1539,26 +1539,24 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS
   tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
   tNameGetFullDbName(&name, pReq->db);
   pReq->numOfVgroups = pStmt->pOptions->numOfVgroups;
+  pReq->numOfStables = pStmt->pOptions->singleStable;
+  pReq->buffer = pStmt->pOptions->buffer;
+  pReq->pageSize = pStmt->pOptions->pagesize;
+  pReq->pages = pStmt->pOptions->pages;
   pReq->daysPerFile = pStmt->pOptions->daysPerFile;
   pReq->daysToKeep0 = pStmt->pOptions->keep[0];
   pReq->daysToKeep1 = pStmt->pOptions->keep[1];
   pReq->daysToKeep2 = pStmt->pOptions->keep[2];
   pReq->minRows = pStmt->pOptions->minRowsPerBlock;
   pReq->maxRows = pStmt->pOptions->maxRowsPerBlock;
-  pReq->commitTime = -1;
   pReq->fsyncPeriod = pStmt->pOptions->fsyncPeriod;
   pReq->walLevel = pStmt->pOptions->walLevel;
   pReq->precision = pStmt->pOptions->precision;
   pReq->compression = pStmt->pOptions->compressionLevel;
   pReq->replications = pStmt->pOptions->replica;
-  pReq->update = -1;
+  pReq->strict = pStmt->pOptions->strict;
   pReq->cacheLastRow = pStmt->pOptions->cachelast;
   pReq->ignoreExist = pStmt->ignoreExists;
-  pReq->singleSTable = pStmt->pOptions->singleStable;
-  pReq->strict = pStmt->pOptions->strict;
-  // pStmt->pOptions->buffer;
-  // pStmt->pOptions->pages;
-  // pStmt->pOptions->pagesize;
   return buildCreateDbRetentions(pStmt->pOptions->pRetentions, pReq);
 }
 
@@ -1698,7 +1696,8 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa
 
 static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions,
                                     bool alter) {
-  int32_t code = checkRangeOption(pCxt, "buffer", pOptions->buffer, TSDB_MIN_BUFFER_PER_VNODE, INT32_MAX);
+  int32_t code =
+      checkRangeOption(pCxt, "buffer", pOptions->buffer, TSDB_MIN_BUFFER_PER_VNODE, TSDB_MAX_BUFFER_PER_VNODE);
   if (TSDB_CODE_SUCCESS == code) {
     code = checkRangeOption(pCxt, "cacheLast", pOptions->cachelast, TSDB_MIN_DB_CACHE_LAST_ROW,
                             TSDB_MAX_DB_CACHE_LAST_ROW);
@@ -1724,7 +1723,7 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName
     code = checkDbKeepOption(pCxt, pOptions);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = checkRangeOption(pCxt, "pages", pOptions->pages, TSDB_MIN_PAGES_PER_VNODE, INT32_MAX);
+    code = checkRangeOption(pCxt, "pages", pOptions->pages, TSDB_MIN_PAGES_PER_VNODE, TSDB_MAX_PAGES_PER_VNODE);
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = checkRangeOption(pCxt, "pagesize", pOptions->pagesize, TSDB_MIN_PAGESIZE_PER_VNODE,
@@ -1810,16 +1809,18 @@ static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt,
   SName name = {0};
   tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
   tNameGetFullDbName(&name, pReq->db);
-  // pStmt->pOptions->buffer
+  pReq->buffer = pStmt->pOptions->buffer;
-  pReq->cacheLastRow = pStmt->pOptions->cachelast;
+  pReq->pageSize = -1;
-  pReq->fsyncPeriod = pStmt->pOptions->fsyncPeriod;
+  pReq->pages = pStmt->pOptions->pages;
+  pReq->daysPerFile = -1;
   pReq->daysToKeep0 = pStmt->pOptions->keep[0];
   pReq->daysToKeep1 = pStmt->pOptions->keep[1];
   pReq->daysToKeep2 = pStmt->pOptions->keep[2];
-  // pStmt->pOptions->pages
+  pReq->fsyncPeriod = pStmt->pOptions->fsyncPeriod;
-  pReq->replications = pStmt->pOptions->replica;
-  pReq->strict = pStmt->pOptions->strict;
   pReq->walLevel = pStmt->pOptions->walLevel;
+  pReq->strict = pStmt->pOptions->strict;
+  pReq->cacheLastRow = pStmt->pOptions->cachelast;
+  pReq->replications = pStmt->pOptions->replica;
   return;
 }
 
@@ -2658,7 +2659,7 @@ static int32_t translateCreateStream(STranslateContext* pCxt, SCreateStreamStmt*
   if ('\0' != pStmt->targetTabName[0]) {
     strcpy(name.dbname, pStmt->targetDbName);
     strcpy(name.tname, pStmt->targetTabName);
-    tNameExtractFullName(&name, createReq.outputSTbName);
+    tNameExtractFullName(&name, createReq.targetStbFullName);
   }
 
   int32_t code = translateQuery(pCxt, pStmt->pQuery);

source/libs/planner/src/planner.c

@@ -19,26 +19,23 @@
 
 typedef struct SCollectPlaceholderValuesCxt {
   int32_t    errCode;
-  SNodeList* pValues;
+  SArray* pValues;
 } SCollectPlaceholderValuesCxt;
 
 static EDealRes collectPlaceholderValuesImpl(SNode* pNode, void* pContext) {
   if (QUERY_NODE_VALUE == nodeType(pNode) && ((SValueNode*)pNode)->placeholderNo > 0) {
     SCollectPlaceholderValuesCxt* pCxt = pContext;
-    pCxt->errCode = nodesListMakeAppend(&pCxt->pValues, pNode);
+    taosArrayInsert(pCxt->pValues, ((SValueNode*)pNode)->placeholderNo - 1, &pNode);
     return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR;
   }
   return DEAL_RES_CONTINUE;
 }
 
 static int32_t collectPlaceholderValues(SPlanContext* pCxt, SQueryPlan* pPlan) {
-  SCollectPlaceholderValuesCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pValues = NULL};
+  pPlan->pPlaceholderValues = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
+
+  SCollectPlaceholderValuesCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pValues = pPlan->pPlaceholderValues};
   nodesWalkPhysiPlan((SNode*)pPlan, collectPlaceholderValuesImpl, &cxt);
-  if (TSDB_CODE_SUCCESS == cxt.errCode) {
-    pPlan->pPlaceholderValues = cxt.pValues;
-  } else {
-    nodesDestroyList(cxt.pValues);
-  }
   return cxt.errCode;
 }
 
@@ -60,7 +57,7 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo
   if (TSDB_CODE_SUCCESS == code) {
     code = createPhysiPlan(pCxt, pLogicPlan, pPlan, pExecNodeList);
   }
-  if (TSDB_CODE_SUCCESS == code && pCxt->isStmtQuery) {
+  if (TSDB_CODE_SUCCESS == code && pCxt->placeholderNum > 0) {
     code = collectPlaceholderValues(pCxt, *pPlan);
   }
 
@@ -108,7 +105,7 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) {
     return TSDB_CODE_SUCCESS;
   }
   pVal->node.resType.type = pParam->buffer_type;
-  pVal->node.resType.bytes = *(pParam->length);
+  pVal->node.resType.bytes = NULL != pParam->length ? *(pParam->length) : tDataTypes[pParam->buffer_type].bytes;
   switch (pParam->buffer_type) {
     case TSDB_DATA_TYPE_BOOL:
       pVal->datum.b = *((bool*)pParam->buffer);
@@ -133,6 +130,7 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) {
       break;
     case TSDB_DATA_TYPE_VARCHAR:
     case TSDB_DATA_TYPE_VARBINARY:
+    case TSDB_DATA_TYPE_NCHAR:
       pVal->datum.p = taosMemoryCalloc(1, pVal->node.resType.bytes + VARSTR_HEADER_SIZE + 1);
       if (NULL == pVal->datum.p) {
         return TSDB_CODE_OUT_OF_MEMORY;
@@ -155,7 +153,6 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) {
     case TSDB_DATA_TYPE_UBIGINT:
       pVal->datum.u = *((uint64_t*)pParam->buffer);
       break;
-    case TSDB_DATA_TYPE_NCHAR:
     case TSDB_DATA_TYPE_JSON:
     case TSDB_DATA_TYPE_DECIMAL:
     case TSDB_DATA_TYPE_BLOB:
@@ -168,18 +165,35 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx) {
+static EDealRes updatePlanQueryId(SNode* pNode, void* pContext) {
-  if (colIdx < 0) {
+  int64_t queryId = *(uint64_t *)pContext;
-    int32_t index = 0;
-    SNode* pNode = NULL;
   
-    FOREACH(pNode, pPlan->pPlaceholderValues) {
+  if (QUERY_NODE_PHYSICAL_PLAN == nodeType(pNode)) {
-      setValueByBindParam((SValueNode*)pNode, pParams + index);
+    SQueryPlan* planNode = (SQueryPlan*)pNode;
-      ++index;
+    planNode->queryId = queryId;
+  } else if (QUERY_NODE_PHYSICAL_SUBPLAN == nodeType(pNode)) {
+    SSubplan* subplanNode = (SSubplan*)pNode;
+    subplanNode->id.queryId = queryId;
+  }
+
+  return DEAL_RES_CONTINUE;
+}
+
+int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId) {
+  int32_t size = taosArrayGetSize(pPlan->pPlaceholderValues);
+
+  if (colIdx < 0) {
+    for (int32_t i = 0; i < size; ++i) {
+      setValueByBindParam((SValueNode*)taosArrayGetP(pPlan->pPlaceholderValues, i), pParams + i);
     }
   } else {
-    setValueByBindParam((SValueNode*)nodesListGetNode(pPlan->pPlaceholderValues, colIdx), pParams);
+    setValueByBindParam((SValueNode*)taosArrayGetP(pPlan->pPlaceholderValues, colIdx), pParams);
   }
+
+  if (colIdx < 0 || ((colIdx + 1) == size)) {
+    nodesWalkPhysiPlan((SNode*)pPlan, updatePlanQueryId, &queryId);
+  }
+  
   return TSDB_CODE_SUCCESS;
 }
 

source/libs/scalar/inc/sclInt.h

@@ -34,7 +34,7 @@ typedef struct SScalarCtx {
 
 #define SCL_IS_CONST_NODE(_node) ((NULL == (_node)) || (QUERY_NODE_VALUE == (_node)->type) || (QUERY_NODE_NODE_LIST == (_node)->type))
 #define SCL_IS_CONST_CALC(_ctx) (NULL == (_ctx)->pBlockList)
-#define SCL_IS_NULL_VALUE_NODE(_node) ((QUERY_NODE_VALUE == nodeType(_node)) && (TSDB_DATA_TYPE_NULL == ((SValueNode *)_node)->node.resType.type))
+#define SCL_IS_NULL_VALUE_NODE(_node) ((QUERY_NODE_VALUE == nodeType(_node)) && (TSDB_DATA_TYPE_NULL == ((SValueNode *)_node)->node.resType.type) && (((SValueNode *)_node)->placeholderNo <= 0))
 
 #define sclFatal(...)  qFatal(__VA_ARGS__)
 #define sclError(...)  qError(__VA_ARGS__)

source/libs/scalar/src/filter.c

@@ -3541,11 +3541,16 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
   }
 
   if (QUERY_NODE_VALUE == nodeType(*pNode)) {
+    SValueNode *valueNode = (SValueNode *)*pNode;
+    if (valueNode->placeholderNo >= 1) {
+      stat->scalarMode = true;
+      return DEAL_RES_CONTINUE;
+    }
+    
     if (!FILTER_GET_FLAG(stat->info->options, FLT_OPTION_TIMESTAMP)) {
       return DEAL_RES_CONTINUE;
     }
     
-    SValueNode *valueNode = (SValueNode *)*pNode;
     if (TSDB_DATA_TYPE_BINARY != valueNode->node.resType.type && TSDB_DATA_TYPE_NCHAR != valueNode->node.resType.type) {
       return DEAL_RES_CONTINUE;
     }
@@ -3587,7 +3592,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
       return DEAL_RES_CONTINUE;
     }
 
-    if (node->opType == OP_TYPE_NOT_IN || node->opType == OP_TYPE_NOT_LIKE || node->opType > OP_TYPE_IS_NOT_NULL) {
+    if (node->opType == OP_TYPE_NOT_IN || node->opType == OP_TYPE_NOT_LIKE || node->opType > OP_TYPE_IS_NOT_NULL || node->opType == OP_TYPE_NOT_EQUAL) {
       stat->scalarMode = true;
       return DEAL_RES_CONTINUE;
     }

source/libs/scalar/src/scalar.c

@@ -7,6 +7,7 @@
 #include "tcommon.h"
 #include "tdatablock.h"
 #include "scalar.h"
+#include "tudf.h"
 
 int32_t scalarGetOperatorParamNum(EOperatorType type) {
   if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || OP_TYPE_IS_NOT_TRUE == type 
@@ -336,14 +337,12 @@ int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outp
   SCL_ERR_RET(sclInitParamList(&params, node->pParameterList, ctx, &paramNum, &rowNum));
 
   if (fmIsUserDefinedFunc(node->funcId)) {
-#if 0  
     UdfcFuncHandle udfHandle = NULL;
     
     SCL_ERR_JRET(setupUdf(node->functionName, &udfHandle));
     code = callUdfScalarFunc(udfHandle, params, paramNum, output);
     teardownUdf(udfHandle);
     SCL_ERR_JRET(code);
-#endif    
   } else {
     SScalarFuncExecFuncs ffpSet = {0};
     code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet);
@@ -505,6 +504,7 @@ EDealRes sclRewriteBasedOnOptr(SNode** pNode, SScalarCtx *ctx, EOperatorType opT
     }
     
     res->node.resType.type = TSDB_DATA_TYPE_BOOL;
+    res->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
     res->datum.b = false;
     
     nodesDestroyNode(*pNode);
@@ -520,14 +520,14 @@ EDealRes sclRewriteOperatorForNullValue(SNode** pNode, SScalarCtx *ctx) {
 
   if (node->pLeft && (QUERY_NODE_VALUE == nodeType(node->pLeft))) {
     SValueNode *valueNode = (SValueNode *)node->pLeft;
-    if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type && (node->opType != OP_TYPE_IS_NULL && node->opType != OP_TYPE_IS_NOT_NULL)) {
+    if (SCL_IS_NULL_VALUE_NODE(valueNode) && (node->opType != OP_TYPE_IS_NULL && node->opType != OP_TYPE_IS_NOT_NULL)) {
       return sclRewriteBasedOnOptr(pNode, ctx, node->opType);
     }
   }
 
   if (node->pRight && (QUERY_NODE_VALUE == nodeType(node->pRight))) {
     SValueNode *valueNode = (SValueNode *)node->pRight;
-    if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type && (node->opType != OP_TYPE_IS_NULL && node->opType != OP_TYPE_IS_NOT_NULL)) {
+    if (SCL_IS_NULL_VALUE_NODE(valueNode) && (node->opType != OP_TYPE_IS_NULL && node->opType != OP_TYPE_IS_NOT_NULL)) {
       return sclRewriteBasedOnOptr(pNode, ctx, node->opType);
     }
   }
@@ -589,7 +589,10 @@ EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) {
   if (colDataIsNull_s(output.columnData, 0)) {
     res->node.resType.type = TSDB_DATA_TYPE_NULL;
   } else {
-    res->node.resType = node->node.resType;
+    res->node.resType.type = output.columnData->info.type;
+    res->node.resType.bytes = output.columnData->info.bytes;
+    res->node.resType.scale = output.columnData->info.scale;
+    res->node.resType.precision = output.columnData->info.precision;
     int32_t type = output.columnData->info.type;
     if (IS_VAR_DATA_TYPE(type)) {
       res->datum.p = taosMemoryCalloc(res->node.resType.bytes + VARSTR_HEADER_SIZE + 1, 1);

source/libs/scalar/src/sclfunc.c

@@ -15,7 +15,15 @@ typedef int16_t (*_len_fn)(char *, int32_t);
 
 /** Math functions **/
 static double tlog(double v, double base) {
-  return log(v) / log(base);
+  double a = log(v);
+  double b = log(base);
+  if (isnan(a) || isinf(a)) {
+    return a;
+  } else if (isnan(b) || isinf(b)) {
+    return b;
+  } else {
+    return a / b;
+  }
 }
 
 int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
@@ -160,22 +168,64 @@ static int32_t doScalarFunctionUnique2(SScalarParam *pInput, int32_t inputNum, S
   }
 
   double *out = (double *)pOutputData->pData;
+  double result;
 
-  for (int32_t i = 0; i < pInput->numOfRows; ++i) {
+  int32_t numOfRows = MAX(pInput[0].numOfRows, pInput[1].numOfRows);
+  if (pInput[0].numOfRows == pInput[1].numOfRows) {
+    for (int32_t i = 0; i < numOfRows; ++i) {
       if (colDataIsNull_s(pInputData[0], i) ||
-        colDataIsNull_s(pInputData[1], 0)) {
+          colDataIsNull_s(pInputData[1], i)) {
         colDataAppendNULL(pOutputData, i);
         continue;
       }
-    double result = valFn(getValueFn[0](pInputData[0]->pData, i), getValueFn[1](pInputData[1]->pData, 0));
+      result = valFn(getValueFn[0](pInputData[0]->pData, i), getValueFn[1](pInputData[1]->pData, i));
       if (isinf(result) || isnan(result)) {
         colDataAppendNULL(pOutputData, i);
       } else {
         out[i] = result;
       }
     }
+  } else if (pInput[0].numOfRows == 1) { //left operand is constant
+    if (colDataIsNull_s(pInputData[0], 0)) {
+      colDataAppendNNULL(pOutputData, 0, pInput[1].numOfRows);
+    } else {
+      for (int32_t i = 0; i < numOfRows; ++i) {
+        if (colDataIsNull_s(pInputData[1], i)) {
+          colDataAppendNULL(pOutputData, i);
+          continue;
+        }
 
-  pOutput->numOfRows = pInput->numOfRows;
+        result = valFn(getValueFn[0](pInputData[0]->pData, 0), getValueFn[1](pInputData[1]->pData, i));
+        if (isinf(result) || isnan(result)) {
+          colDataAppendNULL(pOutputData, i);
+          continue;
+        }
+
+        out[i] = result;
+      }
+    }
+  } else if (pInput[1].numOfRows == 1) {
+    if (colDataIsNull_s(pInputData[1], 0)) {
+      colDataAppendNNULL(pOutputData, 0, pInput[0].numOfRows);
+    } else {
+      for (int32_t i = 0; i < numOfRows; ++i) {
+        if (colDataIsNull_s(pInputData[0], i)) {
+          colDataAppendNULL(pOutputData, i);
+          continue;
+        }
+
+        result = valFn(getValueFn[0](pInputData[0]->pData, i), getValueFn[1](pInputData[1]->pData, 0));
+        if (isinf(result) || isnan(result)) {
+          colDataAppendNULL(pOutputData, i);
+          continue;
+        }
+
+        out[i] = result;
+      }
+    }
+  }
+
+  pOutput->numOfRows = numOfRows;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -658,11 +708,6 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
   int16_t outputType = GET_PARAM_TYPE(&pOutput[0]);
   int64_t outputLen  = GET_PARAM_BYTES(&pOutput[0]);
 
-  if (IS_VAR_DATA_TYPE(outputType)) {
-    int32_t factor = (TSDB_DATA_TYPE_NCHAR == outputType) ? TSDB_NCHAR_SIZE : 1;
-    outputLen = outputLen * factor + VARSTR_HEADER_SIZE;
-  }
-
   char *outputBuf = taosMemoryCalloc(outputLen * pInput[0].numOfRows, 1);
   char *output = outputBuf;
 

source/libs/stream/src/tstream.c

@@ -158,8 +158,6 @@ int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, in
     //
   } else if (pTask->sinkType == TASK_SINK__FETCH) {
     //
-  } else if (pTask->sinkType == TASK_SINK__SHOW) {
-    blockDebugShowData(pRes);
   } else {
     ASSERT(pTask->sinkType == TASK_SINK__NONE);
   }
@@ -280,8 +278,6 @@ int32_t tEncodeSStreamTask(SCoder* pEncoder, const SStreamTask* pTask) {
     if (tEncodeI64(pEncoder, pTask->smaSink.smaId) < 0) return -1;
   } else if (pTask->sinkType == TASK_SINK__FETCH) {
     if (tEncodeI8(pEncoder, pTask->fetchSink.reserved) < 0) return -1;
-  } else if (pTask->sinkType == TASK_SINK__SHOW) {
-    if (tEncodeI8(pEncoder, pTask->showSink.reserved) < 0) return -1;
   } else {
     ASSERT(pTask->sinkType == TASK_SINK__NONE);
   }
@@ -326,8 +322,6 @@ int32_t tDecodeSStreamTask(SCoder* pDecoder, SStreamTask* pTask) {
     if (tDecodeI64(pDecoder, &pTask->smaSink.smaId) < 0) return -1;
   } else if (pTask->sinkType == TASK_SINK__FETCH) {
     if (tDecodeI8(pDecoder, &pTask->fetchSink.reserved) < 0) return -1;
-  } else if (pTask->sinkType == TASK_SINK__SHOW) {
-    if (tDecodeI8(pDecoder, &pTask->showSink.reserved) < 0) return -1;
   } else {
     ASSERT(pTask->sinkType == TASK_SINK__NONE);
   }

source/libs/stream/src/tstreamUpdate.c

@@ -0,0 +1,180 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tstreamUpdate.h"
+#include "ttime.h"
+
+#define DEFAULT_FALSE_POSITIVE 0.01
+#define DEFAULT_BUCKET_SIZE 1024
+#define ROWS_PER_MILLISECOND 1
+#define MAX_NUM_SCALABLE_BF 120
+#define MIN_NUM_SCALABLE_BF 10
+#define DEFAULT_PREADD_BUCKET 1
+#define MAX_INTERVAL MILLISECOND_PER_MINUTE
+#define MIN_INTERVAL (MILLISECOND_PER_SECOND * 10)
+
+static void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) {
+  if (pInfo->numSBFs < count ) {
+    count = pInfo->numSBFs;
+  }
+  for (uint64_t i = 0; i < count; ++i) {
+    SScalableBf *tsSBF = tScalableBfInit(pInfo->interval * ROWS_PER_MILLISECOND,
+                                         DEFAULT_FALSE_POSITIVE);
+    taosArrayPush(pInfo->pTsSBFs, &tsSBF);
+  }
+}
+
+static void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count) {
+  if (count < pInfo->numSBFs - 1) {
+    for (uint64_t i = 0; i < count; ++i) {
+      SScalableBf *pTsSBFs = taosArrayGetP(pInfo->pTsSBFs, i);
+      tScalableBfDestroy(pTsSBFs);
+      taosArrayRemove(pInfo->pTsSBFs, i);
+    }
+  } else {
+    taosArrayClearP(pInfo->pTsSBFs, (FDelete)tScalableBfDestroy);
+  }
+  pInfo->minTS += pInfo->interval * count;
+}
+
+static int64_t adjustInterval(int64_t interval, int32_t precision) {
+  int64_t val = interval;
+  if (precision != TSDB_TIME_PRECISION_MILLI) {
+    val = convertTimePrecision(interval, precision, TSDB_TIME_PRECISION_MILLI);
+  }
+
+  if (val <= 0 || val > MAX_INTERVAL) {
+    val = MAX_INTERVAL;
+  } else if (val < MIN_INTERVAL) {
+    val = MIN_INTERVAL;
+  }
+
+  if (precision != TSDB_TIME_PRECISION_MILLI) {
+    val = convertTimePrecision(val, TSDB_TIME_PRECISION_MILLI, precision);
+  }
+  return val;
+}
+
+static int64_t adjustWatermark(int64_t interval, int32_t watermark) {
+  if (watermark <= 0 || watermark > MAX_NUM_SCALABLE_BF * interval) {
+    watermark = MAX_NUM_SCALABLE_BF * interval;
+  } else if (watermark < MIN_NUM_SCALABLE_BF * interval) {
+    watermark = MIN_NUM_SCALABLE_BF * interval;
+  }
+  return watermark;
+}
+
+SUpdateInfo *updateInfoInitP(SInterval* pInterval, int64_t watermark) {
+  return updateInfoInit(pInterval->interval, pInterval->precision, watermark);
+}
+
+SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark) {
+  SUpdateInfo *pInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo));
+  if (pInfo == NULL) {
+    return NULL;
+  }
+  pInfo->pTsBuckets = NULL;
+  pInfo->pTsSBFs = NULL;
+  pInfo->minTS = -1;
+  pInfo->interval = adjustInterval(interval, precision);
+  pInfo->watermark = adjustWatermark(pInfo->interval, watermark);
+
+  uint64_t bfSize = (uint64_t)(pInfo->watermark / pInfo->interval);
+
+  pInfo->pTsSBFs = taosArrayInit(bfSize, sizeof(SScalableBf));
+  if (pInfo->pTsSBFs == NULL) {
+    updateInfoDestroy(pInfo);
+    return NULL;
+  }
+  pInfo->numSBFs = bfSize;
+  windowSBfAdd(pInfo, bfSize);
+
+  pInfo->pTsBuckets = taosArrayInit(DEFAULT_BUCKET_SIZE, sizeof(TSKEY));
+  if (pInfo->pTsBuckets == NULL) {
+    updateInfoDestroy(pInfo);
+    return NULL;
+  }
+
+  TSKEY dumy = 0;
+  for(uint64_t i=0; i < DEFAULT_BUCKET_SIZE; ++i) {
+    taosArrayPush(pInfo->pTsBuckets, &dumy);
+  }
+  pInfo->numBuckets = DEFAULT_BUCKET_SIZE;
+  return pInfo;
+}
+
+static SScalableBf* getSBf(SUpdateInfo *pInfo, TSKEY ts) {
+  if (ts <= 0) {
+    return NULL;
+  }
+  if (pInfo->minTS < 0) {
+    pInfo->minTS = (TSKEY)(ts / pInfo->interval * pInfo->interval);
+  }
+  uint64_t index = (uint64_t)((ts - pInfo->minTS) / pInfo->interval);
+  if (index >= pInfo->numSBFs) {
+    uint64_t count = index + 1 - pInfo->numSBFs;
+    windowSBfDelete(pInfo, count);
+    windowSBfAdd(pInfo, count);
+    index = pInfo->numSBFs - 1;
+  }
+  SScalableBf *res = taosArrayGetP(pInfo->pTsSBFs, index);
+  if (res == NULL) {
+    res = tScalableBfInit(pInfo->interval * ROWS_PER_MILLISECOND,
+                          DEFAULT_FALSE_POSITIVE);
+    taosArrayPush(pInfo->pTsSBFs, &res);
+  }
+  return res;
+}
+
+bool isUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts) {
+  int32_t res = TSDB_CODE_FAILED;
+  uint64_t index = ((uint64_t)tableId) % pInfo->numBuckets;
+  SScalableBf* pSBf = getSBf(pInfo, ts);
+  // pSBf may be a null pointer
+  if (pSBf) {
+    res = tScalableBfPut(pSBf, &ts, sizeof(TSKEY));
+  }
+
+  TSKEY maxTs = *(TSKEY *)taosArrayGet(pInfo->pTsBuckets, index);
+  if (maxTs < ts ) {
+    taosArraySet(pInfo->pTsBuckets, index, &ts);
+    return false;
+  }
+
+  if (ts < pInfo->minTS) {
+    return true;
+  } else if (res == TSDB_CODE_SUCCESS) {
+    return false;
+  }
+
+  //check from tsdb api
+  return true;
+}
+
+void updateInfoDestroy(SUpdateInfo *pInfo) {
+  if (pInfo == NULL) {
+    return;
+  }
+  taosArrayDestroy(pInfo->pTsBuckets);
+
+  uint64_t size = taosArrayGetSize(pInfo->pTsSBFs);
+  for (uint64_t i = 0; i < size; i++) {
+    SScalableBf *pSBF = taosArrayGetP(pInfo->pTsSBFs, i);
+    tScalableBfDestroy(pSBF);
+  }
+  
+  taosArrayDestroy(pInfo->pTsSBFs);
+  taosMemoryFree(pInfo);
+}

source/libs/stream/test/CMakeLists.txt

@@ -0,0 +1,20 @@
+
+MESSAGE(STATUS "build stream unit test")
+
+# GoogleTest requires at least C++11
+SET(CMAKE_CXX_STANDARD 11)
+AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+
+# bloomFilterTest
+ADD_EXECUTABLE(streamUpdateTest "tstreamUpdateTest.cpp")
+
+TARGET_LINK_LIBRARIES(
+        streamUpdateTest
+        PUBLIC os util common gtest stream
+)
+
+TARGET_INCLUDE_DIRECTORIES(
+        streamUpdateTest
+        PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/"
+        PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc"
+)

source/libs/stream/test/tstreamUpdateTest.cpp

@@ -0,0 +1,113 @@
+#include <gtest/gtest.h>
+
+#include "tstreamUpdate.h"
+#include "ttime.h"
+
+using namespace std;
+
+TEST(TD_STREAM_UPDATE_TEST, update) {
+  int64_t interval = 20 * 1000;
+  int64_t watermark = 10 * 60 * 1000;
+  SUpdateInfo *pSU = updateInfoInit(interval, TSDB_TIME_PRECISION_MILLI, watermark);
+  GTEST_ASSERT_EQ(isUpdated(pSU,1, 0), true);
+  GTEST_ASSERT_EQ(isUpdated(pSU,1, -1), true);
+
+  for(int i=0; i < 1024; i++) {
+    GTEST_ASSERT_EQ(isUpdated(pSU,i, 1), false);
+  }
+  for(int i=0; i < 1024; i++) {
+    GTEST_ASSERT_EQ(isUpdated(pSU,i, 1), true);
+  }
+
+  for(int i=0; i < 1024; i++) {
+    GTEST_ASSERT_EQ(isUpdated(pSU,i, 2), false);
+  }
+  for(int i=0; i < 1024; i++) {
+    GTEST_ASSERT_EQ(isUpdated(pSU,i, 2), true);
+  }
+
+  for(int i=0; i < 1024; i++) {
+    GTEST_ASSERT_EQ(isUpdated(pSU,i, 1), true);
+  }
+
+  for(int i=3; i < 1024; i++) {
+    GTEST_ASSERT_EQ(isUpdated(pSU,0, i), false);
+  }
+  GTEST_ASSERT_EQ(*(int64_t*)taosArrayGet(pSU->pTsBuckets,0), 1023);
+
+  for(int i=3; i < 1024; i++) {
+    GTEST_ASSERT_EQ(isUpdated(pSU,0, i), true);
+  }
+  GTEST_ASSERT_EQ(*(int64_t*)taosArrayGet(pSU->pTsBuckets,0), 1023);
+
+  SUpdateInfo *pSU1 = updateInfoInit(interval, TSDB_TIME_PRECISION_MILLI, watermark);
+  for(int i=1; i <= watermark / interval; i++) {
+    GTEST_ASSERT_EQ(isUpdated(pSU1, 1, i * interval + 5), false);
+    GTEST_ASSERT_EQ(pSU1->minTS, interval);
+    GTEST_ASSERT_EQ(pSU1->numSBFs, watermark / interval);
+  }
+  for(int i=0; i < pSU1->numSBFs; i++) {
+    SScalableBf *pSBF = (SScalableBf *)taosArrayGetP(pSU1->pTsSBFs, i);
+    SBloomFilter *pBF = (SBloomFilter *)taosArrayGetP(pSBF->bfArray, 0);
+    GTEST_ASSERT_EQ(pBF->size, 1);
+  }
+
+  for(int i= watermark / interval + 1, j = 2 ; i <= watermark / interval + 10; i++,j++) {
+    GTEST_ASSERT_EQ(isUpdated(pSU1, 1, i * interval + 5), false);
+    GTEST_ASSERT_EQ(pSU1->minTS, interval*j);
+    GTEST_ASSERT_EQ(pSU1->numSBFs, watermark / interval);
+    SScalableBf *pSBF = (SScalableBf *)taosArrayGetP(pSU1->pTsSBFs, pSU1->numSBFs - 1);
+    SBloomFilter *pBF = (SBloomFilter *)taosArrayGetP(pSBF->bfArray, 0);
+    GTEST_ASSERT_EQ(pBF->size, 1);
+  }
+  
+  for(int i= watermark / interval * 100, j = 0; j < 10; i+= (watermark / interval * 2), j++) {
+    GTEST_ASSERT_EQ(isUpdated(pSU1, 1, i * interval + 5), false);
+    GTEST_ASSERT_EQ(pSU1->minTS, (i-(pSU1->numSBFs-1))*interval);
+    GTEST_ASSERT_EQ(pSU1->numSBFs, watermark / interval);
+  }
+
+  SUpdateInfo *pSU2 = updateInfoInit(interval, TSDB_TIME_PRECISION_MILLI, watermark);
+  GTEST_ASSERT_EQ(isUpdated(pSU2, 1, 1 * interval + 5), false);
+  GTEST_ASSERT_EQ(pSU2->minTS, interval);
+  for(int i= watermark / interval * 100, j = 0; j < 10; i+= (watermark / interval * 10), j++) {
+    GTEST_ASSERT_EQ(isUpdated(pSU2, 1, i * interval + 5), false);
+    GTEST_ASSERT_EQ(pSU2->minTS, (i-(pSU2->numSBFs-1))*interval);
+    GTEST_ASSERT_EQ(pSU2->numSBFs, watermark / interval);
+    GTEST_ASSERT_EQ(*(int64_t*)taosArrayGet(pSU2->pTsBuckets,1), i * interval + 5);
+  }
+  
+  SUpdateInfo *pSU3 = updateInfoInit(interval, TSDB_TIME_PRECISION_MILLI, watermark);
+  for(int j = 1; j < 100; j++) {
+    for(int i = 0; i < pSU3->numSBFs; i++) {
+      GTEST_ASSERT_EQ(isUpdated(pSU3, i, i * interval + 5 * j), false);
+      GTEST_ASSERT_EQ(pSU3->minTS, 0);
+      GTEST_ASSERT_EQ(pSU3->numSBFs, watermark / interval);
+      GTEST_ASSERT_EQ(*(int64_t*)taosArrayGet(pSU3->pTsBuckets, i), i * interval + 5 * j);
+      SScalableBf *pSBF = (SScalableBf *)taosArrayGetP(pSU3->pTsSBFs, i);
+      SBloomFilter *pBF = (SBloomFilter *)taosArrayGetP(pSBF->bfArray, 0);
+      GTEST_ASSERT_EQ(pBF->size, j);
+    }
+  }
+
+  SUpdateInfo *pSU4 = updateInfoInit(-1, TSDB_TIME_PRECISION_MILLI, -1);
+  GTEST_ASSERT_EQ(pSU4->watermark, 120 * pSU4->interval);
+  GTEST_ASSERT_EQ(pSU4->interval, MILLISECOND_PER_MINUTE);
+
+  SUpdateInfo *pSU5 = updateInfoInit(0, TSDB_TIME_PRECISION_MILLI, 0);
+  GTEST_ASSERT_EQ(pSU5->watermark, 120 * pSU4->interval);
+  GTEST_ASSERT_EQ(pSU5->interval, MILLISECOND_PER_MINUTE);
+
+
+  updateInfoDestroy(pSU);
+  updateInfoDestroy(pSU1);
+  updateInfoDestroy(pSU2);
+  updateInfoDestroy(pSU3);
+  updateInfoDestroy(pSU4);
+  updateInfoDestroy(pSU5);
+}
+
+int main(int argc, char* argv[]) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

source/libs/transport/inc/transComm.h

@@ -28,6 +28,7 @@ extern "C" {
 #include "taoserror.h"
 #include "tglobal.h"
 #include "thash.h"
+#include "theap.h"
 #include "tidpool.h"
 #include "tmd5.h"
 #include "tmempool.h"
@@ -328,10 +329,38 @@ void transQueueClear(STransQueue* queue);
  */
 void transQueueDestroy(STransQueue* queue);
 
+/*
+ * delay queue based on uv loop and uv timer, and only used in retry
+ */
+typedef struct STaskArg {
+  void* param1;
+  void* param2;
+} STaskArg;
+
+typedef struct SDelayTask {
+  void (*func)(void* arg);
+  void*    arg;
+  uint64_t execTime;
+  HeapNode node;
+} SDelayTask;
+
+typedef struct SDelayQueue {
+  uv_timer_t* timer;
+  Heap*       heap;
+  uv_loop_t*  loop;
+} SDelayQueue;
+
+int transDQCreate(uv_loop_t* loop, SDelayQueue** queue);
+
+void transDQDestroy(SDelayQueue* queue);
+
+int transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs);
+
 /*
  * init global func
  */
 void transThreadOnce();
+
 #ifdef __cplusplus
 }
 #endif

source/libs/transport/src/transCli.c

@@ -60,7 +60,7 @@ typedef struct SCliThrdObj {
   // msg queue
   queue         msg;
   TdThreadMutex msgMtx;
-
+  SDelayQueue*  delayQueue;
   uint64_t      nextTimeout;  // next timeout
   void*         pTransInst;   //
   bool          quit;
@@ -161,8 +161,7 @@ static void cliWalkCb(uv_handle_t* handle, void* arg);
         transUnrefCliHandle(conn);                                                       \
       }                                                                                  \
       if (T_REF_VAL_GET(conn) == 1) {                                                    \
-        SCliThrdObj* thrd = conn->hostThrd;                                              \
+        transUnrefCliHandle(conn);                                                       \
-        addConnToPool(thrd->pool, conn);                                                 \
       }                                                                                  \
       destroyCmsg(pMsg);                                                                 \
       return;                                                                            \
@@ -838,12 +837,13 @@ static SCliThrdObj* createThrdObj() {
   uv_loop_init(pThrd->loop);
 
   pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, 5, pThrd, cliAsyncCb);
-
   uv_timer_init(pThrd->loop, &pThrd->timer);
   pThrd->timer.data = pThrd;
 
   pThrd->pool = createConnPool(4);
 
+  transDQCreate(pThrd->loop, &pThrd->delayQueue);
+
   pThrd->quit = false;
   return pThrd;
 }
@@ -851,12 +851,13 @@ static void destroyThrdObj(SCliThrdObj* pThrd) {
   if (pThrd == NULL) {
     return;
   }
+
   taosThreadJoin(pThrd->thread, NULL);
   CLI_RELEASE_UV(pThrd->loop);
   taosThreadMutexDestroy(&pThrd->msgMtx);
   transDestroyAsyncPool(pThrd->asyncPool);
 
-  uv_timer_stop(&pThrd->timer);
+  transDQDestroy(pThrd->delayQueue);
   taosMemoryFree(pThrd->loop);
   taosMemoryFree(pThrd);
 }
@@ -885,6 +886,16 @@ int cliRBChoseIdx(STrans* pTransInst) {
   }
   return index % pTransInst->numOfThreads;
 }
+static void doDelayTask(void* param) {
+  STaskArg* arg = param;
+
+  SCliMsg*     pMsg = arg->param1;
+  SCliThrdObj* pThrd = arg->param2;
+
+  cliHandleReq(pMsg, pThrd);
+
+  taosMemoryFree(arg);
+}
 int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
   SCliThrdObj* pThrd = pConn->hostThrd;
   STrans*      pTransInst = pThrd->pTransInst;
@@ -908,20 +919,28 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
     if (msgType == TDMT_MND_CONNECT && pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
       if (pCtx->retryCount < pEpSet->numOfEps) {
         pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps;
-        cliHandleReq(pMsg, pThrd);
+
+        STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg));
+        arg->param1 = pMsg;
+        arg->param2 = pThrd;
+        transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL);
+
         cliDestroy((uv_handle_t*)pConn->stream);
         return -1;
       }
     } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) {
       if (pResp->contLen == 0) {
-        pEpSet->inUse = (pEpSet->inUse++) % pEpSet->numOfEps;
+        pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps;
       } else {
         SMEpSet emsg = {0};
         tDeserializeSMEpSet(pResp->pCont, pResp->contLen, &emsg);
         pCtx->epSet = emsg.epSet;
       }
-      cliHandleReq(pMsg, pThrd);
+      STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg));
-      // release pConn
+      arg->param1 = pMsg;
+      arg->param2 = pThrd;
+
+      transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL);
       addConnToPool(pThrd, pConn);
       return -1;
     }

source/libs/transport/src/transComm.c

@@ -358,4 +358,81 @@ void transQueueDestroy(STransQueue* queue) {
   transQueueClear(queue);
   taosArrayDestroy(queue->q);
 }
+
+static int32_t timeCompare(const HeapNode* a, const HeapNode* b) {
+  SDelayTask* arg1 = container_of(a, SDelayTask, node);
+  SDelayTask* arg2 = container_of(b, SDelayTask, node);
+  if (arg1->execTime > arg2->execTime) {
+    return 0;
+  } else {
+    return 1;
+  }
+}
+
+static void transDQTimeout(uv_timer_t* timer) {
+  SDelayQueue* queue = timer->data;
+  tTrace("timer %p timeout", timer);
+  uint64_t timeout = 0;
+  do {
+    HeapNode* minNode = heapMin(queue->heap);
+    if (minNode == NULL) break;
+    SDelayTask* task = container_of(minNode, SDelayTask, node);
+    if (task->execTime <= taosGetTimestampMs()) {
+      heapRemove(queue->heap, minNode);
+      task->func(task->arg);
+      taosMemoryFree(task);
+      timeout = 0;
+    } else {
+      timeout = task->execTime - taosGetTimestampMs();
+      break;
+    }
+  } while (1);
+  if (timeout != 0) {
+    uv_timer_start(queue->timer, transDQTimeout, timeout, 0);
+  }
+}
+int transDQCreate(uv_loop_t* loop, SDelayQueue** queue) {
+  uv_timer_t* timer = taosMemoryCalloc(1, sizeof(uv_timer_t));
+  uv_timer_init(loop, timer);
+
+  Heap* heap = heapCreate(timeCompare);
+
+  SDelayQueue* q = taosMemoryCalloc(1, sizeof(SDelayQueue));
+  q->heap = heap;
+  q->timer = timer;
+  q->loop = loop;
+  q->timer->data = q;
+
+  *queue = q;
+  return 0;
+}
+
+void transDQDestroy(SDelayQueue* queue) {
+  taosMemoryFree(queue->timer);
+
+  while (heapSize(queue->heap) > 0) {
+    HeapNode* minNode = heapMin(queue->heap);
+    if (minNode == NULL) {
+      return;
+    }
+    heapRemove(queue->heap, minNode);
+
+    SDelayTask* task = container_of(minNode, SDelayTask, node);
+    taosMemoryFree(task);
+  }
+  heapDestroy(queue->heap);
+  taosMemoryFree(queue);
+}
+
+int transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs) {
+  SDelayTask* task = taosMemoryCalloc(1, sizeof(SDelayTask));
+  task->func = func;
+  task->arg = arg;
+  task->execTime = taosGetTimestampMs() + timeoutMs;
+
+  tTrace("timer %p put task into queue, timeoutMs: %" PRIu64 "", queue->timer, timeoutMs);
+  heapInsert(queue->heap, &task->node);
+  uv_timer_start(queue->timer, transDQTimeout, timeoutMs, 0);
+  return 0;
+}
 #endif

source/os/src/osDir.c

@@ -72,8 +72,8 @@ void taosRemoveDir(const char *dirname) {
   while ((de = taosReadDir(pDir)) != NULL) {
     if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue;
 
-    char filename[1024];
+    char filename[1024] = {0};
-    snprintf(filename, sizeof(filename), "%s/%s", dirname, taosGetDirEntryName(de));
+    snprintf(filename, sizeof(filename), "%s%s%s", dirname, TD_DIRSEP, taosGetDirEntryName(de));
     if (taosDirEntryIsDir(de)) {
       taosRemoveDir(filename);
     } else {

source/util/src/tarray.c

@@ -318,6 +318,20 @@ void taosArrayClearEx(SArray* pArray, void (*fp)(void*)) {
   pArray->size = 0;
 }
 
+void taosArrayClearP(SArray* pArray, FDelete fp) {
+  if (pArray == NULL) return;
+  if (fp == NULL) {
+    pArray->size = 0;
+    return;
+  }
+
+  for (int32_t i = 0; i < pArray->size; ++i) {
+    fp(*(void**)TARRAY_GET_ELEM(pArray, i));
+  }
+
+  pArray->size = 0;
+}
+
 void* taosArrayDestroy(SArray* pArray) {
   if (pArray) {
     taosMemoryFree(pArray->pData);

source/util/src/tbloomfilter.c

@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+
+#include "tbloomfilter.h"
+#include "taos.h"
+#include "taoserror.h"
+
+#define UNIT_NUM_BITS 64
+#define UNIT_ADDR_NUM_BITS 6
+
+static FORCE_INLINE bool setBit(uint64_t *buf, uint64_t index) {
+  uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS;
+  uint64_t mask = 1 << (index % UNIT_NUM_BITS);
+  uint64_t old = buf[unitIndex];
+  buf[unitIndex] |= mask;
+  return buf[unitIndex] != old;
+}
+
+static FORCE_INLINE bool getBit(uint64_t *buf, uint64_t index) {
+  uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS;
+  uint64_t mask = 1 << (index % UNIT_NUM_BITS);
+  return buf[unitIndex] & mask;
+}
+
+SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate) {
+  if (expectedEntries < 1 || errorRate <= 0 || errorRate >= 1.0) {
+    return NULL;
+  }
+  SBloomFilter *pBF = taosMemoryCalloc(1, sizeof(SBloomFilter));
+  if (pBF == NULL) {
+    return NULL;
+  }
+  pBF->expectedEntries = expectedEntries;
+  pBF->errorRate = errorRate;
+
+  double lnRate = fabs(log(errorRate));
+  // ln(2)^2 = 0.480453013918201
+  // m = - n * ln(P) / ( ln(2) )^2
+  // m is the size of bloom filter, n is expected entries, P is false positive probability
+  pBF->numUnits = (uint64_t) ceil(expectedEntries * lnRate / 0.480453013918201 / UNIT_NUM_BITS);
+  pBF->numBits = pBF->numUnits * 64;
+  pBF->size = 0;
+
+  // ln(2) = 0.693147180559945
+  pBF->hashFunctions = (uint32_t) ceil(lnRate / 0.693147180559945);
+  pBF->hashFn1 = taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP);
+  pBF->hashFn2 = taosGetDefaultHashFunction(TSDB_DATA_TYPE_NCHAR);
+  pBF->buffer = taosMemoryCalloc(pBF->numUnits, sizeof(uint64_t));
+  if (pBF->buffer == NULL) {
+    tBloomFilterDestroy(pBF);
+    return NULL;
+  }
+  return pBF;
+}
+
+int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len) {
+  ASSERT(!tBloomFilterIsFull(pBF));
+  uint64_t h1 = (uint64_t)pBF->hashFn1(keyBuf, len);
+  uint64_t h2 = (uint64_t)pBF->hashFn2(keyBuf, len);
+  bool hasChange = false;
+  const register uint64_t size = pBF->numBits;
+  uint64_t cbHash = h1;
+  for (uint64_t i = 0; i < pBF->hashFunctions; ++i) {
+    hasChange |= setBit(pBF->buffer, cbHash % size);
+    cbHash += h2;
+  }
+  if (hasChange) {
+    pBF->size++;
+    return TSDB_CODE_SUCCESS;
+  }
+  return TSDB_CODE_FAILED;
+}
+
+int32_t tBloomFilterNoContain(const SBloomFilter *pBF, const void *keyBuf,
+                              uint32_t len) {
+  uint64_t h1 = (uint64_t)pBF->hashFn1(keyBuf, len);
+  uint64_t h2 = (uint64_t)pBF->hashFn2(keyBuf, len);
+  const register uint64_t size = pBF->numBits;
+  uint64_t cbHash = h1;
+  for (uint64_t i = 0; i < pBF->hashFunctions; ++i) {
+    if (!getBit(pBF->buffer, cbHash % size)) {
+      return TSDB_CODE_SUCCESS;
+    }
+    cbHash += h2;
+  }
+  return TSDB_CODE_FAILED;
+}
+
+void tBloomFilterDestroy(SBloomFilter *pBF) {
+  if (pBF == NULL) {
+    return;
+  }
+  taosMemoryFree(pBF->buffer);
+  taosMemoryFree(pBF);
+}
+
+void tBloomFilterDump(const struct SBloomFilter *pBF) {
+// ToDo
+}
+
+bool tBloomFilterIsFull(const SBloomFilter *pBF) {
+  return pBF->size >= pBF->expectedEntries;
+}

source/util/src/terror.c

@@ -250,6 +250,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_NOT_EXIST,            "Tag does not exist")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_COLUMNS,         "Too many columns")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_ALREADY_EXIST,     "Column already exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_NOT_EXIST,         "Column does not exist")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_SINGLE_STB_MODE_DB,       "Database is single stable mode")
 
 // mnode-infoSchema
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SYS_TABLENAME,    "Invalid system table name")

source/util/src/tscalablebf.c

@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+
+#include "tscalablebf.h"
+#include "taoserror.h"
+
+#define DEFAULT_GROWTH 2
+#define DEFAULT_TIGHTENING_RATIO 0.5
+
+static SBloomFilter *tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries,
+                                   double errorRate);
+
+SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate) {
+  const uint32_t defaultSize = 8;
+  if (expectedEntries < 1 || errorRate <= 0 || errorRate >= 1.0) {
+    return NULL;
+  }
+  SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf));
+  if (pSBf == NULL) {
+    return NULL;
+  }
+  pSBf->numBits = 0;
+  pSBf->bfArray = taosArrayInit(defaultSize, sizeof(void *));
+  if (tScalableBfAddFilter(pSBf, expectedEntries, errorRate * DEFAULT_TIGHTENING_RATIO) == NULL ) {
+    tScalableBfDestroy(pSBf);
+    return NULL;
+  }
+  pSBf->growth = DEFAULT_GROWTH;
+  return pSBf;
+}
+
+int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) {
+  int32_t size = taosArrayGetSize(pSBf->bfArray);
+  for (int32_t i = size - 2; i >= 0; --i) {
+    if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i),
+                              keyBuf, len) != TSDB_CODE_SUCCESS) {
+      return TSDB_CODE_FAILED;
+    }
+  }
+
+  SBloomFilter *pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1);
+  ASSERT(pNormalBf);
+  if (tBloomFilterIsFull(pNormalBf)) {
+    pNormalBf = tScalableBfAddFilter(pSBf,
+                                     pNormalBf->expectedEntries * pSBf->growth,
+                                     pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO);
+    if (pNormalBf == NULL) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
+  return tBloomFilterPut(pNormalBf, keyBuf, len);
+}
+
+int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf,
+                             uint32_t len) {
+  int32_t size = taosArrayGetSize(pSBf->bfArray);
+  for (int32_t i = size - 1; i >= 0; --i) {
+    if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i),
+                              keyBuf, len) != TSDB_CODE_SUCCESS) {
+      return TSDB_CODE_FAILED;
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static SBloomFilter *tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries,
+                                   double errorRate) {
+  SBloomFilter *pNormalBf = tBloomFilterInit(expectedEntries, errorRate);
+  if (pNormalBf == NULL) {
+    return NULL;
+  }
+  if(taosArrayPush(pSBf->bfArray, &pNormalBf) == NULL) {
+    tBloomFilterDestroy(pNormalBf);
+    return NULL;
+  }
+  pSBf->numBits += pNormalBf->numBits;
+  return pNormalBf;
+}
+
+void tScalableBfDestroy(SScalableBf *pSBf) {
+  if (pSBf == NULL) {
+    return;
+  }
+  if (pSBf->bfArray != NULL) {
+    taosArrayDestroyP(pSBf->bfArray, (FDelete)tBloomFilterDestroy);
+  }
+  taosMemoryFree(pSBf);
+}
+
+void tScalableBfDump(const SScalableBf *pSBf) {
+  // Todo;
+}

source/util/test/CMakeLists.txt

@@ -60,3 +60,11 @@ add_test(
     NAME cfgTest
     COMMAND cfgTest
 )
+
+# bloomFilterTest
+add_executable(bloomFilterTest "bloomFilterTest.cpp")
+target_link_libraries(bloomFilterTest os util gtest_main)   
+add_test(
+    NAME bloomFilterTest
+    COMMAND bloomFilterTest
+)

source/util/test/bloomFilterTest.cpp

@@ -0,0 +1,140 @@
+#include <gtest/gtest.h>
+
+#include "tscalablebf.h"
+#include "taoserror.h"
+
+using namespace std;
+
+TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) {
+  int64_t ts1 = 1650803518000;
+
+  GTEST_ASSERT_EQ(NULL, tBloomFilterInit(100, 0));
+  GTEST_ASSERT_EQ(NULL, tBloomFilterInit(100, 1));
+  GTEST_ASSERT_EQ(NULL, tBloomFilterInit(100, -0.1));
+  GTEST_ASSERT_EQ(NULL, tBloomFilterInit(0, 0.01));
+
+  SBloomFilter *pBF1 = tBloomFilterInit(100, 0.005);
+  GTEST_ASSERT_EQ(pBF1->numBits, 1152);
+  GTEST_ASSERT_EQ(pBF1->numUnits, 1152/64);
+  int64_t count = 0;
+  for(int64_t i = 0; count < 100; i++) {
+    int64_t ts = i + ts1;
+    if(tBloomFilterPut(pBF1, &ts, sizeof(int64_t)) == TSDB_CODE_SUCCESS ) {
+      count++;
+    }
+  }
+  ASSERT_TRUE(tBloomFilterIsFull(pBF1));
+
+  SBloomFilter *pBF2 = tBloomFilterInit(1000*10000, 0.1);
+  GTEST_ASSERT_EQ(pBF2->numBits, 47925312);
+  GTEST_ASSERT_EQ(pBF2->numUnits, 47925312/64);
+
+  SBloomFilter *pBF3 = tBloomFilterInit(10000*10000, 0.001);
+  GTEST_ASSERT_EQ(pBF3->numBits, 1437758784);
+  GTEST_ASSERT_EQ(pBF3->numUnits, 1437758784/64);
+  
+  int64_t size = 10000;
+  SBloomFilter *pBF4 = tBloomFilterInit(size, 0.001);
+  for(int64_t i = 0; i < 1000; i++) {
+    int64_t ts = i + ts1;
+    GTEST_ASSERT_EQ(tBloomFilterPut(pBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS);
+  }
+  ASSERT_TRUE(!tBloomFilterIsFull(pBF4));
+
+  for(int64_t i = 0; i < 1000; i++) {
+    int64_t ts = i + ts1;
+    GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, &ts, sizeof(int64_t)), TSDB_CODE_FAILED);
+  }
+
+  for(int64_t i = 2000; i < 3000; i++) {
+    int64_t ts = i + ts1;
+    GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS);
+  }
+
+  tBloomFilterDestroy(pBF1);
+  tBloomFilterDestroy(pBF2);
+  tBloomFilterDestroy(pBF3);
+  tBloomFilterDestroy(pBF4);
+
+}
+
+TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) {
+  int64_t ts1 = 1650803518000;
+
+  GTEST_ASSERT_EQ(NULL, tScalableBfInit(100, 0));
+  GTEST_ASSERT_EQ(NULL, tScalableBfInit(100, 1));
+  GTEST_ASSERT_EQ(NULL, tScalableBfInit(100, -0.1));
+  GTEST_ASSERT_EQ(NULL, tScalableBfInit(0, 0.01));
+
+  SScalableBf *pSBF1 = tScalableBfInit(100, 0.01);
+  GTEST_ASSERT_EQ(pSBF1->numBits, 1152);
+  int64_t count = 0;
+  int64_t index = 0;
+  for( ; count < 100; index++) {
+    int64_t ts = index + ts1;
+    if(tScalableBfPut(pSBF1, &ts, sizeof(int64_t)) == TSDB_CODE_SUCCESS ) {
+      count++;
+    }
+  }
+  GTEST_ASSERT_EQ(pSBF1->numBits, 1152);
+
+  for( ; count < 300; index++) {
+    int64_t ts = index + ts1;
+    if(tScalableBfPut(pSBF1, &ts, sizeof(int64_t)) == TSDB_CODE_SUCCESS ) {
+      count++;
+    }
+  }
+  GTEST_ASSERT_EQ(pSBF1->numBits, 1152+2496);
+
+  for( ; count < 700; index++) {
+    int64_t ts = index + ts1;
+    if(tScalableBfPut(pSBF1, &ts, sizeof(int64_t)) == TSDB_CODE_SUCCESS ) {
+      count++;
+    }
+  }
+  GTEST_ASSERT_EQ(pSBF1->numBits, 1152+2496+5568);
+
+  for( ; count < 1500; index++) {
+    int64_t ts = index + ts1;
+    if(tScalableBfPut(pSBF1, &ts, sizeof(int64_t)) == TSDB_CODE_SUCCESS ) {
+      count++;
+    }
+  }
+  GTEST_ASSERT_EQ(pSBF1->numBits, 1152+2496+5568+12288);
+  
+  int32_t aSize = taosArrayGetSize(pSBF1->bfArray);
+  int64_t totalBits = 0;
+  for(int64_t i = 0; i < aSize; i++) {
+    SBloomFilter *pBF = (SBloomFilter *)taosArrayGetP(pSBF1->bfArray, i);
+    ASSERT_TRUE(tBloomFilterIsFull(pBF)); 
+    totalBits += pBF->numBits;
+  }
+  GTEST_ASSERT_EQ(pSBF1->numBits, totalBits);
+
+  for(int64_t i = 0; i < index; i++) {
+    int64_t ts = i + ts1;
+    GTEST_ASSERT_EQ(tScalableBfNoContain(pSBF1, &ts, sizeof(int64_t)), TSDB_CODE_FAILED);
+  }
+
+
+  int64_t size = 10000;
+  SScalableBf *pSBF4 = tScalableBfInit(size, 0.001);
+  for(int64_t i = 0; i < 1000; i++) {
+    int64_t ts = i + ts1;
+    GTEST_ASSERT_EQ(tScalableBfPut(pSBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS);
+  }
+
+  for(int64_t i = 0; i < 1000; i++) {
+    int64_t ts = i + ts1;
+    GTEST_ASSERT_EQ(tScalableBfNoContain(pSBF4, &ts, sizeof(int64_t)), TSDB_CODE_FAILED);
+  }
+
+  for(int64_t i = 2000; i < 3000; i++) {
+    int64_t ts = i + ts1;
+    GTEST_ASSERT_EQ(tScalableBfNoContain(pSBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS);
+  }
+
+  tScalableBfDestroy(pSBF1);
+  tScalableBfDestroy(pSBF4);
+
+}

tests/script/jenkins/basic.txt

@@ -81,7 +81,7 @@
 ./test.sh -f tsim/insert/backquote.sim -m
 ./test.sh -f tsim/parser/fourArithmetic-basic.sim -m
 ./test.sh -f tsim/query/interval-offset.sim -m
-#./test.sh -f tsim/tmq/basic1.sim -m
+./test.sh -f tsim/tmq/basic3.sim -m
 ./test.sh -f tsim/stable/vnode3.sim -m
 ./test.sh -f tsim/qnode/basic1.sim -m
 ./test.sh -f tsim/mnode/basic1.sim -m