Merge remote-tracking branch 'origin/3.0' into feature/3.0_liaohj

include/client/taos.h

@@ -54,6 +54,7 @@ typedef void   TAOS_SUB;
 #define TSDB_DATA_TYPE_BLOB       18  // binary
 #define TSDB_DATA_TYPE_MEDIUMBLOB 19
 #define TSDB_DATA_TYPE_BINARY     TSDB_DATA_TYPE_VARCHAR  // string
+#define TSDB_DATA_TYPE_MAX        20
 
 typedef enum {
   TSDB_OPTION_LOCALE,

include/common/taosdef.h

@@ -76,6 +76,13 @@ typedef enum {
   TSDB_SMA_TYPE_ROLLUP = 2,      // Rollup SMA
 } ETsdbSmaType;
 
+typedef enum {
+  TSDB_BSMA_TYPE_NONE = 0,  // no block-wise SMA
+  TSDB_BSMA_TYPE_I = 1,     // sum/min/max(default)
+} ETsdbBSmaType;
+
+#define TSDB_BSMA_TYPE_LATEST TSDB_BSMA_TYPE_I
+
 extern char *qtypeStr[];
 
 #define TSDB_PORT_HTTP      11

include/common/tdataformat.h

@@ -70,11 +70,13 @@ typedef struct {
 #pragma pack(pop)
 
 #define colType(col)   ((col)->type)
+#define colSma(col)    ((col)->sma)
 #define colColId(col)  ((col)->colId)
 #define colBytes(col)  ((col)->bytes)
 #define colOffset(col) ((col)->offset)
 
 #define colSetType(col, t)   (colType(col) = (t))
+#define colSetSma(col, s)    (colSma(col) = (s))
 #define colSetColId(col, id) (colColId(col) = (id))
 #define colSetBytes(col, b)  (colBytes(col) = (b))
 #define colSetOffset(col, o) (colOffset(col) = (o))
@@ -139,7 +141,7 @@ typedef struct {
 int32_t   tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version);
 void      tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder);
 void      tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version);
-int32_t   tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col_bytes_t bytes);
+int32_t   tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t sma, col_id_t colId, col_bytes_t bytes);
 STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
 
 // ----------------- Semantic timestamp key definition

include/common/tmsg.h

@@ -265,6 +265,20 @@ typedef struct SSchema {
   char     name[TSDB_COL_NAME_LEN];
 } SSchema;
 
+typedef struct {
+  int8_t   type;
+  int8_t   sma;  // ETsdbBSmaType and default is TSDB_BSMA_TYPE_I
+  col_id_t colId;
+  int32_t  bytes;
+  char     name[TSDB_COL_NAME_LEN];
+} SSchemaEx;
+
+#define SSCHMEA_TYPE(s) ((s)->type)
+#define SSCHMEA_SMA(s) ((s)->sma)
+#define SSCHMEA_COLID(s) ((s)->colId)
+#define SSCHMEA_BYTES(s) ((s)->bytes)
+#define SSCHMEA_NAME(s) ((s)->name)
+
 typedef struct {
   char    name[TSDB_TABLE_FNAME_LEN];
   int8_t  igExists;
@@ -1381,11 +1395,10 @@ typedef struct {
 } SDDropTopicReq;
 
 typedef struct {
-  float    xFilesFactor;
+  float      xFilesFactor;
-  int8_t   delayUnit;
+  int32_t    delay;
-  int8_t   nFuncIds;
+  int8_t     nFuncIds;
-  int32_t* pFuncIds;
+  func_id_t* pFuncIds;
-  int64_t  delay;
 } SRSmaParam;
 
 typedef struct SVCreateTbReq {
@@ -1403,13 +1416,12 @@ typedef struct SVCreateTbReq {
   };
   union {
     struct {
-      tb_uid_t    suid;
+      tb_uid_t   suid;
-      uint32_t    nCols;
+      col_id_t   nCols;
-      SSchema*    pSchema;
+      col_id_t   nBSmaCols;
-      uint32_t    nTagCols;
+      SSchemaEx* pSchema;
-      SSchema*    pTagSchema;
+      col_id_t   nTagCols;
-      col_id_t    nBSmaCols;
+      SSchema*   pTagSchema;
-      col_id_t*   pBSmaCols;
       SRSmaParam* pRSmaParam;
     } stbCfg;
     struct {
@@ -1417,10 +1429,9 @@ typedef struct SVCreateTbReq {
       SKVRow   pTag;
     } ctbCfg;
     struct {
-      uint32_t    nCols;
+      col_id_t   nCols;
-      SSchema*    pSchema;
+      col_id_t   nBSmaCols;
-      col_id_t    nBSmaCols;
+      SSchemaEx* pSchema;
-      col_id_t*   pBSmaCols;
       SRSmaParam* pRSmaParam;
     } ntbCfg;
   };
@@ -1899,7 +1910,10 @@ int32_t tDecodeSMqCMCommitOffsetReq(SCoder* decoder, SMqCMCommitOffsetReq* pReq)
 
 typedef struct {
   uint32_t nCols;
-  SSchema* pSchema;
+  union {
+    SSchema*   pSchema;
+    SSchemaEx* pSchemaEx;
+  };
 } SSchemaWrapper;
 
 static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema) {

include/common/tmsgcb.h

@@ -50,7 +50,6 @@ typedef struct {
   PutToQueueFp            queueFps[QUEUE_MAX];
   GetQueueSizeFp          qsizeFp;
   SendReqFp               sendReqFp;
-  SendMnodeReqFp          sendMnodeReqFp;
   SendRspFp               sendRspFp;
   RegisterBrokenLinkArgFp registerBrokenLinkArgFp;
   ReleaseHandleFp         releaseHandleFp;
@@ -60,7 +59,6 @@ void    tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb);
 int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq);
 int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype);
 int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq);
-int32_t tmsgSendMnodeReq(const SMsgCb* pMsgCb, SRpcMsg* pReq);
 void    tmsgSendRsp(const SRpcMsg* pRsp);
 void    tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg);
 void    tmsgReleaseHandle(void* handle, int8_t type);

include/common/ttokendef.h

@@ -131,69 +131,91 @@
 #define TK_FUNCTIONS                      113
 #define TK_INDEXES                        114
 #define TK_FROM                           115
-#define TK_LIKE                           116
+#define TK_ACCOUNTS                       116
-#define TK_INDEX                          117
+#define TK_APPS                           117
-#define TK_FULLTEXT                       118
+#define TK_CONNECTIONS                    118
-#define TK_FUNCTION                       119
+#define TK_LICENCE                        119
-#define TK_INTERVAL                       120
+#define TK_QUERIES                        120
-#define TK_TOPIC                          121
+#define TK_SCORES                         121
-#define TK_AS                             122
+#define TK_TOPICS                         122
-#define TK_DESC                           123
+#define TK_VARIABLES                      123
-#define TK_DESCRIBE                       124
+#define TK_LIKE                           124
-#define TK_RESET                          125
+#define TK_INDEX                          125
-#define TK_QUERY                          126
+#define TK_FULLTEXT                       126
-#define TK_EXPLAIN                        127
+#define TK_FUNCTION                       127
-#define TK_ANALYZE                        128
+#define TK_INTERVAL                       128
-#define TK_VERBOSE                        129
+#define TK_TOPIC                          129
-#define TK_NK_BOOL                        130
+#define TK_AS                             130
-#define TK_RATIO                          131
+#define TK_DESC                           131
-#define TK_NULL                           132
+#define TK_DESCRIBE                       132
-#define TK_NK_VARIABLE                    133
+#define TK_RESET                          133
-#define TK_NK_UNDERLINE                   134
+#define TK_QUERY                          134
-#define TK_ROWTS                          135
+#define TK_EXPLAIN                        135
-#define TK_TBNAME                         136
+#define TK_ANALYZE                        136
-#define TK_QSTARTTS                       137
+#define TK_VERBOSE                        137
-#define TK_QENDTS                         138
+#define TK_NK_BOOL                        138
-#define TK_WSTARTTS                       139
+#define TK_RATIO                          139
-#define TK_WENDTS                         140
+#define TK_COMPACT                        140
-#define TK_WDURATION                      141
+#define TK_VNODES                         141
-#define TK_BETWEEN                        142
+#define TK_IN                             142
-#define TK_IS                             143
+#define TK_OUTPUTTYPE                     143
-#define TK_NK_LT                          144
+#define TK_AGGREGATE                      144
-#define TK_NK_GT                          145
+#define TK_BUFSIZE                        145
-#define TK_NK_LE                          146
+#define TK_STREAM                         146
-#define TK_NK_GE                          147
+#define TK_INTO                           147
-#define TK_NK_NE                          148
+#define TK_KILL                           148
-#define TK_MATCH                          149
+#define TK_CONNECTION                     149
-#define TK_NMATCH                         150
+#define TK_MERGE                          150
-#define TK_IN                             151
+#define TK_VGROUP                         151
-#define TK_JOIN                           152
+#define TK_REDISTRIBUTE                   152
-#define TK_INNER                          153
+#define TK_SPLIT                          153
-#define TK_SELECT                         154
+#define TK_SYNCDB                         154
-#define TK_DISTINCT                       155
+#define TK_NULL                           155
-#define TK_WHERE                          156
+#define TK_NK_VARIABLE                    156
-#define TK_PARTITION                      157
+#define TK_NOW                            157
-#define TK_BY                             158
+#define TK_ROWTS                          158
-#define TK_SESSION                        159
+#define TK_TBNAME                         159
-#define TK_STATE_WINDOW                   160
+#define TK_QSTARTTS                       160
-#define TK_SLIDING                        161
+#define TK_QENDTS                         161
-#define TK_FILL                           162
+#define TK_WSTARTTS                       162
-#define TK_VALUE                          163
+#define TK_WENDTS                         163
-#define TK_NONE                           164
+#define TK_WDURATION                      164
-#define TK_PREV                           165
+#define TK_BETWEEN                        165
-#define TK_LINEAR                         166
+#define TK_IS                             166
-#define TK_NEXT                           167
+#define TK_NK_LT                          167
-#define TK_GROUP                          168
+#define TK_NK_GT                          168
-#define TK_HAVING                         169
+#define TK_NK_LE                          169
-#define TK_ORDER                          170
+#define TK_NK_GE                          170
-#define TK_SLIMIT                         171
+#define TK_NK_NE                          171
-#define TK_SOFFSET                        172
+#define TK_MATCH                          172
-#define TK_LIMIT                          173
+#define TK_NMATCH                         173
-#define TK_OFFSET                         174
+#define TK_JOIN                           174
-#define TK_ASC                            175
+#define TK_INNER                          175
-#define TK_NULLS                          176
+#define TK_SELECT                         176
-#define TK_FIRST                          177
+#define TK_DISTINCT                       177
-#define TK_LAST                           178
+#define TK_WHERE                          178
+#define TK_PARTITION                      179
+#define TK_BY                             180
+#define TK_SESSION                        181
+#define TK_STATE_WINDOW                   182
+#define TK_SLIDING                        183
+#define TK_FILL                           184
+#define TK_VALUE                          185
+#define TK_NONE                           186
+#define TK_PREV                           187
+#define TK_LINEAR                         188
+#define TK_NEXT                           189
+#define TK_GROUP                          190
+#define TK_HAVING                         191
+#define TK_ORDER                          192
+#define TK_SLIMIT                         193
+#define TK_SOFFSET                        194
+#define TK_LIMIT                          195
+#define TK_OFFSET                         196
+#define TK_ASC                            197
+#define TK_NULLS                          198
+#define TK_FIRST                          199
+#define TK_LAST                           200
 
 #define TK_NK_SPACE                       300
 #define TK_NK_COMMENT                     301
@@ -207,10 +229,8 @@
 #define TK_NK_COLON                       500
 #define TK_NK_BITNOT                      501
 #define TK_INSERT                         502
-#define TK_INTO                           503
-#define TK_NOW                            504
 #define TK_VALUES                         507
-#define TK_IMPORT                         507
+#define TK_IMPORT                         509
 #define TK_NK_SEMI                        508
 
 #define TK_NK_NIL                         65535

include/common/ttypes.h

@@ -31,6 +31,7 @@ typedef int16_t  col_id_t;
 typedef int8_t   col_type_t;
 typedef int32_t  col_bytes_t;
 typedef uint16_t schema_ver_t;
+typedef int32_t  func_id_t;
 
 #pragma pack(push, 1)
 typedef struct {

include/dnode/mnode/mnode.h

@@ -29,8 +29,7 @@ extern "C" {
 typedef struct SMnode SMnode;
 
 typedef struct {
-  int32_t  dnodeId;
+  bool     deploy;
-  int64_t  clusterId;
   int8_t   replica;
   int8_t   selfIndex;
   SReplica replicas[TSDB_MAX_REPLICA];

include/libs/nodes/cmdnodes.h

@@ -197,6 +197,12 @@ typedef struct SShowStmt {
   SNode* pTbNamePattern; // SValueNode
 } SShowStmt;
 
+typedef struct SShowCreatStmt {
+  ENodeType type;
+  char dbName[TSDB_DB_NAME_LEN];
+  char tableName[TSDB_TABLE_NAME_LEN];
+} SShowCreatStmt;
+
 typedef enum EIndexType {
   INDEX_TYPE_SMA = 1,
   INDEX_TYPE_FULLTEXT

include/libs/nodes/nodes.h

@@ -103,6 +103,15 @@ typedef enum ENodeType {
   QUERY_NODE_EXPLAIN_STMT,
   QUERY_NODE_DESCRIBE_STMT,
   QUERY_NODE_RESET_QUERY_CACHE_STMT,
+  QUERY_NODE_COMPACT_STMT,
+  QUERY_NODE_CREATE_FUNCTION_STMT,
+  QUERY_NODE_DROP_FUNCTION_STMT,
+  QUERY_NODE_CREATE_STREAM_STMT,
+  QUERY_NODE_DROP_STREAM_STMT,
+  QUERY_NODE_MERGE_VGROUP_STMT,
+  QUERY_NODE_REDISTRIBUTE_VGROUP_STMT,
+  QUERY_NODE_SPLIT_VGROUP_STMT,
+  QUERY_NODE_SYNCDB_STMT,
   QUERY_NODE_SHOW_DATABASES_STMT,
   QUERY_NODE_SHOW_TABLES_STMT,
   QUERY_NODE_SHOW_STABLES_STMT,
@@ -115,6 +124,18 @@ typedef enum ENodeType {
   QUERY_NODE_SHOW_FUNCTIONS_STMT,
   QUERY_NODE_SHOW_INDEXES_STMT,
   QUERY_NODE_SHOW_STREAMS_STMT,
+  QUERY_NODE_SHOW_APPS_STMT,
+  QUERY_NODE_SHOW_CONNECTIONS_STMT,
+  QUERY_NODE_SHOW_LICENCE_STMT,
+  QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
+  QUERY_NODE_SHOW_CREATE_TABLE_STMT,
+  QUERY_NODE_SHOW_CREATE_STABLE_STMT, 
+  QUERY_NODE_SHOW_QUERIES_STMT,
+  QUERY_NODE_SHOW_SCORES_STMT,
+  QUERY_NODE_SHOW_TOPICS_STMT,
+  QUERY_NODE_SHOW_VARIABLE_STMT,
+  QUERY_NODE_KILL_CONNECTION_STMT,
+  QUERY_NODE_KILL_QUERY_STMT,
 
   // logic plan node
   QUERY_NODE_LOGIC_PLAN_SCAN,

include/libs/nodes/querynodes.h

@@ -306,6 +306,7 @@ int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNod
 
 bool nodesIsExprNode(const SNode* pNode);
 
+bool nodesIsUnaryOp(const SOperatorNode* pOp);
 bool nodesIsArithmeticOp(const SOperatorNode* pOp);
 bool nodesIsComparisonOp(const SOperatorNode* pOp);
 bool nodesIsJsonOp(const SOperatorNode* pOp);

include/os/osMemory.h

@@ -32,13 +32,16 @@ extern "C" {
 void *taosMemoryMalloc(int32_t size);
 void *taosMemoryCalloc(int32_t num, int32_t size);
 void *taosMemoryRealloc(void *ptr, int32_t size);
+void *taosMemoryStrDup(void *ptr);
 void taosMemoryFree(const void *ptr);
 int32_t taosMemorySize(void *ptr);
 
 #define taosMemoryFreeClear(ptr) \
   do {                           \
-    taosMemoryFree(ptr);         \
+    if (ptr) {                   \
-    (ptr)=NULL;                  \
+      taosMemoryFree(ptr);       \
+      (ptr) = NULL;              \
+    }                            \
   } while (0)
 
 #ifdef __cplusplus

include/os/osShm.h

@@ -26,7 +26,7 @@ typedef struct {
   void*   ptr;
 } SShm;
 
-int32_t taosCreateShm(SShm *pShm, int32_t shmsize) ;
+int32_t taosCreateShm(SShm *pShm, int32_t key, int32_t shmsize) ;
 void    taosDropShm(SShm *pShm);
 int32_t taosAttachShm(SShm *pShm);
 

include/util/tcoding.h

@@ -57,6 +57,8 @@ static FORCE_INLINE void *taosDecodeFixedI8(const void *buf, int8_t *value) {
   return POINTER_SHIFT(buf, sizeof(*value));
 }
 
+static FORCE_INLINE void *taosSkipFixedLen(const void *buf, size_t len) { return POINTER_SHIFT(buf, len); }
+
 // ---- Fixed U16
 static FORCE_INLINE int32_t taosEncodeFixedU16(void **buf, uint16_t value) {
   if (buf != NULL) {

include/util/tdef.h

@@ -128,18 +128,20 @@ extern const int32_t TYPE_BYTES[15];
   } while (0)
 
 typedef enum EOperatorType {
-  // arithmetic operator
+  // binary arithmetic operator
   OP_TYPE_ADD = 1,
   OP_TYPE_SUB,
   OP_TYPE_MULTI,
   OP_TYPE_DIV,
   OP_TYPE_MOD,
+  // unary arithmetic operator
+  OP_TYPE_MINUS,
 
   // bit operator
   OP_TYPE_BIT_AND,
   OP_TYPE_BIT_OR,
 
-  // comparison operator
+  // binary comparison operator
   OP_TYPE_GREATER_THAN,
   OP_TYPE_GREATER_EQUAL,
   OP_TYPE_LOWER_THAN,
@@ -152,6 +154,7 @@ typedef enum EOperatorType {
   OP_TYPE_NOT_LIKE,
   OP_TYPE_MATCH,
   OP_TYPE_NMATCH,
+  // unary comparison operator
   OP_TYPE_IS_NULL,
   OP_TYPE_IS_NOT_NULL,
   OP_TYPE_IS_TRUE,

include/util/tprocess.h

@@ -22,7 +22,7 @@
 extern "C" {
 #endif
 
-typedef enum { PROC_REQ, PROC_RSP, PROC_REG, PROC_RELEASE } ProcFuncType;
+typedef enum { PROC_QUEUE, PROC_REQ, PROC_RSP, PROC_REGIST, PROC_RELEASE } ProcFuncType;
 
 typedef struct SProcQueue SProcQueue;
 typedef struct SProcObj   SProcObj;
@@ -53,7 +53,7 @@ void      taosProcCleanup(SProcObj *pProc);
 int32_t   taosProcRun(SProcObj *pProc);
 int32_t   taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
                               ProcFuncType ftype);
-int32_t   taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
+void      taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
                                ProcFuncType ftype);
 
 #ifdef __cplusplus

source/client/src/tmq.c

@@ -78,6 +78,7 @@ struct tmq_t {
   STscObj*       pTscObj;
   tmq_commit_cb* commit_cb;
   int32_t        nextTopicIdx;
+  int8_t         epStatus;
   int32_t        waitingRequest;
   int32_t        readyRequest;
   SArray*        clientTopics;  // SArray<SMqClientTopic>
@@ -311,6 +312,7 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
   pTmq->epoch = 0;
   pTmq->waitingRequest = 0;
   pTmq->readyRequest = 0;
+  pTmq->epStatus = 0;
   // set conf
   strcpy(pTmq->clientId, conf->clientId);
   strcpy(pTmq->groupId, conf->groupId);
@@ -833,7 +835,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   tmq_t*          tmq = pParam->tmq;
   if (code != 0) {
     tscWarn("msg discard, code:%x", code);
-    goto WRITE_QUEUE_FAIL;
+    goto CREATE_MSG_FAIL;
   }
 
   int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch;
@@ -873,7 +875,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   /*SMqConsumeRsp* pRsp = taosMemoryCalloc(1, sizeof(SMqConsumeRsp));*/
   tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t));
   if (pRsp == NULL) {
-    goto WRITE_QUEUE_FAIL;
+    goto CREATE_MSG_FAIL;
   }
   memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));
   tDecodeSMqPollRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->msg);
@@ -882,11 +884,13 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   // TODO: alloc mem
   /*pRsp->*/
   /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/
+#if 0
   if (pRsp->msg.numOfTopics == 0) {
     /*printf("no data\n");*/
     taosFreeQitem(pRsp);
-    goto WRITE_QUEUE_FAIL;
+    goto CREATE_MSG_FAIL;
   }
+#endif
 
   tscError("tmq recv poll: vg %d, req offset %ld, rsp offset %ld", pParam->pVg->vgId, pRsp->msg.reqOffset,
            pRsp->msg.rspOffset);
@@ -897,7 +901,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   tsem_post(&tmq->rspSem);
   return 0;
 
-WRITE_QUEUE_FAIL:
+CREATE_MSG_FAIL:
   if (pParam->epoch == tmq->epoch) {
     atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
   }
@@ -907,7 +911,7 @@ WRITE_QUEUE_FAIL:
 
 bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
   /*printf("call update ep %d\n", epoch);*/
-  /*printf("tmq update ep epoch %d to epoch %d\n", tmq->epoch, epoch);*/
+  tscDebug("tmq update ep epoch %d to epoch %d", tmq->epoch, epoch);
   bool    set = false;
   int32_t topicNumGet = taosArrayGetSize(pRsp->topics);
   char    vgKey[TSDB_TOPIC_FNAME_LEN + 22];
@@ -938,7 +942,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
         for (int32_t k = 0; k < vgNumCur; k++) {
           SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, k);
           sprintf(vgKey, "%s:%d", topic.topicName, pVgCur->vgId);
-          /*printf("epoch %d vg %d build %s\n", epoch, pVgCur->vgId, vgKey);*/
+          tscDebug("epoch %d vg %d build %s\n", epoch, pVgCur->vgId, vgKey);
           taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t));
         }
         break;
@@ -952,12 +956,12 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
       sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId);
       int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey));
       int64_t  offset = pVgEp->offset;
-      /*printf("epoch %d vg %d offset og to %ld\n", epoch, pVgEp->vgId, offset);*/
+      tscDebug("epoch %d vg %d offset og to %ld\n", epoch, pVgEp->vgId, offset);
       if (pOffset != NULL) {
         offset = *pOffset;
-        /*printf("epoch %d vg %d found %s\n", epoch, pVgEp->vgId, vgKey);*/
+        tscDebug("epoch %d vg %d found %s\n", epoch, pVgEp->vgId, vgKey);
       }
-      /*printf("epoch %d vg %d offset set to %ld\n", epoch, pVgEp->vgId, offset);*/
+      tscDebug("epoch %d vg %d offset set to %ld\n", epoch, pVgEp->vgId, offset);
       SMqClientVg clientVg = {
           .pollCnt = 0,
           .currentOffset = offset,
@@ -980,6 +984,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
 int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
   SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param;
   tmq_t*           tmq = pParam->tmq;
+  tscDebug("consumer %ld recv ep", tmq->consumerId);
   if (code != 0) {
     tscError("get topic endpoint error, not ready, wait:%d\n", pParam->sync);
     goto END;
@@ -1018,6 +1023,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
   }
 
 END:
+  atomic_store_8(&tmq->epStatus, 0);
   if (pParam->sync) {
     tsem_post(&pParam->rspSem);
   }
@@ -1025,10 +1031,16 @@ END:
 }
 
 int32_t tmqAskEp(tmq_t* tmq, bool sync) {
+  int8_t epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1);
+  if (epStatus == 1) {
+    tscDebug("consumer %ld skip ask ep", tmq->consumerId);
+    return 0;
+  }
   int32_t           tlen = sizeof(SMqCMGetSubEpReq);
   SMqCMGetSubEpReq* req = taosMemoryMalloc(tlen);
   if (req == NULL) {
     tscError("failed to malloc get subscribe ep buf");
+    atomic_store_8(&tmq->epStatus, 0);
     return -1;
   }
   req->consumerId = htobe64(tmq->consumerId);
@@ -1039,6 +1051,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) {
   if (pParam == NULL) {
     tscError("failed to malloc subscribe param");
     taosMemoryFree(req);
+    atomic_store_8(&tmq->epStatus, 0);
     return -1;
   }
   pParam->tmq = tmq;
@@ -1050,6 +1063,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) {
     tsem_destroy(&pParam->rspSem);
     taosMemoryFree(pParam);
     taosMemoryFree(req);
+    atomic_store_8(&tmq->epStatus, 0);
     return -1;
   }
 
@@ -1067,6 +1081,8 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) {
 
   SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
 
+  tscDebug("consumer %ld ask ep", tmq->consumerId);
+
   int64_t transporterId = 0;
   asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
 
@@ -1205,7 +1221,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
       SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
       int32_t      vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
       if (vgStatus != TMQ_VG_STATUS__IDLE) {
-        /*printf("skip vg %d\n", pVg->vgId);*/
+        tscDebug("consumer %ld skip vg %d", tmq->consumerId, pVg->vgId);
         continue;
       }
       SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
@@ -1249,7 +1265,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
       int64_t transporterId = 0;
       /*printf("send poll\n");*/
       atomic_add_fetch_32(&tmq->waitingRequest, 1);
-      /*tscDebug("tmq send poll: vg %d, req offset %ld", pVg->vgId, pVg->currentOffset);*/
+      tscDebug("consumer %ld send poll: vg %d, req offset %ld", tmq->consumerId, pVg->vgId, pVg->currentOffset);
       /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/
       asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
       pVg->pollCnt++;
@@ -1260,13 +1276,13 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
 }
 
 // return
-int32_t tmqHandleRes(tmq_t* tmq, SMqRspHead* rspHead, bool* pReset) {
+int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspHead* rspHead, bool* pReset) {
   if (rspHead->mqMsgType == TMQ_MSG_TYPE__EP_RSP) {
     /*printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch);*/
     if (rspHead->epoch > atomic_load_32(&tmq->epoch)) {
       SMqCMGetSubEpRsp* rspMsg = (SMqCMGetSubEpRsp*)rspHead;
       tmqUpdateEp(tmq, rspHead->epoch, rspMsg);
-      tmqClearUnhandleMsg(tmq);
+      /*tmqClearUnhandleMsg(tmq);*/
       *pReset = true;
     } else {
       *pReset = false;
@@ -1297,6 +1313,11 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese
         /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/
         pVg->currentOffset = rspMsg->msg.rspOffset;
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
+        if (rspMsg->msg.numOfTopics == 0) {
+          taosFreeQitem(rspMsg);
+          rspHead = NULL;
+          continue;
+        }
         return rspMsg;
       } else {
         /*printf("epoch mismatch\n");*/
@@ -1305,10 +1326,10 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese
     } else {
       /*printf("handle ep rsp %d\n", rspMsg->head.mqMsgType);*/
       bool reset = false;
-      tmqHandleRes(tmq, rspHead, &reset);
+      tmqHandleNoPollRsp(tmq, rspHead, &reset);
       taosFreeQitem(rspHead);
       if (pollIfReset && reset) {
-        printf("reset and repoll\n");
+        tscDebug("consumer %ld reset and repoll", tmq->consumerId);
         tmqPollImpl(tmq, blockingTime);
       }
     }
@@ -1547,24 +1568,3 @@ TAOS_ROW tmq_get_row(tmq_message_t* message) {
 }
 
 char* tmq_get_topic_name(tmq_message_t* message) { return "not implemented yet"; }
-
-#if 0
-tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) {
-  tmq_t* pTmq = taosMemoryMalloc(sizeof(tmq_t));
-  if (pTmq == NULL) {
-    return NULL;
-  }
-  strcpy(pTmq->groupId, conf->groupId);
-  strcpy(pTmq->clientId, conf->clientId);
-  pTmq->pTscObj = (STscObj*)conn;
-  pTmq->pTscObj->connType = HEARTBEAT_TYPE_MQ;
-  return pTmq;
-}
-
-
-static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) {
-  assert(pMsgBody != NULL);
-  taosMemoryFreeClear(pMsgBody->msgInfo.pData);
-  taosMemoryFreeClear(pMsgBody);
-}
-#endif

source/common/src/tdataformat.c

@@ -85,6 +85,7 @@ int tdEncodeSchema(void **buf, STSchema *pSchema) {
   for (int i = 0; i < schemaNCols(pSchema); i++) {
     STColumn *pCol = schemaColAt(pSchema, i);
     tlen += taosEncodeFixedI8(buf, colType(pCol));
+    tlen += taosEncodeFixedI8(buf, colSma(pCol));
     tlen += taosEncodeFixedI16(buf, colColId(pCol));
     tlen += taosEncodeFixedI16(buf, colBytes(pCol));
   }
@@ -107,12 +108,14 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) {
 
   for (int i = 0; i < numOfCols; i++) {
     col_type_t  type = 0;
+    int8_t      sma = TSDB_BSMA_TYPE_NONE;
     col_id_t    colId = 0;
     col_bytes_t bytes = 0;
     buf = taosDecodeFixedI8(buf, &type);
+    buf = taosDecodeFixedI8(buf, &sma);
     buf = taosDecodeFixedI16(buf, &colId);
     buf = taosDecodeFixedI32(buf, &bytes);
-    if (tdAddColToSchema(&schemaBuilder, type, colId, bytes) < 0) {
+    if (tdAddColToSchema(&schemaBuilder, type, sma, colId, bytes) < 0) {
       tdDestroyTSchemaBuilder(&schemaBuilder);
       return NULL;
     }
@@ -148,7 +151,7 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) {
   pBuilder->version = version;
 }
 
-int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col_bytes_t bytes) {
+int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t sma, col_id_t colId, col_bytes_t bytes) {
   if (!isValidDataType(type)) return -1;
 
   if (pBuilder->nCols >= pBuilder->tCols) {
@@ -161,6 +164,7 @@ int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col
   STColumn *pCol = &(pBuilder->columns[pBuilder->nCols]);
   colSetType(pCol, type);
   colSetColId(pCol, colId);
+  colSetSma(pCol, sma);
   if (pBuilder->nCols == 0) {
     colSetOffset(pCol, 0);
   } else {
@@ -168,9 +172,6 @@ int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col
     colSetOffset(pCol, pTCol->offset + TYPE_BYTES[pTCol->type]);
   }
 
-  // TODO: set sma value by user input
-  pCol->sma = 1;
-
   if (IS_VAR_DATA_TYPE(type)) {
     colSetBytes(pCol, bytes);
     pBuilder->tlen += (TYPE_BYTES[type] + bytes);

source/common/src/tmsg.c

@@ -296,33 +296,30 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
   switch (pReq->type) {
     case TD_SUPER_TABLE:
       tlen += taosEncodeFixedI64(buf, pReq->stbCfg.suid);
-      tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nCols);
+      tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nCols);
-      for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) {
+      tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nBSmaCols);
+      for (col_id_t i = 0; i < pReq->stbCfg.nCols; ++i) {
         tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].type);
+        tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].sma);
         tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pSchema[i].colId);
         tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].bytes);
         tlen += taosEncodeString(buf, pReq->stbCfg.pSchema[i].name);
       }
-      tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nTagCols);
+      tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nTagCols);
-      for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) {
+      for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; ++i) {
         tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].type);
         tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pTagSchema[i].colId);
         tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].bytes);
         tlen += taosEncodeString(buf, pReq->stbCfg.pTagSchema[i].name);
       }
-      tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nBSmaCols);
-      for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) {
-        tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pBSmaCols[i]);
-      }
       if (pReq->rollup && pReq->stbCfg.pRSmaParam) {
         SRSmaParam *param = pReq->stbCfg.pRSmaParam;
-        tlen += taosEncodeFixedU32(buf, (uint32_t)param->xFilesFactor);
+        tlen += taosEncodeBinary(buf, (const void *)&param->xFilesFactor, sizeof(param->xFilesFactor));
-        tlen += taosEncodeFixedI8(buf, param->delayUnit);
+        tlen += taosEncodeFixedI32(buf, param->delay);
         tlen += taosEncodeFixedI8(buf, param->nFuncIds);
         for (int8_t i = 0; i < param->nFuncIds; ++i) {
           tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]);
         }
-        tlen += taosEncodeFixedI64(buf, param->delay);
       }
       break;
     case TD_CHILD_TABLE:
@@ -330,26 +327,23 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
       tlen += tdEncodeKVRow(buf, pReq->ctbCfg.pTag);
       break;
     case TD_NORMAL_TABLE:
-      tlen += taosEncodeFixedU32(buf, pReq->ntbCfg.nCols);
+      tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nCols);
-      for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) {
+      tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nBSmaCols);
+      for (col_id_t i = 0; i < pReq->ntbCfg.nCols; ++i) {
         tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].type);
+        tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].sma);
         tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.pSchema[i].colId);
         tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].bytes);
         tlen += taosEncodeString(buf, pReq->ntbCfg.pSchema[i].name);
       }
-      tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nBSmaCols);
-      for (col_id_t i = 0; i < pReq->ntbCfg.nBSmaCols; ++i) {
-        tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.pBSmaCols[i]);
-      }
       if (pReq->rollup && pReq->ntbCfg.pRSmaParam) {
         SRSmaParam *param = pReq->ntbCfg.pRSmaParam;
-        tlen += taosEncodeFixedU32(buf, (uint32_t)param->xFilesFactor);
+        tlen += taosEncodeBinary(buf, (const void *)&param->xFilesFactor, sizeof(param->xFilesFactor));
-        tlen += taosEncodeFixedI8(buf, param->delayUnit);
+        tlen += taosEncodeFixedI32(buf, param->delay);
         tlen += taosEncodeFixedI8(buf, param->nFuncIds);
         for (int8_t i = 0; i < param->nFuncIds; ++i) {
           tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]);
         }
-        tlen += taosEncodeFixedI64(buf, param->delay);
       }
       break;
     default:
@@ -370,45 +364,38 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
   switch (pReq->type) {
     case TD_SUPER_TABLE:
       buf = taosDecodeFixedI64(buf, &(pReq->stbCfg.suid));
-      buf = taosDecodeFixedU32(buf, &(pReq->stbCfg.nCols));
+      buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nCols));
-      pReq->stbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchema));
+      buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols));
-      for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) {
+      pReq->stbCfg.pSchema = (SSchemaEx *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchemaEx));
+      for (col_id_t i = 0; i < pReq->stbCfg.nCols; i++) {
         buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type));
+        buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].sma));
         buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.pSchema[i].colId));
         buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].bytes));
         buf = taosDecodeStringTo(buf, pReq->stbCfg.pSchema[i].name);
       }
-      buf = taosDecodeFixedU32(buf, &pReq->stbCfg.nTagCols);
+      buf = taosDecodeFixedI16(buf, &pReq->stbCfg.nTagCols);
       pReq->stbCfg.pTagSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nTagCols * sizeof(SSchema));
-      for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) {
+      for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; i++) {
         buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type));
         buf = taosDecodeFixedI16(buf, &pReq->stbCfg.pTagSchema[i].colId);
         buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes);
         buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name);
       }
-      buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols));
-      if (pReq->stbCfg.nBSmaCols > 0) {
-        pReq->stbCfg.pBSmaCols = (col_id_t *)taosMemoryMalloc(pReq->stbCfg.nBSmaCols * sizeof(col_id_t));
-        for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) {
-          buf = taosDecodeFixedI16(buf, pReq->stbCfg.pBSmaCols + i);
-        }
-      } else {
-        pReq->stbCfg.pBSmaCols = NULL;
-      }
       if (pReq->rollup) {
         pReq->stbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam));
         SRSmaParam *param = pReq->stbCfg.pRSmaParam;
-        buf = taosDecodeFixedU32(buf, (uint32_t *)&param->xFilesFactor);
+        buf = taosDecodeBinaryTo(buf, (void*)&param->xFilesFactor, sizeof(param->xFilesFactor));
-        buf = taosDecodeFixedI8(buf, &param->delayUnit);
+        buf = taosDecodeFixedI32(buf, &param->delay);
         buf = taosDecodeFixedI8(buf, &param->nFuncIds);
         if (param->nFuncIds > 0) {
+          param->pFuncIds = (func_id_t *)taosMemoryMalloc(param->nFuncIds * sizeof(func_id_t));
           for (int8_t i = 0; i < param->nFuncIds; ++i) {
             buf = taosDecodeFixedI32(buf, param->pFuncIds + i);
           }
         } else {
           param->pFuncIds = NULL;
         }
-        buf = taosDecodeFixedI64(buf, &param->delay);
       } else {
         pReq->stbCfg.pRSmaParam = NULL;
       }
@@ -418,37 +405,30 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
       buf = tdDecodeKVRow(buf, &pReq->ctbCfg.pTag);
       break;
     case TD_NORMAL_TABLE:
-      buf = taosDecodeFixedU32(buf, &pReq->ntbCfg.nCols);
+      buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.nCols);
-      pReq->ntbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchema));
+      buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols));
-      for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) {
+      pReq->ntbCfg.pSchema = (SSchemaEx *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchemaEx));
+      for (col_id_t i = 0; i < pReq->ntbCfg.nCols; i++) {
         buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type);
+        buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].sma);
         buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.pSchema[i].colId);
         buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].bytes);
         buf = taosDecodeStringTo(buf, pReq->ntbCfg.pSchema[i].name);
       }
-      buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols));
-      if (pReq->ntbCfg.nBSmaCols > 0) {
-        pReq->ntbCfg.pBSmaCols = (col_id_t *)taosMemoryMalloc(pReq->ntbCfg.nBSmaCols * sizeof(col_id_t));
-        for (col_id_t i = 0; i < pReq->ntbCfg.nBSmaCols; ++i) {
-          buf = taosDecodeFixedI16(buf, pReq->ntbCfg.pBSmaCols + i);
-        }
-      } else {
-        pReq->ntbCfg.pBSmaCols = NULL;
-      }
       if (pReq->rollup) {
         pReq->ntbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam));
         SRSmaParam *param = pReq->ntbCfg.pRSmaParam;
-        buf = taosDecodeFixedU32(buf, (uint32_t *)&param->xFilesFactor);
+        buf = taosDecodeBinaryTo(buf, (void*)&param->xFilesFactor, sizeof(param->xFilesFactor));
-        buf = taosDecodeFixedI8(buf, &param->delayUnit);
+        buf = taosDecodeFixedI32(buf, &param->delay);
         buf = taosDecodeFixedI8(buf, &param->nFuncIds);
         if (param->nFuncIds > 0) {
+          param->pFuncIds = (func_id_t *)taosMemoryMalloc(param->nFuncIds * sizeof(func_id_t));
           for (int8_t i = 0; i < param->nFuncIds; ++i) {
             buf = taosDecodeFixedI32(buf, param->pFuncIds + i);
           }
         } else {
           param->pFuncIds = NULL;
         }
-        buf = taosDecodeFixedI64(buf, &param->delay);
       } else {
         pReq->ntbCfg.pRSmaParam = NULL;
       }

source/common/src/tmsgcb.c

@@ -32,10 +32,6 @@ int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq) {
   return (*pMsgCb->sendReqFp)(pMsgCb->pWrapper, epSet, pReq);
 }
 
-int32_t tmsgSendMnodeReq(const SMsgCb* pMsgCb, SRpcMsg* pReq) {
-  return (*pMsgCb->sendMnodeReqFp)(pMsgCb->pWrapper, pReq);
-}
-
 void tmsgSendRsp(const SRpcMsg* pRsp) { return (*tsDefaultMsgCb.sendRspFp)(tsDefaultMsgCb.pWrapper, pRsp); }
 
 void tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg) {

source/common/src/ttypes.c

@@ -431,7 +431,7 @@ FORCE_INLINE void *getDataMax(int32_t type) {
   }
 }
 
-bool isValidDataType(int32_t type) { return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_UBIGINT; }
+bool isValidDataType(int32_t type) { return type >= TSDB_DATA_TYPE_NULL && type < TSDB_DATA_TYPE_MAX; }
 
 void setVardataNull(void *val, int32_t type) {
   if (type == TSDB_DATA_TYPE_BINARY) {

source/dnode/mgmt/bm/src/bmInt.c

@@ -19,12 +19,7 @@
 static int32_t bmRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); }
 
 static void bmInitOption(SBnodeMgmt *pMgmt, SBnodeOpt *pOption) {
-  SMsgCb msgCb = {0};
+  SMsgCb msgCb = dndCreateMsgcb(pMgmt->pWrapper);
-  msgCb.pWrapper = pMgmt->pWrapper;
-  msgCb.sendReqFp = dndSendReqToDnode;
-  msgCb.sendMnodeReqFp = dndSendReqToMnode;
-  msgCb.sendRspFp = dndSendRsp;
-  msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
   pOption->msgCb = msgCb;
 }
 

source/dnode/mgmt/main/exe/dndMain.c

@@ -30,7 +30,6 @@ static struct {
 } global = {0};
 
 static void dndStopDnode(int signum, void *info, void *ctx) {
-  dInfo("system signal:%d received", signum);
   SDnode *pDnode = atomic_val_compare_exchange_ptr(&global.pDnode, 0, global.pDnode);
   if (pDnode != NULL) {
     dndHandleEvent(pDnode, DND_EVENT_STOP);
@@ -41,8 +40,10 @@ static void dndSetSignalHandle() {
   taosSetSignal(SIGTERM, dndStopDnode);
   taosSetSignal(SIGHUP, dndStopDnode);
   taosSetSignal(SIGINT, dndStopDnode);
+  taosSetSignal(SIGTSTP, dndStopDnode);
   taosSetSignal(SIGABRT, dndStopDnode);
   taosSetSignal(SIGBREAK, dndStopDnode);
+  taosSetSignal(SIGQUIT, dndStopDnode);
 
   if (!tsMultiProcess) {
   } else if (global.ntype == DNODE || global.ntype == NODE_MAX) {
@@ -72,7 +73,7 @@ static int32_t dndParseArgs(int32_t argc, char const *argv[]) {
     } else if (strcmp(argv[i], "-n") == 0) {
       global.ntype = atoi(argv[++i]);
       if (global.ntype <= DNODE || global.ntype > NODE_MAX) {
-        printf("'-n' range is [1-5], default is 0\n");
+        printf("'-n' range is [1 - %d], default is 0\n", NODE_MAX - 1);
         return -1;
       }
     } else if (strcmp(argv[i], "-k") == 0) {

source/dnode/mgmt/main/inc/dnd.h

@@ -149,7 +149,7 @@ int32_t dndInitClient(SDnode *pDnode);
 void    dndCleanupClient(SDnode *pDnode);
 int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg);
 int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
-int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pMsg);
+int32_t dndSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pMsg);
 void    dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp);
 void    dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
 SMsgCb  dndCreateMsgcb(SMgmtWrapper *pWrapper);

source/dnode/mgmt/main/src/dndExec.c

@@ -45,7 +45,7 @@ int32_t dndOpenNode(SMgmtWrapper *pWrapper) {
 }
 
 void dndCloseNode(SMgmtWrapper *pWrapper) {
-  dDebug("node:%s, start to close", pWrapper->name);
+  dDebug("node:%s, mgmt start to close", pWrapper->name);
   pWrapper->required = false;
   taosWLockLatch(&pWrapper->latch);
   if (pWrapper->deployed) {
@@ -62,7 +62,7 @@ void dndCloseNode(SMgmtWrapper *pWrapper) {
     taosProcCleanup(pWrapper->pProc);
     pWrapper->pProc = NULL;
   }
-  dDebug("node:%s, has been closed", pWrapper->name);
+  dDebug("node:%s, mgmt has been closed", pWrapper->name);
 }
 
 static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen,
@@ -90,10 +90,10 @@ static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int16_t
 static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen,
                                   ProcFuncType ftype) {
   pMsg->pCont = pCont;
-  dTrace("msg:%p, get from parent queue, handle:%p app:%p", pMsg, pMsg->handle, pMsg->ahandle);
+  dTrace("msg:%p, get from parent queue, ftype:%d handle:%p, app:%p", pMsg, ftype, pMsg->handle, pMsg->ahandle);
 
   switch (ftype) {
-    case PROC_REG:
+    case PROC_REGIST:
       rpcRegisterBrokenLinkArg(pMsg);
       break;
     case PROC_RELEASE:
@@ -101,11 +101,14 @@ static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t
       rpcFreeCont(pCont);
       break;
     case PROC_REQ:
-      // todo send to dnode
       dndSendReqToMnode(pWrapper, pMsg);
-    default:
+      // dndSendReq(pWrapper, (const SEpSet *)((char *)pMsg + sizeof(SRpcMsg)), pMsg);
+      break;
+    case PROC_RSP:
       dndSendRpcRsp(pWrapper, pMsg);
       break;
+    default:
+      break;
   }
   taosMemoryFree(pMsg);
 }
@@ -180,6 +183,7 @@ static int32_t dndRunInSingleProcess(SDnode *pDnode) {
   while (1) {
     if (pDnode->event == DND_EVENT_STOP) {
       dInfo("dnode is about to stop");
+      dndSetStatus(pDnode, DND_STAT_STOPPED);
       break;
     }
     taosMsleep(100);
@@ -202,7 +206,7 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) {
     if (!pWrapper->required) continue;
 
     int32_t shmsize = 1024 * 1024 * 2;  // size will be a configuration item
-    if (taosCreateShm(&pWrapper->shm, shmsize) != 0) {
+    if (taosCreateShm(&pWrapper->shm, n, shmsize) != 0) {
       terrno = TAOS_SYSTEM_ERROR(terrno);
       dError("node:%s, failed to create shm size:%d since %s", pWrapper->name, shmsize, terrstr());
       return -1;
@@ -255,6 +259,8 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) {
   while (1) {
     if (pDnode->event == DND_EVENT_STOP) {
       dInfo("dnode is about to stop");
+      dndSetStatus(pDnode, DND_STAT_STOPPED);
+
       for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) {
         SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
         if (!pWrapper->required) continue;
@@ -263,15 +269,6 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) {
         if (pWrapper->procId > 0 && taosProcExist(pWrapper->procId)) {
           dInfo("node:%s, send kill signal to the child process:%d", pWrapper->name, pWrapper->procId);
           taosKillProc(pWrapper->procId);
-        }
-      }
-
-      for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) {
-        SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
-        if (!pWrapper->required) continue;
-        if (pDnode->ntype == NODE_MAX) continue;
-
-        if (pWrapper->procId > 0 && taosProcExist(pWrapper->procId)) {
           dInfo("node:%s, wait for child process:%d to stop", pWrapper->name, pWrapper->procId);
           taosWaitProc(pWrapper->procId);
           dInfo("node:%s, child process:%d is stopped", pWrapper->name, pWrapper->procId);
@@ -331,6 +328,8 @@ static int32_t dndRunInChildProcess(SDnode *pDnode) {
     }
   }
 
+  dndSetStatus(pDnode, DND_STAT_RUNNING);
+
   if (taosProcRun(pWrapper->pProc) != 0) {
     dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr());
     return -1;
@@ -340,11 +339,14 @@ static int32_t dndRunInChildProcess(SDnode *pDnode) {
   dndReportStartup(pDnode, "TDengine", "initialized successfully");
   while (1) {
     if (pDnode->event == DND_EVENT_STOP) {
-      dInfo("dnode is about to stop");
+      dInfo("%s is about to stop", pWrapper->name);
+      dndSetStatus(pDnode, DND_STAT_STOPPED);
       break;
     }
     taosMsleep(100);
   }
+
+  return 0;
 }
 
 int32_t dndRun(SDnode *pDnode) {

source/dnode/mgmt/main/src/dndInt.c

@@ -133,14 +133,6 @@ _OVER:
 void dndClose(SDnode *pDnode) {
   if (pDnode == NULL) return;
 
-  if (dndGetStatus(pDnode) == DND_STAT_STOPPED) {
-    dError("dnode is shutting down, data:%p", pDnode);
-    return;
-  }
-
-  dInfo("start to close dnode, data:%p", pDnode);
-  dndSetStatus(pDnode, DND_STAT_STOPPED);
-
   for (ENodeType n = 0; n < NODE_MAX; ++n) {
     SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
     dndCloseNode(pWrapper);
@@ -151,7 +143,6 @@ void dndClose(SDnode *pDnode) {
 }
 
 void dndHandleEvent(SDnode *pDnode, EDndEvent event) {
-  dInfo("dnode receive %s event, data:%p", dndEventStr(event), pDnode);
   if (event == DND_EVENT_STOP) {
     pDnode->event = event;
   }

source/dnode/mgmt/main/src/dndTransport.c

@@ -319,22 +319,6 @@ static int32_t dndSendRpcReq(STransMgmt *pMgmt, const SEpSet *pEpSet, SRpcMsg *p
   return 0;
 }
 
-int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pReq) {
-  if (pWrapper->procType != PROC_CHILD) {
-    SDnode *pDnode = pWrapper->pDnode;
-    if (dndGetStatus(pDnode) != DND_STAT_RUNNING) {
-      terrno = TSDB_CODE_DND_OFFLINE;
-      dError("failed to send rpc msg since %s, handle:%p", terrstr(), pReq->handle);
-      return -1;
-    }
-    return dndSendRpcReq(&pDnode->trans, pEpSet, pReq);
-  } else {
-    while (taosProcPutToParentQ(pWrapper->pProc, pReq, sizeof(SRpcMsg), pReq->pCont, pReq->contLen, PROC_REQ) != 0) {
-      taosMsleep(1);
-    }
-  }
-}
-
 int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pReq) {
   SDnode     *pDnode = pWrapper->pDnode;
   STransMgmt *pTrans = &pDnode->trans;
@@ -362,13 +346,37 @@ void dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) {
   }
 }
 
+int32_t dndSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pReq) {
+  SDnode *pDnode = pWrapper->pDnode;
+  if (dndGetStatus(pDnode) != DND_STAT_RUNNING) {
+    terrno = TSDB_CODE_DND_OFFLINE;
+    dError("failed to send rpc msg since %s, handle:%p", terrstr(), pReq->handle);
+    return -1;
+  }
+
+  if (pWrapper->procType != PROC_CHILD) {
+    return dndSendRpcReq(&pDnode->trans, pEpSet, pReq);
+  } else {
+    int32_t headLen = sizeof(SRpcMsg) + sizeof(SEpSet);
+    char   *pHead = taosMemoryMalloc(headLen);
+    if (pHead == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return -1;
+    }
+    memcpy(pHead, pReq, sizeof(SRpcMsg));
+    memcpy(pHead + sizeof(SRpcMsg), pEpSet, sizeof(SEpSet));
+
+    taosProcPutToParentQ(pWrapper->pProc, pReq, headLen, pReq->pCont, pReq->contLen, PROC_REQ);
+    taosMemoryFree(pHead);
+    return 0;
+  }
+}
+
 void dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) {
   if (pWrapper->procType != PROC_CHILD) {
     dndSendRpcRsp(pWrapper, pRsp);
   } else {
-    while (taosProcPutToParentQ(pWrapper->pProc, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen, PROC_RSP) != 0) {
+    taosProcPutToParentQ(pWrapper->pProc, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen, PROC_RSP);
-      taosMsleep(1);
-    }
   }
 }
 
@@ -376,9 +384,7 @@ void dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) {
   if (pWrapper->procType != PROC_CHILD) {
     rpcRegisterBrokenLinkArg(pMsg);
   } else {
-    while (taosProcPutToParentQ(pWrapper->pProc, pMsg, sizeof(SRpcMsg), pMsg->pCont, pMsg->contLen, PROC_REG) != 0) {
+    taosProcPutToParentQ(pWrapper->pProc, pMsg, sizeof(SRpcMsg), pMsg->pCont, pMsg->contLen, PROC_REGIST);
-      taosMsleep(1);
-    }
   }
 }
 
@@ -387,20 +393,17 @@ static void dndReleaseHandle(SMgmtWrapper *pWrapper, void *handle, int8_t type)
     rpcReleaseHandle(handle, type);
   } else {
     SRpcMsg msg = {.handle = handle, .code = type};
-    while (taosProcPutToParentQ(pWrapper->pProc, &msg, sizeof(SRpcMsg), NULL, 0, PROC_RELEASE) != 0) {
+    taosProcPutToParentQ(pWrapper->pProc, &msg, sizeof(SRpcMsg), NULL, 0, PROC_RELEASE);
-      taosMsleep(1);
-    }
   }
 }
 
 SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper) {
   SMsgCb msgCb = {
       .pWrapper = pWrapper,
+      .sendReqFp = dndSendReq,
+      .sendRspFp = dndSendRsp,
       .registerBrokenLinkArgFp = dndRegisterBrokenLinkArg,
       .releaseHandleFp = dndReleaseHandle,
-      .sendMnodeReqFp = dndSendReqToMnode,
-      .sendReqFp = dndSendReqToDnode,
-      .sendRspFp = dndSendRsp,
   };
   return msgCb;
 }

source/dnode/mgmt/mm/src/mmFile.c

@@ -111,7 +111,7 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, bool deployed) {
 
   TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pFile == NULL) {
-    terrno = TAOS_SYSTEM_ERROR(errno);;
+    terrno = TAOS_SYSTEM_ERROR(errno);
     dError("failed to write %s since %s", file, terrstr());
     return -1;
   }
@@ -145,7 +145,7 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, bool deployed) {
   snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP);
 
   if (taosRenameFile(file, realfile) != 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);;
+    terrno = TAOS_SYSTEM_ERROR(errno);
     dError("failed to rename %s since %s", file, terrstr());
     return -1;
   }

source/dnode/mgmt/mm/src/mmInt.c

@@ -39,20 +39,11 @@ static int32_t mmRequire(SMgmtWrapper *pWrapper, bool *required) {
 }
 
 static void mmInitOption(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) {
-  SDnode *pDnode = pMgmt->pDnode;
+  SMsgCb msgCb = dndCreateMsgcb(pMgmt->pWrapper);
-  pOption->dnodeId = pDnode->dnodeId;
-  pOption->clusterId = pDnode->clusterId;
-
-  SMsgCb msgCb = {0};
-  msgCb.pWrapper = pMgmt->pWrapper;
   msgCb.queueFps[QUERY_QUEUE] = mmPutMsgToQueryQueue;
   msgCb.queueFps[READ_QUEUE] = mmPutMsgToReadQueue;
   msgCb.queueFps[WRITE_QUEUE] = mmPutMsgToWriteQueue;
   msgCb.queueFps[SYNC_QUEUE] = mmPutMsgToWriteQueue;
-  msgCb.sendReqFp = dndSendReqToDnode;
-  msgCb.sendMnodeReqFp = dndSendReqToMnode;
-  msgCb.sendRspFp = dndSendRsp;
-  msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
   pOption->msgCb = msgCb;
 }
 
@@ -66,6 +57,7 @@ static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) {
   pReplica->id = 1;
   pReplica->port = pDnode->serverPort;
   tstrncpy(pReplica->fqdn, pDnode->localFqdn, TSDB_FQDN_LEN);
+  pOption->deploy = true;
 
   pMgmt->selfIndex = pOption->selfIndex;
   pMgmt->replica = pOption->replica;
@@ -77,6 +69,7 @@ static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) {
   pOption->selfIndex = pMgmt->selfIndex;
   pOption->replica = pMgmt->replica;
   memcpy(&pOption->replicas, pMgmt->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA);
+  pOption->deploy = false;
 }
 
 static int32_t mmBuildOptionFromReq(SMnodeMgmt *pMgmt, SMnodeOpt *pOption, SDCreateMnodeReq *pCreate) {
@@ -89,7 +82,7 @@ static int32_t mmBuildOptionFromReq(SMnodeMgmt *pMgmt, SMnodeOpt *pOption, SDCre
     pReplica->id = pCreate->replicas[i].id;
     pReplica->port = pCreate->replicas[i].port;
     memcpy(pReplica->fqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN);
-    if (pReplica->id == pOption->dnodeId) {
+    if (pReplica->id == pMgmt->pDnode->dnodeId) {
       pOption->selfIndex = i;
     }
   }
@@ -98,6 +91,7 @@ static int32_t mmBuildOptionFromReq(SMnodeMgmt *pMgmt, SMnodeOpt *pOption, SDCre
     dError("failed to build mnode options since %s", terrstr());
     return -1;
   }
+  pOption->deploy = true;
 
   pMgmt->selfIndex = pOption->selfIndex;
   pMgmt->replica = pOption->replica;
@@ -225,9 +219,7 @@ int32_t mmOpenFromMsg(SMgmtWrapper *pWrapper, SDCreateMnodeReq *pReq) {
   return code;
 }
 
-static int32_t mmOpen(SMgmtWrapper *pWrapper) {
+static int32_t mmOpen(SMgmtWrapper *pWrapper) { return mmOpenFromMsg(pWrapper, NULL); }
-  return mmOpenFromMsg(pWrapper, NULL);
-}
 
 static int32_t mmStart(SMgmtWrapper *pWrapper) {
   dDebug("mnode-mgmt start to run");
@@ -258,7 +250,7 @@ int32_t mmGetUserAuth(SMgmtWrapper *pWrapper, char *user, char *spi, char *encry
 }
 
 int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
-                         SMonGrantInfo *pGrantInfo) {
+                           SMonGrantInfo *pGrantInfo) {
   SMnodeMgmt *pMgmt = pWrapper->pMgmt;
   return mndGetMonitorInfo(pMgmt->pMnode, pClusterInfo, pVgroupInfo, pGrantInfo);
 }

source/dnode/mgmt/qm/src/qmInt.c

@@ -19,15 +19,10 @@
 static int32_t qmRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); }
 
 static void qmInitOption(SQnodeMgmt *pMgmt, SQnodeOpt *pOption) {
-  SMsgCb msgCb = {0};
+  SMsgCb msgCb = dndCreateMsgcb(pMgmt->pWrapper);
-  msgCb.pWrapper = pMgmt->pWrapper;
   msgCb.queueFps[QUERY_QUEUE] = qmPutMsgToQueryQueue;
   msgCb.queueFps[FETCH_QUEUE] = qmPutMsgToFetchQueue;
   msgCb.qsizeFp = qmGetQueueSize;
-  msgCb.sendReqFp = dndSendReqToDnode;
-  msgCb.sendMnodeReqFp = dndSendReqToMnode;
-  msgCb.sendRspFp = dndSendRsp;
-  msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
   pOption->msgCb = msgCb;
 }
 

source/dnode/mgmt/sm/src/smInt.c

@@ -19,12 +19,7 @@
 static int32_t smRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); }
 
 static void smInitOption(SSnodeMgmt *pMgmt, SSnodeOpt *pOption) {
-  SMsgCb msgCb = {0};
+  SMsgCb msgCb = dndCreateMsgcb(pMgmt->pWrapper);
-  msgCb.pWrapper = pMgmt->pWrapper;
-  msgCb.sendReqFp = dndSendReqToDnode;
-  msgCb.sendMnodeReqFp = dndSendReqToMnode;
-  msgCb.sendRspFp = dndSendRsp;
-  msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
   pOption->msgCb = msgCb;
 }
 

source/dnode/mgmt/test/vnode/vnode.cpp

@@ -70,46 +70,6 @@ TEST_F(DndTestVnode, 01_Create_Vnode) {
       ASSERT_EQ(pRsp->code, TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED);
     }
   }
-
-  {
-    SCreateVnodeReq createReq = {0};
-    createReq.vgId = 2;
-    createReq.dnodeId = 3;
-    strcpy(createReq.db, "1.d1");
-    createReq.dbUid = 9527;
-    createReq.vgVersion = 1;
-    createReq.cacheBlockSize = 16;
-    createReq.totalBlocks = 10;
-    createReq.daysPerFile = 10;
-    createReq.daysToKeep0 = 3650;
-    createReq.daysToKeep1 = 3650;
-    createReq.daysToKeep2 = 3650;
-    createReq.minRows = 100;
-    createReq.minRows = 4096;
-    createReq.commitTime = 3600;
-    createReq.fsyncPeriod = 3000;
-    createReq.walLevel = 1;
-    createReq.precision = 0;
-    createReq.compression = 2;
-    createReq.replica = 1;
-    createReq.quorum = 1;
-    createReq.update = 0;
-    createReq.cacheLastRow = 0;
-    createReq.selfIndex = 0;
-    for (int r = 0; r < createReq.replica; ++r) {
-      SReplica* pReplica = &createReq.replicas[r];
-      pReplica->id = 1;
-      pReplica->port = 9527;
-    }
-
-    int32_t contLen = tSerializeSCreateVnodeReq(NULL, 0, &createReq);
-    void*   pReq = rpcMallocCont(contLen);
-    tSerializeSCreateVnodeReq(pReq, contLen, &createReq);
-
-    SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_VNODE, pReq, contLen);
-    ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_VNODE_INVALID_OPTION);
-  }
 }
 
 TEST_F(DndTestVnode, 02_Alter_Vnode) {
@@ -164,37 +124,37 @@ TEST_F(DndTestVnode, 03_Create_Stb) {
     req.keep = 0;
     req.type = TD_SUPER_TABLE;
 
-    SSchema schemas[5] = {0};
+    SSchemaEx schemas[2] = {0};
     {
-      SSchema* pSchema = &schemas[0];
+      SSchemaEx* pSchema = &schemas[0];
       pSchema->bytes = htonl(8);
       pSchema->type = TSDB_DATA_TYPE_TIMESTAMP;
       strcpy(pSchema->name, "ts");
     }
 
     {
-      SSchema* pSchema = &schemas[1];
+      SSchemaEx* pSchema = &schemas[1];
       pSchema->bytes = htonl(4);
       pSchema->type = TSDB_DATA_TYPE_INT;
       strcpy(pSchema->name, "col1");
     }
-
+    SSchema tagSchemas[3] = {0};
     {
-      SSchema* pSchema = &schemas[2];
+      SSchema* pSchema = &tagSchemas[0];
       pSchema->bytes = htonl(2);
       pSchema->type = TSDB_DATA_TYPE_TINYINT;
       strcpy(pSchema->name, "tag1");
     }
 
     {
-      SSchema* pSchema = &schemas[3];
+      SSchema* pSchema = &tagSchemas[1];
       pSchema->bytes = htonl(8);
       pSchema->type = TSDB_DATA_TYPE_BIGINT;
       strcpy(pSchema->name, "tag2");
     }
 
     {
-      SSchema* pSchema = &schemas[4];
+      SSchema* pSchema = &tagSchemas[2];
       pSchema->bytes = htonl(16);
       pSchema->type = TSDB_DATA_TYPE_BINARY;
       strcpy(pSchema->name, "tag3");
@@ -204,7 +164,7 @@ TEST_F(DndTestVnode, 03_Create_Stb) {
     req.stbCfg.nCols = 2;
     req.stbCfg.pSchema = &schemas[0];
     req.stbCfg.nTagCols = 3;
-    req.stbCfg.pTagSchema = &schemas[2];
+    req.stbCfg.pTagSchema = &tagSchemas[0];
 
     int32_t   contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead);
     SMsgHead* pHead = (SMsgHead*)rpcMallocCont(contLen);
@@ -236,37 +196,37 @@ TEST_F(DndTestVnode, 04_Alter_Stb) {
     req.keep = 0;
     req.type = TD_SUPER_TABLE;
 
-    SSchema schemas[5] = {0};
+    SSchemaEx schemas[2] = {0};
     {
-      SSchema* pSchema = &schemas[0];
+      SSchemaEx* pSchema = &schemas[0];
       pSchema->bytes = htonl(8);
       pSchema->type = TSDB_DATA_TYPE_TIMESTAMP;
       strcpy(pSchema->name, "ts");
     }
 
     {
-      SSchema* pSchema = &schemas[1];
+      SSchemaEx* pSchema = &schemas[1];
       pSchema->bytes = htonl(4);
       pSchema->type = TSDB_DATA_TYPE_INT;
       strcpy(pSchema->name, "col1");
     }
-
+    SSchema tagSchemas[3] = {0};
     {
-      SSchema* pSchema = &schemas[2];
+      SSchema* pSchema = &tagSchemas[0];
       pSchema->bytes = htonl(2);
       pSchema->type = TSDB_DATA_TYPE_TINYINT;
       strcpy(pSchema->name, "_tag1");
     }
 
     {
-      SSchema* pSchema = &schemas[3];
+      SSchema* pSchema = &tagSchemas[1];
       pSchema->bytes = htonl(8);
       pSchema->type = TSDB_DATA_TYPE_BIGINT;
       strcpy(pSchema->name, "_tag2");
     }
 
     {
-      SSchema* pSchema = &schemas[4];
+      SSchema* pSchema = &tagSchemas[2];
       pSchema->bytes = htonl(16);
       pSchema->type = TSDB_DATA_TYPE_BINARY;
       strcpy(pSchema->name, "_tag3");
@@ -276,7 +236,7 @@ TEST_F(DndTestVnode, 04_Alter_Stb) {
     req.stbCfg.nCols = 2;
     req.stbCfg.pSchema = &schemas[0];
     req.stbCfg.nTagCols = 3;
-    req.stbCfg.pTagSchema = &schemas[2];
+    req.stbCfg.pTagSchema = &tagSchemas[0];
 
     int32_t   contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead);
     SMsgHead* pHead = (SMsgHead*)rpcMallocCont(contLen);

source/dnode/mgmt/vm/src/vmInt.c

@@ -128,16 +128,12 @@ static void *vmOpenVnodeFunc(void *param) {
              pMgmt->state.openVnodes, pMgmt->state.totalVnodes);
     dndReportStartup(pDnode, "open-vnodes", stepDesc);
 
-    SMsgCb msgCb = {0};
+    SMsgCb msgCb = dndCreateMsgcb(pMgmt->pWrapper);
     msgCb.pWrapper = pMgmt->pWrapper;
     msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue;
     msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue;
     msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue;
     msgCb.qsizeFp = vmGetQueueSize;
-    msgCb.sendReqFp = dndSendReqToDnode;
-    msgCb.sendMnodeReqFp = dndSendReqToMnode;
-    msgCb.sendRspFp = dndSendRsp;
-    msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
     SVnodeCfg cfg = {.msgCb = msgCb, .pTfs = pMgmt->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid};
     SVnode   *pImpl = vnodeOpen(pCfg->path, &cfg);
     if (pImpl == NULL) {

source/dnode/mgmt/vm/src/vmMsg.c

@@ -68,12 +68,6 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
   SWrapperCfg wrapperCfg = {0};
   vmGenerateWrapperCfg(pMgmt, &createReq, &wrapperCfg);
 
-  if (createReq.dnodeId != pMgmt->pDnode->dnodeId) {
-    terrno = TSDB_CODE_DND_VNODE_INVALID_OPTION;
-    dDebug("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr());
-    return -1;
-  }
-
   SVnodeObj *pVnode = vmAcquireVnode(pMgmt, createReq.vgId);
   if (pVnode != NULL) {
     dDebug("vgId:%d, already exist", createReq.vgId);
@@ -82,16 +76,12 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
     return -1;
   }
 
-  SMsgCb msgCb = {0};
+  SMsgCb msgCb = dndCreateMsgcb(pMgmt->pWrapper);
   msgCb.pWrapper = pMgmt->pWrapper;
   msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue;
   msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue;
   msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue;
   msgCb.qsizeFp = vmGetQueueSize;
-  msgCb.sendReqFp = dndSendReqToDnode;
-  msgCb.sendMnodeReqFp = dndSendReqToMnode;
-  msgCb.sendRspFp = dndSendRsp;
-  msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
 
   vnodeCfg.msgCb = msgCb;
   vnodeCfg.pTfs = pMgmt->pTfs;

source/dnode/mnode/impl/inc/mndCluster.h

@@ -25,6 +25,7 @@ extern "C" {
 int32_t mndInitCluster(SMnode *pMnode);
 void    mndCleanupCluster(SMnode *pMnode);
 int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len);
+int64_t mndGetClusterId(SMnode *pMnode);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/inc/mndDef.h

@@ -620,13 +620,13 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub)
 
 static FORCE_INLINE void tDeleteSMqSubscribeObj(SMqSubscribeObj* pSub) {
   if (pSub->consumers) {
-    taosArrayDestroyEx(pSub->consumers, (void (*)(void*))tDeleteSMqSubConsumer);
+    //taosArrayDestroyEx(pSub->consumers, (void (*)(void*))tDeleteSMqSubConsumer);
     // taosArrayDestroy(pSub->consumers);
     pSub->consumers = NULL;
   }
 
   if (pSub->unassignedVg) {
-    taosArrayDestroyEx(pSub->unassignedVg, (void (*)(void*))tDeleteSMqConsumerEp);
+    //taosArrayDestroyEx(pSub->unassignedVg, (void (*)(void*))tDeleteSMqConsumerEp);
     // taosArrayDestroy(pSub->unassignedVg);
     pSub->unassignedVg = NULL;
   }

source/dnode/mnode/impl/inc/mndInt.h

@@ -100,7 +100,6 @@ typedef struct {
 } SGrantInfo;
 
 typedef struct SMnode {
-  int32_t           dnodeId;
   int64_t           clusterId;
   int8_t            replica;
   int8_t            selfIndex;

source/dnode/mnode/impl/src/mndCluster.c

@@ -17,7 +17,7 @@
 #include "mndCluster.h"
 #include "mndShow.h"
 
-#define TSDB_CLUSTER_VER_NUMBE 1
+#define TSDB_CLUSTER_VER_NUMBE    1
 #define TSDB_CLUSTER_RESERVE_SIZE 64
 
 static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster);
@@ -61,6 +61,23 @@ int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) {
   return 0;
 }
 
+int64_t mndGetClusterId(SMnode *pMnode) {
+  SSdb   *pSdb = pMnode->pSdb;
+  void   *pIter = NULL;
+  int64_t clusterId = -1;
+
+  while (1) {
+    SClusterObj *pCluster = NULL;
+    pIter = sdbFetch(pSdb, SDB_CLUSTER, pIter, (void **)&pCluster);
+    if (pIter == NULL) break;
+
+    clusterId = pCluster->id;
+    sdbRelease(pSdb, pCluster);
+  }
+
+  return clusterId;
+}
+
 static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) {
   terrno = TSDB_CODE_OUT_OF_MEMORY;
 

source/dnode/mnode/impl/src/mndConsumer.c

@@ -160,6 +160,7 @@ static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) {
 
 static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) {
   mTrace("consumer:%" PRId64 ", perform update action", pOldConsumer->consumerId);
+  pOldConsumer->epoch++;
 
   // TODO handle update
   /*taosWLockLatch(&pOldConsumer->lock);*/

source/dnode/mnode/impl/src/mndProfile.c

@@ -433,12 +433,6 @@ static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq) {
   pHeartbeat->connId = htonl(pHeartbeat->connId);
   pHeartbeat->pid = htonl(pHeartbeat->pid);
 
-  SRpcConnInfo info = {0};
-  if (rpcGetConnInfo(pReq->rpcMsg.handle, &info) != 0) {
-    mError("user:%s, connId:%d failed to process hb since %s", pReq->user, pHeartbeat->connId, terrstr());
-    return -1;
-  }
-
   SConnObj *pConn = mndAcquireConn(pMnode, pHeartbeat->connId);
   if (pConn == NULL) {
     pConn = mndCreateConn(pMnode, &info, pHeartbeat->pid, pHeartbeat->app, 0);

source/dnode/mnode/impl/src/mndQnode.c

@@ -423,7 +423,7 @@ static int32_t mndProcessDropQnodeReq(SNodeMsg *pReq) {
 
 DROP_QNODE_OVER:
   if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
-    mError("qnode:%d, failed to drop since %s", pMnode->dnodeId, terrstr());
+    mError("qnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr());
   }
 
   mndReleaseQnode(pMnode, pObj);

source/dnode/mnode/impl/src/mndSnode.c

@@ -433,7 +433,7 @@ static int32_t mndProcessDropSnodeReq(SNodeMsg *pReq) {
 
 DROP_SNODE_OVER:
   if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
-    mError("snode:%d, failed to drop since %s", pMnode->dnodeId, terrstr());
+    mError("snode:%d, failed to drop since %s", dropReq.dnodeId, terrstr());
   }
 
   mndReleaseSnode(pMnode, pObj);

source/dnode/mnode/impl/src/mndStb.c

@@ -333,6 +333,15 @@ static SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) {
   return mndAcquireDb(pMnode, db);
 }
 
+static FORCE_INLINE int schemaExColIdCompare(const void *colId, const void *pSchema) {
+  if (*(col_id_t *)colId < ((SSchemaEx *)pSchema)->colId) {
+    return -1;
+  } else if (*(col_id_t *)colId > ((SSchemaEx *)pSchema)->colId) {
+    return 1;
+  }
+  return 0;
+}
+
 static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen) {
   SName name = {0};
   tNameFromString(&name, pStb->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
@@ -345,16 +354,91 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
   req.name = (char *)tNameGetTableName(&name);
   req.ttl = 0;
   req.keep = 0;
+  req.rollup = pStb->aggregationMethod > -1 ? 1 : 0;
   req.type = TD_SUPER_TABLE;
   req.stbCfg.suid = pStb->uid;
   req.stbCfg.nCols = pStb->numOfColumns;
-  req.stbCfg.pSchema = pStb->pColumns;
   req.stbCfg.nTagCols = pStb->numOfTags;
   req.stbCfg.pTagSchema = pStb->pTags;
+  req.stbCfg.nBSmaCols = pStb->numOfSmas;
+  req.stbCfg.pSchema = (SSchemaEx *)taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchemaEx));
+  if (req.stbCfg.pSchema == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
 
-  int32_t   contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead);
+  int bSmaStat = 0;                             // no column has bsma
+  if (pStb->numOfSmas == pStb->numOfColumns) {  // assume pColumns > 0
+    bSmaStat = 1;                               // all columns have bsma
+  } else if (pStb->numOfSmas != 0) {
+    bSmaStat = 2;                  // partial columns have bsma
+    TASSERT(pStb->pSmas != NULL);  // TODO: remove the assert
+  }
+
+  for (int32_t i = 0; i < req.stbCfg.nCols; ++i) {
+    SSchemaEx *pSchemaEx = req.stbCfg.pSchema + i;
+    SSchema   *pSchema = pStb->pColumns + i;
+    pSchemaEx->type = pSchema->type;
+    pSchemaEx->sma = (bSmaStat == 1) ? TSDB_BSMA_TYPE_LATEST : TSDB_BSMA_TYPE_NONE;
+    pSchemaEx->colId = pSchema->colId;
+    pSchemaEx->bytes = pSchema->bytes;
+    memcpy(pSchemaEx->name, pSchema->name, TSDB_COL_NAME_LEN);
+  }
+
+  if (bSmaStat == 2) {
+    if (pStb->pSmas == NULL) {
+      mError("stb:%s, sma options is empty", pStb->name);
+      taosMemoryFreeClear(req.stbCfg.pSchema);
+      terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
+      return NULL;
+    }
+    for (int32_t i = 0; i < pStb->numOfSmas; ++i) {
+      SSchema   *pSmaSchema = pStb->pSmas + i;
+      SSchemaEx *pColSchema = taosbsearch(&pSmaSchema->colId, req.stbCfg.pSchema, req.stbCfg.nCols, sizeof(SSchemaEx),
+                                          schemaExColIdCompare, TD_EQ);
+      if (pColSchema == NULL) {
+        terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
+        taosMemoryFreeClear(req.stbCfg.pSchema);
+        mError("stb:%s, sma col:%s not found in columns", pStb->name, pSmaSchema->name);
+        return NULL;
+      }
+      pColSchema->sma = TSDB_BSMA_TYPE_LATEST;
+    }
+  }
+
+  SRSmaParam *pRSmaParam = NULL;
+  if (req.rollup) {
+    pRSmaParam = (SRSmaParam *)taosMemoryCalloc(1, sizeof(SRSmaParam));
+    if (pRSmaParam == NULL) {
+      taosMemoryFreeClear(req.stbCfg.pSchema);
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return NULL;
+    }
+
+    pRSmaParam->xFilesFactor = pStb->xFilesFactor;
+    pRSmaParam->delay = pStb->delay;
+    pRSmaParam->nFuncIds = 1;  // only 1 aggregation method supported currently
+    pRSmaParam->pFuncIds = (func_id_t *)taosMemoryCalloc(pRSmaParam->nFuncIds, sizeof(func_id_t));
+    if (pRSmaParam->pFuncIds == NULL) {
+      taosMemoryFreeClear(req.stbCfg.pRSmaParam);
+      taosMemoryFreeClear(req.stbCfg.pSchema);
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return NULL;
+    }
+    for (int32_t f = 0; f < pRSmaParam->nFuncIds; ++f) {
+      *(pRSmaParam->pFuncIds + f) = pStb->aggregationMethod;
+    }
+    req.stbCfg.pRSmaParam = pRSmaParam;
+  }  
+
+  int32_t contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead);
   SMsgHead *pHead = taosMemoryMalloc(contLen);
   if (pHead == NULL) {
+    if (pRSmaParam) {
+      taosMemoryFreeClear(pRSmaParam->pFuncIds);
+      taosMemoryFreeClear(pRSmaParam);
+    }
+    taosMemoryFreeClear(req.stbCfg.pSchema);
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
@@ -366,6 +450,11 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
   tSerializeSVCreateTbReq(&pBuf, &req);
 
   *pContLen = contLen;
+  if (pRSmaParam) {
+    taosMemoryFreeClear(pRSmaParam->pFuncIds);
+    taosMemoryFreeClear(pRSmaParam);
+  }
+  taosMemoryFreeClear(req.stbCfg.pSchema);
   return pHead;
 }
 
@@ -498,7 +587,6 @@ static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj
     if (pReq == NULL) {
       sdbCancelFetch(pSdb, pIter);
       sdbRelease(pSdb, pVgroup);
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
     }
 
@@ -559,9 +647,9 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj
 }
 
 static SSchema *mndFindStbColumns(const SStbObj *pStb, const char *colName) {
-  for (int32_t col = 0; col < pStb->numOfColumns; col++) {
+  for (int32_t col = 0; col < pStb->numOfColumns; ++col) {
     SSchema *pSchema = &pStb->pColumns[col];
-    if (strcasecmp(pStb->pColumns[col].name, colName) == 0) {
+    if (strncasecmp(pSchema->name, colName, TSDB_COL_NAME_LEN) == 0) {
       return pSchema;
     }
   }
@@ -578,6 +666,9 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre
   stbObj.dbUid = pDb->uid;
   stbObj.version = 1;
   stbObj.nextColId = 1;
+  stbObj.xFilesFactor = pCreate->xFilesFactor;
+  stbObj.aggregationMethod = pCreate->aggregationMethod;
+  stbObj.delay = pCreate->delay;
   stbObj.ttl = pCreate->ttl;
   stbObj.numOfColumns = pCreate->numOfColumns;
   stbObj.numOfTags = pCreate->numOfTags;
@@ -625,7 +716,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre
     SSchema *pSchema = &stbObj.pSmas[i];
     SSchema *pColSchema = mndFindStbColumns(&stbObj, pField->name);
     if (pColSchema == NULL) {
-      mError("stb:%s, sma:%s not found in columns", stbObj.name, pSchema->name);
+      mError("stb:%s, sma:%s not found in columns", stbObj.name, pField->name);
       terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
       return -1;
     }
@@ -1061,7 +1152,6 @@ static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj
     if (pReq == NULL) {
       sdbCancelFetch(pSdb, pIter);
       sdbRelease(pSdb, pVgroup);
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
     }
 

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -237,7 +237,8 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) {
       for (int32_t j = 0; j < csz; j++) {
         SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, j);
         if (consumerId == pSubConsumer->consumerId) {
-          int32_t       vgsz = taosArrayGetSize(pSubConsumer->vgInfo);
+          int32_t vgsz = taosArrayGetSize(pSubConsumer->vgInfo);
+          mInfo("topic %s has %d vg", topicName, pConsumer->epoch);
           SMqSubTopicEp topicEp;
           strcpy(topicEp.topic, topicName);
           topicEp.vgs = taosArrayInit(vgsz, sizeof(SMqSubVgEp));
@@ -419,7 +420,6 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
       int32_t vgNum = pSub->vgNum;
       int32_t vgEachConsumer = vgNum / consumerNum;
       int32_t imbalanceVg = vgNum % consumerNum;
-      int32_t imbalanceSolved = 0;
 
       // iterate all consumers, set unassignedVgStash
       for (int32_t i = 0; i < consumerNum; i++) {
@@ -446,9 +446,9 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
         if (vgThisConsumerAfterRb != vgThisConsumerBeforeRb ||
             (vgThisConsumerAfterRb != 0 && status != MQ_CONSUMER_STATUS__ACTIVE) ||
             (vgThisConsumerAfterRb == 0 && status != MQ_CONSUMER_STATUS__LOST)) {
-          if (vgThisConsumerAfterRb != vgThisConsumerBeforeRb) {
+          /*if (vgThisConsumerAfterRb != vgThisConsumerBeforeRb) {*/
-            pRebConsumer->epoch++;
+            /*pRebConsumer->epoch++;*/
-          }
+          /*}*/
           if (vgThisConsumerAfterRb != 0) {
             atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__ACTIVE);
           } else {
@@ -460,7 +460,7 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
 
           SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pRebConsumer);
           sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);
-          mndTransAppendRedolog(pTrans, pConsumerRaw);
+          mndTransAppendCommitlog(pTrans, pConsumerRaw);
         }
         mndReleaseConsumer(pMnode, pRebConsumer);
       }
@@ -469,7 +469,6 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
       if (taosArrayGetSize(pSub->unassignedVg) != 0) {
         for (int32_t i = 0; i < consumerNum; i++) {
           SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, i);
-          int32_t         vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo);
           int32_t         vgThisConsumerAfterRb;
           if (i < imbalanceVg)
             vgThisConsumerAfterRb = vgEachConsumer + 1;

source/dnode/mnode/impl/src/mnode.c

@@ -187,7 +187,7 @@ static int32_t mndAllocStep(SMnode *pMnode, char *name, MndInitFp initFp, MndCle
   return 0;
 }
 
-static int32_t mndInitSteps(SMnode *pMnode) {
+static int32_t mndInitSteps(SMnode *pMnode, bool deploy) {
   if (mndAllocStep(pMnode, "mnode-sdb", mndInitSdb, mndCleanupSdb) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-trans", mndInitTrans, mndCleanupTrans) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-cluster", mndInitCluster, mndCleanupCluster) != 0) return -1;
@@ -210,7 +210,7 @@ static int32_t mndInitSteps(SMnode *pMnode) {
   if (mndAllocStep(pMnode, "mnode-infos", mndInitInfos, mndCleanupInfos) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1;
-  if (pMnode->clusterId <= 0) {
+  if (deploy) {
     if (mndAllocStep(pMnode, "mnode-sdb-deploy", mndDeploySdb, NULL) != 0) return -1;
   } else {
     if (mndAllocStep(pMnode, "mnode-sdb-read", mndReadSdb, NULL) != 0) return -1;
@@ -263,23 +263,15 @@ static int32_t mndExecSteps(SMnode *pMnode) {
     }
   }
 
+  pMnode->clusterId = mndGetClusterId(pMnode);
   return 0;
 }
 
-static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) {
+static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) {
-  pMnode->dnodeId = pOption->dnodeId;
-  pMnode->clusterId = pOption->clusterId;
   pMnode->replica = pOption->replica;
   pMnode->selfIndex = pOption->selfIndex;
   memcpy(&pMnode->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA);
   pMnode->msgCb = pOption->msgCb;
-
-  if (pMnode->dnodeId < 0 || pMnode->clusterId < 0) {
-    terrno = TSDB_CODE_MND_INVALID_OPTIONS;
-    return -1;
-  }
-
-  return 0;
 }
 
 SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
@@ -294,6 +286,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
 
   char timestr[24] = "1970-01-01 00:00:00.00";
   (void)taosParseTime(timestr, &pMnode->checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
+  mndSetOptions(pMnode, pOption);
 
   pMnode->pSteps = taosArrayInit(24, sizeof(SMnodeStep));
   if (pMnode->pSteps == NULL) {
@@ -312,16 +305,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
     return NULL;
   }
 
-  code = mndSetOptions(pMnode, pOption);
+  code = mndInitSteps(pMnode, pOption->deploy);
-  if (code != 0) {
-    code = terrno;
-    mError("failed to open mnode since %s", terrstr());
-    mndClose(pMnode);
-    terrno = code;
-    return NULL;
-  }
-
-  code = mndInitSteps(pMnode);
   if (code != 0) {
     code = terrno;
     mError("failed to open mnode since %s", terrstr());

source/dnode/mnode/sdb/src/sdbHash.c

@@ -40,6 +40,10 @@ const char *sdbTableName(ESdbType type) {
       return "auth";
     case SDB_ACCT:
       return "acct";
+    case SDB_STREAM:
+      return "stream";
+    case SDB_OFFSET:
+      return "offset";
     case SDB_SUBSCRIBE:
       return "subscribe";
     case SDB_CONSUMER:

source/dnode/vnode/src/meta/metaBDBImpl.c

@@ -70,9 +70,12 @@ static void    *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg);
 static void     metaClearTbCfg(STbCfg *pTbCfg);
 static int      metaEncodeSchema(void **buf, SSchemaWrapper *pSW);
 static void    *metaDecodeSchema(void *buf, SSchemaWrapper *pSW);
+static int      metaEncodeSchemaEx(void **buf, SSchemaWrapper *pSW);
+static void    *metaDecodeSchemaEx(void *buf, SSchemaWrapper *pSW, bool isGetEx);
 static void     metaDBWLock(SMetaDB *pDB);
 static void     metaDBRLock(SMetaDB *pDB);
 static void     metaDBULock(SMetaDB *pDB);
+static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline, bool isGetEx);
 
 #define BDB_PERR(info, code) fprintf(stderr, info " reason: %s", db_strerror(code))
 
@@ -155,13 +158,13 @@ void metaCloseDB(SMeta *pMeta) {
 }
 
 int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
-  tb_uid_t uid;
+  tb_uid_t   uid;
-  char     buf[512];
+  char       buf[512];
-  char     buf1[512];
+  char       buf1[512];
-  void    *pBuf;
+  void      *pBuf;
-  DBT      key1, value1;
+  DBT        key1, value1;
-  DBT      key2, value2;
+  DBT        key2, value2;
-  SSchema *pSchema = NULL;
+  SSchemaEx *pSchema = NULL;
 
   if (pTbCfg->type == META_SUPER_TABLE) {
     uid = pTbCfg->stbCfg.suid;
@@ -204,8 +207,8 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
     key2.data = &schemaKey;
     key2.size = sizeof(schemaKey);
 
-    SSchemaWrapper sw = {.nCols = ncols, .pSchema = pSchema};
+    SSchemaWrapper sw = {.nCols = ncols, .pSchemaEx = pSchema};
-    metaEncodeSchema(&pBuf, &sw);
+    metaEncodeSchemaEx(&pBuf, &sw);
 
     value2.data = buf1;
     value2.size = POINTER_DISTANCE(pBuf, buf1);
@@ -298,6 +301,8 @@ static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW) {
 
   buf = taosDecodeFixedU32(buf, &pSW->nCols);
   pSW->pSchema = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * pSW->nCols);
+
+  int8_t dummy;
   for (int i = 0; i < pSW->nCols; i++) {
     pSchema = pSW->pSchema + i;
     buf = taosDecodeFixedI8(buf, &pSchema->type);
@@ -309,6 +314,50 @@ static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW) {
   return buf;
 }
 
+static int metaEncodeSchemaEx(void **buf, SSchemaWrapper *pSW) {
+  int        tlen = 0;
+  SSchemaEx *pSchema;
+
+  tlen += taosEncodeFixedU32(buf, pSW->nCols);
+  for (int i = 0; i < pSW->nCols; i++) {
+    pSchema = pSW->pSchemaEx + i;
+    tlen += taosEncodeFixedI8(buf, pSchema->type);
+    tlen += taosEncodeFixedI8(buf, pSchema->sma);
+    tlen += taosEncodeFixedI16(buf, pSchema->colId);
+    tlen += taosEncodeFixedI32(buf, pSchema->bytes);
+    tlen += taosEncodeString(buf, pSchema->name);
+  }
+
+  return tlen;
+}
+
+static void *metaDecodeSchemaEx(void *buf, SSchemaWrapper *pSW, bool isGetEx) {
+  buf = taosDecodeFixedU32(buf, &pSW->nCols);
+  if (isGetEx) {
+    pSW->pSchemaEx = (SSchemaEx *)taosMemoryMalloc(sizeof(SSchemaEx) * pSW->nCols);
+    for (int i = 0; i < pSW->nCols; i++) {
+      SSchemaEx *pSchema = pSW->pSchemaEx + i;
+      buf = taosDecodeFixedI8(buf, &pSchema->type);
+      buf = taosDecodeFixedI8(buf, &pSchema->sma);
+      buf = taosDecodeFixedI16(buf, &pSchema->colId);
+      buf = taosDecodeFixedI32(buf, &pSchema->bytes);
+      buf = taosDecodeStringTo(buf, pSchema->name);
+    }
+  } else {
+    pSW->pSchema = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * pSW->nCols);
+    for (int i = 0; i < pSW->nCols; i++) {
+      SSchema *pSchema = pSW->pSchema + i;
+      buf = taosDecodeFixedI8(buf, &pSchema->type);
+      buf = taosSkipFixedLen(buf, sizeof(int8_t));
+      buf = taosDecodeFixedI16(buf, &pSchema->colId);
+      buf = taosDecodeFixedI32(buf, &pSchema->bytes);
+      buf = taosDecodeStringTo(buf, pSchema->name);
+    }
+  }
+
+  return buf;
+}
+
 static SMetaDB *metaNewDB() {
   SMetaDB *pDB = NULL;
   pDB = (SMetaDB *)taosMemoryCalloc(1, sizeof(*pDB));
@@ -652,12 +701,16 @@ STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) {
 }
 
 SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) {
+  return metaGetTableSchemaImpl(pMeta, uid, sver, isinline, false);
+}
+
+static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline, bool isGetEx) {
   uint32_t        nCols;
   SSchemaWrapper *pSW = NULL;
   SMetaDB        *pDB = pMeta->pDB;
   int             ret;
   void           *pBuf;
-  SSchema        *pSchema;
+  // SSchema        *pSchema;
   SSchemaKey      schemaKey = {uid, sver, 0};
   DBT             key = {0};
   DBT             value = {0};
@@ -678,7 +731,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo
   // Decode the schema
   pBuf = value.data;
   pSW = taosMemoryMalloc(sizeof(*pSW));
-  metaDecodeSchema(pBuf, pSW);
+  metaDecodeSchemaEx(pBuf, pSW, isGetEx);
 
   return pSW;
 }
@@ -755,7 +808,7 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) {
 STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
   STSchemaBuilder sb;
   STSchema       *pTSchema = NULL;
-  SSchema        *pSchema;
+  SSchemaEx      *pSchema;
   SSchemaWrapper *pSW;
   STbCfg         *pTbCfg;
   tb_uid_t        quid;
@@ -767,16 +820,16 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
     quid = uid;
   }
 
-  pSW = metaGetTableSchema(pMeta, quid, sver, true);
+  pSW = metaGetTableSchemaImpl(pMeta, quid, sver, true, true);
   if (pSW == NULL) {
     return NULL;
   }
 
   // Rebuild a schema
   tdInitTSchemaBuilder(&sb, 0);
-  for (int32_t i = 0; i < pSW->nCols; i++) {
+  for (int32_t i = 0; i < pSW->nCols; ++i) {
-    pSchema = pSW->pSchema + i;
+    pSchema = pSW->pSchemaEx + i;
-    tdAddColToSchema(&sb, pSchema->type, pSchema->colId, pSchema->bytes);
+    tdAddColToSchema(&sb, pSchema->type, pSchema->sma, pSchema->colId, pSchema->bytes);
   }
   pTSchema = tdGetSchemaFromBuilder(&sb);
   tdDestroyTSchemaBuilder(&sb);

source/dnode/vnode/src/meta/metaTDBImpl.c

@@ -46,6 +46,10 @@ static int   metaEncodeTbInfo(void **buf, STbCfg *pTbCfg);
 static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg);
 static int   metaEncodeSchema(void **buf, SSchemaWrapper *pSW);
 static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW);
+static int   metaEncodeSchemaEx(void **buf, SSchemaWrapper *pSW);
+static void *metaDecodeSchemaEx(void *buf, SSchemaWrapper *pSW, bool isGetEx);
+
+static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline, bool isGetEx);
 
 static inline int metaUidCmpr(const void *arg1, int len1, const void *arg2, int len2) {
   tb_uid_t uid1, uid2;
@@ -228,7 +232,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
       schemaWrapper.pSchema = pTbCfg->ntbCfg.pSchema;
     }
     pVal = pBuf = buf;
-    metaEncodeSchema(&pBuf, &schemaWrapper);
+    metaEncodeSchemaEx(&pBuf, &schemaWrapper);
     vLen = POINTER_DISTANCE(pBuf, buf);
     ret = tdbDbInsert(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen);
     if (ret < 0) {
@@ -345,6 +349,10 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) {
 }
 
 SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) {
+  return *metaGetTableSchemaImpl(pMeta, uid, sver, isinline, false);
+}
+
+static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline, bool isGetEx) {
   void           *pKey;
   void           *pVal;
   int             kLen;
@@ -368,7 +376,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo
   // decode
   pBuf = pVal;
   pSchemaWrapper = taosMemoryMalloc(sizeof(*pSchemaWrapper));
-  metaDecodeSchema(pBuf, pSchemaWrapper);
+  metaDecodeSchemaEx(pBuf, pSchemaWrapper, isGetEx);
 
   TDB_FREE(pVal);
 
@@ -379,7 +387,7 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
   tb_uid_t        quid;
   SSchemaWrapper *pSW;
   STSchemaBuilder sb;
-  SSchema        *pSchema;
+  SSchemaEx      *pSchema;
   STSchema       *pTSchema;
   STbCfg         *pTbCfg;
 
@@ -390,15 +398,15 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
     quid = uid;
   }
 
-  pSW = metaGetTableSchema(pMeta, quid, sver, true);
+  pSW = metaGetTableSchemaImpl(pMeta, quid, sver, true, true);
   if (pSW == NULL) {
     return NULL;
   }
 
   tdInitTSchemaBuilder(&sb, 0);
   for (int i = 0; i < pSW->nCols; i++) {
-    pSchema = pSW->pSchema + i;
+    pSchema = pSW->pSchemaEx + i;
-    tdAddColToSchema(&sb, pSchema->type, pSchema->colId, pSchema->bytes);
+    tdAddColToSchema(&sb, pSchema->type, pSchema->sma, pSchema->colId, pSchema->bytes);
   }
   pTSchema = tdGetSchemaFromBuilder(&sb);
   tdDestroyTSchemaBuilder(&sb);
@@ -605,6 +613,50 @@ static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW) {
   return buf;
 }
 
+static int metaEncodeSchemaEx(void **buf, SSchemaWrapper *pSW) {
+  int        tlen = 0;
+  SSchemaEx *pSchema;
+
+  tlen += taosEncodeFixedU32(buf, pSW->nCols);
+  for (int i = 0; i < pSW->nCols; ++i) {
+    pSchema = pSW->pSchemaEx + i;
+    tlen += taosEncodeFixedI8(buf, pSchema->type);
+    tlen += taosEncodeFixedI8(buf, pSchema->sma);
+    tlen += taosEncodeFixedI16(buf, pSchema->colId);
+    tlen += taosEncodeFixedI32(buf, pSchema->bytes);
+    tlen += taosEncodeString(buf, pSchema->name);
+  }
+
+  return tlen;
+}
+
+static void *metaDecodeSchemaEx(void *buf, SSchemaWrapper *pSW, bool isGetEx) {
+  buf = taosDecodeFixedU32(buf, &pSW->nCols);
+  if (isGetEx) {
+    pSW->pSchemaEx = (SSchemaEx *)taosMemoryMalloc(sizeof(SSchemaEx) * pSW->nCols);
+    for (int i = 0; i < pSW->nCols; i++) {
+      SSchemaEx *pSchema = pSW->pSchemaEx + i;
+      buf = taosDecodeFixedI8(buf, &pSchema->type);
+      buf = taosDecodeFixedI8(buf, &pSchema->sma);
+      buf = taosDecodeFixedI16(buf, &pSchema->colId);
+      buf = taosDecodeFixedI32(buf, &pSchema->bytes);
+      buf = taosDecodeStringTo(buf, pSchema->name);
+    }
+  } else {
+    pSW->pSchema = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * pSW->nCols);
+    for (int i = 0; i < pSW->nCols; i++) {
+      SSchema *pSchema = pSW->pSchema + i;
+      buf = taosDecodeFixedI8(buf, &pSchema->type);
+      buf = taosSkipFixedLen(buf, sizeof(int8_t));
+      buf = taosDecodeFixedI16(buf, &pSchema->colId);
+      buf = taosDecodeFixedI32(buf, &pSchema->bytes);
+      buf = taosDecodeStringTo(buf, pSchema->name);
+    }
+  }
+
+  return buf;
+}
+
 static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) {
   int tsize = 0;
 

source/dnode/vnode/src/tq/tq.c

@@ -299,8 +299,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
       // response to user
       break;
     }
-    /*printf("vg %d offset %ld msgType %d from epoch %d\n", pTq->pVnode->vgId, fetchOffset, pHead->msgType,
+    /*printf("vg %d offset %ld msgType %d from epoch %d\n", pTq->pVnode->vgId, fetchOffset, pHead->msgType, pReq->epoch);*/
-     * pReq->epoch);*/
     /*int8_t pos = fetchOffset % TQ_BUFFER_SIZE;*/
     /*pHead = pTopic->pReadhandle->pHead;*/
     if (pHead->msgType == TDMT_VND_SUBMIT) {
@@ -353,9 +352,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
       pMsg->pCont = buf;
       pMsg->contLen = tlen;
       pMsg->code = 0;
-      /*printf("vg %d offset %ld msgType %d from epoch %d actual rsp\n", pTq->pVnode->vgId, fetchOffset,
+      /*printf("vg %d offset %ld msgType %d from epoch %d actual rsp\n", pTq->pVnode->vgId, fetchOffset, pHead->msgType, pReq->epoch);*/
-       * pHead->msgType,*/
-      /*pReq->epoch);*/
       tmsgSendRsp(pMsg);
       taosMemoryFree(pHead);
       return 0;
@@ -377,6 +374,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   }
   ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
   ((SMqRspHead*)buf)->epoch = pReq->epoch;
+  rsp.rspOffset = fetchOffset - 1;
 
   void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
   tEncodeSMqPollRsp(&abuf, &rsp);
@@ -452,7 +450,7 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
     pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, &handle);
     ASSERT(pTopic->buffer.output[i].task);
   }
-  printf("set topic %s to consumer %ld\n", pTopic->topicName, req.consumerId);
+  /*printf("set topic %s to consumer %ld on vg %d\n", pTopic->topicName, req.consumerId, pTq->pVnode->vgId);*/
   taosArrayPush(pConsumer->topics, pTopic);
   tqHandleMovePut(pTq->tqMeta, req.consumerId, pConsumer);
   tqHandleCommit(pTq->tqMeta, req.consumerId);

source/dnode/vnode/src/vnd/vnodeWrite.c

@@ -78,11 +78,13 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
         // TODO: handle error
       }
 
-      // TODO: maybe need to clear the request struct
+      // TODO: to encapsule a free API
       taosMemoryFree(vCreateTbReq.stbCfg.pSchema);
       taosMemoryFree(vCreateTbReq.stbCfg.pTagSchema);
-      taosMemoryFree(vCreateTbReq.stbCfg.pBSmaCols);
+      if(vCreateTbReq.stbCfg.pRSmaParam) {
-      taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam);
+        taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->pFuncIds);
+        taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam);
+      }
       taosMemoryFree(vCreateTbReq.dbFName);
       taosMemoryFree(vCreateTbReq.name);
       break;
@@ -111,19 +113,24 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
           // TODO: handle error
           vError("vgId:%d, failed to create table: %s", pVnode->vgId, pCreateTbReq->name);
         }
+        // TODO: to encapsule a free API
         taosMemoryFree(pCreateTbReq->name);
         taosMemoryFree(pCreateTbReq->dbFName);
         if (pCreateTbReq->type == TD_SUPER_TABLE) {
           taosMemoryFree(pCreateTbReq->stbCfg.pSchema);
           taosMemoryFree(pCreateTbReq->stbCfg.pTagSchema);
-          taosMemoryFree(pCreateTbReq->stbCfg.pBSmaCols);
+          if (pCreateTbReq->stbCfg.pRSmaParam) {
-          taosMemoryFree(pCreateTbReq->stbCfg.pRSmaParam);
+            taosMemoryFree(pCreateTbReq->stbCfg.pRSmaParam->pFuncIds);
+            taosMemoryFree(pCreateTbReq->stbCfg.pRSmaParam);
+          }
         } else if (pCreateTbReq->type == TD_CHILD_TABLE) {
           taosMemoryFree(pCreateTbReq->ctbCfg.pTag);
         } else {
           taosMemoryFree(pCreateTbReq->ntbCfg.pSchema);
-          taosMemoryFree(pCreateTbReq->ntbCfg.pBSmaCols);
+          if (pCreateTbReq->ntbCfg.pRSmaParam) {
-          taosMemoryFree(pCreateTbReq->ntbCfg.pRSmaParam);
+            taosMemoryFree(pCreateTbReq->ntbCfg.pRSmaParam->pFuncIds);
+            taosMemoryFree(pCreateTbReq->ntbCfg.pRSmaParam);
+          }
         }
       }
 
@@ -148,10 +155,13 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
       SVCreateTbReq vAlterTbReq = {0};
       vTrace("vgId:%d, process alter stb req", pVnode->vgId);
       tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vAlterTbReq);
+      // TODO: to encapsule a free API
       taosMemoryFree(vAlterTbReq.stbCfg.pSchema);
       taosMemoryFree(vAlterTbReq.stbCfg.pTagSchema);
-      taosMemoryFree(vAlterTbReq.stbCfg.pBSmaCols);
+      if (vAlterTbReq.stbCfg.pRSmaParam) {
-      taosMemoryFree(vAlterTbReq.stbCfg.pRSmaParam);
+        taosMemoryFree(vAlterTbReq.stbCfg.pRSmaParam->pFuncIds);
+        taosMemoryFree(vAlterTbReq.stbCfg.pRSmaParam);
+      }
       taosMemoryFree(vAlterTbReq.dbFName);
       taosMemoryFree(vAlterTbReq.name);
       break;

source/libs/function/src/builtins.c

@@ -361,6 +361,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .initFunc     = NULL,
     .sprocessFunc = winDurFunction,
     .finalizeFunc = NULL
+  },
+  {
+    .name = "now",
+    .type = FUNCTION_TYPE_NOW,
+    .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_DATETIME_FUNC,
+    .checkFunc    = stubCheckAndGetResultType,
+    .getEnvFunc   = getTimePseudoFuncEnv,
+    .initFunc     = NULL,
+    .sprocessFunc = winDurFunction,
+    .finalizeFunc = NULL
   }
 };
 
@@ -436,7 +446,9 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) {
       pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE };
       break;
     }
-
+    case FUNCTION_TYPE_NOW:
+      // todo
+      break;
     default:
       ASSERT(0); // to found the fault ASAP.
   }

source/libs/nodes/src/nodesCloneFuncs.c

@@ -206,6 +206,12 @@ static SNode* orderByExprNodeCopy(const SOrderByExprNode* pSrc, SOrderByExprNode
   return (SNode*)pDst;
 }
 
+static SNode* nodeListNodeCopy(const SNodeListNode* pSrc, SNodeListNode* pDst) {
+  COPY_ALL_SCALAR_FIELDS;
+  CLONE_NODE_LIST_FIELD(pNodeList);
+  return (SNode*)pDst;
+}
+
 static SNode* fillNodeCopy(const SFillNode* pSrc, SFillNode* pDst) {
   COPY_SCALAR_FIELD(mode);
   CLONE_NODE_FIELD(pValues);
@@ -360,6 +366,8 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) {
       return orderByExprNodeCopy((const SOrderByExprNode*)pNode, (SOrderByExprNode*)pDst);
     case QUERY_NODE_LIMIT:
       break;
+    case QUERY_NODE_NODE_LIST:
+      return nodeListNodeCopy((const SNodeListNode*)pNode, (SNodeListNode*)pDst);
     case QUERY_NODE_FILL:
       return fillNodeCopy((const SFillNode*)pNode, (SFillNode*)pDst);
     case QUERY_NODE_DATABLOCK_DESC:

source/libs/nodes/src/nodesCodeFuncs.c

@@ -2015,6 +2015,31 @@ static int32_t jsonToNodeListNode(const SJson* pJson, void* pObj) {
   return code;
 }
 
+static const char* jkFillMode = "Mode";
+static const char* jkFillValues = "Values";
+
+static int32_t fillNodeToJson(const void* pObj, SJson* pJson) {
+  const SFillNode* pNode = (const SFillNode*)pObj;
+
+  int32_t code = tjsonAddIntegerToObject(pJson, jkFillMode, pNode->mode);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddObject(pJson, jkFillValues, nodeToJson, pNode->pValues);
+  }
+
+  return code;
+}
+
+static int32_t jsonToFillNode(const SJson* pJson, void* pObj) {
+  SFillNode* pNode = (SFillNode*)pObj;
+
+  int32_t code = tjsonGetNumberValue(pJson, jkFillMode, pNode->mode);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = jsonToNodeObject(pJson, jkFillValues, &pNode->pValues);
+  }
+
+  return code;
+}
+
 static const char* jkTargetDataBlockId = "DataBlockId";
 static const char* jkTargetSlotId = "SlotId";
 static const char* jkTargetExpr = "Expr";
@@ -2328,6 +2353,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
     case QUERY_NODE_NODE_LIST:
       return nodeListNodeToJson(pObj, pJson);
     case QUERY_NODE_FILL:
+      return fillNodeToJson(pObj, pJson);
     case QUERY_NODE_RAW_EXPR:
       break;
     case QUERY_NODE_TARGET:
@@ -2431,7 +2457,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
       return jsonToIntervalWindowNode(pJson, pObj);
     case QUERY_NODE_NODE_LIST:
       return jsonToNodeListNode(pJson, pObj);
-    // case QUERY_NODE_FILL:
+    case QUERY_NODE_FILL:
+      return jsonToFillNode(pJson, pObj);
     case QUERY_NODE_TARGET:
       return jsonToTargetNode(pJson, pObj);
     // case QUERY_NODE_RAW_EXPR:

source/libs/nodes/src/nodesUtilFuncs.c

@@ -860,6 +860,24 @@ bool nodesIsExprNode(const SNode* pNode) {
   return (QUERY_NODE_COLUMN == type || QUERY_NODE_VALUE == type || QUERY_NODE_OPERATOR == type || QUERY_NODE_FUNCTION == type);
 }
 
+bool nodesIsUnaryOp(const SOperatorNode* pOp) {
+  switch (pOp->opType) {
+    case OP_TYPE_MINUS:
+    case OP_TYPE_IS_NULL:
+    case OP_TYPE_IS_NOT_NULL:
+    case OP_TYPE_IS_TRUE:
+    case OP_TYPE_IS_FALSE:
+    case OP_TYPE_IS_UNKNOWN:
+    case OP_TYPE_IS_NOT_TRUE:
+    case OP_TYPE_IS_NOT_FALSE:
+    case OP_TYPE_IS_NOT_UNKNOWN:
+      return true;
+    default:
+      break;
+  }
+  return false;
+}
+
 bool nodesIsArithmeticOp(const SOperatorNode* pOp) {
   switch (pOp->opType) {
     case OP_TYPE_ADD:

source/libs/parser/inc/parAst.h

@@ -150,6 +150,8 @@ SNode* createAlterTableRenameCol(SAstCreateContext* pCxt, SNode* pRealTable, int
 SNode* createAlterTableSetTag(SAstCreateContext* pCxt, SNode* pRealTable, const SToken* pTagName, SNode* pVal);
 SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName);
 SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pDbName, SNode* pTbNamePattern);
+SNode* createShowCreateDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName);
+SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable);
 SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword);
 SNode* createAlterUserStmt(SAstCreateContext* pCxt, SToken* pUserName, int8_t alterType, const SToken* pVal);
 SNode* createDropUserStmt(SAstCreateContext* pCxt, SToken* pUserName);
@@ -170,6 +172,16 @@ SNode* setExplainRatio(SAstCreateContext* pCxt, SNode* pOptions, const SToken* p
 SNode* createExplainStmt(SAstCreateContext* pCxt, bool analyze, SNode* pOptions, SNode* pQuery);
 SNode* createDescribeStmt(SAstCreateContext* pCxt, SNode* pRealTable);
 SNode* createResetQueryCacheStmt(SAstCreateContext* pCxt);
+SNode* createCompactStmt(SAstCreateContext* pCxt, SNodeList* pVgroups);
+SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool aggFunc, const SToken* pFuncName, const SToken* pLibPath, SDataType dataType, int32_t bufSize);
+SNode* createDropFunctionStmt(SAstCreateContext* pCxt, const SToken* pFuncName);
+SNode* createCreateStreamStmt(SAstCreateContext* pCxt, const SToken* pStreamName, const SToken* pTableName, SNode* pQuery);
+SNode* createDropStreamStmt(SAstCreateContext* pCxt, const SToken* pStreamName);
+SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId);
+SNode* createMergeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId1, const SToken* pVgId2);
+SNode* createRedistributeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId, SNodeList* pDnodes);
+SNode* createSplitVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId);
+SNode* createSyncdbStmt(SAstCreateContext* pCxt, const SToken* pDbName);
 
 #ifdef __cplusplus
 }

source/libs/parser/inc/parInsertData.h

@@ -77,9 +77,6 @@ typedef struct STableDataBlocks {
   STableMeta *pTableMeta;   // the tableMeta of current table, the table meta will be used during submit, keep a ref to avoid to be removed from cache
   char       *pData;
   bool        cloned;
-  STagData    tagData; 
-  char        tableName[TSDB_TABLE_NAME_LEN];
-  char        dbFName[TSDB_DB_FNAME_LEN];
   
   SParsedDataColInfo boundColumnInfo;
   SRowBuilder        rowBuilder;

source/libs/parser/inc/sql.y

@@ -301,6 +301,17 @@ cmd ::= SHOW QNODES.
 cmd ::= SHOW FUNCTIONS.                                                           { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT, NULL, NULL); }
 cmd ::= SHOW INDEXES FROM table_name_cond(A) from_db_opt(B).                      { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, A, B); }
 cmd ::= SHOW STREAMS.                                                             { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT, NULL, NULL); }
+cmd ::= SHOW ACCOUNTS.                                                            { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); }
+cmd ::= SHOW APPS.                                                                { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_APPS_STMT, NULL, NULL); }
+cmd ::= SHOW CONNECTIONS.                                                         { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONNECTIONS_STMT, NULL, NULL); }
+cmd ::= SHOW LICENCE.                                                             { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCE_STMT, NULL, NULL); }
+cmd ::= SHOW CREATE DATABASE db_name(A).                                          { pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &A); }
+cmd ::= SHOW CREATE TABLE full_table_name(A).                                     { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, A); }
+cmd ::= SHOW CREATE STABLE full_table_name(A).                                    { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, A); }
+cmd ::= SHOW QUERIES.                                                             { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT, NULL, NULL); }
+cmd ::= SHOW SCORES.                                                              { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SCORES_STMT, NULL, NULL); }
+cmd ::= SHOW TOPICS.                                                              { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TOPICS_STMT, NULL, NULL); }
+cmd ::= SHOW VARIABLES.                                                           { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VARIABLE_STMT, NULL, NULL); }
 
 db_name_cond_opt(A) ::= .                                                         { A = createDefaultDatabaseCondValue(pCxt); }
 db_name_cond_opt(A) ::= db_name(B) NK_DOT.                                        { A = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B); }
@@ -364,6 +375,45 @@ explain_options(A) ::= .
 explain_options(A) ::= explain_options(B) VERBOSE NK_BOOL(C).                     { A = setExplainVerbose(pCxt, B, &C); }
 explain_options(A) ::= explain_options(B) RATIO NK_FLOAT(C).                      { A = setExplainRatio(pCxt, B, &C); }
 
+/************************************************ compact *************************************************************/
+cmd ::= COMPACT VNODES IN NK_LP integer_list(A) NK_RP.                            { pCxt->pRootNode = createCompactStmt(pCxt, A); }
+
+/************************************************ create/drop function ************************************************/
+cmd ::= CREATE agg_func_opt(A) FUNCTION function_name(B) 
+  AS NK_STRING(C) OUTPUTTYPE type_name(D) bufsize_opt(E).                         { pCxt->pRootNode = createCreateFunctionStmt(pCxt, A, &B, &C, D, E); }
+cmd ::= DROP FUNCTION function_name(A).                                           { pCxt->pRootNode = createDropFunctionStmt(pCxt, &A); }
+
+%type agg_func_opt                                                                { bool }
+%destructor agg_func_opt                                                          { }
+agg_func_opt(A) ::= .                                                             { A = false; }
+agg_func_opt(A) ::= AGGREGATE.                                                    { A = true; }
+
+%type bufsize_opt                                                                 { int32_t }
+%destructor bufsize_opt                                                           { }
+bufsize_opt(A) ::= .                                                              { A = 0; }
+bufsize_opt(A) ::= BUFSIZE NK_INTEGER(B).                                         { A = strtol(B.z, NULL, 10); }
+
+/************************************************ create/drop stream **************************************************/
+cmd ::= CREATE STREAM stream_name(A) INTO table_name(B) AS query_expression(C).   { pCxt->pRootNode = createCreateStreamStmt(pCxt, &A, &B, C); }
+cmd ::= DROP STREAM stream_name(A).                                               { pCxt->pRootNode = createDropStreamStmt(pCxt, &A); }
+
+/************************************************ kill connection/query ***********************************************/
+cmd ::= KILL CONNECTION NK_INTEGER(A).                                            { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &A); }
+cmd ::= KILL QUERY NK_INTEGER(A).                                                 { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_QUERY_STMT, &A); }
+
+/************************************************ merge/redistribute/ vgroup ******************************************/
+cmd ::= MERGE VGROUP NK_INTEGER(A) NK_INTEGER(B).                                 { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &A, &B); }
+cmd ::= REDISTRIBUTE VGROUP NK_INTEGER(A) dnode_list(B).                          { pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &A, B); }
+cmd ::= SPLIT VGROUP NK_INTEGER(A).                                               { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &A); }
+
+%type dnode_list                                                                  { SNodeList* }
+%destructor dnode_list                                                            { nodesDestroyList($$); }
+dnode_list(A) ::= DNODE NK_INTEGER(B).                                            { A = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B)); }
+dnode_list(A) ::= dnode_list(B) DNODE NK_INTEGER(C).                              { A = addNodeToList(pCxt, B, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C)); }
+
+/************************************************ syncdb **************************************************************/
+cmd ::= SYNCDB db_name(A) REPLICA.                                                { pCxt->pRootNode = createSyncdbStmt(pCxt, &A); }
+
 /************************************************ select **************************************************************/
 cmd ::= query_expression(A).                                                      { pCxt->pRootNode = A; }
 
@@ -442,6 +492,10 @@ index_name(A) ::= NK_ID(B).
 %destructor topic_name                                                            { }
 topic_name(A) ::= NK_ID(B).                                                       { A = B; }
 
+%type stream_name                                                                 { SToken }
+%destructor stream_name                                                           { }
+stream_name(A) ::= NK_ID(B).                                                      { A = B; }
+
 /************************************************ expression **********************************************************/
 expression(A) ::= literal(B).                                                     { A = B; }
 //expression(A) ::= NK_QUESTION(B).                                                 { A = B; }
@@ -459,7 +513,7 @@ expression(A) ::= NK_PLUS(B) expression(C).
                                                                                   }
 expression(A) ::= NK_MINUS(B) expression(C).                                      {
                                                                                     SToken t = getTokenFromRawExprNode(pCxt, C);
-                                                                                    A = createRawExprNodeExt(pCxt, &B, &t, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, C), NULL));
+                                                                                    A = createRawExprNodeExt(pCxt, &B, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, C), NULL));
                                                                                   }
 expression(A) ::= expression(B) NK_PLUS expression(C).                            {
                                                                                     SToken s = getTokenFromRawExprNode(pCxt, B);
@@ -495,38 +549,14 @@ expression_list(A) ::= expression_list(B) NK_COMMA expression(C).
 column_reference(A) ::= column_name(B).                                           { A = createRawExprNode(pCxt, &B, createColumnNode(pCxt, NULL, &B)); }
 column_reference(A) ::= table_name(B) NK_DOT column_name(C).                      { A = createRawExprNodeExt(pCxt, &B, &C, createColumnNode(pCxt, &B, &C)); }
 
-//pseudo_column(A) ::= NK_NOW.                                                    { A = createFunctionNode(pCxt, NULL, NULL); }
+pseudo_column(A) ::=  NOW(B).                                                     { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
-pseudo_column(A) ::=  NK_UNDERLINE(B) ROWTS(C).                                   {
+pseudo_column(A) ::=  ROWTS(B).                                                   { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
-                                                                                    SToken t = B;
-                                                                                    t.n = (C.z + C.n) - B.z;
-                                                                                    A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL));
-                                                                                  }
 pseudo_column(A) ::=  TBNAME(B).                                                  { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
-pseudo_column(A) ::=  NK_UNDERLINE(B) QSTARTTS(C).                                {
+pseudo_column(A) ::=  QSTARTTS(B).                                                { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
-                                                                                    SToken t = B;
+pseudo_column(A) ::=  QENDTS(B).                                                  { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
-                                                                                    t.n = (C.z + C.n) - B.z;
+pseudo_column(A) ::=  WSTARTTS(B).                                                { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
-                                                                                    A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL));
+pseudo_column(A) ::=  WENDTS(B).                                                  { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
-                                                                                  }
+pseudo_column(A) ::=  WDURATION(B).                                               { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
-pseudo_column(A) ::=  NK_UNDERLINE(B) QENDTS(C).                                  {
-                                                                                    SToken t = B;
-                                                                                    t.n = (C.z + C.n) - B.z;
-                                                                                    A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL));
-                                                                                  }
-pseudo_column(A) ::=  NK_UNDERLINE(B) WSTARTTS(C).                                {
-                                                                                    SToken t = B;
-                                                                                    t.n = (C.z + C.n) - B.z;
-                                                                                    A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL));
-                                                                                  }
-pseudo_column(A) ::=  NK_UNDERLINE(B) WENDTS(C).                                  {
-                                                                                    SToken t = B;
-                                                                                    t.n = (C.z + C.n) - B.z;
-                                                                                    A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL));
-                                                                                  }
-pseudo_column(A) ::=  NK_UNDERLINE(B) WDURATION(C).                               {
-                                                                                    SToken t = B;
-                                                                                    t.n = (C.z + C.n) - B.z;
-                                                                                    A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL));
-                                                                                  }
 
 /************************************************ predicate ***********************************************************/
 predicate(A) ::= expression(B) compare_op(C) expression(D).                       {

source/libs/parser/src/parAstCreater.c

@@ -1221,6 +1221,18 @@ SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pDbName, S
   return (SNode*)pStmt;
 }
 
+SNode* createShowCreateDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_SHOW_CREATE_DATABASE_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable) {
+  SNode* pStmt = nodesMakeNode(type);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
 SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword) {
   char password[TSDB_USET_PASSWORD_LEN] = {0};
   if (!checkUserName(pCxt, pUserName) || !checkPassword(pCxt, pPassword, password)) {
@@ -1433,3 +1445,63 @@ SNode* createResetQueryCacheStmt(SAstCreateContext* pCxt) {
   CHECK_OUT_OF_MEM(pStmt);
   return pStmt;
 }
+
+SNode* createCompactStmt(SAstCreateContext* pCxt, SNodeList* pVgroups) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_COMPACT_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool aggFunc, const SToken* pFuncName, const SToken* pLibPath, SDataType dataType, int32_t bufSize) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_CREATE_FUNCTION_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createDropFunctionStmt(SAstCreateContext* pCxt, const SToken* pFuncName) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_DROP_FUNCTION_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createCreateStreamStmt(SAstCreateContext* pCxt, const SToken* pStreamName, const SToken* pTableName, SNode* pQuery) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_CREATE_STREAM_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createDropStreamStmt(SAstCreateContext* pCxt, const SToken* pStreamName) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_DROP_STREAM_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId) {
+  SNode* pStmt = nodesMakeNode(type);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createMergeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId1, const SToken* pVgId2) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_MERGE_VGROUP_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createRedistributeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId, SNodeList* pDnodes) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_REDISTRIBUTE_VGROUP_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createSplitVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_SPLIT_VGROUP_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createSyncdbStmt(SAstCreateContext* pCxt, const SToken* pDbName) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_SYNCDB_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}

source/libs/parser/src/parInsert.c

@@ -52,13 +52,13 @@ typedef struct SInsertParseContext {
   SParseContext* pComCxt;       // input
   char          *pSql;          // input
   SMsgBuf        msg;           // input
-  char           dbFName[TSDB_DB_FNAME_LEN];
-  char           tableName[TSDB_TABLE_NAME_LEN];
   STableMeta* pTableMeta;       // each table
   SParsedDataColInfo tags;      // each table
   SKVRowBuilder tagsBuilder;    // each table
+  SVCreateTbReq createTblReq;   // each table
   SHashObj* pVgroupsHashObj;    // global
   SHashObj* pTableBlockHashObj; // global
+  SHashObj* pSubTableHashObj;   // global
   SArray* pTableDataBlocks;     // global
   SArray* pVgDataBlocks;        // global
   int32_t totalNum;
@@ -231,9 +231,6 @@ static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) {
   SVgroupInfo vg;
   CHECK_CODE(catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &vg));
   CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)));
-  pCxt->pTableMeta->vgId = vg.vgId; // todo remove
-  strcpy(pCxt->tableName, name.tname);
-  tNameGetFullDbName(&name, pCxt->dbFName);
   
   return TSDB_CODE_SUCCESS;
 }
@@ -743,8 +740,20 @@ static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void *value, int32_t len, voi
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t buildCreateTbReq(SInsertParseContext* pCxt, const SName* pName, SKVRow row) {
+  char dbFName[TSDB_DB_FNAME_LEN] = {0};
+  tNameGetFullDbName(pName, dbFName);
+  pCxt->createTblReq.type = TD_CHILD_TABLE;
+  pCxt->createTblReq.dbFName = strdup(dbFName);
+  pCxt->createTblReq.name = strdup(pName->tname);
+  pCxt->createTblReq.ctbCfg.suid = pCxt->pTableMeta->suid;
+  pCxt->createTblReq.ctbCfg.pTag = row;
+
+  return TSDB_CODE_SUCCESS;
+}
+
 // pSql -> tag1_value, ...)
-static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pTagsSchema, uint8_t precision) {
+static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pTagsSchema, uint8_t precision, const SName* pName) {
   if (tdInitKVRowBuilder(&pCxt->tagsBuilder) < 0) {
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
@@ -765,23 +774,46 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pTagsSchema,
   }
   tdSortKVRowByColIdx(row);
 
-  // todo construct payload
+  return buildCreateTbReq(pCxt, pName, row);
+}
 
-  taosMemoryFreeClear(row);
+static int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) {
+  *pDst = taosMemoryMalloc(TABLE_META_SIZE(pSrc));
+  if (NULL == *pDst) {
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+  }
+  memcpy(*pDst, pSrc, TABLE_META_SIZE(pSrc));
+  return TSDB_CODE_SUCCESS;
+}
 
-  return 0;
+static int32_t storeTableMeta(SHashObj* pHash, const char* pName, int32_t len, STableMeta* pMeta) {
+  STableMeta* pBackup = NULL;
+  if (TSDB_CODE_SUCCESS != cloneTableMeta(pMeta, &pBackup)) {
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+  }
+  return taosHashPut(pHash, pName, len, &pBackup, POINTER_BYTES);
 }
 
 // pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)
 static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) {
-  SToken sToken;
+  SName name;
+  createSName(&name, pTbnameToken, pCxt->pComCxt, &pCxt->msg);
+  char tbFName[TSDB_TABLE_FNAME_LEN];
+  tNameExtractFullName(&name, tbFName);
+  int32_t len = strlen(tbFName);
+  STableMeta** pMeta = taosHashGet(pCxt->pSubTableHashObj, tbFName, len);
+  if (NULL != pMeta) {
+    return cloneTableMeta(*pMeta, &pCxt->pTableMeta);
+  }
 
+  SToken sToken;
   // pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)
   NEXT_TOKEN(pCxt->pSql, sToken);
   CHECK_CODE(getTableMeta(pCxt, &sToken));
   if (TSDB_SUPER_TABLE != pCxt->pTableMeta->tableType) {
     return buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed");
   }
+  CHECK_CODE(storeTableMeta(pCxt->pSubTableHashObj, tbFName, len, pCxt->pTableMeta));
 
   SSchema* pTagsSchema = getTableTagSchema(pCxt->pTableMeta);
   setBoundColumnInfo(&pCxt->tags, pTagsSchema, getNumOfTags(pCxt->pTableMeta));
@@ -801,7 +833,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken)
   if (TK_NK_LP != sToken.type) {
     return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z);
   }
-  CHECK_CODE(parseTagsClause(pCxt, pTagsSchema, getTableInfo(pCxt->pTableMeta).precision));
+  CHECK_CODE(parseTagsClause(pCxt, pTagsSchema, getTableInfo(pCxt->pTableMeta).precision, &name));
 
   return TSDB_CODE_SUCCESS;
 }
@@ -909,10 +941,17 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* da
   return TSDB_CODE_SUCCESS;
 }
 
+static void destroyCreateSubTbReq(SVCreateTbReq* pReq) {
+  taosMemoryFreeClear(pReq->dbFName);
+  taosMemoryFreeClear(pReq->name);
+  taosMemoryFreeClear(pReq->ctbCfg.pTag);
+}
+
 static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) {
   taosMemoryFreeClear(pCxt->pTableMeta);
   destroyBoundColumnInfo(&pCxt->tags);
   tdDestroyKVRowBuilder(&pCxt->tagsBuilder);
+  destroyCreateSubTbReq(&pCxt->createTblReq);
 }
 
 static void destroyDataBlock(STableDataBlocks* pDataBlock) {
@@ -977,9 +1016,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
     STableDataBlocks *dataBuf = NULL;
     CHECK_CODE(getDataBlockFromList(pCxt->pTableBlockHashObj, pCxt->pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE,
         sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, &dataBuf, NULL));
-    strcpy(dataBuf->tableName, pCxt->tableName);
+        
-    strcpy(dataBuf->dbFName, pCxt->dbFName);
-    
     if (TK_NK_LP == sToken.type) {
       // pSql -> field1_name, ...)
       CHECK_CODE(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pCxt->pTableMeta)));
@@ -1028,11 +1065,13 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) {
     .pTableMeta = NULL,
     .pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false),
     .pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false),
+    .pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, false),
     .totalNum = 0,
     .pOutput = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT)
   };
 
-  if (NULL == context.pVgroupsHashObj || NULL == context.pTableBlockHashObj || NULL == context.pOutput) {
+  if (NULL == context.pVgroupsHashObj || NULL == context.pTableBlockHashObj ||
+      NULL == context.pSubTableHashObj || NULL == context.pOutput) {
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
 

source/libs/parser/src/parTokenizer.c

@@ -29,10 +29,14 @@ typedef struct SKeyword {
 // keywords in sql string
 static SKeyword keywordTable[] = {
     {"ACCOUNT",       TK_ACCOUNT},
+    {"ACCOUNTS",      TK_ACCOUNTS},
+    {"ADD",           TK_ADD},
+    {"AGGREGATE",     TK_AGGREGATE},
     {"ALL",           TK_ALL},
     {"ALTER",         TK_ALTER},
     {"ANALYZE",       TK_ANALYZE},
     {"AND",           TK_AND},
+    {"APPS",          TK_APPS},
     {"AS",            TK_AS},
     {"ASC",           TK_ASC},
     {"BETWEEN",       TK_BETWEEN},
@@ -40,15 +44,22 @@ static SKeyword keywordTable[] = {
     {"BIGINT",        TK_BIGINT},
     {"BLOCKS",        TK_BLOCKS},
     {"BOOL",          TK_BOOL},
+    {"BUFSIZE",       TK_BUFSIZE},
     {"BY",            TK_BY},
     {"CACHE",         TK_CACHE},
     {"CACHELAST",     TK_CACHELAST},
+    {"COLUMN",        TK_COLUMN},
     {"COMMENT",       TK_COMMENT},
     {"COMP",          TK_COMP},
+    {"COMPACT",       TK_COMPACT},
+    {"CONNS",         TK_CONNS},
+    {"CONNECTION",    TK_CONNECTION},
+    {"CONNECTIONS",   TK_CONNECTIONS},
     {"CREATE",        TK_CREATE},
     {"DATABASE",      TK_DATABASE},
     {"DATABASES",     TK_DATABASES},
     {"DAYS",          TK_DAYS},
+    {"DBS",           TK_DBS},
     {"DELAY",         TK_DELAY},
     {"DESC",          TK_DESC},
     {"DESCRIBE",      TK_DESCRIBE},
@@ -83,14 +94,18 @@ static SKeyword keywordTable[] = {
     {"JOIN",          TK_JOIN},
     {"JSON",          TK_JSON},
     {"KEEP",          TK_KEEP},
+    {"KILL",          TK_KILL},
+    {"LICENCE",       TK_LICENCE},
     {"LIKE",          TK_LIKE},
     {"LIMIT",         TK_LIMIT},
     {"LINEAR",        TK_LINEAR},
+    {"LOCAL",         TK_LOCAL},
     {"MATCH",         TK_MATCH},
     {"MAXROWS",       TK_MAXROWS},
     {"MINROWS",       TK_MINROWS},
     {"MINUS",         TK_MINUS},
     {"MNODES",        TK_MNODES},
+    {"MODIFY",        TK_MODIFY},
     {"MODULES",       TK_MODULES},
     {"NCHAR",         TK_NCHAR},
     {"NMATCH",        TK_NMATCH},
@@ -102,16 +117,20 @@ static SKeyword keywordTable[] = {
     {"ON",            TK_ON},
     {"OR",            TK_OR},
     {"ORDER",         TK_ORDER},
+    {"OUTPUTTYPE",    TK_OUTPUTTYPE},
     {"PARTITION",     TK_PARTITION},
     {"PASS",          TK_PASS},
     {"PORT",          TK_PORT},
+    {"PPS",           TK_PPS},
     {"PRECISION",     TK_PRECISION},
     {"PRIVILEGE",     TK_PRIVILEGE},
     {"PREV",          TK_PREV},
-    {"QENDTS",        TK_QENDTS},
+    {"_QENDTS",        TK_QENDTS},
     {"QNODE",         TK_QNODE},
     {"QNODES",        TK_QNODES},
-    {"QSTARTTS",      TK_QSTARTTS},
+    {"_QSTARTTS",      TK_QSTARTTS},
+    {"QTIME",         TK_QTIME},
+    {"QUERIES",       TK_QUERIES},
     {"QUERY",         TK_QUERY},
     {"QUORUM",        TK_QUORUM},
     {"RATIO",         TK_RATIO},
@@ -119,9 +138,11 @@ static SKeyword keywordTable[] = {
     {"RESET",         TK_RESET},
     {"RETENTIONS",    TK_RETENTIONS},
     {"ROLLUP",        TK_ROLLUP},
-    {"ROWTS",         TK_ROWTS},
+    {"_ROWTS",         TK_ROWTS},
+    {"SCORES",        TK_SCORES},
     {"SELECT",        TK_SELECT},
     {"SESSION",       TK_SESSION},
+    {"SET",           TK_SET},
     {"SHOW",          TK_SHOW},
     {"SINGLE_STABLE", TK_SINGLE_STABLE},
     {"SLIDING",       TK_SLIDING},
@@ -131,16 +152,23 @@ static SKeyword keywordTable[] = {
     {"SOFFSET",       TK_SOFFSET},
     {"STABLE",        TK_STABLE},
     {"STABLES",       TK_STABLES},
+    {"STATE",         TK_STATE},
     {"STATE_WINDOW",  TK_STATE_WINDOW},
+    {"STORAGE",       TK_STORAGE},
+    {"STREAM",       TK_STREAM},
     {"STREAMS",       TK_STREAMS},
     {"STREAM_MODE",   TK_STREAM_MODE},
+    {"SYNCDB",        TK_SYNCDB},
     {"TABLE",         TK_TABLE},
     {"TABLES",        TK_TABLES},
+    {"TAG",           TK_TAG},
     {"TAGS",          TK_TAGS},
     {"TBNAME",        TK_TBNAME},
     {"TIMESTAMP",     TK_TIMESTAMP},
     {"TINYINT",       TK_TINYINT},
     {"TOPIC",         TK_TOPIC},
+    {"TOPICS",        TK_TOPICS},
+    {"TSERIES",       TK_TSERIES},
     {"TTL",           TK_TTL},
     {"UNION",         TK_UNION},
     {"UNSIGNED",      TK_UNSIGNED},
@@ -148,15 +176,18 @@ static SKeyword keywordTable[] = {
     {"USER",          TK_USER},
     {"USERS",         TK_USERS},
     {"USING",         TK_USING},
+    {"VALUE",         TK_VALUE},
     {"VALUES",        TK_VALUES},
     {"VARCHAR",       TK_VARCHAR},
+    {"VARIABLES",     TK_VARIABLES},
     {"VERBOSE",       TK_VERBOSE},
     {"VGROUPS",       TK_VGROUPS},
+    {"VNODES",        TK_VNODES},
     {"WAL",           TK_WAL},
-    {"WDURATION",     TK_WDURATION},
+    {"_WDURATION",     TK_WDURATION},
-    {"WENDTS",        TK_WENDTS},
+    {"_WENDTS",        TK_WENDTS},
     {"WHERE",         TK_WHERE},
-    {"WSTARTTS",      TK_WSTARTTS},
+    {"_WSTARTTS",     TK_WSTARTTS},
     // {"ID",           TK_ID},
     // {"STRING",       TK_STRING},
     // {"EQ",           TK_EQ},
@@ -182,22 +213,8 @@ static SKeyword keywordTable[] = {
     // {"UMINUS",       TK_UMINUS},
     // {"UPLUS",        TK_UPLUS},
     // {"BITNOT",       TK_BITNOT},
-    // {"ACCOUNTS",     TK_ACCOUNTS},
-    // {"QUERIES",      TK_QUERIES},
-    // {"CONNECTIONS",  TK_CONNECTIONS},
-    // {"VARIABLES",    TK_VARIABLES},
-    // {"SCORES",       TK_SCORES},
     // {"GRANTS",       TK_GRANTS},
     // {"DOT",          TK_DOT},
-    // {"SYNCDB",       TK_SYNCDB},
-    // {"LOCAL",        TK_LOCAL},
-    // {"PPS",          TK_PPS},
-    // {"TSERIES",      TK_TSERIES},
-    // {"DBS",          TK_DBS},
-    // {"STORAGE",      TK_STORAGE},
-    // {"QTIME",        TK_QTIME},
-    // {"CONNS",        TK_CONNS},
-    // {"STATE",        TK_STATE},
     // {"CTIME",        TK_CTIME},
     // {"LP",           TK_LP},
     // {"RP",           TK_RP},
@@ -205,15 +222,8 @@ static SKeyword keywordTable[] = {
     // {"EVERY",        TK_EVERY},
     // {"VARIABLE",     TK_VARIABLE},
     // {"UPDATE",       TK_UPDATE},
-    // {"ADD",          TK_ADD},
-    // {"COLUMN",       TK_COLUMN},
-    // {"TAG",          TK_TAG},
     // {"CHANGE",       TK_CHANGE},
-    // {"SET",          TK_SET},
-    // {"KILL",         TK_KILL},
-    // {"CONNECTION",   TK_CONNECTION},
     // {"COLON",        TK_COLON},
-    // {"STREAM",       TK_STREAM},
     // {"ABORT",        TK_ABORT},
     // {"AFTER",        TK_AFTER},
     // {"ATTACH",       TK_ATTACH},
@@ -244,14 +254,7 @@ static SKeyword keywordTable[] = {
     // {"TRIGGER",      TK_TRIGGER},
     // {"VIEW",         TK_VIEW},
     // {"SEMI",         TK_SEMI},
-    // {"VNODES",       TK_VNODES},
+    // {"PARTITIONS",   TK_PARTITIONS},
-//    {"PARTITIONS",   TK_PARTITIONS},
-    // {"TOPICS",       TK_TOPICS},
-    // {"COMPACT",      TK_COMPACT},
-    // {"MODIFY",       TK_MODIFY},
-    // {"OUTPUTTYPE",   TK_OUTPUTTYPE},
-    // {"AGGREGATE",    TK_AGGREGATE},
-    // {"BUFSIZE",      TK_BUFSIZE},
     // {"MODE",         TK_MODE},
 };
 
@@ -437,10 +440,6 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) {
       *tokenId = TK_NK_QUESTION;
       return 1;
     }
-    case '_': {
-      *tokenId = TK_NK_UNDERLINE;
-      return 1;
-    }
     case '`':
     case '\'':
     case '"': {

source/libs/parser/src/parTranslater.c

@@ -240,7 +240,11 @@ static void setColumnInfoByExpr(const STableNode* pTable, SExprNode* pExpr, SCol
   if (NULL != pTable) {
     strcpy(pCol->tableAlias, pTable->tableAlias);
   } else if (QUERY_NODE_COLUMN == nodeType(pExpr)) {
-    strcpy(pCol->tableAlias, ((SColumnNode*)pExpr)->tableAlias);
+    SColumnNode* pProjCol = (SColumnNode*)pExpr;
+    strcpy(pCol->tableAlias, pProjCol->tableAlias);
+    pCol->tableId = pProjCol->tableId;
+    pCol->colId = pProjCol->colId;
+    pCol->colType = pProjCol->colType;
   }
   strcpy(pCol->colName, pExpr->aliasName);
   pCol->node.resType = pExpr->resType;
@@ -435,6 +439,9 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) {
 }
 
 static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
+  if (nodesIsUnaryOp(pOp)) {
+    return DEAL_RES_CONTINUE;
+  }
   SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType;
   SDataType rdt = ((SExprNode*)(pOp->pRight))->resType;
   if (nodesIsArithmeticOp(pOp)) {
@@ -1095,10 +1102,6 @@ static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray) {
 }
 
 static int32_t columnNodeToField(SNodeList* pList, SArray** pArray) {
-  if (NULL == pList) {
-    return TSDB_CODE_SUCCESS;
-  }
-
   *pArray = taosArrayInit(LIST_LENGTH(pList), sizeof(SField));
   SNode* pNode;
   FOREACH(pNode, pList) {
@@ -1161,10 +1164,15 @@ static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableSt
   createReq.delay = pStmt->pOptions->delay;
   columnDefNodeToField(pStmt->pCols, &createReq.pColumns);
   columnDefNodeToField(pStmt->pTags, &createReq.pTags);
-  columnNodeToField(pStmt->pOptions->pSma, &createReq.pSmas);
   createReq.numOfColumns = LIST_LENGTH(pStmt->pCols);
   createReq.numOfTags = LIST_LENGTH(pStmt->pTags);
-  createReq.numOfSmas = LIST_LENGTH(pStmt->pOptions->pSma);
+  if (NULL == pStmt->pOptions->pSma) {
+    columnDefNodeToField(pStmt->pCols, &createReq.pSmas);
+    createReq.numOfSmas = createReq.numOfColumns;
+  } else {
+    columnNodeToField(pStmt->pOptions->pSma, &createReq.pSmas);
+    createReq.numOfSmas = LIST_LENGTH(pStmt->pOptions->pSma);
+  }
 
   SName tableName = { .type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId };
   strcpy(tableName.dbname, pStmt->dbName);
@@ -1470,20 +1478,20 @@ static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pSt
 
 static int32_t nodeTypeToShowType(ENodeType nt) {
   switch (nt) {
-    case QUERY_NODE_SHOW_DATABASES_STMT:
+    case QUERY_NODE_SHOW_APPS_STMT:
-      return TSDB_MGMT_TABLE_DB;
+      return 0; // todo
-    case QUERY_NODE_SHOW_STABLES_STMT:
+    case QUERY_NODE_SHOW_CONNECTIONS_STMT:
-      return TSDB_MGMT_TABLE_STB;
+      return TSDB_MGMT_TABLE_CONNS;
-    case QUERY_NODE_SHOW_USERS_STMT:
+    case QUERY_NODE_SHOW_LICENCE_STMT:
-      return TSDB_MGMT_TABLE_USER;
+      return 0; // todo
-    case QUERY_NODE_SHOW_DNODES_STMT:
+    case QUERY_NODE_SHOW_QUERIES_STMT:
-      return TSDB_MGMT_TABLE_DNODE;
+      return TSDB_MGMT_TABLE_QUERIES;
-    case QUERY_NODE_SHOW_VGROUPS_STMT:
+    case QUERY_NODE_SHOW_SCORES_STMT:
-      return TSDB_MGMT_TABLE_VGROUP;
+      return 0; // todo
-    case QUERY_NODE_SHOW_MNODES_STMT:
+    case QUERY_NODE_SHOW_TOPICS_STMT:
-      return TSDB_MGMT_TABLE_MNODE;
+      return 0; // todo
-    case QUERY_NODE_SHOW_QNODES_STMT:
+    case QUERY_NODE_SHOW_VARIABLE_STMT:
-      return TSDB_MGMT_TABLE_QNODE;
+      return TSDB_MGMT_TABLE_VARIABLES;
     default:
       break;
   }
@@ -1509,30 +1517,6 @@ static int32_t translateShow(STranslateContext* pCxt, SShowStmt* pStmt) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t translateShowTables(STranslateContext* pCxt) {
-  SVShowTablesReq* pShowReq = taosMemoryCalloc(1, sizeof(SVShowTablesReq));
-
-  SArray* array = NULL;
-  int32_t code = getDBVgInfo(pCxt, pCxt->pParseCxt->db, &array);
-  if (TSDB_CODE_SUCCESS != code) {
-    return code;
-  }
-  SVgroupInfo* info = taosArrayGet(array, 0);
-  pShowReq->head.vgId = htonl(info->vgId);
-
-  pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo));
-  if (NULL == pCxt->pCmdMsg) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-  pCxt->pCmdMsg->epSet = info->epSet;
-  pCxt->pCmdMsg->msgType = TDMT_VND_SHOW_TABLES;
-  pCxt->pCmdMsg->msgLen = sizeof(SVShowTablesReq);
-  pCxt->pCmdMsg->pMsg = pShowReq;
-  pCxt->pCmdMsg->pExtension = array;
-
-  return TSDB_CODE_SUCCESS;
-}
-
 static int32_t getSmaIndexDstVgId(STranslateContext* pCxt, char* pTableName, int32_t* pVgId) {
   SVgroupInfo vg = {0};
   int32_t code = getTableHashVgroup(pCxt, pCxt->pParseCxt->db, pTableName, &vg);
@@ -1868,17 +1852,19 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
     case QUERY_NODE_ALTER_DNODE_STMT:
       code = translateAlterDnode(pCxt, (SAlterDnodeStmt*)pNode);
       break;
-    case QUERY_NODE_SHOW_DATABASES_STMT:
+    case QUERY_NODE_SHOW_APPS_STMT:
-    case QUERY_NODE_SHOW_STABLES_STMT:
+    case QUERY_NODE_SHOW_CONNECTIONS_STMT:
-    case QUERY_NODE_SHOW_USERS_STMT:
+    case QUERY_NODE_SHOW_LICENCE_STMT:
-    case QUERY_NODE_SHOW_DNODES_STMT:
+    case QUERY_NODE_SHOW_QUERIES_STMT:
-    case QUERY_NODE_SHOW_VGROUPS_STMT:
+    case QUERY_NODE_SHOW_SCORES_STMT:
-    case QUERY_NODE_SHOW_MNODES_STMT:
+    case QUERY_NODE_SHOW_TOPICS_STMT:
-    case QUERY_NODE_SHOW_QNODES_STMT:
+    case QUERY_NODE_SHOW_VARIABLE_STMT:
       code = translateShow(pCxt, (SShowStmt*)pNode);
       break;
-    case QUERY_NODE_SHOW_TABLES_STMT:
+    case QUERY_NODE_SHOW_CREATE_DATABASE_STMT:
-      code = translateShowTables(pCxt);
+    case QUERY_NODE_SHOW_CREATE_TABLE_STMT:
+    case QUERY_NODE_SHOW_CREATE_STABLE_STMT:
+      // todo
       break;
     case QUERY_NODE_CREATE_INDEX_STMT:
       code = translateCreateIndex(pCxt, (SCreateIndexStmt*)pNode);
@@ -2169,10 +2155,11 @@ typedef struct SVgroupTablesBatch {
   char               dbName[TSDB_DB_NAME_LEN];
 } SVgroupTablesBatch;
 
-static void toSchema(const SColumnDefNode* pCol, col_id_t colId, SSchema* pSchema) {
+static void toSchema(const SColumnDefNode* pCol, col_id_t colId, SSchemaEx* pSchema) {
   pSchema->colId = colId;
   pSchema->type = pCol->dataType.type;
   pSchema->bytes = calcTypeBytes(pCol->dataType);
+  pSchema->sma = TSDB_BSMA_TYPE_LATEST;  // TODO: use default value currently, and use the real value later.
   strcpy(pSchema->name, pCol->colName);
 }
 
@@ -2194,7 +2181,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const char* pDbName, con
   req.dbFName = strdup(dbFName);
   req.name = strdup(pTableName);
   req.ntbCfg.nCols = LIST_LENGTH(pColumns);
-  req.ntbCfg.pSchema = taosMemoryCalloc(req.ntbCfg.nCols, sizeof(SSchema));
+  req.ntbCfg.pSchema = taosMemoryCalloc(req.ntbCfg.nCols, sizeof(SSchemaEx));
   if (NULL == req.name || NULL == req.ntbCfg.pSchema) {
     destroyCreateTbReq(&req);
     return TSDB_CODE_OUT_OF_MEMORY;
@@ -2205,6 +2192,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const char* pDbName, con
     toSchema((SColumnDefNode*)pCol, index + 1, req.ntbCfg.pSchema + index);
     ++index;
   }
+  // TODO: use the real sma for normal table.
 
   pBatch->info = *pVgroupInfo;
   strcpy(pBatch->dbName, pDbName);

source/libs/planner/test/plannerTest.cpp

@@ -195,6 +195,12 @@ TEST_F(PlannerTest, interval) {
 
   bind("SELECT _wstartts, _wduration, _wendts, count(*) FROM t1 interval(10s)");
   ASSERT_TRUE(run());
+
+  bind("SELECT count(*) FROM t1 interval(10s) fill(linear)");
+  ASSERT_TRUE(run());
+
+  bind("SELECT count(*), sum(c1) FROM t1 interval(10s) fill(value, 10, 20)");
+  ASSERT_TRUE(run());
 }
 
 TEST_F(PlannerTest, sessionWindow) {

source/libs/scalar/src/sclfunc.c

@@ -23,7 +23,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
           colDataSetNull_f(pOutputData->nullbitmap, i);
           continue;
         }
-        out[i] = (in[i] > 0)? in[i] : -in[i];
+        out[i] = (in[i] >= 0)? in[i] : -in[i];
       }
       break;
     }
@@ -36,7 +36,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
           colDataSetNull_f(pOutputData->nullbitmap, i);
           continue;
         }
-        out[i] = (in[i] > 0)? in[i] : -in[i];
+        out[i] = (in[i] >= 0)? in[i] : -in[i];
       }
       break;
     }
@@ -49,7 +49,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
           colDataSetNull_f(pOutputData->nullbitmap, i);
           continue;
         }
-        out[i] = (in[i] > 0)? in[i] : -in[i];
+        out[i] = (in[i] >= 0)? in[i] : -in[i];
       }
       break;
     }
@@ -62,7 +62,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
           colDataSetNull_f(pOutputData->nullbitmap, i);
           continue;
         }
-        out[i] = (in[i] > 0)? in[i] : -in[i];
+        out[i] = (in[i] >= 0)? in[i] : -in[i];
       }
       break;
     }
@@ -75,7 +75,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
           colDataSetNull_f(pOutputData->nullbitmap, i);
           continue;
         }
-        out[i] = (in[i] > 0)? in[i] : -in[i];
+        out[i] = (in[i] >= 0)? in[i] : -in[i];
       }
       break;
     }
@@ -88,7 +88,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
           colDataSetNull_f(pOutputData->nullbitmap, i);
           continue;
         }
-        out[i] = (in[i] > 0)? in[i] : -in[i];
+        out[i] = (in[i] >= 0)? in[i] : -in[i];
       }
       break;
     }
@@ -407,4 +407,4 @@ int32_t winEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
   ASSERT(inputNum == 1);
   colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t*) colDataGetData(pInput->columnData, 4));
   return TSDB_CODE_SUCCESS;
-}
+}

source/libs/tdb/src/db/tdbBtree.c

@@ -29,15 +29,15 @@ struct SBTree {
   int            minLocal;
   int            maxLeaf;
   int            minLeaf;
-  u8            *pTmp;
+  void          *pBuf;
 };
 
 #define TDB_BTREE_PAGE_COMMON_HDR u8 flags;
 
-#define TDB_BTREE_PAGE_GET_FLAGS(PAGE) (PAGE)->pData[0]
+#define TDB_BTREE_PAGE_GET_FLAGS(PAGE)        (PAGE)->pData[0]
 #define TDB_BTREE_PAGE_SET_FLAGS(PAGE, flags) ((PAGE)->pData[0] = (flags))
-#define TDB_BTREE_PAGE_IS_ROOT(PAGE) (TDB_BTREE_PAGE_GET_FLAGS(PAGE) & TDB_BTREE_ROOT)
+#define TDB_BTREE_PAGE_IS_ROOT(PAGE)          (TDB_BTREE_PAGE_GET_FLAGS(PAGE) & TDB_BTREE_ROOT)
-#define TDB_BTREE_PAGE_IS_LEAF(PAGE) (TDB_BTREE_PAGE_GET_FLAGS(PAGE) & TDB_BTREE_LEAF)
+#define TDB_BTREE_PAGE_IS_LEAF(PAGE)          (TDB_BTREE_PAGE_GET_FLAGS(PAGE) & TDB_BTREE_LEAF)
 #define TDB_BTREE_ASSERT_FLAG(flags)                                                 \
   ASSERT(TDB_FLAG_IS(flags, TDB_BTREE_ROOT) || TDB_FLAG_IS(flags, TDB_BTREE_LEAF) || \
          TDB_FLAG_IS(flags, TDB_BTREE_ROOT | TDB_BTREE_LEAF) || TDB_FLAG_IS(flags, 0))
@@ -101,7 +101,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, S
   // pBt->kcmpr
   pBt->kcmpr = kcmpr ? kcmpr : tdbDefaultKeyCmprFn;
   // pBt->pageSize
-  pBt->pageSize = tdbPagerGetPageSize(pPager);
+  pBt->pageSize = pPager->pageSize;
   // pBt->maxLocal
   pBt->maxLocal = tdbPageCapacity(pBt->pageSize, sizeof(SIntHdr)) / 4;
   // pBt->minLocal: Should not be allowed smaller than 15, which is [nPayload][nKey][nData]
@@ -127,132 +127,144 @@ int tdbBtreeClose(SBTree *pBt) {
   return 0;
 }
 
-int tdbBtCursorInsert(SBTC *pBtc, const void *pKey, int kLen, const void *pVal, int vLen) {
+int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen) {
-  int     ret;
+  SBTC   btc;
-  int     idx;
+  SCell *pCell;
-  SPager *pPager;
+  void  *pBuf;
-  SCell  *pCell;
+  int    szCell;
-  int     szCell;
+  int    szBuf;
-  int     cret;
+  int    ret;
-  SBTree *pBt;
+  int    idx;
+  int    c;
 
-  ret = tdbBtcMoveTo(pBtc, pKey, kLen, &cret);
+  tdbBtcOpen(&btc, pBt);
+
+  // move to the position to insert
+  ret = tdbBtcMoveTo(&btc, pKey, kLen, &c);
   if (ret < 0) {
-    // TODO: handle error
+    tdbBtcClose(&btc);
+    ASSERT(0);
     return -1;
   }
 
-  if (pBtc->idx == -1) {
+  if (btc.idx == -1) {
-    ASSERT(TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0);
     idx = 0;
   } else {
-    if (cret > 0) {
+    if (c > 0) {
-      idx = pBtc->idx + 1;
+      idx = btc.idx + 1;
-    } else if (cret < 0) {
+    } else if (c < 0) {
-      idx = pBtc->idx;
+      idx = btc.idx;
     } else {
-      /* TODO */
+      // TDB does NOT allow same key
+      tdbBtcClose(&btc);
       ASSERT(0);
-    }
-  }
-
-  // TODO: refact code here
-  pBt = pBtc->pBt;
-  if (!pBt->pTmp) {
-    pBt->pTmp = (u8 *)tdbOsMalloc(pBt->pageSize);
-    if (pBt->pTmp == NULL) {
       return -1;
     }
   }
 
-  pCell = pBt->pTmp;
+  // make sure enough space to hold the cell
+  szBuf = kLen + vLen + 14;
+  pBuf = TDB_REALLOC(pBt->pBuf, pBt->pageSize > szBuf ? szBuf : pBt->pageSize);
+  if (pBuf == NULL) {
+    tdbBtcClose(&btc);
+    ASSERT(0);
+    return -1;
+  }
+  pBt->pBuf = pBuf;
+  pCell = (SCell *)pBt->pBuf;
 
-  // Encode the cell
+  // encode cell
-  ret = tdbBtreeEncodeCell(pBtc->pPage, pKey, kLen, pVal, vLen, pCell, &szCell);
+  ret = tdbBtreeEncodeCell(btc.pPage, pKey, kLen, pVal, vLen, pCell, &szCell);
   if (ret < 0) {
+    tdbBtcClose(&btc);
+    ASSERT(0);
     return -1;
   }
 
-  // Insert the cell to the index
+  // mark the page dirty
-  ret = tdbPageInsertCell(pBtc->pPage, idx, pCell, szCell, 0);
+  ret = tdbPagerWrite(pBt->pPager, btc.pPage);
   if (ret < 0) {
+    tdbBtcClose(&btc);
+    ASSERT(0);
     return -1;
   }
 
-  // If page is overflow, balance the tree
+  // insert the cell
-  if (pBtc->pPage->nOverflow > 0) {
+  ret = tdbPageInsertCell(btc.pPage, idx, pCell, szCell, 0);
-    ret = tdbBtreeBalance(pBtc);
+  if (ret < 0) {
+    tdbBtcClose(&btc);
+    ASSERT(0);
+    return -1;
+  }
+
+  // check if need balance
+  if (btc.pPage->nOverflow > 0) {
+    ret = tdbBtreeBalance(&btc);
     if (ret < 0) {
+      tdbBtcClose(&btc);
+      ASSERT(0);
       return -1;
     }
   }
 
+  tdbBtcClose(&btc);
+
   return 0;
 }
 
 int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen) {
-  SBTC         btc;
+  return tdbBtreePGet(pBt, pKey, kLen, NULL, NULL, ppVal, vLen);
-  SCell       *pCell;
-  int          cret;
-  void        *pVal;
-  SCellDecoder cd;
-
-  tdbBtcOpen(&btc, pBt);
-
-  tdbBtcMoveTo(&btc, pKey, kLen, &cret);
-
-  if (cret) {
-    return cret;
-  }
-
-  pCell = tdbPageGetCell(btc.pPage, btc.idx);
-  tdbBtreeDecodeCell(btc.pPage, pCell, &cd);
-
-  *vLen = cd.vLen;
-  pVal = TDB_REALLOC(*ppVal, *vLen);
-  if (pVal == NULL) {
-    return -1;
-  }
-
-  *ppVal = pVal;
-  memcpy(*ppVal, cd.pVal, cd.vLen);
-  return 0;
 }
 
 int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen) {
   SBTC         btc;
   SCell       *pCell;
   int          cret;
-  void        *pTKey;
+  int          ret;
-  void        *pTVal;
+  void        *pTKey = NULL;
+  void        *pTVal = NULL;
   SCellDecoder cd;
 
   tdbBtcOpen(&btc, pBt);
 
-  tdbBtcMoveTo(&btc, pKey, kLen, &cret);
+  ret = tdbBtcMoveTo(&btc, pKey, kLen, &cret);
+  if (ret < 0) {
+    tdbBtcClose(&btc);
+    ASSERT(0);
+  }
+
   if (cret) {
-    return cret;
+    tdbBtcClose(&btc);
+    return -1;
   }
 
   pCell = tdbPageGetCell(btc.pPage, btc.idx);
   tdbBtreeDecodeCell(btc.pPage, pCell, &cd);
 
-  pTKey = TDB_REALLOC(*ppKey, cd.kLen);
+  if (ppKey) {
-  pTVal = TDB_REALLOC(*ppVal, cd.vLen);
+    pTKey = TDB_REALLOC(*ppKey, cd.kLen);
-
+    if (pTKey == NULL) {
-  if (pTKey == NULL || pTVal == NULL) {
+      tdbBtcClose(&btc);
-    TDB_FREE(pTKey);
+      ASSERT(0);
-    TDB_FREE(pTVal);
+      return -1;
+    }
+    *ppKey = pTKey;
+    *pkLen = cd.kLen;
+    memcpy(*ppKey, cd.pKey, cd.kLen);
   }
 
-  *ppKey = pTKey;
+  pTVal = TDB_REALLOC(*ppVal, cd.vLen);
+  if (pTVal == NULL) {
+    tdbBtcClose(&btc);
+    ASSERT(0);
+    return -1;
+  }
   *ppVal = pTVal;
-  *pkLen = cd.kLen;
   *vLen = cd.vLen;
-
-  memcpy(*ppKey, cd.pKey, cd.kLen);
   memcpy(*ppVal, cd.pVal, cd.vLen);
 
+  tdbBtcClose(&btc);
+
   return 0;
 }
 
@@ -300,7 +312,8 @@ static int tdbBtreeOpenImpl(SBTree *pBt) {
     return -1;
   }
 
-  // TODO: Unref the page
+  // TODO: here still has problem
+  tdbPagerReturnPage(pBt->pPager, pPage);
 
   ASSERT(pgno != 0);
   pBt->root = pgno;
@@ -371,17 +384,7 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) {
   return 0;
 }
 
-#ifndef TDB_BTREE_BALANCE
+// TDB_BTREE_BALANCE =====================
-typedef struct {
-  SBTree *pBt;
-  SPage  *pParent;
-  int     idx;
-  i8      nOld;
-  SPage  *pOldPages[3];
-  i8      nNewPages;
-  SPage  *pNewPages[5];
-} SBtreeBalanceHelper;
-
 static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) {
   SPager           *pPager;
   SPage            *pChild;
@@ -408,6 +411,13 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) {
     ((SIntHdr *)pChild->pData)->pgno = ((SIntHdr *)(pRoot->pData))->pgno;
   }
 
+  ret = tdbPagerWrite(pPager, pChild);
+  if (ret < 0) {
+    // TODO
+    ASSERT(0);
+    return 0;
+  }
+
   // Copy the root page content to the child page
   tdbPageCopy(pRoot, pChild);
 
@@ -472,6 +482,13 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) {
         ASSERT(0);
         return -1;
       }
+
+      ret = tdbPagerWrite(pBt->pPager, pOlds[i]);
+      if (ret < 0) {
+        // TODO
+        ASSERT(0);
+        return -1;
+      }
     }
     // copy the parent key out if child pages are not leaf page
     childNotLeaf = !TDB_BTREE_PAGE_IS_LEAF(pOlds[0]);
@@ -492,6 +509,14 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) {
       }
       rPgno = ((SIntHdr *)pOlds[nOlds - 1]->pData)->pgno;
     }
+
+    ret = tdbPagerWrite(pBt->pPager, pParent);
+    if (ret < 0) {
+      // TODO
+      ASSERT(0);
+      return -1;
+    }
+
     // drop the cells on parent page
     for (int i = 0; i < nOlds; i++) {
       nCells = TDB_PAGE_TOTAL_CELLS(pParent);
@@ -619,6 +644,13 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) {
         if (ret < 0) {
           ASSERT(0);
         }
+
+        ret = tdbPagerWrite(pBt->pPager, pNews[iNew]);
+        if (ret < 0) {
+          // TODO
+          ASSERT(0);
+          return -1;
+        }
       }
     }
 
@@ -732,14 +764,24 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) {
     }
   }
 
+  // TODO: here is not corrent for drop case
+  for (int i = 0; i < nNews; i++) {
+    if (i < nOlds) {
+      tdbPagerReturnPage(pBt->pPager, pOlds[i]);
+    } else {
+      tdbPagerReturnPage(pBt->pPager, pNews[i]);
+    }
+  }
+
   return 0;
 }
 
 static int tdbBtreeBalance(SBTC *pBtc) {
   int    iPage;
+  int    ret;
+  int    nFree;
   SPage *pParent;
   SPage *pPage;
-  int    ret;
   u8     flags;
   u8     leaf;
   u8     root;
@@ -750,10 +792,11 @@ static int tdbBtreeBalance(SBTC *pBtc) {
     pPage = pBtc->pPage;
     leaf = TDB_BTREE_PAGE_IS_LEAF(pPage);
     root = TDB_BTREE_PAGE_IS_ROOT(pPage);
+    nFree = TDB_PAGE_FREE_SIZE(pPage);
 
     // when the page is not overflow and not too empty, the balance work
     // is finished. Just break out the balance loop.
-    if (pPage->nOverflow == 0 /* TODO: && pPage->nFree <= */) {
+    if (pPage->nOverflow == 0 && nFree < TDB_PAGE_USABLE_SIZE(pPage) * 2 / 3) {
       break;
     }
 
@@ -781,6 +824,8 @@ static int tdbBtreeBalance(SBTC *pBtc) {
         return -1;
       }
 
+      tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage);
+
       pBtc->iPage--;
       pBtc->pPage = pBtc->pgStack[pBtc->iPage];
     }
@@ -788,7 +833,7 @@ static int tdbBtreeBalance(SBTC *pBtc) {
 
   return 0;
 }
-#endif
+// TDB_BTREE_BALANCE
 
 // TDB_BTREE_CELL =====================
 static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const void *pKey, int kLen, const void *pVal,
@@ -1028,12 +1073,11 @@ int tdbBtcMoveToFirst(SBTC *pBtc) {
 
     // move upward
     for (;;) {
-      if (pBtc->iPage == 0) {
+      if (pBtc->iPage == iPage) {
         pBtc->idx = 0;
         break;
       }
 
-      if (pBtc->iPage < iPage) break;
       tdbBtcMoveUpward(pBtc);
     }
   }
@@ -1056,6 +1100,7 @@ int tdbBtcMoveToFirst(SBTC *pBtc) {
 
 int tdbBtcMoveToLast(SBTC *pBtc) {
   int     ret;
+  int     nCells;
   SBTree *pBt;
   SPager *pPager;
   SPgno   pgno;
@@ -1071,27 +1116,56 @@ int tdbBtcMoveToLast(SBTC *pBtc) {
       return -1;
     }
 
+    nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage);
     pBtc->iPage = 0;
+    if (nCells > 0) {
+      pBtc->idx = TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage) ? nCells - 1 : nCells;
+    } else {
+      // no data at all, point to an invalid position
+      ASSERT(TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage));
+      pBtc->idx = -1;
+      return 0;
+    }
   } else {
-    // move from a position
+    int iPage = 0;
-    ASSERT(0);
+
+    // downward search
+    for (; iPage < pBtc->iPage; iPage++) {
+      ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pBtc->pgStack[iPage]));
+      nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pgStack[iPage]);
+      if (pBtc->idxStack[iPage] != nCells) break;
+    }
+
+    // move upward
+    for (;;) {
+      if (pBtc->iPage == iPage) {
+        if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) {
+          pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage) - 1;
+        } else {
+          pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage);
+        }
+        break;
+      }
+
+      tdbBtcMoveUpward(pBtc);
+    }
   }
 
   // move downward
   for (;;) {
-    if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) {
+    if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) break;
-      // TODO: handle empty case
-      ASSERT(TDB_PAGE_TOTAL_CELLS(pBtc->pPage) > 0);
-      pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage) - 1;
-      break;
-    } else {
-      pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage);
 
-      ret = tdbBtcMoveDownward(pBtc);
+    ret = tdbBtcMoveDownward(pBtc);
-      if (ret < 0) {
+    if (ret < 0) {
-        ASSERT(0);
+      ASSERT(0);
-        return -1;
+      return -1;
-      }
+    }
+
+    nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage);
+    if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) {
+      pBtc->idx = nCells - 1;
+    } else {
+      pBtc->idx = nCells;
     }
   }
 
@@ -1104,6 +1178,7 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
   void        *pKey, *pVal;
   int          ret;
 
+  // current cursor points to an invalid position
   if (pBtc->idx < 0) {
     return -1;
   }
@@ -1134,12 +1209,17 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
   memcpy(pVal, cd.pVal, cd.vLen);
 
   ret = tdbBtcMoveToNext(pBtc);
+  if (ret < 0) {
+    ASSERT(0);
+    return -1;
+  }
 
   return 0;
 }
 
 static int tdbBtcMoveToNext(SBTC *pBtc) {
   int    nCells;
+  int    ret;
   SCell *pCell;
 
   ASSERT(TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage));
@@ -1151,39 +1231,35 @@ static int tdbBtcMoveToNext(SBTC *pBtc) {
     return 0;
   }
 
-  if (pBtc->iPage == 0) {
+  // move upward
-    pBtc->idx = -1;
-    return 0;
-  }
-
-  // Move upward
   for (;;) {
-    tdbBtcMoveUpward(pBtc);
-    pBtc->idx++;
-
-    nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage);
-    if (pBtc->idx <= nCells) {
-      break;
-    }
-
     if (pBtc->iPage == 0) {
       pBtc->idx = -1;
       return 0;
     }
-  }
 
-  // Move downward
+    tdbBtcMoveUpward(pBtc);
-  for (;;) {
+    pBtc->idx++;
-    nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage);
 
-    tdbBtcMoveDownward(pBtc);
+    ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage));
-    pBtc->idx = 0;
+    if (pBtc->idx <= TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) {
-
-    if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) {
       break;
     }
   }
 
+  // move downward
+  for (;;) {
+    if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) break;
+
+    ret = tdbBtcMoveDownward(pBtc);
+    if (ret < 0) {
+      ASSERT(0);
+      return -1;
+    }
+
+    pBtc->idx = 0;
+  }
+
   return 0;
 }
 
@@ -1230,91 +1306,145 @@ static int tdbBtcMoveUpward(SBTC *pBtc) {
 }
 
 static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) {
-  int     ret;
+  int          ret;
-  SBTree *pBt;
+  int          nCells;
-  SPager *pPager;
+  int          c;
+  SBTree      *pBt;
+  SCell       *pCell;
+  SPager      *pPager;
+  SCellDecoder cd = {0};
 
   pBt = pBtc->pBt;
   pPager = pBt->pPager;
 
   if (pBtc->iPage < 0) {
-    ASSERT(pBtc->iPage == -1);
+    // move from a clear cursor
-    ASSERT(pBtc->idx == -1);
-
-    // Move from the root
     ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt);
     if (ret < 0) {
+      // TODO
       ASSERT(0);
-      return -1;
-    }
-
-    pBtc->iPage = 0;
-
-    if (TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0) {
-      // Current page is empty
-      // ASSERT(TDB_FLAG_IS(TDB_PAGE_FLAGS(pBtc->pPage), TDB_BTREE_ROOT | TDB_BTREE_LEAF));
       return 0;
     }
 
-    for (;;) {
+    pBtc->iPage = 0;
-      int          lidx, ridx, midx, c, nCells;
+    pBtc->idx = -1;
-      SCell       *pCell;
+    // for empty tree, just return with an invalid position
-      SPage       *pPage;
+    if (TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0) return 0;
-      SCellDecoder cd = {0};
+  } else {
+    SPage *pPage;
+    int    idx;
+    int    iPage = 0;
 
-      pPage = pBtc->pPage;
+    // downward search
+    for (; iPage < pBtc->iPage; iPage++) {
+      pPage = pBtc->pgStack[iPage];
+      idx = pBtc->idxStack[iPage];
       nCells = TDB_PAGE_TOTAL_CELLS(pPage);
-      lidx = 0;
-      ridx = nCells - 1;
 
-      ASSERT(nCells > 0);
+      ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pPage));
 
-      for (;;) {
+      // check if key <= current position
-        if (lidx > ridx) break;
+      if (idx < nCells) {
-
+        pCell = tdbPageGetCell(pPage, idx);
-        midx = (lidx + ridx) >> 1;
+        tdbBtreeDecodeCell(pPage, pCell, &cd);
-
-        pCell = tdbPageGetCell(pPage, midx);
-        ret = tdbBtreeDecodeCell(pPage, pCell, &cd);
-        if (ret < 0) {
-          // TODO: handle error
-          ASSERT(0);
-          return -1;
-        }
-
-        // Compare the key values
         c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen);
-        if (c < 0) {
+        if (c > 0) break;
-          /* input-key < cell-key */
-          ridx = midx - 1;
-        } else if (c > 0) {
-          /* input-key > cell-key */
-          lidx = midx + 1;
-        } else {
-          /* input-key == cell-key */
-          break;
-        }
       }
 
-      // Move downward or break
+      // check if key > current - 1 position
-      u8 leaf = TDB_BTREE_PAGE_IS_LEAF(pPage);
+      if (idx > 0) {
-      if (leaf) {
+        pCell = tdbPageGetCell(pPage, idx - 1);
-        pBtc->idx = midx;
+        tdbBtreeDecodeCell(pPage, pCell, &cd);
-        *pCRst = c;
+        c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen);
-        break;
+        if (c <= 0) break;
-      } else {
-        if (c <= 0) {
-          pBtc->idx = midx;
-        } else {
-          pBtc->idx = midx + 1;
-        }
-        tdbBtcMoveDownward(pBtc);
       }
     }
 
-  } else {
+    // move upward
-    // TODO: Move the cursor from a some position instead of a clear state
+    for (;;) {
-    ASSERT(0);
+      if (pBtc->iPage == iPage) break;
+      tdbBtcMoveUpward(pBtc);
+    }
+  }
+
+  // search downward to the leaf
+  for (;;) {
+    int    lidx, ridx, midx;
+    SPage *pPage;
+
+    pPage = pBtc->pPage;
+    nCells = TDB_PAGE_TOTAL_CELLS(pPage);
+    lidx = 0;
+    ridx = nCells - 1;
+
+    ASSERT(nCells > 0);
+    ASSERT(pBtc->idx == -1);
+
+    // compare first cell
+    midx = lidx;
+    pCell = tdbPageGetCell(pPage, midx);
+    tdbBtreeDecodeCell(pPage, pCell, &cd);
+    c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen);
+    if (c <= 0) {
+      ridx = lidx - 1;
+    } else {
+      lidx = lidx + 1;
+    }
+
+    // compare last cell
+    if (lidx <= ridx) {
+      midx = ridx;
+      pCell = tdbPageGetCell(pPage, midx);
+      tdbBtreeDecodeCell(pPage, pCell, &cd);
+      c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen);
+      if (c >= 0) {
+        lidx = ridx + 1;
+      } else {
+        ridx = ridx - 1;
+      }
+    }
+
+    // binary search
+    for (;;) {
+      if (lidx > ridx) break;
+
+      midx = (lidx + ridx) >> 1;
+
+      pCell = tdbPageGetCell(pPage, midx);
+      ret = tdbBtreeDecodeCell(pPage, pCell, &cd);
+      if (ret < 0) {
+        // TODO: handle error
+        ASSERT(0);
+        return -1;
+      }
+
+      // Compare the key values
+      c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen);
+      if (c < 0) {
+        // pKey < cd.pKey
+        ridx = midx - 1;
+      } else if (c > 0) {
+        // pKey > cd.pKey
+        lidx = midx + 1;
+      } else {
+        // pKey == cd.pKey
+        break;
+      }
+    }
+
+    // keep search downward or break
+    if (TDB_BTREE_PAGE_IS_LEAF(pPage)) {
+      pBtc->idx = midx;
+      *pCRst = c;
+      break;
+    } else {
+      if (c <= 0) {
+        pBtc->idx = midx;
+      } else {
+        pBtc->idx = midx + 1;
+      }
+      tdbBtcMoveDownward(pBtc);
+    }
   }
 
   return 0;

source/libs/tdb/src/db/tdbDb.c

@@ -49,6 +49,8 @@ int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprF
     if (ret < 0) {
       return -1;
     }
+
+    tdbEnvAddPager(pEnv, pPager);
   }
 
   ASSERT(pPager != NULL);
@@ -74,22 +76,7 @@ int tdbDbDrop(TDB *pDb) {
 }
 
 int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen) {
-  SBTC  btc;
+  return tdbBtreeInsert(pDb->pBt, pKey, keyLen, pVal, valLen);
-  SBTC *pCur;
-  int   ret;
-
-  pCur = &btc;
-  ret = tdbBtcOpen(pCur, pDb->pBt);
-  if (ret < 0) {
-    return -1;
-  }
-
-  ret = tdbBtCursorInsert(pCur, pKey, keyLen, pVal, valLen);
-  if (ret < 0) {
-    return -1;
-  }
-
-  return 0;
 }
 
 int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) {

source/libs/tdb/src/db/tdbEnv.c

@@ -19,6 +19,7 @@ int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, TENV **ppEnv) {
   TENV *pEnv;
   int   dsize;
   int   zsize;
+  int   tsize;
   u8   *pPtr;
   int   ret;
 
@@ -53,6 +54,14 @@ int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, TENV **ppEnv) {
     return -1;
   }
 
+  pEnv->nPgrHash = 8;
+  tsize = sizeof(SPager *) * pEnv->nPgrHash;
+  pEnv->pgrHash = TDB_REALLOC(pEnv->pgrHash, tsize);
+  if (pEnv->pgrHash == NULL) {
+    return -1;
+  }
+  memset(pEnv->pgrHash, 0, tsize);
+
   mkdir(rootDir, 0755);
 
   *ppEnv = pEnv;
@@ -64,7 +73,99 @@ int tdbEnvClose(TENV *pEnv) {
   return 0;
 }
 
+int tdbBegin(TENV *pEnv) {
+  SPager *pPager;
+  int     ret;
+
+  for (pPager = pEnv->pgrList; pPager; pPager = pPager->pNext) {
+    ret = tdbPagerBegin(pPager);
+    if (ret < 0) {
+      ASSERT(0);
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+int tdbCommit(TENV *pEnv) {
+  SPager *pPager;
+  int     ret;
+
+  for (pPager = pEnv->pgrList; pPager; pPager = pPager->pNext) {
+    ret = tdbPagerCommit(pPager);
+    if (ret < 0) {
+      ASSERT(0);
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+int tdbRollback(TENV *pEnv) {
+  ASSERT(0);
+  return 0;
+}
+
 SPager *tdbEnvGetPager(TENV *pEnv, const char *fname) {
-  // TODO
+  u32      hash;
-  return NULL;
+  SPager **ppPager;
+
+  hash = tdbCstringHash(fname);
+  ppPager = &pEnv->pgrHash[hash % pEnv->nPgrHash];
+  for (; *ppPager && (strcmp(fname, (*ppPager)->dbFileName) != 0); ppPager = &((*ppPager)->pHashNext)) {
+  }
+
+  return *ppPager;
+}
+
+void tdbEnvAddPager(TENV *pEnv, SPager *pPager) {
+  u32      hash;
+  SPager **ppPager;
+
+  // rehash if neccessary
+  if (pEnv->nPager + 1 > pEnv->nPgrHash) {
+    // TODO
+  }
+
+  // add to list
+  pPager->pNext = pEnv->pgrList;
+  pEnv->pgrList = pPager;
+
+  // add to hash
+  hash = tdbCstringHash(pPager->dbFileName);
+  ppPager = &pEnv->pgrHash[hash % pEnv->nPgrHash];
+  pPager->pHashNext = *ppPager;
+  *ppPager = pPager;
+
+  // increase the counter
+  pEnv->nPager++;
+}
+
+void tdbEnvRemovePager(TENV *pEnv, SPager *pPager) {
+  u32      hash;
+  SPager **ppPager;
+
+  // remove from the list
+  for (ppPager = &pEnv->pgrList; *ppPager && (*ppPager != pPager); ppPager = &((*ppPager)->pNext)) {
+  }
+  ASSERT(*ppPager == pPager);
+  *ppPager = pPager->pNext;
+
+  // remove from hash
+  hash = tdbCstringHash(pPager->dbFileName);
+  ppPager = &pEnv->pgrHash[hash % pEnv->nPgrHash];
+  for (; *ppPager && *ppPager != pPager; ppPager = &((*ppPager)->pHashNext)) {
+  }
+  ASSERT(*ppPager == pPager);
+  *ppPager = pPager->pNext;
+
+  // decrease the counter
+  pEnv->nPager--;
+
+  // rehash if necessary
+  if (pEnv->nPgrHash > 8 && pEnv->nPager < pEnv->nPgrHash / 2) {
+    // TODO
+  }
 }

source/libs/tdb/src/db/tdbPCache.c

@@ -34,18 +34,6 @@ struct SPCache {
   })
 #define PAGE_IS_PINNED(pPage) ((pPage)->pLruNext == NULL)
 
-// For page ref
-#define TDB_INIT_PAGE_REF(pPage) ((pPage)->nRef = 0)
-#if 0
-#define TDB_REF_PAGE(pPage)     (++(pPage)->nRef)
-#define TDB_UNREF_PAGE(pPage)   (--(pPage)->nRef)
-#define TDB_GET_PAGE_REF(pPage) ((pPage)->nRef)
-#else
-#define TDB_REF_PAGE(pPage)     atomic_add_fetch_32(&((pPage)->nRef), 1)
-#define TDB_UNREF_PAGE(pPage)   atomic_sub_fetch_32(&((pPage)->nRef), 1)
-#define TDB_GET_PAGE_REF(pPage) atomic_load_32(&((pPage)->nRef))
-#endif
-
 static int    tdbPCacheOpenImpl(SPCache *pCache);
 static void   tdbPCacheInitLock(SPCache *pCache);
 static void   tdbPCacheClearLock(SPCache *pCache);
@@ -107,12 +95,7 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage) {
   ASSERT(nRef >= 0);
 
   if (nRef == 0) {
-    if (1 /*TODO: page still clean*/) {
+    tdbPCacheUnpinPage(pCache, pPage);
-      tdbPCacheUnpinPage(pCache, pPage);
-    } else {
-      // TODO
-      ASSERT(0);
-    }
   }
 }
 
@@ -192,6 +175,8 @@ static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) {
 
   tdbPCacheLock(pCache);
 
+  ASSERT(!pPage->isDirty);
+
   nRef = TDB_GET_PAGE_REF(pPage);
   ASSERT(nRef >= 0);
   if (nRef == 0) {

source/libs/tdb/src/db/tdbPager.c

@@ -15,20 +15,6 @@
 
 #include "tdbInt.h"
 
-struct SPager {
-  char    *dbFileName;
-  char    *jFileName;
-  int      pageSize;
-  uint8_t  fid[TDB_FILE_ID_LEN];
-  tdb_fd_t fd;
-  tdb_fd_t jfd;
-  SPCache *pCache;
-  SPgno    dbFileSize;
-  SPgno    dbOrigSize;
-  SPage   *pDirty;
-  u8       inTran;
-};
-
 typedef struct __attribute__((__packed__)) {
   u8    hdrString[16];
   u16   pageSize;
@@ -41,9 +27,8 @@ TDB_STATIC_ASSERT(sizeof(SFileHdr) == 128, "Size of file header is not correct")
 
 #define TDB_PAGE_INITIALIZED(pPage) ((pPage)->pPager != NULL)
 
-static int tdbPagerReadPage(SPager *pPager, SPage *pPage);
 static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno);
-static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg);
+static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg, u8 loadPage);
 static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage);
 static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage);
 
@@ -131,26 +116,36 @@ int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate) {
 }
 
 int tdbPagerWrite(SPager *pPager, SPage *pPage) {
-  int ret;
+  int     ret;
+  SPage **ppPage;
 
+  ASSERT(pPager->inTran);
+#if 0
   if (pPager->inTran == 0) {
     ret = tdbPagerBegin(pPager);
     if (ret < 0) {
       return -1;
     }
   }
+#endif
 
   if (pPage->isDirty) return 0;
 
+  // ref page one more time so the page will not be release
+  TDB_REF_PAGE(pPage);
+
   // Set page as dirty
   pPage->isDirty = 1;
 
-  // Add page to dirty list
+  // Add page to dirty list(TODO: NOT use O(n^2) algorithm)
-  // TODO: sort the list according to the page number
+  for (ppPage = &pPager->pDirty; (*ppPage) && TDB_PAGE_PGNO(*ppPage) < TDB_PAGE_PGNO(pPage);
-  pPage->pDirtyNext = pPager->pDirty;
+       ppPage = &((*ppPage)->pDirtyNext)) {
-  pPager->pDirty = pPage;
+  }
+  ASSERT(*ppPage == NULL || TDB_PAGE_PGNO(*ppPage) > TDB_PAGE_PGNO(pPage));
+  pPage->pDirtyNext = *ppPage;
+  *ppPage = pPage;
 
-  // Write page to journal
+  // Write page to journal if neccessary
   if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize) {
     ret = tdbPagerWritePageToJournal(pPager, pPage);
     if (ret < 0) {
@@ -184,54 +179,46 @@ int tdbPagerCommit(SPager *pPager) {
   SPage *pPage;
   int    ret;
 
-  // Begin commit
+  // sync the journal file
-  {
+  ret = tdbOsFSync(pPager->jfd);
-    // TODO: Sync the journal file (Here or when write ?)
+  if (ret < 0) {
+    // TODO
+    ASSERT(0);
+    return 0;
   }
 
-  for (;;) {
+  // loop to write the dirty pages to file
-    pPage = pPager->pDirty;
+  for (pPage = pPager->pDirty; pPage; pPage = pPage->pDirtyNext) {
-
+    // TODO: update the page footer
-    if (pPage == NULL) break;
-
     ret = tdbPagerWritePageToDB(pPager, pPage);
     if (ret < 0) {
       ASSERT(0);
       return -1;
     }
+  }
 
+  // release the page
+  for (pPage = pPager->pDirty; pPage; pPage = pPager->pDirty) {
     pPager->pDirty = pPage->pDirtyNext;
     pPage->pDirtyNext = NULL;
 
-    // TODO: release the page
+    pPage->isDirty = 0;
+
+    tdbPCacheRelease(pPager->pCache, pPage);
   }
 
+  // sync the db file
   tdbOsFSync(pPager->fd);
 
+  // remote the journal file
   tdbOsClose(pPager->jfd);
   tdbOsRemove(pPager->jFileName);
-  // pPager->jfd = -1;
+  pPager->dbOrigSize = pPager->dbFileSize;
+  pPager->inTran = 0;
 
   return 0;
 }
 
-static int tdbPagerReadPage(SPager *pPager, SPage *pPage) {
-  i64 offset;
-  int ret;
-
-  ASSERT(memcmp(pPager->fid, pPage->pgid.fileid, TDB_FILE_ID_LEN) == 0);
-
-  offset = (pPage->pgid.pgno - 1) * (i64)(pPager->pageSize);
-  ret = tdbOsPRead(pPager->fd, pPage->pData, pPager->pageSize, offset);
-  if (ret < 0) {
-    // TODO: handle error
-    return -1;
-  }
-  return 0;
-}
-
-int tdbPagerGetPageSize(SPager *pPager) { return pPager->pageSize; }
-
 int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg) {
   SPage *pPage;
   SPgid  pgid;
@@ -247,7 +234,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage
 
   // Initialize the page if need
   if (!TDB_PAGE_INITIALIZED(pPage)) {
-    ret = tdbPagerInitPage(pPager, pPage, initPage, arg);
+    ret = tdbPagerInitPage(pPager, pPage, initPage, arg, 1);
     if (ret < 0) {
       return -1;
     }
@@ -284,7 +271,7 @@ int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage
   ASSERT(!TDB_PAGE_INITIALIZED(pPage));
 
   // Initialize the page if need
-  ret = tdbPagerInitPage(pPager, pPage, initPage, arg);
+  ret = tdbPagerInitPage(pPager, pPage, initPage, arg, 0);
   if (ret < 0) {
     return -1;
   }
@@ -332,10 +319,11 @@ static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno) {
   return 0;
 }
 
-static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg) {
+static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg, u8 loadPage) {
   int ret;
   int lcode;
   int nLoops;
+  i64 nRead;
 
   lcode = TDB_TRY_LOCK_PAGE(pPage);
   if (lcode == P_LOCK_SUCC) {
@@ -344,6 +332,19 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage
       return 0;
     }
 
+    if (loadPage) {
+      nRead = tdbOsPRead(pPager->fd, pPage->pData, pPage->pageSize, ((i64)pPage->pageSize) * TDB_PAGE_PGNO(pPage));
+      if (nRead < 0) {
+        // TODO
+        ASSERT(0);
+        return -1;
+      } else if (nRead < pPage->pageSize) {
+        // TODO
+        ASSERT(0);
+        return -1;
+      }
+    }
+
     ret = (*initPage)(pPage, arg);
     if (ret < 0) {
       TDB_UNLOCK_PAGE(pPage);

source/libs/tdb/src/db/tdbTxn.c

@@ -15,17 +15,29 @@
 
 #include "tdbInt.h"
 
-int tdbTxnBegin(TENV *pEnv) {
+// int tdbTxnBegin(TENV *pEnv) {
-  // TODO
+//   // TODO
-  return 0;
+//   return 0;
-}
+// }
 
-int tdbTxnCommit(TENV *pEnv) {
+// int tdbTxnCommit(TENV *pEnv) {
-  // TODO
+//   SPager *pPager = NULL;
-  return 0;
+//   int     ret;
-}
 
-int tdbTxnRollback(TENV *pEnv) {
+//   for (;;) {
-  // TODO
+//     break;
-  return 0;
+//     ret = tdbPagerCommit(pPager);
-}
+//     if (ret < 0) {
+//       ASSERT(0);
+//       return -1;
+//     }
+//   }
+
+//   // TODO
+//   return 0;
+// }
+
+// int tdbTxnRollback(TENV *pEnv) {
+//   // TODO
+//   return 0;
+// }

source/libs/tdb/src/inc/tdbBtree.h

@@ -40,7 +40,7 @@ struct SBTC {
 // SBTree
 int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, FKeyComparator kcmpr, SBTree **ppBt);
 int tdbBtreeClose(SBTree *pBt);
-int tdbBtCursorInsert(SBTC *pCur, const void *pKey, int kLen, const void *pVal, int vLen);
+int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen);
 int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen);
 int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen);
 

source/libs/tdb/src/inc/tdbEnv.h

@@ -25,11 +25,20 @@ typedef struct STEnv {
   char    *jfname;
   int      jfd;
   SPCache *pCache;
+  SPager  *pgrList;
+  int      nPager;
+  int      nPgrHash;
+  SPager **pgrHash;
 } TENV;
 
 int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, TENV **ppEnv);
 int tdbEnvClose(TENV *pEnv);
+int tdbBegin(TENV *pEnv);
+int tdbCommit(TENV *pEnv);
+int tdbRollback(TENV *pEnv);
 
+void    tdbEnvAddPager(TENV *pEnv, SPager *pPager);
+void    tdbEnvRemovePager(TENV *pEnv, SPager *pPager);
 SPager *tdbEnvGetPager(TENV *pEnv, const char *fname);
 
 #ifdef __cplusplus

source/libs/tdb/src/inc/tdbInt.h

@@ -16,8 +16,6 @@
 #ifndef _TD_TDB_INTERNAL_H_
 #define _TD_TDB_INTERNAL_H_
 
-#include "os.h"
-
 #include "tdb.h"
 
 #ifdef __cplusplus
@@ -91,23 +89,6 @@ static FORCE_INLINE int tdbCmprPgId(const void *p1, const void *p2) {
 // dbname
 #define TDB_MAX_DBNAME_LEN 24
 
-// tdb_log
-#define tdbError(var)
-
-#define TERR_A(val, op, flag)  \
-  do {                         \
-    if (((val) = (op)) != 0) { \
-      goto flag;               \
-    }                          \
-  } while (0)
-
-#define TERR_B(val, op, flag)     \
-  do {                            \
-    if (((val) = (op)) == NULL) { \
-      goto flag;                  \
-    }                             \
-  } while (0)
-
 #define TDB_VARIANT_LEN ((int)-1)
 
 typedef int (*FKeyComparator)(const void *pKey1, int kLen1, const void *pKey2, int kLen2);

source/libs/tdb/src/inc/tdbOs.h

@@ -24,6 +24,8 @@ extern "C" {
 #define TDB_FOR_TDENGINE
 
 #ifdef TDB_FOR_TDENGINE
+#include "os.h"
+#include "thash.h"
 
 // For memory -----------------
 #define tdbOsMalloc  taosMemoryMalloc
@@ -95,7 +97,11 @@ typedef int tdb_fd_t;
 
 #define tdbOsOpen(PATH, OPTION, MODE) open((PATH), (OPTION), (MODE))
 
-#define tdbOsClose close
+#define tdbOsClose(FD) \
+  do {                 \
+    close(FD);         \
+    (FD) = -1;         \
+  } while (0)
 
 i64 tdbOsRead(tdb_fd_t fd, void *pData, i64 nBytes);
 i64 tdbOsPRead(tdb_fd_t fd, void *pData, i64 nBytes, i64 offset);

source/libs/tdb/src/inc/tdbPCache.h

@@ -33,6 +33,18 @@ extern "C" {
   SPager *pPager;       \
   SPgid   pgid;
 
+// For page ref
+#define TDB_INIT_PAGE_REF(pPage) ((pPage)->nRef = 0)
+#if 0
+#define TDB_REF_PAGE(pPage)     (++(pPage)->nRef)
+#define TDB_UNREF_PAGE(pPage)   (--(pPage)->nRef)
+#define TDB_GET_PAGE_REF(pPage) ((pPage)->nRef)
+#else
+#define TDB_REF_PAGE(pPage)     atomic_add_fetch_32(&((pPage)->nRef), 1)
+#define TDB_UNREF_PAGE(pPage)   atomic_sub_fetch_32(&((pPage)->nRef), 1)
+#define TDB_GET_PAGE_REF(pPage) atomic_load_32(&((pPage)->nRef))
+#endif
+
 int    tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache);
 int    tdbPCacheClose(SPCache *pCache);
 SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, bool alcNewPage);

source/libs/tdb/src/inc/tdbPage.h

@@ -100,6 +100,7 @@ struct SPage {
 // APIs
 #define TDB_PAGE_TOTAL_CELLS(pPage)        ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage))
 #define TDB_PAGE_USABLE_SIZE(pPage)        ((u8 *)(pPage)->pPageFtr - (pPage)->pCellIdx)
+#define TDB_PAGE_FREE_SIZE(pPage)          (*(pPage)->pPageMethods->getFreeBytes)(pPage)
 #define TDB_PAGE_PGNO(pPage)               ((pPage)->pgid.pgno)
 #define TDB_BYTES_CELL_TAKEN(pPage, pCell) ((*(pPage)->xCellSize)(pPage, pCell) + (pPage)->pPageMethods->szOffset)
 #define TDB_PAGE_OFFSET_SIZE(pPage)        ((pPage)->pPageMethods->szOffset)

source/libs/tdb/src/inc/tdbPager.h

@@ -20,13 +20,28 @@
 extern "C" {
 #endif
 
+struct SPager {
+  char    *dbFileName;
+  char    *jFileName;
+  int      pageSize;
+  uint8_t  fid[TDB_FILE_ID_LEN];
+  tdb_fd_t fd;
+  tdb_fd_t jfd;
+  SPCache *pCache;
+  SPgno    dbFileSize;
+  SPgno    dbOrigSize;
+  SPage   *pDirty;
+  u8       inTran;
+  SPager  *pNext;      // used by TENV
+  SPager  *pHashNext;  // used by TENV
+};
+
 int  tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager);
 int  tdbPagerClose(SPager *pPager);
 int  tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate);
 int  tdbPagerWrite(SPager *pPager, SPage *pPage);
 int  tdbPagerBegin(SPager *pPager);
 int  tdbPagerCommit(SPager *pPager);
-int  tdbPagerGetPageSize(SPager *pPager);
 int  tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg);
 int  tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg);
 void tdbPagerReturnPage(SPager *pPager, SPage *pPage);

source/libs/tdb/src/inc/tdbTxn.h

@@ -28,10 +28,6 @@ struct STxn {
   void *xArg;
 };
 
-int tdbTxnBegin(TENV *pEnv);
-int tdbTxnCommit(TENV *pEnv);
-int tdbTxnRollback(TENV *pEnv);
-
 #ifdef __cplusplus
 }
 #endif

source/libs/tdb/src/inc/tdbUtil.h

@@ -101,6 +101,8 @@ static inline int tdbGetVarInt(const u8 *p, int *v) {
   return n;
 }
 
+static inline u32 tdbCstringHash(const char *s) { return MurmurHash3_32(s, strlen(s)); }
+
 #ifdef __cplusplus
 }
 #endif

source/libs/tdb/test/tdbTest.cpp

@@ -118,10 +118,10 @@ TEST(tdb_test, simple_test) {
   TENV          *pEnv;
   TDB           *pDb;
   FKeyComparator compFunc;
-  int            nData = 1000000;
+  int            nData = 50000000;
 
   // Open Env
-  ret = tdbEnvOpen("tdb", 4096, 256000, &pEnv);
+  ret = tdbEnvOpen("tdb", 4096, 64, &pEnv);
   GTEST_ASSERT_EQ(ret, 0);
 
   // Create a database
@@ -134,36 +134,23 @@ TEST(tdb_test, simple_test) {
     char val[64];
 
     {  // Insert some data
-      int       i = 1;
+      for (int i = 1; i <= nData;) {
-      SPoolMem *pPool;
+        tdbBegin(pEnv);
-      int       memPoolCapacity = 16 * 1024;
 
-      pPool = openPool();
+        for (int k = 0; k < 2000; k++) {
-
+          sprintf(key, "key%d", i);
-      tdbTxnBegin(pEnv);
+          sprintf(val, "value%d", i);
-
+          ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val));
-      for (;;) {
+          GTEST_ASSERT_EQ(ret, 0);
-        if (i > nData) break;
+          i++;
-
-        sprintf(key, "key%d", i);
-        sprintf(val, "value%d", i);
-        ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val));
-        GTEST_ASSERT_EQ(ret, 0);
-
-        if (pPool->size >= memPoolCapacity) {
-          tdbTxnCommit(pEnv);
-
-          clearPool(pPool);
-
-          tdbTxnBegin(pEnv);
         }
 
-        i++;
+        tdbCommit(pEnv);
       }
-
-      closePool(pPool);
     }
 
+    tdbCommit(pEnv);
+
     {  // Query the data
       void *pVal = NULL;
       int   vLen;
@@ -173,6 +160,7 @@ TEST(tdb_test, simple_test) {
         sprintf(val, "value%d", i);
 
         ret = tdbDbGet(pDb, key, strlen(key), &pVal, &vLen);
+        ASSERT(ret == 0);
         GTEST_ASSERT_EQ(ret, 0);
 
         GTEST_ASSERT_EQ(vLen, strlen(val));

source/os/src/osFile.c

@@ -294,7 +294,7 @@ int64_t taosCloseFile(TdFilePtr *ppFile) {
 #if FILE_WITH_LOCK
   taosThreadRwlockWrlock(&((*ppFile)->rwlock));
 #endif
-  if (ppFile == NULL || *ppFile == NULL || (*ppFile)->fd == -1) {
+  if (ppFile == NULL || *ppFile == NULL) {
     return 0;
   }
   if ((*ppFile)->fp != NULL) {

source/os/src/osMemory.c

@@ -23,15 +23,22 @@
 
 #define TD_MEMORY_STACK_TRACE_DEPTH 10
 
+typedef struct TdMemoryInfo *TdMemoryInfoPtr;
+
 typedef struct TdMemoryInfo {
   int32_t symbol;
   int32_t memorySize;
   void *stackTrace[TD_MEMORY_STACK_TRACE_DEPTH];     // gdb: disassemble /m 0xXXX
-} *TdMemoryInfoPtr , TdMemoryInfo;
+  // TdMemoryInfoPtr pNext;
+  // TdMemoryInfoPtr pPrev;
+} TdMemoryInfo;
+
+// static TdMemoryInfoPtr GlobalMemoryPtr = NULL;
 
 #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
-
+  #define tstrdup(str) _strdup(str)
 #else
+  #define tstrdup(str) strdup(str)
 
 #include<execinfo.h>
 
@@ -129,6 +136,26 @@ void *taosMemoryRealloc(void *ptr, int32_t size) {
 #endif
 }
 
+void *taosMemoryStrDup(void *ptr) {
+#ifdef USE_TD_MEMORY
+  if (ptr == NULL) return NULL;
+  
+  TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char*)ptr - sizeof(TdMemoryInfo));
+  assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL);
+
+  void *tmp = tstrdup((const char *)pTdMemoryInfo);
+  if (tmp == NULL) return NULL;
+  
+  memcpy(tmp, pTdMemoryInfo, sizeof(TdMemoryInfo));
+  taosBackTrace(((TdMemoryInfoPtr)tmp)->stackTrace,TD_MEMORY_STACK_TRACE_DEPTH);
+
+  return (char*)tmp  + sizeof(TdMemoryInfo);
+#else
+  return tstrdup((const char *)ptr);
+#endif
+}
+
+
 void taosMemoryFree(const void *ptr) {
   if (ptr == NULL) return;
 

source/os/src/osProc.c

@@ -23,9 +23,9 @@ int32_t taosNewProc(char **args) {
   int32_t pid = fork();
   if (pid == 0) {
     args[0] = tsProcPath;
-    close(STDIN_FILENO);
+    // close(STDIN_FILENO);
     close(STDOUT_FILENO);
-    close(STDERR_FILENO);
+    // close(STDERR_FILENO);
     return execvp(tsProcPath, args);
   } else {
     return pid;
@@ -33,7 +33,7 @@ int32_t taosNewProc(char **args) {
 }
 
 void taosWaitProc(int32_t pid) {
-  int32_t status = 0;
+  int32_t status = -1;
   waitpid(pid, &status, 0);
 }
 

source/os/src/osShm.c

@@ -17,10 +17,10 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 
-int32_t taosCreateShm(SShm* pShm, int32_t shmsize) {
+int32_t taosCreateShm(SShm* pShm, int32_t key, int32_t shmsize) {
   pShm->id = -1;
 
-  int32_t shmid = shmget(0X95279527, shmsize, IPC_CREAT | 0600);
+  int32_t shmid = shmget(0X95270000 + key, shmsize, IPC_CREAT | 0600);
   if (shmid < 0) {
     return -1;
   }

source/os/src/osSignal.c

@@ -59,7 +59,7 @@ void taosSetSignal(int32_t signum, FSignalHandler sigfp) {
   struct sigaction act;
   memset(&act, 0, sizeof(act));
 #if 1
-  act.sa_flags = SA_SIGINFO;
+  act.sa_flags = SA_SIGINFO | SA_RESTART;
   act.sa_sigaction = (FLinuxSignalHandler)sigfp;
 #else
   act.sa_handler = sigfp;

source/util/src/tlog.c

@@ -347,12 +347,16 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
   taosThreadMutexInit(&tsLogObj.logMutex, NULL);
 
   taosUmaskFile(0);
-  tsLogObj.logHandle->pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE);
+  TdFilePtr pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE);
 
-  if (tsLogObj.logHandle->pFile == NULL) {
+  if (pFile == NULL) {
     printf("\nfailed to open log file:%s, reason:%s\n", fileName, strerror(errno));
     return -1;
   }
+  TdFilePtr pOldFile = tsLogObj.logHandle->pFile;
+  tsLogObj.logHandle->pFile = pFile;
+  taosUnLockLogFile(pOldFile);
+  taosCloseFile(&pOldFile);
   taosLockLogFile(tsLogObj.logHandle->pFile);
 
   // only an estimate for number of lines

source/util/src/tprocess.c

@@ -434,7 +434,9 @@ int32_t taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen,
   return taosProcQueuePush(pProc->pChildQueue, pHead, headLen, pBody, bodyLen, ftype);
 }
 
-int32_t taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
+void taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
-                             ProcFuncType ftype) {
+                          ProcFuncType ftype) {
-  return taosProcQueuePush(pProc->pParentQueue, pHead, headLen, pBody, bodyLen, ftype);
+  while (taosProcQueuePush(pProc->pParentQueue, pHead, headLen, pBody, bodyLen, ftype) != 0) {
+    taosMsleep(1);
+  }
 }

tests/script/jenkins/basic.txt

@@ -23,8 +23,8 @@
 ./test.sh -f tsim/insert/null.sim
 
 # ---- parser
-#./test.sh -f tsim/parser/groupby-basic.sim
+./test.sh -f tsim/parser/groupby-basic.sim
-#./test.sh -f tsim/parser/fourArithmetic-basic.sim
+./test.sh -f tsim/parser/fourArithmetic-basic.sim
 
 # ---- query
 ./test.sh -f tsim/query/interval.sim
@@ -40,6 +40,8 @@
 # ---- tmq
 ./test.sh -f tsim/tmq/basic.sim
 ./test.sh -f tsim/tmq/basic1.sim
+#./test.sh -f tsim/tmq/oneTopic.sim
+#./test.sh -f tsim/tmq/multiTopic.sim
 
 # --- stable
 ./test.sh -f tsim/stable/disk.sim
@@ -50,4 +52,10 @@
 ./test.sh -f tsim/stable/values.sim
 ./test.sh -f tsim/stable/vnode3.sim
 
+
+# --- for multi process mode
+./test.sh -f tsim/user/basic1.sim -m
+./test.sh -f tsim/stable/vnode3.sim -m
+./test.sh -f tsim/tmq/basic.sim -m
+
 #======================b1-end===============

tests/script/runAllSimCases.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+##################################################
+# 
+# Do simulation test 
+#
+##################################################
+
+set -e
+#set -x
+
+while read line
+do
+  firstChar=`echo ${line:0:1}`
+  if [[ -n "$line" ]]  && [[ $firstChar != "#" ]]; then
+    echo "======== $line ========"
+    $line 
+  fi
+done < ./jenkins/basic.txt
+
+

tests/script/sh/deploy.sh

@@ -5,18 +5,12 @@ set +e
 
 echo "Executing deploy.sh"
 
-if [ $# != 4 ]; then
-  echo "argument list need input : "
-  echo "  -n nodeName"
-  echo "  -i nodePort"
-  exit 1
-fi
-
 UNAME_BIN=`which uname`
 OS_TYPE=`$UNAME_BIN`
 NODE_NAME=
 NODE=
-while getopts "n:i:" arg
+MULTIPROCESS=0
+while getopts "n:i:m" arg
 do
   case $arg in
     n)
@@ -25,6 +19,9 @@ do
     i)
       NODE=$OPTARG
       ;;
+    m)
+      MULTIPROCESS=1
+      ;;
     ?)
       echo "unkonw argument"
       ;;
@@ -145,5 +142,5 @@ echo "statusInterval         1"                  >> $TAOS_CFG
 echo "asyncLog               0"                  >> $TAOS_CFG
 echo "locale                 en_US.UTF-8"        >> $TAOS_CFG
 echo "telemetryReporting     0"                  >> $TAOS_CFG
-echo "multiProcess           0"                  >> $TAOS_CFG
+echo "multiProcess           ${MULTIPROCESS}"    >> $TAOS_CFG
 echo " "                                         >> $TAOS_CFG  

tests/script/test.sh

@@ -7,7 +7,6 @@
 ##################################################
 
 set +e
-#set -x
 
 FILE_NAME=
 RELEASE=0
@@ -16,7 +15,8 @@ VALGRIND=0
 UNIQUE=0
 UNAME_BIN=`which uname`
 OS_TYPE=`$UNAME_BIN`
-while getopts "f:avu" arg
+MULTIPROCESS=0
+while getopts "f:avum" arg
 do
   case $arg in
     f)
@@ -28,6 +28,9 @@ do
     u)
       UNIQUE=1
       ;;
+    m)
+      MULTIPROCESS=1
+      ;;
     ?)
       echo "unknow argument"
       ;;
@@ -125,8 +128,13 @@ if [ -n "$FILE_NAME" ]; then
     echo valgrind --tool=memcheck --leak-check=full --show-reachable=no  --track-origins=yes --show-leak-kinds=all  -v  --workaround-gcc296-bugs=yes  --log-file=${CODE_DIR}/../script/valgrind.log $PROGRAM -c $CFG_DIR -f $FILE_NAME
     valgrind --tool=memcheck --leak-check=full --show-reachable=no  --track-origins=yes --show-leak-kinds=all  -v  --workaround-gcc296-bugs=yes  --log-file=${CODE_DIR}/../script/valgrind.log $PROGRAM -c $CFG_DIR -f $FILE_NAME
   else
-    echo "ExcuteCmd:" $PROGRAM -c $CFG_DIR -f $FILE_NAME
+    if [[ $MULTIPROCESS -eq 1 ]];then
-    $PROGRAM -c $CFG_DIR -f $FILE_NAME
+      echo "ExcuteCmd(multiprocess):" $PROGRAM -m -c $CFG_DIR -f $FILE_NAME  
+      $PROGRAM -m -c $CFG_DIR -f $FILE_NAME
+    else
+      echo "ExcuteCmd(singleprocess):" $PROGRAM -c $CFG_DIR -f $FILE_NAME  
+      $PROGRAM -c $CFG_DIR -f $FILE_NAME
+    fi
   fi
 else
   echo "ExcuteCmd:" $PROGRAM -c $CFG_DIR -f basicSuite.sim

tests/script/tsim/db/alter_option.sim

@@ -138,7 +138,7 @@ sql_error alter database db ntables 0
 sql_error alter database db ntables 1
 sql_error alter database db ntables 10
 
-#print ============== modify replica
+#print ============== modify replica        # TD-14409
 sql_error alter database db replica 2
 sql_error alter database db replica 5
 sql_error alter database db replica -1
@@ -270,7 +270,7 @@ if $data12_db != 2 then
   return -1
 endi
 
-#sql_error alter database db wal 0     # TD-14436
+sql_error alter database db wal 0     # TD-14436
 sql_error alter database db wal 3
 sql_error alter database db wal 100
 sql_error alter database db wal -1

tests/script/tsim/parser/fourArithmetic-basic.sim

@@ -78,6 +78,10 @@ while $i < $tbNum
   $tstart = 1640966400000
 endw
 
+
+$loop_test = 0
+loop_test_pos:
+
 sql select ts, c2-c1, c3/c1, c4+c1, c1*9, c1%3 from ct0
 print ===> rows: $rows 
 print ===> $data00 $data01 $data02 $data03 $data04 $data05
@@ -107,4 +111,31 @@ endi
 if $data93 != 8.000000000 then
  return -1
 endi
+
+if $loop_test == 0 then
+  print =============== stop and restart taosd
+  system sh/exec.sh -n dnode1 -s stop -x SIGINT
+  system sh/exec.sh -n dnode1 -s start
+  
+  $loop_cnt = 0
+  check_dnode_ready_0:
+  	$loop_cnt = $loop_cnt + 1
+  	sleep 200
+  	if $loop_cnt == 10 then
+  	  print ====> dnode not ready!
+  		return -1
+  	endi
+  sql show dnodes
+  print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
+  if $data00 != 1 then
+    return -1
+  endi
+  if $data04 != ready then
+    goto check_dnode_ready_0
+  endi
+
+  $loop_test = 1  
+  goto loop_test_pos
+endi
+
 #system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/parser/groupby-basic.sim

@@ -45,7 +45,7 @@ $tstart = 1640966400000  # 2022-01-01 00:00:00.000
 print ==== create db, stable, ctables, insert data
 sql drop database if exists $db -x step1
 step1:
-sql create database if not exists $db keep 3650
+sql create database if not exists $db
 sql use $db
 
 sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))
@@ -112,25 +112,26 @@ print $data90 $data91 $data92 $data93
 if $rows != 10 then
   return -1
 endi
-#if $data00 != 10 then
+if $data00 != 10 then
-# return -1
+ return -1
-#endi
+endi
 if $data01 != 0 then
   return -1
 endi
-#if $data10 != 10 then
+if $data10 != 10 then
-#  return -1
+  return -1
-#endi
+endi
 if $data11 != 1 then
   return -1
 endi
-#if $data90 != 10 then
+if $data90 != 10 then
-#  return -1
+  return -1
-#endi
+endi
 if $data91 != 9 then
   return -1
 endi
 
+print ==== select first(ts),c1 from group_tb0 group by c1;
 sql select first(ts),c1 from group_tb0 group by c1;
 print rows: $rows
 print $data00 $data01 $data02 $data03
@@ -155,55 +156,65 @@ if $data91 != 9 then
   return -1
 endi
 
-print ============> filter not supported yet.
+print ==== select first(ts),c1 from interval(5m) group_tb0 group by c1;
-#sql select sum(c1), c1, avg(c1), min(c1), max(c2) from group_tb0 where c1 < 20 group by c1;
+sql select first(ts),c1 from group_tb0 group by c1;
-#if $row != 20 then
+print rows: $rows
-#  return -1
+print $data00 $data01 $data02 $data03
-#endi
+print $data10 $data11 $data12 $data13
-#
+print $data20 $data21 $data22 $data23
-#if $data00 != 0 then
+print $data80 $data81 $data82 $data83
-#  return -1
+print $data90 $data91 $data92 $data93
-#endi
+
-#
+return 
-#if $data01 != 0 then
+
-#  return -1
+sql select sum(c1), c1, avg(c1), min(c1), max(c2) from group_tb0 where c1 < 20 group by c1;
-#endi
+if $row != 20 then
-#
+  return -1
-#print $data02
+endi
-#if $data02 != 0.000000000 then
+
-#  return -1
+if $data00 != 0 then
-#endi
+  return -1
-#
+endi
-#if $data03 != 0 then
+
-#  return -1
+if $data01 != 0 then
-#endi
+  return -1
-#
+endi
-#print $data04
+
-#if $data04 != 0.00000 then
+print $data02
-#  return -1
+if $data02 != 0.000000000 then
-#endi
+  return -1
-#
+endi
-#if $data10 != 100 then
+
-#  return -1
+if $data03 != 0 then
-#endi
+  return -1
-#
+endi
-#if $data11 != 1 then
+
-#  return -1
+print $data04
-#endi
+if $data04 != 0.00000 then
-#
+  return -1
-#print $data12
+endi
-#if $data12 != 1.000000000 then
+
-#  return -1
+if $data10 != 100 then
-#endi
+  return -1
-#
+endi
-#if $data13 != 1 then
+
-#  return -1
+if $data11 != 1 then
-#endi
+  return -1
-#
+endi
-#if $data14 != 1.00000 then
+
-#  print expect 1.00000, actual:$data14
+print $data12
-#  return -1
+if $data12 != 1.000000000 then
-#endi
+  return -1
+endi
+
+if $data13 != 1 then
+  return -1
+endi
+
+if $data14 != 1.00000 then
+  print expect 1.00000, actual:$data14
+  return -1
+endi
 
 sql_error select sum(c1), ts, c1 from group_tb0 where c1<20 group by c1;
 sql_error select first(ts), ts, c2 from group_tb0 where c1 < 20 group by c1;

tests/script/tsim/query/scalarFunction.sim

@@ -202,7 +202,7 @@ if $rows != $rowNum then
 endi
 
 print ====> sin
-#sql select c1, sin(c1), sin(c1) * 3.14159265 / 180 from ct1  # TD-14426
+sql select c1, sin(c1), sin(c1) * 3.14159265 / 180 from ct1
 sql select c1, sin(c1), c2, sin(c2), c3, sin(c3) from ct1
 print ====> select c1, sin(c1), c2, sin(c2), c3, sin(c3) from ct1
 print ====> rows: $rows

tests/script/tsim/query/session.sim

@@ -0,0 +1,338 @@
+#### session windows
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1
+system sh/exec.sh -n dnode1 -s start
+
+$loop_cnt = 0
+check_dnode_ready:
+	$loop_cnt = $loop_cnt + 1
+	sleep 200
+	if $loop_cnt == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
+if $data00 != 1 then
+  return -1
+endi
+if $data04 != ready then
+  goto check_dnode_ready
+endi
+
+sql connect
+
+$vgroups  = 4
+$dbNamme  = d0
+
+print ====> create database d1 precision 'us' 
+sql create database d1 precision 'us'
+sql use d1
+sql create table dev_001 (ts timestamp ,i timestamp ,j int)
+sql insert into dev_001 values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4)
+sql create table secondts(ts timestamp,t2 timestamp,i int)
+sql insert into secondts values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4)
+
+print ====> create database $dbNamme vgroups $vgroups
+sql create database $dbNamme vgroups $vgroups
+sql show databases
+print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+#print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+
+sql use $dbNamme
+
+print =============== create super table, child table and insert data
+sql create table if not exists st (ts timestamp, tagtype int) tags(dev nchar(50), tag2 binary(16))
+sql create table if not exists dev_001 using st tags("dev_01", "tag_01")
+sql create table if not exists dev_002 using st tags("dev_02", "tag_02")
+
+sql INSERT INTO dev_001 VALUES('2020-05-13 10:00:00.000', 1)
+sql INSERT INTO dev_001 VALUES('2020-05-13 10:00:00.005', 2)
+sql INSERT INTO dev_001 VALUES('2020-05-13 10:00:00.011', 3)
+sql INSERT INTO dev_001 VALUES('2020-05-13 10:00:01.011', 4)
+sql INSERT INTO dev_001 VALUES('2020-05-13 10:00:01.611', 5)
+sql INSERT INTO dev_001 VALUES('2020-05-13 10:00:02.612', 6)
+sql INSERT INTO dev_001 VALUES('2020-05-13 10:01:02.612', 7)
+sql INSERT INTO dev_001 VALUES('2020-05-13 10:02:02.612', 8)
+sql INSERT INTO dev_001 VALUES('2020-05-13 10:03:02.613', 9)
+sql INSERT INTO dev_001 VALUES('2020-05-13 11:00:00.000', 10)
+sql INSERT INTO dev_001 VALUES('2020-05-13 12:00:00.000', 11)
+sql INSERT INTO dev_001 VALUES('2020-05-13 13:00:00.001', 12)
+sql INSERT INTO dev_001 VALUES('2020-05-14 13:00:00.001', 13)
+sql INSERT INTO dev_001 VALUES('2020-05-15 14:00:00.000', 14)
+sql INSERT INTO dev_001 VALUES('2020-05-20 10:00:00.000', 15)
+sql INSERT INTO dev_001 VALUES('2020-05-27 10:00:00.001', 16) 
+           
+sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.000', 1)
+sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.005', 2)
+sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.009', 3)
+sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.0021', 4)
+sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.031', 5)
+sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.036', 6)
+sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.51', 7)
+
+$loop_test = 0  
+loop_test_pos:
+
+# session(ts,5a)
+print ====> select count(*) from dev_001 session(ts,5a)
+sql select count(*) from dev_001 session(ts,5a)
+print ====> rows: $rows
+print ====> $data00 $data01 $data02 $data03 $data04 $data05
+print ====> $data10 $data11 $data12 $data13 $data14 $data15
+print ====> $data20 $data21 $data22 $data23 $data24 $data25
+print ====> $data30 $data31 $data32 $data33 $data34 $data35
+print ====> $data40 $data41 $data42 $data43 $data44 $data45
+print ====> $data50 $data51 $data52 $data53 $data54 $data55
+print ====> $data60 $data61 $data62 $data63 $data64 $data65
+print ====> $data70 $data71 $data72 $data73 $data74 $data75
+print ====> $data80 $data81 $data82 $data83 $data84 $data85
+print ====> $data90 $data91 $data92 $data93 $data94 $data95
+if $rows != 15 then
+  return -1
+endi
+if $data01 != 2 then
+  return -1
+endi
+
+
+print ====> select count(*) from (select * from dev_001) session(ts,5a)
+sql select count(*) from (select * from dev_001) session(ts,5a)
+if $rows != 15 then
+  return -1
+endi
+if $data01 != 2 then
+  return -1
+endi
+
+print ====> select count(*) from dev_001 session(ts,1s)
+sql select count(*) from dev_001 session(ts,1s)
+if $rows != 12 then
+  return -1
+endi
+if $data01 != 5 then
+  return -1
+endi
+
+print ====> select count(*) from (select * from dev_001) session(ts,1s)
+sql select count(*) from (select * from dev_001) session(ts,1s)
+if $rows != 12 then
+  return -1
+endi
+if $data01 != 5 then
+  return -1
+endi
+
+print ====> select count(*) from dev_001 session(ts,1000a)
+sql select count(*) from dev_001 session(ts,1000a)
+if $rows != 12 then
+  return -1
+endi
+if $data01 != 5 then
+  return -1
+endi
+
+print ====> select count(*) from  (select * from dev_001) session(ts,1000a)
+sql select count(*) from  (select * from dev_001) session(ts,1000a)
+if $rows != 12 then
+  return -1
+endi
+if $data01 != 5 then
+  return -1
+endi
+
+print ====> select count(*) from dev_001 session(ts,1m)
+sql select count(*) from dev_001 session(ts,1m)
+if $rows != 9 then
+  return -1
+endi
+if $data01 != 8 then
+  return -1
+endi
+
+print ====> select count(*) from (select * from dev_001) session(ts,1m)
+sql select count(*) from (select * from dev_001) session(ts,1m)
+if $rows != 9 then
+  return -1
+endi
+if $data01 != 8 then
+  return -1
+endi
+
+print ====> select count(*) from dev_001 session(ts,1h)
+sql select count(*) from dev_001 session(ts,1h)
+if $rows != 6 then
+  return -1
+endi
+if $data01 != 11 then
+  return -1
+endi
+
+print ====> select count(*) from (select * from dev_001) session(ts,1h)
+sql select count(*) from (select * from dev_001) session(ts,1h)
+if $rows != 6 then
+  return -1
+endi
+if $data01 != 11 then
+  return -1
+endi
+
+print ====> select count(*) from dev_001 session(ts,1d)
+sql select count(*) from dev_001 session(ts,1d)
+if $rows != 4 then
+  return -1
+endi
+if $data01 != 13 then
+  return -1
+endi
+
+print ====> select count(*) from (select * from dev_001) session(ts,1d)
+sql select count(*) from (select * from dev_001) session(ts,1d)
+if $rows != 4 then
+  return -1
+endi
+if $data01 != 13 then
+  return -1
+endi
+
+print ====> select count(*) from dev_001 session(ts,1w)
+sql select count(*) from dev_001 session(ts,1w)
+if $rows != 2 then
+  return -1
+endi
+if $data01 != 15 then
+  return -1
+endi
+
+print ====> select count(*) from (select * from dev_001) session(ts,1w)
+sql select count(*) from (select * from dev_001) session(ts,1w)
+if $rows != 2 then
+  return -1
+endi
+if $data01 != 15 then
+  return -1
+endi
+
+print ====> select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1),spread(tagtype),stddev(tagtype),percentile(tagtype,0)  from dev_001 where ts <'2020-05-20 0:0:0' session(ts,1d)
+sql select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1),spread(tagtype),stddev(tagtype),percentile(tagtype,0)  from dev_001 where ts <'2020-05-20 0:0:0' session(ts,1d)
+if $rows != 2 then
+  return -1
+endi
+if $data01 != 13 then
+  return -1
+endi
+if $data02 != 1 then
+  return -1
+endi
+if $data03 != 13 then
+  return -1
+endi
+if $data04 != 7 then
+  return -1
+endi
+if $data05 != 91 then
+  return -1
+endi
+if $data06 != 1 then
+  return -1
+endi
+if $data07 != 13 then
+  return -1
+endi
+if $data08 != @{slop:1.000000, intercept:0.000000}@ then
+  return -1
+endi
+if $data09 != 12 then
+  return -1
+endi
+# $data0-10 != 3.741657387
+# $data0-11 !=  1
+# $data1-11 !=  14
+
+print ====> select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1) from (select * from dev_001 where ts <'2020-05-20 0:0:0') session(ts,1d)
+sql select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1) from (select * from dev_001 where ts <'2020-05-20 0:0:0') session(ts,1d)
+if $rows != 2 then
+  return -1
+endi
+if $data01 != 13 then
+  return -1
+endi
+if $data02 != 1 then
+  return -1
+endi
+if $data03 != 13 then
+  return -1
+endi
+if $data04 != 7 then
+  return -1
+endi
+if $data05 != 91 then
+  return -1
+endi
+if $data06 != 1 then
+  return -1
+endi
+if $data07 != 13 then
+  return -1
+endi
+if $data08 != @{slop:1.000000, intercept:0.000000}@ then
+  return -1
+endi
+
+sql_error select * from dev_001 session(ts,1w)
+sql_error select count(*) from st session(ts,1w)
+sql_error select count(*) from dev_001 group by tagtype session(ts,1w)
+sql_error select count(*) from dev_001 session(ts,1n)
+sql_error select count(*) from dev_001 session(ts,1y)
+sql_error select count(*) from dev_001 session(ts,0s)
+sql_error select count(*) from dev_001 session(i,1y)
+sql_error select count(*) from dev_001 session(ts,1d) where ts <'2020-05-20 0:0:0'
+
+print ====> create database d1 precision 'us' 
+sql create database d1 precision 'us'
+sql use d1
+sql create table dev_001 (ts timestamp ,i timestamp ,j int)
+sql insert into dev_001 values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4)
+print ====> select count(*) from dev_001 session(ts,1u)
+sql select count(*) from dev_001 session(ts,1u)
+if $rows != 2 then
+  return -1
+endi
+if $data01 != 3 then
+  return -1
+endi
+sql_error select count(*) from dev_001 session(i,1s)
+sql create table secondts(ts timestamp,t2 timestamp,i int)
+sql_error select count(*) from secondts session(t2,2s)
+
+
+if $loop_test == 0 then
+  print =============== stop and restart taosd
+  system sh/exec.sh -n dnode1 -s stop -x SIGINT
+  system sh/exec.sh -n dnode1 -s start
+  
+  $loop_cnt = 0
+  check_dnode_ready_0:
+  	$loop_cnt = $loop_cnt + 1
+  	sleep 200
+  	if $loop_cnt == 10 then
+  	  print ====> dnode not ready!
+  		return -1
+  	endi
+  sql show dnodes
+  print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
+  if $data00 != 1 then
+    return -1
+  endi
+  if $data04 != ready then
+    goto check_dnode_ready_0
+  endi
+
+  $loop_test = 1  
+  goto loop_test_pos
+endi
+
+#system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/test-all.sh

@@ -91,20 +91,23 @@ function runSimCaseOneByOnefq {
   for ((i=$start;i<=$end;i++)) ; do
     line=`sed -n "$i"p jenkins/basic.txt`
     if [[ $line =~ ^./test.sh* ]] || [[ $line =~ ^run* ]]; then
-      case=`echo $line | grep sim$ |awk '{print $NF}'`
+      #case=`echo $line | grep sim$ |awk '{print $NF}'`
+      case=`echo $line | grep -o ".*\.sim" |awk '{print $NF}'`
 
       start_time=`date +%s`
       date +%F\ %T | tee -a out.log
       if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then
-        echo -n $case
+        #echo -n $case
-        ./test.sh -f $case > case.log 2>&1 \
+        echo -n $line
+        $line > case.log 2>&1 \
             && \
         ([ -f ../../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log  ||  echo -e "${GREEN} success${NC}" | tee -a out.log )|| \
         ([ -f ../../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log )  || \
         ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat case.log )
       else
-        echo -n $case
+        #echo -n $case
-        ./test.sh -f $case > ../../sim/case.log 2>&1 && \
+        echo -n $line
+        $line > ../../sim/case.log 2>&1 && \
         ([ -f ../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log  ||  echo -e "${GREEN} success${NC}" | tee -a out.log )|| \
         ([ -f ../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log )  || \
         ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' &&  pwd && cat ../../sim/case.log )

tests/tsim/inc/simInt.h

@@ -155,6 +155,7 @@ extern int32_t  simScriptSucced;
 extern int32_t  simDebugFlag;
 extern char     simScriptDir[];
 extern bool     abortExecution;
+extern bool     useMultiProcess;
 
 SScript *simParseScript(char *fileName);
 SScript *simProcessCallOver(SScript *script);

tests/tsim/src/simExe.c

@@ -305,25 +305,24 @@ bool simExecuteRunBackCmd(SScript *script, char *option) {
   return true;
 }
 
-void simReplaceShToBat(char *dst) {
+void simReplaceStr(char *buf, char *src, char *dst) {
-  char *sh = strstr(dst, ".sh");
+  char *begin = strstr(buf, src);
-  if (sh != NULL) {
+  if (begin != NULL) {
+    int32_t srcLen = (int32_t)strlen(src);
     int32_t dstLen = (int32_t)strlen(dst);
-    char   *end = dst + dstLen;
+    int32_t interval = (dstLen - srcLen);
-    *(end + 1) = 0;
+    int32_t remainLen = (int32_t)strlen(buf);
+    char   *end = buf + remainLen;
+    *(end + interval) = 0;
 
-    for (char *p = end; p >= sh; p--) {
+    for (char *p = end; p >= begin; p--) {
-      *(p + 1) = *p;
+      *(p + interval) = *p;
     }
 
-    sh[0] = '.';
+    memcpy(begin, dst, dstLen);
-    sh[1] = 'b';
-    sh[2] = 'a';
-    sh[3] = 't';
-    sh[4] = ' ';
   }
 
-  simDebug("system cmd is %s", dst);
+  simInfo("system cmd is %s", buf);
 }
 
 bool simExecuteSystemCmd(SScript *script, char *option) {
@@ -334,9 +333,13 @@ bool simExecuteSystemCmd(SScript *script, char *option) {
   simVisuallizeOption(script, option, buf + strlen(buf));
 #else
   sprintf(buf, "%s%s", simScriptDir, option);
-  simReplaceShToBat(buf);
+  simReplaceStr(buf, ".sh", ".bat");
 #endif
 
+  if (useMultiProcess) {
+    simReplaceStr(buf, "deploy.sh", "deploy.sh -m");
+  }
+
   simLogSql(buf, true);
   int32_t code = system(buf);
   int32_t repeatTimes = 0;

tests/tsim/src/simMain.c

@@ -18,6 +18,7 @@
 
 bool simExecSuccess = false;
 bool abortExecution = false;
+bool useMultiProcess = false;
 
 void simHandleSignal(int32_t signo, void *sigInfo, void *context) {
   simSystemCleanUp();
@@ -32,6 +33,8 @@ int32_t main(int32_t argc, char *argv[]) {
       tstrncpy(configDir, argv[++i], 128);
     } else if (strcmp(argv[i], "-f") == 0 && i < argc - 1) {
       strcpy(scriptFile, argv[++i]);
+    } else if (strcmp(argv[i], "-m") == 0) {
+      useMultiProcess = true;
     } else {
       printf("usage: %s [options] \n", argv[0]);
       printf("       [-c config]: config directory, default is: %s\n", configDir);