Merge branch '3.0' into feature/stream

cmake/cmake.define

@@ -14,25 +14,6 @@ MESSAGE(STATUS "Project binary files output path: " ${PROJECT_BINARY_DIR})
 MESSAGE(STATUS "Project executable files output path: " ${EXECUTABLE_OUTPUT_PATH})
 MESSAGE(STATUS "Project library files output path: " ${LIBRARY_OUTPUT_PATH})
 
-find_package(Git QUIET)
-if(GIT_FOUND AND EXISTS "${TD_SOURCE_DIR}/.git")
-# Update submodules as needed
-    option(GIT_SUBMODULE "Check submodules during build" ON)
-    if(GIT_SUBMODULE)
-        message(STATUS "Submodule update")
-        execute_process(COMMAND cd ${TD_SOURCE_DIR} && ${GIT_EXECUTABLE} submodule update --init --recursive
-                        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-                        RESULT_VARIABLE GIT_SUBMOD_RESULT)
-        if(NOT GIT_SUBMOD_RESULT EQUAL "0")
-            message(WARNING "git submodule update --init --recursive failed with ${GIT_SUBMOD_RESULT}, please checkout submodules")
-        endif()
-    endif()
-endif()
-
-if(NOT EXISTS "${TD_SOURCE_DIR}/tools/taos-tools/CMakeLists.txt")
-    message(WARNING "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules manually if you need build them.")
-endif()
-
 if (NOT DEFINED TD_GRANT)
   SET(TD_GRANT FALSE)
 endif()

cmake/cmake.options

@@ -78,6 +78,12 @@ option(
     OFF
 )
 
+option(
+    BUILD_WITH_BDB
+    "If build with BDB" 
+    OFF
+)
+
 option(
     BUILD_WITH_LUCENE 
     "If build with lucene" 

contrib/CMakeLists.txt

@@ -78,9 +78,9 @@ if(${BUILD_WITH_UV})
 endif(${BUILD_WITH_UV})
 
 # bdb
-#if(${BUILD_WITH_BDB})
+if(${BUILD_WITH_BDB})
-    #cat("${TD_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    cat("${TD_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
-#endif(${BUILD_WITH_BDB})
+endif(${BUILD_WITH_BDB})
 
 # sqlite
 if(${BUILD_WITH_SQLITE})

contrib/test/CMakeLists.txt

@@ -7,9 +7,9 @@ if(${BUILD_WITH_LUCENE})
     add_subdirectory(lucene)
 endif(${BUILD_WITH_LUCENE})
 
-#if(${BUILD_WITH_BDB})
+if(${BUILD_WITH_BDB})
-    #add_subdirectory(bdb)
+    add_subdirectory(bdb)
-#endif(${BUILD_WITH_BDB})
+endif(${BUILD_WITH_BDB})
 
 if(${BUILD_WITH_SQLITE})
     add_subdirectory(sqlite)

include/common/tmsg.h

@@ -70,7 +70,7 @@ typedef uint16_t tmsg_t;
 #define TSDB_IE_TYPE_DNODE_EXT   6
 #define TSDB_IE_TYPE_DNODE_STATE 7
 
-enum { CONN_TYPE__QUERY = 1, CONN_TYPE__TMQ, CONN_TYPE__MAX };
+enum { CONN_TYPE__QUERY = 1, CONN_TYPE__TMQ, CONN_TYPE__UDFD, CONN_TYPE__MAX };
 
 enum {
   HEARTBEAT_KEY_USER_AUTHINFO = 1,
@@ -1670,6 +1670,7 @@ typedef struct SVDropStbReq {
 int32_t tEncodeSVDropStbReq(SEncoder* pCoder, const SVDropStbReq* pReq);
 int32_t tDecodeSVDropStbReq(SDecoder* pCoder, SVDropStbReq* pReq);
 
+// TDMT_VND_CREATE_TABLE ==============
 #define TD_CREATE_IF_NOT_EXISTS 0x1
 typedef struct SVCreateTbReq {
   int32_t     flags;
@@ -1759,6 +1760,43 @@ typedef struct {
 int32_t tEncodeSVDropTbBatchRsp(SEncoder* pCoder, const SVDropTbBatchRsp* pRsp);
 int32_t tDecodeSVDropTbBatchRsp(SDecoder* pCoder, SVDropTbBatchRsp* pRsp);
 
+// TDMT_VND_ALTER_TABLE =====================
+typedef struct {
+  const char* tbName;
+  int8_t      action;
+  const char* colName;
+  // TSDB_ALTER_TABLE_ADD_COLUMN
+  int8_t  type;
+  int8_t  flags;
+  int32_t bytes;
+  // TSDB_ALTER_TABLE_DROP_COLUMN
+  // TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES
+  int32_t colModBytes;
+  // TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME
+  const char* colNewName;
+  // TSDB_ALTER_TABLE_UPDATE_TAG_VAL
+  const char*    tagName;
+  int8_t         isNull;
+  uint32_t       nTagVal;
+  const uint8_t* pTagVal;
+  // TSDB_ALTER_TABLE_UPDATE_OPTIONS
+  int8_t      updateTTL;
+  int32_t     newTTL;
+  int8_t      updateComment;
+  const char* newComment;
+} SVAlterTbReq;
+
+int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq);
+int32_t tDecodeSVAlterTbReq(SDecoder* pDecoder, SVAlterTbReq* pReq);
+
+typedef struct {
+  int32_t code;
+} SVAlterTbRsp;
+
+int32_t tEncodeSVAlterTbRsp(SEncoder* pEncoder, const SVAlterTbRsp* pRsp);
+int32_t tDecodeSVAlterTbRsp(SDecoder* pDecoder, SVAlterTbRsp* pRsp);
+// ======================
+
 typedef struct {
   SMsgHead head;
   int64_t  uid;

include/libs/function/tudf.h

@@ -121,6 +121,8 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx);
 int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock);
 
 int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, SScalarParam *output);
+
+int32_t cleanUpUdfs();
 // end API to taosd and qworker
 //=============================================================================================================================
 // begin API to UDF writer.

include/libs/sync/sync.h

@@ -144,6 +144,7 @@ int32_t     syncInit();
 void        syncCleanUp();
 int64_t     syncOpen(const SSyncInfo* pSyncInfo);
 void        syncStart(int64_t rid);
+void        syncStartStandBy(int64_t rid);
 void        syncStop(int64_t rid);
 int32_t     syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
 ESyncState  syncGetMyRole(int64_t rid);

include/util/taoserror.h

@@ -324,6 +324,9 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_VND_SMA_NOT_EXIST             TAOS_DEF_ERROR_CODE(0, 0x0516)
 #define TSDB_CODE_VND_HASH_MISMATCH             TAOS_DEF_ERROR_CODE(0, 0x0517)
 #define TSDB_CODE_VND_TABLE_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x0518)
+#define TSDB_CODE_VND_INVALID_TABLE_ACTION      TAOS_DEF_ERROR_CODE(0, 0x0519)
+#define TSDB_CODE_VND_COL_ALREADY_EXISTS        TAOS_DEF_ERROR_CODE(0, 0x051a)
+#define TSDB_CODE_VND_TABLE_COL_NOT_EXISTS      TAOS_DEF_ERROR_CODE(0, 0x051b)
 
 // tsdb
 #define TSDB_CODE_TDB_INVALID_TABLE_ID          TAOS_DEF_ERROR_CODE(0, 0x0600)
@@ -640,6 +643,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INVALID_INTERNAL_PK       TAOS_DEF_ERROR_CODE(0, 0x2646)
 #define TSDB_CODE_PAR_INVALID_TIMELINE_FUNC     TAOS_DEF_ERROR_CODE(0, 0x2647)
 #define TSDB_CODE_PAR_INVALID_PASSWD            TAOS_DEF_ERROR_CODE(0, 0x2648)
+#define TSDB_CODE_PAR_INVALID_ALTER_TABLE       TAOS_DEF_ERROR_CODE(0, 0x2649)
 
 //planner
 #define TSDB_CODE_PLAN_INTERNAL_ERROR           TAOS_DEF_ERROR_CODE(0, 0x2700)

include/util/tencode.h

@@ -18,7 +18,6 @@
 
 #include "tcoding.h"
 #include "tlist.h"
-// #include "tfreelist.h"
 
 #ifdef __cplusplus
 extern "C" {

include/util/tfreelist.h

@@ -1,60 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_UTIL_FREELIST_H_
-#define _TD_UTIL_FREELIST_H_
-
-#include "tlist.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-struct SFreeListNode {
-  TD_SLIST_NODE(SFreeListNode);
-  char payload[];
-};
-
-typedef TD_SLIST(SFreeListNode) SFreeList;
-
-#define TFL_MALLOC(PTR, TYPE, SIZE, LIST)                      \
-  do {                                                         \
-    void *ptr = taosMemoryMalloc((SIZE) + sizeof(struct SFreeListNode)); \
-    if (ptr) {                                                 \
-      TD_SLIST_PUSH((LIST), (struct SFreeListNode *)ptr);      \
-      ptr = ((struct SFreeListNode *)ptr)->payload;            \
-      (PTR) = (TYPE)(ptr);                                     \
-    }else{                                                     \
-      (PTR) = NULL;                                            \
-    }                                                          \
-  }while(0);
-
-#define tFreeListInit(pFL) TD_SLIST_INIT(pFL)
-
-static FORCE_INLINE void tFreeListClear(SFreeList *pFL) {
-  struct SFreeListNode *pNode;
-  for (;;) {
-    pNode = TD_SLIST_HEAD(pFL);
-    if (pNode == NULL) break;
-    TD_SLIST_POP(pFL);
-    taosMemoryFree(pNode);
-  }
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_UTIL_FREELIST_H_*/

source/client/src/clientImpl.c

@@ -518,8 +518,9 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t
   if (pRequest->code != TSDB_CODE_SUCCESS) {
     const char* errorMsg =
         (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(pRequest->code);
-    printf("failed to connect to server, reason: %s\n\n", errorMsg);
+    fprintf(stderr,"failed to connect to server, reason: %s\n\n", errorMsg);
 
+    terrno = pRequest->code;
     destroyRequest(pRequest);
     taos_close(pTscObj);
     pTscObj = NULL;

source/client/src/clientSml.c

@@ -1007,8 +1007,6 @@ static int32_t smlParseTelnetTags(const char* data, SArray *cols, SHashObj *dump
     while(*sql != '\0') {
       // parse value
       if (*sql == SPACE) {
-        valueLen = sql - value;
-        sql++;
         break;
       }
       if (*sql == EQUAL) {
@@ -1017,9 +1015,7 @@ static int32_t smlParseTelnetTags(const char* data, SArray *cols, SHashObj *dump
       }
       sql++;
     }
-    if(valueLen == 0){
+    valueLen = sql - value;
-      valueLen = sql - value;
-    }
 
     if(valueLen == 0){
       smlBuildInvalidDataMsg(msg, "invalid value", value);
@@ -1365,7 +1361,7 @@ static void smlDestroySTableMeta(SSmlSTableMeta *meta){
 static void smlDestroyCols(SArray *cols) {
   if (!cols) return;
   for (int i = 0; i < taosArrayGetSize(cols); ++i) {
-    void *kv = taosArrayGet(cols, i);
+    void *kv = taosArrayGetP(cols, i);
     taosMemoryFree(kv);
   }
 }
@@ -2077,12 +2073,16 @@ static int32_t smlParseTelnetLine(SSmlHandle* info, void *data) {
   if(ret != TSDB_CODE_SUCCESS){
     uError("SML:0x%"PRIx64" smlParseTelnetLine failed", info->id);
     smlDestroyTableInfo(info, tinfo);
+    smlDestroyCols(cols);
     taosArrayDestroy(cols);
     return ret;
   }
 
   if(taosArrayGetSize(tinfo->tags) <= 0 || taosArrayGetSize(tinfo->tags) > TSDB_MAX_TAGS){
     smlBuildInvalidDataMsg(&info->msgBuf, "invalidate tags length:[1,128]", NULL);
+    smlDestroyTableInfo(info, tinfo);
+    smlDestroyCols(cols);
+    taosArrayDestroy(cols);
     return TSDB_CODE_SML_INVALID_DATA;
   }
   taosHashClear(info->dumplicateKey);

source/client/test/smlTest.cpp

@@ -516,8 +516,8 @@ TEST(testCase, smlProcess_influx_Test) {
   int ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0]));
   ASSERT_EQ(ret, 0);
 
-  TAOS_RES *res = taos_query(taos, "select * from t_6885c584b98481584ee13dac399e173d");
+//  TAOS_RES *res = taos_query(taos, "select * from t_6885c584b98481584ee13dac399e173d");
-  ASSERT_NE(res, nullptr);
+//  ASSERT_NE(res, nullptr);
 //  int fieldNum = taos_field_count(res);
 //  ASSERT_EQ(fieldNum, 5);
 //  int rowNum = taos_affected_rows(res);
@@ -525,7 +525,7 @@ TEST(testCase, smlProcess_influx_Test) {
 //  for (int i = 0; i < rowNum; ++i) {
 //    TAOS_ROW rows = taos_fetch_row(res);
 //  }
-  taos_free_result(res);
+//  taos_free_result(res);
   destroyRequest(request);
   smlDestroyInfo(info);
 }
@@ -605,8 +605,8 @@ TEST(testCase, smlProcess_telnet_Test) {
   int ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0]));
   ASSERT_EQ(ret, 0);
 
-  TAOS_RES *res = taos_query(taos, "select * from t_8c30283b3c4131a071d1e16cf6d7094a");
+//  TAOS_RES *res = taos_query(taos, "select * from t_8c30283b3c4131a071d1e16cf6d7094a");
-  ASSERT_NE(res, nullptr);
+//  ASSERT_NE(res, nullptr);
 //  int fieldNum = taos_field_count(res);
 //  ASSERT_EQ(fieldNum, 2);
 //  int rowNum = taos_affected_rows(res);
@@ -614,7 +614,7 @@ TEST(testCase, smlProcess_telnet_Test) {
 //  for (int i = 0; i < rowNum; ++i) {
 //   TAOS_ROW rows = taos_fetch_row(res);
 //  }
-  taos_free_result(res);
+//  taos_free_result(res);
 
 //  res = taos_query(taos, "select * from t_6931529054e5637ca92c78a1ad441961");
 //  ASSERT_NE(res, nullptr);
@@ -670,16 +670,16 @@ TEST(testCase, smlProcess_json1_Test) {
   int ret = smlProcess(info, (char **)(&sql), -1);
   ASSERT_EQ(ret, 0);
 
-  TAOS_RES *res = taos_query(taos, "select * from t_cb27a7198d637b4f1c6464bd73f756a7");
+//  TAOS_RES *res = taos_query(taos, "select * from t_cb27a7198d637b4f1c6464bd73f756a7");
-  ASSERT_NE(res, nullptr);
+//  ASSERT_NE(res, nullptr);
-  int fieldNum = taos_field_count(res);
+//  int fieldNum = taos_field_count(res);
-  ASSERT_EQ(fieldNum, 2);
+//  ASSERT_EQ(fieldNum, 2);
   //   int rowNum = taos_affected_rows(res);
   //   ASSERT_EQ(rowNum, 1);
   //   for (int i = 0; i < rowNum; ++i) {
   //     TAOS_ROW rows = taos_fetch_row(res);
   //   }
-  taos_free_result(res);
+//  taos_free_result(res);
   destroyRequest(request);
   smlDestroyInfo(info);
 }

source/common/src/tmsg.c

@@ -4163,3 +4163,113 @@ void tFreeSSubmitRsp(SSubmitRsp *pRsp) {
 
   taosMemoryFree(pRsp);
 }
+
+int32_t tEncodeSVAlterTbReq(SEncoder *pEncoder, const SVAlterTbReq *pReq) {
+  if (tStartEncode(pEncoder) < 0) return -1;
+
+  if (tEncodeCStr(pEncoder, pReq->tbName) < 0) return -1;
+  if (tEncodeI8(pEncoder, pReq->action) < 0) return -1;
+  switch (pReq->action) {
+    case TSDB_ALTER_TABLE_ADD_COLUMN:
+      if (tEncodeCStr(pEncoder, pReq->colName) < 0) return -1;
+      if (tEncodeI8(pEncoder, pReq->type) < 0) return -1;
+      if (tEncodeI8(pEncoder, pReq->flags) < 0) return -1;
+      if (tEncodeI32v(pEncoder, pReq->bytes) < 0) return -1;
+      break;
+    case TSDB_ALTER_TABLE_DROP_COLUMN:
+      if (tEncodeCStr(pEncoder, pReq->colName) < 0) return -1;
+      break;
+    case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:
+      if (tEncodeCStr(pEncoder, pReq->colName) < 0) return -1;
+      if (tEncodeI32v(pEncoder, pReq->colModBytes) < 0) return -1;
+      break;
+    case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME:
+      if (tEncodeCStr(pEncoder, pReq->colName) < 0) return -1;
+      if (tEncodeCStr(pEncoder, pReq->colNewName) < 0) return -1;
+      break;
+    case TSDB_ALTER_TABLE_UPDATE_TAG_VAL:
+      if (tEncodeCStr(pEncoder, pReq->tagName) < 0) return -1;
+      if (tEncodeI8(pEncoder, pReq->isNull) < 0) return -1;
+      if (!pReq->isNull) {
+        if (tEncodeBinary(pEncoder, pReq->pTagVal, pReq->nTagVal) < 0) return -1;
+      }
+      break;
+    case TSDB_ALTER_TABLE_UPDATE_OPTIONS:
+      if (tEncodeI8(pEncoder, pReq->updateTTL) < 0) return -1;
+      if (pReq->updateTTL) {
+        if (tEncodeI32v(pEncoder, pReq->newTTL) < 0) return -1;
+      }
+      if (tEncodeI8(pEncoder, pReq->updateComment) < 0) return -1;
+      if (pReq->updateComment) {
+        if (tEncodeCStr(pEncoder, pReq->newComment) < 0) return -1;
+      }
+      break;
+    default:
+      break;
+  }
+
+  tEndEncode(pEncoder);
+  return 0;
+}
+
+int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) {
+  if (tStartDecode(pDecoder) < 0) return -1;
+
+  if (tDecodeCStr(pDecoder, &pReq->tbName) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pReq->action) < 0) return -1;
+  switch (pReq->action) {
+    case TSDB_ALTER_TABLE_ADD_COLUMN:
+      if (tDecodeCStr(pDecoder, &pReq->colName) < 0) return -1;
+      if (tDecodeI8(pDecoder, &pReq->type) < 0) return -1;
+      if (tDecodeI8(pDecoder, &pReq->flags) < 0) return -1;
+      if (tDecodeI32v(pDecoder, &pReq->bytes) < 0) return -1;
+      break;
+    case TSDB_ALTER_TABLE_DROP_COLUMN:
+      if (tDecodeCStr(pDecoder, &pReq->colName) < 0) return -1;
+      break;
+    case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:
+      if (tDecodeCStr(pDecoder, &pReq->colName) < 0) return -1;
+      if (tDecodeI32v(pDecoder, &pReq->colModBytes) < 0) return -1;
+      break;
+    case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME:
+      if (tDecodeCStr(pDecoder, &pReq->colName) < 0) return -1;
+      if (tDecodeCStr(pDecoder, &pReq->colNewName) < 0) return -1;
+      break;
+    case TSDB_ALTER_TABLE_UPDATE_TAG_VAL:
+      if (tDecodeCStr(pDecoder, &pReq->tagName) < 0) return -1;
+      if (tDecodeI8(pDecoder, &pReq->isNull) < 0) return -1;
+      if (!pReq->isNull) {
+        if (tDecodeBinary(pDecoder, &pReq->pTagVal, &pReq->nTagVal) < 0) return -1;
+      }
+      break;
+    case TSDB_ALTER_TABLE_UPDATE_OPTIONS:
+      if (tDecodeI8(pDecoder, &pReq->updateTTL) < 0) return -1;
+      if (pReq->updateTTL) {
+        if (tDecodeI32v(pDecoder, &pReq->newTTL) < 0) return -1;
+      }
+      if (tDecodeI8(pDecoder, &pReq->updateComment) < 0) return -1;
+      if (pReq->updateComment) {
+        if (tDecodeCStr(pDecoder, &pReq->newComment) < 0) return -1;
+      }
+      break;
+    default:
+      break;
+  }
+
+  tEndDecode(pDecoder);
+  return 0;
+}
+
+int32_t tEncodeSVAlterTbRsp(SEncoder *pEncoder, const SVAlterTbRsp *pRsp) {
+  if (tStartEncode(pEncoder) < 0) return -1;
+  if (tEncodeI32(pEncoder, pRsp->code) < 0) return -1;
+  tEndEncode(pEncoder);
+  return 0;
+}
+
+int32_t tDecodeSVAlterTbRsp(SDecoder *pDecoder, SVAlterTbRsp *pRsp) {
+  if (tStartDecode(pDecoder) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pRsp->code) < 0) return -1;
+  tEndDecode(pDecoder);
+  return 0;
+}

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -192,7 +192,8 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
     rpcFreeCont(originalRpcMsg.pCont);
 
     // if leader, send response
-    if (pMsg->rpcMsg.handle != NULL && pMsg->rpcMsg.ahandle != NULL) {
+    //if (pMsg->rpcMsg.handle != NULL && pMsg->rpcMsg.ahandle != NULL) {
+    if (pMsg->rpcMsg.handle != NULL) {
       rsp.ahandle = pMsg->rpcMsg.ahandle;
       rsp.handle = pMsg->rpcMsg.handle;
       rsp.refId = pMsg->rpcMsg.refId;
@@ -213,7 +214,23 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf
 
     // todo
     SRpcMsg *pRsp = NULL;
-    (void)vnodeProcessSyncReq(pVnode->pImpl, &pMsg->rpcMsg, &pRsp);
+    int32_t ret = vnodeProcessSyncReq(pVnode->pImpl, &pMsg->rpcMsg, &pRsp);
+    if (ret != 0) {
+      // if leader, send response
+      if (pMsg->rpcMsg.handle != NULL) {
+        SRpcMsg    rsp;
+        rsp.pCont = NULL;
+        rsp.contLen = 0;
+
+        rsp.code = terrno;
+        dTrace("vmProcessSyncQueue error, code:%d", terrno);
+        
+        rsp.ahandle = pMsg->rpcMsg.ahandle;
+        rsp.handle = pMsg->rpcMsg.handle;
+        rsp.refId = pMsg->rpcMsg.refId;
+        tmsgSendRsp(&rsp);
+      }
+    }
 
     rpcFreeCont(pMsg->rpcMsg.pCont);
     taosFreeQitem(pMsg);

source/dnode/mnode/impl/src/mndStb.c

@@ -383,9 +383,10 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
   req.suid = pStb->uid;
   req.rollup = pStb->ast1Len > 0 ? 1 : 0;
   req.schema.nCols = pStb->numOfColumns;
-  req.schema.sver = 0;
+  req.schema.sver = pStb->version;
   req.schema.pSchema = pStb->pColumns;
   req.schemaTag.nCols = pStb->numOfTags;
+  req.schemaTag.nCols = 0;
   req.schemaTag.pSchema = pStb->pTags;
 
   if (req.rollup) {

source/dnode/vnode/inc/vnode.h

@@ -189,6 +189,7 @@ struct SMetaEntry {
     struct {
       int64_t        ctime;
       int32_t        ttlDays;
+      int32_t        ncid;  // next column id
       SSchemaWrapper schema;
     } ntbEntry;
     struct {

source/dnode/vnode/src/inc/vnodeInt.h

@@ -79,9 +79,11 @@ int             metaClose(SMeta* pMeta);
 int             metaBegin(SMeta* pMeta);
 int             metaCommit(SMeta* pMeta);
 int             metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq);
+int             metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq);
 int             metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq);
 int             metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq);
 int             metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq);
+int             metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq);
 SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline);
 STSchema*       metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver);
 int             metaGetTableEntryByName(SMetaReader* pReader, const char* name);

source/dnode/vnode/src/meta/metaEntry.c

@@ -34,6 +34,7 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) {
   } else if (pME->type == TSDB_NORMAL_TABLE) {
     if (tEncodeI64(pCoder, pME->ntbEntry.ctime) < 0) return -1;
     if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1;
+    if (tEncodeI32v(pCoder, pME->ntbEntry.ncid) < 0) return -1;
     if (tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1;
   } else if (pME->type == TSDB_TSMA_TABLE) {
     if (tEncodeTSma(pCoder, pME->smaEntry.tsma) < 0) return -1;
@@ -65,10 +66,11 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
   } else if (pME->type == TSDB_NORMAL_TABLE) {
     if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1;
     if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1;
+    if (tDecodeI32v(pCoder, &pME->ntbEntry.ncid) < 0) return -1;
     if (tDecodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1;
   } else if (pME->type == TSDB_TSMA_TABLE) {
     if (tDecodeTSma(pCoder, pME->smaEntry.tsma) < 0) return -1;
-  }  else {
+  } else {
     ASSERT(0);
   }
 

source/dnode/vnode/src/meta/metaTable.c

@@ -131,6 +131,75 @@ _err:
   return -1;
 }
 
+int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
+  SMetaEntry  oStbEntry = {0};
+  SMetaEntry  nStbEntry = {0};
+  TDBC       *pUidIdxc = NULL;
+  TDBC       *pTbDbc = NULL;
+  const void *pData;
+  int         nData;
+  int64_t     oversion;
+  SDecoder    dc = {0};
+  int32_t     ret;
+  int32_t     c;
+
+  tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn);
+  ret = tdbDbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c);
+  if (ret < 0 || c) {
+    ASSERT(0);
+    return -1;
+  }
+
+  ret = tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData);
+  if (ret < 0) {
+    ASSERT(0);
+    return -1;
+  }
+
+  oversion = *(int64_t *)pData;
+
+  tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn);
+  ret = tdbDbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c);
+  ASSERT(ret == 0 && c == 0);
+
+  ret = tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData);
+  ASSERT(ret == 0);
+
+  tDecoderInit(&dc, pData, nData);
+  metaDecodeEntry(&dc, &oStbEntry);
+
+  nStbEntry.version = version;
+  nStbEntry.type = TSDB_SUPER_TABLE;
+  nStbEntry.uid = pReq->suid;
+  nStbEntry.name = pReq->name;
+  nStbEntry.stbEntry.schema = pReq->schema;
+  nStbEntry.stbEntry.schemaTag = pReq->schemaTag;
+
+  metaWLock(pMeta);
+  // compare two entry
+  if (oStbEntry.stbEntry.schema.sver != pReq->schema.sver) {
+    if (oStbEntry.stbEntry.schema.nCols != pReq->schema.nCols) {
+      metaSaveToSkmDb(pMeta, &nStbEntry);
+    }
+  }
+
+  // if (oStbEntry.stbEntry.schemaTag.sver != pReq->schemaTag.sver) {
+  //   // change tag schema
+  // }
+
+  // update table.db
+  metaSaveToTbDb(pMeta, &nStbEntry);
+
+  // update uid index
+  tdbDbcUpsert(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &version, sizeof(version), 0);
+
+  metaULock(pMeta);
+  tDecoderClear(&dc);
+  tdbDbcClose(pTbDbc);
+  tdbDbcClose(pUidIdxc);
+  return 0;
+}
+
 int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) {
   SMetaEntry  me = {0};
   SMetaReader mr = {0};
@@ -171,6 +240,7 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) {
     me.ntbEntry.ctime = pReq->ctime;
     me.ntbEntry.ttlDays = pReq->ttl;
     me.ntbEntry.schema = pReq->ntb.schema;
+    me.ntbEntry.ncid = me.ntbEntry.schema.pSchema[me.ntbEntry.schema.nCols - 1].colId + 1;
   }
 
   if (metaHandleEntry(pMeta, &me) < 0) goto _err;
@@ -305,6 +375,170 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) {
   return 0;
 }
 
+static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
+  void           *pVal = NULL;
+  int             nVal = 0;
+  const void     *pData = NULL;
+  int             nData = 0;
+  int             ret = 0;
+  tb_uid_t        uid;
+  int64_t         oversion;
+  SSchema        *pColumn = NULL;
+  SMetaEntry      entry = {0};
+  SSchemaWrapper *pSchema;
+  int             c;
+
+  // search name index
+  ret = tdbDbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal);
+  if (ret < 0) {
+    terrno = TSDB_CODE_VND_TABLE_NOT_EXIST;
+    return -1;
+  }
+
+  uid = *(tb_uid_t *)pVal;
+  tdbFree(pVal);
+  pVal = NULL;
+
+  // search uid index
+  TDBC *pUidIdxc = NULL;
+
+  tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn);
+  tdbDbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c);
+  ASSERT(c == 0);
+
+  tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData);
+  oversion = *(int64_t *)pData;
+
+  // search table.db
+  TDBC *pTbDbc = NULL;
+
+  tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn);
+  tdbDbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c);
+  ASSERT(c == 0);
+  tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData);
+
+  // get table entry
+  SDecoder dc = {0};
+  tDecoderInit(&dc, pData, nData);
+  metaDecodeEntry(&dc, &entry);
+
+  if (entry.type != TSDB_NORMAL_TABLE) {
+    terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION;
+    goto _err;
+  }
+
+  // search the column to add/drop/update
+  pSchema = &entry.ntbEntry.schema;
+  int32_t iCol = 0;
+  for (;;) {
+    pColumn = NULL;
+
+    if (iCol >= pSchema->nCols) break;
+    pColumn = &pSchema->pSchema[iCol];
+
+    if (strcmp(pColumn->name, pAlterTbReq->colName) == 0) break;
+    iCol++;
+  }
+
+  entry.version = version;
+  int tlen;
+  switch (pAlterTbReq->action) {
+    case TSDB_ALTER_TABLE_ADD_COLUMN:
+      if (pColumn) {
+        terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS;
+        goto _err;
+      }
+      pSchema->sver++;
+      pSchema->nCols++;
+      pSchema->pSchema =
+          taosMemoryRealloc(entry.ntbEntry.schema.pSchema, sizeof(SSchema) * entry.ntbEntry.schema.nCols);
+      pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].bytes = pAlterTbReq->bytes;
+      pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].type = pAlterTbReq->type;
+      pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].flags = pAlterTbReq->flags;
+      pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].colId = entry.ntbEntry.ncid++;
+      strcpy(pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].name, pAlterTbReq->colName);
+      break;
+    case TSDB_ALTER_TABLE_DROP_COLUMN:
+      if (pColumn == NULL) {
+        terrno = TSDB_CODE_VND_TABLE_COL_NOT_EXISTS;
+        goto _err;
+      }
+      if (pColumn->colId == 0) {
+        terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION;
+        goto _err;
+      }
+      pSchema->sver++;
+      pSchema->nCols--;
+      tlen = (pSchema->nCols - iCol - 1) * sizeof(SSchema);
+      if (tlen) {
+        memmove(pColumn, pColumn + 1, tlen);
+      }
+      break;
+    case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:
+      if (pColumn == NULL) {
+        terrno = TSDB_CODE_VND_TABLE_COL_NOT_EXISTS;
+        goto _err;
+      }
+      if (!IS_VAR_DATA_TYPE(pColumn->type) || pColumn->bytes <= pAlterTbReq->bytes) {
+        terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION;
+        goto _err;
+      }
+      pSchema->sver++;
+      pColumn->bytes = pAlterTbReq->bytes;
+      break;
+    case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME:
+      if (pColumn == NULL) {
+        terrno = TSDB_CODE_VND_TABLE_COL_NOT_EXISTS;
+        goto _err;
+      }
+      pSchema->sver++;
+      strcpy(pColumn->name, pAlterTbReq->colNewName);
+      break;
+  }
+
+  entry.version = version;
+
+  tDecoderClear(&dc);
+  tdbDbcClose(pTbDbc);
+  tdbDbcClose(pUidIdxc);
+  return 0;
+
+_err:
+  tDecoderClear(&dc);
+  tdbDbcClose(pTbDbc);
+  tdbDbcClose(pUidIdxc);
+  return -1;
+}
+
+static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
+  // TODO
+  return 0;
+}
+
+static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
+  // TODO
+  ASSERT(0);
+  return 0;
+}
+
+int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) {
+  switch (pReq->action) {
+    case TSDB_ALTER_TABLE_ADD_COLUMN:
+    case TSDB_ALTER_TABLE_DROP_COLUMN:
+    case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:
+    case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME:
+      return metaAlterTableColumn(pMeta, version, pReq);
+    case TSDB_ALTER_TABLE_UPDATE_TAG_VAL:
+      return metaUpdateTableTagVal(pMeta, version, pReq);
+    case TSDB_ALTER_TABLE_UPDATE_OPTIONS:
+      return metaUpdateTableOptions(pMeta, version, pReq);
+    default:
+      terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION;
+      return -1;
+      break;
+  }
+}
+
 static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) {
   STbDbKey tbDbKey;
   void    *pKey = NULL;

source/dnode/vnode/src/tq/tqRead.c

@@ -91,7 +91,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
   // TODO set to real sversion
   *pUid = 0;
 
-  int32_t sversion = 0;
+  int32_t sversion = 1;
   if (pHandle->sver != sversion || pHandle->cachedSchemaUid != pHandle->msgIter.suid) {
     pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion);
 

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -465,7 +465,7 @@ static int tsdbCreateCommitIters(SCommitH *pCommith) {
     pTbData = (STbData *)pNode->pData;
 
     pCommitIter = pCommith->iters + i;
-    pTSchema = metaGetTbTSchema(REPO_META(pRepo), pTbData->uid, 0);  // TODO: schema version
+    pTSchema = metaGetTbTSchema(REPO_META(pRepo), pTbData->uid, 1);  // TODO: schema version
 
     if (pTSchema) {
       pCommitIter->pIter = tSkipListCreateIter(pTbData->pData);
@@ -912,7 +912,7 @@ static int tsdbMoveBlkIdx(SCommitH *pCommith, SBlockIdx *pIdx) {
   while (bidx < nBlocks) {
     if (!pTSchema && !tsdbCommitIsSameFile(pCommith, bidx)) {
       // Set commit table
-      pTSchema = metaGetTbTSchema(REPO_META(pTsdb), pIdx->uid, 0);  // TODO: schema version
+      pTSchema = metaGetTbTSchema(REPO_META(pTsdb), pIdx->uid, 1);  // TODO: schema version
       if (!pTSchema) {
         terrno = TSDB_CODE_OUT_OF_MEMORY;
         return -1;

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -490,7 +490,7 @@ tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableG
 
   STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, 0);
 
-  pTsdbReadHandle->pSchema = metaGetTbTSchema(pVnode->pMeta, pCheckInfo->tableId, 0);
+  pTsdbReadHandle->pSchema = metaGetTbTSchema(pVnode->pMeta, pCheckInfo->tableId, 1);
   int32_t  numOfCols = taosArrayGetSize(pTsdbReadHandle->suppInfo.defaultLoadColumn);
   int16_t* ids = pTsdbReadHandle->suppInfo.defaultLoadColumn->pData;
 
@@ -1271,7 +1271,6 @@ _error:
 static int32_t getEndPosInDataBlock(STsdbReadHandle* pTsdbReadHandle, SDataBlockInfo* pBlockInfo);
 static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows,
                                        int32_t start, int32_t end);
-static void    moveDataToFront(STsdbReadHandle* pTsdbReadHandle, int32_t numOfRows, int32_t numOfCols);
 static void    doCheckGeneratedBlockRange(STsdbReadHandle* pTsdbReadHandle);
 static void    copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo,
                                               SDataBlockInfo* pBlockInfo, int32_t endPos);
@@ -1301,7 +1300,7 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock*
     if ((ascScan && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) ||
         (!ascScan && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) {
       // do not load file block into buffer
-      int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1;
+      int32_t step = ascScan ? 1 : -1;
 
       TSKEY maxKey =
           ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? (binfo.window.skey - step) : (binfo.window.ekey - step);
@@ -1618,7 +1617,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa
   if (pSchema1 == NULL) {
     // pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row1));
     // TODO: use the real schemaVersion
-    pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 0);
+    pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 1);
   }
 
 #ifdef TD_DEBUG_PRINT_ROW
@@ -1637,7 +1636,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa
     if (pSchema2 == NULL) {
       // pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row2));
       // TODO: use the real schemaVersion
-      pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 0);
+      pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 1);
     }
     if (isRow2DataRow) {
       numOfColsOfRow2 = schemaNCols(pSchema2);
@@ -1790,22 +1789,6 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa
 #endif
 }
 
-static void moveDataToFront(STsdbReadHandle* pTsdbReadHandle, int32_t numOfRows, int32_t numOfCols) {
-  if (numOfRows == 0 || ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
-    return;
-  }
-
-  // if the buffer is not full in case of descending order query, move the data in the front of the buffer
-  if (numOfRows < pTsdbReadHandle->outputCapacity) {
-    int32_t emptySize = pTsdbReadHandle->outputCapacity - numOfRows;
-    for (int32_t i = 0; i < numOfCols; ++i) {
-      SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
-      memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes,
-              numOfRows * pColInfo->info.bytes);
-    }
-  }
-}
-
 static void getQualifiedRowsPos(STsdbReadHandle* pTsdbReadHandle, int32_t startPos, int32_t endPos,
                                 int32_t numOfExisted, int32_t* start, int32_t* end) {
   *start = -1;
@@ -1891,9 +1874,6 @@ static void copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, STa
   cur->lastKey = tsArray[endPos] + step;
   cur->blockCompleted = true;
 
-  // if the buffer is not full in case of descending order query, move the data in the front of the buffer
-  moveDataToFront(pTsdbReadHandle, numOfRows, numOfCols);
-
   // The value of pos may be -1 or pBlockInfo->rows, and it is invalid in both cases.
   pos = endPos + step;
   updateInfoAfterMerge(pTsdbReadHandle, pCheckInfo, numOfRows, pos);
@@ -1944,18 +1924,18 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
   assert(pCols->numOfRows == pBlock->numOfRows && tsArray[0] == pBlock->keyFirst &&
          tsArray[pBlock->numOfRows - 1] == pBlock->keyLast);
 
+  bool ascScan = ASCENDING_TRAVERSE(pTsdbReadHandle->order);
+  int32_t step = ascScan ? 1 : -1;
+
   // for search the endPos, so the order needs to reverse
-  int32_t order = (pTsdbReadHandle->order == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC;
+  int32_t order = ascScan ? TSDB_ORDER_DESC : TSDB_ORDER_ASC;
 
-  int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1;
   int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pTsdbReadHandle));
-
-  STable* pTable = NULL;
   int32_t endPos = getEndPosInDataBlock(pTsdbReadHandle, &blockInfo);
 
+  STimeWindow* pWin = &blockInfo.window;
   tsdbDebug("%p uid:%" PRIu64 " start merge data block, file block range:%" PRIu64 "-%" PRIu64
-            " rows:%d, start:%d, end:%d, %s",
+            " rows:%d, start:%d, end:%d, %s", pTsdbReadHandle, pCheckInfo->tableId, pWin->skey, pWin->ekey, blockInfo.rows,
-            pTsdbReadHandle, pCheckInfo->tableId, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows,
             cur->pos, endPos, pTsdbReadHandle->idStr);
 
   // compared with the data from in-memory buffer, to generate the correct timestamp array list
@@ -1986,20 +1966,16 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
       }
 
       TSKEY key = TD_ROW_KEY(row1);
-      if ((key > pTsdbReadHandle->window.ekey && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
+      if ((key > pTsdbReadHandle->window.ekey && ascScan) || (key < pTsdbReadHandle->window.ekey && !ascScan)) {
-          (key < pTsdbReadHandle->window.ekey && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) {
         break;
       }
 
-      if (((pos > endPos || tsArray[pos] > pTsdbReadHandle->window.ekey) &&
+      if (((pos > endPos || tsArray[pos] > pTsdbReadHandle->window.ekey) && ascScan) ||
-           ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
+          ((pos < endPos || tsArray[pos] < pTsdbReadHandle->window.ekey) && !ascScan)) {
-          ((pos < endPos || tsArray[pos] < pTsdbReadHandle->window.ekey) &&
-           !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) {
         break;
       }
 
-      if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
+      if ((key < tsArray[pos] && ascScan) || (key > tsArray[pos] && !ascScan)) {
-          (key > tsArray[pos] && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) {
         if (rv1 != TD_ROW_SVER(row1)) {
           //          pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1));
           rv1 = TD_ROW_SVER(row1);
@@ -2054,23 +2030,19 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
         }
 #endif
         if (TD_SUPPORT_UPDATE(pCfg->update)) {
-          if (lastKeyAppend != key) {
-            lastKeyAppend = key;
-            ++curRow;
-          }
           numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, curRow, pos, pos);
+          lastKeyAppend = key;
 
           if (rv1 != TD_ROW_SVER(row1)) {
-            //            pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1));
             rv1 = TD_ROW_SVER(row1);
           }
           if (row2 && rv2 != TD_ROW_SVER(row2)) {
-            //            pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2));
             rv2 = TD_ROW_SVER(row2);
           }
-          numOfRows +=
+
-              mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols,
+          // still assign data into current row
-                                 pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastKeyAppend);
+          mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols,
+                             pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastKeyAppend);
 
           if (cur->win.skey == TSKEY_INITIAL_VAL) {
             cur->win.skey = key;
@@ -2081,12 +2053,13 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
           cur->mixBlock = true;
 
           moveToNextRowInMem(pCheckInfo);
+          ++curRow;
+
           pos += step;
         } else {
           moveToNextRowInMem(pCheckInfo);
         }
-      } else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
+      } else if ((key > tsArray[pos] && ascScan) || (key < tsArray[pos] && !ascScan)) {
-                 (key < tsArray[pos] && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) {
         if (cur->win.skey == TSKEY_INITIAL_VAL) {
           cur->win.skey = tsArray[pos];
         }
@@ -2112,17 +2085,17 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
         int32_t qstart = 0, qend = 0;
         getQualifiedRowsPos(pTsdbReadHandle, pos, end, numOfRows, &qstart, &qend);
 
-        if ((lastKeyAppend != TSKEY_INITIAL_VAL) &&
+        if ((lastKeyAppend != TSKEY_INITIAL_VAL) && (lastKeyAppend != (ascScan ? tsArray[qstart] : tsArray[qend]))) {
-            (lastKeyAppend != (ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? tsArray[qstart] : tsArray[qend]))) {
           ++curRow;
         }
+
         numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, curRow, qstart, qend);
         pos += (qend - qstart + 1) * step;
         if (numOfRows > 0) {
           curRow = numOfRows - 1;
         }
 
-        cur->win.ekey = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? tsArray[qend] : tsArray[qstart];
+        cur->win.ekey = ascScan ? tsArray[qend] : tsArray[qstart];
         cur->lastKey = cur->win.ekey + step;
         lastKeyAppend = cur->win.ekey;
       }
@@ -2134,10 +2107,8 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
        * copy them all to result buffer, since it may be overlapped with file data block.
        */
       if (node == NULL ||
-          ((TD_ROW_KEY((STSRow*)SL_GET_NODE_DATA(node)) > pTsdbReadHandle->window.ekey) &&
+          ((TD_ROW_KEY((STSRow*)SL_GET_NODE_DATA(node)) > pTsdbReadHandle->window.ekey) && ascScan) ||
-           ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
+          ((TD_ROW_KEY((STSRow*)SL_GET_NODE_DATA(node)) < pTsdbReadHandle->window.ekey) && !ascScan)) {
-          ((TD_ROW_KEY((STSRow*)SL_GET_NODE_DATA(node)) < pTsdbReadHandle->window.ekey) &&
-           !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) {
         // no data in cache or data in cache is greater than the ekey of time window, load data from file block
         if (cur->win.skey == TSKEY_INITIAL_VAL) {
           cur->win.skey = tsArray[pos];
@@ -2149,22 +2120,20 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
         numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, start, end);
         pos += (end - start + 1) * step;
 
-        cur->win.ekey = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? tsArray[end] : tsArray[start];
+        cur->win.ekey = ascScan ? tsArray[end] : tsArray[start];
         cur->lastKey = cur->win.ekey + step;
         cur->mixBlock = true;
       }
     }
   }
 
-  cur->blockCompleted =
+  cur->blockCompleted = (((pos > endPos || cur->lastKey > pTsdbReadHandle->window.ekey) && ascScan) ||
-      (((pos > endPos || cur->lastKey > pTsdbReadHandle->window.ekey) && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
+       ((pos < endPos || cur->lastKey < pTsdbReadHandle->window.ekey) && !ascScan));
-       ((pos < endPos || cur->lastKey < pTsdbReadHandle->window.ekey) && !ASCENDING_TRAVERSE(pTsdbReadHandle->order)));
 
-  if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
+  if (!ascScan) {
     TSWAP(cur->win.skey, cur->win.ekey);
   }
 
-  moveDataToFront(pTsdbReadHandle, numOfRows, numOfCols);
   updateInfoAfterMerge(pTsdbReadHandle, pCheckInfo, numOfRows, pos);
   doCheckGeneratedBlockRange(pTsdbReadHandle);
 
@@ -2755,7 +2724,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
 
     win->ekey = key;
     if (rv != TD_ROW_SVER(row)) {
-      pSchema = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), pCheckInfo->tableId, 0);
+      pSchema = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), pCheckInfo->tableId, 1);
       rv = TD_ROW_SVER(row);
     }
     numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, &curRows, row, NULL, numOfCols, pCheckInfo->tableId,
@@ -3877,7 +3846,7 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch
 
   // NOTE: not add ref count for super table
   SArray*         res = taosArrayInit(8, sizeof(STableKeyInfo));
-  SSchemaWrapper* pTagSchema = metaGetTableSchema(pMeta, uid, 0, true);
+  SSchemaWrapper* pTagSchema = metaGetTableSchema(pMeta, uid, 1, true);
 
   // no tags and tbname condition, all child tables of this stable are involved
   if (tbnameCond == NULL && (pTagCond == NULL || len == 0)) {

source/dnode/vnode/src/tsdb/tsdbSma.c

@@ -2084,7 +2084,7 @@ static int32_t tsdbExecuteRSma(STsdb *pTsdb, const void *pMsg, int32_t inputType
 
   if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
     // TODO: use the proper schema instead of 0, and cache STSchema in cache
-    STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, suid, 0);
+    STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, suid, 1);
     if (!pTSchema) {
       terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
       return TSDB_CODE_FAILED;

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -16,10 +16,10 @@
 #include "vnd.h"
 
 static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp);
-static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp);
-static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp);
@@ -73,7 +73,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
       if (vnodeProcessCreateStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
       break;
     case TDMT_VND_ALTER_STB:
-      if (vnodeProcessAlterStbReq(pVnode, pReq, len, pRsp) < 0) goto _err;
+      if (vnodeProcessAlterStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
       break;
     case TDMT_VND_DROP_STB:
       if (vnodeProcessDropStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
@@ -82,7 +82,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
       if (vnodeProcessCreateTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
       break;
     case TDMT_VND_ALTER_TABLE:
-      if (vnodeProcessAlterTbReq(pVnode, pReq, len, pRsp) < 0) goto _err;
+      if (vnodeProcessAlterTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
       break;
     case TDMT_VND_DROP_TABLE:
       if (vnodeProcessDropTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
@@ -203,6 +203,8 @@ void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) {
 }
 
 int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
+  int32_t ret = TAOS_SYNC_PROPOSE_OTHER_ERROR;
+  
   if (syncEnvIsStart()) {
     SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync);
     assert(pSyncNode != NULL);
@@ -224,67 +226,70 @@ int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
       SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pRpcMsg);
       assert(pSyncMsg != NULL);
 
-      syncNodeOnTimeoutCb(pSyncNode, pSyncMsg);
+      ret = syncNodeOnTimeoutCb(pSyncNode, pSyncMsg);
       syncTimeoutDestroy(pSyncMsg);
 
     } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING) {
       SyncPing *pSyncMsg = syncPingFromRpcMsg2(pRpcMsg);
       assert(pSyncMsg != NULL);
 
-      syncNodeOnPingCb(pSyncNode, pSyncMsg);
+      ret = syncNodeOnPingCb(pSyncNode, pSyncMsg);
       syncPingDestroy(pSyncMsg);
 
     } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING_REPLY) {
       SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pRpcMsg);
       assert(pSyncMsg != NULL);
 
-      syncNodeOnPingReplyCb(pSyncNode, pSyncMsg);
+      ret = syncNodeOnPingReplyCb(pSyncNode, pSyncMsg);
       syncPingReplyDestroy(pSyncMsg);
 
     } else if (pRpcMsg->msgType == TDMT_VND_SYNC_CLIENT_REQUEST) {
       SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg);
       assert(pSyncMsg != NULL);
 
-      syncNodeOnClientRequestCb(pSyncNode, pSyncMsg);
+      ret = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg);
       syncClientRequestDestroy(pSyncMsg);
 
     } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE) {
       SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pRpcMsg);
       assert(pSyncMsg != NULL);
 
-      syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg);
+      ret = syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg);
       syncRequestVoteDestroy(pSyncMsg);
 
     } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE_REPLY) {
       SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pRpcMsg);
       assert(pSyncMsg != NULL);
 
-      syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg);
+      ret = syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg);
       syncRequestVoteReplyDestroy(pSyncMsg);
 
     } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES) {
       SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pRpcMsg);
       assert(pSyncMsg != NULL);
 
-      syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg);
+      ret = syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg);
       syncAppendEntriesDestroy(pSyncMsg);
 
     } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES_REPLY) {
       SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pRpcMsg);
       assert(pSyncMsg != NULL);
 
-      syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg);
+      ret = syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg);
       syncAppendEntriesReplyDestroy(pSyncMsg);
 
     } else {
       vError("==vnodeProcessSyncReq== error msg type:%d", pRpcMsg->msgType);
+      ret = TAOS_SYNC_PROPOSE_OTHER_ERROR;
     }
 
     syncNodeRelease(pSyncNode);
   } else {
     vError("==vnodeProcessSyncReq== error syncEnv stop");
+    ret = TAOS_SYNC_PROPOSE_OTHER_ERROR;
   }
-  return 0;
+
+  return ret;
 }
 
 static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp) {
@@ -402,20 +407,32 @@ _exit:
   return rcode;
 }
 
-static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
-  // ASSERT(0);
+  SVCreateStbReq req = {0};
-#if 0
+  SDecoder       dc = {0};
-  SVCreateTbReq vAlterTbReq = {0};
+
-  vTrace("vgId:%d, process alter stb req", TD_VID(pVnode));
+  pRsp->msgType = TDMT_VND_ALTER_STB_RSP;
-  tDeserializeSVCreateTbReq(pReq, &vAlterTbReq);
+  pRsp->code = TSDB_CODE_SUCCESS;
-  // TODO: to encapsule a free API
+  pRsp->pCont = NULL;
-  taosMemoryFree(vAlterTbReq.stbCfg.pSchema);
+  pRsp->contLen = 0;
-  taosMemoryFree(vAlterTbReq.stbCfg.pTagSchema);
+
-  if (vAlterTbReq.stbCfg.pRSmaParam) {
+  tDecoderInit(&dc, pReq, len);
-    taosMemoryFree(vAlterTbReq.stbCfg.pRSmaParam);
+
+  // decode req
+  if (tDecodeSVCreateStbReq(&dc, &req) < 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    tDecoderClear(&dc);
+    return -1;
   }
-  taosMemoryFree(vAlterTbReq.name);
+
-#endif
+  if (metaAlterSTable(pVnode->pMeta, version, &req) < 0) {
+    pRsp->code = terrno;
+    tDecoderClear(&dc);
+    return -1;
+  }
+
+  tDecoderClear(&dc);
+
   return 0;
 }
 
@@ -448,9 +465,32 @@ _exit:
   return 0;
 }
 
-static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
-  // TODO
+  SVAlterTbReq vAlterTbReq = {0};
-  ASSERT(0);
+  SDecoder     dc = {0};
+
+  pRsp->msgType = TDMT_VND_ALTER_TABLE_RSP;
+  pRsp->pCont = NULL;
+  pRsp->contLen = 0;
+  pRsp->code = TSDB_CODE_SUCCESS;
+
+  tDecoderInit(&dc, pReq, len);
+
+  // decode
+  if (tDecodeSVAlterTbReq(&dc, &vAlterTbReq) < 0) {
+    pRsp->code = TSDB_CODE_INVALID_MSG;
+    tDecoderClear(&dc);
+    return -1;
+  }
+
+  // process
+  if (metaAlterTable(pVnode->pMeta, version, &vAlterTbReq) < 0) {
+    pRsp->code = terrno;
+    tDecoderClear(&dc);
+    return -1;
+  }
+
+  tDecoderClear(&dc);
   return 0;
 }
 
@@ -518,7 +558,7 @@ static int vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, SSub
     if (pSchema) {
       taosMemoryFreeClear(pSchema);
     }
-    pSchema = metaGetTbTSchema(pMeta, msgIter->suid, 0);  // TODO: use the real schema
+    pSchema = metaGetTbTSchema(pMeta, msgIter->suid, 1);  // TODO: use the real schema
     if (pSchema) {
       suid = msgIter->suid;
     }

source/libs/catalog/src/catalog.c

@@ -2576,12 +2576,6 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) {
       CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
     }
 
-    SHashObj *metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
-    if (NULL == metaCache) {
-      qError("taosHashInit failed, num:%d", gCtgMgmt.cfg.maxTblCacheNum);
-      CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
-    }
-    
     code = taosHashPut(gCtgMgmt.pCluster, &clusterId, sizeof(clusterId), &clusterCtg, POINTER_BYTES);
     if (code) {
       if (HASH_NODE_EXIST(code)) {

source/libs/executor/inc/executorimpl.h

@@ -658,6 +658,7 @@ void    initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWin
 void    cleanupAggSup(SAggSupporter* pAggSup);
 void    destroyBasicOperatorInfo(void* param, int32_t numOfOutput);
 void    appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle);
+void    setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId);
 SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode);
 SColumn extractColumnFromColumnNode(SColumnNode* pColNode);
 

source/libs/executor/src/executorMain.c

@@ -21,13 +21,14 @@
 #include "tcache.h"
 #include "tglobal.h"
 #include "tmsg.h"
+#include "tudf.h"
 
-#include "thash.h"
-#include "executorimpl.h"
 #include "executor.h"
+#include "executorimpl.h"
+#include "query.h"
+#include "thash.h"
 #include "tlosertree.h"
 #include "ttypes.h"
-#include "query.h"
 
 typedef struct STaskMgmt {
   TdThreadMutex lock;
@@ -156,6 +157,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) {
   int32_t current = (*pRes != NULL)? (*pRes)->info.rows:0;
   pTaskInfo->totalRows += current;
 
+  cleanUpUdfs();
   qDebug("%s task suspended, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms",
          GET_TASKID(pTaskInfo), current, pTaskInfo->totalRows, 0, el/1000.0);
 

source/libs/executor/src/executorimpl.c

@@ -13,7 +13,6 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <libs/function/function.h>
 #include "filter.h"
 #include "function.h"
 #include "functionMgt.h"
@@ -1601,9 +1600,6 @@ void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p) {
   }
 }
 
-static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId);
-static void         doSetTagValueInParam(void* pTable, int32_t tagColId, SVariant* tag, int16_t type, int16_t bytes);
-
 static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) {
   SqlFunctionCtx* pCtx = pTableScanInfo->pCtx;
   uint32_t        status = BLK_DATA_NOT_LOAD;
@@ -1771,100 +1767,6 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc
   return TSDB_CODE_SUCCESS;
 }
 
-/*
- * set tag value in SqlFunctionCtx
- * e.g.,tag information into input buffer
- */
-static void doSetTagValueInParam(void* pTable, int32_t tagColId, SVariant* tag, int16_t type, int16_t bytes) {
-  taosVariantDestroy(tag);
-
-  char* val = NULL;
-  //  if (tagColId == TSDB_TBNAME_COLUMN_INDEX) {
-  //    val = tsdbGetTableName(pTable);
-  //    assert(val != NULL);
-  //  } else {
-  //    val = tsdbGetTableTagVal(pTable, tagColId, type, bytes);
-  //  }
-
-  if (val == NULL || isNull(val, type)) {
-    tag->nType = TSDB_DATA_TYPE_NULL;
-    return;
-  }
-
-  if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
-    int32_t maxLen = bytes - VARSTR_HEADER_SIZE;
-    int32_t len = (varDataLen(val) > maxLen) ? maxLen : varDataLen(val);
-    taosVariantCreateFromBinary(tag, varDataVal(val), len, type);
-    // taosVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), type);
-  } else {
-    taosVariantCreateFromBinary(tag, val, bytes, type);
-  }
-}
-
-static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId) {
-  assert(pTagColList != NULL && numOfTags > 0);
-
-  for (int32_t i = 0; i < numOfTags; ++i) {
-    if (pTagColList[i].colId == colId) {
-      return &pTagColList[i];
-    }
-  }
-
-  return NULL;
-}
-
-void setTagValue(SOperatorInfo* pOperatorInfo, void* pTable, SqlFunctionCtx* pCtx, int32_t numOfOutput) {
-  SExprInfo* pExpr = pOperatorInfo->pExpr;
-  SExprInfo* pExprInfo = &pExpr[0];
-  int32_t    functionId = getExprFunctionId(pExprInfo);
-#if 0
-  if (pQueryAttr->numOfOutput == 1 && functionId == FUNCTION_TS_COMP && pQueryAttr->stableQuery) {
-    assert(pExprInfo->base.numOfParams == 1);
-
-    //    int16_t      tagColId = (int16_t)pExprInfo->base.param[0].i;
-    int16_t      tagColId = -1;
-    SColumnInfo* pColInfo = doGetTagColumnInfoById(pQueryAttr->tagColList, pQueryAttr->numOfTags, tagColId);
-
-    doSetTagValueInParam(pTable, tagColId, &pCtx[0].tag, pColInfo->type, pColInfo->bytes);
-
-  } else {
-    // set tag value, by which the results are aggregated.
-    int32_t offset = 0;
-    memset(pRuntimeEnv->tagVal, 0, pQueryAttr->tagLen);
-
-    for (int32_t idx = 0; idx < numOfOutput; ++idx) {
-      SExprInfo* pLocalExprInfo = &pExpr[idx];
-
-      // ts_comp column required the tag value for join filter
-      if (!TSDB_COL_IS_TAG(pLocalExprInfo->base.pParam[0].pCol->flag)) {
-        continue;
-      }
-
-      // todo use tag column index to optimize performance
-      doSetTagValueInParam(pTable, pLocalExprInfo->base.pParam[0].pCol->colId, &pCtx[idx].tag,
-                           pLocalExprInfo->base.resSchema.type, pLocalExprInfo->base.resSchema.bytes);
-
-      if (IS_NUMERIC_TYPE(pLocalExprInfo->base.resSchema.type) ||
-          pLocalExprInfo->base.resSchema.type == TSDB_DATA_TYPE_BOOL ||
-          pLocalExprInfo->base.resSchema.type == TSDB_DATA_TYPE_TIMESTAMP) {
-        memcpy(pRuntimeEnv->tagVal + offset, &pCtx[idx].tag.i, pLocalExprInfo->base.resSchema.bytes);
-      } else {
-        if (pCtx[idx].tag.pz != NULL) {
-          memcpy(pRuntimeEnv->tagVal + offset, pCtx[idx].tag.pz, pCtx[idx].tag.nLen);
-        }
-      }
-
-      offset += pLocalExprInfo->base.resSchema.bytes;
-    }
-  }
-
-  // set the tsBuf start position before check each data block
-  if (pRuntimeEnv->pTsBuf != NULL) {
-    setCtxTagForJoin(pRuntimeEnv, &pCtx[0], pExprInfo, pTable);
-  }
-#endif
-}
-
 void copyToSDataBlock(SSDataBlock* pBlock, int32_t* offset, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pResBuf) {
   pBlock->info.rows = 0;
 
@@ -2153,6 +2055,11 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR
       SColumnInfoData* pDst = taosArrayGet(px->pDataBlock, i);
       SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, i);
 
+      // it is a reserved column for scalar function, and no data in this column yet.
+      if (pSrc->pData == NULL) {
+        continue;
+      }
+
       int32_t numOfRows = 0;
       for (int32_t j = 0; j < totalRows; ++j) {
         if (rowRes[j] == 0) {
@@ -4038,12 +3945,6 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
   if (pProjectInfo->existDataBlock) {  // TODO refactor
     SSDataBlock* pBlock = pProjectInfo->existDataBlock;
     pProjectInfo->existDataBlock = NULL;
-    *newgroup = true;
-
-    // todo dynamic set tags
-    //    if (pTableQueryInfo != NULL) {
-    //      setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfExprs);
-    //    }
 
     // the pDataBlock are always the same one, no need to call this again
     setInputDataBlock(pOperator, pInfo->pCtx, pBlock, TSDB_ORDER_ASC);
@@ -4084,13 +3985,6 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
       }
     }
 
-    // todo set tags
-
-    //    STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current;
-    //    if (pTableQueryInfo != NULL) {
-    //      setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfExprs);
-    //    }
-
     // the pDataBlock are always the same one, no need to call this again
     int32_t code = getTableScanInfo(pOperator->pDownstream[0], &order, &scanFlag);
 
@@ -4430,10 +4324,6 @@ void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) {
   doDestroyBasicInfo(pInfo, numOfOutput);
 }
 
-void destroyMergeJoinOperator(void* param, int32_t numOfOutput) {
-  SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param;
-}
-
 void destroyAggOperatorInfo(void* param, int32_t numOfOutput) {
   SAggOperatorInfo* pInfo = (SAggOperatorInfo*)param;
   doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
@@ -4487,9 +4377,9 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p
     goto _error;
   }
 
-  pInfo->limit = *pLimit;
+  pInfo->limit      = *pLimit;
-  pInfo->slimit = *pSlimit;
+  pInfo->slimit     = *pSlimit;
-  pInfo->curOffset = pLimit->offset;
+  pInfo->curOffset  = pLimit->offset;
   pInfo->curSOffset = pSlimit->offset;
 
   pInfo->binfo.pRes = pResBlock;
@@ -4778,7 +4668,6 @@ static SArray* extractColumnInfo(SNodeList* pNodeList);
 
 static SArray* createSortInfo(SNodeList* pNodeList);
 static SArray* extractPartitionColInfo(SNodeList* pNodeList);
-static void    setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode);
 
 SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle,
                                   uint64_t queryId, uint64_t taskId, STableGroupInfo* pTableGroupInfo) {
@@ -5447,150 +5336,3 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo
 
   return TSDB_CODE_SUCCESS;
 }
-
-static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) {
-  SJoinOperatorInfo* pJoinInfo = pOperator->info;
-
-  SSDataBlock* pRes = pJoinInfo->pRes;
-  blockDataCleanup(pRes);
-  blockDataEnsureCapacity(pRes, 4096);
-
-  int32_t nrows = 0;
-
-  while (1) {
-    if (pJoinInfo->pLeft == NULL || pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) {
-      SOperatorInfo* ds1 = pOperator->pDownstream[0];
-      publishOperatorProfEvent(ds1, QUERY_PROF_BEFORE_OPERATOR_EXEC);
-      pJoinInfo->pLeft = ds1->fpSet.getNextFn(ds1);
-      publishOperatorProfEvent(ds1, QUERY_PROF_AFTER_OPERATOR_EXEC);
-
-      pJoinInfo->leftPos = 0;
-      if (pJoinInfo->pLeft == NULL) {
-        setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
-        break;
-      }
-    }
-
-    if (pJoinInfo->pRight == NULL || pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) {
-      SOperatorInfo* ds2 = pOperator->pDownstream[1];
-      publishOperatorProfEvent(ds2, QUERY_PROF_BEFORE_OPERATOR_EXEC);
-      pJoinInfo->pRight = ds2->fpSet.getNextFn(ds2);
-      publishOperatorProfEvent(ds2, QUERY_PROF_AFTER_OPERATOR_EXEC);
-
-      pJoinInfo->rightPos = 0;
-      if (pJoinInfo->pRight == NULL) {
-        setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
-        break;
-      }
-    }
-
-    SColumnInfoData* pLeftCol = taosArrayGet(pJoinInfo->pLeft->pDataBlock, pJoinInfo->leftCol.slotId);
-    char*            pLeftVal = colDataGetData(pLeftCol, pJoinInfo->leftPos);
-
-    SColumnInfoData* pRightCol = taosArrayGet(pJoinInfo->pRight->pDataBlock, pJoinInfo->rightCol.slotId);
-    char*            pRightVal = colDataGetData(pRightCol, pJoinInfo->rightPos);
-
-    // only the timestamp match support for ordinary table
-    ASSERT(pLeftCol->info.type == TSDB_DATA_TYPE_TIMESTAMP);
-    if (*(int64_t*)pLeftVal == *(int64_t*)pRightVal) {
-      for (int32_t i = 0; i < pOperator->numOfExprs; ++i) {
-        SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, i);
-
-        SExprInfo* pExprInfo = &pOperator->pExpr[i];
-
-        int32_t blockId = pExprInfo->base.pParam[0].pCol->dataBlockId;
-        int32_t slotId = pExprInfo->base.pParam[0].pCol->slotId;
-        int32_t rowIndex = -1;
-
-        SColumnInfoData* pSrc = NULL;
-        if (pJoinInfo->pLeft->info.blockId == blockId) {
-          pSrc = taosArrayGet(pJoinInfo->pLeft->pDataBlock, slotId);
-          rowIndex = pJoinInfo->leftPos;
-        } else {
-          pSrc = taosArrayGet(pJoinInfo->pRight->pDataBlock, slotId);
-          rowIndex = pJoinInfo->rightPos;
-        }
-
-        if (colDataIsNull_s(pSrc, rowIndex)) {
-          colDataAppendNULL(pDst, nrows);
-        } else {
-          char* p = colDataGetData(pSrc, rowIndex);
-          colDataAppend(pDst, nrows, p, false);
-        }
-      }
-
-      pJoinInfo->leftPos += 1;
-      pJoinInfo->rightPos += 1;
-
-      nrows += 1;
-    } else if (*(int64_t*)pLeftVal < *(int64_t*)pRightVal) {
-      pJoinInfo->leftPos += 1;
-
-      if (pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) {
-        continue;
-      }
-    } else if (*(int64_t*)pLeftVal > *(int64_t*)pRightVal) {
-      pJoinInfo->rightPos += 1;
-      if (pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) {
-        continue;
-      }
-    }
-
-    // the pDataBlock are always the same one, no need to call this again
-    pRes->info.rows = nrows;
-    if (pRes->info.rows >= pOperator->resultInfo.threshold) {
-      break;
-    }
-  }
-
-  return (pRes->info.rows > 0) ? pRes : NULL;
-}
-
-SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo,
-                                           int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition,
-                                           SExecTaskInfo* pTaskInfo) {
-  SJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SJoinOperatorInfo));
-  SOperatorInfo*     pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
-  if (pOperator == NULL || pInfo == NULL) {
-    goto _error;
-  }
-
-  initResultSizeInfo(pOperator, 4096);
-
-  pInfo->pRes = pResBlock;
-  pOperator->name = "MergeJoinOperator";
-  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_JOIN;
-  pOperator->blocking = false;
-  pOperator->status = OP_NOT_OPENED;
-  pOperator->pExpr = pExprInfo;
-  pOperator->numOfExprs = numOfCols;
-  pOperator->info = pInfo;
-  pOperator->pTaskInfo = pTaskInfo;
-
-  SOperatorNode* pNode = (SOperatorNode*)pOnCondition;
-  setJoinColumnInfo(&pInfo->leftCol, (SColumnNode*)pNode->pLeft);
-  setJoinColumnInfo(&pInfo->rightCol, (SColumnNode*)pNode->pRight);
-
-  pOperator->fpSet =
-      createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, NULL, destroyMergeJoinOperator, NULL, NULL, NULL);
-  int32_t code = appendDownstream(pOperator, pDownstream, numOfDownstream);
-  if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
-  }
-
-  return pOperator;
-
-_error:
-  taosMemoryFree(pInfo);
-  taosMemoryFree(pOperator);
-  pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
-  return NULL;
-}
-
-void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) {
-  pColumn->slotId = pColumnNode->slotId;
-  pColumn->type = pColumnNode->node.resType.type;
-  pColumn->bytes = pColumnNode->node.resType.bytes;
-  pColumn->precision = pColumnNode->node.resType.precision;
-  pColumn->scale = pColumnNode->node.resType.scale;
-}

source/libs/executor/src/joinoperator.c

@@ -0,0 +1,199 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "function.h"
+#include "os.h"
+#include "querynodes.h"
+#include "tdatablock.h"
+#include "tmsg.h"
+#include "executorimpl.h"
+#include "tcompare.h"
+#include "thash.h"
+#include "ttypes.h"
+
+static void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode);
+static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator);
+static void destroyMergeJoinOperator(void* param, int32_t numOfOutput);
+static void extractTimeCondition(SJoinOperatorInfo *Info, SLogicConditionNode* pLogicConditionNode);
+
+SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo,
+                                           int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition,
+                                           SExecTaskInfo* pTaskInfo) {
+  SJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SJoinOperatorInfo));
+  SOperatorInfo*     pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+  if (pOperator == NULL || pInfo == NULL) {
+    goto _error;
+  }
+
+  initResultSizeInfo(pOperator, 4096);
+
+  pInfo->pRes           = pResBlock;
+  pOperator->name       = "MergeJoinOperator";
+  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_JOIN;
+  pOperator->blocking   = false;
+  pOperator->status     = OP_NOT_OPENED;
+  pOperator->pExpr      = pExprInfo;
+  pOperator->numOfExprs = numOfCols;
+  pOperator->info       = pInfo;
+  pOperator->pTaskInfo  = pTaskInfo;
+
+  if (nodeType(pOnCondition) == QUERY_NODE_OPERATOR) {
+    SOperatorNode* pNode = (SOperatorNode*)pOnCondition;
+    setJoinColumnInfo(&pInfo->leftCol, (SColumnNode*)pNode->pLeft);
+    setJoinColumnInfo(&pInfo->rightCol, (SColumnNode*)pNode->pRight);
+  } else if (nodeType(pOnCondition) == QUERY_NODE_LOGIC_CONDITION) {
+    extractTimeCondition(pInfo, (SLogicConditionNode*) pOnCondition);
+  }
+
+  pOperator->fpSet =
+      createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, NULL, destroyMergeJoinOperator, NULL, NULL, NULL);
+  int32_t code = appendDownstream(pOperator, pDownstream, numOfDownstream);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
+
+  return pOperator;
+
+  _error:
+  taosMemoryFree(pInfo);
+  taosMemoryFree(pOperator);
+  pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
+  return NULL;
+}
+
+void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) {
+  pColumn->slotId = pColumnNode->slotId;
+  pColumn->type = pColumnNode->node.resType.type;
+  pColumn->bytes = pColumnNode->node.resType.bytes;
+  pColumn->precision = pColumnNode->node.resType.precision;
+  pColumn->scale = pColumnNode->node.resType.scale;
+}
+
+void destroyMergeJoinOperator(void* param, int32_t numOfOutput) {
+  SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param;
+}
+
+SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) {
+  SJoinOperatorInfo* pJoinInfo = pOperator->info;
+
+  SSDataBlock* pRes = pJoinInfo->pRes;
+  blockDataCleanup(pRes);
+  blockDataEnsureCapacity(pRes, 4096);
+
+  int32_t nrows = 0;
+
+  while (1) {
+    // todo extract method
+    if (pJoinInfo->pLeft == NULL || pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) {
+      SOperatorInfo* ds1 = pOperator->pDownstream[0];
+      publishOperatorProfEvent(ds1, QUERY_PROF_BEFORE_OPERATOR_EXEC);
+      pJoinInfo->pLeft = ds1->fpSet.getNextFn(ds1);
+      publishOperatorProfEvent(ds1, QUERY_PROF_AFTER_OPERATOR_EXEC);
+
+      pJoinInfo->leftPos = 0;
+      if (pJoinInfo->pLeft == NULL) {
+        setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
+        break;
+      }
+    }
+
+    if (pJoinInfo->pRight == NULL || pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) {
+      SOperatorInfo* ds2 = pOperator->pDownstream[1];
+      publishOperatorProfEvent(ds2, QUERY_PROF_BEFORE_OPERATOR_EXEC);
+      pJoinInfo->pRight = ds2->fpSet.getNextFn(ds2);
+      publishOperatorProfEvent(ds2, QUERY_PROF_AFTER_OPERATOR_EXEC);
+
+      pJoinInfo->rightPos = 0;
+      if (pJoinInfo->pRight == NULL) {
+        setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
+        break;
+      }
+    }
+
+    SColumnInfoData* pLeftCol = taosArrayGet(pJoinInfo->pLeft->pDataBlock, pJoinInfo->leftCol.slotId);
+    char*            pLeftVal = colDataGetData(pLeftCol, pJoinInfo->leftPos);
+
+    SColumnInfoData* pRightCol = taosArrayGet(pJoinInfo->pRight->pDataBlock, pJoinInfo->rightCol.slotId);
+    char*            pRightVal = colDataGetData(pRightCol, pJoinInfo->rightPos);
+
+    // only the timestamp match support for ordinary table
+    ASSERT(pLeftCol->info.type == TSDB_DATA_TYPE_TIMESTAMP);
+    if (*(int64_t*)pLeftVal == *(int64_t*)pRightVal) {
+      for (int32_t i = 0; i < pOperator->numOfExprs; ++i) {
+        SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, i);
+
+        SExprInfo* pExprInfo = &pOperator->pExpr[i];
+
+        int32_t blockId = pExprInfo->base.pParam[0].pCol->dataBlockId;
+        int32_t slotId = pExprInfo->base.pParam[0].pCol->slotId;
+        int32_t rowIndex = -1;
+
+        SColumnInfoData* pSrc = NULL;
+        if (pJoinInfo->pLeft->info.blockId == blockId) {
+          pSrc = taosArrayGet(pJoinInfo->pLeft->pDataBlock, slotId);
+          rowIndex = pJoinInfo->leftPos;
+        } else {
+          pSrc = taosArrayGet(pJoinInfo->pRight->pDataBlock, slotId);
+          rowIndex = pJoinInfo->rightPos;
+        }
+
+        if (colDataIsNull_s(pSrc, rowIndex)) {
+          colDataAppendNULL(pDst, nrows);
+        } else {
+          char* p = colDataGetData(pSrc, rowIndex);
+          colDataAppend(pDst, nrows, p, false);
+        }
+      }
+
+      pJoinInfo->leftPos += 1;
+      pJoinInfo->rightPos += 1;
+
+      nrows += 1;
+    } else if (*(int64_t*)pLeftVal < *(int64_t*)pRightVal) {
+      pJoinInfo->leftPos += 1;
+
+      if (pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) {
+        continue;
+      }
+    } else if (*(int64_t*)pLeftVal > *(int64_t*)pRightVal) {
+      pJoinInfo->rightPos += 1;
+      if (pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) {
+        continue;
+      }
+    }
+
+    // the pDataBlock are always the same one, no need to call this again
+    pRes->info.rows = nrows;
+    if (pRes->info.rows >= pOperator->resultInfo.threshold) {
+      break;
+    }
+  }
+
+  return (pRes->info.rows > 0) ? pRes : NULL;
+}
+
+static void extractTimeCondition(SJoinOperatorInfo *pInfo, SLogicConditionNode* pLogicConditionNode) {
+  int32_t len = LIST_LENGTH(pLogicConditionNode->pParameterList);
+
+  for(int32_t i = 0; i < len; ++i) {
+    SNode* pNode = nodesListGetNode(pLogicConditionNode->pParameterList, i);
+    if (nodeType(pNode) == QUERY_NODE_OPERATOR) {
+      SOperatorNode* pn1 = (SOperatorNode*)pNode;
+      setJoinColumnInfo(&pInfo->leftCol, (SColumnNode*)pn1->pLeft);
+      setJoinColumnInfo(&pInfo->rightCol, (SColumnNode*)pn1->pRight);
+      break;
+    }
+  }
+}

source/libs/executor/src/scanoperator.c

@@ -291,20 +291,7 @@ void addTagPseudoColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock)
 
     // this is to handle the tbname
     if (fmIsScanPseudoColumnFunc(functionId)) {
-      struct SScalarFuncExecFuncs fpSet = {0};
+      setTbNameColData(pTableScanInfo->readHandle.meta, pBlock, pColInfoData, functionId);
-      fmGetScalarFuncExecFuncs(functionId, &fpSet);
-
-      SColumnInfoData infoData = {0};
-      infoData.info.type = TSDB_DATA_TYPE_BIGINT;
-      infoData.info.bytes = sizeof(uint64_t);
-      colInfoDataEnsureCapacity(&infoData, 0, 1);
-
-      colDataAppendInt64(&infoData, 0, &pBlock->info.uid);
-      SScalarParam srcParam = {
-          .numOfRows = pBlock->info.rows, .param = pTableScanInfo->readHandle.meta, .columnData = &infoData};
-
-      SScalarParam param = {.columnData = pColInfoData};
-      fpSet.process(&srcParam, 1, &param);
     } else {  // these are tags
       const char* p = metaGetTableTagVal(&mr.me, pExpr->base.pParam[0].pCol->colId);
       for (int32_t i = 0; i < pBlock->info.rows; ++i) {
@@ -316,6 +303,23 @@ void addTagPseudoColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock)
   metaReaderClear(&mr);
 }
 
+void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId) {
+  struct SScalarFuncExecFuncs fpSet = {0};
+  fmGetScalarFuncExecFuncs(functionId, &fpSet);
+
+  SColumnInfoData infoData = {0};
+  infoData.info.type = TSDB_DATA_TYPE_BIGINT;
+  infoData.info.bytes = sizeof(uint64_t);
+  colInfoDataEnsureCapacity(&infoData, 0, 1);
+
+  colDataAppendInt64(&infoData, 0, (int64_t*) &pBlock->info.uid);
+  SScalarParam srcParam = {
+      .numOfRows = pBlock->info.rows, .param = pMeta, .columnData = &infoData};
+
+  SScalarParam param = {.columnData = pColInfoData};
+  fpSet.process(&srcParam, 1, &param);
+}
+
 static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
   STableScanInfo* pTableScanInfo = pOperator->info;
   SSDataBlock*    pBlock = pTableScanInfo->pResBlock;

source/libs/executor/src/timewindowoperator.c

@@ -773,7 +773,6 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
       break;
     }
 
-    //    setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfExprs);
     // the pDataBlock are always the same one, no need to call this again
     setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, MAIN_SCAN, true);
     STableQueryInfo* pTableQueryInfo = pInfo->pCurrent;
@@ -1062,8 +1061,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
 
     // The timewindows that overlaps the timestamps of the input pBlock need to be recalculated and return to the
     // caller. Note that all the time window are not close till now.
-
-    //    setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfExprs);
     // the pDataBlock are always the same one, no need to call this again
     setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, MAIN_SCAN, true);
     if (pInfo->invertible) {
@@ -1377,7 +1374,6 @@ static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator) {
       break;
     }
 
-    //    setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfExprs);
     // the pDataBlock are always the same one, no need to call this again
     setInputDataBlock(pOperator, pSliceInfo->binfo.pCtx, pBlock, order, MAIN_SCAN, true);
     //    hashAllIntervalAgg(pOperator, &pSliceInfo->binfo.resultRowInfo, pBlock, 0);

source/libs/function/inc/builtinsimpl.h

@@ -105,7 +105,12 @@ int32_t mavgFunction(SqlFunctionCtx* pCtx);
 bool getSampleFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool sampleFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t sampleFunction(SqlFunctionCtx* pCtx);
-int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+//int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+
+bool getTailFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
+bool tailFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
+int32_t tailFunction(SqlFunctionCtx* pCtx);
+int32_t tailFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 
 bool getSelectivityFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 

source/libs/function/src/builtins.c

@@ -386,6 +386,35 @@ static int32_t translateSample(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t translateTail(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
+  if (2 != numOfParams && 3 != numOfParams) {
+    return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
+  }
+
+  SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0);
+  if (QUERY_NODE_COLUMN != nodeType(pPara)) {
+    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                           "The input parameter of TAIL function can only be column");
+  }
+
+  for (int32_t i = 1; i < numOfParams; ++i) {
+    uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, i))->resType.type;
+    if (!IS_INTEGER_TYPE(paraType)) {
+      return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+  }
+
+  SExprNode* pCol = (SExprNode*)nodesListGetNode(pFunc->pParameterList, 0);
+  uint8_t colType = pCol->resType.type;
+  if (IS_VAR_DATA_TYPE(colType)) {
+    pFunc->node.resType = (SDataType){.bytes = pCol->resType.bytes, .type = colType};
+  } else {
+    pFunc->node.resType = (SDataType){.bytes = tDataTypes[colType].bytes, .type = colType};
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t translateLastRow(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   // todo
   return TSDB_CODE_SUCCESS;
@@ -850,6 +879,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .processFunc  = sampleFunction,
     .finalizeFunc = NULL
   },
+  {
+    .name = "tail",
+    .type = FUNCTION_TYPE_TAIL,
+    .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC,
+    .translateFunc = translateTail,
+    .getEnvFunc   = getTailFuncEnv,
+    .initFunc     = tailFunctionSetup,
+    .processFunc  = tailFunction,
+    .finalizeFunc = tailFinalize
+  },
   {
     .name = "abs",
     .type = FUNCTION_TYPE_ABS,

source/libs/function/src/builtinsimpl.c

@@ -18,12 +18,15 @@
 #include "function.h"
 #include "querynodes.h"
 #include "taggfunction.h"
+#include "tcompare.h"
 #include "tdatablock.h"
 #include "tpercentile.h"
 
 #define HISTOGRAM_MAX_BINS_NUM   1000
 #define MAVG_MAX_POINTS_NUM      1000
 #define SAMPLE_MAX_POINTS_NUM    1000
+#define TAIL_MAX_POINTS_NUM      100
+#define TAIL_MAX_OFFSET          100
 
 typedef struct SSumRes {
   union {
@@ -161,6 +164,21 @@ typedef struct SSampleInfo {
   int64_t *timestamp;
 } SSampleInfo;
 
+typedef struct STailItem {
+  int64_t timestamp;
+  bool    isNull;
+  char    data[];
+} STailItem;
+
+typedef struct STailInfo {
+  int32_t   numOfPoints;
+  int32_t   numAdded;
+  int32_t   offset;
+  uint8_t   colType;
+  int16_t   colBytes;
+  STailItem **pItems;
+} STailInfo;
+
 #define SET_VAL(_info, numOfElem, res) \
   do {                                 \
     if ((numOfElem) <= 0) {            \
@@ -3107,7 +3125,7 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) {
 
   int32_t startOffset = pCtx->offset;
   for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) {
-    if (colDataIsNull_f(pInputCol->nullbitmap, i)) {
+    if (colDataIsNull_s(pInputCol, i)) {
       //colDataAppendNULL(pOutput, i);
       continue;
     }
@@ -3141,3 +3159,132 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) {
 //
 //  return pResInfo->numOfRes;
 //}
+
+
+bool getTailFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
+  SColumnNode* pCol = (SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0);
+  SValueNode*  pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1);
+  int32_t numOfPoints = pVal->datum.i;
+  pEnv->calcMemSize = sizeof(STailInfo) + numOfPoints * (POINTER_BYTES + sizeof(STailItem) + pCol->node.resType.bytes);
+  return true;
+}
+
+bool tailFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo *pResultInfo) {
+  if (!functionSetup(pCtx, pResultInfo)) {
+    return false;
+  }
+
+  STailInfo *pInfo = GET_ROWCELL_INTERBUF(pResultInfo);
+  pInfo->numAdded = 0;
+  pInfo->numOfPoints = pCtx->param[1].param.i;
+  if (pCtx->numOfParams == 4) {
+    pInfo->offset = pCtx->param[2].param.i;
+  } else {
+    pInfo->offset = 0;
+  }
+  pInfo->colType = pCtx->resDataInfo.type;
+  pInfo->colBytes = pCtx->resDataInfo.bytes;
+  if ((pInfo->numOfPoints < 1 || pInfo->numOfPoints > TAIL_MAX_POINTS_NUM) ||
+      (pInfo->numOfPoints < 0 || pInfo->numOfPoints > TAIL_MAX_OFFSET)) {
+    return false;
+  }
+
+  pInfo->pItems = (STailItem **)((char *)pInfo + sizeof(STailInfo));
+  char *pItem = (char *)pInfo->pItems + pInfo->numOfPoints * POINTER_BYTES;
+
+  size_t unitSize = sizeof(STailItem) + pInfo->colBytes;
+  for (int32_t i = 0; i < pInfo->numOfPoints; ++i) {
+    pInfo->pItems[i] = (STailItem *)(pItem + i * unitSize);
+    pInfo->pItems[i]->isNull = false;
+  }
+
+  return true;
+}
+
+static void tailAssignResult(STailItem* pItem, char *data, int32_t colBytes, TSKEY ts, bool isNull) {
+  pItem->timestamp = ts;
+  if (isNull) {
+    pItem->isNull = true;
+  } else {
+    memcpy(pItem->data, data, colBytes);
+  }
+}
+
+static int32_t tailCompFn(const void *p1, const void *p2, const void *param) {
+  STailItem *d1 = *(STailItem **)p1;
+  STailItem *d2 = *(STailItem **)p2;
+  return compareInt64Val(&d1->timestamp, &d2->timestamp);
+}
+
+static void doTailAdd(STailInfo* pInfo, char *data, TSKEY ts, bool isNull) {
+  STailItem **pList = pInfo->pItems;
+  if (pInfo->numAdded < pInfo->numOfPoints) {
+    tailAssignResult(pList[pInfo->numAdded], data, pInfo->colBytes, ts, isNull);
+    taosheapsort((void *)pList, sizeof(STailItem **), pInfo->numAdded + 1, NULL, tailCompFn, 0);
+    pInfo->numAdded++;
+  } else if (pList[0]->timestamp < ts) {
+    tailAssignResult(pList[0], data, pInfo->colBytes, ts, isNull);
+    taosheapadjust((void *)pList, sizeof(STailItem **), 0, pInfo->numOfPoints - 1, NULL, tailCompFn, NULL, 0);
+  }
+}
+
+int32_t tailFunction(SqlFunctionCtx* pCtx) {
+  SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
+  STailInfo*         pInfo = GET_ROWCELL_INTERBUF(pResInfo);
+
+  SInputColumnInfoData* pInput = &pCtx->input;
+  TSKEY* tsList = (int64_t*)pInput->pPTS->pData;
+
+  SColumnInfoData* pInputCol = pInput->pData[0];
+  SColumnInfoData* pTsOutput = pCtx->pTsOutput;
+  SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput;
+
+  int32_t startOffset = pCtx->offset;
+  if (pInfo->offset >= pInput->numOfRows) {
+    return 0;
+  } else {
+    pInfo->numOfPoints = MIN(pInfo->numOfPoints, pInput->numOfRows - pInfo->offset);
+  }
+  for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex - pInfo->offset; i += 1) {
+
+    char* data = colDataGetData(pInputCol, i);
+    doTailAdd(pInfo, data, tsList[i], colDataIsNull_s(pInputCol, i));
+  }
+
+  taosqsort(pInfo->pItems, pInfo->numOfPoints, POINTER_BYTES, NULL, tailCompFn);
+
+  for (int32_t i = 0; i < pInfo->numOfPoints; ++i) {
+    int32_t pos = startOffset + i;
+    STailItem *pItem = pInfo->pItems[i];
+    if (pItem->isNull) {
+      colDataAppendNULL(pOutput, pos);
+    } else {
+      colDataAppend(pOutput, pos, pItem->data, false);
+    }
+  }
+
+  return pInfo->numOfPoints;
+}
+
+int32_t tailFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
+  SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx);
+  STailInfo*                pInfo = GET_ROWCELL_INTERBUF(pEntryInfo);
+  pEntryInfo->complete = true;
+
+  int32_t type = pCtx->input.pData[0]->info.type;
+  int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
+
+  SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
+
+  // todo assign the tag value and the corresponding row data
+  int32_t currentRow = pBlock->info.rows;
+  for (int32_t i = 0; i < pEntryInfo->numOfRes; ++i) {
+    STailItem *pItem = pInfo->pItems[i];
+    colDataAppend(pCol, currentRow, pItem->data, false);
+
+    //setSelectivityValue(pCtx, pBlock, &pInfo->pItems[i].tuplePos, currentRow);
+    currentRow += 1;
+  }
+
+  return pEntryInfo->numOfRes;
+}

source/libs/function/src/tudf.c

@@ -25,8 +25,6 @@
 #include "functionMgt.h"
 
 //TODO: add unit test
-//TODO: include all global variable under context struct
-
 typedef struct SUdfdData {
   bool          startCalled;
   bool          needCleanUp;
@@ -313,6 +311,8 @@ int64_t gUdfTaskSeqNum = 0;
 typedef struct SUdfcFuncStub {
   char udfName[TSDB_FUNC_NAME_LEN];
   UdfcFuncHandle handle;
+  int32_t refCount;
+  int64_t lastRefTime;
 } SUdfcFuncStub;
 
 typedef struct SUdfcProxy {
@@ -338,7 +338,7 @@ typedef struct SUdfcProxy {
 
 SUdfcProxy gUdfdProxy = {0};
 
-typedef struct SClientUdfUvSession {
+typedef struct SUdfcUvSession {
   SUdfcProxy *udfc;
   int64_t severHandle;
   uv_pipe_t *udfUvPipe;
@@ -346,7 +346,9 @@ typedef struct SClientUdfUvSession {
   int8_t  outputType;
   int32_t outputLen;
   int32_t bufSize;
-} SClientUdfUvSession;
+
+  char udfName[TSDB_FUNC_NAME_LEN];
+} SUdfcUvSession;
 
 typedef struct SClientUvTaskNode {
   SUdfcProxy *udfc;
@@ -369,7 +371,7 @@ typedef struct SClientUvTaskNode {
 typedef struct SClientUdfTask {
   int8_t type;
 
-  SClientUdfUvSession *session;
+  SUdfcUvSession *session;
 
   int32_t errCode;
 
@@ -401,7 +403,7 @@ typedef struct SClientUvConn {
   uv_pipe_t *pipe;
   QUEUE taskQueue;
   SClientConnBuf readBuf;
-  SClientUdfUvSession *session;
+  SUdfcUvSession *session;
 } SClientUvConn;
 
 enum {
@@ -825,11 +827,6 @@ void onUdfcPipeClose(uv_handle_t *handle) {
   taosMemoryFree(conn->readBuf.buf);
   taosMemoryFree(conn);
   taosMemoryFree((uv_pipe_t *) handle);
-
-  //clear the udf handles cache
-  uv_mutex_lock(&gUdfdProxy.udfStubsMutex);
-  taosArrayClear(gUdfdProxy.udfStubs);
-  uv_mutex_unlock(&gUdfdProxy.udfStubsMutex);
 }
 
 int32_t udfcGetUdfTaskResultFromUvTask(SClientUdfTask *task, SClientUvTaskNode *uvTask) {
@@ -1277,13 +1274,12 @@ int32_t udfcRunUdfUvTask(SClientUdfTask *task, int8_t uvTaskType) {
 }
 
 int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) {
-  fnInfo("udfc setup udf. udfName: %s", udfName);
   if (gUdfdProxy.udfcState != UDFC_STATE_READY) {
     return TSDB_CODE_UDF_INVALID_STATE;
   }
   SClientUdfTask *task = taosMemoryCalloc(1,sizeof(SClientUdfTask));
   task->errCode = 0;
-  task->session = taosMemoryCalloc(1, sizeof(SClientUdfUvSession));
+  task->session = taosMemoryCalloc(1, sizeof(SUdfcUvSession));
   task->session->udfc = &gUdfdProxy;
   task->type = UDF_TASK_SETUP;
 
@@ -1303,10 +1299,11 @@ int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) {
   task->session->outputType = rsp->outputType;
   task->session->outputLen = rsp->outputLen;
   task->session->bufSize = rsp->bufSize;
+  strcpy(task->session->udfName, udfName);
   if (task->errCode != 0) {
     fnError("failed to setup udf. udfname: %s, err: %d", udfName, task->errCode)
   } else {
-    fnInfo("sucessfully setup udf func handle. handle: %p", task->session);
+    fnInfo("sucessfully setup udf func handle. udfName: %s, handle: %p", udfName, task->session);
     *funcHandle = task->session;
   }
   int32_t err = task->errCode;
@@ -1317,14 +1314,14 @@ int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) {
 int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2,
                 SSDataBlock* output, SUdfInterBuf *newState) {
   fnTrace("udfc call udf. callType: %d, funcHandle: %p", callType, handle);
-  SClientUdfUvSession *session = (SClientUdfUvSession *) handle;
+  SUdfcUvSession *session = (SUdfcUvSession *) handle;
   if (session->udfUvPipe == NULL) {
     fnError("No pipe to udfd");
     return TSDB_CODE_UDF_PIPE_NO_PIPE;
   }
   SClientUdfTask *task = taosMemoryCalloc(1, sizeof(SClientUdfTask));
   task->errCode = 0;
-  task->session = (SClientUdfUvSession *) handle;
+  task->session = (SUdfcUvSession *) handle;
   task->type = UDF_TASK_CALL;
 
   SUdfCallRequest *req = &task->_call.req;
@@ -1440,7 +1437,7 @@ int compareUdfcFuncSub(const void* elem1, const void* elem2) {
   return strcmp(stub1->udfName, stub2->udfName);
 }
 
-int32_t setupUdf(char* udfName, UdfcFuncHandle* pHandle) {
+int32_t accquireUdfFuncHandle(char* udfName, UdfcFuncHandle* pHandle) {
   int32_t code = 0;
   uv_mutex_lock(&gUdfdProxy.udfStubsMutex);
   SUdfcFuncStub key = {0};
@@ -1449,6 +1446,8 @@ int32_t setupUdf(char* udfName, UdfcFuncHandle* pHandle) {
   if (foundStub != NULL) {
     uv_mutex_unlock(&gUdfdProxy.udfStubsMutex);
     *pHandle = foundStub->handle;
+    ++foundStub->refCount;
+    foundStub->lastRefTime = taosGetTimestampUs();
     return 0;
   }
   *pHandle = NULL;
@@ -1457,6 +1456,8 @@ int32_t setupUdf(char* udfName, UdfcFuncHandle* pHandle) {
     SUdfcFuncStub stub = {0};
     strcpy(stub.udfName, udfName);
     stub.handle = *pHandle;
+    ++stub.refCount;
+    stub.lastRefTime = taosGetTimestampUs();
     taosArrayPush(gUdfdProxy.udfStubs, &stub);
     taosArraySort(gUdfdProxy.udfStubs, compareUdfcFuncSub);
   } else {
@@ -1467,23 +1468,33 @@ int32_t setupUdf(char* udfName, UdfcFuncHandle* pHandle) {
   return code;
 }
 
+void releaseUdfFuncHandle(char* udfName) {
+  uv_mutex_lock(&gUdfdProxy.udfStubsMutex);
+  SUdfcFuncStub key = {0};
+  strcpy(key.udfName, udfName);
+  SUdfcFuncStub *foundStub = taosArraySearch(gUdfdProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ);
+  ASSERT(foundStub);
+  --foundStub->refCount;
+  ASSERT(foundStub->refCount>=0);
+  uv_mutex_unlock(&gUdfdProxy.udfStubsMutex);
+}
+
 int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, SScalarParam *output) {
   UdfcFuncHandle handle = NULL;
-  int32_t code = setupUdf(udfName, &handle);
+  int32_t code = accquireUdfFuncHandle(udfName, &handle);
   if (code != 0) {
     return code;
   }
   code = doCallUdfScalarFunc(handle, input, numOfCols, output);
+  releaseUdfFuncHandle(udfName);
   return code;
 }
 
-//TODO: when to teardown udf. teardown udf is not called
 int32_t doTeardownUdf(UdfcFuncHandle handle) {
-  fnInfo("tear down udf. udf func handle: %p", handle);
+  SUdfcUvSession *session = (SUdfcUvSession *) handle;
 
-  SClientUdfUvSession *session = (SClientUdfUvSession *) handle;
   if (session->udfUvPipe == NULL) {
-    fnError("pipe to udfd does not exist");
+    fnError("tear down udf. pipe to udfd does not exist. udf name: %s", session->udfName);
     return TSDB_CODE_UDF_PIPE_NO_PIPE;
   }
 
@@ -1498,7 +1509,6 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) {
   udfcRunUdfUvTask(task, UV_TASK_REQ_RSP);
 
   SUdfTeardownResponse *rsp = &task->_teardown.rsp;
-
   int32_t err = task->errCode;
 
   udfcRunUdfUvTask(task, UV_TASK_DISCONNECT);
@@ -1506,12 +1516,14 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) {
   taosMemoryFree(task->session);
   taosMemoryFree(task);
 
+  fnInfo("tear down udf. udf name: %s, udf func handle: %p", session->udfName, handle);
+
   return err;
 }
 
 //memory layout |---SUdfAggRes----|-----final result-----|---inter result----|
 typedef struct SUdfAggRes {
-  SClientUdfUvSession *session;
+  SUdfcUvSession *session;
   int8_t finalResNum;
   int8_t interResNum;
   char* finalResBuf;
@@ -1532,11 +1544,11 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult
   }
   UdfcFuncHandle handle;
   int32_t udfCode = 0;
-  if ((udfCode = setupUdf((char *)pCtx->udfName, &handle)) != 0) {
+  if ((udfCode = accquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) {
     fnError("udfAggInit error. step doSetupUdf. udf code: %d", udfCode);
     return false;
   }
-  SClientUdfUvSession *session = (SClientUdfUvSession *)handle;
+  SUdfcUvSession *session = (SUdfcUvSession *)handle;
   SUdfAggRes *udfRes = (SUdfAggRes*)GET_ROWCELL_INTERBUF(pResultCellInfo);
   int32_t envSize = sizeof(SUdfAggRes) + session->outputLen + session->bufSize;
   memset(udfRes, 0, envSize);
@@ -1544,7 +1556,7 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult
   udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes);
   udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen;
 
-  udfRes->session = (SClientUdfUvSession *)handle;
+  udfRes->session = (SUdfcUvSession *)handle;
   SUdfInterBuf buf = {0};
   if ((udfCode = doCallUdfAggInit(handle, &buf)) != 0) {
     fnError("udfAggInit error. step doCallUdfAggInit. udf code: %d", udfCode);
@@ -1560,7 +1572,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
   int32_t numOfCols = pInput->numOfInputCols;
 
   SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
-  SClientUdfUvSession *session = udfRes->session;
+  SUdfcUvSession *session = udfRes->session;
   if (session == NULL) {
     return TSDB_CODE_UDF_NO_FUNC_HANDLE;
   }
@@ -1615,7 +1627,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
 
 int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) {
   SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
-  SClientUdfUvSession *session = udfRes->session;
+  SUdfcUvSession *session = udfRes->session;
   if (session == NULL) {
     return TSDB_CODE_UDF_NO_FUNC_HANDLE;
   }
@@ -1638,11 +1650,29 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) {
     GET_RES_INFO(pCtx)->numOfRes = udfRes->finalResNum;
   }
 
-//  int32_t code = doTeardownUdf(session);
-//  if (code != 0) {
-//    fnError("udfAggFinalize error. doTeardownUdf step. udf code: %d", code);
-//  }
-
   int32_t numOfResults = functionFinalizeWithResultBuf(pCtx, pBlock, udfRes->finalResBuf);
+  releaseUdfFuncHandle(pCtx->udfName);
   return udfCallCode == 0 ? numOfResults : udfCallCode;
+}
+
+int32_t cleanUpUdfs() {
+  uv_mutex_lock(&gUdfdProxy.udfStubsMutex);
+  int32_t i = 0;
+  SArray* udfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub));
+  while (i < taosArrayGetSize(gUdfdProxy.udfStubs)) {
+    SUdfcFuncStub *stub = taosArrayGet(gUdfdProxy.udfStubs, i);
+    if (stub->refCount == 0) {
+      fnInfo("tear down udf. udf name: %s, handle: %p", stub->udfName, stub->handle);
+      doTeardownUdf(stub->handle);
+    } else {
+      fnInfo("udf still in use. udf name: %s, ref count: %d, last ref time: %"PRId64", handle: %p",
+             stub->udfName, stub->refCount, stub->lastRefTime, stub->handle);
+      taosArrayPush(udfStubs, stub);
+    }
+    ++i;
+  }
+  taosArrayDestroy(gUdfdProxy.udfStubs);
+  gUdfdProxy.udfStubs = udfStubs;
+  uv_mutex_unlock(&gUdfdProxy.udfStubsMutex);
+  return 0;
 }

source/libs/function/src/udfd.c

@@ -485,7 +485,146 @@ void udfdIntrSignalHandler(uv_signal_t *handle, int signum) {
   uv_stop(global.loop);
 }
 
-void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { return; }
+typedef enum EUdfdRpcReqRspType {
+  UDFD_RPC_MNODE_CONNECT = 0,
+  UDFD_RPC_RETRIVE_FUNC,
+} EUdfdRpcReqRspType;
+
+typedef struct SUdfdRpcSendRecvInfo {
+  EUdfdRpcReqRspType rpcType;
+  int32_t code;
+  void* param;
+  uv_sem_t resultSem;
+} SUdfdRpcSendRecvInfo;
+
+
+void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
+  SUdfdRpcSendRecvInfo *msgInfo = (SUdfdRpcSendRecvInfo *)pMsg->ahandle;
+  ASSERT(pMsg->ahandle != NULL);
+
+  if (pEpSet) {
+    if (!isEpsetEqual(&global.mgmtEp.epSet, pEpSet)) {
+      updateEpSet_s(&global.mgmtEp, pEpSet);
+    }
+  }
+
+  if (pMsg->code != TSDB_CODE_SUCCESS) {
+    fnError("udfd rpc error. code: %s", tstrerror(pMsg->code));
+    msgInfo->code = pMsg->code;
+    goto _return;
+  }
+
+  if (msgInfo->rpcType == UDFD_RPC_MNODE_CONNECT) {
+    SConnectRsp connectRsp = {0};
+    tDeserializeSConnectRsp(pMsg->pCont, pMsg->contLen, &connectRsp);
+    if (connectRsp.epSet.numOfEps == 0) {
+      msgInfo->code = TSDB_CODE_MND_APP_ERROR;
+      goto _return;
+    }
+
+    if (connectRsp.dnodeNum > 1 && !isEpsetEqual(&global.mgmtEp.epSet, &connectRsp.epSet)) {
+      updateEpSet_s(&global.mgmtEp, &connectRsp.epSet);
+    }
+    msgInfo->code = 0;
+  } else if (msgInfo->rpcType == UDFD_RPC_RETRIVE_FUNC) {
+    SRetrieveFuncRsp retrieveRsp = {0};
+    tDeserializeSRetrieveFuncRsp(pMsg->pCont, pMsg->contLen, &retrieveRsp);
+
+    SFuncInfo *pFuncInfo = (SFuncInfo *)taosArrayGet(retrieveRsp.pFuncInfos, 0);
+    SUdf* udf = msgInfo->param;
+    udf->funcType = pFuncInfo->funcType;
+    udf->scriptType = pFuncInfo->scriptType;
+    udf->outputType = pFuncInfo->funcType;
+    udf->outputLen = pFuncInfo->outputLen;
+    udf->bufSize = pFuncInfo->bufSize;
+
+    char path[PATH_MAX] = {0};
+    snprintf(path, sizeof(path), "%s/lib%s.so", "/tmp", pFuncInfo->name);
+    TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL);
+    // TODO check for failure of flush to disk
+    taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize);
+    taosCloseFile(&file);
+    strncpy(udf->path, path, strlen(path));
+    taosArrayDestroy(retrieveRsp.pFuncInfos);
+    msgInfo->code = 0;
+  }
+
+_return:
+  rpcFreeCont(pMsg->pCont);
+  uv_sem_post(&msgInfo->resultSem);
+  return;
+}
+
+int32_t udfdConnectToMNode() {
+  SConnectReq connReq = {0};
+  connReq.connType = CONN_TYPE__UDFD;
+  tstrncpy(connReq.app, "udfd",sizeof(connReq.app));
+  tstrncpy(connReq.user, TSDB_DEFAULT_USER, sizeof(connReq.user));
+  char pass[TSDB_PASSWORD_LEN + 1] = {0};
+  taosEncryptPass_c((uint8_t *)(TSDB_DEFAULT_PASS), strlen(TSDB_DEFAULT_PASS), pass);
+  tstrncpy(connReq.passwd, pass, sizeof(connReq.passwd));
+  connReq.pid = htonl(taosGetPId());
+  connReq.startTime = htobe64(taosGetTimestampMs());
+
+  int32_t contLen = tSerializeSConnectReq(NULL, 0, &connReq);
+  void*   pReq = rpcMallocCont(contLen);
+  tSerializeSConnectReq(pReq, contLen, &connReq);
+
+  SUdfdRpcSendRecvInfo *msgInfo = taosMemoryCalloc(1, sizeof(SUdfdRpcSendRecvInfo));
+  msgInfo->rpcType = UDFD_RPC_MNODE_CONNECT;
+  uv_sem_init(&msgInfo->resultSem, 0);
+
+  SRpcMsg rpcMsg = {0};
+  rpcMsg.msgType = TDMT_MND_CONNECT;
+  rpcMsg.pCont = pReq;
+  rpcMsg.contLen = contLen;
+  rpcMsg.ahandle = msgInfo;
+  rpcSendRequest(global.clientRpc, &global.mgmtEp.epSet, &rpcMsg, NULL);
+
+  uv_sem_wait(&msgInfo->resultSem);
+  int32_t code = msgInfo->code;
+  uv_sem_destroy(&msgInfo->resultSem);
+  taosMemoryFree(msgInfo);
+  return code;
+}
+
+int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf) {
+  SRetrieveFuncReq retrieveReq = {0};
+  retrieveReq.numOfFuncs = 1;
+  retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
+  taosArrayPush(retrieveReq.pFuncNames, udfName);
+
+  int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
+  void   *pReq = rpcMallocCont(contLen);
+  tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq);
+  taosArrayDestroy(retrieveReq.pFuncNames);
+
+  SUdfdRpcSendRecvInfo* msgInfo = taosMemoryCalloc(1, sizeof(SUdfdRpcSendRecvInfo));
+  msgInfo->rpcType = UDFD_RPC_RETRIVE_FUNC;
+  msgInfo->param = udf;
+  uv_sem_init(&msgInfo->resultSem, 0);
+
+  SRpcMsg rpcMsg = {0};
+  rpcMsg.pCont = pReq;
+  rpcMsg.contLen = contLen;
+  rpcMsg.msgType = TDMT_MND_RETRIEVE_FUNC;
+  rpcMsg.ahandle = msgInfo;
+  rpcSendRequest(clientRpc, &global.mgmtEp.epSet, &rpcMsg, NULL);
+
+  uv_sem_wait(&msgInfo->resultSem);
+  uv_sem_destroy(&msgInfo->resultSem);
+  int32_t code = msgInfo->code;
+  taosMemoryFree(msgInfo);
+  return code;
+}
+
+static bool udfdRpcRfp(int32_t code) {
+  if (code == TSDB_CODE_RPC_REDIRECT) {
+    return true;
+  } else {
+    return false;
+  }
+}
 
 int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet) {
   pEpSet->version = 0;
@@ -528,69 +667,30 @@ int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSe
   return 0;
 }
 
-int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf) {
-  SRetrieveFuncReq retrieveReq = {0};
-  retrieveReq.numOfFuncs = 1;
-  retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
-  taosArrayPush(retrieveReq.pFuncNames, udfName);
-
-  int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
-  void   *pReq = rpcMallocCont(contLen);
-  tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq);
-  taosArrayDestroy(retrieveReq.pFuncNames);
-
-  SRpcMsg rpcMsg = {0};
-  rpcMsg.pCont = pReq;
-  rpcMsg.contLen = contLen;
-  rpcMsg.msgType = TDMT_MND_RETRIEVE_FUNC;
-
-  SRpcMsg rpcRsp = {0};
-  rpcSendRecv(clientRpc, &global.mgmtEp.epSet, &rpcMsg, &rpcRsp);
-  SRetrieveFuncRsp retrieveRsp = {0};
-  tDeserializeSRetrieveFuncRsp(rpcRsp.pCont, rpcRsp.contLen, &retrieveRsp);
-
-  SFuncInfo *pFuncInfo = (SFuncInfo *)taosArrayGet(retrieveRsp.pFuncInfos, 0);
-
-  udf->funcType = pFuncInfo->funcType;
-  udf->scriptType = pFuncInfo->scriptType;
-  udf->outputType = pFuncInfo->funcType;
-  udf->outputLen = pFuncInfo->outputLen;
-  udf->bufSize = pFuncInfo->bufSize;
-
-  char path[PATH_MAX] = {0};
-  snprintf(path, sizeof(path), "%s/lib%s.so", "/tmp", udfName);
-  TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL);
-  // TODO check for failure of flush to disk
-  taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize);
-  taosCloseFile(&file);
-  strncpy(udf->path, path, strlen(path));
-  taosArrayDestroy(retrieveRsp.pFuncInfos);
-
-  rpcFreeCont(rpcRsp.pCont);
-  return 0;
-}
 
 int32_t udfdOpenClientRpc() {
-  char *pass = "taosdata";
-  char *user = "root";
-  char  secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0};
-  taosEncryptPass_c((uint8_t *)pass, strlen(pass), secretEncrypt);
   SRpcInit rpcInit = {0};
-  rpcInit.label = (char *)"UDFD";
+  rpcInit.label = "UDFD";
   rpcInit.numOfThreads = 1;
-  rpcInit.cfp = udfdProcessRpcRsp;
+  rpcInit.cfp = (RpcCfp)udfdProcessRpcRsp;
   rpcInit.sessions = 1024;
   rpcInit.connType = TAOS_CONN_CLIENT;
-  rpcInit.idleTime = 30 * 1000;
+  rpcInit.idleTime = tsShellActivityTimer * 1000;
-  rpcInit.parent = &global;
+  rpcInit.user = TSDB_DEFAULT_USER;
-
+  rpcInit.ckey = "key";
-  rpcInit.user = (char *)user;
-  rpcInit.ckey = (char *)"key";
-  rpcInit.secret = (char *)secretEncrypt;
   rpcInit.spi = 1;
+  rpcInit.parent = &global;
+  rpcInit.rfp = udfdRpcRfp;
+
+  char pass[TSDB_PASSWORD_LEN + 1] = {0};
+  taosEncryptPass_c((uint8_t *)(TSDB_DEFAULT_PASS), strlen(TSDB_DEFAULT_PASS), pass);
+  rpcInit.secret = pass;
 
   global.clientRpc = rpcOpen(&rpcInit);
-
+  if (global.clientRpc == NULL) {
+    fnError("failed to init dnode rpc client");
+    return -1;
+  }
   return 0;
 }
 
@@ -700,12 +800,6 @@ static int32_t udfdRun() {
   global.udfsHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
   uv_mutex_init(&global.udfsMutex);
 
-  // TOOD: client rpc to fetch udf function info from mnode
-  if (udfdOpenClientRpc() != 0) {
-    fnError("open rpc connection to mnode failure");
-    return -1;
-  }
-
   if (udfdUvInit() != 0) {
     fnError("uv init failure");
     return -2;
@@ -717,7 +811,6 @@ static int32_t udfdRun() {
   int codeClose = uv_loop_close(global.loop);
   fnDebug("uv loop close. result: %s", uv_err_name(codeClose));
   removeListeningPipe();
-  udfdCloseClientRpc();
   uv_mutex_destroy(&global.udfsMutex);
   taosHashCleanup(global.udfsHash);
   return 0;
@@ -746,9 +839,22 @@ int main(int argc, char *argv[]) {
 
   if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) {
     fnError("failed to start since read config error");
-    return -1;
+    return -2;
   }
 
   initEpSetFromCfg(tsFirst, tsSecond, &global.mgmtEp);
-  return udfdRun();
+  if (udfdOpenClientRpc() != 0) {
+    fnError("open rpc connection to mnode failure");
+    return -3;
+  }
+
+  if (udfdConnectToMNode() != 0) {
+    fnError("failed to start since can not connect to mnode");
+    return -4;
+  }
+
+  udfdRun();
+
+  udfdCloseClientRpc();
+
 }

source/libs/nodes/src/nodesCodeFuncs.c

@@ -584,6 +584,37 @@ static int32_t jsonToLogicProjectNode(const SJson* pJson, void* pObj) {
   return code;
 }
 
+static const char* jkExchangeLogicPlanSrcGroupId = "SrcGroupId";
+static const char* jkExchangeLogicPlanSrcPrecision = "Precision";
+
+static int32_t logicExchangeNodeToJson(const void* pObj, SJson* pJson) {
+  const SExchangeLogicNode* pNode = (const SExchangeLogicNode*)pObj;
+
+  int32_t code = logicPlanNodeToJson(pObj, pJson);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkExchangeLogicPlanSrcGroupId, pNode->srcGroupId);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkExchangeLogicPlanSrcPrecision, pNode->precision);
+  }
+
+  return code;
+}
+
+static int32_t jsonToLogicExchangeNode(const SJson* pJson, void* pObj) {
+  SExchangeLogicNode* pNode = (SExchangeLogicNode*)pObj;
+
+  int32_t code = jsonToLogicPlanNode(pJson, pObj);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetIntValue(pJson, jkExchangeLogicPlanSrcGroupId, &pNode->srcGroupId);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetUTinyIntValue(pJson, jkExchangeLogicPlanSrcPrecision, &pNode->precision);
+  }
+
+  return code;
+}
+
 static const char* jkFillLogicPlanMode = "Mode";
 static const char* jkFillLogicPlanWStartTs = "WStartTs";
 static const char* jkFillLogicPlanValues = "Values";
@@ -2987,6 +3018,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
       return logicProjectNodeToJson(pObj, pJson);
     case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF:
       break;
+    case QUERY_NODE_LOGIC_PLAN_EXCHANGE:
+      return logicExchangeNodeToJson(pObj, pJson);
     case QUERY_NODE_LOGIC_PLAN_FILL:
       return logicFillNodeToJson(pObj, pJson);
     case QUERY_NODE_LOGIC_PLAN_SORT:
@@ -3083,6 +3116,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
       return jsonToLogicScanNode(pJson, pObj);
     case QUERY_NODE_LOGIC_PLAN_PROJECT:
       return jsonToLogicProjectNode(pJson, pObj);
+    case QUERY_NODE_LOGIC_PLAN_EXCHANGE:
+      return jsonToLogicExchangeNode(pJson, pObj);
     case QUERY_NODE_LOGIC_PLAN_FILL:
       return jsonToLogicFillNode(pJson, pObj);
     case QUERY_NODE_LOGIC_PLAN_SORT:

source/libs/parser/inc/sql.y

@@ -241,7 +241,7 @@ alter_table_clause(A) ::=
 alter_table_clause(A) ::= 
   full_table_name(B) RENAME TAG column_name(C) column_name(D).                    { A = createAlterTableRenameCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &C, &D); }
 alter_table_clause(A) ::=
-  full_table_name(B) SET TAG column_name(C) NK_EQ literal(D).                     { A = createAlterTableSetTag(pCxt, B, &C, D); }
+  full_table_name(B) SET TAG column_name(C) NK_EQ literal(D).                     { A = createAlterTableSetTag(pCxt, B, &C, releaseRawExprNode(pCxt, D)); }
 
 %type multi_create_clause                                                         { SNodeList* }
 %destructor multi_create_clause                                                   { nodesDestroyList($$); }

source/libs/parser/src/parAuthenticator.c

@@ -14,6 +14,7 @@
  */
 
 #include "catalog.h"
+#include "cmdnodes.h"
 #include "parInt.h"
 
 typedef struct SAuthCxt {
@@ -65,8 +66,8 @@ static int32_t authSetOperator(SAuthCxt* pCxt, SSetOperator* pSetOper) {
   return code;
 }
 
-static int32_t authDropUser(SAuthCxt* pCxt, SDropUserReq* pStmt) {
+static int32_t authDropUser(SAuthCxt* pCxt, SDropUserStmt* pStmt) {
-  if (!pCxt->pParseCxt->isSuperUser || 0 == strcmp(pStmt->user, TSDB_DEFAULT_USER)) {
+  if (!pCxt->pParseCxt->isSuperUser || 0 == strcmp(pStmt->useName, TSDB_DEFAULT_USER)) {
     return TSDB_CODE_PAR_PERMISSION_DENIED;
   }
   return TSDB_CODE_SUCCESS;
@@ -92,7 +93,7 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) {
     case QUERY_NODE_ALTER_USER_STMT:
       break;
     case QUERY_NODE_DROP_USER_STMT: {
-      return authDropUser(pCxt, (SDropUserReq*)pStmt);
+      return authDropUser(pCxt, (SDropUserStmt*)pStmt);
     }
     case QUERY_NODE_USE_DATABASE_STMT:
     case QUERY_NODE_CREATE_DNODE_STMT:

source/libs/parser/src/parInsert.c

@@ -1041,18 +1041,6 @@ static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) {
   destroyCreateSubTbReq(&pCxt->createTblReq);
 }
 
-static void destroyDataBlock(STableDataBlocks* pDataBlock) {
-  if (pDataBlock == NULL) {
-    return;
-  }
-
-  taosMemoryFreeClear(pDataBlock->pData);
-  if (!pDataBlock->cloned) {
-    destroyBoundColumnInfo(&pDataBlock->boundColumnInfo);
-  }
-  taosMemoryFreeClear(pDataBlock);
-}
-
 static void destroyInsertParseContext(SInsertParseContext* pCxt) {
   destroyInsertParseContextForTable(pCxt);
   taosHashCleanup(pCxt->pVgroupsHashObj);
@@ -1301,6 +1289,7 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash
 
   CHECK_CODE(buildOutput(&insertCtx));
 
+  destroyBlockArrayList(insertCtx.pVgDataBlocks);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -1580,16 +1569,25 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD** fields
 
 // schemaless logic start
 
-typedef struct SmlExecHandle {
+typedef struct SmlExecTableHandle {
-  SHashObj* pBlockHash;
-
   SParsedDataColInfo tags;          // each table
   SKVRowBuilder      tagsBuilder;   // each table
   SVCreateTbReq      createTblReq;  // each table
+} SmlExecTableHandle;
 
-  SQuery* pQuery;
+typedef struct SmlExecHandle {
+  SHashObj*           pBlockHash;
+  SmlExecTableHandle  tableExecHandle;
+  SQuery              *pQuery;
 } SSmlExecHandle;
 
+static void smlDestroyTableHandle(void* pHandle) {
+  SmlExecTableHandle* handle = (SmlExecTableHandle*)pHandle;
+  tdDestroyKVRowBuilder(&handle->tagsBuilder);
+  destroyBoundColumnInfo(&handle->tags);
+  destroyCreateSubTbReq(&handle->createTblReq);
+}
+
 static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SSchema* pSchema) {
   col_id_t nCols = pColList->numOfCols;
 
@@ -1692,25 +1690,26 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsSchema, SArray *cols
   SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen};
 
   SSmlExecHandle* smlHandle = (SSmlExecHandle*)handle;
+  smlDestroyTableHandle(&smlHandle->tableExecHandle);   // free for each table
   SSchema*        pTagsSchema = getTableTagSchema(pTableMeta);
-  setBoundColumnInfo(&smlHandle->tags, pTagsSchema, getNumOfTags(pTableMeta));
+  setBoundColumnInfo(&smlHandle->tableExecHandle.tags, pTagsSchema, getNumOfTags(pTableMeta));
-  int ret = smlBoundColumnData(tags, &smlHandle->tags, pTagsSchema);
+  int ret = smlBoundColumnData(tags, &smlHandle->tableExecHandle.tags, pTagsSchema);
   if (ret != TSDB_CODE_SUCCESS) {
     buildInvalidOperationMsg(&pBuf, "bound tags error");
     return ret;
   }
   SKVRow row = NULL;
-  ret = smlBuildTagRow(tags, &smlHandle->tagsBuilder, &smlHandle->tags, pTagsSchema, &row, &pBuf);
+  ret = smlBuildTagRow(tags, &smlHandle->tableExecHandle.tagsBuilder, &smlHandle->tableExecHandle.tags, pTagsSchema, &row, &pBuf);
   if (ret != TSDB_CODE_SUCCESS) {
     return ret;
   }
 
-  buildCreateTbReq(&smlHandle->createTblReq, tableName, row, pTableMeta->suid);
+  buildCreateTbReq(&smlHandle->tableExecHandle.createTblReq, tableName, row, pTableMeta->suid);
 
   STableDataBlocks* pDataBlock = NULL;
   ret = getDataBlockFromList(smlHandle->pBlockHash, &pTableMeta->uid, sizeof(pTableMeta->uid),
                              TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), getTableInfo(pTableMeta).rowSize,
-                             pTableMeta, &pDataBlock, NULL, &smlHandle->createTblReq);
+                             pTableMeta, &pDataBlock, NULL, &smlHandle->tableExecHandle.createTblReq);
   if (ret != TSDB_CODE_SUCCESS) {
     buildInvalidOperationMsg(&pBuf, "create data block error");
     return ret;
@@ -1826,9 +1825,7 @@ void smlDestroyHandle(void* pHandle) {
   if (!pHandle) return;
   SSmlExecHandle* handle = (SSmlExecHandle*)pHandle;
   destroyBlockHashmap(handle->pBlockHash);
-  tdDestroyKVRowBuilder(&handle->tagsBuilder);
+  smlDestroyTableHandle(&handle->tableExecHandle);
-  destroyBoundColumnInfo(&handle->tags);
-  destroyCreateSubTbReq(&handle->createTblReq);
   taosMemoryFree(handle);
 }
 

source/libs/parser/src/parInsertData.c

@@ -237,9 +237,7 @@ static void destroyDataBlock(STableDataBlocks* pDataBlock) {
   taosMemoryFreeClear(pDataBlock->pData);
   if (!pDataBlock->cloned) {
     // free the refcount for metermeta
-    if (pDataBlock->pTableMeta != NULL) {
+    taosMemoryFreeClear(pDataBlock->pTableMeta);
-      taosMemoryFreeClear(pDataBlock->pTableMeta);
-    }
 
     destroyBoundColumnInfo(&pDataBlock->boundColumnInfo);
   }

source/libs/parser/src/parTranslater.c

@@ -2949,8 +2949,8 @@ static int32_t translateCreateIndex(STranslateContext* pCxt, SCreateIndexStmt* p
 }
 
 static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt) {
-  SEncoder       encoder = {0};
+  SEncoder      encoder = {0};
-  int32_t        contLen = 0;
+  int32_t       contLen = 0;
   SVDropTSmaReq dropSmaReq = {0};
   strcpy(dropSmaReq.indexName, pStmt->indexName);
 
@@ -2958,7 +2958,7 @@ static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt
   if (NULL == pCxt->pCmdMsg) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
-  
+
   int32_t ret = 0;
   tEncodeSize(tEncodeSVDropTSmaReq, &dropSmaReq, contLen, ret);
   if (ret < 0) {
@@ -3725,7 +3725,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt*
   req.type = TD_NORMAL_TABLE;
   req.name = strdup(pStmt->tableName);
   req.ntb.schema.nCols = LIST_LENGTH(pStmt->pCols);
-  req.ntb.schema.sver = 0;
+  req.ntb.schema.sver = 1;
   req.ntb.schema.pSchema = taosMemoryCalloc(req.ntb.schema.nCols, sizeof(SSchema));
   if (NULL == req.name || NULL == req.ntb.schema.pSchema) {
     destroyCreateTbReq(&req);
@@ -3800,7 +3800,7 @@ static void destroyCreateTbReqBatch(SVgroupCreateTableBatch* pTbBatch) {
   taosArrayDestroy(pTbBatch->req.pArray);
 }
 
-static int32_t rewriteToVnodeModifOpStmt(SQuery* pQuery, SArray* pBufArray) {
+static int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray) {
   SVnodeModifOpStmt* pNewStmt = nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT);
   if (pNewStmt == NULL) {
     return TSDB_CODE_OUT_OF_MEMORY;
@@ -3855,7 +3855,7 @@ static int32_t rewriteCreateTable(STranslateContext* pCxt, SQuery* pQuery) {
     code = buildCreateTableDataBlock(pCxt->pParseCxt->acctId, pStmt, &info, &pBufArray);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = rewriteToVnodeModifOpStmt(pQuery, pBufArray);
+    code = rewriteToVnodeModifyOpStmt(pQuery, pBufArray);
     if (TSDB_CODE_SUCCESS != code) {
       destroyCreateTbReqArray(pBufArray);
     }
@@ -4111,7 +4111,7 @@ static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery)
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
-  return rewriteToVnodeModifOpStmt(pQuery, pBufArray);
+  return rewriteToVnodeModifyOpStmt(pQuery, pBufArray);
 }
 
 typedef struct SVgroupDropTableBatch {
@@ -4251,12 +4251,160 @@ static int32_t rewriteDropTable(STranslateContext* pCxt, SQuery* pQuery) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
-  return rewriteToVnodeModifOpStmt(pQuery, pBufArray);
+  return rewriteToVnodeModifyOpStmt(pQuery, pBufArray);
+}
+
+static int32_t buildAlterTbReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, SVAlterTbReq* pReq) {
+  pReq->tbName = strdup(pStmt->tableName);
+  if (NULL == pReq->tbName) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  pReq->action = pStmt->alterType;
+
+  switch (pStmt->alterType) {
+    case TSDB_ALTER_TABLE_ADD_TAG:
+    case TSDB_ALTER_TABLE_DROP_TAG:
+    case TSDB_ALTER_TABLE_UPDATE_TAG_NAME:
+    case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES:
+      return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE);
+    case TSDB_ALTER_TABLE_UPDATE_TAG_VAL:
+      pReq->tagName = strdup(pStmt->colName);
+      if (NULL == pReq->tagName) {
+        return TSDB_CODE_OUT_OF_MEMORY;
+      }
+      if (DEAL_RES_ERROR == translateValue(pCxt, pStmt->pVal)) {
+        return pCxt->errCode;
+      }
+      pReq->isNull = (TSDB_DATA_TYPE_NULL == pStmt->pVal->node.resType.type);
+      pReq->nTagVal = pStmt->pVal->node.resType.bytes;
+      char* pVal = nodesGetValueFromNode(pStmt->pVal);
+      pReq->pTagVal = IS_VAR_DATA_TYPE(pStmt->pVal->node.resType.type) ? pVal + VARSTR_HEADER_SIZE : pVal;
+      break;
+    case TSDB_ALTER_TABLE_ADD_COLUMN:
+    case TSDB_ALTER_TABLE_DROP_COLUMN:
+      pReq->colName = strdup(pStmt->colName);
+      if (NULL == pReq->colName) {
+        return TSDB_CODE_OUT_OF_MEMORY;
+      }
+      pReq->type = pStmt->dataType.type;
+      pReq->flags = COL_SMA_ON;
+      pReq->bytes = pStmt->dataType.bytes;
+      break;
+    case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:
+      pReq->colName = strdup(pStmt->colName);
+      if (NULL == pReq->colName) {
+        return TSDB_CODE_OUT_OF_MEMORY;
+      }
+      pReq->colModBytes = calcTypeBytes(pStmt->dataType);
+      break;
+    case TSDB_ALTER_TABLE_UPDATE_OPTIONS:
+      if (-1 != pStmt->pOptions->ttl) {
+        pReq->updateTTL = true;
+        pReq->newTTL = pStmt->pOptions->ttl;
+      }
+      if ('\0' != pStmt->pOptions->comment[0]) {
+        pReq->updateComment = true;
+        pReq->newComment = strdup(pStmt->pOptions->comment);
+        if (NULL == pReq->newComment) {
+          return TSDB_CODE_OUT_OF_MEMORY;
+        }
+      }
+      break;
+    case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME:
+      pReq->colName = strdup(pStmt->colName);
+      pReq->colNewName = strdup(pStmt->newColName);
+      if (NULL == pReq->colName || NULL == pReq->colNewName) {
+        return TSDB_CODE_OUT_OF_MEMORY;
+      }
+      break;
+    default:
+      break;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t serializeAlterTbReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, SVAlterTbReq* pReq,
+                                   SArray* pArray) {
+  SVgroupInfo vg = {0};
+  int32_t     code = getTableHashVgroup(pCxt, pStmt->dbName, pStmt->tableName, &vg);
+  int         tlen = 0;
+  if (TSDB_CODE_SUCCESS == code) {
+    tEncodeSize(tEncodeSVAlterTbReq, pReq, tlen, code);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    tlen += sizeof(SMsgHead);
+    void* pMsg = taosMemoryMalloc(tlen);
+    if (NULL == pMsg) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+    ((SMsgHead*)pMsg)->vgId = htonl(vg.vgId);
+    ((SMsgHead*)pMsg)->contLen = htonl(tlen);
+    void*    pBuf = POINTER_SHIFT(pMsg, sizeof(SMsgHead));
+    SEncoder coder = {0};
+    tEncoderInit(&coder, pBuf, tlen - sizeof(SMsgHead));
+    tEncodeSVAlterTbReq(&coder, pReq);
+    tEncoderClear(&coder);
+
+    SVgDataBlocks* pVgData = taosMemoryCalloc(1, sizeof(SVgDataBlocks));
+    if (NULL == pVgData) {
+      taosMemoryFree(pMsg);
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+    pVgData->vg = vg;
+    pVgData->pData = pMsg;
+    pVgData->size = tlen;
+    pVgData->numOfTables = 1;
+    taosArrayPush(pArray, &pVgData);
+  }
+
+  return code;
+}
+
+static int32_t buildModifyVnodeArray(STranslateContext* pCxt, SAlterTableStmt* pStmt, SVAlterTbReq* pReq,
+                                     SArray** pArray) {
+  SArray* pTmpArray = taosArrayInit(1, sizeof(void*));
+  if (NULL == pTmpArray) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  int32_t code = serializeAlterTbReq(pCxt, pStmt, pReq, pTmpArray);
+  if (TSDB_CODE_SUCCESS == code) {
+    *pArray = pTmpArray;
+  } else {
+    taosArrayDestroy(pTmpArray);
+  }
+
+  return code;
 }
 
 static int32_t rewriteAlterTable(STranslateContext* pCxt, SQuery* pQuery) {
-  // todo
+  SAlterTableStmt* pStmt = (SAlterTableStmt*)pQuery->pRoot;
-  return TSDB_CODE_SUCCESS;
+
+  STableMeta* pTableMeta = NULL;
+  int32_t     code = getTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pTableMeta);
+  if (TSDB_CODE_SUCCESS != code) {
+    return code;
+  }
+
+  if (TSDB_SUPER_TABLE == pTableMeta->tableType) {
+    return TSDB_CODE_SUCCESS;
+  } else if (TSDB_CHILD_TABLE != pTableMeta->tableType && TSDB_NORMAL_TABLE != pTableMeta->tableType) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE);
+  }
+
+  SVAlterTbReq req = {0};
+  code = buildAlterTbReq(pCxt, pStmt, &req);
+
+  SArray* pArray = NULL;
+  if (TSDB_CODE_SUCCESS == code) {
+    code = buildModifyVnodeArray(pCxt, pStmt, &req, &pArray);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = rewriteToVnodeModifyOpStmt(pQuery, pArray);
+  }
+
+  return code;
 }
 
 static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
@@ -4296,9 +4444,7 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
       code = rewriteDropTable(pCxt, pQuery);
       break;
     case QUERY_NODE_ALTER_TABLE_STMT:
-      if (TSDB_ALTER_TABLE_UPDATE_TAG_VAL == ((SAlterTableStmt*)pQuery->pRoot)->alterType) {
+      code = rewriteAlterTable(pCxt, pQuery);
-        code = rewriteAlterTable(pCxt, pQuery);
-      }
       break;
     default:
       break;

source/libs/parser/src/parUtil.c

@@ -152,6 +152,8 @@ static char* getSyntaxErrFormat(int32_t errCode) {
       return "Invalid timeline function";
     case TSDB_CODE_PAR_INVALID_PASSWD:
       return "Invalid password";
+    case TSDB_CODE_PAR_INVALID_ALTER_TABLE:
+      return "Invalid alter table statement";
     case TSDB_CODE_OUT_OF_MEMORY:
       return "Out of memory";
     default:

source/libs/parser/src/sql.c

@@ -3500,7 +3500,7 @@ static YYACTIONTYPE yy_reduce(
   yymsp[-4].minor.yy172 = yylhsminor.yy172;
         break;
       case 121: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal */
-{ yylhsminor.yy172 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy172, &yymsp[-2].minor.yy105, yymsp[0].minor.yy172); }
+{ yylhsminor.yy172 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy172, &yymsp[-2].minor.yy105, releaseRawExprNode(pCxt, yymsp[0].minor.yy172)); }
   yymsp[-5].minor.yy172 = yylhsminor.yy172;
         break;
       case 123: /* multi_create_clause ::= multi_create_clause create_subtable_clause */

source/libs/parser/test/parInitialATest.cpp

@@ -69,7 +69,7 @@ TEST_F(ParserInitialATest, alterDatabase) {
  *   | COMMENT 'string_value'
  * }
  */
-TEST_F(ParserInitialATest, alterTable) {
+TEST_F(ParserInitialATest, alterSTable) {
   useDb("root", "test");
 
   SMAlterStbReq expect = {0};
@@ -119,7 +119,7 @@ TEST_F(ParserInitialATest, alterTable) {
   setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) {
     ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_ALTER_TABLE_STMT);
     SMAlterStbReq req = {0};
-    ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSMAlterStbReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req));
+    ASSERT_EQ(tDeserializeSMAlterStbReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req), TSDB_CODE_SUCCESS);
     ASSERT_EQ(std::string(req.name), std::string(expect.name));
     ASSERT_EQ(req.alterType, expect.alterType);
     ASSERT_EQ(req.numOfFields, expect.numOfFields);
@@ -139,24 +139,24 @@ TEST_F(ParserInitialATest, alterTable) {
     }
   });
 
-  setAlterStbReqFunc("t1", TSDB_ALTER_TABLE_UPDATE_OPTIONS, 0, nullptr, 0, 0, nullptr, nullptr, 10);
+  setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_OPTIONS, 0, nullptr, 0, 0, nullptr, nullptr, 10);
-  run("ALTER TABLE t1 TTL 10");
+  run("ALTER TABLE st1 TTL 10");
 
-  setAlterStbReqFunc("t1", TSDB_ALTER_TABLE_UPDATE_OPTIONS, 0, nullptr, 0, 0, nullptr, "test");
+  setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_OPTIONS, 0, nullptr, 0, 0, nullptr, "test");
-  run("ALTER TABLE t1 COMMENT 'test'");
+  run("ALTER TABLE st1 COMMENT 'test'");
 
-  setAlterStbReqFunc("t1", TSDB_ALTER_TABLE_ADD_COLUMN, 1, "cc1", TSDB_DATA_TYPE_BIGINT);
+  setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_ADD_COLUMN, 1, "cc1", TSDB_DATA_TYPE_BIGINT);
-  run("ALTER TABLE t1 ADD COLUMN cc1 BIGINT");
+  run("ALTER TABLE st1 ADD COLUMN cc1 BIGINT");
 
-  setAlterStbReqFunc("t1", TSDB_ALTER_TABLE_DROP_COLUMN, 1, "c1");
+  setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_DROP_COLUMN, 1, "c1");
-  run("ALTER TABLE t1 DROP COLUMN c1");
+  run("ALTER TABLE st1 DROP COLUMN c1");
 
-  setAlterStbReqFunc("t1", TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, 1, "c1", TSDB_DATA_TYPE_VARCHAR,
+  setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, 1, "c1", TSDB_DATA_TYPE_VARCHAR,
                      20 + VARSTR_HEADER_SIZE);
-  run("ALTER TABLE t1 MODIFY COLUMN c1 VARCHAR(20)");
+  run("ALTER TABLE st1 MODIFY COLUMN c1 VARCHAR(20)");
 
-  setAlterStbReqFunc("t1", TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, 2, "c1", 0, 0, "cc1");
+  setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, 2, "c1", 0, 0, "cc1");
-  run("ALTER TABLE t1 RENAME COLUMN c1 cc1");
+  run("ALTER TABLE st1 RENAME COLUMN c1 cc1");
 
   setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_ADD_TAG, 1, "tag11", TSDB_DATA_TYPE_BIGINT);
   run("ALTER TABLE st1 ADD TAG tag11 BIGINT");
@@ -171,7 +171,127 @@ TEST_F(ParserInitialATest, alterTable) {
   setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_TAG_NAME, 2, "tag1", 0, 0, "tag11");
   run("ALTER TABLE st1 RENAME TAG tag1 tag11");
 
-  // run("ALTER TABLE st1s1 SET TAG tag1=10");
+  // todo
+  // ADD {FULLTEXT | SMA} INDEX index_name (col_name [, col_name] ...) [index_option]
+}
+
+TEST_F(ParserInitialATest, alterTable) {
+  useDb("root", "test");
+
+  SVAlterTbReq expect = {0};
+
+  auto setAlterColFunc = [&](const char* pTbname, int8_t alterType, const char* pColName, int8_t dataType = 0,
+                             int32_t dataBytes = 0, const char* pNewColName = nullptr) {
+    memset(&expect, 0, sizeof(SVAlterTbReq));
+    expect.tbName = strdup(pTbname);
+    expect.action = alterType;
+    expect.colName = strdup(pColName);
+
+    switch (alterType) {
+      case TSDB_ALTER_TABLE_ADD_COLUMN:
+        expect.type = dataType;
+        expect.flags = COL_SMA_ON;
+        expect.bytes = dataBytes > 0 ? dataBytes : (dataType > 0 ? tDataTypes[dataType].bytes : 0);
+        break;
+      case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:
+        expect.colModBytes = dataBytes;
+        break;
+      case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME:
+        expect.colNewName = strdup(pNewColName);
+        break;
+      default:
+        break;
+    }
+  };
+
+  auto setAlterTagFunc = [&](const char* pTbname, const char* pTagName, const uint8_t* pNewVal, uint32_t bytes) {
+    memset(&expect, 0, sizeof(SVAlterTbReq));
+    expect.tbName = strdup(pTbname);
+    expect.action = TSDB_ALTER_TABLE_UPDATE_TAG_VAL;
+    expect.tagName = strdup(pTagName);
+
+    expect.isNull = (nullptr == pNewVal);
+    expect.nTagVal = bytes;
+    expect.pTagVal = pNewVal;
+  };
+
+  auto setAlterOptionsFunc = [&](const char* pTbname, int32_t ttl, const char* pComment = nullptr) {
+    memset(&expect, 0, sizeof(SVAlterTbReq));
+    expect.tbName = strdup(pTbname);
+    expect.action = TSDB_ALTER_TABLE_UPDATE_OPTIONS;
+    if (-1 != ttl) {
+      expect.updateTTL = true;
+      expect.newTTL = ttl;
+    }
+    if (nullptr != pComment) {
+      expect.updateComment = true;
+      expect.newComment = pComment;
+    }
+  };
+
+  setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) {
+    ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_VNODE_MODIF_STMT);
+    SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pQuery->pRoot;
+
+    ASSERT_EQ(pStmt->sqlNodeType, QUERY_NODE_ALTER_TABLE_STMT);
+    ASSERT_NE(pStmt->pDataBlocks, nullptr);
+    ASSERT_EQ(taosArrayGetSize(pStmt->pDataBlocks), 1);
+    SVgDataBlocks* pVgData = (SVgDataBlocks*)taosArrayGetP(pStmt->pDataBlocks, 0);
+    void*          pBuf = POINTER_SHIFT(pVgData->pData, sizeof(SMsgHead));
+    SVAlterTbReq   req = {0};
+    SDecoder       coder = {0};
+    tDecoderInit(&coder, (const uint8_t*)pBuf, pVgData->size);
+    ASSERT_EQ(tDecodeSVAlterTbReq(&coder, &req), TSDB_CODE_SUCCESS);
+
+    ASSERT_EQ(std::string(req.tbName), std::string(expect.tbName));
+    ASSERT_EQ(req.action, expect.action);
+    if (nullptr != expect.colName) {
+      ASSERT_EQ(std::string(req.colName), std::string(expect.colName));
+    }
+    ASSERT_EQ(req.type, expect.type);
+    ASSERT_EQ(req.flags, expect.flags);
+    ASSERT_EQ(req.bytes, expect.bytes);
+    ASSERT_EQ(req.colModBytes, expect.colModBytes);
+    if (nullptr != expect.colNewName) {
+      ASSERT_EQ(std::string(req.colNewName), std::string(expect.colNewName));
+    }
+    if (nullptr != expect.tagName) {
+      ASSERT_EQ(std::string(req.tagName), std::string(expect.tagName));
+    }
+    ASSERT_EQ(req.isNull, expect.isNull);
+    ASSERT_EQ(req.nTagVal, expect.nTagVal);
+    ASSERT_EQ(memcmp(req.pTagVal, expect.pTagVal, expect.nTagVal), 0);
+    ASSERT_EQ(req.updateTTL, expect.updateTTL);
+    ASSERT_EQ(req.newTTL, expect.newTTL);
+    ASSERT_EQ(req.updateComment, expect.updateComment);
+    if (nullptr != expect.newComment) {
+      ASSERT_EQ(std::string(req.newComment), std::string(expect.newComment));
+    }
+
+    tDecoderClear(&coder);
+  });
+
+  setAlterOptionsFunc("t1", 10, nullptr);
+  run("ALTER TABLE t1 TTL 10");
+
+  setAlterOptionsFunc("t1", -1, "test");
+  run("ALTER TABLE t1 COMMENT 'test'");
+
+  setAlterColFunc("t1", TSDB_ALTER_TABLE_ADD_COLUMN, "cc1", TSDB_DATA_TYPE_BIGINT);
+  run("ALTER TABLE t1 ADD COLUMN cc1 BIGINT");
+
+  setAlterColFunc("t1", TSDB_ALTER_TABLE_DROP_COLUMN, "c1");
+  run("ALTER TABLE t1 DROP COLUMN c1");
+
+  setAlterColFunc("t1", TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, "c1", TSDB_DATA_TYPE_VARCHAR, 20 + VARSTR_HEADER_SIZE);
+  run("ALTER TABLE t1 MODIFY COLUMN c1 VARCHAR(20)");
+
+  setAlterColFunc("t1", TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, "c1", 0, 0, "cc1");
+  run("ALTER TABLE t1 RENAME COLUMN c1 cc1");
+
+  int64_t val = 10;
+  setAlterTagFunc("st1s1", "tag1", (const uint8_t*)&val, sizeof(val));
+  run("ALTER TABLE st1s1 SET TAG tag1=10");
 
   // todo
   // ADD {FULLTEXT | SMA} INDEX index_name (col_name [, col_name] ...) [index_option]

source/libs/parser/test/parSelectTest.cpp

@@ -235,11 +235,17 @@ TEST_F(ParserSelectTest, semanticError) {
 TEST_F(ParserSelectTest, setOperator) {
   useDb("root", "test");
 
-  // run("SELECT * FROM t1 UNION ALL SELECT * FROM t1");
+  run("SELECT * FROM t1 UNION ALL SELECT * FROM t1");
 
-  // run("(SELECT * FROM t1) UNION ALL (SELECT * FROM t1)");
+  run("(SELECT * FROM t1) UNION ALL (SELECT * FROM t1)");
 
   run("SELECT c1 FROM (SELECT c1 FROM t1 UNION ALL SELECT c1 FROM t1)");
 }
 
+TEST_F(ParserSelectTest, informationSchema) {
+  useDb("root", "test");
+
+  run("SELECT * FROM information_schema.user_databases WHERE name = 'information_schema'");
+}
+
 }  // namespace ParserTest

source/libs/planner/src/planLogicCreater.c

@@ -985,6 +985,8 @@ static int32_t getMsgType(ENodeType sqlType) {
       return TDMT_VND_CREATE_TABLE;
     case QUERY_NODE_DROP_TABLE_STMT:
       return TDMT_VND_DROP_TABLE;
+    case QUERY_NODE_ALTER_TABLE_STMT:
+      return TDMT_VND_ALTER_TABLE;
     default:
       break;
   }

source/libs/planner/src/planOptimizer.c

@@ -392,7 +392,8 @@ static int32_t cpdCalcTimeRange(SScanLogicNode* pScan, SNode** pPrimaryKeyCond,
 }
 
 static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* pScan) {
-  if (NULL == pScan->node.pConditions || OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD)) {
+  if (NULL == pScan->node.pConditions || OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD) ||
+      TSDB_SYSTEM_TABLE == pScan->pMeta->tableType) {
     return TSDB_CODE_SUCCESS;
   }
 

source/libs/planner/src/planSpliter.c

@@ -303,7 +303,7 @@ static SLogicNode* unMatchByNode(SLogicNode* pNode) {
   }
   SNode* pChild;
   FOREACH(pChild, pNode->pChildren) {
-    SLogicNode* pSplitNode = uaMatchByNode((SLogicNode*)pChild);
+    SLogicNode* pSplitNode = unMatchByNode((SLogicNode*)pChild);
     if (NULL != pSplitNode) {
       return pSplitNode;
     }
@@ -318,7 +318,7 @@ static int32_t unCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan
   }
   pExchange->srcGroupId = pCxt->groupId;
   // pExchange->precision = pScan->pMeta->tableInfo.precision;
-  pExchange->node.pTargets = nodesCloneList(pAgg->node.pTargets);
+  pExchange->node.pTargets = nodesCloneList(pAgg->pGroupKeys);
   if (NULL == pExchange->node.pTargets) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }

source/libs/planner/test/planOtherTest.cpp

@@ -47,4 +47,10 @@ TEST_F(PlanOtherTest, explain) {
   run("explain analyze SELECT * FROM t1");
 
   run("explain analyze verbose true ratio 0.01 SELECT * FROM t1");
-}
+}
+
+TEST_F(PlanOtherTest, show) {
+  useDb("root", "test");
+
+  run("SHOW DATABASES");
+}

source/libs/planner/test/planStateTest.cpp

@@ -23,13 +23,13 @@ class PlanStateTest : public PlannerTestBase {};
 TEST_F(PlanStateTest, basic) {
   useDb("root", "test");
 
-  run("select count(*) from t1 state_window(c1)");
+  run("SELECT COUNT(*) FROM t1 STATE_WINDOW(c1)");
 }
 
 TEST_F(PlanStateTest, stateExpr) {
   useDb("root", "test");
 
-  run("select count(*) from t1 state_window(c1 + 10)");
+  run("SELECT COUNT(*) FROM t1 STATE_WINDOW(c1 + 10)");
 }
 
 TEST_F(PlanStateTest, selectFunc) {

source/libs/planner/test/planSubqueryTest.cpp

@@ -25,11 +25,9 @@ TEST_F(PlanSubqeuryTest, basic) {
 
   if (0 == g_skipSql) {
     run("SELECT * FROM (SELECT * FROM t1)");
-
-    run("SELECT LAST(c1) FROM (SELECT * FROM t1)");
   }
 
-  run("SELECT c1 FROM (SELECT c1 FROM t1 UNION ALL SELECT c1 FROM t1)");
+  run("SELECT LAST(c1) FROM (SELECT * FROM t1)");
 }
 
 TEST_F(PlanSubqeuryTest, doubleGroupBy) {
@@ -39,3 +37,11 @@ TEST_F(PlanSubqeuryTest, doubleGroupBy) {
       "SELECT c1 + c3 a, c1 + COUNT(*) b FROM t1 WHERE c2 = 'abc' GROUP BY c1, c3) "
       "WHERE a > 100 GROUP BY b");
 }
+
+TEST_F(PlanSubqeuryTest, withSetOperator) {
+  useDb("root", "test");
+
+  run("SELECT c1 FROM (SELECT c1 FROM t1 UNION ALL SELECT c1 FROM t1)");
+
+  run("SELECT c1 FROM (SELECT c1 FROM t1 UNION SELECT c1 FROM t1)");
+}

source/libs/planner/test/planSysTbTest.cpp

@@ -27,8 +27,8 @@ TEST_F(PlanSysTableTest, show) {
   run("show stables");
 }
 
-TEST_F(PlanSysTableTest, information) {
+TEST_F(PlanSysTableTest, informationSchema) {
   useDb("root", "information_schema");
 
-  run("show tables");
+  run("SELECT * FROM information_schema.user_databases WHERE name = 'information_schema'");
 }

source/libs/qworker/src/qworker.c

@@ -812,13 +812,11 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu
   }
 
   if (ctx->rspCode) {
-    QW_TASK_ELOG("task already failed at phase %s, error:%x - %s", qwPhaseStr(phase), ctx->rspCode,
+    QW_TASK_ELOG("task already failed at phase %s, code:%s", qwPhaseStr(phase), tstrerror(ctx->rspCode));
-                 tstrerror(ctx->rspCode));
     QW_ERR_JRET(ctx->rspCode);
   }
 
 _return:
-
   if (ctx) {
     QW_UPDATE_RSP_CODE(ctx, code);
 
@@ -836,7 +834,11 @@ _return:
     QW_TASK_DLOG("cancel rsp send, handle:%p, code:%x - %s", cancelConnection->handle, code, tstrerror(code));
   }
 
-  QW_TASK_DLOG("end to handle event at phase %s, code:%x - %s", qwPhaseStr(phase), code, tstrerror(code));
+  if (code != TSDB_CODE_SUCCESS) {
+    QW_TASK_ELOG("end to handle event at phase %s, code:%s", qwPhaseStr(phase), tstrerror(code));
+  } else {
+    QW_TASK_DLOG("end to handle event at phase %s, code:%s", qwPhaseStr(phase), tstrerror(code));
+  }
 
   QW_RET(code);
 }

source/libs/scheduler/src/scheduler.c

@@ -256,6 +256,7 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m
       return TSDB_CODE_SUCCESS;
     case TDMT_VND_CREATE_TABLE_RSP:
     case TDMT_VND_DROP_TABLE_RSP:
+    case TDMT_VND_ALTER_TABLE_RSP:
     case TDMT_VND_SUBMIT_RSP:
       break;
     default:
@@ -1131,6 +1132,24 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch
       SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask));
       break;
     }
+    case TDMT_VND_ALTER_TABLE_RSP: {
+      SVAlterTbRsp rsp = {0};
+      if (msg) {
+        SDecoder coder = {0};
+        tDecoderInit(&coder, msg, msgSize);
+        code = tDecodeSVAlterTbRsp(&coder, &rsp);
+        tDecoderClear(&coder);
+        SCH_ERR_JRET(code);
+        SCH_ERR_JRET(rsp.code);
+      }
+
+      SCH_ERR_JRET(rspCode);
+
+      if (NULL == msg) {
+        SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
+      }
+      break;
+    }
     case TDMT_VND_SUBMIT_RSP: {
       SCH_ERR_JRET(rspCode);
 
@@ -1391,6 +1410,10 @@ int32_t schHandleDropTableCallback(void *param, const SDataBuf *pMsg, int32_t co
   return schHandleCallback(param, pMsg, TDMT_VND_DROP_TABLE_RSP, code);
 }
 
+int32_t schHandleAlterTableCallback(void *param, const SDataBuf *pMsg, int32_t code) {
+  return schHandleCallback(param, pMsg, TDMT_VND_ALTER_TABLE_RSP, code);
+}
+
 int32_t schHandleQueryCallback(void *param, const SDataBuf *pMsg, int32_t code) {
   return schHandleCallback(param, pMsg, TDMT_VND_QUERY_RSP, code);
 }
@@ -1490,6 +1513,9 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) {
     case TDMT_VND_DROP_TABLE:
       *fp = schHandleDropTableCallback;
       break;
+    case TDMT_VND_ALTER_TABLE:
+      *fp = schHandleAlterTableCallback;
+      break;
     case TDMT_VND_SUBMIT:
       *fp = schHandleSubmitCallback;
       break;
@@ -2010,6 +2036,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
   switch (msgType) {
     case TDMT_VND_CREATE_TABLE:
     case TDMT_VND_DROP_TABLE:
+    case TDMT_VND_ALTER_TABLE:
     case TDMT_VND_SUBMIT: {
       msgSize = pTask->msgLen;
       msg = taosMemoryCalloc(1, msgSize);

source/libs/sync/inc/syncIndexMgr.h

@@ -35,6 +35,7 @@ typedef struct SSyncIndexMgr {
 } SSyncIndexMgr;
 
 SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode);
+void           syncIndexMgrUpdate(SSyncIndexMgr *pSyncIndexMgr, SSyncNode *pSyncNode);
 void           syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr);
 void           syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr);
 void           syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncIndex index);

source/libs/sync/inc/syncInt.h

@@ -247,6 +247,7 @@ typedef struct SSyncNode {
 // open/close --------------
 SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo);
 void       syncNodeStart(SSyncNode* pSyncNode);
+void       syncNodeStartStandBy(SSyncNode* pSyncNode);
 void       syncNodeClose(SSyncNode* pSyncNode);
 
 // ping --------------
@@ -271,7 +272,7 @@ int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, S
 cJSON*  syncNode2Json(const SSyncNode* pSyncNode);
 char*   syncNode2Str(const SSyncNode* pSyncNode);
 char*   syncNode2SimpleStr(const SSyncNode* pSyncNode);
-void    syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg *newConfig);
+void    syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig);
 
 SSyncNode* syncNodeAcquire(int64_t rid);
 void       syncNodeRelease(SSyncNode* pNode);

source/libs/sync/inc/syncUtil.h

@@ -62,7 +62,6 @@ bool        syncUtilUserPreCommit(tmsg_t msgType);
 bool        syncUtilUserCommit(tmsg_t msgType);
 bool        syncUtilUserRollback(tmsg_t msgType);
 
-
 #ifdef __cplusplus
 }
 #endif

source/libs/sync/src/syncAppendEntries.c

@@ -15,11 +15,11 @@
 
 #include "syncAppendEntries.h"
 #include "syncInt.h"
+#include "syncRaftCfg.h"
 #include "syncRaftLog.h"
 #include "syncRaftStore.h"
 #include "syncUtil.h"
 #include "syncVoteMgr.h"
-#include "syncRaftCfg.h"
 
 // TLA+ Spec
 // HandleAppendEntriesRequest(i, j, m) ==
@@ -200,7 +200,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
             SSyncRaftEntry* pRollBackEntry = logStoreGetEntry(ths->pLogStore, index);
             assert(pRollBackEntry != NULL);
 
-            //if (pRollBackEntry->msgType != TDMT_VND_SYNC_NOOP) {
+            // if (pRollBackEntry->msgType != TDMT_VND_SYNC_NOOP) {
             if (syncUtilUserRollback(pRollBackEntry->msgType)) {
               SRpcMsg rpcMsg;
               syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg);
@@ -229,7 +229,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
         SRpcMsg rpcMsg;
         syncEntry2OriginalRpc(pAppendEntry, &rpcMsg);
         if (ths->pFsm != NULL) {
-          //if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+          // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
           if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) {
             SFsmCbMeta cbMeta;
             cbMeta.index = pAppendEntry->index;
@@ -261,7 +261,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
       SRpcMsg rpcMsg;
       syncEntry2OriginalRpc(pAppendEntry, &rpcMsg);
       if (ths->pFsm != NULL) {
-        //if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+        // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
         if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) {
           SFsmCbMeta cbMeta;
           cbMeta.index = pAppendEntry->index;
@@ -324,7 +324,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
               SRpcMsg rpcMsg;
               syncEntry2OriginalRpc(pEntry, &rpcMsg);
 
-              //if (ths->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+              // if (ths->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
               if (ths->pFsm->FpCommitCb != NULL && syncUtilUserCommit(pEntry->originalRpcType)) {
                 SFsmCbMeta cbMeta;
                 cbMeta.index = pEntry->index;
@@ -338,10 +338,15 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
               // config change
               if (pEntry->originalRpcType == TDMT_VND_SYNC_CONFIG_CHANGE) {
                 SSyncCfg newSyncCfg;
-                int32_t ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg);
+                int32_t  ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg);
                 ASSERT(ret == 0);
 
                 syncNodeUpdateConfig(ths, &newSyncCfg);
+                if (ths->state == TAOS_SYNC_STATE_LEADER) {
+                  syncNodeBecomeLeader(ths);
+                } else {
+                  syncNodeBecomeFollower(ths);
+                }
               }
 
               rpcFreeCont(rpcMsg.pCont);

source/libs/sync/src/syncCommit.c

@@ -16,10 +16,10 @@
 #include "syncCommit.h"
 #include "syncIndexMgr.h"
 #include "syncInt.h"
+#include "syncRaftCfg.h"
 #include "syncRaftLog.h"
 #include "syncRaftStore.h"
 #include "syncUtil.h"
-#include "syncRaftCfg.h"
 
 // \* Leader i advances its commitIndex.
 // \* This is done as a separate step from handling AppendEntries responses,
@@ -102,7 +102,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
           SRpcMsg rpcMsg;
           syncEntry2OriginalRpc(pEntry, &rpcMsg);
 
-          //if (pSyncNode->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+          // if (pSyncNode->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
           if (pSyncNode->pFsm->FpCommitCb != NULL && syncUtilUserCommit(pEntry->originalRpcType)) {
             SFsmCbMeta cbMeta;
             cbMeta.index = pEntry->index;
@@ -114,12 +114,17 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
           }
 
           // config change
-            if (pEntry->originalRpcType == TDMT_VND_SYNC_CONFIG_CHANGE) {
+          if (pEntry->originalRpcType == TDMT_VND_SYNC_CONFIG_CHANGE) {
-              SSyncCfg newSyncCfg;
+            SSyncCfg newSyncCfg;
-              int32_t ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg);
+            int32_t  ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg);
-              ASSERT(ret == 0);
+            ASSERT(ret == 0);
 
-              syncNodeUpdateConfig(pSyncNode, &newSyncCfg);
+            syncNodeUpdateConfig(pSyncNode, &newSyncCfg);
+            if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
+              syncNodeBecomeLeader(pSyncNode);
+            } else {
+              syncNodeBecomeFollower(pSyncNode);
+            }
           }
 
           rpcFreeCont(rpcMsg.pCont);

source/libs/sync/src/syncIO.c

@@ -15,6 +15,7 @@
 
 #include "syncIO.h"
 #include <tdatablock.h>
+#include "os.h"
 #include "syncMessage.h"
 #include "syncUtil.h"
 #include "tglobal.h"
@@ -198,6 +199,7 @@ static int32_t syncIOStartInternal(SSyncIO *io) {
   {
     SRpcInit rpcInit;
     memset(&rpcInit, 0, sizeof(rpcInit));
+    snprintf(rpcInit.localFqdn, sizeof(rpcInit.localFqdn), "%s", "127.0.0.1");
     rpcInit.localPort = io->myAddr.eps[0].port;
     rpcInit.label = "SYNC-IO-SERVER";
     rpcInit.numOfThreads = 1;

source/libs/sync/src/syncIndexMgr.c

@@ -31,6 +31,13 @@ SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode) {
   return pSyncIndexMgr;
 }
 
+void syncIndexMgrUpdate(SSyncIndexMgr *pSyncIndexMgr, SSyncNode *pSyncNode) {
+  pSyncIndexMgr->replicas = &(pSyncNode->replicasId);
+  pSyncIndexMgr->replicaNum = pSyncNode->replicaNum;
+  pSyncIndexMgr->pSyncNode = pSyncNode;
+  syncIndexMgrClear(pSyncIndexMgr);
+}
+
 void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr) {
   if (pSyncIndexMgr != NULL) {
     taosMemoryFree(pSyncIndexMgr);

source/libs/sync/src/syncMain.c

@@ -103,6 +103,16 @@ void syncStart(int64_t rid) {
   taosReleaseRef(tsNodeRefId, pSyncNode->rid);
 }
 
+void syncStartStandBy(int64_t rid) {
+  SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid);
+  if (pSyncNode == NULL) {
+    return;
+  }
+  syncNodeStartStandBy(pSyncNode);
+
+  taosReleaseRef(tsNodeRefId, pSyncNode->rid);
+}
+
 void syncStop(int64_t rid) {
   SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid);
   if (pSyncNode == NULL) {
@@ -116,7 +126,7 @@ void syncStop(int64_t rid) {
 
 int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) {
   int32_t ret = 0;
-  char *configChange = syncCfg2Str((SSyncCfg*)pSyncCfg);
+  char*   configChange = syncCfg2Str((SSyncCfg*)pSyncCfg);
   SRpcMsg rpcMsg = {0};
   rpcMsg.msgType = TDMT_VND_SYNC_CONFIG_CHANGE;
   rpcMsg.noResp = 1;
@@ -188,10 +198,9 @@ void syncGetEpSet(int64_t rid, SEpSet* pEpSet) {
     (pEpSet->numOfEps)++;
 
     sInfo("syncGetEpSet index:%d %s:%d", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port);
-
   }
   pEpSet->inUse = pSyncNode->pRaftCfg->cfg.myIndex;
-  
+
   sInfo("syncGetEpSet pEpSet->inUse:%d ", pEpSet->inUse);
 
   taosReleaseRef(tsNodeRefId, pSyncNode->rid);
@@ -524,6 +533,17 @@ void syncNodeStart(SSyncNode* pSyncNode) {
   assert(ret == 0);
 }
 
+void syncNodeStartStandBy(SSyncNode* pSyncNode) {
+  // state change
+  pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER;
+  syncNodeStopHeartbeatTimer(pSyncNode);
+
+  // reset elect timer, long enough
+  int32_t electMS = TIMER_MAX_MS;
+  int32_t ret = syncNodeRestartElectTimer(pSyncNode, electMS);
+  ASSERT(ret == 0);
+}
+
 void syncNodeClose(SSyncNode* pSyncNode) {
   int32_t ret;
   assert(pSyncNode != NULL);
@@ -858,7 +878,7 @@ char* syncNode2SimpleStr(const SSyncNode* pSyncNode) {
   return s;
 }
 
-void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg *newConfig) {
+void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig) {
   pSyncNode->pRaftCfg->cfg = *newConfig;
   int32_t ret = raftCfgPersist(pSyncNode->pRaftCfg);
   ASSERT(ret == 0);
@@ -885,6 +905,11 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg *newConfig) {
   for (int i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) {
     syncUtilnodeInfo2raftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]);
   }
+
+  syncIndexMgrUpdate(pSyncNode->pNextIndex, pSyncNode);
+  syncIndexMgrUpdate(pSyncNode->pMatchIndex, pSyncNode);
+
+  syncNodeLog2("==syncNodeUpdateConfig==", pSyncNode);
 }
 
 SSyncNode* syncNodeAcquire(int64_t rid) {
@@ -1245,7 +1270,7 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) {
     syncEntry2OriginalRpc(pEntry, &rpcMsg);
 
     if (ths->pFsm != NULL) {
-      //if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+      // if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
       if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) {
         SFsmCbMeta cbMeta;
         cbMeta.index = pEntry->index;
@@ -1267,7 +1292,7 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) {
     syncEntry2OriginalRpc(pEntry, &rpcMsg);
 
     if (ths->pFsm != NULL) {
-      //if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+      // if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
       if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) {
         SFsmCbMeta cbMeta;
         cbMeta.index = pEntry->index;

source/libs/sync/src/syncRaftLog.c

@@ -58,14 +58,15 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
   syncMeta.term = pEntry->term;
   code = walWriteWithSyncInfo(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen);
   if (code != 0) {
-    int32_t err = terrno;
+    int32_t     err = terrno;
-    const char *errStr = tstrerror(err);
+    const char* errStr = tstrerror(err);
-    int32_t linuxErr = errno;
+    int32_t     linuxErr = errno;
-    const char *linuxErrMsg = strerror(errno);
+    const char* linuxErrMsg = strerror(errno);
-    sError("walWriteWithSyncInfo error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg);
+    sError("walWriteWithSyncInfo error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr,
+           linuxErrMsg);
     ASSERT(0);
-  }    
+  }
-  //assert(code == 0);
+  // assert(code == 0);
 
   walFsync(pWal, true);
   return code;
@@ -77,16 +78,17 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
 
   if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) {
     SWalReadHandle* pWalHandle = walOpenReadHandle(pWal);
-    int32_t code = walReadWithHandle(pWalHandle, index);
+    int32_t         code = walReadWithHandle(pWalHandle, index);
     if (code != 0) {
-      int32_t err = terrno;
+      int32_t     err = terrno;
-      const char *errStr = tstrerror(err);
+      const char* errStr = tstrerror(err);
-      int32_t linuxErr = errno;
+      int32_t     linuxErr = errno;
-      const char *linuxErrMsg = strerror(errno);
+      const char* linuxErrMsg = strerror(errno);
-      sError("walReadWithHandle error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg);
+      sError("walReadWithHandle error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr,
+             linuxErrMsg);
       ASSERT(0);
-    }    
+    }
-    //assert(walReadWithHandle(pWalHandle, index) == 0);
+    // assert(walReadWithHandle(pWalHandle, index) == 0);
 
     SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen);
     assert(pEntry != NULL);
@@ -112,16 +114,17 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
 int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) {
   SSyncLogStoreData* pData = pLogStore->data;
   SWal*              pWal = pData->pWal;
-  //assert(walRollback(pWal, fromIndex) == 0);
+  // assert(walRollback(pWal, fromIndex) == 0);
   int32_t code = walRollback(pWal, fromIndex);
   if (code != 0) {
-    int32_t err = terrno;
+    int32_t     err = terrno;
-    const char *errStr = tstrerror(err);
+    const char* errStr = tstrerror(err);
-    int32_t linuxErr = errno;
+    int32_t     linuxErr = errno;
-    const char *linuxErrMsg = strerror(errno);
+    const char* linuxErrMsg = strerror(errno);
-    sError("walRollback error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg);
+    sError("walRollback error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr,
+           linuxErrMsg);
     ASSERT(0);
-  } 
+  }
   return 0;  // to avoid compiler error
 }
 
@@ -145,16 +148,16 @@ SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore) {
 int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) {
   SSyncLogStoreData* pData = pLogStore->data;
   SWal*              pWal = pData->pWal;
-  //assert(walCommit(pWal, index) == 0);
+  // assert(walCommit(pWal, index) == 0);
   int32_t code = walCommit(pWal, index);
   if (code != 0) {
-    int32_t err = terrno;
+    int32_t     err = terrno;
-    const char *errStr = tstrerror(err);
+    const char* errStr = tstrerror(err);
-    int32_t linuxErr = errno;
+    int32_t     linuxErr = errno;
-    const char *linuxErrMsg = strerror(errno);
+    const char* linuxErrMsg = strerror(errno);
     sError("walCommit error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg);
     ASSERT(0);
-  } 
+  }
   return 0;  // to avoid compiler error
 }
 

source/libs/sync/test/CMakeLists.txt

@@ -37,6 +37,7 @@ add_executable(syncRaftCfgTest "")
 add_executable(syncRespMgrTest "")
 add_executable(syncSnapshotTest "")
 add_executable(syncApplyMsgTest "")
+add_executable(syncConfigChangeTest "")
 
 
 target_sources(syncTest
@@ -195,6 +196,10 @@ target_sources(syncApplyMsgTest
     PRIVATE
     "syncApplyMsgTest.cpp"
 )
+target_sources(syncConfigChangeTest
+    PRIVATE
+    "syncConfigChangeTest.cpp"
+)
 
 
 target_include_directories(syncTest
@@ -392,6 +397,11 @@ target_include_directories(syncApplyMsgTest
     "${TD_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
+target_include_directories(syncConfigChangeTest
+    PUBLIC
+    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
 
 
 target_link_libraries(syncTest
@@ -550,6 +560,10 @@ target_link_libraries(syncApplyMsgTest
     sync
     gtest_main
 )
+target_link_libraries(syncConfigChangeTest
+    sync
+    gtest_main
+)
 
 
 enable_testing()

source/libs/sync/test/syncConfigChangeTest.cpp

@@ -0,0 +1,259 @@
+#include <gtest/gtest.h>
+#include <stdio.h>
+#include "os.h"
+#include "syncEnv.h"
+#include "syncIO.h"
+#include "syncInt.h"
+#include "syncUtil.h"
+#include "wal.h"
+
+void logTest() {
+  sTrace("--- sync log test: trace");
+  sDebug("--- sync log test: debug");
+  sInfo("--- sync log test: info");
+  sWarn("--- sync log test: warn");
+  sError("--- sync log test: error");
+  sFatal("--- sync log test: fatal");
+}
+
+uint16_t    gPorts[] = {7010, 7110, 7210, 7310, 7410};
+const char* gDir = "./syncReplicateTest";
+int32_t     gVgId = 1234;
+SyncIndex   gSnapshotLastApplyIndex;
+
+void init() {
+  int code = walInit();
+  assert(code == 0);
+
+  code = syncInit();
+  assert(code == 0);
+
+  sprintf(tsTempDir, "%s", ".");
+}
+
+void cleanup() { walCleanUp(); }
+
+void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {
+  SyncIndex beginIndex = SYNC_INDEX_INVALID;
+  if (pFsm->FpGetSnapshot != NULL) {
+    SSnapshot snapshot;
+    pFsm->FpGetSnapshot(pFsm, &snapshot);
+    beginIndex = snapshot.lastApplyIndex;
+  }
+
+  if (cbMeta.index > beginIndex) {
+    char logBuf[256];
+    snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n",
+             pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state));
+    syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg);
+  } else {
+    sTrace("==callback== ==CommitCb== do not apply again %ld", cbMeta.index);
+  }
+}
+
+void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {
+  char logBuf[256];
+  snprintf(logBuf, sizeof(logBuf),
+           "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index,
+           cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state));
+  syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg);
+}
+
+void RollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {
+  char logBuf[256];
+  snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n",
+           pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state));
+  syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg);
+}
+
+int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) {
+  pSnapshot->data = NULL;
+  pSnapshot->lastApplyIndex = gSnapshotLastApplyIndex;
+  pSnapshot->lastApplyTerm = 100;
+  return 0;
+}
+
+SSyncFSM* createFsm() {
+  SSyncFSM* pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM));
+  pFsm->FpCommitCb = CommitCb;
+  pFsm->FpPreCommitCb = PreCommitCb;
+  pFsm->FpRollBackCb = RollBackCb;
+  pFsm->FpGetSnapshot = GetSnapshotCb;
+  return pFsm;
+}
+
+SWal* createWal(char* path, int32_t vgId) {
+  SWalCfg walCfg;
+  memset(&walCfg, 0, sizeof(SWalCfg));
+  walCfg.vgId = vgId;
+  walCfg.fsyncPeriod = 1000;
+  walCfg.retentionPeriod = 1000;
+  walCfg.rollPeriod = 1000;
+  walCfg.retentionSize = 1000;
+  walCfg.segSize = 1000;
+  walCfg.level = TAOS_WAL_FSYNC;
+  SWal* pWal = walOpen(path, &walCfg);
+  assert(pWal != NULL);
+  return pWal;
+}
+
+int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* pWal, char* path, bool isStandBy) {
+  SSyncInfo syncInfo;
+  syncInfo.vgId = vgId;
+  syncInfo.rpcClient = gSyncIO->clientRpc;
+  syncInfo.FpSendMsg = syncIOSendMsg;
+  syncInfo.queue = gSyncIO->pMsgQ;
+  syncInfo.FpEqMsg = syncIOEqMsg;
+  syncInfo.pFsm = createFsm();
+  snprintf(syncInfo.path, sizeof(syncInfo.path), "%s_sync_replica%d_index%d", path, replicaNum, myIndex);
+  syncInfo.pWal = pWal;
+
+  SSyncCfg* pCfg = &syncInfo.syncCfg;
+
+  if (isStandBy) {
+    pCfg->myIndex = 0;
+    pCfg->replicaNum = 1;
+    pCfg->nodeInfo[0].nodePort = gPorts[myIndex];
+    taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
+
+  } else {
+    pCfg->myIndex = myIndex;
+    pCfg->replicaNum = replicaNum;
+
+    for (int i = 0; i < replicaNum; ++i) {
+      pCfg->nodeInfo[i].nodePort = gPorts[i];
+      taosGetFqdn(pCfg->nodeInfo[i].nodeFqdn);
+      // snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1");
+    }
+  }
+
+  int64_t rid = syncOpen(&syncInfo);
+  assert(rid > 0);
+
+  SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid);
+  assert(pSyncNode != NULL);
+  gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
+  gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
+  gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote;
+  gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply;
+  gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries;
+  gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply;
+  gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
+  gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
+  gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout;
+  gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest;
+  gSyncIO->pSyncNode = pSyncNode;
+  syncNodeRelease(pSyncNode);
+
+  return rid;
+}
+
+void configChange(int64_t rid, int32_t replicaNum, int32_t myIndex) {
+  SSyncCfg syncCfg;
+
+  syncCfg.myIndex = myIndex;
+  syncCfg.replicaNum = replicaNum;
+
+  for (int i = 0; i < replicaNum; ++i) {
+    syncCfg.nodeInfo[i].nodePort = gPorts[i];
+    taosGetFqdn(syncCfg.nodeInfo[i].nodeFqdn);
+  }
+
+  syncReconfig(rid, &syncCfg);
+}
+
+void usage(char* exe) {
+  printf("usage: %s replicaNum myIndex lastApplyIndex writeRecordNum isStandBy isConfigChange \n", exe);
+}
+
+SRpcMsg* createRpcMsg(int i, int count, int myIndex) {
+  SRpcMsg* pMsg = (SRpcMsg*)taosMemoryMalloc(sizeof(SRpcMsg));
+  memset(pMsg, 0, sizeof(SRpcMsg));
+  pMsg->msgType = 9999;
+  pMsg->contLen = 256;
+  pMsg->pCont = rpcMallocCont(pMsg->contLen);
+  snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-%ld", myIndex, i, count, taosGetTimestampMs());
+  return pMsg;
+}
+
+int main(int argc, char** argv) {
+  tsAsyncLog = 0;
+  sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE;
+  if (argc != 7) {
+    usage(argv[0]);
+    exit(-1);
+  }
+
+  int32_t replicaNum = atoi(argv[1]);
+  int32_t myIndex = atoi(argv[2]);
+  int32_t lastApplyIndex = atoi(argv[3]);
+  int32_t writeRecordNum = atoi(argv[4]);
+  bool    isStandBy = atoi(argv[5]);
+  bool    isConfigChange = atoi(argv[6]);
+  gSnapshotLastApplyIndex = lastApplyIndex;
+
+  if (!isStandBy) {
+    assert(replicaNum >= 1 && replicaNum <= 5);
+    assert(myIndex >= 0 && myIndex < replicaNum);
+    assert(lastApplyIndex >= -1);
+    assert(writeRecordNum >= 0);
+  }
+
+  init();
+  int32_t ret = syncIOStart((char*)"127.0.0.1", gPorts[myIndex]);
+  assert(ret == 0);
+
+  char walPath[128];
+  snprintf(walPath, sizeof(walPath), "%s_wal_replica%d_index%d", gDir, replicaNum, myIndex);
+  SWal* pWal = createWal(walPath, gVgId);
+
+  int64_t rid = createSyncNode(replicaNum, myIndex, gVgId, pWal, (char*)gDir, isStandBy);
+  assert(rid > 0);
+
+  if (isStandBy) {
+    syncStartStandBy(rid);
+  } else {
+    syncStart(rid);
+  }
+
+  SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid);
+  assert(pSyncNode != NULL);
+
+  if (isConfigChange) {
+    configChange(rid, 3, myIndex);
+  }
+
+  //---------------------------
+  int32_t alreadySend = 0;
+  while (1) {
+    char* s = syncNode2SimpleStr(pSyncNode);
+
+    if (alreadySend < writeRecordNum) {
+      SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex);
+      int32_t  ret = syncPropose(rid, pRpcMsg, false);
+      if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) {
+        sTrace("%s value%d write not leader", s, alreadySend);
+      } else {
+        assert(ret == 0);
+        sTrace("%s value%d write ok", s, alreadySend);
+      }
+      alreadySend++;
+
+      rpcFreeCont(pRpcMsg->pCont);
+      taosMemoryFree(pRpcMsg);
+    } else {
+      sTrace("%s", s);
+    }
+
+    taosMsleep(1000);
+    taosMemoryFree(s);
+    taosMsleep(1000);
+  }
+
+  syncNodeRelease(pSyncNode);
+  syncStop(rid);
+  walClose(pWal);
+  syncIOStop();
+  cleanup();
+  return 0;
+}

source/libs/transport/src/transSrv.c

@@ -137,7 +137,9 @@ static void destroySmsg(SSrvMsg* smsg);
 // check whether already read complete packet
 static SSrvConn* createConn(void* hThrd);
 static void      destroyConn(SSrvConn* conn, bool clear /*clear handle or not*/);
-static int       reallocConnRefHandle(SSrvConn* conn);
+static void      destroyConnRegArg(SSrvConn* conn);
+
+static int reallocConnRefHandle(SSrvConn* conn);
 
 static void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd);
 static void uvHandleRelease(SSrvMsg* msg, SWorkThrdObj* thrd);
@@ -429,6 +431,8 @@ static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) {
     if (smsg->type == Release) {
       pHead->msgType = 0;
       pConn->status = ConnNormal;
+
+      destroyConnRegArg(pConn);
       transUnrefSrvHandle(pConn);
     } else {
       pHead->msgType = pMsg->msgType;
@@ -800,6 +804,12 @@ static void destroyConn(SSrvConn* conn, bool clear) {
     // uv_shutdown(req, (uv_stream_t*)conn->pTcp, uvShutDownCb);
   }
 }
+static void destroyConnRegArg(SSrvConn* conn) {
+  if (conn->regArg.init == 1) {
+    transFreeMsg(conn->regArg.msg.pCont);
+    conn->regArg.init = 0;
+  }
+}
 static int reallocConnRefHandle(SSrvConn* conn) {
   uvReleaseExHandle(conn->refId);
   uvRemoveExHandle(conn->refId);
@@ -827,16 +837,9 @@ static void uvDestroyConn(uv_handle_t* handle) {
   // uv_timer_stop(&conn->pTimer);
   transQueueDestroy(&conn->srvMsgs);
 
-  if (conn->regArg.init == 1) {
-    transFreeMsg(conn->regArg.msg.pCont);
-    conn->regArg.init = 0;
-  }
   QUEUE_REMOVE(&conn->queue);
   taosMemoryFree(conn->pTcp);
-  if (conn->regArg.init == 1) {
+  destroyConnRegArg(conn);
-    transFreeMsg(conn->regArg.msg.pCont);
-    conn->regArg.init = 0;
-  }
   taosMemoryFree(conn);
 
   if (thrd->quit && QUEUE_IS_EMPTY(&thrd->conn)) {

source/util/src/terror.c

@@ -323,6 +323,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_TB_NOT_EXIST,             "Table not exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_SMA_NOT_EXIST,            "SMA not exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_HASH_MISMATCH,            "Hash value mismatch")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_TABLE_NOT_EXIST,          "Table does not exists")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_TABLE_ACTION,     "Invalid table action")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_ALREADY_EXISTS,       "Table column already exists")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_TABLE_COL_NOT_EXISTS,     "Table column not exists")
+
 
 // tsdb
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID,         "Invalid table ID")

source/util/test/CMakeLists.txt

@@ -33,13 +33,13 @@ ENDIF()
 
 INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/src/util/inc)
 
-# freelistTest
+# # freelistTest
-add_executable(freelistTest "")
+# add_executable(freelistTest "")
-target_sources(freelistTest
+# target_sources(freelistTest
-    PRIVATE
+#     PRIVATE
-    "freelistTest.cpp"
+#     "freelistTest.cpp"
-)
+# )
-target_link_libraries(freelistTest os util gtest gtest_main)
+# target_link_libraries(freelistTest os util gtest gtest_main)
 
 # # encodeTest
 # add_executable(encodeTest "encodeTest.cpp")

source/util/test/freelistTest.cpp

@@ -1,17 +0,0 @@
-#include <gtest/gtest.h>
-
-#include "tfreelist.h"
-
-TEST(TD_UTIL_FREELIST_TEST, simple_test) {
-  SFreeList fl;
-
-  tFreeListInit(&fl);
-
-  for (size_t i = 0; i < 1000; i++) {
-    void *ptr = NULL;
-    TFL_MALLOC(ptr, void*, 1024, &fl);
-    GTEST_ASSERT_NE(ptr, nullptr);
-  }
-
-  tFreeListClear(&fl);
-}

tests/script/tsim/query/udf.sim

@@ -7,7 +7,7 @@ system sh/cfg.sh -n dnode1 -c udf -v 1
 
 print ========= start dnode1 as LEADER
 system sh/exec.sh -n dnode1 -s start
-sleep 2000
+sleep 1000
 sql connect
 
 print ======== step1 udf 

tests/script/tsim/sync/threeReplica1VgElect.sim

@@ -220,7 +220,6 @@ if $data[0][4] == LEADER then
   print ---- vgroup $data[0][0] leader switch to dnode $data[0][3]
 elif $data[0][6] == LEADER then
   print ---- vgroup $data[0][0] leader switch to dnode $data[0][5]
-endi
 elif $data[0][8] == LEADER then
   print ---- vgroup $data[0][0] leader switch to dnode $data[0][7]
 else 
@@ -342,7 +341,6 @@ elif $data[0][6] == LEADER then
     goto check_vg_ready_3
   endi
   print ---- vgroup $data[0][0] leader locating dnode $data[0][7]
-endi
 elif $data[0][8] == LEADER then
   if $data[0][4] == LEADER then
     goto check_vg_ready_3

tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim

@@ -420,7 +420,6 @@ elif $data[0][6] == LEADER then
     goto check_vg_ready_3
   endi
   print ---- vgroup $data[0][0] leader locating dnode $data[0][7]
-endi
 elif $data[0][8] == LEADER then
   if $data[0][4] == LEADER then
     goto check_vg_ready_3

tests/script/tsim/tmq/basic1.sim

@@ -1,278 +1,288 @@
-#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
+#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
-#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics
+#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics
-#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics
+#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics
-#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics
+#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics
-#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics
+#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics
-
+
-# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
+# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
-# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
+# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
-#
+#
-# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
+# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
-#
+#
-
+
-run tsim/tmq/prepareBasicEnv-1vgrp.sim
+run tsim/tmq/prepareBasicEnv-1vgrp.sim
-
+
-#---- global parameters start ----#
+#---- global parameters start ----#
-$dbName    = db
+$dbName    = db
-$vgroups    = 1
+$vgroups    = 1
-$stbPrefix  = stb
+$stbPrefix  = stb
-$ctbPrefix  = ctb
+$ctbPrefix  = ctb
-$ntbPrefix  = ntb
+$ntbPrefix  = ntb
-$stbNum     = 1
+$stbNum     = 1
-$ctbNum     = 10
+$ctbNum     = 10
-$ntbNum     = 10
+$ntbNum     = 10
-$rowsPerCtb = 10
+$rowsPerCtb = 10
-$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
+$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
-#---- global parameters end ----#
+#---- global parameters end ----#
-
+
-$pullDelay    = 3
+$pullDelay    = 3
-$ifcheckdata  = 1
+$ifcheckdata  = 1
-$showMsg      = 1
+$ifmanualcommit = 1
-$showRow      = 0
+$showMsg      = 1
-
+$showRow      = 0
-sql connect
+
-sql use $dbName
+sql connect
-
+sql use $dbName
-print == create topics from super table
+
-sql create topic topic_stb_column as select ts, c3 from stb
+print == create topics from super table
-sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
+sql create topic topic_stb_column as select ts, c3 from stb
-sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
+sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
-
+sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
-print == create topics from child table
+
-sql create topic topic_ctb_column as select ts, c3 from ctb0
+print == create topics from child table
-sql create topic topic_ctb_all as select * from ctb0
+sql create topic topic_ctb_column as select ts, c3 from ctb0
-sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
+sql create topic topic_ctb_all as select * from ctb0
-
+sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
-print == create topics from normal table
+
-sql create topic topic_ntb_column as select ts, c3 from ntb0
+print == create topics from normal table
-sql create topic topic_ntb_all as select * from ntb0
+sql create topic topic_ntb_column as select ts, c3 from ntb0
-sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
+sql create topic topic_ntb_all as select * from ntb0
-
+sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
-#sql show topics
+
-#if $rows != 9 then 
+#sql show topics
-#  return -1
+#if $rows != 9 then 
-#endi
+#  return -1
-
+#endi
-$keyList = ' . group.id:cgrp1
+
-$keyList = $keyList . '
+#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest'
-
+$keyList = ' . group.id:cgrp1
-$cdb_index = 0
+$keyList = $keyList . ,
-#=============================== start consume =============================#
+$keyList = $keyList . enable.auto.commit:false
-
+#$keyList = $keyList . ,
-print ================ test consume from stb
+#$keyList = $keyList . auto.commit.interval.ms:6000
-$loop_cnt = 0
+#$keyList = $keyList . ,
-loop_consume_diff_topic_from_stb:
+#$keyList = $keyList . auto.offset.reset:earliest
-
+$keyList = $keyList . '
-#######################################################################################
+print ========== key list:  $keyList
-# clear consume info and consume result 
+
-#run tsim/tmq/clearConsume.sim
+
-# because drop table function no stable, so by create new db for consume info and result. Modify it later
+$cdb_index = 0
-$cdb_index = $cdb_index + 1
+#=============================== start consume =============================#
-$cdbName = cdb . $cdb_index
+
-sql create database $cdbName vgroups 1
+print ================ test consume from stb
-sleep 500
+$loop_cnt = 0
-sql use $cdbName
+loop_consume_diff_topic_from_stb:
-
+
-print == create consume info table and consume result table
+#######################################################################################
-sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+# clear consume info and consume result 
-sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+#run tsim/tmq/clearConsume.sim
-
+# because drop table function no stable, so by create new db for consume info and result. Modify it later
-sql show tables
+$cdb_index = $cdb_index + 1
-if $rows != 2 then 
+$cdbName = cdb . $cdb_index
-  return -1
+sql create database $cdbName vgroups 1
-endi
+sleep 500
-#######################################################################################
+sql use $cdbName
-
+
-if $loop_cnt == 0 then
+print == create consume info table and consume result table
-  print == scenario 1: topic_stb_column
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
-  $topicList = ' . topic_stb_column
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
-  $topicList = $topicList . '
+
-elif $loop_cnt == 1 then
+sql show tables
-  print == scenario 2: topic_stb_all
+if $rows != 2 then 
-  $topicList = ' . topic_stb_all
+  return -1
-  $topicList = $topicList . '
+endi
-elif $loop_cnt == 2 then
+#######################################################################################
-  print == scenario 3: topic_stb_function
+
-  $topicList = ' . topic_stb_function
+if $loop_cnt == 0 then
-  $topicList = $topicList . '
+  print == scenario 1: topic_stb_column
-else
+  $topicList = ' . topic_stb_column
-  goto loop_consume_diff_topic_from_stb_end
+  $topicList = $topicList . '
-endi
+elif $loop_cnt == 1 then
-
+  print == scenario 2: topic_stb_all
-$consumerId    = 0
+  $topicList = ' . topic_stb_all
-$totalMsgOfStb = $ctbNum * $rowsPerCtb
+  $topicList = $topicList . '
-$expectmsgcnt  = $totalMsgOfStb
+elif $loop_cnt == 2 then
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+  print == scenario 3: topic_stb_function
-
+  $topicList = ' . topic_stb_function
-print == start consumer to pull msgs from stb
+  $topicList = $topicList . '
-print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+else
-system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+  goto loop_consume_diff_topic_from_stb_end
-
+endi
-print == check consume result
+
-wait_consumer_end_from_stb:
+$consumerId    = 0
-sql select * from consumeresult
+$totalMsgOfStb = $ctbNum * $rowsPerCtb
-print ==> rows: $rows 
+$expectmsgcnt  = $totalMsgOfStb
-print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-if $rows != 1 then
+
-  sleep 1000
+print == start consumer to pull msgs from stb
-  goto wait_consumer_end_from_stb
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
-endi
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
-if $data[0][1] != $consumerId then
+
-  return -1
+print == check consume result
-endi
+wait_consumer_end_from_stb:
-if $data[0][2] != $expectmsgcnt then
+sql select * from consumeresult
-  return -1
+print ==> rows: $rows 
-endi
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
-if $data[0][3] != $expectmsgcnt then
+if $rows != 1 then
-  return -1
+  sleep 1000
-endi
+  goto wait_consumer_end_from_stb
-$loop_cnt = $loop_cnt + 1
+endi
-goto loop_consume_diff_topic_from_stb
+if $data[0][1] != $consumerId then
-loop_consume_diff_topic_from_stb_end:
+  return -1
-
+endi
-print ================ test consume from ctb
+if $data[0][2] != $expectmsgcnt then
-$loop_cnt = 0
+  return -1
-loop_consume_diff_topic_from_ctb:
+endi
-
+if $data[0][3] != $expectmsgcnt then
-#######################################################################################
+  return -1
-# clear consume info and consume result 
+endi
-#run tsim/tmq/clearConsume.sim
+$loop_cnt = $loop_cnt + 1
-# because drop table function no stable, so by create new db for consume info and result. Modify it later
+goto loop_consume_diff_topic_from_stb
-$cdb_index = $cdb_index + 1
+loop_consume_diff_topic_from_stb_end:
-$cdbName = cdb . $cdb_index
+
-sql create database $cdbName vgroups 1
+print ================ test consume from ctb
-sleep 500
+$loop_cnt = 0
-sql use $cdbName
+loop_consume_diff_topic_from_ctb:
-
+
-print == create consume info table and consume result table
+#######################################################################################
-sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+# clear consume info and consume result 
-sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+#run tsim/tmq/clearConsume.sim
-
+# because drop table function no stable, so by create new db for consume info and result. Modify it later
-sql show tables
+$cdb_index = $cdb_index + 1
-if $rows != 2 then 
+$cdbName = cdb . $cdb_index
-  return -1
+sql create database $cdbName vgroups 1
-endi
+sleep 500
-#######################################################################################
+sql use $cdbName
-
+
-if $loop_cnt == 0 then
+print == create consume info table and consume result table
-  print == scenario 1: topic_ctb_column
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
-  $topicList = ' . topic_ctb_column
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
-  $topicList = $topicList . '
+
-elif $loop_cnt == 1 then
+sql show tables
-  print == scenario 2: topic_ctb_all
+if $rows != 2 then 
-  $topicList = ' . topic_ctb_all
+  return -1
-  $topicList = $topicList . '
+endi
-elif $loop_cnt == 2 then
+#######################################################################################
-  print == scenario 3: topic_ctb_function
+
-  $topicList = ' . topic_ctb_function
+if $loop_cnt == 0 then
-  $topicList = $topicList . '
+  print == scenario 1: topic_ctb_column
-else
+  $topicList = ' . topic_ctb_column
-  goto loop_consume_diff_topic_from_ctb_end
+  $topicList = $topicList . '
-endi
+elif $loop_cnt == 1 then
-
+  print == scenario 2: topic_ctb_all
-$consumerId    = 0
+  $topicList = ' . topic_ctb_all
-$totalMsgOfCtb = $rowsPerCtb
+  $topicList = $topicList . '
-$expectmsgcnt  = $totalMsgOfCtb
+elif $loop_cnt == 2 then
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+  print == scenario 3: topic_ctb_function
-
+  $topicList = ' . topic_ctb_function
-print == start consumer to pull msgs from ctb
+  $topicList = $topicList . '
-print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
+else
-system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+  goto loop_consume_diff_topic_from_ctb_end
-
+endi
-print == check consume result
+
-wait_consumer_end_from_ctb:
+$consumerId    = 0
-sql select * from consumeresult
+$totalMsgOfCtb = $rowsPerCtb
-print ==> rows: $rows 
+$expectmsgcnt  = $totalMsgOfCtb
-print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-if $rows != 1 then
+
-  sleep 1000
+print == start consumer to pull msgs from ctb
-  goto wait_consumer_end_from_ctb
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
-endi
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
-if $data[0][1] != $consumerId then
+
-  return -1
+print == check consume result
-endi
+wait_consumer_end_from_ctb:
-if $data[0][2] != $totalMsgOfCtb then
+sql select * from consumeresult
-  return -1
+print ==> rows: $rows 
-endi
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
-if $data[0][3] != $totalMsgOfCtb then
+if $rows != 1 then
-  return -1
+  sleep 1000
-endi
+  goto wait_consumer_end_from_ctb
-$loop_cnt = $loop_cnt + 1
+endi
-goto loop_consume_diff_topic_from_ctb
+if $data[0][1] != $consumerId then
-loop_consume_diff_topic_from_ctb_end:
+  return -1
-
+endi
-print ================ test consume from ntb
+if $data[0][2] != $totalMsgOfCtb then
-$loop_cnt = 0
+  return -1
-loop_consume_diff_topic_from_ntb:
+endi
-
+if $data[0][3] != $totalMsgOfCtb then
-#######################################################################################
+  return -1
-# clear consume info and consume result 
+endi
-#run tsim/tmq/clearConsume.sim
+$loop_cnt = $loop_cnt + 1
-# because drop table function no stable, so by create new db for consume info and result. Modify it later
+goto loop_consume_diff_topic_from_ctb
-$cdb_index = $cdb_index + 1
+loop_consume_diff_topic_from_ctb_end:
-$cdbName = cdb . $cdb_index
+
-sql create database $cdbName vgroups 1
+print ================ test consume from ntb
-sleep 500
+$loop_cnt = 0
-sql use $cdbName
+loop_consume_diff_topic_from_ntb:
-
+
-print == create consume info table and consume result table
+#######################################################################################
-sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+# clear consume info and consume result 
-sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+#run tsim/tmq/clearConsume.sim
-
+# because drop table function no stable, so by create new db for consume info and result. Modify it later
-sql show tables
+$cdb_index = $cdb_index + 1
-if $rows != 2 then 
+$cdbName = cdb . $cdb_index
-  return -1
+sql create database $cdbName vgroups 1
-endi
+sleep 500
-#######################################################################################
+sql use $cdbName
-
+
-if $loop_cnt == 0 then
+print == create consume info table and consume result table
-  print == scenario 1: topic_ntb_column
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
-  $topicList = ' . topic_ntb_column
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
-  $topicList = $topicList . '
+
-elif $loop_cnt == 1 then
+sql show tables
-  print == scenario 2: topic_ntb_all
+if $rows != 2 then 
-  $topicList = ' . topic_ntb_all
+  return -1
-  $topicList = $topicList . '
+endi
-elif $loop_cnt == 2 then
+#######################################################################################
-  print == scenario 3: topic_ntb_function
+
-  $topicList = ' . topic_ntb_function
+if $loop_cnt == 0 then
-  $topicList = $topicList . '
+  print == scenario 1: topic_ntb_column
-else
+  $topicList = ' . topic_ntb_column
-  goto loop_consume_diff_topic_from_ntb_end
+  $topicList = $topicList . '
-endi
+elif $loop_cnt == 1 then
-
+  print == scenario 2: topic_ntb_all
-$consumerId    = 0
+  $topicList = ' . topic_ntb_all
-$totalMsgOfNtb = $rowsPerCtb
+  $topicList = $topicList . '
-$expectmsgcnt  = $totalMsgOfNtb
+elif $loop_cnt == 2 then
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+  print == scenario 3: topic_ntb_function
-
+  $topicList = ' . topic_ntb_function
-print == start consumer to pull msgs from ntb
+  $topicList = $topicList . '
-print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
+else
-system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+  goto loop_consume_diff_topic_from_ntb_end
-
+endi
-print == check consume result from ntb
+
-wait_consumer_end_from_ntb:
+$consumerId    = 0
-sql select * from consumeresult
+$totalMsgOfNtb = $rowsPerCtb
-print ==> rows: $rows 
+$expectmsgcnt  = $totalMsgOfNtb
-print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-if $rows != 1 then
+
-  sleep 1000
+print == start consumer to pull msgs from ntb
-  goto wait_consumer_end_from_ntb
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
-endi
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
-if $data[0][1] != $consumerId then
+
-  return -1
+print == check consume result from ntb
-endi
+wait_consumer_end_from_ntb:
-if $data[0][2] != $totalMsgOfNtb then
+sql select * from consumeresult
-  return -1
+print ==> rows: $rows 
-endi
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
-if $data[0][3] != $totalMsgOfNtb then
+if $rows != 1 then
-  return -1
+  sleep 1000
-endi
+  goto wait_consumer_end_from_ntb
-$loop_cnt = $loop_cnt + 1
+endi
-goto loop_consume_diff_topic_from_ntb
+if $data[0][1] != $consumerId then
-loop_consume_diff_topic_from_ntb_end:
+  return -1
-
+endi
-#------ not need stop consumer, because it exit after pull msg overthan expect msg
+if $data[0][2] != $totalMsgOfNtb then
-#system tsim/tmq/consume.sh -s stop -x SIGINT
+  return -1
-
+endi
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
+if $data[0][3] != $totalMsgOfNtb then
+  return -1
+endi
+$loop_cnt = $loop_cnt + 1
+goto loop_consume_diff_topic_from_ntb
+loop_consume_diff_topic_from_ntb_end:
+
+#------ not need stop consumer, because it exit after pull msg overthan expect msg
+#system tsim/tmq/consume.sh -s stop -x SIGINT
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/tmq/basic1Of2Cons.sim

@@ -1,372 +1,382 @@
-#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
+#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
-#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
+#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
-#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
+#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
-#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
+#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
-#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
+#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
-
+
-# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
+# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
-# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
+# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
-#
+#
-# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
+# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
-#
+#
-
+
-run tsim/tmq/prepareBasicEnv-1vgrp.sim
+run tsim/tmq/prepareBasicEnv-1vgrp.sim
-
+
-#---- global parameters start ----#
+#---- global parameters start ----#
-$dbName    = db
+$dbName    = db
-$vgroups    = 1
+$vgroups    = 1
-$stbPrefix  = stb
+$stbPrefix  = stb
-$ctbPrefix  = ctb
+$ctbPrefix  = ctb
-$ntbPrefix  = ntb
+$ntbPrefix  = ntb
-$stbNum     = 1
+$stbNum     = 1
-$ctbNum     = 10
+$ctbNum     = 10
-$ntbNum     = 10
+$ntbNum     = 10
-$rowsPerCtb = 10
+$rowsPerCtb = 10
-$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
+$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
-#---- global parameters end ----#
+#---- global parameters end ----#
-
+
-$pullDelay    = 5
+$pullDelay    = 5
-$ifcheckdata  = 1
+$ifcheckdata  = 1
-$showMsg      = 1
+$ifmanualcommit = 1
-$showRow      = 0
+$showMsg      = 1
-
+$showRow      = 0
-sql connect
+
-sql use $dbName
+sql connect
-
+sql use $dbName
-print == create topics from super table
+
-sql create topic topic_stb_column as select ts, c3 from stb
+print == create topics from super table
-sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
+sql create topic topic_stb_column as select ts, c3 from stb
-sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
+sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
-
+sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
-print == create topics from child table
+
-sql create topic topic_ctb_column as select ts, c3 from ctb0
+print == create topics from child table
-sql create topic topic_ctb_all as select * from ctb0
+sql create topic topic_ctb_column as select ts, c3 from ctb0
-sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
+sql create topic topic_ctb_all as select * from ctb0
-
+sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
-print == create topics from normal table
+
-sql create topic topic_ntb_column as select ts, c3 from ntb0
+print == create topics from normal table
-sql create topic topic_ntb_all as select * from ntb0
+sql create topic topic_ntb_column as select ts, c3 from ntb0
-sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
+sql create topic topic_ntb_all as select * from ntb0
-
+sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
-#sql show topics
+
-#if $rows != 9 then 
+#sql show topics
-#  return -1
+#if $rows != 9 then 
-#endi
+#  return -1
-
+#endi
-$keyList = ' . group.id:cgrp1
+
-$keyList = $keyList . '
+#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest'
-
+$keyList = ' . group.id:cgrp1
-$cdb_index = 0
+$keyList = $keyList . ,
-#=============================== start consume =============================#
+$keyList = $keyList . enable.auto.commit:false
-
+#$keyList = $keyList . ,
-print ================ test consume from stb
+#$keyList = $keyList . auto.commit.interval.ms:6000
-$loop_cnt = 0
+#$keyList = $keyList . ,
-loop_consume_diff_topic_from_stb:
+#$keyList = $keyList . auto.offset.reset:earliest
-
+$keyList = $keyList . '
-#######################################################################################
+print ========== key list:  $keyList
-# clear consume info and consume result 
+
-#run tsim/tmq/clearConsume.sim
+$cdb_index = 0
-# because drop table function no stable, so by create new db for consume info and result. Modify it later
+
-$cdb_index = $cdb_index + 1
+#=============================== start consume =============================#
-$cdbName = cdb . $cdb_index
+
-sql create database $cdbName vgroups 1
+print ================ test consume from stb
-sleep 500
+$loop_cnt = 0
-sql use $cdbName
+loop_consume_diff_topic_from_stb:
-
+
-print == create consume info table and consume result table for stb
+#######################################################################################
-sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+# clear consume info and consume result 
-sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+#run tsim/tmq/clearConsume.sim
-
+# because drop table function no stable, so by create new db for consume info and result. Modify it later
-sql show tables
+$cdb_index = $cdb_index + 1
-if $rows != 2 then 
+$cdbName = cdb . $cdb_index
-  return -1
+sql create database $cdbName vgroups 1
-endi
+sleep 500
-#######################################################################################
+sql use $cdbName
-
+
-if $loop_cnt == 0 then
+print == create consume info table and consume result table for stb
-  print == scenario 1: topic_stb_column
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
-  $topicList = ' . topic_stb_column
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
-  $topicList = $topicList . '
+
-elif $loop_cnt == 1 then
+sql show tables
-  print == scenario 2: topic_stb_all
+if $rows != 2 then 
-  $topicList = ' . topic_stb_all
+  return -1
-  $topicList = $topicList . '
+endi
-elif $loop_cnt == 2 then
+#######################################################################################
-  print == scenario 3: topic_stb_function
+
-  $topicList = ' . topic_stb_function
+if $loop_cnt == 0 then
-  $topicList = $topicList . '
+  print == scenario 1: topic_stb_column
-else
+  $topicList = ' . topic_stb_column
-  goto loop_consume_diff_topic_from_stb_end
+  $topicList = $topicList . '
-endi
+elif $loop_cnt == 1 then
-
+  print == scenario 2: topic_stb_all
-$consumerId    = 0
+  $topicList = ' . topic_stb_all
-$totalMsgOfStb = $ctbNum * $rowsPerCtb
+  $topicList = $topicList . '
-$expectmsgcnt  = $totalMsgOfStb
+elif $loop_cnt == 2 then
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+  print == scenario 3: topic_stb_function
-
+  $topicList = ' . topic_stb_function
-$consumerId    = 1
+  $topicList = $topicList . '
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+else
-
+  goto loop_consume_diff_topic_from_stb_end
-print == start consumer to pull msgs from stb
+endi
-print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+
-system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+$consumerId    = 0
-
+$totalMsgOfStb = $ctbNum * $rowsPerCtb
-print == check consume result
+$expectmsgcnt  = $totalMsgOfStb
-wait_consumer_end_from_stb:
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-sql select * from consumeresult
+
-print ==> rows: $rows 
+$consumerId    = 1
-print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
+
-if $rows != 2 then
+print == start consumer to pull msgs from stb
-  sleep 1000
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
-  goto wait_consumer_end_from_stb
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
-endi
+
-if $data[0][1] == 0 then
+print == check consume result
-  if $data[1][1] != 1 then
+wait_consumer_end_from_stb:
-    return -1
+sql select * from consumeresult
-  endi
+print ==> rows: $rows 
-endi
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
-if $data[0][1] == 1 then
+print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
-  if $data[1][1] != 0 then
+if $rows != 2 then
-    return -1
+  sleep 1000
-  endi
+  goto wait_consumer_end_from_stb
-endi
+endi
-
+if $data[0][1] == 0 then
-# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0
+  if $data[1][1] != 1 then
-# or     $data[0][2] == 0              and $data[1][2] == $totalMsgOfStb
+    return -1
-if $data[0][2] == $totalMsgOfStb then
+  endi
-  if $data[1][2] == 0 then
+endi
-    goto check_ok_0
+if $data[0][1] == 1 then
-  endi
+  if $data[1][1] != 0 then
-elif $data[0][2] == 0 then
+    return -1
-  if $data[1][2] == $totalMsgOfStb then
+  endi
-    goto check_ok_0
+endi
-  endi
+
-endi
+# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0
-return -1 
+# or     $data[0][2] == 0              and $data[1][2] == $totalMsgOfStb
-check_ok_0:
+if $data[0][2] == $totalMsgOfStb then
-
+  if $data[1][2] == 0 then
-if $data[0][3] == $totalMsgOfStb then
+    goto check_ok_0
-  if $data[1][3] == 0 then
+  endi
-    goto check_ok_1
+elif $data[0][2] == 0 then
-  endi
+  if $data[1][2] == $totalMsgOfStb then
-elif $data[0][3] == 0 then
+    goto check_ok_0
-  if $data[1][3] == $totalMsgOfStb then
+  endi
-    goto check_ok_1
+endi
-  endi
+return -1 
-endi
+check_ok_0:
-return -1 
+
-check_ok_1:
+if $data[0][3] == $totalMsgOfStb then
-
+  if $data[1][3] == 0 then
-$loop_cnt = $loop_cnt + 1
+    goto check_ok_1
-goto loop_consume_diff_topic_from_stb
+  endi
-loop_consume_diff_topic_from_stb_end:
+elif $data[0][3] == 0 then
-
+  if $data[1][3] == $totalMsgOfStb then
-print ================ test consume from ctb
+    goto check_ok_1
-$loop_cnt = 0
+  endi
-loop_consume_diff_topic_from_ctb:
+endi
-
+return -1 
-#######################################################################################
+check_ok_1:
-# clear consume info and consume result 
+
-#run tsim/tmq/clearConsume.sim
+$loop_cnt = $loop_cnt + 1
-# because drop table function no stable, so by create new db for consume info and result. Modify it later
+goto loop_consume_diff_topic_from_stb
-$cdb_index = $cdb_index + 1
+loop_consume_diff_topic_from_stb_end:
-$cdbName = cdb . $cdb_index
+
-sql create database $cdbName vgroups 1
+print ================ test consume from ctb
-sleep 500
+$loop_cnt = 0
-sql use $cdbName
+loop_consume_diff_topic_from_ctb:
-
+
-print == create consume info table and consume result table for ctb
+#######################################################################################
-sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+# clear consume info and consume result 
-sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+#run tsim/tmq/clearConsume.sim
-
+# because drop table function no stable, so by create new db for consume info and result. Modify it later
-sql show tables
+$cdb_index = $cdb_index + 1
-if $rows != 2 then 
+$cdbName = cdb . $cdb_index
-  return -1
+sql create database $cdbName vgroups 1
-endi
+sleep 500
-#######################################################################################
+sql use $cdbName
-
+
-if $loop_cnt == 0 then
+print == create consume info table and consume result table for ctb
-  print == scenario 1: topic_ctb_column
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
-  $topicList = ' . topic_ctb_column
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
-  $topicList = $topicList . '
+
-elif $loop_cnt == 1 then
+sql show tables
-  print == scenario 2: topic_ctb_all
+if $rows != 2 then 
-  $topicList = ' . topic_ctb_all
+  return -1
-  $topicList = $topicList . '
+endi
-elif $loop_cnt == 2 then
+#######################################################################################
-  print == scenario 3: topic_ctb_function
+
-  $topicList = ' . topic_ctb_function
+if $loop_cnt == 0 then
-  $topicList = $topicList . '
+  print == scenario 1: topic_ctb_column
-else
+  $topicList = ' . topic_ctb_column
-  goto loop_consume_diff_topic_from_ctb_end
+  $topicList = $topicList . '
-endi
+elif $loop_cnt == 1 then
-
+  print == scenario 2: topic_ctb_all
-$consumerId    = 0
+  $topicList = ' . topic_ctb_all
-$totalMsgOfCtb = $rowsPerCtb
+  $topicList = $topicList . '
-$expectmsgcnt  = $totalMsgOfCtb
+elif $loop_cnt == 2 then
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+  print == scenario 3: topic_ctb_function
-$consumerId    = 1
+  $topicList = ' . topic_ctb_function
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+  $topicList = $topicList . '
-
+else
-print == start consumer to pull msgs from ctb
+  goto loop_consume_diff_topic_from_ctb_end
-print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
+endi
-system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+
-
+$consumerId    = 0
-print == check consume result
+$totalMsgOfCtb = $rowsPerCtb
-wait_consumer_end_from_ctb:
+$expectmsgcnt  = $totalMsgOfCtb
-sql select * from consumeresult
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-print ==> rows: $rows 
+$consumerId    = 1
-print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
+
-if $rows != 2 then
+print == start consumer to pull msgs from ctb
-  sleep 1000
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
-  goto wait_consumer_end_from_ctb
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
-endi
+
-if $data[0][1] == 0 then
+print == check consume result
-  if $data[1][1] != 1 then
+wait_consumer_end_from_ctb:
-    return -1
+sql select * from consumeresult
-  endi
+print ==> rows: $rows 
-endi
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
-if $data[0][1] == 1 then
+print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
-  if $data[1][1] != 0 then
+if $rows != 2 then
-    return -1
+  sleep 1000
-  endi
+  goto wait_consumer_end_from_ctb
-endi
+endi
-
+if $data[0][1] == 0 then
-# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0
+  if $data[1][1] != 1 then
-# or     $data[0][2] == 0              and $data[1][2] == $totalMsgOfCtb
+    return -1
-if $data[0][2] == $totalMsgOfCtb then
+  endi
-  if $data[1][2] == 0 then
+endi
-    goto check_ok_2
+if $data[0][1] == 1 then
-  endi
+  if $data[1][1] != 0 then
-elif $data[0][2] == 0 then
+    return -1
-  if $data[1][2] == $totalMsgOfCtb then
+  endi
-    goto check_ok_2
+endi
-  endi
+
-endi
+# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0
-return -1 
+# or     $data[0][2] == 0              and $data[1][2] == $totalMsgOfCtb
-check_ok_2:
+if $data[0][2] == $totalMsgOfCtb then
-
+  if $data[1][2] == 0 then
-if $data[0][3] == $totalMsgOfCtb then
+    goto check_ok_2
-  if $data[1][3] == 0 then
+  endi
-    goto check_ok_3
+elif $data[0][2] == 0 then
-  endi
+  if $data[1][2] == $totalMsgOfCtb then
-elif $data[0][3] == 0 then
+    goto check_ok_2
-  if $data[1][3] == $totalMsgOfCtb then
+  endi
-    goto check_ok_3
+endi
-  endi
+return -1 
-endi
+check_ok_2:
-return -1 
+
-check_ok_3:
+if $data[0][3] == $totalMsgOfCtb then
-
+  if $data[1][3] == 0 then
-$loop_cnt = $loop_cnt + 1
+    goto check_ok_3
-goto loop_consume_diff_topic_from_ctb
+  endi
-loop_consume_diff_topic_from_ctb_end:
+elif $data[0][3] == 0 then
-
+  if $data[1][3] == $totalMsgOfCtb then
-print ================ test consume from ntb
+    goto check_ok_3
-$loop_cnt = 0
+  endi
-loop_consume_diff_topic_from_ntb:
+endi
-
+return -1 
-#######################################################################################
+check_ok_3:
-# clear consume info and consume result 
+
-#run tsim/tmq/clearConsume.sim
+$loop_cnt = $loop_cnt + 1
-# because drop table function no stable, so by create new db for consume info and result. Modify it later
+goto loop_consume_diff_topic_from_ctb
-$cdb_index = $cdb_index + 1
+loop_consume_diff_topic_from_ctb_end:
-$cdbName = cdb . $cdb_index
+
-sql create database $cdbName vgroups 1
+print ================ test consume from ntb
-sleep 500
+$loop_cnt = 0
-sql use $cdbName
+loop_consume_diff_topic_from_ntb:
-
+
-print == create consume info table and consume result table for ntb
+#######################################################################################
-sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+# clear consume info and consume result 
-sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+#run tsim/tmq/clearConsume.sim
-
+# because drop table function no stable, so by create new db for consume info and result. Modify it later
-sql show tables
+$cdb_index = $cdb_index + 1
-if $rows != 2 then 
+$cdbName = cdb . $cdb_index
-  return -1
+sql create database $cdbName vgroups 1
-endi
+sleep 500
-#######################################################################################
+sql use $cdbName
-
+
-if $loop_cnt == 0 then
+print == create consume info table and consume result table for ntb
-  print == scenario 1: topic_ntb_column
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
-  $topicList = ' . topic_ntb_column
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
-  $topicList = $topicList . '
+
-elif $loop_cnt == 1 then
+sql show tables
-  print == scenario 2: topic_ntb_all
+if $rows != 2 then 
-  $topicList = ' . topic_ntb_all
+  return -1
-  $topicList = $topicList . '
+endi
-elif $loop_cnt == 2 then
+#######################################################################################
-  print == scenario 3: topic_ntb_function
+
-  $topicList = ' . topic_ntb_function
+if $loop_cnt == 0 then
-  $topicList = $topicList . '
+  print == scenario 1: topic_ntb_column
-else
+  $topicList = ' . topic_ntb_column
-  goto loop_consume_diff_topic_from_ntb_end
+  $topicList = $topicList . '
-endi
+elif $loop_cnt == 1 then
-
+  print == scenario 2: topic_ntb_all
-$consumerId    = 0
+  $topicList = ' . topic_ntb_all
-$totalMsgOfNtb = $rowsPerCtb
+  $topicList = $topicList . '
-$expectmsgcnt  = $totalMsgOfNtb
+elif $loop_cnt == 2 then
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+  print == scenario 3: topic_ntb_function
-$consumerId    = 1
+  $topicList = ' . topic_ntb_function
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+  $topicList = $topicList . '
-
+else
-print == start consumer to pull msgs from ntb
+  goto loop_consume_diff_topic_from_ntb_end
-print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
+endi
-system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+
-
+$consumerId    = 0
-print == check consume result from ntb
+$totalMsgOfNtb = $rowsPerCtb
-wait_consumer_end_from_ntb:
+$expectmsgcnt  = $totalMsgOfNtb
-sql select * from consumeresult
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-print ==> rows: $rows 
+$consumerId    = 1
-print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
+
-if $rows != 2 then
+print == start consumer to pull msgs from ntb
-  sleep 1000
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
-  goto wait_consumer_end_from_ntb
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
-endi
+
-if $data[0][1] == 0 then
+print == check consume result from ntb
-  if $data[1][1] != 1 then
+wait_consumer_end_from_ntb:
-    return -1
+sql select * from consumeresult
-  endi
+print ==> rows: $rows 
-endi
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
-if $data[1][1] == 0 then
+print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
-  if $data[0][1] != 1 then
+if $rows != 2 then
-    return -1
+  sleep 1000
-  endi
+  goto wait_consumer_end_from_ntb
-endi
+endi
-
+if $data[0][1] == 0 then
-# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0
+  if $data[1][1] != 1 then
-# or     $data[0][2] == 0              and $data[1][2] == $totalMsgOfNtb
+    return -1
-if $data[0][2] == $totalMsgOfNtb then
+  endi
-  if $data[1][2] == 0 then
+endi
-    goto check_ok_4
+if $data[1][1] == 0 then
-  endi
+  if $data[0][1] != 1 then
-elif $data[0][2] == 0 then
+    return -1
-  if $data[1][2] == $totalMsgOfNtb then
+  endi
-    goto check_ok_4
+endi
-  endi
+
-endi
+# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0
-return -1 
+# or     $data[0][2] == 0              and $data[1][2] == $totalMsgOfNtb
-check_ok_4:
+if $data[0][2] == $totalMsgOfNtb then
-
+  if $data[1][2] == 0 then
-if $data[0][3] == $totalMsgOfNtb then
+    goto check_ok_4
-  if $data[1][3] == 0 then
+  endi
-    goto check_ok_5
+elif $data[0][2] == 0 then
-  endi
+  if $data[1][2] == $totalMsgOfNtb then
-elif $data[0][3] == 0 then
+    goto check_ok_4
-  if $data[1][3] == $totalMsgOfNtb then
+  endi
-    goto check_ok_5
+endi
-  endi
+return -1 
-endi
+check_ok_4:
-return -1 
+
-check_ok_5:
+if $data[0][3] == $totalMsgOfNtb then
-
+  if $data[1][3] == 0 then
-$loop_cnt = $loop_cnt + 1
+    goto check_ok_5
-goto loop_consume_diff_topic_from_ntb
+  endi
-loop_consume_diff_topic_from_ntb_end:
+elif $data[0][3] == 0 then
-
+  if $data[1][3] == $totalMsgOfNtb then
-#------ not need stop consumer, because it exit after pull msg overthan expect msg
+    goto check_ok_5
-#system tsim/tmq/consume.sh -s stop -x SIGINT
+  endi
-
+endi
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
+return -1 
+check_ok_5:
+
+$loop_cnt = $loop_cnt + 1
+goto loop_consume_diff_topic_from_ntb
+loop_consume_diff_topic_from_ntb_end:
+
+#------ not need stop consumer, because it exit after pull msg overthan expect msg
+#system tsim/tmq/consume.sh -s stop -x SIGINT
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/tmq/basic2.sim

@@ -1,219 +1,229 @@
-#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
+#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
-#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics
+#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics
-#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics
+#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics
-#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics
+#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics
-#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics
+#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics
-
+
-# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
+# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
-# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
+# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
-#
+#
-# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
+# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
-#
+#
-
+
-run tsim/tmq/prepareBasicEnv-1vgrp.sim
+run tsim/tmq/prepareBasicEnv-1vgrp.sim
-
+
-#---- global parameters start ----#
+#---- global parameters start ----#
-$dbName    = db
+$dbName    = db
-$vgroups    = 1
+$vgroups    = 1
-$stbPrefix  = stb
+$stbPrefix  = stb
-$ctbPrefix  = ctb
+$ctbPrefix  = ctb
-$ntbPrefix  = ntb
+$ntbPrefix  = ntb
-$stbNum     = 1
+$stbNum     = 1
-$ctbNum     = 10
+$ctbNum     = 10
-$ntbNum     = 10
+$ntbNum     = 10
-$rowsPerCtb = 10
+$rowsPerCtb = 10
-$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
+$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
-#---- global parameters end ----#
+#---- global parameters end ----#
-
+
-$pullDelay    = 3
+$pullDelay    = 3
-$ifcheckdata  = 1
+$ifcheckdata  = 1
-$showMsg      = 1
+$ifmanualcommit = 1
-$showRow      = 0
+$showMsg      = 1
-
+$showRow      = 0
-sql connect
+
-sql use $dbName
+sql connect
-
+sql use $dbName
-print == create topics from super table
+
-sql create topic topic_stb_column as select ts, c3 from stb
+print == create topics from super table
-sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
+sql create topic topic_stb_column as select ts, c3 from stb
-sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
+sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
-
+sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
-print == create topics from child table
+
-sql create topic topic_ctb_column as select ts, c3 from ctb0
+print == create topics from child table
-sql create topic topic_ctb_all as select * from ctb0
+sql create topic topic_ctb_column as select ts, c3 from ctb0
-sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
+sql create topic topic_ctb_all as select * from ctb0
-
+sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
-print == create topics from normal table
+
-sql create topic topic_ntb_column as select ts, c3 from ntb0
+print == create topics from normal table
-sql create topic topic_ntb_all as select * from ntb0
+sql create topic topic_ntb_column as select ts, c3 from ntb0
-sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
+sql create topic topic_ntb_all as select * from ntb0
-
+sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
-#sql show topics
+
-#if $rows != 9 then 
+#sql show topics
-#  return -1
+#if $rows != 9 then 
-#endi
+#  return -1
-
+#endi
-$keyList = ' . group.id:cgrp1
+
-$keyList = $keyList . '
+#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest'
-
+$keyList = ' . group.id:cgrp1
-$topicNum = 3
+$keyList = $keyList . ,
-
+$keyList = $keyList . enable.auto.commit:false
-#=============================== start consume =============================#
+#$keyList = $keyList . ,
-
+#$keyList = $keyList . auto.commit.interval.ms:6000
-
+#$keyList = $keyList . ,
-print ================ test consume from stb
+#$keyList = $keyList . auto.offset.reset:earliest
-print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function
+$keyList = $keyList . '
-$topicList = ' . topic_stb_column
+print ========== key list:  $keyList
-$topicList = $topicList . ,
+
-$topicList = $topicList . topic_stb_all
+
-$topicList = $topicList . ,
+$topicNum = 3
-$topicList = $topicList . topic_stb_function
+
-$topicList = $topicList . '
+#=============================== start consume =============================#
-
+
-$consumerId    = 0
+
-$totalMsgOfStb = $ctbNum * $rowsPerCtb
+print ================ test consume from stb
-$totalMsgOfStb = $totalMsgOfStb * $topicNum
+print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function
-$expectmsgcnt  = $totalMsgOfStb
+$topicList = ' . topic_stb_column
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+$topicList = $topicList . ,
-
+$topicList = $topicList . topic_stb_all
-print == start consumer to pull msgs from stb
+$topicList = $topicList . ,
-print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
+$topicList = $topicList . topic_stb_function
-system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
+$topicList = $topicList . '
-
+
-print == check consume result
+$consumerId    = 0
-wait_consumer_end_from_stb:
+$totalMsgOfStb = $ctbNum * $rowsPerCtb
-sql select * from consumeresult
+$totalMsgOfStb = $totalMsgOfStb * $topicNum
-print ==> rows: $rows 
+$expectmsgcnt  = $totalMsgOfStb
-print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-if $rows != 1 then
+
-  sleep 1000
+print == start consumer to pull msgs from stb
-  goto wait_consumer_end_from_stb
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
-endi
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
-if $data[0][1] != $consumerId then
+
-  return -1
+print == check consume result
-endi
+wait_consumer_end_from_stb:
-if $data[0][2] != $expectmsgcnt then
+sql select * from consumeresult
-  return -1
+print ==> rows: $rows 
-endi
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
-if $data[0][3] != $expectmsgcnt then
+if $rows != 1 then
-  return -1
+  sleep 1000
-endi
+  goto wait_consumer_end_from_stb
-
+endi
-#######################################################################################
+if $data[0][1] != $consumerId then
-# clear consume info and consume result 
+  return -1
-#run tsim/tmq/clearConsume.sim
+endi
-# because drop table function no stable, so by create new db for consume info and result. Modify it later
+if $data[0][2] != $expectmsgcnt then
-$cdbName = cdb1
+  return -1
-sql create database $cdbName vgroups 1
+endi
-sleep 500
+if $data[0][3] != $expectmsgcnt then
-sql use $cdbName
+  return -1
-
+endi
-print == create consume info table and consume result table
+
-sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+#######################################################################################
-sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+# clear consume info and consume result 
-
+#run tsim/tmq/clearConsume.sim
-sql show tables
+# because drop table function no stable, so by create new db for consume info and result. Modify it later
-if $rows != 2 then 
+$cdbName = cdb1
-  return -1
+sql create database $cdbName vgroups 1
-endi
+sleep 500
-#######################################################################################
+sql use $cdbName
-
+
-
+print == create consume info table and consume result table
-print ================ test consume from ctb
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
-print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
-$topicList = ' . topic_ctb_column
+
-$topicList = $topicList . ,
+sql show tables
-$topicList = $topicList . topic_ctb_all
+if $rows != 2 then 
-$topicList = $topicList . ,
+  return -1
-$topicList = $topicList . topic_ctb_function
+endi
-$topicList = $topicList . '
+#######################################################################################
-
+
-$consumerId    = 0
+
-$totalMsgOfCtb = $rowsPerCtb * $topicNum
+print ================ test consume from ctb
-$expectmsgcnt  = $totalMsgOfCtb
+print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+$topicList = ' . topic_ctb_column
-
+$topicList = $topicList . ,
-print == start consumer to pull msgs from ctb
+$topicList = $topicList . topic_ctb_all
-print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
+$topicList = $topicList . ,
-system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+$topicList = $topicList . topic_ctb_function
-
+$topicList = $topicList . '
-print == check consume result
+
-wait_consumer_end_from_ctb:
+$consumerId    = 0
-sql select * from consumeresult
+$totalMsgOfCtb = $rowsPerCtb * $topicNum
-print ==> rows: $rows 
+$expectmsgcnt  = $totalMsgOfCtb
-print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-if $rows != 1 then
+
-  sleep 1000
+print == start consumer to pull msgs from ctb
-  goto wait_consumer_end_from_ctb
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
-endi
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
-if $data[0][1] != $consumerId then
+
-  return -1
+print == check consume result
-endi
+wait_consumer_end_from_ctb:
-if $data[0][2] != $totalMsgOfCtb then
+sql select * from consumeresult
-  return -1
+print ==> rows: $rows 
-endi
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
-if $data[0][3] != $totalMsgOfCtb then
+if $rows != 1 then
-  return -1
+  sleep 1000
-endi
+  goto wait_consumer_end_from_ctb
-
+endi
-#######################################################################################
+if $data[0][1] != $consumerId then
-# clear consume info and consume result 
+  return -1
-#run tsim/tmq/clearConsume.sim
+endi
-# because drop table function no stable, so by create new db for consume info and result. Modify it later
+if $data[0][2] != $totalMsgOfCtb then
-$cdbName = cdb2
+  return -1
-sql create database $cdbName vgroups 1
+endi
-sleep 500
+if $data[0][3] != $totalMsgOfCtb then
-sql use $cdbName
+  return -1
-
+endi
-print == create consume info table and consume result table
+
-sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+#######################################################################################
-sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+# clear consume info and consume result 
-
+#run tsim/tmq/clearConsume.sim
-sql show tables
+# because drop table function no stable, so by create new db for consume info and result. Modify it later
-if $rows != 2 then 
+$cdbName = cdb2
-  return -1
+sql create database $cdbName vgroups 1
-endi
+sleep 500
-#######################################################################################
+sql use $cdbName
-
+
-
+print == create consume info table and consume result table
-print ================ test consume from ntb
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
-print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
-$topicList = ' . topic_ntb_column
+
-$topicList = $topicList . ,
+sql show tables
-$topicList = $topicList . topic_ntb_all
+if $rows != 2 then 
-$topicList = $topicList . ,
+  return -1
-$topicList = $topicList . topic_ntb_function
+endi
-$topicList = $topicList . '
+#######################################################################################
-
+
-$consumerId    = 0
+
-$totalMsgOfNtb = $rowsPerCtb * $topicNum
+print ================ test consume from ntb
-$expectmsgcnt  = $totalMsgOfNtb
+print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+$topicList = ' . topic_ntb_column
-
+$topicList = $topicList . ,
-print == start consumer to pull msgs from ntb
+$topicList = $topicList . topic_ntb_all
-print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
+$topicList = $topicList . ,
-system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+$topicList = $topicList . topic_ntb_function
-
+$topicList = $topicList . '
-print == check consume result from ntb
+
-wait_consumer_end_from_ntb:
+$consumerId    = 0
-sql select * from consumeresult
+$totalMsgOfNtb = $rowsPerCtb * $topicNum
-print ==> rows: $rows 
+$expectmsgcnt  = $totalMsgOfNtb
-print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-if $rows != 1 then
+
-  sleep 1000
+print == start consumer to pull msgs from ntb
-  goto wait_consumer_end_from_ntb
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
-endi
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
-if $data[0][1] != $consumerId then
+
-  return -1
+print == check consume result from ntb
-endi
+wait_consumer_end_from_ntb:
-if $data[0][2] != $totalMsgOfNtb then
+sql select * from consumeresult
-  return -1
+print ==> rows: $rows 
-endi
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
-if $data[0][3] != $totalMsgOfNtb then
+if $rows != 1 then
-  return -1
+  sleep 1000
-endi
+  goto wait_consumer_end_from_ntb
-
+endi
-#------ not need stop consumer, because it exit after pull msg overthan expect msg
+if $data[0][1] != $consumerId then
-#system tsim/tmq/consume.sh -s stop -x SIGINT
+  return -1
-
+endi
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
+if $data[0][2] != $totalMsgOfNtb then
+  return -1
+endi
+if $data[0][3] != $totalMsgOfNtb then
+  return -1
+endi
+
+#------ not need stop consumer, because it exit after pull msg overthan expect msg
+#system tsim/tmq/consume.sh -s stop -x SIGINT
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/tmq/basic2Of2Cons.sim

@@ -1,309 +1,318 @@
-#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
+#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
-#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
+#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
-#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
+#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
-#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
+#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
-#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
+#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
-
+
-# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
+# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
-# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
+# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
-#
+#
-# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
+# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
-#
+#
-
+
-run tsim/tmq/prepareBasicEnv-1vgrp.sim
+run tsim/tmq/prepareBasicEnv-1vgrp.sim
-
+
-#---- global parameters start ----#
+#---- global parameters start ----#
-$dbName    = db
+$dbName    = db
-$vgroups    = 1
+$vgroups    = 1
-$stbPrefix  = stb
+$stbPrefix  = stb
-$ctbPrefix  = ctb
+$ctbPrefix  = ctb
-$ntbPrefix  = ntb
+$ntbPrefix  = ntb
-$stbNum     = 1
+$stbNum     = 1
-$ctbNum     = 10
+$ctbNum     = 10
-$ntbNum     = 10
+$ntbNum     = 10
-$rowsPerCtb = 10
+$rowsPerCtb = 10
-$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
+$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
-#---- global parameters end ----#
+#---- global parameters end ----#
-
+
-$pullDelay    = 5
+$pullDelay    = 5
-$ifcheckdata  = 1
+$ifcheckdata  = 1
-$showMsg      = 1
+$ifmanualcommit = 1
-$showRow      = 0
+$showMsg      = 1
-
+$showRow      = 0
-sql connect
+
-sql use $dbName
+sql connect
-
+sql use $dbName
-print == create topics from super table
+
-sql create topic topic_stb_column as select ts, c3 from stb
+print == create topics from super table
-sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
+sql create topic topic_stb_column as select ts, c3 from stb
-sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
+sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
-
+sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
-print == create topics from child table
+
-sql create topic topic_ctb_column as select ts, c3 from ctb0
+print == create topics from child table
-sql create topic topic_ctb_all as select * from ctb0
+sql create topic topic_ctb_column as select ts, c3 from ctb0
-sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
+sql create topic topic_ctb_all as select * from ctb0
-
+sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
-print == create topics from normal table
+
-sql create topic topic_ntb_column as select ts, c3 from ntb0
+print == create topics from normal table
-sql create topic topic_ntb_all as select * from ntb0
+sql create topic topic_ntb_column as select ts, c3 from ntb0
-sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
+sql create topic topic_ntb_all as select * from ntb0
-
+sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
-#sql show topics
+
-#if $rows != 9 then 
+#sql show topics
-#  return -1
+#if $rows != 9 then 
-#endi
+#  return -1
-
+#endi
-$keyList = ' . group.id:cgrp1
+
-$keyList = $keyList . '
+#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest'
-
+$keyList = ' . group.id:cgrp1
-$topicNum = 3
+$keyList = $keyList . ,
-
+$keyList = $keyList . enable.auto.commit:false
-#=============================== start consume =============================#
+#$keyList = $keyList . ,
-
+#$keyList = $keyList . auto.commit.interval.ms:6000
-
+#$keyList = $keyList . ,
-print ================ test consume from stb
+#$keyList = $keyList . auto.offset.reset:earliest
-print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function
+$keyList = $keyList . '
-$topicList = ' . topic_stb_column
+print ========== key list:  $keyList
-$topicList = $topicList . ,
+
-$topicList = $topicList . topic_stb_all
+$topicNum = 3
-$topicList = $topicList . ,
+
-$topicList = $topicList . topic_stb_function
+#=============================== start consume =============================#
-$topicList = $topicList . '
+
-
+
-$consumerId    = 0
+print ================ test consume from stb
-$totalMsgOfStb = $ctbNum * $rowsPerCtb
+print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function
-$totalMsgOfStb = $totalMsgOfStb * $topicNum
+$topicList = ' . topic_stb_column
-$expectmsgcnt  = $totalMsgOfStb
+$topicList = $topicList . ,
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+$topicList = $topicList . topic_stb_all
-$consumerId    = 1
+$topicList = $topicList . ,
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+$topicList = $topicList . topic_stb_function
-
+$topicList = $topicList . '
-print == start consumer to pull msgs from stb
+
-print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
+$consumerId    = 0
-system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
+$totalMsgOfStb = $ctbNum * $rowsPerCtb
-
+$totalMsgOfStb = $totalMsgOfStb * $topicNum
-print == check consume result
+$expectmsgcnt  = $totalMsgOfStb
-wait_consumer_end_from_stb:
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-sql select * from consumeresult
+$consumerId    = 1
-print ==> rows: $rows 
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+
-print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
+print == start consumer to pull msgs from stb
-if $rows != 2 then
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
-  sleep 1000
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
-  goto wait_consumer_end_from_stb
+
-endi
+print == check consume result
-if $data[0][1] == 0 then
+wait_consumer_end_from_stb:
-  if $data[1][1] != 1 then
+sql select * from consumeresult
-    return -1
+print ==> rows: $rows 
-  endi
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
-endi
+print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
-if $data[0][1] == 1 then
+if $rows != 2 then
-  if $data[1][1] != 0 then
+  sleep 1000
-    return -1
+  goto wait_consumer_end_from_stb
-  endi
+endi
-endi
+if $data[0][1] == 0 then
-
+  if $data[1][1] != 1 then
-# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0
+    return -1
-# or     $data[0][2] == 0              and $data[1][2] == $totalMsgOfStb
+  endi
-if $data[0][2] == $totalMsgOfStb then
+endi
-  if $data[1][2] == 0 then
+if $data[0][1] == 1 then
-    goto check_ok_0
+  if $data[1][1] != 0 then
-  endi
+    return -1
-elif $data[0][2] == 0 then
+  endi
-  if $data[1][2] == $totalMsgOfStb then
+endi
-    goto check_ok_0
+
-  endi
+# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0
-endi
+# or     $data[0][2] == 0              and $data[1][2] == $totalMsgOfStb
-return -1
+if $data[0][2] == $totalMsgOfStb then
-check_ok_0:
+  if $data[1][2] == 0 then
-
+    goto check_ok_0
-if $data[0][3] == $totalMsgOfStb then
+  endi
-  if $data[1][3] == 0 then
+elif $data[0][2] == 0 then
-    goto check_ok_1
+  if $data[1][2] == $totalMsgOfStb then
-  endi
+    goto check_ok_0
-elif $data[0][3] == 0 then
+  endi
-  if $data[1][3] == $totalMsgOfStb then
+endi
-    goto check_ok_1
+return -1
-  endi
+check_ok_0:
-endi
+
-return -1
+if $data[0][3] == $totalMsgOfStb then
-check_ok_1:
+  if $data[1][3] == 0 then
-
+    goto check_ok_1
-#######################################################################################
+  endi
-# clear consume info and consume result 
+elif $data[0][3] == 0 then
-#run tsim/tmq/clearConsume.sim
+  if $data[1][3] == $totalMsgOfStb then
-# because drop table function no stable, so by create new db for consume info and result. Modify it later
+    goto check_ok_1
-$cdbName = cdb1
+  endi
-sql create database $cdbName vgroups 1
+endi
-sleep 500
+return -1
-sql use $cdbName
+check_ok_1:
-
+
-print == create consume info table and consume result table for ctb
+#######################################################################################
-sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+# clear consume info and consume result 
-sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+#run tsim/tmq/clearConsume.sim
-
+# because drop table function no stable, so by create new db for consume info and result. Modify it later
-sql show tables
+$cdbName = cdb1
-if $rows != 2 then 
+sql create database $cdbName vgroups 1
-  return -1
+sleep 500
-endi
+sql use $cdbName
-#######################################################################################
+
-
+print == create consume info table and consume result table for ctb
-
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
-print ================ test consume from ctb
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
-print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function
+
-$topicList = ' . topic_ctb_column
+sql show tables
-$topicList = $topicList . ,
+if $rows != 2 then 
-$topicList = $topicList . topic_ctb_all
+  return -1
-$topicList = $topicList . ,
+endi
-$topicList = $topicList . topic_ctb_function
+#######################################################################################
-$topicList = $topicList . '
+
-
+
-$consumerId    = 0
+print ================ test consume from ctb
-$totalMsgOfCtb = $rowsPerCtb * $topicNum
+print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function
-$expectmsgcnt  = $totalMsgOfCtb
+$topicList = ' . topic_ctb_column
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+$topicList = $topicList . ,
-$consumerId    = 1
+$topicList = $topicList . topic_ctb_all
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+$topicList = $topicList . ,
-
+$topicList = $topicList . topic_ctb_function
-print == start consumer to pull msgs from ctb
+$topicList = $topicList . '
-print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
+
-system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+$consumerId    = 0
-
+$totalMsgOfCtb = $rowsPerCtb * $topicNum
-print == check consume result
+$expectmsgcnt  = $totalMsgOfCtb
-wait_consumer_end_from_ctb:
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-sql select * from consumeresult
+$consumerId    = 1
-print ==> rows: $rows 
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+
-print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
+print == start consumer to pull msgs from ctb
-if $rows != 2 then
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
-  sleep 1000
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
-  goto wait_consumer_end_from_ctb
+
-endi
+print == check consume result
-if $data[0][1] == 0 then
+wait_consumer_end_from_ctb:
-  if $data[1][1] != 1 then
+sql select * from consumeresult
-    return -1
+print ==> rows: $rows 
-  endi
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
-endi
+print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
-if $data[0][1] == 1 then
+if $rows != 2 then
-  if $data[1][1] != 0 then
+  sleep 1000
-    return -1
+  goto wait_consumer_end_from_ctb
-  endi
+endi
-endi
+if $data[0][1] == 0 then
-
+  if $data[1][1] != 1 then
-# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0
+    return -1
-# or     $data[0][2] == 0              and $data[1][2] == $totalMsgOfCtb
+  endi
-if $data[0][2] == $totalMsgOfCtb then
+endi
-  if $data[1][2] == 0 then
+if $data[0][1] == 1 then
-    goto check_ok_2
+  if $data[1][1] != 0 then
-  endi
+    return -1
-elif $data[0][2] == 0 then
+  endi
-  if $data[1][2] == $totalMsgOfCtb then
+endi
-    goto check_ok_2
+
-  endi
+# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0
-endi
+# or     $data[0][2] == 0              and $data[1][2] == $totalMsgOfCtb
-return -1
+if $data[0][2] == $totalMsgOfCtb then
-check_ok_2:
+  if $data[1][2] == 0 then
-
+    goto check_ok_2
-if $data[0][3] == $totalMsgOfCtb then
+  endi
-  if $data[1][3] == 0 then
+elif $data[0][2] == 0 then
-    goto check_ok_3
+  if $data[1][2] == $totalMsgOfCtb then
-  endi
+    goto check_ok_2
-elif $data[0][3] == 0 then
+  endi
-  if $data[1][3] == $totalMsgOfCtb then
+endi
-    goto check_ok_3
+return -1
-  endi
+check_ok_2:
-endi
+
-return -1
+if $data[0][3] == $totalMsgOfCtb then
-check_ok_3:
+  if $data[1][3] == 0 then
-
+    goto check_ok_3
-#######################################################################################
+  endi
-# clear consume info and consume result 
+elif $data[0][3] == 0 then
-#run tsim/tmq/clearConsume.sim
+  if $data[1][3] == $totalMsgOfCtb then
-# because drop table function no stable, so by create new db for consume info and result. Modify it later
+    goto check_ok_3
-$cdbName = cdb2
+  endi
-sql create database $cdbName vgroups 1
+endi
-sleep 500
+return -1
-sql use $cdbName
+check_ok_3:
-
+
-print == create consume info table and consume result table for ntb
+#######################################################################################
-sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+# clear consume info and consume result 
-sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+#run tsim/tmq/clearConsume.sim
-
+# because drop table function no stable, so by create new db for consume info and result. Modify it later
-sql show tables
+$cdbName = cdb2
-if $rows != 2 then 
+sql create database $cdbName vgroups 1
-  return -1
+sleep 500
-endi
+sql use $cdbName
-#######################################################################################
+
-
+print == create consume info table and consume result table for ntb
-
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
-print ================ test consume from ntb
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
-print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function
+
-$topicList = ' . topic_ntb_column
+sql show tables
-$topicList = $topicList . ,
+if $rows != 2 then 
-$topicList = $topicList . topic_ntb_all
+  return -1
-$topicList = $topicList . ,
+endi
-$topicList = $topicList . topic_ntb_function
+#######################################################################################
-$topicList = $topicList . '
+
-
+
-$consumerId    = 0
+print ================ test consume from ntb
-$totalMsgOfNtb = $rowsPerCtb * $topicNum
+print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function
-$expectmsgcnt  = $totalMsgOfNtb
+$topicList = ' . topic_ntb_column
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+$topicList = $topicList . ,
-$consumerId    = 1
+$topicList = $topicList . topic_ntb_all
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+$topicList = $topicList . ,
-
+$topicList = $topicList . topic_ntb_function
-print == start consumer to pull msgs from ntb
+$topicList = $topicList . '
-print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
+
-system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+$consumerId    = 0
-
+$totalMsgOfNtb = $rowsPerCtb * $topicNum
-print == check consume result from ntb
+$expectmsgcnt  = $totalMsgOfNtb
-wait_consumer_end_from_ntb:
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-sql select * from consumeresult
+$consumerId    = 1
-print ==> rows: $rows 
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+
-print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
+print == start consumer to pull msgs from ntb
-if $rows != 2 then
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
-  sleep 1000
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
-  goto wait_consumer_end_from_ntb
+
-endi
+print == check consume result from ntb
-if $data[0][1] == 0 then
+wait_consumer_end_from_ntb:
-  if $data[1][1] != 1 then
+sql select * from consumeresult
-    return -1
+print ==> rows: $rows 
-  endi
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
-endi
+print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
-if $data[1][1] == 0 then
+if $rows != 2 then
-  if $data[0][1] != 1 then
+  sleep 1000
-    return -1
+  goto wait_consumer_end_from_ntb
-  endi
+endi
-endi
+if $data[0][1] == 0 then
-
+  if $data[1][1] != 1 then
-# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0
+    return -1
-# or     $data[0][2] == 0              and $data[1][2] == $totalMsgOfNtb
+  endi
-if $data[0][2] == $totalMsgOfNtb then
+endi
-  if $data[1][2] == 0 then
+if $data[1][1] == 0 then
-    goto check_ok_4
+  if $data[0][1] != 1 then
-  endi
+    return -1
-elif $data[0][2] == 0 then
+  endi
-  if $data[1][2] == $totalMsgOfNtb then
+endi
-    goto check_ok_4
+
-  endi
+# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0
-endi
+# or     $data[0][2] == 0              and $data[1][2] == $totalMsgOfNtb
-return -1
+if $data[0][2] == $totalMsgOfNtb then
-check_ok_4:
+  if $data[1][2] == 0 then
-
+    goto check_ok_4
-if $data[0][3] == $totalMsgOfNtb then
+  endi
-  if $data[1][3] == 0 then
+elif $data[0][2] == 0 then
-    goto check_ok_5
+  if $data[1][2] == $totalMsgOfNtb then
-  endi
+    goto check_ok_4
-elif $data[0][3] == 0 then
+  endi
-  if $data[1][3] == $totalMsgOfNtb then
+endi
-    goto check_ok_5
+return -1
-  endi
+check_ok_4:
-endi
+
-return -1
+if $data[0][3] == $totalMsgOfNtb then
-check_ok_5:
+  if $data[1][3] == 0 then
-
+    goto check_ok_5
-#------ not need stop consumer, because it exit after pull msg overthan expect msg
+  endi
-#system tsim/tmq/consume.sh -s stop -x SIGINT
+elif $data[0][3] == 0 then
-
+  if $data[1][3] == $totalMsgOfNtb then
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
+    goto check_ok_5
+  endi
+endi
+return -1
+check_ok_5:
+
+#------ not need stop consumer, because it exit after pull msg overthan expect msg
+#system tsim/tmq/consume.sh -s stop -x SIGINT
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/tmq/basic2Of2ConsOverlap.sim

@@ -27,6 +27,7 @@ $tstart     = 1640966400000  # 2022-01-01 00:00:00.000
 
 $pullDelay    = 5
 $ifcheckdata  = 1
+$ifmanualcommit = 1
 $showMsg      = 1
 $showRow      = 0
 
@@ -53,8 +54,16 @@ sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
 #  return -1
 #endi
 
+#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest'
 $keyList = ' . group.id:cgrp1
+$keyList = $keyList . ,
+$keyList = $keyList . enable.auto.commit:false
+#$keyList = $keyList . ,
+#$keyList = $keyList . auto.commit.interval.ms:6000
+#$keyList = $keyList . ,
+#$keyList = $keyList . auto.offset.reset:earliest
 $keyList = $keyList . '
+print ========== key list:  $keyList
 
 $topicNum = 2
 
@@ -72,7 +81,7 @@ $consumerId    = 0
 $totalMsgOfOneTopic = $ctbNum * $rowsPerCtb
 $totalMsgOfStb = $totalMsgOfOneTopic * $topicNum
 $expectmsgcnt  = $totalMsgOfStb
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
 
 
 $topicList = ' . topic_stb_all
@@ -80,7 +89,7 @@ $topicList = $topicList . ,
 $topicList = $topicList . topic_stb_function
 $topicList = $topicList . '
 $consumerId    = 1
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
 
 print == start consumer to pull msgs from stb
 print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
@@ -158,7 +167,7 @@ sleep 500
 sql use $cdbName
 
 print == create consume info table and consume result table for ctb
-sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
 sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
 
 sql show tables
@@ -179,14 +188,14 @@ $consumerId    = 0
 $totalMsgOfOneTopic = $rowsPerCtb
 $totalMsgOfCtb = $totalMsgOfOneTopic * $topicNum
 $expectmsgcnt  = $totalMsgOfCtb
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
 
 $topicList = ' . topic_ctb_function
 $topicList = $topicList . ,
 $topicList = $topicList . topic_ctb_all
 $topicList = $topicList . '
 $consumerId    = 1
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
 
 print == start consumer to pull msgs from ctb
 print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
@@ -249,7 +258,7 @@ sleep 500
 sql use $cdbName
 
 print == create consume info table and consume result table for ntb
-sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
 sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
 
 sql show tables
@@ -270,7 +279,7 @@ $consumerId    = 0
 $totalMsgOfOneTopic = $rowsPerCtb
 $totalMsgOfNtb = $totalMsgOfOneTopic * $topicNum
 $expectmsgcnt  = $totalMsgOfNtb
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
 
 
 $topicList = ' . topic_ntb_function
@@ -278,7 +287,7 @@ $topicList = $topicList . ,
 $topicList = $topicList . topic_ntb_all
 $topicList = $topicList . '
 $consumerId    = 1
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
 
 print == start consumer to pull msgs from ntb
 print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start

tests/script/tsim/tmq/basic3.sim

@@ -1,278 +1,288 @@
-#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
+#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
-#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics
+#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics
-#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics
+#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics
-#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics
+#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics
-#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics
+#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics
-
+
-# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
+# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
-# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
+# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
-#
+#
-# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
+# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
-#
+#
-
+
-run tsim/tmq/prepareBasicEnv-4vgrp.sim
+run tsim/tmq/prepareBasicEnv-4vgrp.sim
-
+
-#---- global parameters start ----#
+#---- global parameters start ----#
-$dbName    = db
+$dbName    = db
-$vgroups    = 4
+$vgroups    = 4
-$stbPrefix  = stb
+$stbPrefix  = stb
-$ctbPrefix  = ctb
+$ctbPrefix  = ctb
-$ntbPrefix  = ntb
+$ntbPrefix  = ntb
-$stbNum     = 1
+$stbNum     = 1
-$ctbNum     = 10
+$ctbNum     = 10
-$ntbNum     = 10
+$ntbNum     = 10
-$rowsPerCtb = 10
+$rowsPerCtb = 10
-$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
+$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
-#---- global parameters end ----#
+#---- global parameters end ----#
-
+
-$pullDelay    = 3
+$pullDelay    = 3
-$ifcheckdata  = 1
+$ifcheckdata  = 1
-$showMsg      = 1
+$ifmanualcommit = 1
-$showRow      = 0
+$showMsg      = 1
-
+$showRow      = 0
-sql connect
+
-sql use $dbName
+sql connect
-
+sql use $dbName
-print == create topics from super table
+
-sql create topic topic_stb_column as select ts, c3 from stb
+print == create topics from super table
-sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
+sql create topic topic_stb_column as select ts, c3 from stb
-sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
+sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
-
+sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
-print == create topics from child table
+
-sql create topic topic_ctb_column as select ts, c3 from ctb0
+print == create topics from child table
-sql create topic topic_ctb_all as select * from ctb0
+sql create topic topic_ctb_column as select ts, c3 from ctb0
-sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
+sql create topic topic_ctb_all as select * from ctb0
-
+sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
-print == create topics from normal table
+
-sql create topic topic_ntb_column as select ts, c3 from ntb0
+print == create topics from normal table
-sql create topic topic_ntb_all as select * from ntb0
+sql create topic topic_ntb_column as select ts, c3 from ntb0
-sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
+sql create topic topic_ntb_all as select * from ntb0
-
+sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
-#sql show topics
+
-#if $rows != 9 then 
+#sql show topics
-#  return -1
+#if $rows != 9 then 
-#endi
+#  return -1
-
+#endi
-$keyList = ' . group.id:cgrp1
+
-$keyList = $keyList . '
+#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest'
-
+$keyList = ' . group.id:cgrp1
-$cdb_index = 0
+$keyList = $keyList . ,
-#=============================== start consume =============================#
+$keyList = $keyList . enable.auto.commit:false
-
+#$keyList = $keyList . ,
-print ================ test consume from stb
+#$keyList = $keyList . auto.commit.interval.ms:6000
-$loop_cnt = 0
+#$keyList = $keyList . ,
-loop_consume_diff_topic_from_stb:
+#$keyList = $keyList . auto.offset.reset:earliest
-
+$keyList = $keyList . '
-#######################################################################################
+print ========== key list:  $keyList
-# clear consume info and consume result 
+
-#run tsim/tmq/clearConsume.sim
+
-# because drop table function no stable, so by create new db for consume info and result. Modify it later
+$cdb_index = 0
-$cdb_index = $cdb_index + 1
+#=============================== start consume =============================#
-$cdbName = cdb . $cdb_index
+
-sql create database $cdbName vgroups 1
+print ================ test consume from stb
-sleep 500
+$loop_cnt = 0
-sql use $cdbName
+loop_consume_diff_topic_from_stb:
-
+
-print == create consume info table and consume result table
+#######################################################################################
-sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+# clear consume info and consume result 
-sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+#run tsim/tmq/clearConsume.sim
-
+# because drop table function no stable, so by create new db for consume info and result. Modify it later
-sql show tables
+$cdb_index = $cdb_index + 1
-if $rows != 2 then 
+$cdbName = cdb . $cdb_index
-  return -1
+sql create database $cdbName vgroups 1
-endi
+sleep 500
-#######################################################################################
+sql use $cdbName
-
+
-if $loop_cnt == 0 then
+print == create consume info table and consume result table
-  print == scenario 1: topic_stb_column
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
-  $topicList = ' . topic_stb_column
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
-  $topicList = $topicList . '
+
-elif $loop_cnt == 1 then
+sql show tables
-  print == scenario 2: topic_stb_all
+if $rows != 2 then 
-  $topicList = ' . topic_stb_all
+  return -1
-  $topicList = $topicList . '
+endi
-elif $loop_cnt == 2 then
+#######################################################################################
-  print == scenario 3: topic_stb_function
+
-  $topicList = ' . topic_stb_function
+if $loop_cnt == 0 then
-  $topicList = $topicList . '
+  print == scenario 1: topic_stb_column
-else
+  $topicList = ' . topic_stb_column
-  goto loop_consume_diff_topic_from_stb_end
+  $topicList = $topicList . '
-endi
+elif $loop_cnt == 1 then
-
+  print == scenario 2: topic_stb_all
-$consumerId    = 0
+  $topicList = ' . topic_stb_all
-$totalMsgOfStb = $ctbNum * $rowsPerCtb
+  $topicList = $topicList . '
-$expectmsgcnt  = $totalMsgOfStb
+elif $loop_cnt == 2 then
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+  print == scenario 3: topic_stb_function
-
+  $topicList = ' . topic_stb_function
-print == start consumer to pull msgs from stb
+  $topicList = $topicList . '
-print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+else
-system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+  goto loop_consume_diff_topic_from_stb_end
-
+endi
-print == check consume result
+
-wait_consumer_end_from_stb:
+$consumerId    = 0
-sql select * from consumeresult
+$totalMsgOfStb = $ctbNum * $rowsPerCtb
-print ==> rows: $rows 
+$expectmsgcnt  = $totalMsgOfStb
-print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-if $rows != 1 then
+
-  sleep 1000
+print == start consumer to pull msgs from stb
-  goto wait_consumer_end_from_stb
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
-endi
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
-if $data[0][1] != $consumerId then
+
-  return -1
+print == check consume result
-endi
+wait_consumer_end_from_stb:
-if $data[0][2] != $expectmsgcnt then
+sql select * from consumeresult
-  return -1
+print ==> rows: $rows 
-endi
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
-if $data[0][3] != $expectmsgcnt then
+if $rows != 1 then
-  return -1
+  sleep 1000
-endi
+  goto wait_consumer_end_from_stb
-$loop_cnt = $loop_cnt + 1
+endi
-goto loop_consume_diff_topic_from_stb
+if $data[0][1] != $consumerId then
-loop_consume_diff_topic_from_stb_end:
+  return -1
-
+endi
-print ================ test consume from ctb
+if $data[0][2] != $expectmsgcnt then
-$loop_cnt = 0
+  return -1
-loop_consume_diff_topic_from_ctb:
+endi
-
+if $data[0][3] != $expectmsgcnt then
-#######################################################################################
+  return -1
-# clear consume info and consume result 
+endi
-#run tsim/tmq/clearConsume.sim
+$loop_cnt = $loop_cnt + 1
-# because drop table function no stable, so by create new db for consume info and result. Modify it later
+goto loop_consume_diff_topic_from_stb
-$cdb_index = $cdb_index + 1
+loop_consume_diff_topic_from_stb_end:
-$cdbName = cdb . $cdb_index
+
-sql create database $cdbName vgroups 1
+print ================ test consume from ctb
-sleep 500
+$loop_cnt = 0
-sql use $cdbName
+loop_consume_diff_topic_from_ctb:
-
+
-print == create consume info table and consume result table
+#######################################################################################
-sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+# clear consume info and consume result 
-sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+#run tsim/tmq/clearConsume.sim
-
+# because drop table function no stable, so by create new db for consume info and result. Modify it later
-sql show tables
+$cdb_index = $cdb_index + 1
-if $rows != 2 then 
+$cdbName = cdb . $cdb_index
-  return -1
+sql create database $cdbName vgroups 1
-endi
+sleep 500
-#######################################################################################
+sql use $cdbName
-
+
-if $loop_cnt == 0 then
+print == create consume info table and consume result table
-  print == scenario 1: topic_ctb_column
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
-  $topicList = ' . topic_ctb_column
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
-  $topicList = $topicList . '
+
-elif $loop_cnt == 1 then
+sql show tables
-  print == scenario 2: topic_ctb_all
+if $rows != 2 then 
-  $topicList = ' . topic_ctb_all
+  return -1
-  $topicList = $topicList . '
+endi
-elif $loop_cnt == 2 then
+#######################################################################################
-  print == scenario 3: topic_ctb_function
+
-  $topicList = ' . topic_ctb_function
+if $loop_cnt == 0 then
-  $topicList = $topicList . '
+  print == scenario 1: topic_ctb_column
-else
+  $topicList = ' . topic_ctb_column
-  goto loop_consume_diff_topic_from_ctb_end
+  $topicList = $topicList . '
-endi
+elif $loop_cnt == 1 then
-
+  print == scenario 2: topic_ctb_all
-$consumerId    = 0
+  $topicList = ' . topic_ctb_all
-$totalMsgOfCtb = $rowsPerCtb
+  $topicList = $topicList . '
-$expectmsgcnt  = $totalMsgOfCtb
+elif $loop_cnt == 2 then
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+  print == scenario 3: topic_ctb_function
-
+  $topicList = ' . topic_ctb_function
-print == start consumer to pull msgs from ctb
+  $topicList = $topicList . '
-print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
+else
-system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+  goto loop_consume_diff_topic_from_ctb_end
-
+endi
-print == check consume result
+
-wait_consumer_end_from_ctb:
+$consumerId    = 0
-sql select * from consumeresult
+$totalMsgOfCtb = $rowsPerCtb
-print ==> rows: $rows 
+$expectmsgcnt  = $totalMsgOfCtb
-print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-if $rows != 1 then
+
-  sleep 1000
+print == start consumer to pull msgs from ctb
-  goto wait_consumer_end_from_ctb
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
-endi
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
-if $data[0][1] != $consumerId then
+
-  return -1
+print == check consume result
-endi
+wait_consumer_end_from_ctb:
-if $data[0][2] != $totalMsgOfCtb then
+sql select * from consumeresult
-  return -1
+print ==> rows: $rows 
-endi
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
-if $data[0][3] != $totalMsgOfCtb then
+if $rows != 1 then
-  return -1
+  sleep 1000
-endi
+  goto wait_consumer_end_from_ctb
-$loop_cnt = $loop_cnt + 1
+endi
-goto loop_consume_diff_topic_from_ctb
+if $data[0][1] != $consumerId then
-loop_consume_diff_topic_from_ctb_end:
+  return -1
-
+endi
-print ================ test consume from ntb
+if $data[0][2] != $totalMsgOfCtb then
-$loop_cnt = 0
+  return -1
-loop_consume_diff_topic_from_ntb:
+endi
-
+if $data[0][3] != $totalMsgOfCtb then
-#######################################################################################
+  return -1
-# clear consume info and consume result 
+endi
-#run tsim/tmq/clearConsume.sim
+$loop_cnt = $loop_cnt + 1
-# because drop table function no stable, so by create new db for consume info and result. Modify it later
+goto loop_consume_diff_topic_from_ctb
-$cdb_index = $cdb_index + 1
+loop_consume_diff_topic_from_ctb_end:
-$cdbName = cdb . $cdb_index
+
-sql create database $cdbName vgroups 1
+print ================ test consume from ntb
-sleep 500
+$loop_cnt = 0
-sql use $cdbName
+loop_consume_diff_topic_from_ntb:
-
+
-print == create consume info table and consume result table
+#######################################################################################
-sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+# clear consume info and consume result 
-sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+#run tsim/tmq/clearConsume.sim
-
+# because drop table function no stable, so by create new db for consume info and result. Modify it later
-sql show tables
+$cdb_index = $cdb_index + 1
-if $rows != 2 then 
+$cdbName = cdb . $cdb_index
-  return -1
+sql create database $cdbName vgroups 1
-endi
+sleep 500
-#######################################################################################
+sql use $cdbName
-
+
-if $loop_cnt == 0 then
+print == create consume info table and consume result table
-  print == scenario 1: topic_ntb_column
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
-  $topicList = ' . topic_ntb_column
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
-  $topicList = $topicList . '
+
-elif $loop_cnt == 1 then
+sql show tables
-  print == scenario 2: topic_ntb_all
+if $rows != 2 then 
-  $topicList = ' . topic_ntb_all
+  return -1
-  $topicList = $topicList . '
+endi
-elif $loop_cnt == 2 then
+#######################################################################################
-  print == scenario 3: topic_ntb_function
+
-  $topicList = ' . topic_ntb_function
+if $loop_cnt == 0 then
-  $topicList = $topicList . '
+  print == scenario 1: topic_ntb_column
-else
+  $topicList = ' . topic_ntb_column
-  goto loop_consume_diff_topic_from_ntb_end
+  $topicList = $topicList . '
-endi
+elif $loop_cnt == 1 then
-
+  print == scenario 2: topic_ntb_all
-$consumerId    = 0
+  $topicList = ' . topic_ntb_all
-$totalMsgOfNtb = $rowsPerCtb
+  $topicList = $topicList . '
-$expectmsgcnt  = $totalMsgOfNtb
+elif $loop_cnt == 2 then
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+  print == scenario 3: topic_ntb_function
-
+  $topicList = ' . topic_ntb_function
-print == start consumer to pull msgs from ntb
+  $topicList = $topicList . '
-print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
+else
-system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+  goto loop_consume_diff_topic_from_ntb_end
-
+endi
-print == check consume result from ntb
+
-wait_consumer_end_from_ntb:
+$consumerId    = 0
-sql select * from consumeresult
+$totalMsgOfNtb = $rowsPerCtb
-print ==> rows: $rows 
+$expectmsgcnt  = $totalMsgOfNtb
-print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-if $rows != 1 then
+
-  sleep 1000
+print == start consumer to pull msgs from ntb
-  goto wait_consumer_end_from_ntb
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
-endi
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
-if $data[0][1] != $consumerId then
+
-  return -1
+print == check consume result from ntb
-endi
+wait_consumer_end_from_ntb:
-if $data[0][2] != $totalMsgOfNtb then
+sql select * from consumeresult
-  return -1
+print ==> rows: $rows 
-endi
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
-if $data[0][3] != $totalMsgOfNtb then
+if $rows != 1 then
-  return -1
+  sleep 1000
-endi
+  goto wait_consumer_end_from_ntb
-$loop_cnt = $loop_cnt + 1
+endi
-goto loop_consume_diff_topic_from_ntb
+if $data[0][1] != $consumerId then
-loop_consume_diff_topic_from_ntb_end:
+  return -1
-
+endi
-#------ not need stop consumer, because it exit after pull msg overthan expect msg
+if $data[0][2] != $totalMsgOfNtb then
-#system tsim/tmq/consume.sh -s stop -x SIGINT
+  return -1
-
+endi
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
+if $data[0][3] != $totalMsgOfNtb then
+  return -1
+endi
+$loop_cnt = $loop_cnt + 1
+goto loop_consume_diff_topic_from_ntb
+loop_consume_diff_topic_from_ntb_end:
+
+#------ not need stop consumer, because it exit after pull msg overthan expect msg
+#system tsim/tmq/consume.sh -s stop -x SIGINT
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/tmq/basic3Of2Cons.sim

@@ -1,384 +1,393 @@
-#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
+#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
-#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
+#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
-#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
+#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
-#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
+#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
-#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
+#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
-
+
-# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
+# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
-# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
+# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
-#
+#
-# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
+# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
-#
+#
-
+
-run tsim/tmq/prepareBasicEnv-4vgrp.sim
+run tsim/tmq/prepareBasicEnv-4vgrp.sim
-
+
-#---- global parameters start ----#
+#---- global parameters start ----#
-$dbName    = db
+$dbName    = db
-$vgroups    = 4
+$vgroups    = 4
-$stbPrefix  = stb
+$stbPrefix  = stb
-$ctbPrefix  = ctb
+$ctbPrefix  = ctb
-$ntbPrefix  = ntb
+$ntbPrefix  = ntb
-$stbNum     = 1
+$stbNum     = 1
-$ctbNum     = 10
+$ctbNum     = 10
-$ntbNum     = 10
+$ntbNum     = 10
-$rowsPerCtb = 10
+$rowsPerCtb = 10
-$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
+$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
-#---- global parameters end ----#
+#---- global parameters end ----#
-
+
-$pullDelay    = 5
+$pullDelay    = 5
-$ifcheckdata  = 1
+$ifcheckdata  = 1
-$showMsg      = 1
+$ifmanualcommit = 1
-$showRow      = 0
+$showMsg      = 1
-
+$showRow      = 0
-sql connect
+
-sql use $dbName
+sql connect
-
+sql use $dbName
-print == create topics from super table
+
-sql create topic topic_stb_column as select ts, c3 from stb
+print == create topics from super table
-sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
+sql create topic topic_stb_column as select ts, c3 from stb
-sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
+sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
-
+sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
-print == create topics from child table
+
-sql create topic topic_ctb_column as select ts, c3 from ctb0
+print == create topics from child table
-sql create topic topic_ctb_all as select * from ctb0
+sql create topic topic_ctb_column as select ts, c3 from ctb0
-sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
+sql create topic topic_ctb_all as select * from ctb0
-
+sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
-print == create topics from normal table
+
-sql create topic topic_ntb_column as select ts, c3 from ntb0
+print == create topics from normal table
-sql create topic topic_ntb_all as select * from ntb0
+sql create topic topic_ntb_column as select ts, c3 from ntb0
-sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
+sql create topic topic_ntb_all as select * from ntb0
-
+sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
-#sql show topics
+
-#if $rows != 9 then 
+#sql show topics
-#  return -1
+#if $rows != 9 then 
-#endi
+#  return -1
-
+#endi
-$keyList = ' . group.id:cgrp1
+
-$keyList = $keyList . '
+#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest'
-
+$keyList = ' . group.id:cgrp1
-$cdb_index = 0
+$keyList = $keyList . ,
-#=============================== start consume =============================#
+$keyList = $keyList . enable.auto.commit:false
-
+#$keyList = $keyList . ,
-print ================ test consume from stb
+#$keyList = $keyList . auto.commit.interval.ms:6000
-$loop_cnt = 0
+#$keyList = $keyList . ,
-loop_consume_diff_topic_from_stb:
+#$keyList = $keyList . auto.offset.reset:earliest
-
+$keyList = $keyList . '
-#######################################################################################
+print ========== key list:  $keyList
-# clear consume info and consume result 
+
-#run tsim/tmq/clearConsume.sim
+$cdb_index = 0
-# because drop table function no stable, so by create new db for consume info and result. Modify it later
+#=============================== start consume =============================#
-$cdb_index = $cdb_index + 1
+
-$cdbName = cdb . $cdb_index
+print ================ test consume from stb
-sql create database $cdbName vgroups 1
+$loop_cnt = 0
-sleep 500
+loop_consume_diff_topic_from_stb:
-sql use $cdbName
+
-
+#######################################################################################
-print == create consume info table and consume result table
+# clear consume info and consume result 
-sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+#run tsim/tmq/clearConsume.sim
-sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+# because drop table function no stable, so by create new db for consume info and result. Modify it later
-
+$cdb_index = $cdb_index + 1
-sql show tables
+$cdbName = cdb . $cdb_index
-if $rows != 2 then 
+sql create database $cdbName vgroups 1
-  return -1
+sleep 500
-endi
+sql use $cdbName
-#######################################################################################
+
-
+print == create consume info table and consume result table
-if $loop_cnt == 0 then
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
-  print == scenario 1: topic_stb_column
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
-  $topicList = ' . topic_stb_column
+
-  $topicList = $topicList . '
+sql show tables
-elif $loop_cnt == 1 then
+if $rows != 2 then 
-  print == scenario 2: topic_stb_all
+  return -1
-  $topicList = ' . topic_stb_all
+endi
-  $topicList = $topicList . '
+#######################################################################################
-elif $loop_cnt == 2 then
+
-  print == scenario 3: topic_stb_function
+if $loop_cnt == 0 then
-  $topicList = ' . topic_stb_function
+  print == scenario 1: topic_stb_column
-  $topicList = $topicList . '
+  $topicList = ' . topic_stb_column
-else
+  $topicList = $topicList . '
-  goto loop_consume_diff_topic_from_stb_end
+elif $loop_cnt == 1 then
-endi
+  print == scenario 2: topic_stb_all
-
+  $topicList = ' . topic_stb_all
-$consumerId    = 0
+  $topicList = $topicList . '
-$totalMsgOfStb = $ctbNum * $rowsPerCtb
+elif $loop_cnt == 2 then
-$expectmsgcnt  = $totalMsgOfStb
+  print == scenario 3: topic_stb_function
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+  $topicList = ' . topic_stb_function
-$consumerId    = 1
+  $topicList = $topicList . '
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+else
-
+  goto loop_consume_diff_topic_from_stb_end
-print == start consumer to pull msgs from stb
+endi
-print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+
-system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+$consumerId    = 0
-
+$totalMsgOfStb = $ctbNum * $rowsPerCtb
-print == check consume result
+$expectmsgcnt  = $totalMsgOfStb
-wait_consumer_end_from_stb:
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-sql select * from consumeresult
+$consumerId    = 1
-print ==> rows: $rows 
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+
-print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
+print == start consumer to pull msgs from stb
-if $rows != 2 then
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
-  sleep 1000
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
-  goto wait_consumer_end_from_stb
+
-endi
+print == check consume result
-if $data[0][1] == 0 then
+wait_consumer_end_from_stb:
-  if $data[1][1] != 1 then
+sql select * from consumeresult
-    return -1
+print ==> rows: $rows 
-  endi
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
-endi
+print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
-if $data[0][1] == 1 then
+if $rows != 2 then
-  if $data[1][1] != 0 then
+  sleep 1000
-    return -1
+  goto wait_consumer_end_from_stb
-  endi
+endi
-endi
+if $data[0][1] == 0 then
-
+  if $data[1][1] != 1 then
-if $data[0][2] <= 0 then
+    return -1
-  return -1
+  endi
-endi
+endi
-if $data[0][2] >= $expectmsgcnt then
+if $data[0][1] == 1 then
-  return -1
+  if $data[1][1] != 0 then
-endi
+    return -1
-
+  endi
-if $data[1][2] <= 0 then
+endi
-  return -1
+
-endi
+if $data[0][2] <= 0 then
-if $data[1][2] >= $expectmsgcnt then
+  return -1
-  return -1
+endi
-endi
+if $data[0][2] >= $expectmsgcnt then
-
+  return -1
-$sumOfMsgCnt = $data[0][2] + $data[1][2]
+endi
-if $sumOfMsgCnt != $expectmsgcnt then
+
-  return -1
+if $data[1][2] <= 0 then
-endi
+  return -1
-
+endi
-
+if $data[1][2] >= $expectmsgcnt then
-if $data[0][3] <= 0 then
+  return -1
-  return -1
+endi
-endi
+
-if $data[0][3] >= $expectmsgcnt then
+$sumOfMsgCnt = $data[0][2] + $data[1][2]
-  return -1
+if $sumOfMsgCnt != $expectmsgcnt then
-endi
+  return -1
-
+endi
-if $data[1][3] <= 0 then
+
-  return -1
+
-endi
+if $data[0][3] <= 0 then
-if $data[1][3] >= $expectmsgcnt then
+  return -1
-  return -1
+endi
-endi
+if $data[0][3] >= $expectmsgcnt then
-
+  return -1
-$sumOfMsgRows = $data[0][3] + $data[1][3]
+endi
-if $sumOfMsgRows != $expectmsgcnt then
+
-  return -1
+if $data[1][3] <= 0 then
-endi
+  return -1
-
+endi
-$loop_cnt = $loop_cnt + 1
+if $data[1][3] >= $expectmsgcnt then
-goto loop_consume_diff_topic_from_stb
+  return -1
-loop_consume_diff_topic_from_stb_end:
+endi
-
+
-print ================ test consume from ctb
+$sumOfMsgRows = $data[0][3] + $data[1][3]
-$loop_cnt = 0
+if $sumOfMsgRows != $expectmsgcnt then
-loop_consume_diff_topic_from_ctb:
+  return -1
-
+endi
-#######################################################################################
+
-# clear consume info and consume result 
+$loop_cnt = $loop_cnt + 1
-#run tsim/tmq/clearConsume.sim
+goto loop_consume_diff_topic_from_stb
-# because drop table function no stable, so by create new db for consume info and result. Modify it later
+loop_consume_diff_topic_from_stb_end:
-$cdb_index = $cdb_index + 1
+
-$cdbName = cdb . $cdb_index
+print ================ test consume from ctb
-sql create database $cdbName vgroups 1
+$loop_cnt = 0
-sleep 500
+loop_consume_diff_topic_from_ctb:
-sql use $cdbName
+
-
+#######################################################################################
-print == create consume info table and consume result table
+# clear consume info and consume result 
-sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+#run tsim/tmq/clearConsume.sim
-sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+# because drop table function no stable, so by create new db for consume info and result. Modify it later
-
+$cdb_index = $cdb_index + 1
-sql show tables
+$cdbName = cdb . $cdb_index
-if $rows != 2 then 
+sql create database $cdbName vgroups 1
-  return -1
+sleep 500
-endi
+sql use $cdbName
-#######################################################################################
+
-
+print == create consume info table and consume result table
-if $loop_cnt == 0 then
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
-  print == scenario 1: topic_ctb_column
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
-  $topicList = ' . topic_ctb_column
+
-  $topicList = $topicList . '
+sql show tables
-elif $loop_cnt == 1 then
+if $rows != 2 then 
-  print == scenario 2: topic_ctb_all
+  return -1
-  $topicList = ' . topic_ctb_all
+endi
-  $topicList = $topicList . '
+#######################################################################################
-elif $loop_cnt == 2 then
+
-  print == scenario 3: topic_ctb_function
+if $loop_cnt == 0 then
-  $topicList = ' . topic_ctb_function
+  print == scenario 1: topic_ctb_column
-  $topicList = $topicList . '
+  $topicList = ' . topic_ctb_column
-else
+  $topicList = $topicList . '
-  goto loop_consume_diff_topic_from_ctb_end
+elif $loop_cnt == 1 then
-endi
+  print == scenario 2: topic_ctb_all
-
+  $topicList = ' . topic_ctb_all
-$consumerId    = 0
+  $topicList = $topicList . '
-$totalMsgOfCtb = $rowsPerCtb
+elif $loop_cnt == 2 then
-$expectmsgcnt  = $totalMsgOfCtb
+  print == scenario 3: topic_ctb_function
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+  $topicList = ' . topic_ctb_function
-$consumerId    = 1
+  $topicList = $topicList . '
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+else
-
+  goto loop_consume_diff_topic_from_ctb_end
-print == start consumer to pull msgs from ctb
+endi
-print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
+
-system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+$consumerId    = 0
-
+$totalMsgOfCtb = $rowsPerCtb
-print == check consume result
+$expectmsgcnt  = $totalMsgOfCtb
-wait_consumer_end_from_ctb:
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-sql select * from consumeresult
+$consumerId    = 1
-print ==> rows: $rows 
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+
-print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
+print == start consumer to pull msgs from ctb
-if $rows != 2 then
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
-  sleep 1000
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
-  goto wait_consumer_end_from_ctb
+
-endi
+print == check consume result
-if $data[0][1] == 0 then
+wait_consumer_end_from_ctb:
-  if $data[1][1] != 1 then
+sql select * from consumeresult
-    return -1
+print ==> rows: $rows 
-  endi
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
-endi
+print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
-if $data[0][1] == 1 then
+if $rows != 2 then
-  if $data[1][1] != 0 then
+  sleep 1000
-    return -1
+  goto wait_consumer_end_from_ctb
-  endi
+endi
-endi
+if $data[0][1] == 0 then
-
+  if $data[1][1] != 1 then
-# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0
+    return -1
-# or     $data[0][2] == 0              and $data[1][2] == $totalMsgOfCtb
+  endi
-if $data[0][2] == $totalMsgOfCtb then
+endi
-  if $data[1][2] == 0 then
+if $data[0][1] == 1 then
-    goto check_ok_0
+  if $data[1][1] != 0 then
-  endi
+    return -1
-elif $data[1][2] == $totalMsgOfCtb then
+  endi
-  if $data[0][2] == 0 then
+endi
-    goto check_ok_0
+
-  endi
+# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0
-endi
+# or     $data[0][2] == 0              and $data[1][2] == $totalMsgOfCtb
-return -1
+if $data[0][2] == $totalMsgOfCtb then
-check_ok_0:
+  if $data[1][2] == 0 then
-
+    goto check_ok_0
-if $data[0][3] == $totalMsgOfCtb then
+  endi
-  if $data[1][3] == 0 then
+elif $data[1][2] == $totalMsgOfCtb then
-    goto check_ok_1
+  if $data[0][2] == 0 then
-  endi
+    goto check_ok_0
-elif $data[1][3] == $totalMsgOfCtb then
+  endi
-  if $data[0][3] == 0 then
+endi
-    goto check_ok_1
+return -1
-  endi
+check_ok_0:
-endi
+
-return -1
+if $data[0][3] == $totalMsgOfCtb then
-check_ok_1:
+  if $data[1][3] == 0 then
-
+    goto check_ok_1
-$loop_cnt = $loop_cnt + 1
+  endi
-goto loop_consume_diff_topic_from_ctb
+elif $data[1][3] == $totalMsgOfCtb then
-loop_consume_diff_topic_from_ctb_end:
+  if $data[0][3] == 0 then
-
+    goto check_ok_1
-print ================ test consume from ntb
+  endi
-$loop_cnt = 0
+endi
-loop_consume_diff_topic_from_ntb:
+return -1
-
+check_ok_1:
-#######################################################################################
+
-# clear consume info and consume result 
+$loop_cnt = $loop_cnt + 1
-#run tsim/tmq/clearConsume.sim
+goto loop_consume_diff_topic_from_ctb
-# because drop table function no stable, so by create new db for consume info and result. Modify it later
+loop_consume_diff_topic_from_ctb_end:
-$cdb_index = $cdb_index + 1
+
-$cdbName = cdb . $cdb_index
+print ================ test consume from ntb
-sql create database $cdbName vgroups 1
+$loop_cnt = 0
-sleep 500
+loop_consume_diff_topic_from_ntb:
-sql use $cdbName
+
-
+#######################################################################################
-print == create consume info table and consume result table
+# clear consume info and consume result 
-sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+#run tsim/tmq/clearConsume.sim
-sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+# because drop table function no stable, so by create new db for consume info and result. Modify it later
-
+$cdb_index = $cdb_index + 1
-sql show tables
+$cdbName = cdb . $cdb_index
-if $rows != 2 then 
+sql create database $cdbName vgroups 1
-  return -1
+sleep 500
-endi
+sql use $cdbName
-#######################################################################################
+
-
+print == create consume info table and consume result table
-if $loop_cnt == 0 then
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
-  print == scenario 1: topic_ntb_column
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
-  $topicList = ' . topic_ntb_column
+
-  $topicList = $topicList . '
+sql show tables
-elif $loop_cnt == 1 then
+if $rows != 2 then 
-  print == scenario 2: topic_ntb_all
+  return -1
-  $topicList = ' . topic_ntb_all
+endi
-  $topicList = $topicList . '
+#######################################################################################
-elif $loop_cnt == 2 then
+
-  print == scenario 3: topic_ntb_function
+if $loop_cnt == 0 then
-  $topicList = ' . topic_ntb_function
+  print == scenario 1: topic_ntb_column
-  $topicList = $topicList . '
+  $topicList = ' . topic_ntb_column
-else
+  $topicList = $topicList . '
-  goto loop_consume_diff_topic_from_ntb_end
+elif $loop_cnt == 1 then
-endi
+  print == scenario 2: topic_ntb_all
-
+  $topicList = ' . topic_ntb_all
-$consumerId    = 0
+  $topicList = $topicList . '
-$totalMsgOfNtb = $rowsPerCtb
+elif $loop_cnt == 2 then
-$expectmsgcnt  = $totalMsgOfNtb
+  print == scenario 3: topic_ntb_function
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+  $topicList = ' . topic_ntb_function
-$consumerId    = 1
+  $topicList = $topicList . '
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+else
-
+  goto loop_consume_diff_topic_from_ntb_end
-print == start consumer to pull msgs from ntb
+endi
-print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
+
-system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+$consumerId    = 0
-
+$totalMsgOfNtb = $rowsPerCtb
-print == check consume result from ntb
+$expectmsgcnt  = $totalMsgOfNtb
-wait_consumer_end_from_ntb:
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-sql select * from consumeresult
+$consumerId    = 1
-print ==> rows: $rows 
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+
-print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
+print == start consumer to pull msgs from ntb
-if $rows != 2 then
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
-  sleep 1000
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
-  goto wait_consumer_end_from_ntb
+
-endi
+print == check consume result from ntb
-if $data[0][1] == 0 then
+wait_consumer_end_from_ntb:
-  if $data[1][1] != 1 then
+sql select * from consumeresult
-    return -1
+print ==> rows: $rows 
-  endi
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
-endi
+print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
-if $data[1][1] == 0 then
+if $rows != 2 then
-  if $data[0][1] != 1 then
+  sleep 1000
-    return -1
+  goto wait_consumer_end_from_ntb
-  endi
+endi
-endi
+if $data[0][1] == 0 then
-
+  if $data[1][1] != 1 then
-# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0
+    return -1
-# or     $data[0][2] == 0              and $data[1][2] == $totalMsgOfNtb
+  endi
-if $data[0][2] == $totalMsgOfNtb then
+endi
-  if $data[1][2] == 0 then
+if $data[1][1] == 0 then
-    goto check_ok_2
+  if $data[0][1] != 1 then
-  endi
+    return -1
-elif $data[1][2] == $totalMsgOfNtb then
+  endi
-  if $data[0][2] == 0 then
+endi
-    goto check_ok_2
+
-  endi
+# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0
-endi
+# or     $data[0][2] == 0              and $data[1][2] == $totalMsgOfNtb
-return -1
+if $data[0][2] == $totalMsgOfNtb then
-check_ok_2:
+  if $data[1][2] == 0 then
-
+    goto check_ok_2
-if $data[0][3] == $totalMsgOfNtb then
+  endi
-  if $data[1][3] == 0 then
+elif $data[1][2] == $totalMsgOfNtb then
-    goto check_ok_3
+  if $data[0][2] == 0 then
-  endi
+    goto check_ok_2
-elif $data[1][3] == $totalMsgOfNtb then
+  endi
-  if $data[0][3] == 0 then
+endi
-    goto check_ok_3
+return -1
-  endi
+check_ok_2:
-endi
+
-return -1
+if $data[0][3] == $totalMsgOfNtb then
-check_ok_3:
+  if $data[1][3] == 0 then
-
+    goto check_ok_3
-$loop_cnt = $loop_cnt + 1
+  endi
-goto loop_consume_diff_topic_from_ntb
+elif $data[1][3] == $totalMsgOfNtb then
-loop_consume_diff_topic_from_ntb_end:
+  if $data[0][3] == 0 then
-
+    goto check_ok_3
-#------ not need stop consumer, because it exit after pull msg overthan expect msg
+  endi
-#system tsim/tmq/consume.sh -s stop -x SIGINT
+endi
-
+return -1
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
+check_ok_3:
+
+$loop_cnt = $loop_cnt + 1
+goto loop_consume_diff_topic_from_ntb
+loop_consume_diff_topic_from_ntb_end:
+
+#------ not need stop consumer, because it exit after pull msg overthan expect msg
+#system tsim/tmq/consume.sh -s stop -x SIGINT
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/tmq/basic4.sim

@@ -1,216 +1,226 @@
-#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
+#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
-#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics
+#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics
-#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics
+#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics
-#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics
+#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics
-#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics
+#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics
-
+
-# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
+# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
-# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
+# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
-#
+#
-# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
+# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
-#
+#
-
+
-run tsim/tmq/prepareBasicEnv-4vgrp.sim
+run tsim/tmq/prepareBasicEnv-4vgrp.sim
-
+
-#---- global parameters start ----#
+#---- global parameters start ----#
-$dbName    = db
+$dbName    = db
-$vgroups    = 4
+$vgroups    = 4
-$stbPrefix  = stb
+$stbPrefix  = stb
-$ctbPrefix  = ctb
+$ctbPrefix  = ctb
-$ntbPrefix  = ntb
+$ntbPrefix  = ntb
-$stbNum     = 1
+$stbNum     = 1
-$ctbNum     = 10
+$ctbNum     = 10
-$ntbNum     = 10
+$ntbNum     = 10
-$rowsPerCtb = 10
+$rowsPerCtb = 10
-$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
+$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
-#---- global parameters end ----#
+#---- global parameters end ----#
-
+
-$pullDelay    = 3
+$pullDelay    = 3
-$ifcheckdata  = 1
+$ifcheckdata  = 1
-$showMsg      = 1
+$ifmanualcommit = 1
-$showRow      = 0
+$showMsg      = 1
-
+$showRow      = 0
-sql connect
+
-sql use $dbName
+sql connect
-
+sql use $dbName
-print == create topics from super table
+
-sql create topic topic_stb_column as select ts, c3 from stb
+print == create topics from super table
-sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
+sql create topic topic_stb_column as select ts, c3 from stb
-sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
+sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
-
+sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
-print == create topics from child table
+
-sql create topic topic_ctb_column as select ts, c3 from ctb0
+print == create topics from child table
-sql create topic topic_ctb_all as select * from ctb0
+sql create topic topic_ctb_column as select ts, c3 from ctb0
-sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
+sql create topic topic_ctb_all as select * from ctb0
-
+sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
-print == create topics from normal table
+
-sql create topic topic_ntb_column as select ts, c3 from ntb0
+print == create topics from normal table
-sql create topic topic_ntb_all as select * from ntb0
+sql create topic topic_ntb_column as select ts, c3 from ntb0
-sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
+sql create topic topic_ntb_all as select * from ntb0
-
+sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
-#sql show topics
+
-#if $rows != 9 then 
+#sql show topics
-#  return -1
+#if $rows != 9 then 
-#endi
+#  return -1
-
+#endi
-$keyList = ' . group.id:cgrp1
+
-$keyList = $keyList . '
+#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest'
-
+$keyList = ' . group.id:cgrp1
-$topicNum = 3
+$keyList = $keyList . ,
-
+$keyList = $keyList . enable.auto.commit:false
-print ================ test consume from stb
+#$keyList = $keyList . ,
-print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function
+#$keyList = $keyList . auto.commit.interval.ms:6000
-$topicList = ' . topic_stb_column
+#$keyList = $keyList . ,
-$topicList = $topicList . ,
+#$keyList = $keyList . auto.offset.reset:earliest
-$topicList = $topicList . topic_stb_all
+$keyList = $keyList . '
-$topicList = $topicList . ,
+print ========== key list:  $keyList
-$topicList = $topicList . topic_stb_function
+
-$topicList = $topicList . '
+
-
+$topicNum = 3
-$consumerId    = 0
+
-$totalMsgOfStb = $ctbNum * $rowsPerCtb
+print ================ test consume from stb
-$totalMsgOfStb = $totalMsgOfStb * $topicNum
+print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function
-$expectmsgcnt  = $totalMsgOfStb
+$topicList = ' . topic_stb_column
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+$topicList = $topicList . ,
-
+$topicList = $topicList . topic_stb_all
-print == start consumer to pull msgs from stb
+$topicList = $topicList . ,
-print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
+$topicList = $topicList . topic_stb_function
-system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
+$topicList = $topicList . '
-
+
-print == check consume result
+$consumerId    = 0
-wait_consumer_end_from_stb:
+$totalMsgOfStb = $ctbNum * $rowsPerCtb
-sql select * from consumeresult
+$totalMsgOfStb = $totalMsgOfStb * $topicNum
-print ==> rows: $rows 
+$expectmsgcnt  = $totalMsgOfStb
-print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-if $rows != 1 then
+
-  sleep 1000
+print == start consumer to pull msgs from stb
-  goto wait_consumer_end_from_stb
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
-endi
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
-if $data[0][1] != $consumerId then
+
-  return -1
+print == check consume result
-endi
+wait_consumer_end_from_stb:
-if $data[0][2] != $expectmsgcnt then
+sql select * from consumeresult
-  return -1
+print ==> rows: $rows 
-endi
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
-if $data[0][3] != $expectmsgcnt then
+if $rows != 1 then
-  return -1
+  sleep 1000
-endi
+  goto wait_consumer_end_from_stb
-
+endi
-#######################################################################################
+if $data[0][1] != $consumerId then
-# clear consume info and consume result 
+  return -1
-#run tsim/tmq/clearConsume.sim
+endi
-# because drop table function no stable, so by create new db for consume info and result. Modify it later
+if $data[0][2] != $expectmsgcnt then
-$cdbName = cdb1
+  return -1
-sql create database $cdbName vgroups 1
+endi
-sleep 500
+if $data[0][3] != $expectmsgcnt then
-sql use $cdbName
+  return -1
-
+endi
-print == create consume info table and consume result table
+
-sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+#######################################################################################
-sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+# clear consume info and consume result 
-
+#run tsim/tmq/clearConsume.sim
-sql show tables
+# because drop table function no stable, so by create new db for consume info and result. Modify it later
-if $rows != 2 then 
+$cdbName = cdb1
-  return -1
+sql create database $cdbName vgroups 1
-endi
+sleep 500
-#######################################################################################
+sql use $cdbName
-
+
-
+print == create consume info table and consume result table
-print ================ test consume from ctb
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
-print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
-$topicList = ' . topic_ctb_column
+
-$topicList = $topicList . ,
+sql show tables
-$topicList = $topicList . topic_ctb_all
+if $rows != 2 then 
-$topicList = $topicList . ,
+  return -1
-$topicList = $topicList . topic_ctb_function
+endi
-$topicList = $topicList . '
+#######################################################################################
-
+
-$consumerId    = 0
+
-$totalMsgOfCtb = $rowsPerCtb * $topicNum
+print ================ test consume from ctb
-$expectmsgcnt  = $totalMsgOfCtb
+print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+$topicList = ' . topic_ctb_column
-
+$topicList = $topicList . ,
-print == start consumer to pull msgs from ctb
+$topicList = $topicList . topic_ctb_all
-print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
+$topicList = $topicList . ,
-system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+$topicList = $topicList . topic_ctb_function
-
+$topicList = $topicList . '
-print == check consume result
+
-wait_consumer_end_from_ctb:
+$consumerId    = 0
-sql select * from consumeresult
+$totalMsgOfCtb = $rowsPerCtb * $topicNum
-print ==> rows: $rows 
+$expectmsgcnt  = $totalMsgOfCtb
-print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-if $rows != 1 then
+
-  sleep 1000
+print == start consumer to pull msgs from ctb
-  goto wait_consumer_end_from_ctb
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
-endi
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
-if $data[0][1] != $consumerId then
+
-  return -1
+print == check consume result
-endi
+wait_consumer_end_from_ctb:
-if $data[0][2] != $totalMsgOfCtb then
+sql select * from consumeresult
-  return -1
+print ==> rows: $rows 
-endi
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
-if $data[0][3] != $totalMsgOfCtb then
+if $rows != 1 then
-  return -1
+  sleep 1000
-endi
+  goto wait_consumer_end_from_ctb
-
+endi
-#######################################################################################
+if $data[0][1] != $consumerId then
-# clear consume info and consume result 
+  return -1
-#run tsim/tmq/clearConsume.sim
+endi
-# because drop table function no stable, so by create new db for consume info and result. Modify it later
+if $data[0][2] != $totalMsgOfCtb then
-$cdbName = cdb2
+  return -1
-sql create database $cdbName vgroups 1
+endi
-sleep 500
+if $data[0][3] != $totalMsgOfCtb then
-sql use $cdbName
+  return -1
-
+endi
-print == create consume info table and consume result table
+
-sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+#######################################################################################
-sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+# clear consume info and consume result 
-
+#run tsim/tmq/clearConsume.sim
-sql show tables
+# because drop table function no stable, so by create new db for consume info and result. Modify it later
-if $rows != 2 then 
+$cdbName = cdb2
-  return -1
+sql create database $cdbName vgroups 1
-endi
+sleep 500
-#######################################################################################
+sql use $cdbName
-
+
-
+print == create consume info table and consume result table
-print ================ test consume from ntb
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
-print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
-$topicList = ' . topic_ntb_column
+
-$topicList = $topicList . ,
+sql show tables
-$topicList = $topicList . topic_ntb_all
+if $rows != 2 then 
-$topicList = $topicList . ,
+  return -1
-$topicList = $topicList . topic_ntb_function
+endi
-$topicList = $topicList . '
+#######################################################################################
-
+
-$consumerId    = 0
+
-$totalMsgOfNtb = $rowsPerCtb * $topicNum
+print ================ test consume from ntb
-$expectmsgcnt  = $totalMsgOfNtb
+print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+$topicList = ' . topic_ntb_column
-
+$topicList = $topicList . ,
-print == start consumer to pull msgs from ntb
+$topicList = $topicList . topic_ntb_all
-print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
+$topicList = $topicList . ,
-system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+$topicList = $topicList . topic_ntb_function
-
+$topicList = $topicList . '
-print == check consume result from ntb
+
-wait_consumer_end_from_ntb:
+$consumerId    = 0
-sql select * from consumeresult
+$totalMsgOfNtb = $rowsPerCtb * $topicNum
-print ==> rows: $rows 
+$expectmsgcnt  = $totalMsgOfNtb
-print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-if $rows != 1 then
+
-  sleep 1000
+print == start consumer to pull msgs from ntb
-  goto wait_consumer_end_from_ntb
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
-endi
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
-if $data[0][1] != $consumerId then
+
-  return -1
+print == check consume result from ntb
-endi
+wait_consumer_end_from_ntb:
-if $data[0][2] != $totalMsgOfNtb then
+sql select * from consumeresult
-  return -1
+print ==> rows: $rows 
-endi
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
-if $data[0][3] != $totalMsgOfNtb then
+if $rows != 1 then
-  return -1
+  sleep 1000
-endi
+  goto wait_consumer_end_from_ntb
-
+endi
-#------ not need stop consumer, because it exit after pull msg overthan expect msg
+if $data[0][1] != $consumerId then
-#system tsim/tmq/consume.sh -s stop -x SIGINT
+  return -1
-
+endi
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
+if $data[0][2] != $totalMsgOfNtb then
+  return -1
+endi
+if $data[0][3] != $totalMsgOfNtb then
+  return -1
+endi
+
+#------ not need stop consumer, because it exit after pull msg overthan expect msg
+#system tsim/tmq/consume.sh -s stop -x SIGINT
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/tmq/basic4Of2Cons.sim

@@ -1,319 +1,328 @@
-#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
+#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
-#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
+#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
-#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
+#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
-#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
+#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
-#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
+#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
-
+
-# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
+# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
-# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
+# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
-#
+#
-# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
+# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
-#
+#
-
+
-run tsim/tmq/prepareBasicEnv-4vgrp.sim
+run tsim/tmq/prepareBasicEnv-4vgrp.sim
-
+
-#---- global parameters start ----#
+#---- global parameters start ----#
-$dbName    = db
+$dbName    = db
-$vgroups    = 4
+$vgroups    = 4
-$stbPrefix  = stb
+$stbPrefix  = stb
-$ctbPrefix  = ctb
+$ctbPrefix  = ctb
-$ntbPrefix  = ntb
+$ntbPrefix  = ntb
-$stbNum     = 1
+$stbNum     = 1
-$ctbNum     = 10
+$ctbNum     = 10
-$ntbNum     = 10
+$ntbNum     = 10
-$rowsPerCtb = 10
+$rowsPerCtb = 10
-$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
+$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
-#---- global parameters end ----#
+#---- global parameters end ----#
-
+
-$pullDelay    = 5
+$pullDelay    = 5
-$ifcheckdata  = 1
+$ifcheckdata  = 1
-$showMsg      = 1
+$ifmanualcommit = 1
-$showRow      = 0
+$showMsg      = 1
-
+$showRow      = 0
-sql connect
+
-sql use $dbName
+sql connect
-
+sql use $dbName
-print == create topics from super table
+
-sql create topic topic_stb_column as select ts, c3 from stb
+print == create topics from super table
-sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
+sql create topic topic_stb_column as select ts, c3 from stb
-sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
+sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
-
+sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
-print == create topics from child table
+
-sql create topic topic_ctb_column as select ts, c3 from ctb0
+print == create topics from child table
-sql create topic topic_ctb_all as select * from ctb0
+sql create topic topic_ctb_column as select ts, c3 from ctb0
-sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
+sql create topic topic_ctb_all as select * from ctb0
-
+sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
-print == create topics from normal table
+
-sql create topic topic_ntb_column as select ts, c3 from ntb0
+print == create topics from normal table
-sql create topic topic_ntb_all as select * from ntb0
+sql create topic topic_ntb_column as select ts, c3 from ntb0
-sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
+sql create topic topic_ntb_all as select * from ntb0
-
+sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
-#sql show topics
+
-#if $rows != 9 then 
+#sql show topics
-#  return -1
+#if $rows != 9 then 
-#endi
+#  return -1
-
+#endi
-$keyList = ' . group.id:cgrp1
+
-$keyList = $keyList . '
+#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest'
-
+$keyList = ' . group.id:cgrp1
-$topicNum = 3
+$keyList = $keyList . ,
-
+$keyList = $keyList . enable.auto.commit:false
-print ================ test consume from stb
+#$keyList = $keyList . ,
-print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function
+#$keyList = $keyList . auto.commit.interval.ms:6000
-$topicList = ' . topic_stb_column
+#$keyList = $keyList . ,
-$topicList = $topicList . ,
+#$keyList = $keyList . auto.offset.reset:earliest
-$topicList = $topicList . topic_stb_all
+$keyList = $keyList . '
-$topicList = $topicList . ,
+print ========== key list:  $keyList
-$topicList = $topicList . topic_stb_function
+
-$topicList = $topicList . '
+$topicNum = 3
-
+
-$consumerId    = 0
+print ================ test consume from stb
-$totalMsgOfStb = $ctbNum * $rowsPerCtb
+print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function
-$totalMsgOfStb = $totalMsgOfStb * $topicNum
+$topicList = ' . topic_stb_column
-$expectmsgcnt  = $totalMsgOfStb
+$topicList = $topicList . ,
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+$topicList = $topicList . topic_stb_all
-$consumerId    = 1
+$topicList = $topicList . ,
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+$topicList = $topicList . topic_stb_function
-
+$topicList = $topicList . '
-print == start consumer to pull msgs from stb
+
-print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
+$consumerId    = 0
-system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
+$totalMsgOfStb = $ctbNum * $rowsPerCtb
-
+$totalMsgOfStb = $totalMsgOfStb * $topicNum
-print == check consume result
+$expectmsgcnt  = $totalMsgOfStb
-wait_consumer_end_from_stb:
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-sql select * from consumeresult
+$consumerId    = 1
-print ==> rows: $rows 
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+
-print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
+print == start consumer to pull msgs from stb
-if $rows != 2 then
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
-  sleep 1000
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
-  goto wait_consumer_end_from_stb
+
-endi
+print == check consume result
-if $data[0][1] == 0 then
+wait_consumer_end_from_stb:
-  if $data[1][1] != 1 then
+sql select * from consumeresult
-    return -1
+print ==> rows: $rows 
-  endi
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
-endi
+print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
-if $data[0][1] == 1 then
+if $rows != 2 then
-  if $data[1][1] != 0 then
+  sleep 1000
-    return -1
+  goto wait_consumer_end_from_stb
-  endi
+endi
-endi
+if $data[0][1] == 0 then
-
+  if $data[1][1] != 1 then
-if $data[0][2] <= 0 then
+    return -1
-  return -1
+  endi
-endi
+endi
-if $data[0][2] >= $expectmsgcnt then
+if $data[0][1] == 1 then
-  return -1
+  if $data[1][1] != 0 then
-endi
+    return -1
-
+  endi
-if $data[1][2] <= 0 then
+endi
-  return -1
+
-endi
+if $data[0][2] <= 0 then
-if $data[1][2] >= $expectmsgcnt then
+  return -1
-  return -1
+endi
-endi
+if $data[0][2] >= $expectmsgcnt then
-
+  return -1
-$sumOfConsMsg = $data[0][2] + $data[1][2]
+endi
-if $sumOfConsMsg != $expectmsgcnt then
+
-  return -1
+if $data[1][2] <= 0 then
-endi
+  return -1
-
+endi
-if $data[0][3] <= 0 then
+if $data[1][2] >= $expectmsgcnt then
-  return -1
+  return -1
-endi
+endi
-if $data[0][3] >= $expectmsgcnt then
+
-  return -1
+$sumOfConsMsg = $data[0][2] + $data[1][2]
-endi
+if $sumOfConsMsg != $expectmsgcnt then
-
+  return -1
-if $data[1][3] <= 0 then
+endi
-  return -1
+
-endi
+if $data[0][3] <= 0 then
-if $data[1][3] >= $expectmsgcnt then
+  return -1
-  return -1
+endi
-endi
+if $data[0][3] >= $expectmsgcnt then
-
+  return -1
-$sumOfConsRow = $data[0][3] + $data[1][3]
+endi
-if $sumOfConsRow != $expectmsgcnt then
+
-  return -1
+if $data[1][3] <= 0 then
-endi
+  return -1
-
+endi
-#######################################################################################
+if $data[1][3] >= $expectmsgcnt then
-# clear consume info and consume result 
+  return -1
-#run tsim/tmq/clearConsume.sim
+endi
-# because drop table function no stable, so by create new db for consume info and result. Modify it later
+
-$cdbName = cdb1
+$sumOfConsRow = $data[0][3] + $data[1][3]
-sql create database $cdbName vgroups 1
+if $sumOfConsRow != $expectmsgcnt then
-sleep 500
+  return -1
-sql use $cdbName
+endi
-
+
-print == create consume info table and consume result table
+#######################################################################################
-sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+# clear consume info and consume result 
-sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+#run tsim/tmq/clearConsume.sim
-
+# because drop table function no stable, so by create new db for consume info and result. Modify it later
-sql show tables
+$cdbName = cdb1
-if $rows != 2 then 
+sql create database $cdbName vgroups 1
-  return -1
+sleep 500
-endi
+sql use $cdbName
-#######################################################################################
+
-
+print == create consume info table and consume result table
-
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
-print ================ test consume from ctb
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
-print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function
+
-$topicList = ' . topic_ctb_column
+sql show tables
-$topicList = $topicList . ,
+if $rows != 2 then 
-$topicList = $topicList . topic_ctb_all
+  return -1
-$topicList = $topicList . ,
+endi
-$topicList = $topicList . topic_ctb_function
+#######################################################################################
-$topicList = $topicList . '
+
-
+
-$consumerId    = 0
+print ================ test consume from ctb
-$totalMsgOfCtb = $rowsPerCtb * $topicNum
+print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function
-$expectmsgcnt  = $totalMsgOfCtb
+$topicList = ' . topic_ctb_column
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+$topicList = $topicList . ,
-$consumerId    = 1
+$topicList = $topicList . topic_ctb_all
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+$topicList = $topicList . ,
-
+$topicList = $topicList . topic_ctb_function
-print == start consumer to pull msgs from ctb
+$topicList = $topicList . '
-print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
+
-system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+$consumerId    = 0
-
+$totalMsgOfCtb = $rowsPerCtb * $topicNum
-print == check consume result
+$expectmsgcnt  = $totalMsgOfCtb
-wait_consumer_end_from_ctb:
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-sql select * from consumeresult
+$consumerId    = 1
-print ==> rows: $rows 
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+
-print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
+print == start consumer to pull msgs from ctb
-if $rows != 2 then
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
-  sleep 1000
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
-  goto wait_consumer_end_from_ctb
+
-endi
+print == check consume result
-if $data[0][1] == 0 then
+wait_consumer_end_from_ctb:
-  if $data[1][1] != 1 then
+sql select * from consumeresult
-    return -1
+print ==> rows: $rows 
-  endi
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
-endi
+print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
-if $data[0][1] == 1 then
+if $rows != 2 then
-  if $data[1][1] != 0 then
+  sleep 1000
-    return -1
+  goto wait_consumer_end_from_ctb
-  endi
+endi
-endi
+if $data[0][1] == 0 then
-
+  if $data[1][1] != 1 then
-# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0
+    return -1
-# or     $data[0][2] == 0              and $data[1][2] == $totalMsgOfCtb
+  endi
-if $data[0][2] == $totalMsgOfCtb then
+endi
-  if $data[1][2] == 0 then
+if $data[0][1] == 1 then
-    goto check_ok_0
+  if $data[1][1] != 0 then
-  endi
+    return -1
-elif $data[0][2] == 0 then
+  endi
-  if $data[1][2] == $totalMsgOfCtb then
+endi
-    goto check_ok_0
+
-  endi
+# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0
-endi
+# or     $data[0][2] == 0              and $data[1][2] == $totalMsgOfCtb
-return -1
+if $data[0][2] == $totalMsgOfCtb then
-check_ok_0:
+  if $data[1][2] == 0 then
-
+    goto check_ok_0
-if $data[0][3] == $totalMsgOfCtb then
+  endi
-  if $data[1][3] == 0 then
+elif $data[0][2] == 0 then
-    goto check_ok_1
+  if $data[1][2] == $totalMsgOfCtb then
-  endi
+    goto check_ok_0
-elif $data[0][3] == 0 then
+  endi
-  if $data[1][3] == $totalMsgOfCtb then
+endi
-    goto check_ok_1
+return -1
-  endi
+check_ok_0:
-endi
+
-return -1
+if $data[0][3] == $totalMsgOfCtb then
-check_ok_1:
+  if $data[1][3] == 0 then
-
+    goto check_ok_1
-
+  endi
-#######################################################################################
+elif $data[0][3] == 0 then
-# clear consume info and consume result 
+  if $data[1][3] == $totalMsgOfCtb then
-#run tsim/tmq/clearConsume.sim
+    goto check_ok_1
-# because drop table function no stable, so by create new db for consume info and result. Modify it later
+  endi
-$cdbName = cdb2
+endi
-sql create database $cdbName vgroups 1
+return -1
-sleep 500
+check_ok_1:
-sql use $cdbName
+
-
+
-print == create consume info table and consume result table
+#######################################################################################
-sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+# clear consume info and consume result 
-sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+#run tsim/tmq/clearConsume.sim
-
+# because drop table function no stable, so by create new db for consume info and result. Modify it later
-sql show tables
+$cdbName = cdb2
-if $rows != 2 then 
+sql create database $cdbName vgroups 1
-  return -1
+sleep 500
-endi
+sql use $cdbName
-#######################################################################################
+
-
+print == create consume info table and consume result table
-
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
-print ================ test consume from ntb
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
-print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function
+
-$topicList = ' . topic_ntb_column
+sql show tables
-$topicList = $topicList . ,
+if $rows != 2 then 
-$topicList = $topicList . topic_ntb_all
+  return -1
-$topicList = $topicList . ,
+endi
-$topicList = $topicList . topic_ntb_function
+#######################################################################################
-$topicList = $topicList . '
+
-
+
-$consumerId    = 0
+print ================ test consume from ntb
-$totalMsgOfNtb = $rowsPerCtb * $topicNum
+print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function
-$expectmsgcnt  = $totalMsgOfNtb
+$topicList = ' . topic_ntb_column
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+$topicList = $topicList . ,
-$consumerId    = 1
+$topicList = $topicList . topic_ntb_all
-sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+$topicList = $topicList . ,
-
+$topicList = $topicList . topic_ntb_function
-print == start consumer to pull msgs from ntb
+$topicList = $topicList . '
-print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
+
-system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+$consumerId    = 0
-
+$totalMsgOfNtb = $rowsPerCtb * $topicNum
-print == check consume result from ntb
+$expectmsgcnt  = $totalMsgOfNtb
-wait_consumer_end_from_ntb:
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-sql select * from consumeresult
+$consumerId    = 1
-print ==> rows: $rows 
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
-print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+
-print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
+print == start consumer to pull msgs from ntb
-if $rows != 2 then
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
-  sleep 1000
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
-  goto wait_consumer_end_from_ntb
+
-endi
+print == check consume result from ntb
-if $data[0][1] == 0 then
+wait_consumer_end_from_ntb:
-  if $data[1][1] != 1 then
+sql select * from consumeresult
-    return -1
+print ==> rows: $rows 
-  endi
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
-endi
+print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
-if $data[1][1] == 0 then
+if $rows != 2 then
-  if $data[0][1] != 1 then
+  sleep 1000
-    return -1
+  goto wait_consumer_end_from_ntb
-  endi
+endi
-endi
+if $data[0][1] == 0 then
-
+  if $data[1][1] != 1 then
-# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0
+    return -1
-# or     $data[0][2] == 0              and $data[1][2] == $totalMsgOfNtb
+  endi
-if $data[0][2] == $totalMsgOfNtb then
+endi
-  if $data[1][2] == 0 then
+if $data[1][1] == 0 then
-    goto check_ok_2
+  if $data[0][1] != 1 then
-  endi
+    return -1
-elif $data[0][2] == 0 then
+  endi
-  if $data[1][2] == $totalMsgOfNtb then
+endi
-    goto check_ok_2
+
-  endi
+# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0
-endi
+# or     $data[0][2] == 0              and $data[1][2] == $totalMsgOfNtb
-return -1
+if $data[0][2] == $totalMsgOfNtb then
-check_ok_2:
+  if $data[1][2] == 0 then
-
+    goto check_ok_2
-if $data[0][3] == $totalMsgOfNtb then
+  endi
-  if $data[1][3] == 0 then
+elif $data[0][2] == 0 then
-    goto check_ok_3
+  if $data[1][2] == $totalMsgOfNtb then
-  endi
+    goto check_ok_2
-elif $data[0][3] == 0 then
+  endi
-  if $data[1][3] == $totalMsgOfNtb then
+endi
-    goto check_ok_3
+return -1
-  endi
+check_ok_2:
-endi
+
-return -1
+if $data[0][3] == $totalMsgOfNtb then
-check_ok_3:
+  if $data[1][3] == 0 then
-
+    goto check_ok_3
-#------ not need stop consumer, because it exit after pull msg overthan expect msg
+  endi
-#system tsim/tmq/consume.sh -s stop -x SIGINT
+elif $data[0][3] == 0 then
-
+  if $data[1][3] == $totalMsgOfNtb then
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
+    goto check_ok_3
+  endi
+endi
+return -1
+check_ok_3:
+
+#------ not need stop consumer, because it exit after pull msg overthan expect msg
+#system tsim/tmq/consume.sh -s stop -x SIGINT
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim

@@ -1,88 +1,88 @@
-# stop all dnodes before start this case
+# stop all dnodes before start this case
-system sh/stop_dnodes.sh
+system sh/stop_dnodes.sh
-
+
-# deploy dnode 1
+# deploy dnode 1
-system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode1 -i 1
-
+
-# add some config items for this case
+# add some config items for this case
-#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
+#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
-
+
-# start dnode 1
+# start dnode 1
-system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode1 -s start
-
+
-sql connect
+sql connect
-
+
-#---- global parameters start ----#
+#---- global parameters start ----#
-$dbName     = db
+$dbName     = db
-$vgroups    = 1
+$vgroups    = 1
-$stbPrefix  = stb
+$stbPrefix  = stb
-$ctbPrefix  = ctb
+$ctbPrefix  = ctb
-$ntbPrefix  = ntb
+$ntbPrefix  = ntb
-$stbNum     = 1
+$stbNum     = 1
-$ctbNum     = 10
+$ctbNum     = 10
-$ntbNum     = 10
+$ntbNum     = 10
-$rowsPerCtb = 10
+$rowsPerCtb = 10
-$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
+$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
-#---- global parameters end ----#
+#---- global parameters end ----#
-
+
-print == create database $dbName vgroups $vgroups
+print == create database $dbName vgroups $vgroups
-sql create database $dbName vgroups $vgroups
+sql create database $dbName vgroups $vgroups
-
+
-#wait database ready
+#wait database ready
-$loop_cnt = 0
+$loop_cnt = 0
-check_db_ready:
+check_db_ready:
-if $loop_cnt == 10 then
+if $loop_cnt == 10 then
-  print ====> database not ready!
+  print ====> database not ready!
-	return -1
+	return -1
-endi
+endi
-sql show databases
+sql show databases
-print ==> rows: $rows
+print ==> rows: $rows
-print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12]
+print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12]
-print     $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20]
+print     $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20]
-if $data(db)[19] != nostrict then
+if $data(db)[19] != nostrict then
-  sleep 100
+  sleep 100
-  $loop_cnt = $loop_cnt + 1
+  $loop_cnt = $loop_cnt + 1
-  goto check_db_ready  
+  goto check_db_ready  
-endi
+endi
-
+
-sql use $dbName
+sql use $dbName
-
+
-print == create consume info table and consume result table
+print == create consume info table and consume result table
-sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
-sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
-
+
-sql show tables
+sql show tables
-if $rows != 2 then 
+if $rows != 2 then 
-  return -1
+  return -1
-endi
+endi
-
+
-print == create super table
+print == create super table
-sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int)
+sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int)
-sql show stables
+sql show stables
-if $rows != 1 then 
+if $rows != 1 then 
-  return -1
+  return -1
-endi
+endi
-
+
-print == create child table, normal table and insert data
+print == create child table, normal table and insert data
-$i = 0
+$i = 0
-while $i < $ctbNum
+while $i < $ctbNum
-  $ctb = $ctbPrefix . $i
+  $ctb = $ctbPrefix . $i
-  $ntb = $ntbPrefix . $i
+  $ntb = $ntbPrefix . $i
-  sql create table $ctb using $stbPrefix tags( $i )
+  sql create table $ctb using $stbPrefix tags( $i )
-  sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16))
+  sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16))
-
+
-  $x = 0
+  $x = 0
-  while $x < $rowsPerCtb
+  while $x < $rowsPerCtb
-    $binary = ' . binary-
+    $binary = ' . binary-
-    $binary = $binary . $i
+    $binary = $binary . $i
-    $binary = $binary . '
+    $binary = $binary . '
-
+
-    sql insert into $ctb values ($tstart , $i , $x , $binary )
+    sql insert into $ctb values ($tstart , $i , $x , $binary )
-    sql insert into $ntb values ($tstart , $i , $x , $binary )
+    sql insert into $ntb values ($tstart , $i , $x , $binary )
-    $tstart = $tstart + 1
+    $tstart = $tstart + 1
-    $x = $x + 1
+    $x = $x + 1
-  endw
+  endw
-
+
-  $i = $i + 1
+  $i = $i + 1
-  $tstart = 1640966400000
+  $tstart = 1640966400000
-endw
+endw

tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim

@@ -1,88 +1,88 @@
-# stop all dnodes before start this case
+# stop all dnodes before start this case
-system sh/stop_dnodes.sh
+system sh/stop_dnodes.sh
-
+
-# deploy dnode 1
+# deploy dnode 1
-system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode1 -i 1
-
+
-# add some config items for this case
+# add some config items for this case
-#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
+#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
-
+
-# start dnode 1
+# start dnode 1
-system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode1 -s start
-
+
-sql connect
+sql connect
-
+
-#---- global parameters start ----#
+#---- global parameters start ----#
-$dbName     = db
+$dbName     = db
-$vgroups    = 4
+$vgroups    = 4
-$stbPrefix  = stb
+$stbPrefix  = stb
-$ctbPrefix  = ctb
+$ctbPrefix  = ctb
-$ntbPrefix  = ntb
+$ntbPrefix  = ntb
-$stbNum     = 1
+$stbNum     = 1
-$ctbNum     = 10
+$ctbNum     = 10
-$ntbNum     = 10
+$ntbNum     = 10
-$rowsPerCtb = 10
+$rowsPerCtb = 10
-$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
+$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
-#---- global parameters end ----#
+#---- global parameters end ----#
-
+
-print == create database $dbName vgroups $vgroups
+print == create database $dbName vgroups $vgroups
-sql create database $dbName vgroups $vgroups
+sql create database $dbName vgroups $vgroups
-
+
-#wait database ready
+#wait database ready
-$loop_cnt = 0
+$loop_cnt = 0
-check_db_ready:
+check_db_ready:
-if $loop_cnt == 10 then
+if $loop_cnt == 10 then
-  print ====> database not ready!
+  print ====> database not ready!
-	return -1
+	return -1
-endi
+endi
-sql show databases
+sql show databases
-print ==> rows: $rows
+print ==> rows: $rows
-print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12]
+print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12]
-print     $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20]
+print     $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20]
-if $data(db)[19] != nostrict then
+if $data(db)[19] != nostrict then
-  sleep 100
+  sleep 100
-  $loop_cnt = $loop_cnt + 1
+  $loop_cnt = $loop_cnt + 1
-  goto check_db_ready  
+  goto check_db_ready  
-endi
+endi
-
+
-sql use $dbName
+sql use $dbName
-
+
-print == create consume info table and consume result table
+print == create consume info table and consume result table
-sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)
-sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
-
+
-sql show tables
+sql show tables
-if $rows != 2 then 
+if $rows != 2 then 
-  return -1
+  return -1
-endi
+endi
-
+
-print == create super table
+print == create super table
-sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int)
+sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int)
-sql show stables
+sql show stables
-if $rows != 1 then 
+if $rows != 1 then 
-  return -1
+  return -1
-endi
+endi
-
+
-print == create child table, normal table and insert data
+print == create child table, normal table and insert data
-$i = 0
+$i = 0
-while $i < $ctbNum
+while $i < $ctbNum
-  $ctb = $ctbPrefix . $i
+  $ctb = $ctbPrefix . $i
-  $ntb = $ntbPrefix . $i
+  $ntb = $ntbPrefix . $i
-  sql create table $ctb using $stbPrefix tags( $i )
+  sql create table $ctb using $stbPrefix tags( $i )
-  sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16))
+  sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16))
-
+
-  $x = 0
+  $x = 0
-  while $x < $rowsPerCtb
+  while $x < $rowsPerCtb
-    $binary = ' . binary-
+    $binary = ' . binary-
-    $binary = $binary . $i
+    $binary = $binary . $i
-    $binary = $binary . '
+    $binary = $binary . '
-
+
-    sql insert into $ctb values ($tstart , $i , $x , $binary )
+    sql insert into $ctb values ($tstart , $i , $x , $binary )
-    sql insert into $ntb values ($tstart , $i , $x , $binary )
+    sql insert into $ntb values ($tstart , $i , $x , $binary )
-    $tstart = $tstart + 1
+    $tstart = $tstart + 1
-    $x = $x + 1
+    $x = $x + 1
-  endw
+  endw
-
+
-  $i = $i + 1
+  $i = $i + 1
-  $tstart = 1640966400000
+  $tstart = 1640966400000
-endw
+endw

tests/system-test/2-query/query_cols_tags_and_or.py

@@ -22,18 +22,6 @@ class TDTestCase:
         tdSql.init(conn.cursor(), logSql)
 
     def insertData(self, tb_name):
-        # insert_sql_list = [f'insert into {tb_name} values ("2021-01-01 12:00:00", 1, 1, 1, 3, 1.1, 1.1, "binary", "nchar", true, 1)',
-        #                    f'insert into {tb_name} values ("2021-01-05 12:00:00", 2, 2, 1, 3, 1.1, 1.1, "binary", "nchar", true, 2)',
-        #                    f'insert into {tb_name} values ("2021-01-07 12:00:00", 1, 3, 1, 2, 1.1, 1.1, "binary", "nchar", true, 3)',
-        #                    f'insert into {tb_name} values ("2021-01-09 12:00:00", 1, 2, 4, 3, 1.1, 1.1, "binary", "nchar", true, 4)',
-        #                    f'insert into {tb_name} values ("2021-01-11 12:00:00", 1, 2, 5, 5, 1.1, 1.1, "binary", "nchar", true, 5)',
-        #                    f'insert into {tb_name} values ("2021-01-13 12:00:00", 1, 2, 1, 3, 6.6, 1.1, "binary", "nchar", true, 6)',
-        #                    f'insert into {tb_name} values ("2021-01-15 12:00:00", 1, 2, 1, 3, 1.1, 7.7, "binary", "nchar", true, 7)',
-        #                    f'insert into {tb_name} values ("2021-01-17 12:00:00", 1, 2, 1, 3, 1.1, 1.1, "binary8", "nchar", true, 8)',
-        #                    f'insert into {tb_name} values ("2021-01-19 12:00:00", 1, 2, 1, 3, 1.1, 1.1, "binary", "nchar9", true, 9)',
-        #                    f'insert into {tb_name} values ("2021-01-21 12:00:00", 1, 2, 1, 3, 1.1, 1.1, "binary", "nchar", false, 10)',
-        #                    f'insert into {tb_name} values ("2021-01-23 12:00:00", 1, 3, 1, 3, 1.1, 1.1, Null, Null, false, 11)'
-        #                    ]
         insert_sql_list = [f'insert into {tb_name} values ("2021-01-01 12:00:00", 1, 1, 1, 3, 1.1, 1.1, "binary", "nchar", true, 1, 2, 3, 4)',
                            f'insert into {tb_name} values ("2021-01-05 12:00:00", 2, 2, 1, 3, 1.1, 1.1, "binary", "nchar", true, 2, 3, 4, 5)',
                            f'insert into {tb_name} values ("2021-01-07 12:00:00", 1, 3, 1, 2, 1.1, 1.1, "binary", "nchar", true, 3, 4, 5, 6)',
@@ -54,7 +42,6 @@ class TDTestCase:
         tb_name = tdCom.getLongName(8, "letters")
         tdSql.execute(
             f"CREATE TABLE {tb_name} (ts timestamp, c1 tinyint, c2 smallint, c3 int, c4 bigint, c5 float, c6 double, c7 binary(100), c8 nchar(200), c9 bool, c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned)")
-            # f"CREATE TABLE {tb_name} (ts timestamp, c1 tinyint, c2 smallint, c3 int, c4 bigint, c5 float, c6 double, c7 binary(100), c8 nchar(200), c9 bool, c10 int)")
         self.insertData(tb_name)
         return tb_name
 
@@ -95,6 +82,31 @@ class TDTestCase:
 
     def queryTsCol(self, tb_name, check_elm=None):
         select_elm = "*" if check_elm is None else check_elm
+        # ts in
+        query_sql = f'select {select_elm} from {tb_name} where ts in ("2021-01-11 12:00:00", "2021-01-13 12:00:00")'
+        tdSql.query(query_sql)
+        tdSql.checkRows(2)
+        tdSql.checkEqual(self.queryLastC10(query_sql), 6) if select_elm == "*" else False
+        # ts not in
+        query_sql = f'select {select_elm} from {tb_name} where ts not in ("2021-01-11 12:00:00", "2021-01-13 12:00:00")'
+        tdSql.query(query_sql)
+        tdSql.checkRows(9)
+        tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
+        # ts not null
+        query_sql = f'select {select_elm} from {tb_name} where ts is not Null'
+        tdSql.query(query_sql)
+        tdSql.checkRows(11)
+        tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
+        # ts null
+        query_sql = f'select {select_elm} from {tb_name} where ts is Null'
+        tdSql.query(query_sql)
+        tdSql.checkRows(0)
+        # not support like not like match nmatch
+        tdSql.error(f'select {select_elm} from {tb_name} where ts like ("2021-01-11 12:00:00%")')
+        tdSql.error(f'select {select_elm} from {tb_name} where ts not like ("2021-01-11 12:00:0_")')
+        tdSql.error(f'select {select_elm} from {tb_name} where ts match "2021-01-11 12:00:00%"')
+        tdSql.error(f'select {select_elm} from {tb_name} where ts nmatch "2021-01-11 12:00:00%"')
+
         # ts and ts
         query_sql = f'select {select_elm} from {tb_name} where ts > "2021-01-11 12:00:00" or ts < "2021-01-13 12:00:00"'
         tdSql.query(query_sql)
@@ -1422,9 +1434,9 @@ class TDTestCase:
         tdSql.query(query_sql)
         tdSql.checkRows(11)
         tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
-        query_sql = f'select c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13 from {tb_name} where c9 > "binary" and c9 >= "binary8" or c9 < "binary9" and c9 <= "binary" and c9 != 2 and c9 <> 2 and c9 = 4 or c9 is not null and c9 between 2 and 4 and c9 not between 1 and 2 and c9 in (2,4) and c9 not in (1,2) or c9 match "binary[28]" or c9 nmatch "binary"'
+        query_sql = f'select c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13 from {tb_name} where c9 > "binary" and c9 >= "binary8" or c9 < "binary9" and c9 <= "binary" and c9 != 2 and c9 <> 2 and c9 = 4 or c9 is not null and c9 between 2 and 4 and c9 not between 1 and 2 and c9 in (2,4) and c9 not in (1,2)'
         tdSql.query(query_sql)
-        tdSql.checkRows(11)
+        tdSql.checkRows(9)
 
     def queryFullColType(self, tb_name, check_elm=None):
         select_elm = "*" if check_elm is None else check_elm

tests/system-test/2-query/union.py

@@ -0,0 +1,412 @@
+import datetime
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+
+PRIMARY_COL = "ts"
+
+INT_COL     = "c1"
+BINT_COL    = "c2"
+SINT_COL    = "c3"
+TINT_COL    = "c4"
+FLOAT_COL   = "c5"
+DOUBLE_COL  = "c6"
+BOOL_COL    = "c7"
+
+BINARY_COL  = "c8"
+NCHAR_COL   = "c9"
+TS_COL      = "c10"
+
+NUM_COL     = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ]
+CHAR_COL    = [ BINARY_COL, NCHAR_COL, ]
+BOOLEAN_COL = [ BOOL_COL, ]
+TS_TYPE_COL = [ TS_COL, ]
+
+class TDTestCase:
+
+    def init(self, conn, logSql):
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor())
+
+    def __query_condition(self,tbname):
+        query_condition = []
+        for char_col in CHAR_COL:
+            query_condition.extend(
+                (
+                    f"{tbname}.{char_col}",
+                    f"upper( {tbname}.{char_col} )",
+                    f"char_length( {tbname}.{char_col} )",
+                    f"concat( {tbname}.{char_col}, {tbname}.{char_col} )",
+                    f"concat_ws( '_', {tbname}.{char_col}, {tbname}.{char_col} )",
+                    f"length( {tbname}.{char_col} )",
+                    f"lower( {tbname}.{char_col} )",
+                    f"ltrim( {tbname}.{char_col} )",
+                    f"rtrim( {tbname}.{char_col} )",
+                    f"substr( {tbname}.{char_col}, 1 )",
+                    f"count( {tbname}.{char_col} )",
+                    f"cast( {tbname}.{char_col} as nchar(3) )",
+                    f"cast( {tbname}.{char_col} as nchar(8) )",
+                )
+            )
+            query_condition.extend( f"cast( {tbname}.{un_char_col} as binary(16) ) " for un_char_col in NUM_COL)
+            query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{char_col_2} as binary(32) ) " for char_col_2 in CHAR_COL )
+            query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{un_char_col} as binary(32) ) " for un_char_col in NUM_COL )
+
+        for num_col in NUM_COL:
+            query_condition.extend(
+                (
+                    f"{tbname}.{num_col}",
+                    f"ceil( {tbname}.{num_col} )",
+                    f"abs( {tbname}.{num_col} )",
+                    f"acos( {tbname}.{num_col} )",
+                    f"asin( {tbname}.{num_col} )",
+                    f"atan( {tbname}.{num_col} )",
+                    f"cos( {tbname}.{num_col} )",
+                    f"floor( {tbname}.{num_col} )",
+                    f"log( {tbname}.{num_col},  {tbname}.{num_col})",
+                    f"sin( {tbname}.{num_col} )",
+                    f"sqrt( {tbname}.{num_col} )",
+                    f"tan( {tbname}.{num_col} )",
+                    f"round( {tbname}.{num_col} )",
+                    f"max( {tbname}.{num_col} )",
+                    f"sum( {tbname}.{num_col} )",
+                    f"count( {tbname}.{num_col} )",
+                    f"min( {tbname}.{num_col} )",
+                )
+            )
+            query_condition.extend( f"{tbname}.{num_col} + {tbname}.{num_col_2}" for num_col_2 in NUM_COL )
+            query_condition.extend( f"{tbname}.{num_col} + {tbname}.{char_col} " for char_col in CHAR_COL )
+
+        query_condition.extend(
+            (
+                ''' "test1234!@#$%^&*():'><?/.,][}{" ''',
+                ''' "test12" ''',
+                # 1010,
+            )
+        )
+
+        return query_condition
+
+    def __join_condition(self, tb_list, filter=PRIMARY_COL, INNER=False):
+        table_reference = tb_list[0]
+        join_condition = table_reference
+        join = "inner join" if INNER else "join"
+        for i in range(len(tb_list[1:])):
+            join_condition += f" {join} {tb_list[i+1]} on {table_reference}.{filter}={tb_list[i+1]}.{filter}"
+
+        return join_condition
+
+    def __where_condition(self, col=None, tbname=None, query_conditon=None):
+        if query_conditon and isinstance(query_conditon, str):
+            if query_conditon.startswith("count"):
+                query_conditon = query_conditon[6:-1]
+            elif query_conditon.startswith("max"):
+                query_conditon = query_conditon[4:-1]
+            elif query_conditon.startswith("sum"):
+                query_conditon = query_conditon[4:-1]
+            elif query_conditon.startswith("min"):
+                query_conditon = query_conditon[4:-1]
+
+
+        if query_conditon:
+            return f" where {query_conditon} is not null"
+        if col in NUM_COL:
+            return f" where abs( {tbname}.{col} ) >= 0"
+        if col in CHAR_COL:
+            return f" where lower( {tbname}.{col} ) like 'bina%' or lower( {tbname}.{col} ) like '_cha%' "
+        if col in BOOLEAN_COL:
+            return f" where {tbname}.{col} in (false, true)  "
+        if col in TS_TYPE_COL or col in PRIMARY_COL:
+            return f" where cast( {tbname}.{col} as binary(16) ) is not null "
+
+        return ""
+
+
+    def __group_condition(self, col, having = None):
+        if isinstance(col, str):
+            if col.startswith("count"):
+                col = col[6:-1]
+            elif col.startswith("max"):
+                col = col[4:-1]
+            elif col.startswith("sum"):
+                col = col[4:-1]
+            elif col.startswith("min"):
+                col = col[4:-1]
+        return f" group by {col} having {having}" if having else f" group by {col} "
+
+    def __single_sql(self, select_clause, from_clause, where_condition="", group_condition=""):
+        if isinstance(select_clause, str) and "on" not in from_clause and select_clause.split(".")[0] != from_clause.split(".")[0]:
+            return
+        return f"select {select_clause} from {from_clause} {where_condition} {group_condition}"
+
+
+    @property
+    def __join_tblist(self):
+        return [
+            ["ct1", "ct2"],
+            ["ct1", "ct4"],
+            ["ct1", "t1"],
+            ["ct2", "ct4"],
+            ["ct2", "t1"],
+            ["ct4", "t1"],
+            # ["ct1", "ct2", "ct4"],
+            # ["ct1", "ct2", "t1"],
+            # ["ct1", "ct4", "t1"],
+            # ["ct2", "ct4", "t1"],
+            # ["ct1", "ct2", "ct4", "t1"],
+        ]
+
+    @property
+    def __tb_liast(self):
+        return [
+            "ct1",
+            "ct2",
+            "ct4",
+            "t1",
+        ]
+
+    def sql_list(self):
+        sqls = []
+        __join_tblist = self.__join_tblist
+        for join_tblist in __join_tblist:
+            for join_tb in join_tblist:
+                select_claus_list = self.__query_condition(join_tb)
+                for select_claus in select_claus_list:
+                    group_claus = self.__group_condition( col=select_claus)
+                    where_claus = self.__where_condition(query_conditon=select_claus)
+                    having_claus = self.__group_condition( col=select_claus, having=f"{select_claus} is not null")
+                    sqls.extend(
+                        (
+                            self.__single_sql(select_claus, join_tb, where_claus, group_claus),
+                            self.__single_sql(select_claus, join_tb, where_claus, having_claus),
+                            self.__single_sql(select_claus, self.__join_condition(join_tblist), where_claus, having_claus),
+                            self.__single_sql(select_claus, self.__join_condition(join_tblist, INNER=True), where_claus, having_claus),
+                            self.__single_sql(select_claus, join_tb, where_claus),
+                            self.__single_sql(select_claus, join_tb, having_claus),
+                            self.__single_sql(select_claus, join_tb, group_claus),
+                            self.__single_sql(select_claus, join_tb),
+
+                        )
+                    )
+        __no_join_tblist = self.__tb_liast
+        for tb in __no_join_tblist:
+                select_claus_list = self.__query_condition(tb)
+                for select_claus in select_claus_list:
+                    group_claus = self.__group_condition(col=select_claus)
+                    where_claus = self.__where_condition(query_conditon=select_claus)
+                    having_claus = self.__group_condition(col=select_claus, having=f"{select_claus} is not null")
+                    sqls.extend(
+                        (
+                            self.__single_sql(select_claus, join_tb, where_claus, group_claus),
+                            self.__single_sql(select_claus, join_tb, where_claus, having_claus),
+                            self.__single_sql(select_claus, join_tb, where_claus),
+                            self.__single_sql(select_claus, join_tb, group_claus),
+                            self.__single_sql(select_claus, join_tb, having_claus),
+                            self.__single_sql(select_claus, join_tb),
+                        )
+                    )
+
+        return filter(None, sqls)
+        # return list(filter(None, sqls))
+
+    def __get_type(self, col):
+        if tdSql.cursor.istype(col, "BOOL"):
+            return "BOOL"
+        if tdSql.cursor.istype(col, "INT"):
+            return "INT"
+        if tdSql.cursor.istype(col, "BIGINT"):
+            return "BIGINT"
+        if tdSql.cursor.istype(col, "TINYINT"):
+            return "TINYINT"
+        if tdSql.cursor.istype(col, "SMALLINT"):
+            return "SMALLINT"
+        if tdSql.cursor.istype(col, "FLOAT"):
+            return "FLOAT"
+        if tdSql.cursor.istype(col, "DOUBLE"):
+            return "DOUBLE"
+        if tdSql.cursor.istype(col, "BINARY"):
+            return "BINARY"
+        if tdSql.cursor.istype(col, "NCHAR"):
+            return "NCHAR"
+        if tdSql.cursor.istype(col, "TIMESTAMP"):
+            return "TIMESTAMP"
+        if tdSql.cursor.istype(col, "JSON"):
+            return "JSON"
+        if tdSql.cursor.istype(col, "TINYINT UNSIGNED"):
+            return "TINYINT UNSIGNED"
+        if tdSql.cursor.istype(col, "SMALLINT UNSIGNED"):
+            return "SMALLINT UNSIGNED"
+        if tdSql.cursor.istype(col, "INT UNSIGNED"):
+            return "INT UNSIGNED"
+        if tdSql.cursor.istype(col, "BIGINT UNSIGNED"):
+            return "BIGINT UNSIGNED"
+
+    def union_check(self):
+        sqls = self.sql_list()
+        for sql1 in sqls:
+            tdSql.query(sql1)
+            res1_type = self.__get_type(0)
+            for sql2 in sqls:
+                tdSql.query(sql2)
+                union_type = False
+                res2_type =  self.__get_type(0)
+
+                if res1_type in  ( "BIGINT" , "NCHAR" ):
+                    union_type = True
+                elif res2_type == res1_type:
+                    union_type = True
+                elif res1_type == "TIMESAMP" and res2_type not in ("BINARY", "NCHAR"):
+                    union_type = True
+                elif res1_type == "BINARY" and res2_type != "NCHAR":
+                    union_type = True
+
+                if union_type:
+                    tdSql.query(f"{sql1} union {sql2}")
+                    tdSql.checkCols(1)
+                    tdSql.query(f"{sql1} union all {sql2}")
+                    tdSql.checkCols(1)
+                else:
+                    tdSql.error(f"{sql1} union {sql2}")
+
+    def __test_error(self):
+
+
+        tdSql.error( "show tables union show tables" )
+        tdSql.error( "create table errtb1 union all create table errtb2" )
+        tdSql.error( "drop table ct1 union all drop table ct3" )
+        tdSql.error( "select c1 from ct1 union all drop table ct3" )
+        tdSql.error( "select c1 from ct1 union all '' " )
+        tdSql.error( " '' union all select c1 from ct1 " )
+        tdSql.error( "select c1 from ct1 union select c1 from ct2 union select c1 from ct4 ")
+
+    def all_test(self):
+        self.__test_error()
+        self.union_check()
+
+
+    def __create_tb(self):
+
+        tdLog.printNoPrefix("==========step1:create table")
+        create_stb_sql  =  f'''create table stb1(
+                ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
+                 {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
+                 {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp
+            ) tags (t1 int)
+            '''
+        create_ntb_sql = f'''create table t1(
+                ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
+                 {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
+                 {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp
+            )
+            '''
+        tdSql.execute(create_stb_sql)
+        tdSql.execute(create_ntb_sql)
+
+        for i in range(4):
+            tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
+            { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}
+
+    def __insert_data(self, rows):
+        now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000)
+        for i in range(rows):
+            tdSql.execute(
+                f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
+            )
+            tdSql.execute(
+                f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
+            )
+            tdSql.execute(
+                f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i},  {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
+            )
+        tdSql.execute(
+            f'''insert into ct1 values
+            ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 } )
+            ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 } )
+            '''
+        )
+
+        tdSql.execute(
+            f'''insert into ct4 values
+            ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time +  7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            (
+                { now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127,
+                { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000}
+                )
+            (
+                { now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126,
+                { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000}
+                )
+            '''
+        )
+
+        tdSql.execute(
+            f'''insert into ct2 values
+            ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            (
+                { now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126,
+                { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }
+                )
+            (
+                { now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127,
+                { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }
+                )
+            '''
+        )
+
+        for i in range(rows):
+            insert_data = f'''insert into t1 values
+                ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2},
+                "binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i } )
+                '''
+            tdSql.execute(insert_data)
+        tdSql.execute(
+            f'''insert into t1 values
+            ( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127,
+                { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 },
+                "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }
+                )
+            (
+                { now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126,
+                { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 },
+                "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }
+                )
+            '''
+        )
+
+
+    def run(self):
+        tdSql.prepare()
+
+        tdLog.printNoPrefix("==========step1:create table")
+        self.__create_tb()
+
+        tdLog.printNoPrefix("==========step2:insert data")
+        self.rows = 10
+        self.__insert_data(self.rows)
+
+        tdLog.printNoPrefix("==========step3:all check")
+        self.all_test()
+
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+
+        tdSql.execute("use db")
+
+        tdLog.printNoPrefix("==========step4:after wal, all check again ")
+        self.all_test()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/7-tmq/basic5.py

@@ -52,7 +52,7 @@ class TDTestCase:
     def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl):
         tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups))        
         tsql.execute("use %s" %dbName)
-        tsql.execute("create table %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName)
+        tsql.execute("create table  if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName)
         pre_create = "create table"
         sql = pre_create
         #tdLog.debug("doing create one  stable %s and %d  child table in %s  ..." %(stbname, count ,dbname))
@@ -345,11 +345,11 @@ class TDTestCase:
                              after starting consumer, create ctables ")
         # create and start thread
         parameterDict = {'cfg':        '',       \
-                         'dbName':     'db2',    \
+                         'dbName':     'db3',    \
                          'vgroups':    1,        \
                          'stbName':    'stb',    \
                          'ctbNum':     10,       \
-                         'rowsPerTbl': 10000,   \
+                         'rowsPerTbl': 30000,    \
                          'batchNum':   100,       \
                          'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
         parameterDict['cfg'] = cfgPath
@@ -374,22 +374,33 @@ class TDTestCase:
                 break
             else:
                 time.sleep(1)
-
+        
+        tdLog.info("create stable2 for the seconde topic")
+        parameterDict2 = {'cfg':        '',       \
+                         'dbName':     'db3',    \
+                         'vgroups':    1,        \
+                         'stbName':    'stb2',    \
+                         'ctbNum':     10,       \
+                         'rowsPerTbl': 30000,    \
+                         'batchNum':   100,       \
+                         'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
+        parameterDict2['cfg'] = cfgPath
+        tdSql.execute("create stable  if not exists %s.%s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%(parameterDict2['dbName'], parameterDict2['stbName']))
+       
         tdLog.info("create topics from super table")
-        topicFromStb = 'topic_stb_column2'
+        topicFromStb  = 'topic_stb_column3'
-        topicFromCtb = 'topic_ctb_column2'
+        topicFromStb2 = 'topic_stb_column32'
         
         tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb, parameterDict['dbName'], parameterDict['stbName']))
-        tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s_0" %(topicFromCtb, parameterDict['dbName'], parameterDict['stbName']))
+        tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb2, parameterDict2['dbName'], parameterDict2['stbName']))
         
-        time.sleep(1)
         tdSql.query("show topics")
         topic1 = tdSql.getData(0 , 0)
         topic2 = tdSql.getData(1 , 0)
         tdLog.info("show topics: %s, %s"%(topic1, topic2))
-        if topic1 != topicFromStb and topic1 != topicFromCtb:
+        if topic1 != topicFromStb and topic1 != topicFromStb2:
             tdLog.exit("topic error1") 
-        if topic2 != topicFromStb and topic2 != topicFromCtb:
+        if topic2 != topicFromStb and topic2 != topicFromStb2:
             tdLog.exit("topic error2") 
          
         tdLog.info("create consume info table and consume result table")
@@ -397,10 +408,9 @@ class TDTestCase:
         tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)"%cdbName)
         tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName)
 
-        rowsOfNewCtb = 1000
         consumerId   = 0
-        expectrowcnt = parameterDict["rowsPerTbl"]  * parameterDict["ctbNum"] + rowsOfNewCtb
+        expectrowcnt = parameterDict["rowsPerTbl"]  * parameterDict["ctbNum"] +  parameterDict2["rowsPerTbl"]  * parameterDict2["ctbNum"]
-        topicList    = topicFromStb
+        topicList    = topicFromStb + ',' + topicFromStb2
         ifcheckdata  = 0
         keyList      = 'group.id:cgrp1,\
                         enable.auto.commit:false,\
@@ -432,17 +442,13 @@ class TDTestCase:
         tdLog.info(shellCmd)
         os.system(shellCmd)        
 
-        # create new child table and insert data
+        # start the second thread to create new child table and insert data
-        newCtbName = 'newctb'
+        prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2)
-        tdSql.query("create table %s.%s using %s.%s tags(9999)"%(parameterDict["dbName"], newCtbName, parameterDict["dbName"], parameterDict["stbName"]))
+        prepareEnvThread2.start()
-        startTs = parameterDict["startTs"]
-        for j in range(rowsOfNewCtb):
-            sql = "insert into %s.%s values (%d, %d, 'tmqrow_%d') "%(parameterDict["dbName"], newCtbName, startTs + j, j, j)
-            tdSql.execute(sql)
-        tdLog.debug("insert data into new child table ............ [OK]")
 
         # wait for data ready
         prepareEnvThread.join()
+        prepareEnvThread2.join()
         
         tdLog.info("insert process end, and start to check consume result")
         while 1:
@@ -457,7 +463,7 @@ class TDTestCase:
         tdSql.checkData(0 , 3, expectrowcnt)
         
         tdSql.query("drop topic %s"%topicFromStb)
-        tdSql.query("drop topic %s"%topicFromCtb)
+        tdSql.query("drop topic %s"%topicFromStb2)
 
         tdLog.printNoPrefix("======== test case 3 end ...... ")
 
@@ -474,7 +480,7 @@ class TDTestCase:
 
         self.tmqCase1(cfgPath, buildPath)
         self.tmqCase2(cfgPath, buildPath)        
-        #self.tmqCase3(cfgPath, buildPath)
+        self.tmqCase3(cfgPath, buildPath)
 
     def stop(self):
         tdSql.close()

tests/system-test/7-tmq/subscribeDb.py

@@ -49,7 +49,38 @@ class TDTestCase:
         print(cur)
         return cur
 
-    def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg,showRow,cdbName,valgrind=0):
+    def initConsumerTable(self,cdbName='cdb'):        
+        tdLog.info("create consume database, and consume info table, and consume result table")
+        tdSql.query("create database if not exists %s vgroups 1"%(cdbName))
+        tdSql.query("drop table if exists %s.consumeinfo "%(cdbName))
+        tdSql.query("drop table if exists %s.consumeresult "%(cdbName))
+
+        tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName)
+        tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName)
+
+    def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,cdbName='cdb'):    
+        sql = "insert into %s.consumeinfo values "%cdbName
+        sql += "(now, %d, '%s', '%s', %d, %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit)
+        tdLog.info("consume info sql: %s"%sql)
+        tdSql.query(sql)
+
+    def selectConsumeResult(self,expectRows,cdbName='cdb'):
+        resultList=[]
+        while 1:
+            tdSql.query("select * from %s.consumeresult"%cdbName)
+            #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
+            if tdSql.getRows() == expectRows:
+                break
+            else:
+                time.sleep(5)
+        
+        for i in range(expectRows):
+            tdLog.info ("consume id: %d, consume msgs: %d, consume rows: %d"%(tdSql.getData(i , 1), tdSql.getData(i , 2), tdSql.getData(i , 3)))
+            resultList.append(tdSql.getData(i , 3))
+        
+        return resultList
+
+    def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0):
         shellCmd = 'nohup '
         if valgrind == 1:
             logFile = cfgPath + '/../log/valgrind-tmq.log'
@@ -58,7 +89,7 @@ class TDTestCase:
         
         shellCmd += buildPath + '/build/bin/tmq_sim -c ' + cfgPath
         shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) 
-        shellCmd += "> /dev/null 2>&1 &"        
+        shellCmd += "> /dev/null 2>&1 &"
         tdLog.info(shellCmd)
         os.system(shellCmd)
 
@@ -87,6 +118,8 @@ class TDTestCase:
         pre_insert = "insert into "
         sql = pre_insert
 
+        t = time.time()
+        startTs = int(round(t * 1000))
         #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
         for i in range(ctbNum):
             sql += " %s_%d values "%(stbName,i)
@@ -127,7 +160,7 @@ class TDTestCase:
         return
 
     def tmqCase1(self, cfgPath, buildPath):
-        tdLog.printNoPrefix("======== test case 1: Produce while one consume to subscribe one db")
+        tdLog.printNoPrefix("======== test case 1: Produce while one consume to subscribe one db, inclue 1 stb")
         tdLog.info("step 1: create database, stb, ctb and insert data")
         # create and start thread
         parameterDict = {'cfg':        '',       \
@@ -135,11 +168,13 @@ class TDTestCase:
                          'vgroups':    4,        \
                          'stbName':    'stb',    \
                          'ctbNum':     10,       \
-                         'rowsPerTbl': 100000,    \
+                         'rowsPerTbl': 10000,    \
-                         'batchNum':   200,      \
+                         'batchNum':   100,      \
                          'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
         parameterDict['cfg'] = cfgPath
 
+        self.initConsumerTable()
+
         tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups']))
 
         prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict)
@@ -149,23 +184,16 @@ class TDTestCase:
         topicName1 = 'topic_db1'
         
         tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName']))
-        
-        tdLog.info("create consume info table and consume result table")
-        cdbName = parameterDict["dbName"]
-        tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)"%cdbName)
-        tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName)
-
         consumerId   = 0
         expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"]
         topicList    = topicName1
         ifcheckdata  = 0
+        ifManualCommit = 0
         keyList      = 'group.id:cgrp1,\
                         enable.auto.commit:false,\
                         auto.commit.interval.ms:6000,\
                         auto.offset.reset:earliest'
-        sql = "insert into %s.consumeinfo values "%cdbName
+        self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
-        sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata)
-        tdSql.query(sql)
         
         event.wait()
 
@@ -173,32 +201,28 @@ class TDTestCase:
         pollDelay = 5
         showMsg   = 1
         showRow   = 1
-        self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow, cdbName)
+        self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
 
         # wait for data ready
         prepareEnvThread.join()
         
         tdLog.info("insert process end, and start to check consume result")
-        while 1:
+        expectRows = 1
-            tdSql.query("select * from %s.consumeresult"%cdbName)
+        resultList = self.selectConsumeResult(expectRows)
-            #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
+        totalConsumeRows = 0
-            if tdSql.getRows() == 1:
+        for i in range(expectRows):
-                break
+            totalConsumeRows += resultList[i]
-            else:
+        
-                time.sleep(5)
+        if totalConsumeRows != expectrowcnt:
-
+            tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
-        tdLog.info("consumer result: %d, %d"%(tdSql.getData(0 , 2), tdSql.getData(0 , 3)))
+            tdLog.exit("tmq consume rows error!")
-        tdSql.checkData(0 , 1, consumerId)
-         # mulit rows and mulit tables in one sql, this num of msg is not sure
-        #tdSql.checkData(0 , 2, expectmsgcnt)
-        tdSql.checkData(0 , 3, expectrowcnt+1)
 
         tdSql.query("drop topic %s"%topicName1)
 
         tdLog.printNoPrefix("======== test case 1 end ...... ")
 
     def tmqCase2(self, cfgPath, buildPath):
-        tdLog.printNoPrefix("======== test case 2: Produce while two consumers to subscribe one db")
+        tdLog.printNoPrefix("======== test case 2: Produce while two consumers to subscribe one db, inclue 1 stb")
         tdLog.info("step 1: create database, stb, ctb and insert data")
         # create and start thread
         parameterDict = {'cfg':        '',       \
@@ -206,11 +230,13 @@ class TDTestCase:
                          'vgroups':    4,        \
                          'stbName':    'stb',    \
                          'ctbNum':     10,       \
-                         'rowsPerTbl': 100000,    \
+                         'rowsPerTbl': 10000,    \
                          'batchNum':   100,      \
                          'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
         parameterDict['cfg'] = cfgPath
 
+        self.initConsumerTable()
+
         tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups']))
 
         prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict)
@@ -221,27 +247,19 @@ class TDTestCase:
         
         tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName']))
         
-        tdLog.info("create consume info table and consume result table")
-        cdbName = parameterDict["dbName"]
-        tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)"%cdbName)
-        tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName)
-
         consumerId   = 0
         expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"]
         topicList    = topicName1
         ifcheckdata  = 0
+        ifManualCommit = 0
         keyList      = 'group.id:cgrp1,\
                         enable.auto.commit:false,\
                         auto.commit.interval.ms:6000,\
                         auto.offset.reset:earliest'
-        sql = "insert into %s.consumeinfo values "%cdbName
+        self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
-        sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata)
+
-        tdSql.query(sql)
-        
         consumerId   = 1
-        sql = "insert into %s.consumeinfo values "%cdbName
+        self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
-        sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata)
-        tdSql.query(sql)
         
         event.wait()
 
@@ -249,30 +267,20 @@ class TDTestCase:
         pollDelay = 5
         showMsg   = 1
         showRow   = 1        
-        self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow, cdbName)
+        self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
 
         # wait for data ready
         prepareEnvThread.join()
         
         tdLog.info("insert process end, and start to check consume result")
-        while 1:
+        expectRows = 2
-            tdSql.query("select * from %s.consumeresult"%cdbName)
+        resultList = self.selectConsumeResult(expectRows)
-            #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
+        totalConsumeRows = 0
-            if tdSql.getRows() == 2:
+        for i in range(expectRows):
-                break
+            totalConsumeRows += resultList[i]
-            else:
+        
-                time.sleep(5)
+        if totalConsumeRows != expectrowcnt:
-
+            tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
-        consumerId0     = tdSql.getData(0 , 1)
-        consumerId1     = tdSql.getData(1 , 1)
-        actConsumeRows0 = tdSql.getData(0 , 3)
-        actConsumeRows1 = tdSql.getData(1 , 3)
-
-        tdLog.info("consumer %d rows: %d"%(consumerId0, actConsumeRows0))
-        tdLog.info("consumer %d rows: %d"%(consumerId1, actConsumeRows1))
-
-        totalConsumeRows = actConsumeRows0 + actConsumeRows1
-        if totalConsumeRows != expectrowcnt + 2:
             tdLog.exit("tmq consume rows error!")
 
         tdSql.query("drop topic %s"%topicName1)
@@ -288,11 +296,13 @@ class TDTestCase:
                          'vgroups':    4,        \
                          'stbName':    'stb',    \
                          'ctbNum':     10,       \
-                         'rowsPerTbl': 100000,    \
+                         'rowsPerTbl': 10000,    \
                          'batchNum':   100,      \
                          'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
         parameterDict['cfg'] = cfgPath
 
+        self.initConsumerTable()
+
         tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups']))
 
         prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict)
@@ -303,7 +313,7 @@ class TDTestCase:
                          'vgroups':    4,        \
                          'stbName':    'stb2',    \
                          'ctbNum':     10,       \
-                         'rowsPerTbl': 100000,    \
+                         'rowsPerTbl': 10000,    \
                          'batchNum':   100,      \
                          'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
         parameterDict['cfg'] = cfgPath
@@ -316,65 +326,377 @@ class TDTestCase:
         
         tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName']))
         
-        tdLog.info("create consume info table and consume result table")
-        cdbName = parameterDict["dbName"]
-        tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)"%cdbName)
-        tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName)
-
         consumerId   = 0
         expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] +  parameterDict2["rowsPerTbl"] * parameterDict2["ctbNum"]
         topicList    = topicName1
         ifcheckdata  = 0
+        ifManualCommit = 0
         keyList      = 'group.id:cgrp1,\
                         enable.auto.commit:false,\
                         auto.commit.interval.ms:6000,\
                         auto.offset.reset:earliest'
-        sql = "insert into %s.consumeinfo values "%cdbName
+        self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
-        sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata)
-        tdSql.query(sql)
         
         # consumerId   = 1
-        # sql = "insert into %s.consumeinfo values "%cdbName
+        # self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
-        # sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata)
-        # tdSql.query(sql)
         
         event.wait()
 
         tdLog.info("start consume processor")
         pollDelay = 5
         showMsg   = 1
-        showRow   = 1        
+        showRow   = 1
-        self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow, cdbName)
+        self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
 
         # wait for data ready
         prepareEnvThread.join()        
         prepareEnvThread2.join()
         
         tdLog.info("insert process end, and start to check consume result")
-        while 1:
+        expectRows = 1
-            tdSql.query("select * from %s.consumeresult"%cdbName)
+        resultList = self.selectConsumeResult(expectRows)
-            #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
+        totalConsumeRows = 0
-            if tdSql.getRows() == 1:
+        for i in range(expectRows):
-                break
+            totalConsumeRows += resultList[i]
-            else:
+        
-                time.sleep(5)
+        if totalConsumeRows != expectrowcnt:
-
+            tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
-        consumerId0     = tdSql.getData(0 , 1)
-        #consumerId1     = tdSql.getData(1 , 1)
-        actConsumeRows0 = tdSql.getData(0 , 3)
-        #actConsumeRows1 = tdSql.getData(1 , 3)
-
-        tdLog.info("consumer %d rows: %d"%(consumerId0, actConsumeRows0))
-        #tdLog.info("consumer %d rows: %d"%(consumerId1, actConsumeRows1))
-
-        #totalConsumeRows = actConsumeRows0 + actConsumeRows1
-        if actConsumeRows0 != expectrowcnt + 1:
             tdLog.exit("tmq consume rows error!")
 
         tdSql.query("drop topic %s"%topicName1)
 
         tdLog.printNoPrefix("======== test case 3 end ...... ")
 
+    def tmqCase4(self, cfgPath, buildPath):
+        tdLog.printNoPrefix("======== test case 4: Produce while two consumers to subscribe one db, include 2 stb")
+        tdLog.info("step 1: create database, stb, ctb and insert data")
+        # create and start thread
+        parameterDict = {'cfg':        '',       \
+                         'dbName':     'db4',    \
+                         'vgroups':    4,        \
+                         'stbName':    'stb',    \
+                         'ctbNum':     10,       \
+                         'rowsPerTbl': 10000,    \
+                         'batchNum':   100,      \
+                         'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
+        parameterDict['cfg'] = cfgPath
+
+        self.initConsumerTable()
+
+        tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups']))
+
+        prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict)
+        prepareEnvThread.start()
+
+        parameterDict2 = {'cfg':        '',       \
+                         'dbName':     'db4',    \
+                         'vgroups':    4,        \
+                         'stbName':    'stb2',    \
+                         'ctbNum':     10,       \
+                         'rowsPerTbl': 10000,    \
+                         'batchNum':   100,      \
+                         'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
+        parameterDict['cfg'] = cfgPath
+
+        prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2)
+        prepareEnvThread2.start()
+
+        tdLog.info("create topics from db")
+        topicName1 = 'topic_db1'
+        
+        tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName']))
+        
+        consumerId   = 0
+        expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] +  parameterDict2["rowsPerTbl"] * parameterDict2["ctbNum"]
+        topicList    = topicName1
+        ifcheckdata  = 0
+        ifManualCommit = 0
+        keyList      = 'group.id:cgrp1,\
+                        enable.auto.commit:false,\
+                        auto.commit.interval.ms:6000,\
+                        auto.offset.reset:earliest'
+        self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+        
+        consumerId   = 1
+        self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+        
+        event.wait()
+
+        tdLog.info("start consume processor")
+        pollDelay = 5
+        showMsg   = 1
+        showRow   = 1
+        self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
+
+        # wait for data ready
+        prepareEnvThread.join()        
+        prepareEnvThread2.join()
+        
+        tdLog.info("insert process end, and start to check consume result")
+        expectRows = 2
+        resultList = self.selectConsumeResult(expectRows)
+        totalConsumeRows = 0
+        for i in range(expectRows):
+            totalConsumeRows += resultList[i]
+        
+        if totalConsumeRows != expectrowcnt:
+            tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
+            tdLog.exit("tmq consume rows error!")
+
+        tdSql.query("drop topic %s"%topicName1)
+
+        tdLog.printNoPrefix("======== test case 4 end ...... ")
+
+    def tmqCase5(self, cfgPath, buildPath):
+        tdLog.printNoPrefix("======== test case 5: Produce while two consumers to subscribe one db, firstly create one stb, after start consume create other stb")
+        tdLog.info("step 1: create database, stb, ctb and insert data")
+        # create and start thread
+        parameterDict = {'cfg':        '',       \
+                         'dbName':     'db5',    \
+                         'vgroups':    4,        \
+                         'stbName':    'stb',    \
+                         'ctbNum':     10,       \
+                         'rowsPerTbl': 10000,    \
+                         'batchNum':   100,      \
+                         'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
+        parameterDict['cfg'] = cfgPath
+
+        self.initConsumerTable()
+
+        tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups']))
+
+        prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict)
+        prepareEnvThread.start()
+
+        parameterDict2 = {'cfg':        '',       \
+                         'dbName':     'db5',    \
+                         'vgroups':    4,        \
+                         'stbName':    'stb2',    \
+                         'ctbNum':     10,       \
+                         'rowsPerTbl': 10000,    \
+                         'batchNum':   100,      \
+                         'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
+        parameterDict['cfg'] = cfgPath
+
+        tdLog.info("create topics from db")
+        topicName1 = 'topic_db1'
+        
+        tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName']))
+        
+        consumerId   = 0
+        expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] +  parameterDict2["rowsPerTbl"] * parameterDict2["ctbNum"]
+        topicList    = topicName1
+        ifcheckdata  = 0
+        ifManualCommit = 0
+        keyList      = 'group.id:cgrp1,\
+                        enable.auto.commit:false,\
+                        auto.commit.interval.ms:6000,\
+                        auto.offset.reset:earliest'
+        self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+        
+        consumerId   = 1
+        self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+        
+        event.wait()
+
+        tdLog.info("start consume processor")
+        pollDelay = 5
+        showMsg   = 1
+        showRow   = 1
+        self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
+
+        prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2)
+        prepareEnvThread2.start()
+
+        # wait for data ready
+        prepareEnvThread.join()        
+        prepareEnvThread2.join()
+        
+        tdLog.info("insert process end, and start to check consume result")
+        expectRows = 2
+        resultList = self.selectConsumeResult(expectRows)
+        totalConsumeRows = 0
+        for i in range(expectRows):
+            totalConsumeRows += resultList[i]
+        
+        if totalConsumeRows != expectrowcnt:
+            tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
+            tdLog.exit("tmq consume rows error!")
+
+        tdSql.query("drop topic %s"%topicName1)
+
+        tdLog.printNoPrefix("======== test case 5 end ...... ")
+
+    def tmqCase6(self, cfgPath, buildPath):
+        tdLog.printNoPrefix("======== test case 6: Produce while one consumers to subscribe tow topic, Each contains one db")
+        tdLog.info("step 1: create database, stb, ctb and insert data")
+        # create and start thread
+        parameterDict = {'cfg':        '',       \
+                         'dbName':     'db60',    \
+                         'vgroups':    4,        \
+                         'stbName':    'stb',    \
+                         'ctbNum':     10,       \
+                         'rowsPerTbl': 10000,    \
+                         'batchNum':   100,      \
+                         'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
+        parameterDict['cfg'] = cfgPath
+
+        self.initConsumerTable()
+
+        tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups']))
+
+        prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict)
+        prepareEnvThread.start()
+
+        parameterDict2 = {'cfg':        '',       \
+                         'dbName':     'db61',    \
+                         'vgroups':    4,        \
+                         'stbName':    'stb2',    \
+                         'ctbNum':     10,       \
+                         'rowsPerTbl': 10000,    \
+                         'batchNum':   100,      \
+                         'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
+        parameterDict['cfg'] = cfgPath
+
+        tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict2['dbName'], parameterDict2['vgroups']))
+
+        prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2)
+        prepareEnvThread2.start()
+
+        tdLog.info("create topics from db")
+        topicName1 = 'topic_db60'
+        topicName2 = 'topic_db61'
+        
+        tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName']))        
+        tdSql.execute("create topic %s as %s" %(topicName2, parameterDict2['dbName']))
+        
+        consumerId   = 0
+        expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] +  parameterDict2["rowsPerTbl"] * parameterDict2["ctbNum"]
+        topicList    = topicName1 + ',' + topicName2
+        ifcheckdata  = 0
+        ifManualCommit = 0
+        keyList      = 'group.id:cgrp1,\
+                        enable.auto.commit:false,\
+                        auto.commit.interval.ms:6000,\
+                        auto.offset.reset:earliest'
+        self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+        
+        #consumerId   = 1
+        #self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+        
+        event.wait()
+
+        tdLog.info("start consume processor")
+        pollDelay = 5
+        showMsg   = 1
+        showRow   = 1
+        self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
+
+        # wait for data ready
+        prepareEnvThread.join()        
+        prepareEnvThread2.join()
+        
+        tdLog.info("insert process end, and start to check consume result")
+        expectRows = 1
+        resultList = self.selectConsumeResult(expectRows)
+        totalConsumeRows = 0
+        for i in range(expectRows):
+            totalConsumeRows += resultList[i]
+        
+        if totalConsumeRows != expectrowcnt:
+            tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
+            tdLog.exit("tmq consume rows error!")
+
+        tdSql.query("drop topic %s"%topicName1)
+        tdSql.query("drop topic %s"%topicName2)
+
+        tdLog.printNoPrefix("======== test case 6 end ...... ")
+
+    def tmqCase7(self, cfgPath, buildPath):
+        tdLog.printNoPrefix("======== test case 7: Produce while two consumers to subscribe tow topic, Each contains one db")
+        tdLog.info("step 1: create database, stb, ctb and insert data")
+        # create and start thread
+        parameterDict = {'cfg':        '',       \
+                         'dbName':     'db70',    \
+                         'vgroups':    4,        \
+                         'stbName':    'stb',    \
+                         'ctbNum':     10,       \
+                         'rowsPerTbl': 10000,    \
+                         'batchNum':   100,      \
+                         'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
+        parameterDict['cfg'] = cfgPath
+
+        self.initConsumerTable()
+
+        tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups']))
+
+        prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict)
+        prepareEnvThread.start()
+
+        parameterDict2 = {'cfg':        '',       \
+                         'dbName':     'db71',    \
+                         'vgroups':    4,        \
+                         'stbName':    'stb2',    \
+                         'ctbNum':     10,       \
+                         'rowsPerTbl': 10000,    \
+                         'batchNum':   100,      \
+                         'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
+        parameterDict['cfg'] = cfgPath
+
+        tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict2['dbName'], parameterDict2['vgroups']))
+
+        prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2)
+        prepareEnvThread2.start()
+
+        tdLog.info("create topics from db")
+        topicName1 = 'topic_db60'
+        topicName2 = 'topic_db61'
+        
+        tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName']))        
+        tdSql.execute("create topic %s as %s" %(topicName2, parameterDict2['dbName']))
+        
+        consumerId   = 0
+        expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] +  parameterDict2["rowsPerTbl"] * parameterDict2["ctbNum"]
+        topicList    = topicName1 + ',' + topicName2
+        ifcheckdata  = 0
+        ifManualCommit = 0
+        keyList      = 'group.id:cgrp1,\
+                        enable.auto.commit:false,\
+                        auto.commit.interval.ms:6000,\
+                        auto.offset.reset:earliest'
+        self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+        
+        consumerId   = 1
+        self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+        
+        event.wait()
+
+        tdLog.info("start consume processor")
+        pollDelay = 5
+        showMsg   = 1
+        showRow   = 1
+        self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
+
+        # wait for data ready
+        prepareEnvThread.join()        
+        prepareEnvThread2.join()
+        
+        tdLog.info("insert process end, and start to check consume result")
+        expectRows = 2
+        resultList = self.selectConsumeResult(expectRows)
+        totalConsumeRows = 0
+        for i in range(expectRows):
+            totalConsumeRows += resultList[i]
+        
+        if totalConsumeRows != expectrowcnt:
+            tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
+            tdLog.exit("tmq consume rows error!")
+
+        tdSql.query("drop topic %s"%topicName1)
+        tdSql.query("drop topic %s"%topicName2)
+
+        tdLog.printNoPrefix("======== test case 7 end ...... ")
+
     def run(self):
         tdSql.prepare()
 
@@ -387,8 +709,13 @@ class TDTestCase:
         tdLog.info("cfgPath: %s" % cfgPath)
 
         #self.tmqCase1(cfgPath, buildPath)
-        self.tmqCase2(cfgPath, buildPath) 
+        #self.tmqCase2(cfgPath, buildPath) 
         #self.tmqCase3(cfgPath, buildPath)
+        self.tmqCase4(cfgPath, buildPath)
+        self.tmqCase5(cfgPath, buildPath)
+        self.tmqCase6(cfgPath, buildPath)
+        self.tmqCase7(cfgPath, buildPath)
+
 
     def stop(self):
         tdSql.close()

tests/system-test/fulltest.sh

@@ -53,6 +53,9 @@ python3 ./test.py -f 2-query/tan.py
 python3 ./test.py -f 2-query/arcsin.py
 python3 ./test.py -f 2-query/arccos.py
 python3 ./test.py -f 2-query/arctan.py
-# python3 ./test.py -f 2-query/query_cols_tags_and_or.py
+python3 ./test.py -f 2-query/query_cols_tags_and_or.py
+python3 ./test.py -f 2-query/nestedQuery.py
 
 python3 ./test.py -f 7-tmq/basic5.py
+python3 ./test.py -f 7-tmq/subscribeDb.py
+

tests/test/c/tmqSim.c

@@ -37,9 +37,10 @@ typedef struct {
   TdThread thread;
   int32_t  consumerId;
 
-  int32_t  autoCommitIntervalMs;  // 1000 ms  
+  int32_t  ifManualCommit;
-  char     autoCommit[8];         // true, false
+  //int32_t  autoCommitIntervalMs;  // 1000 ms  
-  char     autoOffsetRest[16];    // none, earliest, latest
+  //char     autoCommit[8];         // true, false
+  //char     autoOffsetRest[16];    // none, earliest, latest
 
   int32_t ifCheckData;
   int64_t expectMsgCnt;
@@ -136,9 +137,9 @@ void saveConfigToLogFile() {
 
   for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) {
     taosFprintfFile(g_fp, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId);
-	taosFprintfFile(g_fp, "  auto commit:              %s\n", g_stConfInfo.stThreads[i].autoCommit);
+	//taosFprintfFile(g_fp, "  auto commit:              %s\n", g_stConfInfo.stThreads[i].autoCommit);
-	taosFprintfFile(g_fp, "  auto commit interval ms:  %d\n", g_stConfInfo.stThreads[i].autoCommitIntervalMs);
+	//taosFprintfFile(g_fp, "  auto commit interval ms:  %d\n", g_stConfInfo.stThreads[i].autoCommitIntervalMs);
-	taosFprintfFile(g_fp, "  auto offset rest:         %s\n", g_stConfInfo.stThreads[i].autoOffsetRest);
+	//taosFprintfFile(g_fp, "  auto offset rest:         %s\n", g_stConfInfo.stThreads[i].autoOffsetRest);
     taosFprintfFile(g_fp, "  Topics: ");
     for (int j = 0; j < g_stConfInfo.stThreads[i].numOfTopic; j++) {
       taosFprintfFile(g_fp, "%s, ", g_stConfInfo.stThreads[i].topics[j]);
@@ -232,13 +233,18 @@ static int32_t msg_process(TAOS_RES* msg, int64_t msgIndex, int32_t threadLable)
 
   while (1) {
     TAOS_ROW row = taos_fetch_row(msg);
-    if (row == NULL) break;
+
-    if (0 != g_stConfInfo.showRowFlag) {
+	if (row == NULL) break;
-      TAOS_FIELD* fields = taos_fetch_fields(msg);
+
-      int32_t     numOfFields = taos_field_count(msg);
+	TAOS_FIELD* fields = taos_fetch_fields(msg);
-      taos_print_row(buf, row, fields, numOfFields);
+    int32_t     numOfFields = taos_field_count(msg);
+	
+    taos_print_row(buf, row, fields, numOfFields);
+	
+    if (0 != g_stConfInfo.showRowFlag) {  
       taosFprintfFile(g_fp, "rows[%d]: %s\n", totalRows, buf);
     }
+	
     totalRows++;
   }
 
@@ -316,6 +322,8 @@ int32_t saveConsumeResult(SThreadInfo* pInfo) {
   sprintf(sqlStr, "insert into %s.consumeresult values (now, %d, %" PRId64 ", %" PRId64 ", %d)", g_stConfInfo.cdbName,
           pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt, pInfo->checkresult);
 
+  taosFprintfFile(g_fp, "== save result sql: %s \n", sqlStr);
+  
   TAOS_RES* pRes = taos_query(pConn, sqlStr);
   if (taos_errno(pRes) != 0) {
     pError("error in save consumeinfo, reason:%s\n", taos_errstr(pRes));
@@ -384,8 +392,12 @@ void* consumeThreadFunc(void* param) {
 
   loop_consume(pInfo);
 
-  tmq_commit(pInfo->tmq, NULL, 0);
+  if (pInfo->ifManualCommit) {
-
+    taosFprintfFile(g_fp, "tmq_commit() manual commit when consume end.\n");
+  	pPrint("tmq_commit() manual commit when consume end.\n");
+    tmq_commit(pInfo->tmq, NULL, 0);
+  }
+  
   err = tmq_unsubscribe(pInfo->tmq);
   if (err) {
     pError("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err));
@@ -470,9 +482,9 @@ int32_t getConsumeInfo() {
     int32_t* lengths = taos_fetch_lengths(pRes);
 
     // set default value
-    g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = 5000;
+    //g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = 5000;
-    memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, "true", strlen("true"));
+    //memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, "true", strlen("true"));
-    memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, "earlieast", strlen("earlieast"));  
+    //memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, "earlieast", strlen("earlieast"));  
 
     for (int i = 0; i < num_fields; ++i) {
       if (row[i] == NULL || 0 == i) {
@@ -489,12 +501,8 @@ int32_t getConsumeInfo() {
         g_stConfInfo.stThreads[numOfThread].expectMsgCnt = *((int64_t*)row[i]);
       } else if ((5 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) {
         g_stConfInfo.stThreads[numOfThread].ifCheckData = *((int32_t*)row[i]);
-      } else if ((6 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) {		
+      } else if ((6 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) {
-        memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, row[i], lengths[i]);
+        g_stConfInfo.stThreads[numOfThread].ifManualCommit = *((int32_t*)row[i]);
-      } else if ((7 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) {
-        g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = *((int32_t*)row[i]);
-      } else if ((8 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) {
-        memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, row[i], lengths[i]);
       }
     }
     numOfThread++;