fix:bugs in get_assignment

docs/en/07-develop/07-tmq.mdx

@@ -81,10 +81,6 @@ Set<String> subscription() throws SQLException;
 
 ConsumerRecords<V> poll(Duration timeout) throws SQLException;
 
-void commitAsync();
-
-void commitAsync(OffsetCommitCallback callback);
-
 void commitSync() throws SQLException;
 
 void close() throws SQLException;

docs/en/14-reference/03-connector/04-java.mdx

@@ -36,15 +36,16 @@ REST connection supports all platforms that can run Java.
 
 | taos-jdbcdriver version |                                                                   major changes                                                                    | TDengine version |
 | :---------------------: | :------------------------------------------------------------------------------------------------------------------------------------------------: | :--------------: |
+|          3.2.4          |                     Subscription add the enable.auto.commit parameter and the unsubscribe() method in the WebSocket connection                     | 3.0.5.0 or later |
 |          3.2.3          |                                                 Fixed resultSet data parsing failure in some cases                                                 | 3.0.5.0 or later |
-|          3.2.2          |                                                           subscription add seek function                                                           | 3.0.5.0 or later |
+|          3.2.2          |                                                           Subscription add seek function                                                           | 3.0.5.0 or later |
 |          3.2.1          |                                      JDBC REST connection supports schemaless/prepareStatement over WebSocket                                      | 3.0.3.0 or later |
 |          3.2.0          |                                                          This version has been deprecated                                                          |        -         |
 |          3.1.0          |                                             JDBC REST connection supports subscription over WebSocket                                              |        -         |
 |      3.0.1 - 3.0.4      | fix the resultSet data is parsed incorrectly sometimes. 3.0.1 is compiled on JDK 11, you are advised to use other version in the JDK 8 environment |        -         |
 |          3.0.0          |                                                              Support for TDengine 3.0                                                              | 3.0.0.0 or later |
-|         2.0.42          |                                             fix wasNull interface return value in WebSocket connection                                             |        -         |
+|         2.0.42          |                                             Fix wasNull interface return value in WebSocket connection                                             |        -         |
-|         2.0.41          |                                           fix decode method of username and password in REST connection                                            |        -         |
+|         2.0.41          |                                           Fix decode method of username and password in REST connection                                            |        -         |
 |     2.0.39 - 2.0.40     |                                                   Add REST connection/request timeout parameters                                                   |        -         |
 |         2.0.38          |                                                    JDBC REST connections add bulk pull function                                                    |        -         |
 |         2.0.37          |                                                                 Support json tags                                                                  |        -         |

docs/examples/python/tmq_assignment_example.py

@@ -55,4 +55,4 @@ def taos_get_assignment_and_seek_demo():
 
 
 if __name__ == '__main__':
-    taosws_get_assignment_and_seek_demo()
+    taos_get_assignment_and_seek_demo()

docs/zh/07-develop/07-tmq.mdx

@@ -81,10 +81,6 @@ Set<String> subscription() throws SQLException;
 
 ConsumerRecords<V> poll(Duration timeout) throws SQLException;
 
-void commitAsync();
-
-void commitAsync(OffsetCommitCallback callback);
-
 void commitSync() throws SQLException;
 
 void close() throws SQLException;

docs/zh/08-connector/14-java.mdx

@@ -36,6 +36,7 @@ REST 连接支持所有能运行 Java 的平台。
 
 | taos-jdbcdriver 版本 |                                                                        主要变化                                                                        |   TDengine 版本    |
 | :------------------: | :----------------------------------------------------------------------------------------------------------------------------------------------------: | :----------------: |
+|        3.2.4         |                                   数据订阅在 WebSocket 连接下增加 enable.auto.commit 参数，以及 unsubscribe() 方法。                                   |         -          |
 |        3.2.3         |                                                         修复 ResultSet 在一些情况数据解析失败                                                          |         -          |
 |        3.2.2         |                                                           新增功能：数据订阅支持 seek 功能。                                                           | 3.0.5.0 及更高版本 |
 |        3.2.1         | 新增功能：WebSocket 连接支持 schemaless 与 prepareStatement 写入。变更：consumer poll 返回结果集为 ConsumerRecord，可通过 value() 获取指定结果集数据。 | 3.0.3.0 及更高版本 |

include/libs/transport/trpc.h

@@ -46,6 +46,7 @@ typedef struct SRpcHandleInfo {
   int8_t  noResp;         // has response or not(default 0, 0: resp, 1: no resp)
   int8_t  persistHandle;  // persist handle or not
   int8_t  hasEpSet;
+  int32_t cliVer;
 
   // app info
   void *ahandle;  // app handle set by client
@@ -83,6 +84,7 @@ typedef struct SRpcInit {
   int32_t  sessions;      // number of sessions allowed
   int8_t   connType;      // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS
   int32_t  idleTime;      // milliseconds, 0 means idle timer is disabled
+  int32_t  compatibilityVer;
 
   int32_t retryMinInterval;  // retry init interval
   int32_t retryStepFactor;   // retry interval factor

source/client/src/clientEnv.c

@@ -29,6 +29,7 @@
 #include "trpc.h"
 #include "tsched.h"
 #include "ttime.h"
+#include "tversion.h"
 
 #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
 #include "cus_name.h"
@@ -111,7 +112,8 @@ static void deregisterRequest(SRequestObj *pRequest) {
     atomic_add_fetch_64((int64_t *)&pActivity->numOfSlowQueries, 1);
     if (tsSlowLogScope & reqType) {
       taosPrintSlowLog("PID:%d, Conn:%u, QID:0x%" PRIx64 ", Start:%" PRId64 ", Duration:%" PRId64 "us, SQL:%s",
-        taosGetPId(), pTscObj->connId, pRequest->requestId, pRequest->metric.start, duration, pRequest->sqlstr);
+                       taosGetPId(), pTscObj->connId, pRequest->requestId, pRequest->metric.start, duration,
+                       pRequest->sqlstr);
     }
   }
 
@@ -175,6 +177,8 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) {
   rpcInit.connLimitNum = connLimitNum;
   rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
 
+  taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
+
   void *pDnodeConn = rpcOpen(&rpcInit);
   if (pDnodeConn == NULL) {
     tscError("failed to init connection to server");
@@ -358,17 +362,16 @@ int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, ri
 
 int32_t removeRequest(int64_t rid) { return taosRemoveRef(clientReqRefPool, rid); }
 
-
 void destroySubRequests(SRequestObj *pRequest) {
-  int32_t reqIdx = -1;
+  int32_t      reqIdx = -1;
   SRequestObj *pReqList[16] = {NULL};
-  uint64_t tmpRefId = 0;
+  uint64_t     tmpRefId = 0;
 
   if (pRequest->relation.userRefId && pRequest->relation.userRefId != pRequest->self) {
     return;
   }
-  
+
-  SRequestObj* pTmp = pRequest;
+  SRequestObj *pTmp = pRequest;
   while (pTmp->relation.prevRefId) {
     tmpRefId = pTmp->relation.prevRefId;
     pTmp = acquireRequest(tmpRefId);
@@ -376,9 +379,9 @@ void destroySubRequests(SRequestObj *pRequest) {
       pReqList[++reqIdx] = pTmp;
       releaseRequest(tmpRefId);
     } else {
-      tscError("0x%" PRIx64 ", prev req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pTmp->self,
+      tscError("0x%" PRIx64 ", prev req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pTmp->self, tmpRefId,
-               tmpRefId, pTmp->requestId);
+               pTmp->requestId);
-      break;         
+      break;
     }
   }
 
@@ -391,16 +394,15 @@ void destroySubRequests(SRequestObj *pRequest) {
     pTmp = acquireRequest(tmpRefId);
     if (pTmp) {
       tmpRefId = pTmp->relation.nextRefId;
-      removeRequest(pTmp->self);      
+      removeRequest(pTmp->self);
       releaseRequest(pTmp->self);
     } else {
       tscError("0x%" PRIx64 " is not there", tmpRefId);
-      break;         
+      break;
     }
   }
 }
 
-
 void doDestroyRequest(void *p) {
   if (NULL == p) {
     return;
@@ -412,7 +414,7 @@ void doDestroyRequest(void *p) {
   tscTrace("begin to destroy request %" PRIx64 " p:%p", reqId, pRequest);
 
   destroySubRequests(pRequest);
-  
+
   taosHashRemove(pRequest->pTscObj->pRequests, &pRequest->self, sizeof(pRequest->self));
 
   schedulerFreeJob(&pRequest->body.queryJob, 0);
@@ -473,15 +475,15 @@ void taosStopQueryImpl(SRequestObj *pRequest) {
 }
 
 void stopAllQueries(SRequestObj *pRequest) {
-  int32_t reqIdx = -1;
+  int32_t      reqIdx = -1;
   SRequestObj *pReqList[16] = {NULL};
-  uint64_t tmpRefId = 0;
+  uint64_t     tmpRefId = 0;
 
   if (pRequest->relation.userRefId && pRequest->relation.userRefId != pRequest->self) {
     return;
   }
-  
+
-  SRequestObj* pTmp = pRequest;
+  SRequestObj *pTmp = pRequest;
   while (pTmp->relation.prevRefId) {
     tmpRefId = pTmp->relation.prevRefId;
     pTmp = acquireRequest(tmpRefId);
@@ -489,9 +491,9 @@ void stopAllQueries(SRequestObj *pRequest) {
       pReqList[++reqIdx] = pTmp;
       releaseRequest(tmpRefId);
     } else {
-      tscError("0x%" PRIx64 ", prev req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pTmp->self,
+      tscError("0x%" PRIx64 ", prev req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pTmp->self, tmpRefId,
-               tmpRefId, pTmp->requestId);
+               pTmp->requestId);
-      break;         
+      break;
     }
   }
 
@@ -510,12 +512,11 @@ void stopAllQueries(SRequestObj *pRequest) {
       releaseRequest(pTmp->self);
     } else {
       tscError("0x%" PRIx64 " is not there", tmpRefId);
-      break;         
+      break;
     }
   }
 }
 
-
 void crashReportThreadFuncUnexpectedStopped(void) { atomic_store_32(&clientStop, -1); }
 
 static void *tscCrashReportThreadFp(void *param) {

source/client/src/clientHb.c

@@ -137,7 +137,6 @@ static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) {
   vgInfo->hashSuffix = rsp->hashSuffix;
   vgInfo->vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
   if (NULL == vgInfo->vgHash) {
-    taosMemoryFree(vgInfo);
     tscError("hash init[%d] failed", rsp->vgNum);
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _return;
@@ -147,8 +146,6 @@ static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) {
     SVgroupInfo *pInfo = taosArrayGet(rsp->pVgroupInfos, j);
     if (taosHashPut(vgInfo->vgHash, &pInfo->vgId, sizeof(int32_t), pInfo, sizeof(SVgroupInfo)) != 0) {
       tscError("hash push failed, errno:%d", errno);
-      taosHashCleanup(vgInfo->vgHash);
-      taosMemoryFree(vgInfo);
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto _return;
     }
@@ -486,7 +483,6 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) {
       if (code) {
         taosArrayDestroy(desc.subDesc);
         desc.subDesc = NULL;
-        desc.subPlanNum = 0;
       }
       desc.subPlanNum = taosArrayGetSize(desc.subDesc);
     } else {
@@ -592,7 +588,7 @@ static int32_t hbGetUserAuthInfo(SClientHbKey *connKey, SHbParam *param, SClient
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _return;
   }
-  strncpy(user->user, pTscObj->user, TSDB_USER_LEN);
+  tstrncpy(user->user, pTscObj->user, TSDB_USER_LEN);
   user->version = htonl(-1);  // force get userAuthInfo
   kv.valueLen = sizeof(SUserAuthVersion);
   kv.value = user;

source/client/src/clientImpl.c

@@ -26,7 +26,7 @@
 #include "tpagedbuf.h"
 #include "tref.h"
 #include "tsched.h"
-
+#include "tversion.h"
 static int32_t       initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet);
 static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest);
 
@@ -237,8 +237,9 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t buildPreviousRequest(SRequestObj *pRequest, const char* sql, SRequestObj** pNewRequest) {
+int32_t buildPreviousRequest(SRequestObj* pRequest, const char* sql, SRequestObj** pNewRequest) {
-  int32_t code = buildRequest(pRequest->pTscObj->id, sql, strlen(sql), pRequest, pRequest->validateOnly, pNewRequest, 0);
+  int32_t code =
+      buildRequest(pRequest->pTscObj->id, sql, strlen(sql), pRequest, pRequest->validateOnly, pNewRequest, 0);
   if (TSDB_CODE_SUCCESS == code) {
     pRequest->relation.prevRefId = (*pNewRequest)->self;
     (*pNewRequest)->relation.nextRefId = pRequest->self;
@@ -502,8 +503,7 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t
     pResInfo->userFields[i].bytes = pSchema[i].bytes;
     pResInfo->userFields[i].type = pSchema[i].type;
 
-    if (pSchema[i].type == TSDB_DATA_TYPE_VARCHAR ||
+    if (pSchema[i].type == TSDB_DATA_TYPE_VARCHAR || pSchema[i].type == TSDB_DATA_TYPE_GEOMETRY) {
-        pSchema[i].type == TSDB_DATA_TYPE_GEOMETRY) {
       pResInfo->userFields[i].bytes -= VARSTR_HEADER_SIZE;
     } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON) {
       pResInfo->userFields[i].bytes = (pResInfo->userFields[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
@@ -891,7 +891,7 @@ static bool incompletaFileParsing(SNode* pStmt) {
 
 void continuePostSubQuery(SRequestObj* pRequest, TAOS_ROW row) {
   SSqlCallbackWrapper* pWrapper = pRequest->pWrapper;
-  int32_t code = nodesAcquireAllocator(pWrapper->pParseCtx->allocatorId);
+  int32_t              code = nodesAcquireAllocator(pWrapper->pParseCtx->allocatorId);
   if (TSDB_CODE_SUCCESS == code) {
     int64_t analyseStart = taosGetTimestampUs();
     code = qContinueParsePostQuery(pWrapper->pParseCtx, pRequest->pQuery, (void**)row);
@@ -934,7 +934,7 @@ void postSubQueryFetchCb(void* param, TAOS_RES* res, int32_t rowNum) {
 
   TAOS_ROW row = NULL;
   if (rowNum > 0) {
-    row = taos_fetch_row(res); // for single row only now
+    row = taos_fetch_row(res);  // for single row only now
   }
 
   SRequestObj* pNextReq = acquireRequest(pRequest->relation.nextRefId);
@@ -2135,6 +2135,7 @@ TSDB_SERVER_STATUS taos_check_server_status(const char* fqdn, int port, char* de
   connLimitNum = TMIN(connLimitNum, 500);
   rpcInit.connLimitNum = connLimitNum;
   rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
+  taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
 
   clientRpc = rpcOpen(&rpcInit);
   if (clientRpc == NULL) {
@@ -2494,11 +2495,10 @@ TAOS_RES* taosQueryImplWithReqid(TAOS* taos, const char* sql, bool validateOnly,
   return pRequest;
 }
 
+static void fetchCallback(void* pResult, void* param, int32_t code) {
+  SRequestObj* pRequest = (SRequestObj*)param;
 
-static void fetchCallback(void *pResult, void *param, int32_t code) {
+  SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
-  SRequestObj *pRequest = (SRequestObj *)param;
-
-  SReqResultInfo *pResultInfo = &pRequest->body.resInfo;
 
   tscDebug("0x%" PRIx64 " enter scheduler fetch cb, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code,
            tstrerror(code), pRequest->requestId);
@@ -2520,7 +2520,7 @@ static void fetchCallback(void *pResult, void *param, int32_t code) {
   }
 
   pRequest->code =
-      setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp *)pResultInfo->pData, pResultInfo->convertUcs4, true);
+      setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4, true);
   if (pRequest->code != TSDB_CODE_SUCCESS) {
     pResultInfo->numOfRows = 0;
     pRequest->code = code;
@@ -2531,19 +2531,19 @@ static void fetchCallback(void *pResult, void *param, int32_t code) {
              pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed,
              pRequest->requestId);
 
-    STscObj            *pTscObj = pRequest->pTscObj;
+    STscObj*            pTscObj = pRequest->pTscObj;
-    SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary;
+    SAppClusterSummary* pActivity = &pTscObj->pAppInfo->summary;
-    atomic_add_fetch_64((int64_t *)&pActivity->fetchBytes, pRequest->body.resInfo.payloadLen);
+    atomic_add_fetch_64((int64_t*)&pActivity->fetchBytes, pRequest->body.resInfo.payloadLen);
   }
 
   pRequest->body.fetchFp(pRequest->body.param, pRequest, pResultInfo->numOfRows);
 }
 
-void taosAsyncFetchImpl(SRequestObj *pRequest, __taos_async_fn_t fp, void *param) {
+void taosAsyncFetchImpl(SRequestObj* pRequest, __taos_async_fn_t fp, void* param) {
   pRequest->body.fetchFp = fp;
   pRequest->body.param = param;
 
-  SReqResultInfo *pResultInfo = &pRequest->body.resInfo;
+  SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
 
   // this query has no results or error exists, return directly
   if (taos_num_fields(pRequest) == 0 || pRequest->code != TSDB_CODE_SUCCESS) {
@@ -2578,5 +2578,3 @@ void taosAsyncFetchImpl(SRequestObj *pRequest, __taos_async_fn_t fp, void *param
 
   schedulerFetchRows(pRequest->body.queryJob, &req);
 }
-
-

source/client/src/clientMain.c

@@ -558,13 +558,12 @@ int taos_select_db(TAOS *taos, const char *db) {
   return code;
 }
 
-
 void taos_stop_query(TAOS_RES *res) {
   if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res)) {
     return;
   }
 
-  stopAllQueries((SRequestObj*)res);
+  stopAllQueries((SRequestObj *)res);
 }
 
 bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) {
@@ -785,7 +784,7 @@ void destorySqlCallbackWrapper(SSqlCallbackWrapper *pWrapper) {
   taosMemoryFree(pWrapper);
 }
 
-void destroyCtxInRequest(SRequestObj* pRequest) {
+void destroyCtxInRequest(SRequestObj *pRequest) {
   schedulerFreeJob(&pRequest->body.queryJob, 0);
   qDestroyQuery(pRequest->pQuery);
   pRequest->pQuery = NULL;
@@ -793,7 +792,6 @@ void destroyCtxInRequest(SRequestObj* pRequest) {
   pRequest->pWrapper = NULL;
 }
 
-
 static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t code) {
   SSqlCallbackWrapper *pWrapper = (SSqlCallbackWrapper *)param;
   SRequestObj         *pRequest = pWrapper->pRequest;
@@ -807,15 +805,15 @@ static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t
   if (TSDB_CODE_SUCCESS == code) {
     code = qAnalyseSqlSemantic(pWrapper->pParseCtx, pWrapper->pCatalogReq, pResultMeta, pQuery);
   }
-  
+
   pRequest->metric.analyseCostUs += taosGetTimestampUs() - analyseStart;
-  
+
   handleQueryAnslyseRes(pWrapper, pResultMeta, code);
 }
 
-int32_t cloneCatalogReq(SCatalogReq*     * ppTarget, SCatalogReq* pSrc) {
+int32_t cloneCatalogReq(SCatalogReq **ppTarget, SCatalogReq *pSrc) {
-  int32_t code = TSDB_CODE_SUCCESS;
+  int32_t      code = TSDB_CODE_SUCCESS;
-  SCatalogReq* pTarget = taosMemoryCalloc(1, sizeof(SCatalogReq));
+  SCatalogReq *pTarget = taosMemoryCalloc(1, sizeof(SCatalogReq));
   if (pTarget == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
   } else {
@@ -842,17 +840,16 @@ int32_t cloneCatalogReq(SCatalogReq*     * ppTarget, SCatalogReq* pSrc) {
   return code;
 }
 
-
+void handleSubQueryFromAnalyse(SSqlCallbackWrapper *pWrapper, SMetaData *pResultMeta, SNode *pRoot) {
-void handleSubQueryFromAnalyse(SSqlCallbackWrapper *pWrapper, SMetaData *pResultMeta, SNode* pRoot) {
+  SRequestObj         *pNewRequest = NULL;
-  SRequestObj* pNewRequest = NULL;
+  SSqlCallbackWrapper *pNewWrapper = NULL;
-  SSqlCallbackWrapper* pNewWrapper = NULL;
+  int32_t              code = buildPreviousRequest(pWrapper->pRequest, pWrapper->pRequest->sqlstr, &pNewRequest);
-  int32_t code = buildPreviousRequest(pWrapper->pRequest, pWrapper->pRequest->sqlstr, &pNewRequest);
   if (code) {
     handleQueryAnslyseRes(pWrapper, pResultMeta, code);
     return;
   }
 
-  pNewRequest->pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY);
+  pNewRequest->pQuery = (SQuery *)nodesMakeNode(QUERY_NODE_QUERY);
   if (NULL == pNewRequest->pQuery) {
     code = TSDB_CODE_OUT_OF_MEMORY;
   } else {
@@ -871,16 +868,16 @@ void handleSubQueryFromAnalyse(SSqlCallbackWrapper *pWrapper, SMetaData *pResult
 }
 
 void handleQueryAnslyseRes(SSqlCallbackWrapper *pWrapper, SMetaData *pResultMeta, int32_t code) {
-  SRequestObj         *pRequest = pWrapper->pRequest;
+  SRequestObj *pRequest = pWrapper->pRequest;
-  SQuery              *pQuery = pRequest->pQuery;
+  SQuery      *pQuery = pRequest->pQuery;
 
   if (code == TSDB_CODE_SUCCESS && pQuery->pPrevRoot) {
-    SNode* prevRoot = pQuery->pPrevRoot;
+    SNode *prevRoot = pQuery->pPrevRoot;
     pQuery->pPrevRoot = NULL;
     handleSubQueryFromAnalyse(pWrapper, pResultMeta, prevRoot);
     return;
   }
-  
+
   if (code == TSDB_CODE_SUCCESS) {
     pRequest->stableQuery = pQuery->stableQuery;
     if (pQuery->pRoot) {
@@ -1043,7 +1040,7 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) {
 }
 
 int32_t prepareAndParseSqlSyntax(SSqlCallbackWrapper **ppWrapper, SRequestObj *pRequest, bool updateMetaForce) {
-  int32_t code = TSDB_CODE_SUCCESS;
+  int32_t              code = TSDB_CODE_SUCCESS;
   STscObj             *pTscObj = pRequest->pTscObj;
   SSqlCallbackWrapper *pWrapper = taosMemoryCalloc(1, sizeof(SSqlCallbackWrapper));
   if (pWrapper == NULL) {
@@ -1081,7 +1078,6 @@ int32_t prepareAndParseSqlSyntax(SSqlCallbackWrapper **ppWrapper, SRequestObj *p
   return code;
 }
 
-
 void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
   SSqlCallbackWrapper *pWrapper = NULL;
   int32_t              code = TSDB_CODE_SUCCESS;
@@ -1128,12 +1124,12 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
 }
 
 void restartAsyncQuery(SRequestObj *pRequest, int32_t code) {
-  int32_t reqIdx = 0;
+  int32_t      reqIdx = 0;
   SRequestObj *pReqList[16] = {NULL};
   SRequestObj *pUserReq = NULL;
   pReqList[0] = pRequest;
-  uint64_t tmpRefId = 0;
+  uint64_t     tmpRefId = 0;
-  SRequestObj* pTmp = pRequest;
+  SRequestObj *pTmp = pRequest;
   while (pTmp->relation.prevRefId) {
     tmpRefId = pTmp->relation.prevRefId;
     pTmp = acquireRequest(tmpRefId);
@@ -1141,9 +1137,9 @@ void restartAsyncQuery(SRequestObj *pRequest, int32_t code) {
       pReqList[++reqIdx] = pTmp;
       releaseRequest(tmpRefId);
     } else {
-      tscError("0x%" PRIx64 ", prev req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pTmp->self,
+      tscError("0x%" PRIx64 ", prev req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pTmp->self, tmpRefId,
-               tmpRefId, pTmp->requestId);
+               pTmp->requestId);
-      break;         
+      break;
     }
   }
 
@@ -1152,11 +1148,11 @@ void restartAsyncQuery(SRequestObj *pRequest, int32_t code) {
     pTmp = acquireRequest(tmpRefId);
     if (pTmp) {
       tmpRefId = pTmp->relation.nextRefId;
-      removeRequest(pTmp->self);      
+      removeRequest(pTmp->self);
       releaseRequest(pTmp->self);
     } else {
       tscError("0x%" PRIx64 " is not there", tmpRefId);
-      break;         
+      break;
     }
   }
 

source/client/src/clientRawBlockWrite.c

@@ -1286,6 +1286,10 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
   taosArrayPush(pArray, &pVgData);
 
   pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY);
+  if (NULL == pQuery) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto end;
+  }
   pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
   pQuery->msgType = TDMT_VND_ALTER_TABLE;
   pQuery->stableQuery = false;

source/client/src/clientTmq.c

@@ -636,6 +636,7 @@ static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, int32_t type, tm
   pParamSet->callbackFn = pCommitFp;
   pParamSet->userParam = userParam;
 
+  taosRLockLatch(&tmq->lock);
   int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics);
 
   tscDebug("consumer:0x%" PRIx64 " do manual commit offset for %s, vgId:%d", tmq->consumerId, pTopicName, vgId);
@@ -646,6 +647,7 @@ static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, int32_t type, tm
             pTopicName, numOfTopics);
     taosMemoryFree(pParamSet);
     pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam);
+    taosRUnLockLatch(&tmq->lock);
     return;
   }
 
@@ -663,6 +665,7 @@ static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, int32_t type, tm
             vgId, numOfVgroups, pTopicName);
     taosMemoryFree(pParamSet);
     pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam);
+    taosRUnLockLatch(&tmq->lock);
     return;
   }
 
@@ -679,6 +682,7 @@ static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, int32_t type, tm
     taosMemoryFree(pParamSet);
     pCommitFp(tmq, code, userParam);
   }
+  taosRUnLockLatch(&tmq->lock);
 }
 
 static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam) {
@@ -696,6 +700,7 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us
   // init as 1 to prevent concurrency issue
   pParamSet->waitingRspNum = 1;
 
+  taosRLockLatch(&tmq->lock);
   int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics);
   tscDebug("consumer:0x%" PRIx64 " start to commit offset for %d topics", tmq->consumerId, numOfTopics);
 
@@ -725,6 +730,7 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us
       }
     }
   }
+  taosRUnLockLatch(&tmq->lock);
 
   tscDebug("consumer:0x%" PRIx64 " total commit:%d for %d topics", tmq->consumerId, pParamSet->waitingRspNum - 1,
            numOfTopics);
@@ -799,6 +805,7 @@ void tmqSendHbReq(void* param, void* tmrId) {
   SMqHbReq req = {0};
   req.consumerId = tmq->consumerId;
   req.epoch = tmq->epoch;
+  taosRLockLatch(&tmq->lock);
 //  if(tmq->needReportOffsetRows){
     req.topics = taosArrayInit(taosArrayGetSize(tmq->clientTopics), sizeof(TopicOffsetRows));
     for(int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++){
@@ -820,6 +827,7 @@ void tmqSendHbReq(void* param, void* tmrId) {
     }
 //    tmq->needReportOffsetRows = false;
 //  }
+  taosRUnLockLatch(&tmq->lock);
 
   int32_t tlen = tSerializeSMqHbReq(NULL, 0, &req);
   if (tlen < 0) {
@@ -986,10 +994,12 @@ int32_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) {
   if (*topics == NULL) {
     *topics = tmq_list_new();
   }
+  taosRLockLatch(&tmq->lock);
   for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
     SMqClientTopic* topic = taosArrayGet(tmq->clientTopics, i);
     tmq_list_append(*topics, strchr(topic->topicName, '.') + 1);
   }
+  taosRUnLockLatch(&tmq->lock);
   return 0;
 }
 
@@ -1527,12 +1537,7 @@ static void freeClientVgInfo(void* param) {
 static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) {
   bool set = false;
 
-  int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics);
   int32_t topicNumGet = taosArrayGetSize(pRsp->topics);
-
-  char vgKey[TSDB_TOPIC_FNAME_LEN + 22];
-  tscInfo("consumer:0x%" PRIx64 " update ep epoch from %d to epoch %d, incoming topics:%d, existed topics:%d",
-           tmq->consumerId, tmq->epoch, epoch, topicNumGet, topicNumCur);
   if (epoch <= tmq->epoch) {
     return false;
   }
@@ -1548,6 +1553,12 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp)
     return false;
   }
 
+  taosWLockLatch(&tmq->lock);
+  int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics);
+
+  char vgKey[TSDB_TOPIC_FNAME_LEN + 22];
+  tscInfo("consumer:0x%" PRIx64 " update ep epoch from %d to epoch %d, incoming topics:%d, existed topics:%d",
+          tmq->consumerId, tmq->epoch, epoch, topicNumGet, topicNumCur);
   // todo extract method
   for (int32_t i = 0; i < topicNumCur; i++) {
     // find old topic
@@ -1579,7 +1590,6 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp)
 
   taosHashCleanup(pVgOffsetHashMap);
 
-  taosWLockLatch(&tmq->lock);
   // destroy current buffered existed topics info
   if (tmq->clientTopics) {
     taosArrayDestroyEx(tmq->clientTopics, freeClientVgInfo);
@@ -1807,6 +1817,9 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
   if(atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER){
     return 0;
   }
+  int32_t code = 0;
+
+  taosWLockLatch(&tmq->lock);
   int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics);
   tscDebug("consumer:0x%" PRIx64 " start to poll data, numOfTopics:%d", tmq->consumerId, numOfTopics);
 
@@ -1816,7 +1829,7 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
 
     for (int j = 0; j < numOfVg; j++) {
       SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
-      if (taosGetTimestampMs() - pVg->emptyBlockReceiveTs < EMPTY_BLOCK_POLL_IDLE_DURATION) {  // less than 100ms
+      if (taosGetTimestampMs() - pVg->emptyBlockReceiveTs < EMPTY_BLOCK_POLL_IDLE_DURATION) {  // less than 10ms
         tscTrace("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId,
                  tmq->epoch, pVg->vgId);
         continue;
@@ -1831,15 +1844,17 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
       }
 
       atomic_store_32(&pVg->vgSkipCnt, 0);
-      int32_t code = doTmqPollImpl(tmq, pTopic, pVg, timeout);
+      code = doTmqPollImpl(tmq, pTopic, pVg, timeout);
       if (code != TSDB_CODE_SUCCESS) {
-        return code;
+        goto end;
       }
     }
   }
 
-  tscDebug("consumer:0x%" PRIx64 " end to poll data", tmq->consumerId);
+end:
-  return 0;
+  taosWUnLockLatch(&tmq->lock);
+  tscDebug("consumer:0x%" PRIx64 " end to poll data, code:%d", tmq->consumerId, code);
+  return code;
 }
 
 static int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset) {
@@ -1862,6 +1877,23 @@ static int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* p
   return 0;
 }
 
+static void updateVgInfo(SMqClientVg* pVg, STqOffsetVal* offset, int64_t sver, int64_t ever, int64_t consumerId){
+  if (!pVg->seekUpdated) {
+    tscDebug("consumer:0x%" PRIx64" local offset is update, since seekupdate not set", consumerId);
+    pVg->offsetInfo.currentOffset = *offset;
+  } else {
+    tscDebug("consumer:0x%" PRIx64" local offset is NOT update, since seekupdate is set", consumerId);
+  }
+
+  // update the status
+  atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
+
+  // update the valid wal version range
+  pVg->offsetInfo.walVerBegin = sver;
+  pVg->offsetInfo.walVerEnd = ever;
+  pVg->receivedInfoFromVnode = true;
+}
+
 static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
   tscDebug("consumer:0x%" PRIx64 " start to handle the rsp, total:%d", tmq->consumerId, tmq->qall->numOfItems);
 
@@ -1891,12 +1923,14 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
       SMqDataRsp* pDataRsp = &pollRspWrapper->dataRsp;
 
       if (pDataRsp->head.epoch == consumerEpoch) {
+        taosWLockLatch(&tmq->lock);
         SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId);
         pollRspWrapper->vgHandle = pVg;
         pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName);
         if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){
           tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId,
                    pollRspWrapper->topicName, pollRspWrapper->vgId);
+          taosWUnLockLatch(&tmq->lock);
           return NULL;
         }
         // update the epset
@@ -1908,22 +1942,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
           pVg->epSet = *pollRspWrapper->pEpset;
         }
 
-        // update the local offset value only for the returned values, only when the local offset is NOT updated
+        updateVgInfo(pVg, &pDataRsp->rspOffset, pDataRsp->head.walsver, pDataRsp->head.walever, tmq->consumerId);
-        // by tmq_offset_seek function
-        if (!pVg->seekUpdated) {
-          tscDebug("consumer:0x%" PRIx64" local offset is update, since seekupdate not set", tmq->consumerId);
-          pVg->offsetInfo.currentOffset = pDataRsp->rspOffset;
-        } else {
-          tscDebug("consumer:0x%" PRIx64" local offset is NOT update, since seekupdate is set", tmq->consumerId);
-        }
-
-        // update the status
-        atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-
-        // update the valid wal version range
-        pVg->offsetInfo.walVerBegin = pDataRsp->head.walsver;
-        pVg->offsetInfo.walVerEnd = pDataRsp->head.walever;
-        pVg->receivedInfoFromVnode = true;
 
         char buf[TSDB_OFFSET_LEN];
         tFormatOffset(buf, TSDB_OFFSET_LEN, &pDataRsp->rspOffset);
@@ -1944,8 +1963,10 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
                    tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows,
                    pollRspWrapper->reqId);
           taosFreeQitem(pollRspWrapper);
+          taosWUnLockLatch(&tmq->lock);
           return pRsp;
         }
+        taosWUnLockLatch(&tmq->lock);
       } else {
         tscDebug("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d",
                  tmq->consumerId, pollRspWrapper->vgId, pDataRsp->head.epoch, consumerEpoch);
@@ -1960,23 +1981,22 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
       tscDebug("consumer:0x%" PRIx64 " process meta rsp", tmq->consumerId);
 
       if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) {
+        taosWLockLatch(&tmq->lock);
         SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId);
         pollRspWrapper->vgHandle = pVg;
         pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName);
         if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){
           tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId,
                    pollRspWrapper->topicName, pollRspWrapper->vgId);
+          taosWUnLockLatch(&tmq->lock);
           return NULL;
         }
 
-        if(pollRspWrapper->metaRsp.rspOffset.type != 0){    // if offset is validate
+        updateVgInfo(pVg, &pollRspWrapper->metaRsp.rspOffset, pollRspWrapper->metaRsp.head.walsver, pollRspWrapper->metaRsp.head.walever, tmq->consumerId);
-          pVg->offsetInfo.currentOffset = pollRspWrapper->metaRsp.rspOffset;
-        }
-
-        atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
         // build rsp
         SMqMetaRspObj* pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper);
         taosFreeQitem(pollRspWrapper);
+        taosWUnLockLatch(&tmq->lock);
         return pRsp;
       } else {
         tscDebug("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d",
@@ -1989,27 +2009,18 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
       int32_t            consumerEpoch = atomic_load_32(&tmq->epoch);
 
       if (pollRspWrapper->taosxRsp.head.epoch == consumerEpoch) {
+        taosWLockLatch(&tmq->lock);
         SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId);
         pollRspWrapper->vgHandle = pVg;
         pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName);
         if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){
           tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId,
                    pollRspWrapper->topicName, pollRspWrapper->vgId);
+          taosWUnLockLatch(&tmq->lock);
           return NULL;
         }
 
-        // update the local offset value only for the returned values, only when the local offset is NOT updated
+        updateVgInfo(pVg, &pollRspWrapper->taosxRsp.rspOffset, pollRspWrapper->taosxRsp.head.walsver, pollRspWrapper->taosxRsp.head.walever, tmq->consumerId);
-        // by tmq_offset_seek function
-        if (!pVg->seekUpdated) {
-          if(pollRspWrapper->taosxRsp.rspOffset.type != 0) {    // if offset is validate
-            tscDebug("consumer:0x%" PRIx64" local offset is update, since seekupdate not set", tmq->consumerId);
-            pVg->offsetInfo.currentOffset = pollRspWrapper->taosxRsp.rspOffset;
-          }
-        } else {
-          tscDebug("consumer:0x%" PRIx64" local offset is NOT update, since seekupdate is set", tmq->consumerId);
-        }
-
-        atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
 
         if (pollRspWrapper->taosxRsp.blockNum == 0) {
           tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ", reqId:0x%" PRIx64,
@@ -2017,32 +2028,31 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
           pVg->emptyBlockReceiveTs = taosGetTimestampMs();
           pRspWrapper = tmqFreeRspWrapper(pRspWrapper);
           taosFreeQitem(pollRspWrapper);
-          continue;
         } else {
           pVg->emptyBlockReceiveTs = 0;  // reset the ts
+          // build rsp
+          void*   pRsp = NULL;
+          int64_t numOfRows = 0;
+          if (pollRspWrapper->taosxRsp.createTableNum == 0) {
+            pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg, &numOfRows);
+          } else {
+            pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows);
+          }
+
+          tmq->totalRows += numOfRows;
+
+          char buf[TSDB_OFFSET_LEN];
+          tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.currentOffset);
+          tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64
+                       ", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64,
+                   tmq->consumerId, pVg->vgId, buf, pollRspWrapper->dataRsp.blockNum, numOfRows, pVg->numOfRows,
+                   tmq->totalRows, pollRspWrapper->reqId);
+
+          taosFreeQitem(pollRspWrapper);
+          taosWUnLockLatch(&tmq->lock);
+          return pRsp;
         }
-
+        taosWUnLockLatch(&tmq->lock);
-        // build rsp
-        void*   pRsp = NULL;
-        int64_t numOfRows = 0;
-        if (pollRspWrapper->taosxRsp.createTableNum == 0) {
-          pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg, &numOfRows);
-        } else {
-          pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows);
-        }
-
-        tmq->totalRows += numOfRows;
-
-        char buf[TSDB_OFFSET_LEN];
-        tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.currentOffset);
-        tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64
-                 ", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64,
-                 tmq->consumerId, pVg->vgId, buf, pollRspWrapper->dataRsp.blockNum, numOfRows, pVg->numOfRows,
-                 tmq->totalRows, pollRspWrapper->reqId);
-
-        taosFreeQitem(pollRspWrapper);
-        return pRsp;
-
       } else {
         tscDebug("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d",
                  tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->taosxRsp.head.epoch, consumerEpoch);
@@ -2121,7 +2131,8 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
   }
 }
 
-static void displayConsumeStatistics(const tmq_t* pTmq) {
+static void displayConsumeStatistics(tmq_t* pTmq) {
+  taosRLockLatch(&pTmq->lock);
   int32_t numOfTopics = taosArrayGetSize(pTmq->clientTopics);
   tscDebug("consumer:0x%" PRIx64 " closing poll:%" PRId64 " rows:%" PRId64 " topics:%d, final epoch:%d",
            pTmq->consumerId, pTmq->pollCnt, pTmq->totalRows, numOfTopics, pTmq->epoch);
@@ -2137,7 +2148,7 @@ static void displayConsumeStatistics(const tmq_t* pTmq) {
       tscDebug("topic:%s, %d. vgId:%d rows:%" PRId64, pTopics->topicName, j, pVg->vgId, pVg->numOfRows);
     }
   }
-
+  taosRUnLockLatch(&pTmq->lock);
   tscDebug("consumer:0x%" PRIx64 " rows dist end", pTmq->consumerId);
 }
 
@@ -2533,6 +2544,9 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) {
 }
 
 static void destroyCommonInfo(SMqVgCommon* pCommon) {
+  if(pCommon == NULL){
+    return;
+  }
   taosArrayDestroy(pCommon->pList);
   tsem_destroy(&pCommon->rsp);
   taosThreadMutexDestroy(&pCommon->mutex);
@@ -2544,14 +2558,18 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
                                  int32_t* numOfAssignment) {
   *numOfAssignment = 0;
   *assignment = NULL;
+  SMqVgCommon* pCommon = NULL;
 
   int32_t accId = tmq->pTscObj->acctId;
   char    tname[128] = {0};
   sprintf(tname, "%d.%s", accId, pTopicName);
+  int32_t code = TSDB_CODE_SUCCESS;
 
+  taosWLockLatch(&tmq->lock);
   SMqClientTopic* pTopic = getTopicByName(tmq, tname);
   if (pTopic == NULL) {
-    return TSDB_CODE_INVALID_PARA;
+    code = TSDB_CODE_INVALID_PARA;
+    goto end;
   }
 
   // in case of snapshot is opened, no valid offset will return
@@ -2561,19 +2579,20 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
   if (*assignment == NULL) {
     tscError("consumer:0x%" PRIx64 " failed to malloc buffer, size:%" PRIzu, tmq->consumerId,
              (*numOfAssignment) * sizeof(tmq_topic_assignment));
-    return TSDB_CODE_OUT_OF_MEMORY;
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto end;
   }
 
   bool needFetch = false;
-  int32_t index = 0;
+
   for (int32_t j = 0; j < (*numOfAssignment); ++j) {
     SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j);
     if (!pClientVg->receivedInfoFromVnode) {
       needFetch = true;
-      continue;
+      break;
     }
 
-    tmq_topic_assignment* pAssignment = &(*assignment)[index++];
+    tmq_topic_assignment* pAssignment = &(*assignment)[j];
     if (pClientVg->offsetInfo.currentOffset.type == TMQ_OFFSET__LOG) {
       pAssignment->currentOffset = pClientVg->offsetInfo.currentOffset.version;
     } else {
@@ -2588,10 +2607,11 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
   }
 
   if (needFetch) {
-    SMqVgCommon* pCommon = taosMemoryCalloc(1, sizeof(SMqVgCommon));
+    pCommon = taosMemoryCalloc(1, sizeof(SMqVgCommon));
     if (pCommon == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
-      return terrno;
+      code = terrno;
+      goto end;
     }
 
     pCommon->pList= taosArrayInit(4, sizeof(tmq_topic_assignment));
@@ -2603,13 +2623,11 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     for (int32_t i = 0; i < (*numOfAssignment); ++i) {
       SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i);
-      if (pClientVg->receivedInfoFromVnode) {
+
-        continue;
-      }
       SMqVgWalInfoParam* pParam = taosMemoryMalloc(sizeof(SMqVgWalInfoParam));
       if (pParam == NULL) {
-        destroyCommonInfo(pCommon);
+        code = terrno;
-        return terrno;
+        goto end;
       }
 
       pParam->epoch = tmq->epoch;
@@ -2623,30 +2641,30 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
       int32_t msgSize = tSerializeSMqPollReq(NULL, 0, &req);
       if (msgSize < 0) {
         taosMemoryFree(pParam);
-        destroyCommonInfo(pCommon);
+        code = terrno;
-        return terrno;
+        goto end;
       }
 
       char* msg = taosMemoryCalloc(1, msgSize);
       if (NULL == msg) {
         taosMemoryFree(pParam);
-        destroyCommonInfo(pCommon);
+        code = terrno;
-        return terrno;
+        goto end;
       }
 
       if (tSerializeSMqPollReq(msg, msgSize, &req) < 0) {
         taosMemoryFree(msg);
         taosMemoryFree(pParam);
-        destroyCommonInfo(pCommon);
+        code = terrno;
-        return terrno;
+        goto end;
       }
 
       SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
       if (sendInfo == NULL) {
         taosMemoryFree(pParam);
         taosMemoryFree(msg);
-        destroyCommonInfo(pCommon);
+        code = terrno;
-        return terrno;
+        goto end;
       }
 
       sendInfo->msgInfo = (SDataBuf){.pData = msg, .len = msgSize, .handle = NULL};
@@ -2661,58 +2679,58 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
       tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pClientVg->offsetInfo.currentOffset);
 
       tscInfo("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64,
-               tmq->consumerId, pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId);
+              tmq->consumerId, pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId);
       asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pClientVg->epSet, &transporterId, sendInfo);
     }
 
     tsem_wait(&pCommon->rsp);
-    int32_t code = pCommon->code;
+    code = pCommon->code;
 
     terrno = code;
     if (code != TSDB_CODE_SUCCESS) {
-      taosMemoryFree(*assignment);
+      goto end;
-      *assignment = NULL;
-      *numOfAssignment = 0;
-    } else {
-      int32_t num = taosArrayGetSize(pCommon->pList);
-      for(int32_t i = 0; i < num; ++i) {
-        (*assignment)[index++] = *(tmq_topic_assignment*)taosArrayGet(pCommon->pList, i);
-        tscInfo("consumer:0x%" PRIx64 " get assignment from server:%d->%" PRId64, tmq->consumerId,
-                (*assignment)[i].vgId, (*assignment)[i].currentOffset);
-      }
-      *numOfAssignment = index;
     }
+    int32_t num = taosArrayGetSize(pCommon->pList);
+    for(int32_t i = 0; i < num; ++i) {
+      (*assignment)[i] = *(tmq_topic_assignment*)taosArrayGet(pCommon->pList, i);
+    }
+    *numOfAssignment = num;
+
+    for (int32_t j = 0; j < (*numOfAssignment); ++j) {
+      tmq_topic_assignment* p = &(*assignment)[j];
+
+      for(int32_t i = 0; i < taosArrayGetSize(pTopic->vgs); ++i) {
+        SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i);
+        if (pClientVg->vgId != p->vgId) {
+          continue;
+        }
+
+        SVgOffsetInfo* pOffsetInfo = &pClientVg->offsetInfo;
 
-//    for (int32_t j = 0; j < (*numOfAssignment); ++j) {
-//      tmq_topic_assignment* p = &(*assignment)[j];
-//
-//      for(int32_t i = 0; i < taosArrayGetSize(pTopic->vgs); ++i) {
-//        SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i);
-//        if (pClientVg->vgId != p->vgId) {
-//          continue;
-//        }
-//
-//        SVgOffsetInfo* pOffsetInfo = &pClientVg->offsetInfo;
-//
 //        pOffsetInfo->currentOffset.type = TMQ_OFFSET__LOG;
-//
+
-//        char offsetBuf[80] = {0};
+//        char offsetBuf[TSDB_OFFSET_LEN] = {0};
 //        tFormatOffset(offsetBuf, tListLen(offsetBuf), &pOffsetInfo->currentOffset);
-//
+
-//        tscDebug("vgId:%d offset is update to:%s", p->vgId, offsetBuf);
+        tscInfo("vgId:%d offset is old to:%"PRId64, p->vgId, p->currentOffset);
-//
+
-//        pOffsetInfo->walVerBegin = p->begin;
+        pOffsetInfo->walVerBegin = p->begin;
-//        pOffsetInfo->walVerEnd = p->end;
+        pOffsetInfo->walVerEnd = p->end;
 //        pOffsetInfo->currentOffset.version = p->currentOffset;
 //        pOffsetInfo->committedOffset.version = p->currentOffset;
-//      }
+      }
-//    }
+    }
-
-    destroyCommonInfo(pCommon);
-    return code;
-  } else {
-    return TSDB_CODE_SUCCESS;
   }
+
+end:
+  if(code != TSDB_CODE_SUCCESS){
+    taosMemoryFree(*assignment);
+    *assignment = NULL;
+    *numOfAssignment = 0;
+  }
+  destroyCommonInfo(pCommon);
+  taosWUnLockLatch(&tmq->lock);
+  return code;
 }
 
 void tmq_free_assignment(tmq_topic_assignment* pAssignment) {
@@ -2733,9 +2751,11 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_
   char tname[128] = {0};
   sprintf(tname, "%d.%s", accId, pTopicName);
 
+  taosWLockLatch(&tmq->lock);
   SMqClientTopic* pTopic = getTopicByName(tmq, tname);
   if (pTopic == NULL) {
     tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName);
+    taosWUnLockLatch(&tmq->lock);
     return TSDB_CODE_INVALID_PARA;
   }
 
@@ -2751,6 +2771,7 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_
 
   if (pVg == NULL) {
     tscError("consumer:0x%" PRIx64 " invalid vgroup id:%d", tmq->consumerId, vgId);
+    taosWUnLockLatch(&tmq->lock);
     return TSDB_CODE_INVALID_PARA;
   }
 
@@ -2759,12 +2780,14 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_
   int32_t type = pOffsetInfo->currentOffset.type;
   if (type != TMQ_OFFSET__LOG && !OFFSET_IS_RESET_OFFSET(type)) {
     tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, seek not allowed", tmq->consumerId, type);
+    taosWUnLockLatch(&tmq->lock);
     return TSDB_CODE_INVALID_PARA;
   }
 
   if (type == TMQ_OFFSET__LOG && (offset < pOffsetInfo->walVerBegin || offset > pOffsetInfo->walVerEnd)) {
     tscError("consumer:0x%" PRIx64 " invalid seek params, offset:%" PRId64 ", valid range:[%" PRId64 ", %" PRId64 "]",
              tmq->consumerId, offset, pOffsetInfo->walVerBegin, pOffsetInfo->walVerEnd);
+    taosWUnLockLatch(&tmq->lock);
     return TSDB_CODE_INVALID_PARA;
   }
 
@@ -2779,6 +2802,7 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_
   tstrncpy(rspObj.topic, tname, tListLen(rspObj.topic));
 
   tscInfo("consumer:0x%" PRIx64 " seek to %" PRId64 " on vgId:%d", tmq->consumerId, offset, pVg->vgId);
+  taosWUnLockLatch(&tmq->lock);
 
   SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo));
   if (pInfo == NULL) {

source/common/src/trow.c

@@ -423,7 +423,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
         val = (const void *)&pColVal->value.val;
       }
     } else {
-      pColVal = NULL;
+      // pColVal = NULL;
       valType = TD_VTYPE_NONE;
     }
 

source/dnode/mgmt/node_mgmt/src/dmTransport.c

@@ -16,6 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "dmMgmt.h"
 #include "qworker.h"
+#include "tversion.h"
 
 static inline void dmSendRsp(SRpcMsg *pMsg) { rpcSendResponse(pMsg); }
 
@@ -73,6 +74,13 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
   dGTrace("msg:%s is received, handle:%p len:%d code:0x%x app:%p refId:%" PRId64, TMSG_INFO(pRpc->msgType),
           pRpc->info.handle, pRpc->contLen, pRpc->code, pRpc->info.ahandle, pRpc->info.refId);
 
+  int32_t svrVer = 0;
+  taosVersionStrToInt(version, &svrVer);
+  if (0 != taosCheckVersionCompatible(pRpc->info.cliVer, svrVer, 3)) {
+    dError("Version not compatible, cli ver: %d, svr ver: %d", pRpc->info.cliVer, svrVer);
+    goto _OVER;
+  }
+
   switch (pRpc->msgType) {
     case TDMT_DND_NET_TEST:
       dmProcessNetTestReq(pDnode, pRpc);
@@ -305,6 +313,7 @@ int32_t dmInitClient(SDnode *pDnode) {
   rpcInit.supportBatch = 1;
   rpcInit.batchSize = 8 * 1024;
   rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
+  taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
 
   pTrans->clientRpc = rpcOpen(&rpcInit);
   if (pTrans->clientRpc == NULL) {
@@ -339,7 +348,7 @@ int32_t dmInitServer(SDnode *pDnode) {
   rpcInit.idleTime = tsShellActivityTimer * 1000;
   rpcInit.parent = pDnode;
   rpcInit.compressSize = tsCompressMsgSize;
-
+  taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
   pTrans->serverRpc = rpcOpen(&rpcInit);
   if (pTrans->serverRpc == NULL) {
     dError("failed to init dnode rpc server");

source/dnode/mgmt/test/sut/src/client.cpp

@@ -16,6 +16,7 @@
 #include "sut.h"
 #include "tdatablock.h"
 #include "tmisce.h"
+#include "tversion.h"
 
 static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) {
   TestClient* client = (TestClient*)parent;
@@ -53,6 +54,7 @@ void TestClient::DoInit() {
   rpcInit.parent = this;
   // rpcInit.secret = (char*)secretEncrypt;
   // rpcInit.spi = 1;
+  taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
 
   clientRpc = rpcOpen(&rpcInit);
   ASSERT(clientRpc);

source/dnode/vnode/src/inc/metaTtl.h

@@ -79,16 +79,18 @@ typedef struct {
   TXN*     pTxn;
 } STtlDelTtlCtx;
 
-int ttlMgrOpen(STtlManger** ppTtlMgr, TDB* pEnv, int8_t rollback);
+int  ttlMgrOpen(STtlManger** ppTtlMgr, TDB* pEnv, int8_t rollback);
-int ttlMgrClose(STtlManger* pTtlMgr);
+void ttlMgrClose(STtlManger* pTtlMgr);
-int ttlMgrBegin(STtlManger* pTtlMgr, void* pMeta);
+int  ttlMgrPostOpen(STtlManger* pTtlMgr, void* pMeta);
 
-int ttlMgrConvert(TTB* pOldTtlIdx, TTB* pNewTtlIdx, void* pMeta);
+bool ttlMgrNeedUpgrade(TDB* pEnv);
-int ttlMgrFlush(STtlManger* pTtlMgr, TXN* pTxn);
+int  ttlMgrUpgrade(STtlManger* pTtlMgr, void* pMeta);
 
 int ttlMgrInsertTtl(STtlManger* pTtlMgr, const STtlUpdTtlCtx* pUpdCtx);
 int ttlMgrDeleteTtl(STtlManger* pTtlMgr, const STtlDelTtlCtx* pDelCtx);
 int ttlMgrUpdateChangeTime(STtlManger* pTtlMgr, const STtlUpdCtimeCtx* pUpdCtimeCtx);
+
+int ttlMgrFlush(STtlManger* pTtlMgr, TXN* pTxn);
 int ttlMgrFindExpired(STtlManger* pTtlMgr, int64_t timePointMs, SArray* pTbUids);
 
 #ifdef __cplusplus

source/dnode/vnode/src/inc/vnodeInt.h

@@ -136,6 +136,7 @@ typedef struct STbUidStore STbUidStore;
 #define META_BEGIN_HEAP_NIL        2
 
 int             metaOpen(SVnode* pVnode, SMeta** ppMeta, int8_t rollback);
+int             metaUpgrade(SVnode* pVnode, SMeta** ppMeta);
 int             metaClose(SMeta** pMeta);
 int             metaBegin(SMeta* pMeta, int8_t fromSys);
 TXN*            metaGetTxn(SMeta* pMeta);

source/dnode/vnode/src/meta/metaCommit.c

@@ -40,10 +40,6 @@ int metaBegin(SMeta *pMeta, int8_t heap) {
     return -1;
   }
 
-  if (ttlMgrBegin(pMeta->pTtlMgr, pMeta) < 0) {
-    return -1;
-  }
-
   tdbCommit(pMeta->pEnv, pMeta->txn);
 
   return 0;

source/dnode/vnode/src/meta/metaOpen.c

@@ -29,6 +29,8 @@ static int ncolIdxCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen
 static int32_t metaInitLock(SMeta *pMeta) { return taosThreadRwlockInit(&pMeta->lock, NULL); }
 static int32_t metaDestroyLock(SMeta *pMeta) { return taosThreadRwlockDestroy(&pMeta->lock); }
 
+static void metaCleanup(SMeta **ppMeta);
+
 int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) {
   SMeta *pMeta = NULL;
   int    ret;
@@ -180,51 +182,43 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) {
   return 0;
 
 _err:
-  if (pMeta->pIdx) metaCloseIdx(pMeta);
+  metaCleanup(&pMeta);
-  if (pMeta->pStreamDb) tdbTbClose(pMeta->pStreamDb);
-  if (pMeta->pNcolIdx) tdbTbClose(pMeta->pNcolIdx);
-  if (pMeta->pBtimeIdx) tdbTbClose(pMeta->pBtimeIdx);
-  if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx);
-  if (pMeta->pTtlMgr) ttlMgrClose(pMeta->pTtlMgr);
-  if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx);
-  if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx);
-  if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx);
-  if (pMeta->pSuidIdx) tdbTbClose(pMeta->pSuidIdx);
-  if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx);
-  if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx);
-  if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb);
-  if (pMeta->pTbDb) tdbTbClose(pMeta->pTbDb);
-  if (pMeta->pEnv) tdbClose(pMeta->pEnv);
-  metaDestroyLock(pMeta);
-  taosMemoryFree(pMeta);
   return -1;
 }
 
-int metaClose(SMeta **ppMeta) {
+int metaUpgrade(SVnode *pVnode, SMeta **ppMeta) {
+  int    code = TSDB_CODE_SUCCESS;
   SMeta *pMeta = *ppMeta;
-  if (pMeta) {
-    if (pMeta->pEnv) metaAbort(pMeta);
-    if (pMeta->pCache) metaCacheClose(pMeta);
-    if (pMeta->pIdx) metaCloseIdx(pMeta);
-    if (pMeta->pStreamDb) tdbTbClose(pMeta->pStreamDb);
-    if (pMeta->pNcolIdx) tdbTbClose(pMeta->pNcolIdx);
-    if (pMeta->pBtimeIdx) tdbTbClose(pMeta->pBtimeIdx);
-    if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx);
-    if (pMeta->pTtlMgr) ttlMgrClose(pMeta->pTtlMgr);
-    if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx);
-    if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx);
-    if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx);
-    if (pMeta->pSuidIdx) tdbTbClose(pMeta->pSuidIdx);
-    if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx);
-    if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx);
-    if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb);
-    if (pMeta->pTbDb) tdbTbClose(pMeta->pTbDb);
-    if (pMeta->pEnv) tdbClose(pMeta->pEnv);
-    metaDestroyLock(pMeta);
 
-    taosMemoryFreeClear(*ppMeta);
+  if (ttlMgrNeedUpgrade(pMeta->pEnv)) {
+    code = metaBegin(pMeta, META_BEGIN_HEAP_OS);
+    if (code < 0) {
+      metaError("vgId:%d, failed to upgrade meta, meta begin failed since %s", TD_VID(pVnode), tstrerror(terrno));
+      goto _err;
+    }
+
+    code = ttlMgrUpgrade(pMeta->pTtlMgr, pMeta);
+    if (code < 0) {
+      metaError("vgId:%d, failed to upgrade meta ttl since %s", TD_VID(pVnode), tstrerror(terrno));
+      goto _err;
+    }
+
+    code = metaCommit(pMeta, pMeta->txn);
+    if (code < 0) {
+      metaError("vgId:%d, failed to upgrade meta ttl, meta commit failed since %s", TD_VID(pVnode), tstrerror(terrno));
+      goto _err;
+    }
   }
 
+  return TSDB_CODE_SUCCESS;
+
+_err:
+  metaCleanup(ppMeta);
+  return code;
+}
+
+int metaClose(SMeta **ppMeta) {
+  metaCleanup(ppMeta);
   return 0;
 }
 
@@ -270,6 +264,32 @@ int32_t metaULock(SMeta *pMeta) {
   return ret;
 }
 
+static void metaCleanup(SMeta **ppMeta) {
+  SMeta *pMeta = *ppMeta;
+  if (pMeta) {
+    if (pMeta->pEnv) metaAbort(pMeta);
+    if (pMeta->pCache) metaCacheClose(pMeta);
+    if (pMeta->pIdx) metaCloseIdx(pMeta);
+    if (pMeta->pStreamDb) tdbTbClose(pMeta->pStreamDb);
+    if (pMeta->pNcolIdx) tdbTbClose(pMeta->pNcolIdx);
+    if (pMeta->pBtimeIdx) tdbTbClose(pMeta->pBtimeIdx);
+    if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx);
+    if (pMeta->pTtlMgr) ttlMgrClose(pMeta->pTtlMgr);
+    if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx);
+    if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx);
+    if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx);
+    if (pMeta->pSuidIdx) tdbTbClose(pMeta->pSuidIdx);
+    if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx);
+    if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx);
+    if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb);
+    if (pMeta->pTbDb) tdbTbClose(pMeta->pTbDb);
+    if (pMeta->pEnv) tdbClose(pMeta->pEnv);
+    metaDestroyLock(pMeta);
+
+    taosMemoryFreeClear(*ppMeta);
+  }
+}
+
 static int tbDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
   STbDbKey *pTbDbKey1 = (STbDbKey *)pKey1;
   STbDbKey *pTbDbKey2 = (STbDbKey *)pKey2;

source/dnode/vnode/src/meta/metaTable.c

@@ -207,7 +207,10 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
     tb_uid_t uid = *(tb_uid_t *)pData;
     tdbFree(pData);
     SMetaInfo info;
-    metaGetInfo(pMeta, uid, &info, NULL);
+    if (metaGetInfo(pMeta, uid, &info, NULL) == TSDB_CODE_NOT_FOUND) {
+      terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
+      return -1;
+    }
     if (info.uid == info.suid) {
       return 0;
     } else {
@@ -939,7 +942,7 @@ int metaTtlDropTable(SMeta *pMeta, int64_t timePointMs, SArray *tbUids) {
     return 0;
   }
 
-  metaInfo("ttl find expired table count: %zu" , TARRAY_SIZE(tbUids));
+  metaInfo("ttl find expired table count: %zu", TARRAY_SIZE(tbUids));
 
   metaDropTables(pMeta, tbUids);
   return 0;

source/dnode/vnode/src/meta/metaTtl.c

@@ -21,6 +21,10 @@ typedef struct {
   SMeta *pMeta;
 } SConvertData;
 
+static void ttlMgrCleanup(STtlManger *pTtlMgr);
+
+static int ttlMgrConvert(TTB *pOldTtlIdx, TTB *pNewTtlIdx, void *pMeta);
+
 static void    ttlMgrBuildKey(STtlIdxKeyV1 *pTtlKey, int64_t ttlDays, int64_t changeTimeMs, tb_uid_t uid);
 static int     ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
 static int     ttlIdxKeyV1Cmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
@@ -36,27 +40,17 @@ const char *ttlTbname = "ttl.idx";
 const char *ttlV1Tbname = "ttlv1.idx";
 
 int ttlMgrOpen(STtlManger **ppTtlMgr, TDB *pEnv, int8_t rollback) {
-  int ret;
+  int ret = TSDB_CODE_SUCCESS;
+  int64_t startNs = taosGetTimestampNs();
 
   *ppTtlMgr = NULL;
 
   STtlManger *pTtlMgr = (STtlManger *)tdbOsCalloc(1, sizeof(*pTtlMgr));
-  if (pTtlMgr == NULL) {
+  if (pTtlMgr == NULL) return TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  if (tdbTbExist(ttlTbname, pEnv)) {
-    ret = tdbTbOpen(ttlTbname, sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pEnv, &pTtlMgr->pOldTtlIdx, rollback);
-    if (ret < 0) {
-      metaError("failed to open %s index since %s", ttlTbname, tstrerror(terrno));
-      return ret;
-    }
-  }
 
   ret = tdbTbOpen(ttlV1Tbname, TDB_VARIANT_LEN, TDB_VARIANT_LEN, ttlIdxKeyV1Cmpr, pEnv, &pTtlMgr->pTtlIdx, rollback);
   if (ret < 0) {
     metaError("failed to open %s since %s", ttlV1Tbname, tstrerror(terrno));
-
     tdbOsFree(pTtlMgr);
     return ret;
   }
@@ -66,42 +60,57 @@ int ttlMgrOpen(STtlManger **ppTtlMgr, TDB *pEnv, int8_t rollback) {
 
   taosThreadRwlockInit(&pTtlMgr->lock, NULL);
 
+  ret = ttlMgrFillCache(pTtlMgr);
+  if (ret < 0) {
+    metaError("failed to fill hash since %s", tstrerror(terrno));
+    ttlMgrCleanup(pTtlMgr);
+    return ret;
+  }
+
+  int64_t endNs = taosGetTimestampNs();
+  metaInfo("ttl mgr open end, hash size: %d, time consumed: %" PRId64 " ns", taosHashGetSize(pTtlMgr->pTtlCache),
+           endNs - startNs);
+
   *ppTtlMgr = pTtlMgr;
-  return 0;
+  return TSDB_CODE_SUCCESS;
 }
 
-int ttlMgrClose(STtlManger *pTtlMgr) {
+void ttlMgrClose(STtlManger *pTtlMgr) { ttlMgrCleanup(pTtlMgr); }
-  taosHashCleanup(pTtlMgr->pTtlCache);
+
-  taosHashCleanup(pTtlMgr->pDirtyUids);
+bool ttlMgrNeedUpgrade(TDB *pEnv) {
-  tdbTbClose(pTtlMgr->pTtlIdx);
+  bool needUpgrade = tdbTbExist(ttlTbname, pEnv);
-  taosThreadRwlockDestroy(&pTtlMgr->lock);
+  if (needUpgrade) {
-  tdbOsFree(pTtlMgr);
+    metaInfo("find ttl idx in old version , will convert");
-  return 0;
+  }
+  return needUpgrade;
 }
 
-int ttlMgrBegin(STtlManger *pTtlMgr, void *pMeta) {
+int ttlMgrUpgrade(STtlManger *pTtlMgr, void *pMeta) {
-  metaInfo("ttl mgr start open");
+  SMeta *meta = (SMeta *)pMeta;
-  int ret;
+  int    ret = TSDB_CODE_SUCCESS;
+
+  if (!tdbTbExist(ttlTbname, meta->pEnv)) return TSDB_CODE_SUCCESS;
+
+  metaInfo("ttl mgr start upgrade");
 
   int64_t startNs = taosGetTimestampNs();
 
-  SMeta *meta = (SMeta *)pMeta;
+  ret = tdbTbOpen(ttlTbname, sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, meta->pEnv, &pTtlMgr->pOldTtlIdx, 0);
+  if (ret < 0) {
+    metaError("failed to open %s index since %s", ttlTbname, tstrerror(terrno));
+    goto _out;
+  }
 
-  if (pTtlMgr->pOldTtlIdx) {
+  ret = ttlMgrConvert(pTtlMgr->pOldTtlIdx, pTtlMgr->pTtlIdx, pMeta);
-    ret = ttlMgrConvert(pTtlMgr->pOldTtlIdx, pTtlMgr->pTtlIdx, pMeta);
+  if (ret < 0) {
-    if (ret < 0) {
+    metaError("failed to convert ttl index since %s", tstrerror(terrno));
-      metaError("failed to convert ttl index since %s", tstrerror(terrno));
+    goto _out;
-      goto _out;
+  }
-    }
 
-    ret = tdbTbDropByName(ttlTbname, meta->pEnv, meta->txn);
+  ret = tdbTbDropByName(ttlTbname, meta->pEnv, meta->txn);
-    if (ret < 0) {
+  if (ret < 0) {
-      metaError("failed to drop old ttl index since %s", tstrerror(terrno));
+    metaError("failed to drop old ttl index since %s", tstrerror(terrno));
-      goto _out;
+    goto _out;
-    }
-
-    tdbTbClose(pTtlMgr->pOldTtlIdx);
-    pTtlMgr->pOldTtlIdx = NULL;
   }
 
   ret = ttlMgrFillCache(pTtlMgr);
@@ -111,13 +120,23 @@ int ttlMgrBegin(STtlManger *pTtlMgr, void *pMeta) {
   }
 
   int64_t endNs = taosGetTimestampNs();
-
+  metaInfo("ttl mgr upgrade end, hash size: %d, time consumed: %" PRId64 " ns", taosHashGetSize(pTtlMgr->pTtlCache),
-  metaInfo("ttl mgr open end, hash size: %d, time consumed: %" PRId64 " ns", taosHashGetSize(pTtlMgr->pTtlCache),
            endNs - startNs);
 _out:
+  tdbTbClose(pTtlMgr->pOldTtlIdx);
+  pTtlMgr->pOldTtlIdx = NULL;
+
   return ret;
 }
 
+static void ttlMgrCleanup(STtlManger *pTtlMgr) {
+  taosHashCleanup(pTtlMgr->pTtlCache);
+  taosHashCleanup(pTtlMgr->pDirtyUids);
+  tdbTbClose(pTtlMgr->pTtlIdx);
+  taosThreadRwlockDestroy(&pTtlMgr->lock);
+  tdbOsFree(pTtlMgr);
+}
+
 static void ttlMgrBuildKey(STtlIdxKeyV1 *pTtlKey, int64_t ttlDays, int64_t changeTimeMs, tb_uid_t uid) {
   if (ttlDays <= 0) return;
 
@@ -205,7 +224,7 @@ _out:
   return ret;
 }
 
-int ttlMgrConvert(TTB *pOldTtlIdx, TTB *pNewTtlIdx, void *pMeta) {
+static int ttlMgrConvert(TTB *pOldTtlIdx, TTB *pNewTtlIdx, void *pMeta) {
   SMeta *meta = pMeta;
 
   metaInfo("ttlMgr convert ttl start.");

source/dnode/vnode/src/tq/tq.c

@@ -560,43 +560,32 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) {
   SMqDataRsp dataRsp = {0};
   tqInitDataRsp(&dataRsp, &req);
 
-  STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, req.subKey);
+  if (req.useSnapshot == true) {
-  if (pOffset != NULL) {
+    tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s snapshot not support wal info", consumerId, vgId, req.subKey);
-    if (pOffset->val.type != TMQ_OFFSET__LOG) {
+    terrno = TSDB_CODE_INVALID_PARA;
-      tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s use snapshot, no valid wal info", consumerId, vgId, req.subKey);
+    tDeleteMqDataRsp(&dataRsp);
-      terrno = TSDB_CODE_INVALID_PARA;
+    return -1;
-      tDeleteMqDataRsp(&dataRsp);
-      return -1;
-    }
-
-    dataRsp.rspOffset.type = TMQ_OFFSET__LOG;
-    dataRsp.rspOffset.version = pOffset->val.version;
-    tqInfo("consumer:0x%" PRIx64 " vgId:%d subkey:%s get assignment from store:%"PRId64, consumerId, vgId, req.subKey, dataRsp.rspOffset.version);
-  } else {
-    if (req.useSnapshot == true) {
-      tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s snapshot not support wal info", consumerId, vgId, req.subKey);
-      terrno = TSDB_CODE_INVALID_PARA;
-      tDeleteMqDataRsp(&dataRsp);
-      return -1;
-    }
-
-    dataRsp.rspOffset.type = TMQ_OFFSET__LOG;
-
-    if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEST) {
-      dataRsp.rspOffset.version = sver;  // not consume yet, set the earliest position
-    } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) {
-      dataRsp.rspOffset.version = ever;
-    } else {
-      tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s invalid offset type:%d", consumerId, vgId, req.subKey,
-              reqOffset.type);
-      terrno = TSDB_CODE_INVALID_PARA;
-      tDeleteMqDataRsp(&dataRsp);
-      return -1;
-    }
-    tqInfo("consumer:0x%" PRIx64 " vgId:%d subkey:%s get assignment from init:%"PRId64, consumerId, vgId, req.subKey, dataRsp.rspOffset.version);
   }
 
+  dataRsp.rspOffset.type = TMQ_OFFSET__LOG;
+
+  if (reqOffset.type == TMQ_OFFSET__LOG) {
+    dataRsp.rspOffset.version = reqOffset.version;
+  } else if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEST) {
+    dataRsp.rspOffset.version = sver;  // not consume yet, set the earliest position
+  } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) {
+    dataRsp.rspOffset.version = ever;
+  } else {
+    tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s invalid offset type:%d", consumerId, vgId, req.subKey,
+            reqOffset.type);
+    terrno = TSDB_CODE_INVALID_PARA;
+    tDeleteMqDataRsp(&dataRsp);
+    return -1;
+  }
+  tqInfo("consumer:0x%" PRIx64 " vgId:%d subkey:%s get assignment from init:%"PRId64, consumerId, vgId, req.subKey, dataRsp.rspOffset.version);
+
   tqDoSendDataRsp(&pMsg->info, &dataRsp, req.epoch, req.consumerId, TMQ_MSG_TYPE__WALINFO_RSP, sver, ever);
+  tDeleteMqDataRsp(&dataRsp);
   return 0;
 }
 

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -1031,7 +1031,7 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache
 
   return code;
 }
-
+/*
 int32_t tsdbCacheGet(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, int8_t ltype) {
   int32_t    code = 0;
   SLRUCache *pCache = pTsdb->lruCache;
@@ -1079,7 +1079,7 @@ int32_t tsdbCacheGet(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsR
 
   return code;
 }
-
+*/
 int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey) {
   int32_t code = 0;
   // fetch schema
@@ -1829,10 +1829,11 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
       }
 
       *pIgnoreEarlierTs = false;
+      /*
       if (!hasVal) {
         state->state = SFSLASTNEXTROW_FILESET;
       }
-
+      */
       if (!state->checkRemainingRow) {
         state->checkRemainingRow = true;
       }
@@ -2020,10 +2021,9 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
         tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetDataBlk);
         if (block.maxKey.ts <= state->lastTs) {
           *pIgnoreEarlierTs = true;
-          if (state->pBlockData) {
+
-            tBlockDataDestroy(state->pBlockData);
+          tBlockDataDestroy(state->pBlockData);
-            state->pBlockData = NULL;
+          state->pBlockData = NULL;
-          }
 
           *ppRow = NULL;
           return code;
@@ -3176,97 +3176,46 @@ static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray,
 
     TSKEY rowTs = TSDBROW_TS(pRow);
 
-    if (lastRowTs == TSKEY_MAX) {
+    lastRowTs = rowTs;
-      lastRowTs = rowTs;
 
-      for (int16_t iCol = noneCol; iCol < nCols; ++iCol) {
-        if (iCol >= nLastCol) {
-          break;
-        }
-        SLastCol *pCol = taosArrayGet(pColArray, iCol);
-        if (pCol->colVal.cid != pTSchema->columns[slotIds[iCol]].colId) {
-          continue;
-        }
-        if (slotIds[iCol] == 0) {
-          STColumn *pTColumn = &pTSchema->columns[0];
-
-          *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = rowTs});
-          taosArraySet(pColArray, 0, &(SLastCol){.ts = rowTs, .colVal = *pColVal});
-          continue;
-        }
-        tsdbRowGetColVal(pRow, pTSchema, slotIds[iCol], pColVal);
-
-        *pCol = (SLastCol){.ts = rowTs, .colVal = *pColVal};
-        if (IS_VAR_DATA_TYPE(pColVal->type) /*&& pColVal->value.nData > 0*/) {
-          pCol->colVal.value.pData = taosMemoryMalloc(pCol->colVal.value.nData);
-          if (pCol->colVal.value.pData == NULL) {
-            terrno = TSDB_CODE_OUT_OF_MEMORY;
-            code = TSDB_CODE_OUT_OF_MEMORY;
-            goto _err;
-          }
-          if (pColVal->value.nData > 0) {
-            memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData);
-          }
-        }
-
-        /*if (COL_VAL_IS_NONE(pColVal)) {
-          if (!setNoneCol) {
-            noneCol = iCol;
-            setNoneCol = true;
-          }
-          } else {*/
-        int32_t aColIndex = taosArraySearchIdx(aColArray, &pColVal->cid, compareInt16Val, TD_EQ);
-        if (aColIndex >= 0) {
-          taosArrayRemove(aColArray, aColIndex);
-        }
-        //}
-      }
-      if (!setNoneCol) {
-        // done, goto return pColArray
-        break;
-      } else {
-        continue;
-      }
-    }
-
-    // merge into pColArray
-    setNoneCol = false;
     for (int16_t iCol = noneCol; iCol < nCols; ++iCol) {
       if (iCol >= nLastCol) {
         break;
       }
-      // high version's column value
+      SLastCol *pCol = taosArrayGet(pColArray, iCol);
-      SLastCol *lastColVal = (SLastCol *)taosArrayGet(pColArray, iCol);
+      if (pCol->colVal.cid != pTSchema->columns[slotIds[iCol]].colId) {
-      if (lastColVal->colVal.cid != pTSchema->columns[slotIds[iCol]].colId) {
         continue;
       }
-      SColVal *tColVal = &lastColVal->colVal;
+      if (slotIds[iCol] == 0) {
+        STColumn *pTColumn = &pTSchema->columns[0];
 
+        *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = rowTs});
+        taosArraySet(pColArray, 0, &(SLastCol){.ts = rowTs, .colVal = *pColVal});
+        continue;
+      }
       tsdbRowGetColVal(pRow, pTSchema, slotIds[iCol], pColVal);
-      if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) {
-        SLastCol lastCol = {.ts = rowTs, .colVal = *pColVal};
-        if (IS_VAR_DATA_TYPE(pColVal->type) && pColVal->value.nData > 0) {
-          SLastCol *pLastCol = (SLastCol *)taosArrayGet(pColArray, iCol);
-          taosMemoryFree(pLastCol->colVal.value.pData);
 
-          lastCol.colVal.value.pData = taosMemoryMalloc(lastCol.colVal.value.nData);
+      *pCol = (SLastCol){.ts = rowTs, .colVal = *pColVal};
-          if (lastCol.colVal.value.pData == NULL) {
+      if (IS_VAR_DATA_TYPE(pColVal->type) /*&& pColVal->value.nData > 0*/) {
-            terrno = TSDB_CODE_OUT_OF_MEMORY;
+        pCol->colVal.value.pData = taosMemoryMalloc(pCol->colVal.value.nData);
-            code = TSDB_CODE_OUT_OF_MEMORY;
+        if (pCol->colVal.value.pData == NULL) {
-            goto _err;
+          terrno = TSDB_CODE_OUT_OF_MEMORY;
-          }
+          code = TSDB_CODE_OUT_OF_MEMORY;
-          memcpy(lastCol.colVal.value.pData, pColVal->value.pData, pColVal->value.nData);
+          goto _err;
         }
+        if (pColVal->value.nData > 0) {
+          memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData);
+        }
+      }
 
-        taosArraySet(pColArray, iCol, &lastCol);
+      int32_t aColIndex = taosArraySearchIdx(aColArray, &pColVal->cid, compareInt16Val, TD_EQ);
-        int32_t aColIndex = taosArraySearchIdx(aColArray, &lastCol.colVal.cid, compareInt16Val, TD_EQ);
+      if (aColIndex >= 0) {
         taosArrayRemove(aColArray, aColIndex);
-      } else if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal) && !setNoneCol) {
-        noneCol = iCol;
-        setNoneCol = true;
       }
     }
-  } while (setNoneCol);
+
+    break;
+  } while (1);
 
   if (!hasRow) {
     if (ignoreEarlierTs) {

source/dnode/vnode/src/vnd/vnodeCommit.c

@@ -360,7 +360,12 @@ static int32_t vnodeCommitTask(void *arg) {
 
   // commit
   code = vnodeCommitImpl(pInfo);
-  if (code) goto _exit;
+  if (code) {
+    vFatal("vgId:%d, failed to commit vnode since %s", TD_VID(pVnode), terrstr());
+    taosMsleep(100);
+    exit(EXIT_FAILURE);
+    goto _exit;
+  }
 
   vnodeReturnBufPool(pVnode);
 

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -76,7 +76,7 @@ int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, STfs *p
   }
 
   SSyncCfg *pCfg = &info.config.syncCfg;
- 
+
   pCfg->replicaNum = 0;
   pCfg->totalReplicaNum = 0;
   memset(&pCfg->nodeInfo, 0, sizeof(pCfg->nodeInfo));
@@ -109,7 +109,7 @@ int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, STfs *p
     pCfg->myIndex = pReq->replica + pReq->learnerSelfIndex;
   }
 
-  vInfo("vgId:%d, save config while alter, replicas:%d totalReplicas:%d selfIndex:%d", 
+  vInfo("vgId:%d, save config while alter, replicas:%d totalReplicas:%d selfIndex:%d",
             pReq->vgId, pCfg->replicaNum, pCfg->totalReplicaNum, pCfg->myIndex);
 
   info.config.syncCfg = *pCfg;
@@ -371,6 +371,10 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
     goto _err;
   }
 
+  if (metaUpgrade(pVnode, &pVnode->pMeta) < 0) {
+    vError("vgId:%d, failed to upgrade meta since %s", TD_VID(pVnode), tstrerror(terrno));
+  }
+
   // open tsdb
   if (!VND_IS_RSMA(pVnode) && tsdbOpen(pVnode, &VND_TSDB(pVnode), VNODE_TSDB_DIR, NULL, rollback) < 0) {
     vError("vgId:%d, failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno));

source/libs/catalog/src/catalog.c

@@ -341,13 +341,10 @@ int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pReq,
   SCtgAuthReq req = {0};
   req.pRawReq = pReq;
   req.pConn = pConn;
-  req.onlyCache = exists ? true : false;
+  req.onlyCache = false;
   CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pConn, pReq->user, &req.authInfo, NULL));
 
   CTG_ERR_JRET(ctgChkSetAuthRes(pCtg, &req, &rsp));
-  if (rsp.metaNotExists && exists) {
-    *exists = false;
-  }
 
 _return:
 

source/libs/catalog/src/ctgCache.c

@@ -1721,9 +1721,7 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam
   ctgDebug("stb 0x%" PRIx64 " updated to cache, dbFName:%s, tbName:%s, tbType:%d", meta->suid, dbFName, tbName,
            meta->tableType);
 
-  if (pCache) {
+  CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbId, meta->suid, pCache));
-    CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbId, meta->suid, pCache));
-  }
 
   return TSDB_CODE_SUCCESS;
 }

source/libs/catalog/src/ctgUtil.c

@@ -926,7 +926,6 @@ int32_t ctgGenerateVgList(SCatalog* pCtg, SHashObj* vgHash, SArray** pList) {
     }
 
     pIter = taosHashIterate(vgHash, pIter);
-    vgInfo = NULL;
   }
 
   *pList = vgList;

source/libs/command/src/command.c

@@ -87,7 +87,7 @@ static int32_t buildDescResultDataBlock(SSDataBlock** pOutput) {
   return code;
 }
 
-static void setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock, int32_t numOfRows, STableMeta* pMeta) {
+static int32_t setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock, int32_t numOfRows, STableMeta* pMeta) {
   blockDataEnsureCapacity(pBlock, numOfRows);
   pBlock->info.rows = 0;
 
@@ -114,6 +114,11 @@ static void setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock, in
     colDataSetVal(pCol4, pBlock->info.rows, buf, false);
     ++(pBlock->info.rows);
   }
+  if (pBlock->info.rows <= 0) {
+    qError("no permission to view any columns");
+    return TSDB_CODE_PAR_PERMISSION_DENIED;
+  }
+  return TSDB_CODE_SUCCESS;
 }
 
 static int32_t execDescribe(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** pRsp) {
@@ -123,7 +128,7 @@ static int32_t execDescribe(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp**
   SSDataBlock* pBlock = NULL;
   int32_t      code = buildDescResultDataBlock(&pBlock);
   if (TSDB_CODE_SUCCESS == code) {
-    setDescResultIntoDataBlock(sysInfoUser, pBlock, numOfRows, pDesc->pMeta);
+    code = setDescResultIntoDataBlock(sysInfoUser, pBlock, numOfRows, pDesc->pMeta);
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = buildRetrieveTableRsp(pBlock, DESCRIBE_RESULT_COLS, pRsp);

source/libs/function/src/udfd.c

@@ -29,6 +29,7 @@
 #include "tmsg.h"
 #include "trpc.h"
 #include "tmisce.h"
+#include "tversion.h"
 // clang-format on
 
 #define UDFD_MAX_SCRIPT_PLUGINS 64
@@ -1038,7 +1039,7 @@ int32_t udfdOpenClientRpc() {
   connLimitNum = TMIN(connLimitNum, 500);
   rpcInit.connLimitNum = connLimitNum;
   rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
-
+  taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
   global.clientRpc = rpcOpen(&rpcInit);
   if (global.clientRpc == NULL) {
     fnError("failed to init dnode rpc client");

source/libs/parser/src/parTranslater.c

@@ -6044,6 +6044,9 @@ static int32_t checkCollectTopicTags(STranslateContext* pCxt, SCreateTopicStmt*
 //    for (int32_t i = 0; i < pMeta->tableInfo.numOfColumns; ++i) {
       SSchema* column = &pMeta->schema[0];
       SColumnNode* col = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
+      if (NULL == col) {
+        return TSDB_CODE_OUT_OF_MEMORY;
+      }
       strcpy(col->colName, column->name);
       strcpy(col->node.aliasName, col->colName);
       strcpy(col->node.userAlias, col->colName);
@@ -6154,7 +6157,7 @@ static int32_t translateAlterLocal(STranslateContext* pCxt, SAlterLocalStmt* pSt
   char* p = strchr(pStmt->config, ' ');
   if (NULL != p) {
     *p = 0;
-    strcpy(pStmt->value, p + 1);
+    tstrncpy(pStmt->value, p + 1, sizeof(pStmt->value));
   }
   return TSDB_CODE_SUCCESS;
 }

source/libs/scheduler/src/schJob.c

@@ -135,6 +135,7 @@ int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus) {
         break;
       case JOB_TASK_STATUS_DROP:
         SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED);
+        break;
 
       default:
         SCH_JOB_ELOG("invalid job status:%s", jobTaskStatusStr(oriStatus));

source/libs/scheduler/src/schRemote.c

@@ -392,6 +392,7 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD
       // NEVER REACH HERE
       SCH_TASK_ELOG("invalid status to handle drop task rsp, refId:0x%" PRIx64, pJob->refId);
       SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
+      break;
     }
     case TDMT_SCH_LINK_BROKEN:
       SCH_TASK_ELOG("link broken received, error:%x - %s", rspCode, tstrerror(rspCode));

source/libs/scheduler/src/schTask.c

@@ -961,7 +961,6 @@ int32_t schHandleExplainRes(SArray *pExplainRes) {
     localRsp->rsp.numOfPlans = 0;
     localRsp->rsp.subplanInfo = NULL;
     pTask = NULL;
-    pJob = NULL;
   }
 
 _return:

source/libs/sync/test/sync_test_lib/src/syncIO.c

@@ -21,6 +21,7 @@
 #include "tglobal.h"
 #include "ttimer.h"
 #include "tutil.h"
+#include "tversion.h"
 
 bool     gRaftDetailLog = false;
 SSyncIO *gSyncIO = NULL;
@@ -188,7 +189,7 @@ static int32_t syncIOStartInternal(SSyncIO *io) {
     rpcInit.idleTime = 100;
     rpcInit.user = "sync-io";
     rpcInit.connType = TAOS_CONN_CLIENT;
-
+    taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
     io->clientRpc = rpcOpen(&rpcInit);
     if (io->clientRpc == NULL) {
       sError("failed to initialize RPC");
@@ -209,7 +210,7 @@ static int32_t syncIOStartInternal(SSyncIO *io) {
     rpcInit.idleTime = 2 * 1500;
     rpcInit.parent = io;
     rpcInit.connType = TAOS_CONN_SERVER;
-
+    taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
     void *pRpc = rpcOpen(&rpcInit);
     if (pRpc == NULL) {
       sError("failed to start RPC server");
@@ -470,11 +471,10 @@ static void syncIOTickPing(void *param, void *tmrId) {
   taosTmrReset(syncIOTickPing, io->pingTimerMS, io, io->timerMgr, &io->pingTimer);
 }
 
-void syncEntryDestory(SSyncRaftEntry* pEntry) {}
+void syncEntryDestory(SSyncRaftEntry *pEntry) {}
 
-
+void syncUtilMsgNtoH(void *msg) {
-void syncUtilMsgNtoH(void* msg) {
+  SMsgHead *pHead = msg;
-  SMsgHead* pHead = msg;
   pHead->contLen = ntohl(pHead->contLen);
   pHead->vgId = ntohl(pHead->vgId);
 }
@@ -487,9 +487,9 @@ static inline bool syncUtilCanPrint(char c) {
   }
 }
 
-char* syncUtilPrintBin(char* ptr, uint32_t len) {
+char *syncUtilPrintBin(char *ptr, uint32_t len) {
   int64_t memLen = (int64_t)(len + 1);
-  char*   s = taosMemoryMalloc(memLen);
+  char   *s = taosMemoryMalloc(memLen);
   ASSERT(s != NULL);
   memset(s, 0, len + 1);
   memcpy(s, ptr, len);
@@ -502,13 +502,13 @@ char* syncUtilPrintBin(char* ptr, uint32_t len) {
   return s;
 }
 
-char* syncUtilPrintBin2(char* ptr, uint32_t len) {
+char *syncUtilPrintBin2(char *ptr, uint32_t len) {
   uint32_t len2 = len * 4 + 1;
-  char*    s = taosMemoryMalloc(len2);
+  char    *s = taosMemoryMalloc(len2);
   ASSERT(s != NULL);
   memset(s, 0, len2);
 
-  char* p = s;
+  char *p = s;
   for (int32_t i = 0; i < len; ++i) {
     int32_t n = sprintf(p, "%d,", ptr[i]);
     p += n;
@@ -516,7 +516,7 @@ char* syncUtilPrintBin2(char* ptr, uint32_t len) {
   return s;
 }
 
-void syncUtilU642Addr(uint64_t u64, char* host, int64_t len, uint16_t* port) {
+void syncUtilU642Addr(uint64_t u64, char *host, int64_t len, uint16_t *port) {
   uint32_t hostU32 = (uint32_t)((u64 >> 32) & 0x00000000FFFFFFFF);
 
   struct in_addr addr = {.s_addr = hostU32};
@@ -524,7 +524,7 @@ void syncUtilU642Addr(uint64_t u64, char* host, int64_t len, uint16_t* port) {
   *port = (uint16_t)((u64 & 0x00000000FFFF0000) >> 16);
 }
 
-uint64_t syncUtilAddr2U64(const char* host, uint16_t port) {
+uint64_t syncUtilAddr2U64(const char *host, uint16_t port) {
   uint32_t hostU32 = taosGetIpv4FromFqdn(host);
   if (hostU32 == (uint32_t)-1) {
     sError("failed to resolve ipv4 addr, host:%s", host);

source/libs/tdb/src/db/tdbBtree.c

@@ -345,7 +345,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL
     }
     *ppKey = pTKey;
     *pkLen = cd.kLen;
-    memcpy(*ppKey, cd.pKey, cd.kLen);
+    memcpy(*ppKey, cd.pKey, (size_t)cd.kLen);
   }
 
   if (ppVal) {
@@ -357,7 +357,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL
     }
     *ppVal = pTVal;
     *vLen = cd.vLen;
-    memcpy(*ppVal, cd.pVal, cd.vLen);
+    memcpy(*ppVal, cd.pVal, (size_t)cd.vLen);
   }
 
   if (TDB_CELLDECODER_FREE_KEY(&cd)) {
@@ -1793,7 +1793,7 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
 
   *ppKey = pKey;
   *kLen = cd.kLen;
-  memcpy(pKey, cd.pKey, cd.kLen);
+  memcpy(pKey, cd.pKey, (size_t)cd.kLen);
 
   if (ppVal) {
     if (cd.vLen > 0) {
@@ -1852,7 +1852,7 @@ int tdbBtreePrev(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
 
   *ppKey = pKey;
   *kLen = cd.kLen;
-  memcpy(pKey, cd.pKey, cd.kLen);
+  memcpy(pKey, cd.pKey, (size_t)cd.kLen);
 
   if (ppVal) {
     // TODO: vLen may be zero
@@ -1864,7 +1864,7 @@ int tdbBtreePrev(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
 
     *ppVal = pVal;
     *vLen = cd.vLen;
-    memcpy(pVal, cd.pVal, cd.vLen);
+    memcpy(pVal, cd.pVal, (size_t)cd.vLen);
   }
 
   ret = tdbBtcMoveToPrev(pBtc);

source/libs/tdb/src/db/tdbDb.c

@@ -62,7 +62,10 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i
   }
   memset(pDb->pgrHash, 0, tsize);
 
-  taosMulModeMkDir(dbname, 0755);
+  ret = taosMulModeMkDir(dbname, 0755);
+  if (ret < 0) {
+    return -1;
+  }
 
 #ifdef USE_MAINDB
   // open main db

source/libs/tdb/src/db/tdbPager.c

@@ -980,6 +980,7 @@ int tdbPagerRestoreJournals(SPager *pPager) {
     jname[dirLen] = '/';
     sprintf(jname + dirLen + 1, TDB_MAINDB_NAME "-journal.%" PRId64, *pTxnId);
     if (tdbPagerRestore(pPager, jname) < 0) {
+      taosArrayDestroy(pTxnList);
       tdbCloseDir(&pDir);
 
       tdbError("failed to restore file due to %s. jFileName:%s", strerror(errno), jname);

source/libs/transport/inc/transComm.h

@@ -154,6 +154,7 @@ typedef struct {
 
 #pragma pack(push, 1)
 
+#define TRANS_VER 2
 typedef struct {
   char version : 4;  // RPC version
   char comp : 2;     // compression algorithm, 0:no compression 1:lz4
@@ -166,6 +167,7 @@ typedef struct {
 
   uint64_t timestamp;
   char     user[TSDB_UNI_LEN];
+  int32_t  compatibilityVer;
   uint32_t magicNum;
   STraceId traceId;
   uint64_t ahandle;  // ahandle assigned by client

source/libs/transport/inc/transportInt.h

@@ -46,10 +46,10 @@ typedef struct {
   int8_t   connType;
   char     label[TSDB_LABEL_LEN];
   char     user[TSDB_UNI_LEN];  // meter ID
-
+  int32_t compatibilityVer;
-  int32_t compressSize;  // -1: no compress, 0 : all data compressed, size: compress data if larger than size
+  int32_t  compressSize;  // -1: no compress, 0 : all data compressed, size: compress data if larger than size
-  int8_t  encryption;    // encrypt or not
+  int8_t   encryption;    // encrypt or not
-
+  
   int32_t retryMinInterval;  // retry init interval
   int32_t retryStepFactor;   // retry interval factor
   int32_t retryMaxInterval;  // retry max interval

source/libs/transport/src/trans.c

@@ -50,6 +50,7 @@ void* rpcOpen(const SRpcInit* pInit) {
   }
 
   pRpc->encryption = pInit->encryption;
+  pRpc->compatibilityVer = pInit->compatibilityVer;
 
   pRpc->retryMinInterval = pInit->retryMinInterval;  // retry init interval
   pRpc->retryStepFactor = pInit->retryStepFactor;

source/libs/transport/src/transCli.c

@@ -391,6 +391,7 @@ void cliHandleResp(SCliConn* conn) {
   transMsg.info.ahandle = NULL;
   transMsg.info.traceId = pHead->traceId;
   transMsg.info.hasEpSet = pHead->hasEpSet;
+  transMsg.info.cliVer = htonl(pHead->compatibilityVer);
 
   SCliMsg*       pMsg = NULL;
   STransConnCtx* pCtx = NULL;
@@ -488,6 +489,7 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) {
     transMsg.code = code == -1 ? (pConn->broken ? TSDB_CODE_RPC_BROKEN_LINK : TSDB_CODE_RPC_NETWORK_UNAVAIL) : code;
     transMsg.msgType = pMsg ? pMsg->msg.msgType + 1 : 0;
     transMsg.info.ahandle = NULL;
+    transMsg.info.cliVer = pTransInst->compatibilityVer;
 
     if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(pConn)) {
       transMsg.info.ahandle = transCtxDumpVal(&pConn->ctx, transMsg.msgType);
@@ -984,11 +986,10 @@ void cliSendBatch(SCliConn* pConn) {
   SCliThrd* pThrd = pConn->hostThrd;
   STrans*   pTransInst = pThrd->pTransInst;
 
-  SCliBatch*     pBatch = pConn->pBatch;
+  SCliBatch* pBatch = pConn->pBatch;
-  SCliBatchList* pList = pBatch->pList;
+  int32_t    wLen = pBatch->wLen;
-  pList->connCnt += 1;
 
-  int32_t wLen = pBatch->wLen;
+  pBatch->pList->connCnt += 1;
 
   uv_buf_t* wb = taosMemoryCalloc(wLen, sizeof(uv_buf_t));
   int       i = 0;
@@ -1018,6 +1019,8 @@ void cliSendBatch(SCliConn* pConn) {
       memcpy(pHead->user, pTransInst->user, strlen(pTransInst->user));
       pHead->traceId = pMsg->info.traceId;
       pHead->magicNum = htonl(TRANS_MAGIC_NUM);
+      pHead->version = TRANS_VER;
+      pHead->compatibilityVer = htonl(pTransInst->compatibilityVer);
     }
     pHead->timestamp = taosHton64(taosGetTimestampUs());
 
@@ -1074,6 +1077,8 @@ void cliSend(SCliConn* pConn) {
     memcpy(pHead->user, pTransInst->user, strlen(pTransInst->user));
     pHead->traceId = pMsg->info.traceId;
     pHead->magicNum = htonl(TRANS_MAGIC_NUM);
+    pHead->version = TRANS_VER;
+    pHead->compatibilityVer = htonl(pTransInst->compatibilityVer);
   }
   pHead->timestamp = taosHton64(taosGetTimestampUs());
 
@@ -1346,6 +1351,7 @@ static void doNotifyApp(SCliMsg* pMsg, SCliThrd* pThrd) {
   transMsg.info.ahandle = pMsg->ctx->ahandle;
   transMsg.info.traceId = pMsg->msg.info.traceId;
   transMsg.info.hasEpSet = false;
+  transMsg.info.cliVer = pTransInst->compatibilityVer;
   if (pCtx->pSem != NULL) {
     if (pCtx->pRsp == NULL) {
     } else {
@@ -1527,6 +1533,9 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) {
     // persist conn already release by server
     STransMsg resp;
     cliBuildExceptResp(pMsg, &resp);
+    // refactorr later
+    resp.info.cliVer = pTransInst->compatibilityVer;
+
     if (pMsg->type != Release) {
       pTransInst->cfp(pTransInst->parent, &resp, NULL);
     }
@@ -1836,6 +1845,7 @@ void cliIteraConnMsgs(SCliConn* conn) {
     if (-1 == cliBuildExceptResp(cmsg, &resp)) {
       continue;
     }
+    resp.info.cliVer = pTransInst->compatibilityVer;
     pTransInst->cfp(pTransInst->parent, &resp, NULL);
 
     cmsg->ctx->ahandle = NULL;

source/libs/transport/src/transComm.c

@@ -192,7 +192,7 @@ bool transReadComplete(SConnBuffer* connBuf) {
       memcpy((char*)&head, connBuf->buf, sizeof(head));
       int32_t msgLen = (int32_t)htonl(head.msgLen);
       p->total = msgLen;
-      p->invalid = TRANS_NOVALID_PACKET(htonl(head.magicNum));
+      p->invalid = TRANS_NOVALID_PACKET(htonl(head.magicNum)) || head.version != TRANS_VER;
     }
     if (p->total >= p->len) {
       p->left = p->total - p->len;

source/libs/transport/src/transSvr.c

@@ -196,6 +196,7 @@ static bool uvHandleReq(SSvrConn* pConn) {
     tError("%s conn %p recv invalid packet, failed to decompress", transLabel(pTransInst), pConn);
     return false;
   }
+  tDebug("head version: %d 2", pHead->version);
 
   pHead->code = htonl(pHead->code);
   pHead->msgLen = htonl(pHead->msgLen);
@@ -236,8 +237,8 @@ static bool uvHandleReq(SSvrConn* pConn) {
   if (pConn->status == ConnNormal && pHead->noResp == 0) {
     transRefSrvHandle(pConn);
     if (cost >= EXCEPTION_LIMIT_US) {
-      tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, cost:%dus, recv exception", transLabel(pTransInst),
+      tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, cost:%dus, recv exception",
-             pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, (int)cost);
+              transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, (int)cost);
     } else {
       tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, cost:%dus", transLabel(pTransInst), pConn,
               TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, (int)cost);
@@ -245,8 +246,8 @@ static bool uvHandleReq(SSvrConn* pConn) {
   } else {
     if (cost >= EXCEPTION_LIMIT_US) {
       tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, noResp:%d, code:%d, cost:%dus, recv exception",
-             transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, pHead->noResp,
+              transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, pHead->noResp,
-             transMsg.code, (int)(cost));
+              transMsg.code, (int)(cost));
     } else {
       tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, noResp:%d, code:%d, cost:%dus",
               transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, pHead->noResp,
@@ -262,6 +263,7 @@ static bool uvHandleReq(SSvrConn* pConn) {
   transMsg.info.handle = (void*)transAcquireExHandle(transGetRefMgt(), pConn->refId);
   transMsg.info.refId = pConn->refId;
   transMsg.info.traceId = pHead->traceId;
+  transMsg.info.cliVer = htonl(pHead->compatibilityVer);
 
   tGTrace("%s handle %p conn:%p translated to app, refId:%" PRIu64, transLabel(pTransInst), transMsg.info.handle, pConn,
           pConn->refId);
@@ -410,6 +412,8 @@ static int uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) {
   pHead->traceId = pMsg->info.traceId;
   pHead->hasEpSet = pMsg->info.hasEpSet;
   pHead->magicNum = htonl(TRANS_MAGIC_NUM);
+  pHead->compatibilityVer = htonl(((STrans*)pConn->pTransInst)->compatibilityVer);
+  pHead->version = TRANS_VER;
 
   // handle invalid drop_task resp, TD-20098
   if (pConn->inType == TDMT_SCH_DROP_TASK && pMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) {

source/libs/transport/test/cliBench.c

@@ -19,6 +19,7 @@
 #include "transLog.h"
 #include "trpc.h"
 #include "tutil.h"
+#include "tversion.h"
 
 typedef struct {
   int      index;
@@ -155,7 +156,7 @@ int main(int argc, char *argv[]) {
   }
 
   initLogEnv();
-
+  taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
   void *pRpc = rpcOpen(&rpcInit);
   if (pRpc == NULL) {
     tError("failed to initialize RPC");

source/libs/transport/test/svrBench.c

@@ -13,12 +13,13 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-//#define _DEFAULT_SOURCE
+// #define _DEFAULT_SOURCE
 #include "os.h"
 #include "tglobal.h"
 #include "tqueue.h"
 #include "transLog.h"
 #include "trpc.h"
+#include "tversion.h"
 
 int         msgSize = 128;
 int         commit = 0;
@@ -151,6 +152,8 @@ int main(int argc, char *argv[]) {
   rpcInit.numOfThreads = 1;
   rpcInit.cfp = processRequestMsg;
   rpcInit.idleTime = 2 * 1500;
+
+  taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
   rpcDebugFlag = 131;
 
   for (int i = 1; i < argc; ++i) {
@@ -187,7 +190,7 @@ int main(int argc, char *argv[]) {
   rpcInit.connType = TAOS_CONN_SERVER;
 
   initLogEnv();
-
+  taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
   void *pRpc = rpcOpen(&rpcInit);
   if (pRpc == NULL) {
     tError("failed to start RPC server");

source/libs/transport/test/transUT.cpp

@@ -18,10 +18,10 @@
 #include "tdatablock.h"
 #include "tglobal.h"
 #include "tlog.h"
+#include "tmisce.h"
 #include "transLog.h"
 #include "trpc.h"
-#include "tmisce.h"
+#include "tversion.h"
-
 using namespace std;
 
 const char *label = "APP";
@@ -54,6 +54,8 @@ class Client {
     rpcInit_.user = (char *)user;
     rpcInit_.parent = this;
     rpcInit_.connType = TAOS_CONN_CLIENT;
+
+    taosVersionStrToInt(version, &(rpcInit_.compatibilityVer));
     this->transCli = rpcOpen(&rpcInit_);
     tsem_init(&this->sem, 0, 0);
   }
@@ -66,6 +68,7 @@ class Client {
   void Restart(CB cb) {
     rpcClose(this->transCli);
     rpcInit_.cfp = cb;
+    taosVersionStrToInt(version, &(rpcInit_.compatibilityVer));
     this->transCli = rpcOpen(&rpcInit_);
   }
   void Stop() {
@@ -117,6 +120,7 @@ class Server {
     rpcInit_.cfp = processReq;
     rpcInit_.user = (char *)user;
     rpcInit_.connType = TAOS_CONN_SERVER;
+    taosVersionStrToInt(version, &(rpcInit_.compatibilityVer));
   }
   void Start() {
     this->transSrv = rpcOpen(&this->rpcInit_);

source/libs/wal/src/walMeta.c

@@ -602,18 +602,18 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
   // ftruncate idx file
   if (offset < fileSize) {
     if (taosFtruncateFile(pIdxFile, offset) < 0) {
-      wError("vgId:%d, failed to ftruncate file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId,
-             strerror(errno), offset, fnameStr);
       terrno = TAOS_SYSTEM_ERROR(errno);
+      wError("vgId:%d, failed to ftruncate file since %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, terrstr(),
+             offset, fnameStr);
       goto _err;
     }
   }
 
   // rebuild idx file
   if (taosLSeekFile(pIdxFile, 0, SEEK_END) < 0) {
-    wError("vgId:%d, failed to seek file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno),
-           offset, fnameStr);
     terrno = TAOS_SYSTEM_ERROR(errno);
+    wError("vgId:%d, failed to seek file since %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, terrstr(), offset,
+           fnameStr);
     goto _err;
   }
 
@@ -625,11 +625,12 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
     idxEntry.offset += sizeof(SWalCkHead) + ckHead.head.bodyLen;
 
     if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) {
-      wError("vgId:%d, failed to read wal log head since %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, terrstr(),
+      wError("vgId:%d, failed to read wal log head since %s. index:%" PRId64 ", offset:%" PRId64 ", file:%s",
-             idxEntry.offset, fLogNameStr);
+             pWal->cfg.vgId, terrstr(), idxEntry.ver, idxEntry.offset, fLogNameStr);
       goto _err;
     }
     if (taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) < 0) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
       wError("vgId:%d, failed to append file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr);
       goto _err;
     }
@@ -637,6 +638,7 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
   }
 
   if (taosFsyncFile(pIdxFile) < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
     wError("vgId:%d, faild to fsync file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr);
     goto _err;
   }

source/libs/wal/src/walWrite.c

@@ -473,7 +473,10 @@ static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
   // check alignment of idx entries
   int64_t endOffset = taosLSeekFile(pWal->pIdxFile, 0, SEEK_END);
   if (endOffset < 0) {
-    wFatal("vgId:%d, failed to seek end of idxfile due to %s. ver:%" PRId64 "", pWal->cfg.vgId, strerror(errno), ver);
+    wFatal("vgId:%d, failed to seek end of WAL idxfile due to %s. ver:%" PRId64 "", pWal->cfg.vgId, strerror(errno),
+           ver);
+    taosMsleep(100);
+    exit(EXIT_FAILURE);
   }
   return 0;
 }
@@ -533,16 +536,20 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
 END:
   // recover in a reverse order
   if (taosFtruncateFile(pWal->pLogFile, offset) < 0) {
-    wFatal("vgId:%d, failed to ftruncate logfile to offset:%" PRId64 " during recovery due to %s", pWal->cfg.vgId,
-           offset, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
+    wFatal("vgId:%d, failed to recover WAL logfile from write error since %s, offset:%" PRId64, pWal->cfg.vgId,
+           terrstr(), offset);
+    taosMsleep(100);
+    exit(EXIT_FAILURE);
   }
 
   int64_t idxOffset = (index - pFileInfo->firstVer) * sizeof(SWalIdxEntry);
   if (taosFtruncateFile(pWal->pIdxFile, idxOffset) < 0) {
-    wFatal("vgId:%d, failed to ftruncate idxfile to offset:%" PRId64 "during recovery due to %s", pWal->cfg.vgId,
-           idxOffset, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
+    wFatal("vgId:%d, failed to recover WAL idxfile from write error since %s, offset:%" PRId64, pWal->cfg.vgId,
+           terrstr(), idxOffset);
+    taosMsleep(100);
+    exit(EXIT_FAILURE);
   }
   return -1;
 }

source/util/src/tlog.c

@@ -486,24 +486,11 @@ static inline int32_t taosBuildLogHead(char *buffer, const char *flags) {
 static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *buffer, int32_t len) {
   if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile != NULL && osLogSpaceAvailable()) {
     taosUpdateLogNums(level);
-#if 0 
-    // DEBUG_FATAL and DEBUG_ERROR are duplicated
-    // fsync will cause thread blocking and may also generate log misalignment in case of asyncLog
-    if (tsAsyncLog && level != DEBUG_FATAL) {
-      taosPushLogBuffer(tsLogObj.logHandle, buffer, len);
-    } else {
-      taosWriteFile(tsLogObj.logHandle->pFile, buffer, len);
-      if (level == DEBUG_FATAL) {
-        taosFsyncFile(tsLogObj.logHandle->pFile);
-      }
-    }
-#else
     if (tsAsyncLog) {
       taosPushLogBuffer(tsLogObj.logHandle, buffer, len);
     } else {
       taosWriteFile(tsLogObj.logHandle->pFile, buffer, len);
     }
-#endif
 
     if (tsLogObj.maxLines > 0) {
       atomic_add_fetch_32(&tsLogObj.lines, 1);

tests/parallel_test/cases.task

@@ -36,6 +36,10 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxTopic.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqParamsTest.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqClientConsLog.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxGroupIds.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumeDiscontinuousData.py
+,,n,system-test,python3 ./test.py -f 7-tmq/tmqDropConsumer.py
+
 
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_stable.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 3

tests/system-test/0-others/compatibility.py

@@ -152,6 +152,7 @@ class TDTestCase:
         os.system("LD_LIBRARY_PATH=/usr/lib  taos -f 0-others/TS-3131.tsql")
 
         cmd = f" LD_LIBRARY_PATH={bPath}/build/lib  {bPath}/build/bin/taos -h localhost ;"
+        tdLog.info(f"new  client version  connect to old version taosd, commad return value:{cmd}")
         if os.system(cmd) == 0:
             raise Exception("failed to execute system command. cmd: %s" % cmd)
                 

tests/system-test/7-tmq/tmqCommon.py

@@ -37,6 +37,9 @@ from util.common import *
 #     INSERT_DATA     = 3
 
 class TMQCom:
+    def __init__(self):
+        self.g_end_insert_flag = 0
+    
     def init(self, conn, logSql, replicaVar=1):
         self.replicaVar = int(replicaVar)
         tdSql.init(conn.cursor())
@@ -330,8 +333,11 @@ class TMQCom:
             ctbDict[i] = 0
 
         #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
-        rowsOfCtb = 0
+        rowsOfCtb = 0        
         while rowsOfCtb < rowsPerTbl:
+            if (0 != self.g_end_insert_flag):
+                tdLog.debug("get signal to stop insert data")
+                break
             for i in range(ctbNum):
                 sql += " %s.%s%d values "%(dbName,ctbPrefix,i+ctbStartIdx)
                 rowsBatched = 0
@@ -571,6 +577,20 @@ class TMQCom:
         tdLog.info(tsql.queryResult)
         tdLog.info("wait subscriptions exit for %d s"%wait_cnt)
 
+    def killProcesser(self, processerName):
+        killCmd = (
+            "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -TERM > /dev/null 2>&1"
+            % processerName
+        )
+
+        psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % processerName
+        processID = subprocess.check_output(psCmd, shell=True)
+
+        while processID:
+            os.system(killCmd)
+            time.sleep(1)
+            processID = subprocess.check_output(psCmd, shell=True)
+
     def close(self):
         self.cursor.close()
 

tests/system-test/7-tmq/tmqConsumeDiscontinuousData.py

@@ -0,0 +1,248 @@
+
+import sys
+import time
+import datetime
+import threading
+from taos.tmq import Consumer
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+class TDTestCase:
+    updatecfgDict = {'debugFlag': 135}    
+    
+    def __init__(self):
+        self.vgroups    = 1
+        self.ctbNum     = 10
+        self.rowsPerTbl = 100
+        self.tmqMaxTopicNum = 1
+        self.tmqMaxGroups = 1
+        self.walRetentionPeriod = 3
+        self.actConsumeTotalRows = 0
+        self.retryPoll = 0
+        self.lock = threading.Lock()
+
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor(), False)
+
+    def getPath(self, tool="taosBenchmark"):
+        if (platform.system().lower() == 'windows'):
+            tool = tool + ".exe"
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        paths = []
+        for root, dirs, files in os.walk(projPath):
+            if ((tool) in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    paths.append(os.path.join(root, tool))
+                    break
+        if (len(paths) == 0):
+            tdLog.exit("taosBenchmark not found!")
+            return
+        else:
+            tdLog.info("taosBenchmark found in %s" % paths[0])
+            return paths[0]
+
+    def prepareTestEnv(self):
+        tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 10,
+                    'batchNum':   1,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  10,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   1}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+
+        tmqCom.initConsumerTable()
+        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
+        tdSql.execute("alter database %s wal_retention_period %d" % (paraDict['dbName'], self.walRetentionPeriod))
+        tdLog.info("create stb")
+        tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
+        tdLog.info("create ctb")
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
+                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
+        # tdLog.info("insert data")
+        # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
+        #                                        ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+        #                                        startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+
+        # tdLog.info("restart taosd to ensure that the data falls into the disk")
+        # tdDnodes.stop(1)
+        # tdDnodes.start(1)
+        # tdSql.query("flush database %s"%(paraDict['dbName']))
+        return
+
+    def tmqSubscribe(self, **inputDict):
+        consumer_dict = {
+            "group.id": inputDict['group_id'],
+            "client.id": "client",
+            "td.connect.user": "root",
+            "td.connect.pass": "taosdata",
+            "auto.commit.interval.ms": "100",
+            "enable.auto.commit": "true",
+            "auto.offset.reset": "earliest",
+            "experimental.snapshot.enable": "false",
+            "msg.with.table.name": "false"
+        }
+        
+        consumer = Consumer(consumer_dict)
+        consumer.subscribe([inputDict['topic_name']])
+        onceFlag = 0
+        try:
+            while True:
+                if (1 == self.retryPoll):
+                    time.sleep(2)
+                    continue
+                res = consumer.poll(inputDict['pollDelay'])        
+                if not res:
+                    break
+                err = res.error()
+                if err is not None:
+                    raise err
+                
+                val = res.value()
+                for block in val:
+                    # print(block.fetchall())
+                    data = block.fetchall()
+                    for row in data:
+                        # print("===================================")
+                        # print(row)
+                        self.actConsumeTotalRows += 1
+                    if (0 == onceFlag):
+                        onceFlag = 1
+                        with self.lock:
+                            self.retryPoll = 1
+                            currentTime = datetime.now()      
+                            print("%s temp stop consume"%(str(currentTime)))
+                    
+                currentTime = datetime.now()        
+                print("%s already consume rows: %d, and sleep for a while"%(str(currentTime), self.actConsumeTotalRows))
+                # time.sleep(self.walRetentionPeriod * 3)
+        finally:
+            consumer.unsubscribe()
+            consumer.close()
+
+        return
+
+    def asyncSubscribe(self, inputDict):
+        pThread = threading.Thread(target=self.tmqSubscribe, kwargs=inputDict)
+        pThread.start()
+        return pThread
+    
+    def tmqCase1(self):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 100,
+                    'batchNum':   1,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  3,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   1}
+
+        # create topic
+        topicNameList = ['dbtstb_0001']
+        tdLog.info("create topics from stb")
+        queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
+        for i in range(len(topicNameList)):
+            sqlString = "create topic %s as %s" %(topicNameList[i], queryString)
+            tdLog.info("create topic sql: %s"%sqlString)
+            tdSql.execute(sqlString)
+
+
+
+        # start consumer
+        inputDict = {'group_id':   "grpid_0001",
+                     'topic_name': topicNameList[0],
+                     'pollDelay':  10
+        }
+        
+        pThread2 = self.asyncSubscribe(inputDict)
+        
+        pThread1 = tmqCom.asyncInsertDataByInterlace(paraDict)
+        pThread1.join()
+        tdLog.info("firstly call to flash database")
+        tdSql.query("flush database %s"%(paraDict['dbName']))
+        time.sleep(self.walRetentionPeriod + 1)
+        tdLog.info("secondely call to flash database")
+        tdSql.query("flush database %s"%(paraDict['dbName']))
+        
+        # wait the consumer to complete one poll
+        while (0 == self.retryPoll):
+            time.sleep(1)
+            continue
+        
+        with self.lock:
+            self.retryPoll = 0            
+            currentTime = datetime.now()      
+            print("%s restart consume"%(str(currentTime)))
+        
+        paraDict["startTs"] = 1640966400000 + paraDict["ctbNum"] * paraDict["rowsPerTbl"]
+        pThread3 = tmqCom.asyncInsertDataByInterlace(paraDict)
+        
+                
+        tdLog.debug("wait sub-thread to end insert data")        
+        pThread3.join()
+        
+        totalInsertRows = paraDict["ctbNum"] * paraDict["rowsPerTbl"] * 2
+        tdLog.debug("wait sub-thread to end consume data")
+        pThread2.join()
+        
+        tdLog.info("act consume total rows: %d, act insert total rows: %d"%(self.actConsumeTotalRows, totalInsertRows))
+        
+        if (self.actConsumeTotalRows >= totalInsertRows):
+            tdLog.exit("act consume rows: %d not equal expect: %d"%(self.actConsumeTotalRows, totalInsertRows))
+        
+        tdLog.printNoPrefix("======== test case 1 end ...... ")
+
+    def run(self):
+        self.prepareTestEnv()
+        self.tmqCase1()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/7-tmq/tmqDropConsumer.json

@@ -0,0 +1,28 @@
+{
+    "filetype": "subscribe",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "result_file": "tmq_res.txt",
+    "tmq_info": {
+        "concurrent": 2,
+        "poll_delay": 100000,
+        "group.id": "",
+        "group_mode": "independent",
+        "create_mode": "parallel",
+        "client.id": "cliid_0001",
+        "auto.offset.reset": "earliest",
+        "enable.manual.commit": "false",
+        "enable.auto.commit": "false",
+        "auto.commit.interval.ms": 1000,
+        "experimental.snapshot.enable": "false",
+        "msg.with.table.name": "false",
+        "rows_file": "",
+        "topic_list": [
+            {"name": "dbtstb_0001", "sql": "select * from dbt.stb;"},
+            {"name": "dbtstb_0002", "sql": "select * from dbt.stb;"}
+        ]
+    }
+}

tests/system-test/7-tmq/tmqDropConsumer.py

@@ -0,0 +1,293 @@
+
+import sys
+import time
+import threading
+from taos.tmq import Consumer
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+class TDTestCase:
+    updatecfgDict = {'debugFlag': 135}    
+    
+    def __init__(self):
+        self.vgroups    = 2
+        self.ctbNum     = 10
+        self.rowsPerTbl = 10
+        self.tmqMaxTopicNum = 2
+        self.tmqMaxGroups = 2
+
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor(), False)
+
+    def getPath(self, tool="taosBenchmark"):
+        if (platform.system().lower() == 'windows'):
+            tool = tool + ".exe"
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        paths = []
+        for root, dirs, files in os.walk(projPath):
+            if ((tool) in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    paths.append(os.path.join(root, tool))
+                    break
+        if (len(paths) == 0):
+            tdLog.exit("taosBenchmark not found!")
+            return
+        else:
+            tdLog.info("taosBenchmark found in %s" % paths[0])
+            return paths[0]
+
+    def prepareTestEnv(self):
+        tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    2,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 10,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  10,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   1}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+
+        tmqCom.initConsumerTable()
+        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
+        tdSql.execute("alter database %s wal_retention_period 360000" % (paraDict['dbName']))
+        tdLog.info("create stb")
+        tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
+        tdLog.info("create ctb")
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
+                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
+        tdLog.info("insert data")
+        tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
+                                               ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+                                               startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+
+        tdLog.info("restart taosd to ensure that the data falls into the disk")
+        # tdDnodes.stop(1)
+        # tdDnodes.start(1)
+        tdSql.query("flush database %s"%(paraDict['dbName']))
+        return
+
+    def tmqSubscribe(self, topicName, newGroupId, expectResult):
+        # create new connector for new tdSql instance in my thread
+        # newTdSql = tdCom.newTdSql()
+        # topicName = inputDict['topic_name']
+        # group_id = inputDict['group_id']
+        
+        consumer_dict = {
+            "group.id": newGroupId,
+            "client.id": "client",
+            "td.connect.user": "root",
+            "td.connect.pass": "taosdata",
+            "auto.commit.interval.ms": "1000",
+            "enable.auto.commit": "true",
+            "auto.offset.reset": "earliest",
+            "experimental.snapshot.enable": "false",
+            "msg.with.table.name": "false"
+        }
+        
+        ret = 'success'
+        consumer = Consumer(consumer_dict)
+        # print("======%s"%(inputDict['topic_name']))
+        try:
+            consumer.subscribe([topicName])
+        except Exception as e:
+            tdLog.info("consumer.subscribe() fail ")
+            tdLog.info("%s"%(e))
+            if (expectResult == "fail"):
+                consumer.close()
+                return 'success'
+            else: 
+                consumer.close()
+                return 'fail'
+    
+        tdLog.info("consumer.subscribe() success ")
+        if (expectResult == "success"):
+            consumer.close()
+            return 'success'
+        else: 
+            consumer.close()
+            return 'fail'
+
+    def tmqCase1(self):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 100000000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  3,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   1}
+
+        topicNameList = ['dbtstb_0001','dbtstb_0002']
+        tdLog.info("create topics from stb")
+        queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
+        for i in range(len(topicNameList)):
+            sqlString = "create topic %s as %s" %(topicNameList[i], queryString)
+            tdLog.info("create topic sql: %s"%sqlString)
+            tdSql.execute(sqlString)
+
+        # tdSql.query('show topics;')
+        # topicNum = tdSql.queryRows
+        # tdLog.info(" topic count: %d"%(topicNum))
+        # if topicNum != len(topicNameList):
+        #     tdLog.exit("show topics %d not equal expect num: %d"%(topicNum, len(topicNameList)))
+        
+        pThread = tmqCom.asyncInsertDataByInterlace(paraDict)
+        
+        # use taosBenchmark to subscribe  
+        binPath = self.getPath()
+        cmd = "nohup %s -f ./7-tmq/tmqDropConsumer.json > /dev/null 2>&1 & " % binPath
+        tdLog.info("%s"%(cmd))
+        os.system(cmd)
+                
+        expectTopicNum = len(topicNameList)
+        consumerThreadNum = 2
+        expectConsumerNUm = expectTopicNum * consumerThreadNum
+        expectSubscribeNum = self.vgroups * expectTopicNum * consumerThreadNum
+                
+        tdSql.query('show topics;')
+        topicNum = tdSql.queryRows
+        tdLog.info(" get topic count: %d"%(topicNum))
+        if topicNum != expectTopicNum:
+            tdLog.exit("show topics %d not equal expect num: %d"%(topicNum, expectTopicNum))
+        
+        flag = 0
+        while (1):        
+            tdSql.query('show consumers;')
+            consumerNUm = tdSql.queryRows
+            tdLog.info(" get consumers count: %d"%(consumerNUm))
+            if consumerNUm == expectConsumerNUm:
+                flag = 1
+                break
+            else:
+                time.sleep(1)
+
+        if (0 == flag):
+            tmqCom.g_end_insert_flag = 1
+            tdLog.exit("show consumers %d not equal expect num: %d"%(topicNum, expectConsumerNUm))
+
+        flag = 0
+        for i in range(10):                  
+            tdSql.query('show subscriptions;')
+            subscribeNum = tdSql.queryRows
+            tdLog.info(" get subscriptions count: %d"%(subscribeNum))
+            if subscribeNum == expectSubscribeNum:
+                flag = 1
+                break
+            else:
+                time.sleep(1)
+                
+        if (0 == flag):
+            tmqCom.g_end_insert_flag = 1
+            tdLog.exit("show subscriptions %d not equal expect num: %d"%(subscribeNum, expectSubscribeNum))
+                
+        # get all consumer group id
+        tdSql.query('show consumers;')
+        consumerNUm = tdSql.queryRows 
+        groupIdList = []
+        for i in range(consumerNUm): 
+            groupId = tdSql.getData(i,1)
+            existFlag = 0
+            for j in range(len(groupIdList)): 
+                if (groupId == groupIdList[j]):
+                    existFlag = 1
+                    break
+            if (0 == existFlag):
+                groupIdList.append(groupId)
+        
+        # kill taosBenchmark        
+        tmqCom.killProcesser("taosBenchmark")
+        tdLog.info("kill taosBenchmak end")
+        
+        # wait the status to "lost"
+        while (1):
+            exitFlag = 1
+            tdSql.query('show consumers;')
+            consumerNUm = tdSql.queryRows 
+            for i in range(consumerNUm): 
+                status = tdSql.getData(i,3)
+                if (status != "lost"):
+                    exitFlag = 0
+                    time.sleep(2)
+                    break
+            if (1 == exitFlag):
+                break
+        
+        tdLog.info("all consumers status into 'lost'")
+        
+        # drop consumer groups
+        tdLog.info("drop all consumers")
+        for i in range(len(groupIdList)): 
+            for j in range(len(topicNameList)): 
+                sqlCmd = f"drop consumer group `%s` on %s"%(groupIdList[i], topicNameList[j])
+                tdLog.info("drop consumer cmd: %s"%(sqlCmd))
+                tdSql.execute(sqlCmd)
+                
+        tmqCom.g_end_insert_flag = 1
+        tdLog.debug("notify sub-thread to stop insert data")
+        pThread.join()
+        
+        tdSql.query('show consumers;')
+        consumerNUm = tdSql.queryRows
+
+        tdSql.query('show subscriptions;')
+        subscribeNum = tdSql.queryRows
+        
+        if (0 != consumerNUm or 0 != subscribeNum):
+            tdLog.exit("drop consumer fail! consumerNUm %d, subscribeNum: %d"%(consumerNUm, subscribeNum))
+
+        tdLog.info("drop consuer success, there is no consumers and subscribes")
+        tdLog.printNoPrefix("======== test case 1 end ...... ")
+
+    def run(self):
+        self.prepareTestEnv()
+        self.tmqCase1()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/7-tmq/tmqMaxGroupIds.json

@@ -0,0 +1,27 @@
+{
+    "filetype": "subscribe",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "result_file": "tmq_res.txt",
+    "tmq_info": {
+        "concurrent": 99,
+        "poll_delay": 100000,
+        "group.id": "",
+        "group_mode": "independent",
+        "create_mode": "parallel",
+        "client.id": "cliid_0001",
+        "auto.offset.reset": "earliest",
+        "enable.manual.commit": "false",
+        "enable.auto.commit": "false",
+        "auto.commit.interval.ms": 1000,
+        "experimental.snapshot.enable": "false",
+        "msg.with.table.name": "false",
+        "rows_file": "",
+        "topic_list": [
+            {"name": "dbtstb_0001", "sql": "select * from dbt.stb;"}
+        ]
+    }
+}

tests/system-test/7-tmq/tmqMaxGroupIds.py

@@ -0,0 +1,246 @@
+
+import sys
+import time
+import threading
+from taos.tmq import Consumer
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+class TDTestCase:
+    updatecfgDict = {'debugFlag': 135}    
+    
+    def __init__(self):
+        self.vgroups    = 1
+        self.ctbNum     = 10
+        self.rowsPerTbl = 10
+        self.tmqMaxTopicNum = 20
+        self.tmqMaxGroups = 100
+
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor(), False)
+
+    def getPath(self, tool="taosBenchmark"):
+        if (platform.system().lower() == 'windows'):
+            tool = tool + ".exe"
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        paths = []
+        for root, dirs, files in os.walk(projPath):
+            if ((tool) in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    paths.append(os.path.join(root, tool))
+                    break
+        if (len(paths) == 0):
+            tdLog.exit("taosBenchmark not found!")
+            return
+        else:
+            tdLog.info("taosBenchmark found in %s" % paths[0])
+            return paths[0]
+
+    def prepareTestEnv(self):
+        tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 10,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  10,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   1}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+
+        tmqCom.initConsumerTable()
+        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
+        tdSql.execute("alter database %s wal_retention_period 360000" % (paraDict['dbName']))
+        tdLog.info("create stb")
+        tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
+        tdLog.info("create ctb")
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
+                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
+        tdLog.info("insert data")
+        tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
+                                               ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+                                               startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+
+        tdLog.info("restart taosd to ensure that the data falls into the disk")
+        # tdDnodes.stop(1)
+        # tdDnodes.start(1)
+        tdSql.query("flush database %s"%(paraDict['dbName']))
+        return
+
+    def tmqSubscribe(self, topicName, newGroupId, expectResult):
+        # create new connector for new tdSql instance in my thread
+        # newTdSql = tdCom.newTdSql()
+        # topicName = inputDict['topic_name']
+        # group_id = inputDict['group_id']
+        
+        consumer_dict = {
+            "group.id": newGroupId,
+            "client.id": "client",
+            "td.connect.user": "root",
+            "td.connect.pass": "taosdata",
+            "auto.commit.interval.ms": "1000",
+            "enable.auto.commit": "true",
+            "auto.offset.reset": "earliest",
+            "experimental.snapshot.enable": "false",
+            "msg.with.table.name": "false"
+        }
+        
+        ret = 'success'
+        consumer = Consumer(consumer_dict)
+        # print("======%s"%(inputDict['topic_name']))
+        try:
+            consumer.subscribe([topicName])
+        except Exception as e:
+            tdLog.info("consumer.subscribe() fail ")
+            tdLog.info("%s"%(e))
+            if (expectResult == "fail"):
+                consumer.close()
+                return 'success'
+            else: 
+                consumer.close()
+                return 'fail'
+    
+        tdLog.info("consumer.subscribe() success ")
+        if (expectResult == "success"):
+            consumer.close()
+            return 'success'
+        else: 
+            consumer.close()
+            return 'fail'
+
+    def tmqCase1(self):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 100000000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  3,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   1}
+
+        topicNameList = ['dbtstb_0001']
+        tdLog.info("create topics from stb")
+        queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
+        for i in range(len(topicNameList)):
+            sqlString = "create topic %s as %s" %(topicNameList[i], queryString)
+            tdLog.info("create topic sql: %s"%sqlString)
+            tdSql.execute(sqlString)
+
+        # tdSql.query('show topics;')
+        # topicNum = tdSql.queryRows
+        # tdLog.info(" topic count: %d"%(topicNum))
+        # if topicNum != len(topicNameList):
+        #     tdLog.exit("show topics %d not equal expect num: %d"%(topicNum, len(topicNameList)))
+        
+        pThread = tmqCom.asyncInsertDataByInterlace(paraDict)
+        
+        # use taosBenchmark to subscribe  
+        binPath = self.getPath()
+        cmd = "nohup %s -f ./7-tmq/tmqMaxGroupIds.json > /dev/null 2>&1 & " % binPath
+        tdLog.info("%s"%(cmd))
+        os.system(cmd)
+                
+        expectTopicNum = 1
+        expectConsumerNUm = 99
+        expectSubscribeNum = 99
+                
+        tdSql.query('show topics;')
+        topicNum = tdSql.queryRows
+        tdLog.info(" get topic count: %d"%(topicNum))
+        if topicNum != expectTopicNum:
+            tdLog.exit("show topics %d not equal expect num: %d"%(topicNum, expectTopicNum))
+        
+        flag = 0
+        while (1):        
+            tdSql.query('show consumers;')
+            consumerNUm = tdSql.queryRows
+            tdLog.info(" get consumers count: %d"%(consumerNUm))
+            if consumerNUm == expectConsumerNUm:
+                flag = 1
+                break
+            else:
+                time.sleep(1)
+
+        if (0 == flag):
+            tdLog.exit("show consumers %d not equal expect num: %d"%(topicNum, expectConsumerNUm))
+
+        flag = 0
+        for i in range(10):                  
+            tdSql.query('show subscriptions;')
+            subscribeNum = tdSql.queryRows
+            tdLog.info(" get subscriptions count: %d"%(subscribeNum))
+            if subscribeNum == expectSubscribeNum:
+                flag = 1
+                break
+            else:
+                time.sleep(1)
+                
+        if (0 == flag):
+            tdLog.exit("show subscriptions %d not equal expect num: %d"%(subscribeNum, expectSubscribeNum))
+        
+        res = self.tmqSubscribe(topicNameList[0], "newGroupId_001", "success")
+        if res != 'success':
+            tdLog.exit("limit max groupid fail")
+        
+        res = self.tmqSubscribe(topicNameList[0], "newGroupId_002", "fail")
+        if res != 'success':
+            tdLog.exit("limit max groupid fail")
+        
+        tmqCom.g_end_insert_flag = 1
+        tdLog.debug("notify sub-thread to stop insert data")
+        pThread.join()
+
+        tdLog.printNoPrefix("======== test case 1 end ...... ")
+
+    def run(self):
+        self.prepareTestEnv()
+        self.tmqCase1()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tools/shell/src/shellNettest.c

@@ -15,6 +15,7 @@
 
 #define _GNU_SOURCE
 #include "shellInt.h"
+#include "tversion.h"
 
 static void shellWorkAsClient() {
   SShellArgs *pArgs = &shell.args;
@@ -33,6 +34,7 @@ static void shellWorkAsClient() {
   rpcInit.user = "_dnd";
   rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
 
+  taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
   clientRpc = rpcOpen(&rpcInit);
   if (clientRpc == NULL) {
     printf("failed to init net test client since %s\r\n", terrstr());
@@ -123,6 +125,8 @@ static void shellWorkAsServer() {
   rpcInit.connType = TAOS_CONN_SERVER;
   rpcInit.idleTime = tsShellActivityTimer * 1000;
 
+  taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
+
   void *serverRpc = rpcOpen(&rpcInit);
   if (serverRpc == NULL) {
     printf("failed to init net test server since %s\r\n", terrstr());