Merge remote-tracking branch 'origin/3.0' into fix/TD-20095

Jenkinsfile2

@@ -175,6 +175,7 @@ def pre_test_build_mac() {
         cd ${WK}/debug
         cmake .. -DBUILD_TEST=true -DBUILD_HTTPS=false
         make -j10
+        ctest -j10 || exit 7
     '''
     sh '''
         date

include/common/tmsg.h

@@ -297,7 +297,6 @@ typedef struct {
 
 typedef struct {
   int32_t        code;
-  int8_t         hashMeta;
   int64_t        uid;
   char*          tblFName;
   int32_t        numOfRows;

include/common/tmsgdef.h

@@ -272,6 +272,8 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_SYNC_HEARTBEAT, "sync-heartbeat", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_HEARTBEAT_REPLY, "sync-heartbeat-reply", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_LOCAL_CMD, "sync-local-cmd", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_SYNC_PRE_SNAPSHOT, "sync-pre-snapshot", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_SYNC_PRE_SNAPSHOT_REPLY, "sync-pre-snapshot-reply", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_MAX_MSG, "sync-max", NULL, NULL)
   
   TD_NEW_MSG_SEG(TDMT_VND_STREAM_MSG)

include/libs/catalog/catalog.h

@@ -203,13 +203,11 @@ int32_t catalogUpdateTableMeta(SCatalog* pCatalog, STableMetaRsp* rspMsg);
 
 int32_t catalogUpdateTableMeta(SCatalog* pCatalog, STableMetaRsp* rspMsg);
 
-int32_t catalogGetCachedTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableMeta** pTableMeta);
+int32_t catalogGetCachedTableMeta(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta);
 
-int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
+int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta);
-                            STableMeta** pTableMeta);
 
-int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
+int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, const SName* pTableName,           SVgroupInfo* pVgroup, bool* exists);
-                                  SVgroupInfo* pVgroup, bool* exists);
 
 /**
  * Force refresh DB's local cached vgroup info.
@@ -309,7 +307,7 @@ int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* f
 int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type,
                        bool* pass);
 
-int32_t catalogChkAuthFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type,
+int32_t catalogChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type,
                                        bool* pass, bool* exists);
 
 int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth);
@@ -326,6 +324,10 @@ SMetaData* catalogCloneMetaData(SMetaData* pData);
 
 void catalogFreeMetaData(SMetaData* pData);
 
+int32_t ctgdEnableDebug(char *option, bool enable);
+
+int32_t ctgdHandleDbgCommand(char *command);
+
 /**
  * Destroy catalog and relase all resources
  */

include/libs/qcom/query.h

@@ -248,7 +248,7 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
    (_code) == TSDB_CODE_PAR_INVALID_DROP_COL || ((_code) == TSDB_CODE_TDB_INVALID_TABLE_ID))
 #define NEED_CLIENT_REFRESH_VG_ERROR(_code) \
   ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID)
-#define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_TABLE_RECREATED)
+#define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER)
 #define NEED_CLIENT_HANDLE_ERROR(_code)                                          \
   (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \
    NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code))

include/libs/sync/sync.h

@@ -37,6 +37,7 @@ extern "C" {
 #define SYNC_ADD_QUORUM_COUNT        3
 #define SYNC_MNODE_LOG_RETENTION     10000
 #define SYNC_VNODE_LOG_RETENTION     100
+#define SNAPSHOT_MAX_CLOCK_SKEW_MS   1000 * 10
 
 #define SYNC_APPEND_ENTRIES_TIMEOUT_MS 10000
 

include/util/taoserror.h

@@ -340,7 +340,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_TDB_TABLE_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x0618)
 #define TSDB_CODE_TDB_STB_ALREADY_EXIST         TAOS_DEF_ERROR_CODE(0, 0x0619)
 #define TSDB_CODE_TDB_STB_NOT_EXIST             TAOS_DEF_ERROR_CODE(0, 0x061A)
-#define TSDB_CODE_TDB_TABLE_RECREATED           TAOS_DEF_ERROR_CODE(0, 0x061B)
+#define TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER  TAOS_DEF_ERROR_CODE(0, 0x061B)
 #define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR        TAOS_DEF_ERROR_CODE(0, 0x061C)
 
 // query

packaging/testpackage.sh

@@ -211,7 +211,8 @@ elif [[ ${packgeName} =~ "tar" ]];then
 
     if [ ${diffNumbers} != 0 ];then
         echoColor R "The number and names of files is different from the previous installation package"
-        echoColor Y `cat ${installPath}/diffFile.log`
+        diffLog=`cat ${installPath}/diffFile.log`
+        echoColor Y "${diffLog}"
         exit -1
     else 
         echoColor G "The number and names of files are the same as previous installation packages"

source/client/inc/clientInt.h

@@ -318,6 +318,7 @@ void*    createTscObj(const char* user, const char* auth, const char* db, int32_
 void     destroyTscObj(void* pObj);
 STscObj* acquireTscObj(int64_t rid);
 int32_t  releaseTscObj(int64_t rid);
+void destroyAppInst(SAppInstInfo *pAppInfo);
 
 uint64_t generateRequestId();
 

source/client/inc/clientStmt.h

@@ -39,6 +39,7 @@ typedef enum {
   STMT_BIND_COL,
   STMT_ADD_BATCH,
   STMT_EXECUTE,
+  STMT_MAX,
 } STMT_STATUS;
 
 typedef struct SStmtTableCache {
@@ -94,12 +95,18 @@ typedef struct STscStmt {
   STscObj  *taos;
   SCatalog *pCatalog;
   int32_t   affectedRows;
+  uint32_t  seqId;
+  uint32_t  seqIds[STMT_MAX];
 
   SStmtSQLInfo  sql;
   SStmtExecInfo exec;
   SStmtBindInfo bInfo;
 } STscStmt;
 
+extern char *gStmtStatusStr[];
+
+#define STMT_LOG_SEQ(n) do { (pStmt)->seqId++; (pStmt)->seqIds[n]++; STMT_DLOG("the %dth:%d %s", (pStmt)->seqIds[n], (pStmt)->seqId, gStmtStatusStr[n]); } while (0)
+
 #define STMT_STATUS_NE(S) (pStmt->sql.status != STMT_##S)
 #define STMT_STATUS_EQ(S) (pStmt->sql.status == STMT_##S)
 
@@ -128,6 +135,12 @@ typedef struct STscStmt {
     }                                \
   } while (0)
 
+#define STMT_ELOG(param, ...) qError("stmt:%p " param, pStmt, __VA_ARGS__)
+#define STMT_DLOG(param, ...) qDebug("stmt:%p " param, pStmt, __VA_ARGS__)
+
+#define STMT_ELOG_E(param) qError("stmt:%p " param, pStmt)
+#define STMT_DLOG_E(param) qDebug("stmt:%p " param, pStmt)
+
 TAOS_STMT  *stmtInit(STscObj *taos);
 int         stmtClose(TAOS_STMT *stmt);
 int         stmtExec(TAOS_STMT *stmt);

source/client/src/clientImpl.c

@@ -133,6 +133,12 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas
     taosThreadMutexInit(&p->qnodeMutex, NULL);
     p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores);
     p->pAppHbMgr = appHbMgrInit(p, key);
+    if (NULL == p->pAppHbMgr) {
+      destroyAppInst(p);
+      taosThreadMutexUnlock(&appInfo.mutex);
+      taosMemoryFreeClear(key);
+      return NULL;
+    }
     taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES);
     p->instKey = key;
     key = NULL;
@@ -1266,7 +1272,9 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) {
   pMsgSendInfo->requestObjRefId = pRequest->self;
   pMsgSendInfo->requestId = pRequest->requestId;
   pMsgSendInfo->fp = getMsgRspHandle(pMsgSendInfo->msgType);
-  pMsgSendInfo->param = pRequest;
+  pMsgSendInfo->param = taosMemoryCalloc(1, sizeof(pRequest->self));
+
+  *(int64_t*)pMsgSendInfo->param = pRequest->self;
 
   SConnectReq connectReq = {0};
   STscObj*    pObj = pRequest->pTscObj;

source/client/src/clientMain.c

@@ -286,7 +286,8 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
     return NULL;
   } else {
     // assert to avoid un-initialization error
-    ASSERT(0);
+    tscError("invalid result passed to taos_fetch_row");
+    return NULL;
   }
   return NULL;
 }

source/client/src/clientMsgHandler.c

@@ -45,8 +45,13 @@ int32_t genericRspCallback(void* param, SDataBuf* pMsg, int32_t code) {
 }
 
 int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
-  SRequestObj* pRequest = param;
+  SRequestObj *pRequest = acquireRequest(*(int64_t*)param);
-
+  if (NULL == pRequest) {
+    setErrno(pRequest, TSDB_CODE_TSC_DISCONNECTED);
+    tsem_post(&pRequest->body.rspSem);
+    goto End;
+  }
+  
   if (code != TSDB_CODE_SUCCESS) {
     setErrno(pRequest, code);
     tsem_post(&pRequest->body.rspSem);
@@ -55,6 +60,12 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
 
   STscObj* pTscObj = pRequest->pTscObj;
 
+  if (NULL == pTscObj->pAppInfo || NULL == pTscObj->pAppInfo->pAppHbMgr) {
+    setErrno(pRequest, TSDB_CODE_TSC_DISCONNECTED);
+    tsem_post(&pRequest->body.rspSem);
+    goto End;
+  }
+  
   SConnectRsp connectRsp = {0};
   if (tDeserializeSConnectRsp(pMsg->pData, pMsg->len, &connectRsp) != 0) {
     code = TSDB_CODE_TSC_INVALID_VERSION;
@@ -115,10 +126,15 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
 
   tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, connectRsp.clusterId,
            pTscObj->pAppInfo->numOfConns);
-
+           
   tsem_post(&pRequest->body.rspSem);
 End:
 
+  if (pRequest) {
+    releaseRequest(pRequest->self);
+  }
+  
+  taosMemoryFree(param);
   taosMemoryFree(pMsg->pEpSet);
   taosMemoryFree(pMsg->pData);
   return code;

source/client/src/clientStmt.c

@@ -5,6 +5,8 @@
 
 #include "clientStmt.h"
 
+char *gStmtStatusStr[] = {"unknown", "init", "prepare", "settbname", "settags", "fetchFields", "bind", "bindCol", "addBatch", "exec"};
+
 static int32_t stmtCreateRequest(STscStmt* pStmt) {
   int32_t code = 0;
 
@@ -21,6 +23,10 @@ static int32_t stmtCreateRequest(STscStmt* pStmt) {
 int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) {
   int32_t code = 0;
 
+  if (newStatus >= STMT_INIT && newStatus < STMT_MAX) {
+    STMT_LOG_SEQ(newStatus);
+  }
+
   switch (newStatus) {
     case STMT_PREPARE:
       break;
@@ -528,13 +534,17 @@ TAOS_STMT* stmtInit(STscObj* taos) {
   pStmt->bInfo.needParse = true;
   pStmt->sql.status = STMT_INIT;
 
+  STMT_LOG_SEQ(STMT_INIT);
+  
+  tscDebug("stmt:%p initialized", pStmt);
+
   return pStmt;
 }
 
 int stmtPrepare(TAOS_STMT* stmt, const char* sql, unsigned long length) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
-  tscDebug("stmt start to prepare");
+  STMT_DLOG_E("start to prepare");
 
   if (pStmt->sql.status >= STMT_PREPARE) {
     STMT_ERR_RET(stmtResetStmt(pStmt));
@@ -555,7 +565,7 @@ int stmtPrepare(TAOS_STMT* stmt, const char* sql, unsigned long length) {
 int stmtSetTbName(TAOS_STMT* stmt, const char* tbName) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
-  tscDebug("stmt start to set tbName: %s", tbName);
+  STMT_DLOG("start to set tbName: %s", tbName);
 
   STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTBNAME));
 
@@ -587,7 +597,7 @@ int stmtSetTbName(TAOS_STMT* stmt, const char* tbName) {
 int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
-  tscDebug("stmt start to set tbTags");
+  STMT_DLOG_E("start to set tbTags");
 
   STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS));
 
@@ -649,7 +659,7 @@ int stmtFetchColFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields
 int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
-  tscDebug("start to bind stmt data, colIdx: %d", colIdx);
+  STMT_DLOG("start to bind stmt data, colIdx: %d", colIdx);
 
   STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND));
 
@@ -743,7 +753,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
 int stmtAddBatch(TAOS_STMT* stmt) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
-  tscDebug("stmt start to add batch");
+  STMT_DLOG_E("start to add batch");
 
   STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_ADD_BATCH));
 
@@ -756,8 +766,7 @@ int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp* pRsp) {
   tscDebug("stmt start to update tbUid, blockNum: %d", pRsp->nBlocks);
 
   if (pRsp->nBlocks <= 0) {
-    tscError("invalid submit resp block number %d", pRsp->nBlocks);
+    return TSDB_CODE_SUCCESS;
-    STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR);
   }
 
   size_t             keyLen = 0;
@@ -810,7 +819,7 @@ int stmtExec(TAOS_STMT* stmt) {
   SSubmitRsp* pRsp = NULL;
   bool        autoCreateTbl = pStmt->exec.autoCreateTbl;
 
-  tscDebug("stmt start to exec");
+  STMT_DLOG_E("start to exec");
 
   STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE));
 
@@ -885,6 +894,8 @@ int stmtAffectedRowsOnce(TAOS_STMT* stmt) { return ((STscStmt*)stmt)->exec.affec
 int stmtIsInsert(TAOS_STMT* stmt, int* insert) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
+  STMT_DLOG_E("start is insert");
+
   if (pStmt->sql.type) {
     *insert = (STMT_TYPE_INSERT == pStmt->sql.type || STMT_TYPE_MULTI_INSERT == pStmt->sql.type);
   } else {
@@ -897,6 +908,8 @@ int stmtIsInsert(TAOS_STMT* stmt, int* insert) {
 int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
+  STMT_DLOG_E("start to get tag fields");
+
   if (STMT_TYPE_QUERY == pStmt->sql.type) {
     STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR);
   }
@@ -927,6 +940,8 @@ int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
 int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
+  STMT_DLOG_E("start to get col fields");
+
   if (STMT_TYPE_QUERY == pStmt->sql.type) {
     STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR);
   }
@@ -957,6 +972,8 @@ int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
 int stmtGetParamNum(TAOS_STMT* stmt, int* nums) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
+  STMT_DLOG_E("start to get param num");
+
   STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS));
 
   if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 &&
@@ -986,6 +1003,8 @@ int stmtGetParamNum(TAOS_STMT* stmt, int* nums) {
 int stmtGetParam(TAOS_STMT* stmt, int idx, int* type, int* bytes) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
+  STMT_DLOG_E("start to get param");
+
   if (STMT_TYPE_QUERY == pStmt->sql.type) {
     STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR);
   }
@@ -1028,6 +1047,8 @@ int stmtGetParam(TAOS_STMT* stmt, int idx, int* type, int* bytes) {
 TAOS_RES* stmtUseResult(TAOS_STMT* stmt) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
+  STMT_DLOG_E("start to use result");
+
   if (STMT_TYPE_QUERY != pStmt->sql.type) {
     tscError("useResult only for query statement");
     return NULL;

source/common/src/tmsg.c

@@ -5436,9 +5436,12 @@ static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBl
   if (tStartEncode(pEncoder) < 0) return -1;
 
   if (tEncodeI32(pEncoder, pBlock->code) < 0) return -1;
-  if (tEncodeI8(pEncoder, pBlock->hashMeta) < 0) return -1;
   if (tEncodeI64(pEncoder, pBlock->uid) < 0) return -1;
-  if (tEncodeCStr(pEncoder, pBlock->tblFName) < 0) return -1;
+  if (pBlock->tblFName) {
+    if (tEncodeCStr(pEncoder, pBlock->tblFName) < 0) return -1;
+  } else {
+    if (tEncodeCStr(pEncoder, "") < 0) return -1;
+  }
   if (tEncodeI32v(pEncoder, pBlock->numOfRows) < 0) return -1;
   if (tEncodeI32v(pEncoder, pBlock->affectedRows) < 0) return -1;
   if (tEncodeI64v(pEncoder, pBlock->sver) < 0) return -1;
@@ -5455,7 +5458,6 @@ static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) {
   if (tStartDecode(pDecoder) < 0) return -1;
 
   if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1;
-  if (tDecodeI8(pDecoder, &pBlock->hashMeta) < 0) return -1;
   if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1;
   pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1);
   if (NULL == pBlock->tblFName) return -1;

source/dnode/mgmt/mgmt_mnode/src/mmHandle.c

@@ -195,6 +195,8 @@ SArray *mmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
 
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, mmPutMsgToSyncCtrlQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, mmPutMsgToSyncCtrlQueue, 1) == NULL) goto _OVER;

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -435,6 +435,9 @@ SArray *vmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
+
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, vmPutMsgToSyncCtrlQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, vmPutMsgToSyncCtrlQueue, 0) == NULL) goto _OVER;
 

source/dnode/vnode/src/inc/tsdb.h

@@ -643,6 +643,7 @@ typedef struct SSttBlockLoadInfo {
   STSchema  *pSchema;
   int16_t   *colIds;
   int32_t    numOfCols;
+  bool       sttBlockLoaded;
 } SSttBlockLoadInfo;
 
 typedef struct SMergeTree {

source/dnode/vnode/src/tsdb/tsdbMemTable.c

@@ -116,6 +116,13 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI
   if (info.suid) {
     metaGetInfo(pTsdb->pVnode->pMeta, info.suid, &info);
   }
+  if (pMsgIter->sversion != info.skmVer) {
+    tsdbError("vgId:%d, req sver:%d, skmVer:%d suid:%" PRId64 " uid:%" PRId64,
+             TD_VID(pTsdb->pVnode), pMsgIter->sversion, info.skmVer, suid, uid);
+    code = TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER;
+    goto _err;
+  }
+  
   pRsp->sver = info.skmVer;
 
   // create/get STbData to op
@@ -133,6 +140,7 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI
   return code;
 
 _err:
+  terrno = code;
   return code;
 }
 

source/dnode/vnode/src/tsdb/tsdbMergeTree.c

@@ -72,6 +72,7 @@ void resetLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
 
     pLoadInfo[i].elapsedTime = 0;
     pLoadInfo[i].loadBlocks = 0;
+    pLoadInfo[i].sttBlockLoaded = false;
   }
 }
 
@@ -278,9 +279,10 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
 
   (*pIter)->pBlockLoadInfo = pBlockLoadInfo;
 
-  size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
+//  size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
-  if (size == 0) {
+  if (!pBlockLoadInfo->sttBlockLoaded) {
     int64_t st = taosGetTimestampUs();
+    pBlockLoadInfo->sttBlockLoaded = true;
 
     code = tsdbReadSttBlk(pReader, iStt, pBlockLoadInfo->aSttBlk);
     if (code) {
@@ -288,7 +290,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
     }
 
     // only apply to the child tables, ordinary tables will not incur this filter procedure.
-    size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
+    size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
 
     if (size >= 1) {
       SSttBlk *pStart = taosArrayGet(pBlockLoadInfo->aSttBlk, 0);
@@ -296,10 +298,10 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
 
       // all identical
       if (pStart->suid == pEnd->suid) {
-        if (pStart->suid == suid) {
+        if (pStart->suid != suid) {
-          // do nothing
-        } else if (pStart->suid != suid) {
           // no qualified stt block existed
+          taosArrayClear(pBlockLoadInfo->aSttBlk);
+
           (*pIter)->iSttBlk = -1;
           double el = (taosGetTimestampUs() - st) / 1000.0;
           tsdbDebug("load the last file info completed, elapsed time:%.2fms, %s", el, idStr);
@@ -330,7 +332,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
     tsdbDebug("load the last file info completed, elapsed time:%.2fms, %s", el, idStr);
   }
 
-  size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
+  size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
 
   // find the start block
   (*pIter)->iSttBlk = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, uid, backward);

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -2186,7 +2186,7 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum
 static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) {
   // the last block reader has been initialized for this table.
   if (pLBlockReader->uid == pScanInfo->uid) {
-    return true;
+    return hasDataInLastBlock(pLBlockReader);
   }
 
   if (pLBlockReader->uid != 0) {
@@ -2319,6 +2319,9 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
       if (pReader->order == TSDB_ORDER_ASC ||
           (pReader->order == TSDB_ORDER_DESC && (!hasDataInLastBlock(pLastBlockReader)))) {
         copyBlockDataToSDataBlock(pReader, pBlockScanInfo);
+
+        // record the last key value
+        pBlockScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->order)? pBlock->maxKey.ts:pBlock->minKey.ts;
         goto _end;
       }
     }
@@ -2710,6 +2713,9 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
       pInfo->window = (STimeWindow){.skey = pBlock->minKey.ts, .ekey = pBlock->maxKey.ts};
       setComposedBlockFlag(pReader, false);
       setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlock->maxKey.ts, pReader->order);
+
+      // update the last key for the corresponding table
+      pScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->order)? pInfo->window.ekey:pInfo->window.skey;
     }
   }
 

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -861,6 +861,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
   SEncoder       encoder = {0};
   SArray        *newTbUids = NULL;
   SVStatis       statis = {0};
+  bool           tbCreated = false;
   terrno = TSDB_CODE_SUCCESS;
 
   pRsp->code = 0;
@@ -894,11 +895,10 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
     if (pBlock == NULL) break;
 
     SSubmitBlkRsp submitBlkRsp = {0};
+    tbCreated = false;
 
     // create table for auto create table mode
     if (msgIter.schemaLen > 0) {
-      submitBlkRsp.hashMeta = 1;
-
       tDecoderInit(&decoder, pBlock->data, msgIter.schemaLen);
       if (tDecodeSVCreateTbReq(&decoder, &createTbReq) < 0) {
         pRsp->code = TSDB_CODE_INVALID_MSG;
@@ -935,12 +935,13 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
         }
 
         taosArrayPush(newTbUids, &createTbReq.uid);
+
+        submitBlkRsp.uid = createTbReq.uid;
+        submitBlkRsp.tblFName = taosMemoryMalloc(strlen(pVnode->config.dbname) + strlen(createTbReq.name) + 2);
+        sprintf(submitBlkRsp.tblFName, "%s.%s", pVnode->config.dbname, createTbReq.name);
+        tbCreated = true;
       }
-
+      
-      submitBlkRsp.uid = createTbReq.uid;
-      submitBlkRsp.tblFName = taosMemoryMalloc(strlen(pVnode->config.dbname) + strlen(createTbReq.name) + 2);
-      sprintf(submitBlkRsp.tblFName, "%s.%s", pVnode->config.dbname, createTbReq.name);
-
       msgIter.uid = createTbReq.uid;
       if (createTbReq.type == TSDB_CHILD_TABLE) {
         msgIter.suid = createTbReq.ctb.suid;
@@ -953,10 +954,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
 #endif
       tDecoderClear(&decoder);
       taosArrayDestroy(createTbReq.ctb.tagName);
-    } else {
+    } 
-      submitBlkRsp.tblFName = taosMemoryMalloc(TSDB_TABLE_FNAME_LEN);
-      sprintf(submitBlkRsp.tblFName, "%s.", pVnode->config.dbname);
-    }
 
     if (tsdbInsertTableData(pVnode->pTsdb, version, &msgIter, pBlock, &submitBlkRsp) < 0) {
       submitBlkRsp.code = terrno;
@@ -964,7 +962,9 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
 
     submitRsp.numOfRows += submitBlkRsp.numOfRows;
     submitRsp.affectedRows += submitBlkRsp.affectedRows;
-    taosArrayPush(submitRsp.pArray, &submitBlkRsp);
+    if (tbCreated || submitBlkRsp.code) {
+      taosArrayPush(submitRsp.pArray, &submitBlkRsp);
+    }
   }
 
   if (taosArrayGetSize(newTbUids) > 0) {

source/libs/catalog/inc/catalogInt.h

@@ -33,6 +33,7 @@ extern "C" {
 #define CTG_DEFAULT_MAX_RETRY_TIMES      3
 #define CTG_DEFAULT_BATCH_NUM            64
 #define CTG_DEFAULT_FETCH_NUM            8
+#define CTG_MAX_COMMAND_LEN              512
 
 #define CTG_RENT_SLOT_SECOND 1.5
 
@@ -223,6 +224,7 @@ typedef struct SCtgUserAuth {
 
 typedef struct SCatalog {
   uint64_t     clusterId;
+  bool         stopUpdate;
   SHashObj*    userCache;  // key:user, value:SCtgUserAuth
   SHashObj*    dbCache;    // key:dbname, value:SCtgDBCache
   SCtgRentMgmt dbRent;
@@ -671,7 +673,7 @@ void    ctgdShowClusterCache(SCatalog* pCtg);
 int32_t ctgdShowCacheInfo(void);
 
 int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq);
-int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta);
+int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta);
 int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetasCtx* ctx, int32_t dbIdx,
                                int32_t* fetchIdx, int32_t baseResIdx, SArray* pList);
 
@@ -786,6 +788,7 @@ void    ctgFreeTbCacheImpl(SCtgTbCache* pCache);
 int32_t ctgRemoveTbMeta(SCatalog* pCtg, SName* pTableName);
 int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* pVgroup, bool* exists);
 SName*  ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch);
+int32_t ctgdGetOneHandle(SCatalog **pHandle);
 
 extern SCatalogMgmt gCtgMgmt;
 extern SCtgDebug    gCTGDebug;

source/libs/catalog/src/catalog.c

@@ -202,7 +202,7 @@ int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx
   int32_t           code = 0;
   STableMetaOutput* output = NULL;
 
-  CTG_ERR_RET(ctgGetTbMetaFromCache(pCtg, pConn, ctx, pTableMeta));
+  CTG_ERR_RET(ctgGetTbMetaFromCache(pCtg, ctx, pTableMeta));
   if (*pTableMeta || (ctx->flag & CTG_FLAG_ONLY_CACHE)) {
     goto _return;
   }
@@ -959,14 +959,14 @@ int32_t catalogGetTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName
   CTG_API_LEAVE(ctgGetTbMeta(pCtg, pConn, &ctx, pTableMeta));
 }
 
-int32_t catalogGetCachedTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableMeta** pTableMeta) {
+int32_t catalogGetCachedTableMeta(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta) {
   CTG_API_ENTER();
 
   SCtgTbMetaCtx ctx = {0};
   ctx.pName = (SName*)pTableName;
   ctx.flag = CTG_FLAG_UNKNOWN_STB | CTG_FLAG_ONLY_CACHE;
 
-  CTG_API_LEAVE(ctgGetTbMeta(pCtg, pConn, &ctx, pTableMeta));
+  CTG_API_LEAVE(ctgGetTbMeta(pCtg, NULL, &ctx, pTableMeta));
 }
 
 
@@ -981,15 +981,14 @@ int32_t catalogGetSTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SNam
   CTG_API_LEAVE(ctgGetTbMeta(pCtg, pConn, &ctx, pTableMeta));
 }
 
-int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
+int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, const SName* pTableName,          STableMeta** pTableMeta) {
-                            STableMeta** pTableMeta) {
   CTG_API_ENTER();
 
   SCtgTbMetaCtx ctx = {0};
   ctx.pName = (SName*)pTableName;
   ctx.flag = CTG_FLAG_STB | CTG_FLAG_ONLY_CACHE;
 
-  CTG_API_LEAVE(ctgGetTbMeta(pCtg, pConn, &ctx, pTableMeta));
+  CTG_API_LEAVE(ctgGetTbMeta(pCtg, NULL, &ctx, pTableMeta));
 }
 
 
@@ -1114,11 +1113,10 @@ int32_t catalogGetTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const
   CTG_API_LEAVE(ctgGetTbHashVgroup(pCtg, pConn, pTableName, pVgroup, NULL));
 }
 
-int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
+int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, const SName* pTableName,           SVgroupInfo* pVgroup, bool* exists) {
-                                  SVgroupInfo* pVgroup, bool* exists) {
   CTG_API_ENTER();
 
-  CTG_API_LEAVE(ctgGetTbHashVgroup(pCtg, pConn, pTableName, pVgroup, exists));
+  CTG_API_LEAVE(ctgGetTbHashVgroup(pCtg, NULL, pTableName, pVgroup, exists));
 }
 
 #if 0
@@ -1387,16 +1385,16 @@ _return:
   CTG_API_LEAVE(code);
 }
 
-int32_t catalogChkAuthFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type,
+int32_t catalogChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type,
                                         bool* pass, bool* exists) {
   CTG_API_ENTER();
 
-  if (NULL == pCtg || NULL == pConn || NULL == user || NULL == dbFName || NULL == pass || NULL == exists) {
+  if (NULL == pCtg || NULL == user || NULL == dbFName || NULL == pass || NULL == exists) {
     CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
   }
 
   int32_t code = 0;
-  CTG_ERR_JRET(ctgChkAuth(pCtg, pConn, user, dbFName, type, pass, exists));
+  CTG_ERR_JRET(ctgChkAuth(pCtg, NULL, user, dbFName, type, pass, exists));
 
 _return:
 

source/libs/catalog/src/ctgAsync.c

@@ -1204,11 +1204,15 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu
           stbCtx.flag = flag;
           stbCtx.pName = &stbName;
 
-          taosMemoryFreeClear(pOut->tbMeta);
+          STableMeta *stbMeta = NULL;
-          CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, &pOut->tbMeta));
+          ctgReadTbMetaFromCache(pCtg, &stbCtx, &stbMeta);
-          if (pOut->tbMeta) {
+          if (stbMeta && stbMeta->sversion >= pOut->tbMeta->sversion) {
             ctgDebug("use cached stb meta, tbName:%s", tNameGetTableName(pName));
             exist = 1;
+          } else {
+            ctgDebug("need to get/update stb meta, tbName:%s", tNameGetTableName(pName));
+            taosMemoryFreeClear(pOut->tbMeta);
+            taosMemoryFreeClear(stbMeta);
           }
         }
 
@@ -1641,7 +1645,7 @@ int32_t ctgLaunchGetTbMetaTask(SCtgTask* pTask) {
     pMsgCtx->pBatchs = pJob->pBatchs;
   }
 
-  CTG_ERR_RET(ctgGetTbMetaFromCache(pCtg, pConn, (SCtgTbMetaCtx*)pTask->taskCtx, (STableMeta**)&pTask->res));
+  CTG_ERR_RET(ctgGetTbMetaFromCache(pCtg, (SCtgTbMetaCtx*)pTask->taskCtx, (STableMeta**)&pTask->res));
   if (pTask->res) {
     CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0));
     return TSDB_CODE_SUCCESS;

source/libs/catalog/src/ctgCache.c

@@ -248,6 +248,8 @@ int32_t ctgAcquireStbMetaFromCache(SCatalog *pCtg, char *dbFName, uint64_t suid,
     goto _return;
   }
 
+  taosHashRelease(dbCache->stbCache, stName);
+
   CTG_LOCK(CTG_READ, &pCache->metaLock);
   if (NULL == pCache->pMeta) {
     ctgDebug("stb 0x%" PRIx64 " meta not in cache, dbFName:%s", suid, dbFName);
@@ -1550,7 +1552,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) {
   char            *dbFName = msg->dbFName;
   SCatalog        *pCtg = msg->pCtg;
 
-  if (NULL == dbInfo->vgHash) {
+  if (pCtg->stopUpdate || NULL == dbInfo->vgHash) {
     goto _return;
   }
 
@@ -1620,6 +1622,10 @@ int32_t ctgOpDropDbCache(SCtgCacheOperation *operation) {
   SCtgDropDBMsg *msg = operation->data;
   SCatalog      *pCtg = msg->pCtg;
 
+  if (pCtg->stopUpdate) {
+    goto _return;
+  }
+
   SCtgDBCache *dbCache = NULL;
   ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache);
   if (NULL == dbCache) {
@@ -1646,6 +1652,10 @@ int32_t ctgOpDropDbVgroup(SCtgCacheOperation *operation) {
   SCtgDropDbVgroupMsg *msg = operation->data;
   SCatalog            *pCtg = msg->pCtg;
 
+  if (pCtg->stopUpdate) {
+    goto _return;
+  }
+
   SCtgDBCache *dbCache = NULL;
   ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache);
   if (NULL == dbCache) {
@@ -1675,6 +1685,10 @@ int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *operation) {
   STableMetaOutput    *pMeta = msg->pMeta;
   SCtgDBCache         *dbCache = NULL;
 
+  if (pCtg->stopUpdate) {
+    goto _return;
+  }
+  
   if ((!CTG_IS_META_CTABLE(pMeta->metaType)) && NULL == pMeta->tbMeta) {
     ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", pMeta->dbFName, pMeta->tbName);
     CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
@@ -1723,6 +1737,10 @@ int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) {
   SCtgDropStbMetaMsg *msg = operation->data;
   SCatalog           *pCtg = msg->pCtg;
 
+  if (pCtg->stopUpdate) {
+    goto _return;
+  }
+
   SCtgDBCache *dbCache = NULL;
   ctgGetDBCache(pCtg, msg->dbFName, &dbCache);
   if (NULL == dbCache) {
@@ -1776,6 +1794,10 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) {
   SCtgDropTblMetaMsg *msg = operation->data;
   SCatalog           *pCtg = msg->pCtg;
 
+  if (pCtg->stopUpdate) {
+    goto _return;
+  }
+
   SCtgDBCache *dbCache = NULL;
   ctgGetDBCache(pCtg, msg->dbFName, &dbCache);
   if (NULL == dbCache) {
@@ -1819,6 +1841,10 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) {
   SCtgUpdateUserMsg *msg = operation->data;
   SCatalog          *pCtg = msg->pCtg;
 
+  if (pCtg->stopUpdate) {
+    goto _return;
+  }
+
   SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user));
   if (NULL == pUser) {
     SCtgUserAuth userAuth = {0};
@@ -1872,8 +1898,12 @@ int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation) {
   int32_t             code = 0;
   SCtgUpdateEpsetMsg *msg = operation->data;
   SCatalog           *pCtg = msg->pCtg;
-
   SCtgDBCache *dbCache = NULL;
+
+  if (pCtg->stopUpdate) {
+    goto _return;
+  }
+
   CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache));
   if (NULL == dbCache) {
     ctgDebug("db %s not exist, ignore epset update", msg->dbFName);
@@ -1920,6 +1950,10 @@ int32_t ctgOpUpdateTbIndex(SCtgCacheOperation *operation) {
   STableIndex          *pIndex = msg->pIndex;
   SCtgDBCache          *dbCache = NULL;
 
+  if (pCtg->stopUpdate) {
+    goto _return;
+  }
+
   CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pIndex->dbFName, 0, &dbCache));
 
   CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, pIndex->dbFName, pIndex->tbName, &pIndex));
@@ -1942,6 +1976,10 @@ int32_t ctgOpDropTbIndex(SCtgCacheOperation *operation) {
   SCatalog           *pCtg = msg->pCtg;
   SCtgDBCache        *dbCache = NULL;
 
+  if (pCtg->stopUpdate) {
+    goto _return;
+  }
+
   CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache));
   if (NULL == dbCache) {
     return TSDB_CODE_SUCCESS;
@@ -2154,7 +2192,7 @@ int32_t ctgStartUpdateThread() {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t ctgGetTbMetaFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMetaCtx *ctx, STableMeta **pTableMeta) {
+int32_t ctgGetTbMetaFromCache(SCatalog *pCtg, SCtgTbMetaCtx *ctx, STableMeta **pTableMeta) {
   if (IS_SYS_DBNAME(ctx->pName->dbname)) {
     CTG_FLAG_SET_SYS_DB(ctx->flag);
   }

source/libs/catalog/src/ctgDbg.c

@@ -226,28 +226,45 @@ _return:
   CTG_RET(code);
 }
 
-int32_t ctgdEnableDebug(char *option) {
+int32_t ctgdEnableDebug(char *option, bool enable) {
   if (0 == strcasecmp(option, "lock")) {
-    gCTGDebug.lockEnable = true;
+    gCTGDebug.lockEnable = enable;
-    qDebug("lock debug enabled");
+    qDebug("catalog lock debug set to %d", enable);
     return TSDB_CODE_SUCCESS;
   }
 
   if (0 == strcasecmp(option, "cache")) {
-    gCTGDebug.cacheEnable = true;
+    gCTGDebug.cacheEnable = enable;
-    qDebug("cache debug enabled");
+    qDebug("catalog cache debug set to %d", enable);
     return TSDB_CODE_SUCCESS;
   }
 
   if (0 == strcasecmp(option, "api")) {
-    gCTGDebug.apiEnable = true;
+    gCTGDebug.apiEnable = enable;
-    qDebug("api debug enabled");
+    qDebug("catalog api debug set to %d", enable);
     return TSDB_CODE_SUCCESS;
   }
 
   if (0 == strcasecmp(option, "meta")) {
-    gCTGDebug.metaEnable = true;
+    gCTGDebug.metaEnable = enable;
-    qDebug("api debug enabled");
+    qDebug("catalog meta debug set to %d", enable);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (0 == strcasecmp(option, "stopUpdate")) {
+    SCatalog *pCtg = NULL;
+    
+    void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL);
+    while (pIter) {
+      pCtg = *(SCatalog **)pIter;
+
+      pCtg->stopUpdate = enable;
+      
+      pIter = taosHashIterate(gCtgMgmt.pCluster, pIter);
+    }
+    
+    qDebug("catalog stopUpdate set to %d", enable);
+    
     return TSDB_CODE_SUCCESS;
   }
 
@@ -256,6 +273,77 @@ int32_t ctgdEnableDebug(char *option) {
   return TSDB_CODE_CTG_INTERNAL_ERROR;
 }
 
+int32_t ctgdHandleDbgCommand(char *command) {
+  if (NULL == command) {
+    CTG_RET(TSDB_CODE_INVALID_PARA);
+  }
+
+  if (strlen(command) > CTG_MAX_COMMAND_LEN) {
+    CTG_RET(TSDB_CODE_INVALID_PARA);
+  }
+
+  char *dup = strdup(command);
+  char *option = NULL;
+  char *param = NULL;
+  
+  int32_t i = 0;
+  bool newItem = true;
+  while (*(dup + i)) {
+    if (isspace(*(dup + i))) {
+      *(dup + i) = 0;
+      ++i;
+      newItem = true;
+      continue;
+    }
+
+    if (!newItem) {
+      ++i;
+      continue;
+    }
+    
+    newItem = false;
+    if (NULL == option) {
+      option = dup + i;
+      ++i;
+      continue;
+    }
+
+    if (NULL == param) {
+      param = dup + i;
+      ++i;
+      continue;
+    }
+
+    taosMemoryFree(dup);
+    CTG_RET(TSDB_CODE_INVALID_PARA);
+  }
+
+  bool enable = atoi(param);
+
+  int32_t code = ctgdEnableDebug(option, enable);
+
+  taosMemoryFree(dup);
+
+  CTG_RET(code);
+}
+
+int32_t ctgdGetOneHandle(SCatalog **pHandle) {
+  SCatalog *pCtg = NULL;
+
+  void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL);
+  while (pIter) {
+    pCtg = *(SCatalog **)pIter;
+
+    taosHashCancelIterate(gCtgMgmt.pCluster, pIter);
+    break;
+  }
+
+  *pHandle = pCtg;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
 int32_t ctgdGetStatNum(char *option, void *res) {
   if (0 == strcasecmp(option, "runtime.numOfOpDequeue")) {
     *(uint64_t *)res = atomic_load_64(&gCtgMgmt.stat.runtime.numOfOpDequeue);

source/libs/catalog/test/catalogTests.cpp

@@ -41,7 +41,6 @@
 namespace {
 
 extern "C" int32_t ctgdGetClusterCacheNum(struct SCatalog *pCatalog, int32_t type);
-extern "C" int32_t ctgdEnableDebug(char *option);
 extern "C" int32_t ctgdGetStatNum(char *option, void *res);
 
 void ctgTestSetRspTableMeta();
@@ -49,6 +48,8 @@ void ctgTestSetRspCTableMeta();
 void ctgTestSetRspSTableMeta();
 void ctgTestSetRspMultiSTableMeta();
 
+extern int32_t  clientConnRefPool;
+
 enum {
   CTGT_RSP_VGINFO = 1,
   CTGT_RSP_TBMETA,
@@ -151,10 +152,10 @@ void ctgTestInitLogFile() {
   qDebugFlag = 159;
   strcpy(tsLogDir, TD_LOG_DIR_PATH);
 
-  ctgdEnableDebug("api");
+  ctgdEnableDebug("api", true);
-  ctgdEnableDebug("meta");
+  ctgdEnableDebug("meta", true);
-  ctgdEnableDebug("cache");
+  ctgdEnableDebug("cache", true);
-  ctgdEnableDebug("lock");
+  ctgdEnableDebug("lock", true);
 
   if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
     printf("failed to open log file in directory:%s\n", tsLogDir);
@@ -1204,6 +1205,34 @@ void *ctgTestSetCtableMetaThread(void *param) {
 }
 
 
+void ctgTestFetchRows(TAOS_RES *result, int32_t *rows) {
+  TAOS_ROW    row;
+  int         num_fields = taos_num_fields(result);
+  TAOS_FIELD *fields = taos_fetch_fields(result);
+  char        temp[256];
+
+  // fetch the records row by row
+  while ((row = taos_fetch_row(result))) {
+    (*rows)++;
+    memset(temp, 0, sizeof(temp));
+    taos_print_row(temp, row, fields, num_fields);
+    printf("\t[%s]\n", temp);
+  }
+}
+
+void ctgTestExecQuery(TAOS    * taos, char* sql, bool fetch, int32_t *rows) {
+  TAOS_RES *result = taos_query(taos, sql);
+  int code = taos_errno(result);
+  ASSERT_EQ(code, 0);
+
+  if (fetch) {
+    ctgTestFetchRows(result, rows);
+  }
+  
+  taos_free_result(result);
+}
+
+
 TEST(tableMeta, normalTable) {
   struct SCatalog  *pCtg = NULL;
   SVgroupInfo       vgInfo = {0};
@@ -1245,7 +1274,7 @@ TEST(tableMeta, normalTable) {
 
   memset(&vgInfo, 0, sizeof(vgInfo));
   bool exists = false;
-  code = catalogGetCachedTableHashVgroup(pCtg, mockPointer, &n, &vgInfo, &exists);
+  code = catalogGetCachedTableHashVgroup(pCtg, &n, &vgInfo, &exists);
   ASSERT_EQ(code, 0);
   ASSERT_EQ(vgInfo.vgId, 8);
   ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
@@ -1292,7 +1321,7 @@ TEST(tableMeta, normalTable) {
   taosMemoryFree(tableMeta);
 
   tableMeta = NULL;
-  catalogGetCachedTableMeta(pCtg, mockPointer, &n, &tableMeta);
+  catalogGetCachedTableMeta(pCtg, &n, &tableMeta);
   ASSERT_EQ(code, 0);
   ASSERT_EQ(tableMeta->vgId, 8);
   ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE);
@@ -1500,7 +1529,7 @@ TEST(tableMeta, superTableCase) {
   }
 
   tableMeta = NULL;
-  code = catalogGetCachedSTableMeta(pCtg, mockPointer, &n, &tableMeta);
+  code = catalogGetCachedSTableMeta(pCtg, &n, &tableMeta);
   ASSERT_EQ(code, 0);
   ASSERT_EQ(tableMeta->vgId, 0);
   ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE);
@@ -2772,7 +2801,7 @@ TEST(apiTest, catalogChkAuth_test) {
 
   bool pass = false;
   bool exists = false;
-  code = catalogChkAuthFromCache(pCtg, mockPointer, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists);
+  code = catalogChkAuthFromCache(pCtg, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists);
   ASSERT_EQ(code, 0);
   ASSERT_EQ(exists, false);
   
@@ -2790,7 +2819,7 @@ TEST(apiTest, catalogChkAuth_test) {
     }
   }
 
-  code = catalogChkAuthFromCache(pCtg, mockPointer, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists);
+  code = catalogChkAuthFromCache(pCtg, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists);
   ASSERT_EQ(code, 0);
   ASSERT_EQ(pass, true);
   ASSERT_EQ(exists, true);
@@ -3063,6 +3092,58 @@ TEST(apiTest, catalogGetDnodeList_test) {
   catalogDestroy();
 }
 
+#ifdef INTEGRATION_TEST
+TEST(intTest, autoCreateTableTest) {
+  struct SCatalog  *pCtg = NULL;
+
+  TAOS     *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
+  ASSERT_TRUE(NULL != taos);   
+
+  ctgdEnableDebug("api", true);
+  ctgdEnableDebug("meta", true);
+  ctgdEnableDebug("cache", true);
+  ctgdEnableDebug("lock", true);
+  
+  ctgTestExecQuery(taos, "drop database if exists db1", false, NULL);
+  ctgTestExecQuery(taos, "create database db1", false, NULL);
+  ctgTestExecQuery(taos, "create stable db1.st1 (ts timestamp, f1 int) tags(tg1 int)", false, NULL);
+  ctgTestExecQuery(taos, "insert into db1.tb1 using db1.st1 tags(1) values(now, 1)", false, NULL);
+
+  ctgdGetOneHandle(&pCtg);
+  
+  while (true) {
+    uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
+    if (2 != n) {
+      taosMsleep(50);
+    } else {
+      break;
+    }
+  }
+
+  uint64_t n = 0, m = 0;
+  ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n);
+
+  ctgTestExecQuery(taos, "insert into db1.tb1 using db1.st1 tags(1) values(now, 2)", false, NULL);
+
+  ctgTestExecQuery(taos, "insert into db1.tb1 values(now, 3)", false, NULL);
+
+  taosMsleep(1000);
+  ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&m);
+
+  ASSERT_EQ(n, m);
+
+  ctgdEnableDebug("stopUpdate", true);
+  ctgTestExecQuery(taos, "alter table db1.st1 add column f2 double", false, NULL);
+
+  ctgdEnableDebug("stopUpdate", false);
+  
+  ctgTestExecQuery(taos, "insert into db1.tb1 (ts, f1) values(now, 4)", false, NULL);
+  
+  taos_close(taos);
+}
+
+#endif
+
 
 int main(int argc, char **argv) {
   testing::InitGoogleTest(&argc, argv);

source/libs/command/inc/commandInt.h

@@ -96,6 +96,7 @@ extern "C" {
 #define COMMAND_RESET_LOG "resetLog"
 #define COMMAND_SCHEDULE_POLICY "schedulePolicy"
 #define COMMAND_ENABLE_RESCHEDULE "enableReSchedule"
+#define COMMAND_CATALOG_DEBUG "catalogDebug"
 
 typedef struct SExplainGroup {
   int32_t   nodeNum;

source/libs/command/src/command.c

@@ -571,6 +571,8 @@ static int32_t execAlterCmd(char* cmd, char* value, bool* processed) {
     code = schedulerUpdatePolicy(atoi(value));
   } else if (0 == strcasecmp(cmd, COMMAND_ENABLE_RESCHEDULE)) {
     code = schedulerEnableReSchedule(atoi(value));
+  } else if (0 == strcasecmp(cmd, COMMAND_CATALOG_DEBUG)) {
+    code = ctgdHandleDbgCommand(value);
   } else {
     goto _return;
   }

source/libs/index/src/indexFstFile.c

@@ -152,7 +152,7 @@ IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int
   if (ctx->type == TFILE) {
     // ugly code, refactor later
     ctx->file.readOnly = readOnly;
-    memcpy(ctx->file.buf, path, sizeof(ctx->file.buf));
+    memcpy(ctx->file.buf, path, strlen(path));
     if (readOnly == false) {
       ctx->file.pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
       taosFtruncateFile(ctx->file.pFile, 0);

source/libs/scheduler/src/schRemote.c

@@ -280,7 +280,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa
         }
 
         atomic_add_fetch_32(&pJob->resNumOfRows, rsp->affectedRows);
-        SCH_TASK_DLOG("submit succeed, affectedRows:%d", rsp->affectedRows);
+        SCH_TASK_DLOG("submit succeed, affectedRows:%d, blocks:%d", rsp->affectedRows, rsp->nBlocks);
 
         SCH_LOCK(SCH_WRITE, &pJob->resLock);
         if (pJob->execRes.res) {

source/libs/sync/inc/syncInt.h

@@ -314,6 +314,7 @@ int32_t syncNodePeerStateInit(SSyncNode* pSyncNode);
 
 // trace log
 void syncLogRecvTimer(SSyncNode* pSyncNode, const SyncTimeout* pMsg, const char* s);
+void syncLogRecvLocalCmd(SSyncNode* pSyncNode, const SyncLocalCmd* pMsg, const char* s);
 
 void syncLogSendRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s);
 void syncLogRecvRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s);
@@ -336,7 +337,17 @@ void syncLogRecvHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, const
 void syncLogSendHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* pMsg, const char* s);
 void syncLogRecvHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* pMsg, const char* s);
 
-void syncLogRecvLocalCmd(SSyncNode* pSyncNode, const SyncLocalCmd* pMsg, const char* s);
+void syncLogSendSyncPreSnapshot(SSyncNode* pSyncNode, const SyncPreSnapshot* pMsg, const char* s);
+void syncLogRecvSyncPreSnapshot(SSyncNode* pSyncNode, const SyncPreSnapshot* pMsg, const char* s);
+
+void syncLogSendSyncPreSnapshotReply(SSyncNode* pSyncNode, const SyncPreSnapshotReply* pMsg, const char* s);
+void syncLogRecvSyncPreSnapshotReply(SSyncNode* pSyncNode, const SyncPreSnapshotReply* pMsg, const char* s);
+
+void syncLogSendSyncSnapshotSend(SSyncNode* pSyncNode, const SyncSnapshotSend* pMsg, const char* s);
+void syncLogRecvSyncSnapshotSend(SSyncNode* pSyncNode, const SyncSnapshotSend* pMsg, const char* s);
+
+void syncLogSendSyncSnapshotRsp(SSyncNode* pSyncNode, const SyncSnapshotRsp* pMsg, const char* s);
+void syncLogRecvSyncSnapshotRsp(SSyncNode* pSyncNode, const SyncSnapshotRsp* pMsg, const char* s);
 
 // for debug --------------
 void syncNodePrint(SSyncNode* pObj);

source/libs/sync/inc/syncSnapshot.h

@@ -28,10 +28,11 @@ extern "C" {
 #include "syncMessage.h"
 #include "taosdef.h"
 
-#define SYNC_SNAPSHOT_SEQ_INVALID -1
+#define SYNC_SNAPSHOT_SEQ_INVALID      -2
-#define SYNC_SNAPSHOT_SEQ_FORCE_CLOSE -2
+#define SYNC_SNAPSHOT_SEQ_FORCE_CLOSE  -3
-#define SYNC_SNAPSHOT_SEQ_BEGIN 0
+#define SYNC_SNAPSHOT_SEQ_PRE_SNAPSHOT -1
-#define SYNC_SNAPSHOT_SEQ_END 0x7FFFFFFF
+#define SYNC_SNAPSHOT_SEQ_BEGIN        0
+#define SYNC_SNAPSHOT_SEQ_END          0x7FFFFFFF
 
 #define SYNC_SNAPSHOT_RETRY_MS 5000
 
@@ -47,19 +48,19 @@ typedef struct SSyncSnapshotSender {
   SSnapshot      snapshot;
   SSyncCfg       lastConfig;
   int64_t        sendingMS;
-  SSyncNode     *pSyncNode;
-  int32_t        replicaIndex;
   SyncTerm       term;
-  SyncTerm       privateTerm;
   int64_t        startTime;
   bool           finish;
+
+  // init when create
+  SSyncNode *pSyncNode;
+  int32_t    replicaIndex;
 } SSyncSnapshotSender;
 
 SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex);
 void                 snapshotSenderDestroy(SSyncSnapshotSender *pSender);
 bool                 snapshotSenderIsStart(SSyncSnapshotSender *pSender);
-int32_t              snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshotParam snapshotParam, SSnapshot snapshot,
+int32_t              snapshotSenderStart(SSyncSnapshotSender *pSender);
-                                         void *pReader);
 int32_t              snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish);
 int32_t              snapshotSend(SSyncSnapshotSender *pSender);
 int32_t              snapshotReSend(SSyncSnapshotSender *pSender);
@@ -76,11 +77,13 @@ typedef struct SSyncSnapshotReceiver {
   int32_t        ack;
   void          *pWriter;
   SyncTerm       term;
-  SyncTerm       privateTerm;
   SSnapshotParam snapshotParam;
   SSnapshot      snapshot;
   SRaftId        fromId;
-  SSyncNode     *pSyncNode;
+  int64_t        startTime;
+
+  // init when create
+  SSyncNode *pSyncNode;
 
 } SSyncSnapshotReceiver;
 

source/libs/sync/inc/syncTools.h

@@ -496,6 +496,69 @@ void syncHeartbeatReplyPrint2(char* s, const SyncHeartbeatReply* pMsg);
 void syncHeartbeatReplyLog(const SyncHeartbeatReply* pMsg);
 void syncHeartbeatReplyLog2(char* s, const SyncHeartbeatReply* pMsg);
 
+// ---------------------------------------------
+typedef struct SyncPreSnapshot {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+
+  // private data
+  SyncTerm term;
+
+} SyncPreSnapshot;
+
+SyncPreSnapshot* syncPreSnapshotBuild(int32_t vgId);
+void             syncPreSnapshotDestroy(SyncPreSnapshot* pMsg);
+void             syncPreSnapshotSerialize(const SyncPreSnapshot* pMsg, char* buf, uint32_t bufLen);
+void             syncPreSnapshotDeserialize(const char* buf, uint32_t len, SyncPreSnapshot* pMsg);
+char*            syncPreSnapshotSerialize2(const SyncPreSnapshot* pMsg, uint32_t* len);
+SyncPreSnapshot* syncPreSnapshotDeserialize2(const char* buf, uint32_t len);
+void             syncPreSnapshot2RpcMsg(const SyncPreSnapshot* pMsg, SRpcMsg* pRpcMsg);
+void             syncPreSnapshotFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPreSnapshot* pMsg);
+SyncPreSnapshot* syncPreSnapshotFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*           syncPreSnapshot2Json(const SyncPreSnapshot* pMsg);
+char*            syncPreSnapshot2Str(const SyncPreSnapshot* pMsg);
+
+// for debug ----------------------
+void syncPreSnapshotPrint(const SyncPreSnapshot* pMsg);
+void syncPreSnapshotPrint2(char* s, const SyncPreSnapshot* pMsg);
+void syncPreSnapshotLog(const SyncPreSnapshot* pMsg);
+void syncPreSnapshotLog2(char* s, const SyncPreSnapshot* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncPreSnapshotReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+
+  // private data
+  SyncTerm  term;
+  SyncIndex snapStart;
+
+} SyncPreSnapshotReply;
+
+SyncPreSnapshotReply* syncPreSnapshotReplyBuild(int32_t vgId);
+void                  syncPreSnapshotReplyDestroy(SyncPreSnapshotReply* pMsg);
+void                  syncPreSnapshotReplySerialize(const SyncPreSnapshotReply* pMsg, char* buf, uint32_t bufLen);
+void                  syncPreSnapshotReplyDeserialize(const char* buf, uint32_t len, SyncPreSnapshotReply* pMsg);
+char*                 syncPreSnapshotReplySerialize2(const SyncPreSnapshotReply* pMsg, uint32_t* len);
+SyncPreSnapshotReply* syncPreSnapshotReplyDeserialize2(const char* buf, uint32_t len);
+void                  syncPreSnapshotReply2RpcMsg(const SyncPreSnapshotReply* pMsg, SRpcMsg* pRpcMsg);
+void                  syncPreSnapshotReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPreSnapshotReply* pMsg);
+SyncPreSnapshotReply* syncPreSnapshotReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*                syncPreSnapshotReply2Json(const SyncPreSnapshotReply* pMsg);
+char*                 syncPreSnapshotReply2Str(const SyncPreSnapshotReply* pMsg);
+
+// for debug ----------------------
+void syncPreSnapshotReplyPrint(const SyncPreSnapshotReply* pMsg);
+void syncPreSnapshotReplyPrint2(char* s, const SyncPreSnapshotReply* pMsg);
+void syncPreSnapshotReplyLog(const SyncPreSnapshotReply* pMsg);
+void syncPreSnapshotReplyLog2(char* s, const SyncPreSnapshotReply* pMsg);
+
 // ---------------------------------------------
 typedef struct SyncApplyMsg {
   uint32_t   bytes;
@@ -541,7 +604,7 @@ typedef struct SyncSnapshotSend {
   SyncTerm  lastTerm;         // snapshot.lastTerm
   SyncIndex lastConfigIndex;  // snapshot.lastConfigIndex
   SSyncCfg  lastConfig;
-  SyncTerm  privateTerm;
+  int64_t   startTime;
   int32_t   seq;
   uint32_t  dataLen;
   char      data[];
@@ -576,9 +639,10 @@ typedef struct SyncSnapshotRsp {
   SyncTerm  term;
   SyncIndex lastIndex;
   SyncTerm  lastTerm;
-  SyncTerm  privateTerm;
+  int64_t   startTime;
   int32_t   ack;
   int32_t   code;
+  SyncIndex snapBeginIndex;  // when ack = SYNC_SNAPSHOT_SEQ_BEGIN, it's valid
 } SyncSnapshotRsp;
 
 SyncSnapshotRsp* syncSnapshotRspBuild(int32_t vgId);
@@ -709,6 +773,9 @@ int32_t syncNodeOnRequestVoteReply(SSyncNode* ths, SyncRequestVoteReply* pMsg);
 int32_t syncNodeOnAppendEntries(SSyncNode* ths, SyncAppendEntries* pMsg);
 int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, SyncAppendEntriesReply* pMsg);
 
+int32_t syncNodeOnPreSnapshot(SSyncNode* ths, SyncPreSnapshot* pMsg);
+int32_t syncNodeOnPreSnapshotReply(SSyncNode* ths, SyncPreSnapshotReply* pMsg);
+
 int32_t syncNodeOnSnapshot(SSyncNode* ths, SyncSnapshotSend* pMsg);
 int32_t syncNodeOnSnapshotReply(SSyncNode* ths, SyncSnapshotRsp* pMsg);
 

source/libs/sync/src/syncAppendEntries.c

@@ -142,7 +142,6 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, SyncAppendEntries* pMsg) {
   // pReply->matchIndex = ths->pLogStore->syncLogLastIndex(ths->pLogStore);
   pReply->matchIndex = SYNC_INDEX_INVALID;
   pReply->lastSendIndex = pMsg->prevLogIndex + 1;
-  pReply->privateTerm = ths->pNewNodeReceiver->privateTerm;
   pReply->startTime = ths->startTime;
 
   if (pMsg->term < ths->pRaftStore->currentTerm) {

source/libs/sync/src/syncAppendEntriesReply.c

@@ -73,6 +73,7 @@ static void syncNodeStartSnapshotOnce(SSyncNode* ths, SyncIndex beginIndex, Sync
   int32_t        code = ths->pFsm->FpSnapshotStartRead(ths->pFsm, &readerParam, &pReader);
   ASSERT(code == 0);
 
+#if 0
   if (pMsg->privateTerm < pSender->privateTerm) {
     ASSERT(pReader != NULL);
     snapshotSenderStart(pSender, readerParam, snapshot, pReader);
@@ -82,6 +83,7 @@ static void syncNodeStartSnapshotOnce(SSyncNode* ths, SyncIndex beginIndex, Sync
       ths->pFsm->FpSnapshotStopRead(ths->pFsm, pReader);
     }
   }
+#endif
 }
 
 int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, SyncAppendEntriesReply* pMsg) {

source/libs/sync/src/syncMain.c

@@ -2236,6 +2236,7 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) {
   // init peer mgr
   syncNodePeerStateInit(pSyncNode);
 
+#if 0
   // update sender private term
   SSyncSnapshotSender* pMySender = syncNodeGetSnapshotSender(pSyncNode, &(pSyncNode->myRaftId));
   if (pMySender != NULL) {
@@ -2246,6 +2247,7 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) {
     }
     (pMySender->privateTerm) += 100;
   }
+#endif
 
   // close receiver
   if (snapshotReceiverIsStart(pSyncNode->pNewNodeReceiver)) {
@@ -3561,3 +3563,52 @@ void syncLogRecvLocalCmd(SSyncNode* pSyncNode, const SyncLocalCmd* pMsg, const c
            syncLocalCmdGetStr(pMsg->cmd), pMsg->sdNewTerm, s);
   syncNodeEventLog(pSyncNode, logBuf);
 }
+
+void syncLogSendSyncPreSnapshot(SSyncNode* pSyncNode, const SyncPreSnapshot* pMsg, const char* s) {
+  char     host[64];
+  uint16_t port;
+  syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port);
+  char logBuf[256];
+  snprintf(logBuf, sizeof(logBuf), "send sync-pre-snapshot to %s:%d {term:%" PRIu64 "}, %s", host, port, pMsg->term, s);
+  syncNodeEventLog(pSyncNode, logBuf);
+}
+
+void syncLogRecvSyncPreSnapshot(SSyncNode* pSyncNode, const SyncPreSnapshot* pMsg, const char* s) {
+  char     host[64];
+  uint16_t port;
+  syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
+  char logBuf[256];
+  snprintf(logBuf, sizeof(logBuf), "recv sync-pre-snapshot from %s:%d {term:%" PRIu64 "}, %s", host, port, pMsg->term,
+           s);
+  syncNodeEventLog(pSyncNode, logBuf);
+}
+
+void syncLogSendSyncPreSnapshotReply(SSyncNode* pSyncNode, const SyncPreSnapshotReply* pMsg, const char* s) {
+  char     host[64];
+  uint16_t port;
+  syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port);
+  char logBuf[256];
+  snprintf(logBuf, sizeof(logBuf),
+           "send sync-pre-snapshot-reply to %s:%d {term:%" PRIu64 ", snap-start:%" PRId64 "}, %s", host, port,
+           pMsg->term, pMsg->snapStart, s);
+  syncNodeEventLog(pSyncNode, logBuf);
+}
+
+void syncLogRecvSyncPreSnapshotReply(SSyncNode* pSyncNode, const SyncPreSnapshotReply* pMsg, const char* s) {
+  char     host[64];
+  uint16_t port;
+  syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
+  char logBuf[256];
+  snprintf(logBuf, sizeof(logBuf),
+           "recv sync-pre-snapshot-reply from %s:%d {term:%" PRIu64 ", snap-start:%" PRId64 "}, %s", host, port,
+           pMsg->term, pMsg->snapStart, s);
+  syncNodeEventLog(pSyncNode, logBuf);
+}
+
+void syncLogSendSyncSnapshotSend(SSyncNode* pSyncNode, const SyncSnapshotSend* pMsg, const char* s) {}
+
+void syncLogRecvSyncSnapshotSend(SSyncNode* pSyncNode, const SyncSnapshotSend* pMsg, const char* s) {}
+
+void syncLogSendSyncSnapshotRsp(SSyncNode* pSyncNode, const SyncSnapshotRsp* pMsg, const char* s) {}
+
+void syncLogRecvSyncSnapshotRsp(SSyncNode* pSyncNode, const SyncSnapshotRsp* pMsg, const char* s) {}

source/libs/sync/src/syncMessage.c

@@ -2315,6 +2315,303 @@ void syncHeartbeatReplyLog2(char* s, const SyncHeartbeatReply* pMsg) {
   }
 }
 
+// ---- message process SyncPreSnapshot----
+SyncPreSnapshot* syncPreSnapshotBuild(int32_t vgId) {
+  uint32_t         bytes = sizeof(SyncPreSnapshot);
+  SyncPreSnapshot* pMsg = taosMemoryMalloc(bytes);
+  memset(pMsg, 0, bytes);
+  pMsg->bytes = bytes;
+  pMsg->vgId = vgId;
+  pMsg->msgType = TDMT_SYNC_PRE_SNAPSHOT;
+  return pMsg;
+}
+
+void syncPreSnapshotDestroy(SyncPreSnapshot* pMsg) {
+  if (pMsg != NULL) {
+    taosMemoryFree(pMsg);
+  }
+}
+
+void syncPreSnapshotSerialize(const SyncPreSnapshot* pMsg, char* buf, uint32_t bufLen) {
+  ASSERT(pMsg->bytes <= bufLen);
+  memcpy(buf, pMsg, pMsg->bytes);
+}
+
+void syncPreSnapshotDeserialize(const char* buf, uint32_t len, SyncPreSnapshot* pMsg) {
+  memcpy(pMsg, buf, len);
+  ASSERT(len == pMsg->bytes);
+}
+
+char* syncPreSnapshotSerialize2(const SyncPreSnapshot* pMsg, uint32_t* len) {
+  char* buf = taosMemoryMalloc(pMsg->bytes);
+  ASSERT(buf != NULL);
+  syncPreSnapshotSerialize(pMsg, buf, pMsg->bytes);
+  if (len != NULL) {
+    *len = pMsg->bytes;
+  }
+  return buf;
+}
+
+SyncPreSnapshot* syncPreSnapshotDeserialize2(const char* buf, uint32_t len) {
+  uint32_t         bytes = *((uint32_t*)buf);
+  SyncPreSnapshot* pMsg = taosMemoryMalloc(bytes);
+  ASSERT(pMsg != NULL);
+  syncPreSnapshotDeserialize(buf, len, pMsg);
+  ASSERT(len == pMsg->bytes);
+  return pMsg;
+}
+
+void syncPreSnapshot2RpcMsg(const SyncPreSnapshot* pMsg, SRpcMsg* pRpcMsg) {
+  memset(pRpcMsg, 0, sizeof(*pRpcMsg));
+  pRpcMsg->msgType = pMsg->msgType;
+  pRpcMsg->contLen = pMsg->bytes;
+  pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen);
+  syncPreSnapshotSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen);
+}
+
+void syncPreSnapshotFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPreSnapshot* pMsg) {
+  syncPreSnapshotDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg);
+}
+
+SyncPreSnapshot* syncPreSnapshotFromRpcMsg2(const SRpcMsg* pRpcMsg) {
+  SyncPreSnapshot* pMsg = syncPreSnapshotDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen);
+  ASSERT(pMsg != NULL);
+  return pMsg;
+}
+
+cJSON* syncPreSnapshot2Json(const SyncPreSnapshot* pMsg) {
+  char   u64buf[128] = {0};
+  cJSON* pRoot = cJSON_CreateObject();
+
+  if (pMsg != NULL) {
+    cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
+    cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId);
+    cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
+
+    cJSON* pSrcId = cJSON_CreateObject();
+    snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr);
+    cJSON_AddStringToObject(pSrcId, "addr", u64buf);
+    {
+      uint64_t u64 = pMsg->srcId.addr;
+      cJSON*   pTmp = pSrcId;
+      char     host[128] = {0};
+      uint16_t port;
+      syncUtilU642Addr(u64, host, sizeof(host), &port);
+      cJSON_AddStringToObject(pTmp, "addr_host", host);
+      cJSON_AddNumberToObject(pTmp, "addr_port", port);
+    }
+    cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId);
+    cJSON_AddItemToObject(pRoot, "srcId", pSrcId);
+
+    cJSON* pDestId = cJSON_CreateObject();
+    snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr);
+    cJSON_AddStringToObject(pDestId, "addr", u64buf);
+    {
+      uint64_t u64 = pMsg->destId.addr;
+      cJSON*   pTmp = pDestId;
+      char     host[128] = {0};
+      uint16_t port;
+      syncUtilU642Addr(u64, host, sizeof(host), &port);
+      cJSON_AddStringToObject(pTmp, "addr_host", host);
+      cJSON_AddNumberToObject(pTmp, "addr_port", port);
+    }
+    cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId);
+    cJSON_AddItemToObject(pRoot, "destId", pDestId);
+
+    snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term);
+    cJSON_AddStringToObject(pRoot, "term", u64buf);
+  }
+
+  cJSON* pJson = cJSON_CreateObject();
+  cJSON_AddItemToObject(pJson, "SyncPreSnapshot", pRoot);
+  return pJson;
+}
+
+char* syncPreSnapshot2Str(const SyncPreSnapshot* pMsg) {
+  cJSON* pJson = syncPreSnapshot2Json(pMsg);
+  char*  serialized = cJSON_Print(pJson);
+  cJSON_Delete(pJson);
+  return serialized;
+}
+
+void syncPreSnapshotPrint(const SyncPreSnapshot* pMsg) {
+  char* serialized = syncPreSnapshot2Str(pMsg);
+  printf("syncPreSnapshotPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized);
+  fflush(NULL);
+  taosMemoryFree(serialized);
+}
+
+void syncPreSnapshotPrint2(char* s, const SyncPreSnapshot* pMsg) {
+  char* serialized = syncPreSnapshot2Str(pMsg);
+  printf("syncPreSnapshotPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized);
+  fflush(NULL);
+  taosMemoryFree(serialized);
+}
+
+void syncPreSnapshotLog(const SyncPreSnapshot* pMsg) {
+  char* serialized = syncPreSnapshot2Str(pMsg);
+  sTrace("syncPreSnapshotLog | len:%d | %s", (int32_t)strlen(serialized), serialized);
+  taosMemoryFree(serialized);
+}
+
+void syncPreSnapshotLog2(char* s, const SyncPreSnapshot* pMsg) {
+  if (gRaftDetailLog) {
+    char* serialized = syncPreSnapshot2Str(pMsg);
+    sTrace("syncPreSnapshotLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized);
+    taosMemoryFree(serialized);
+  }
+}
+
+// ---- message process SyncPreSnapshotReply----
+SyncPreSnapshotReply* syncPreSnapshotReplyBuild(int32_t vgId) {
+  uint32_t              bytes = sizeof(SyncPreSnapshotReply);
+  SyncPreSnapshotReply* pMsg = taosMemoryMalloc(bytes);
+  memset(pMsg, 0, bytes);
+  pMsg->bytes = bytes;
+  pMsg->vgId = vgId;
+  pMsg->msgType = TDMT_SYNC_PRE_SNAPSHOT_REPLY;
+  return pMsg;
+}
+
+void syncPreSnapshotReplyDestroy(SyncPreSnapshotReply* pMsg) {
+  if (pMsg != NULL) {
+    taosMemoryFree(pMsg);
+  }
+}
+
+void syncPreSnapshotReplySerialize(const SyncPreSnapshotReply* pMsg, char* buf, uint32_t bufLen) {
+  ASSERT(pMsg->bytes <= bufLen);
+  memcpy(buf, pMsg, pMsg->bytes);
+}
+
+void syncPreSnapshotReplyDeserialize(const char* buf, uint32_t len, SyncPreSnapshotReply* pMsg) {
+  memcpy(pMsg, buf, len);
+  ASSERT(len == pMsg->bytes);
+}
+
+char* syncPreSnapshotReplySerialize2(const SyncPreSnapshotReply* pMsg, uint32_t* len) {
+  char* buf = taosMemoryMalloc(pMsg->bytes);
+  ASSERT(buf != NULL);
+  syncPreSnapshotReplySerialize(pMsg, buf, pMsg->bytes);
+  if (len != NULL) {
+    *len = pMsg->bytes;
+  }
+  return buf;
+}
+
+SyncPreSnapshotReply* syncPreSnapshotReplyDeserialize2(const char* buf, uint32_t len) {
+  uint32_t              bytes = *((uint32_t*)buf);
+  SyncPreSnapshotReply* pMsg = taosMemoryMalloc(bytes);
+  ASSERT(pMsg != NULL);
+  syncPreSnapshotReplyDeserialize(buf, len, pMsg);
+  ASSERT(len == pMsg->bytes);
+  return pMsg;
+}
+
+void syncPreSnapshotReply2RpcMsg(const SyncPreSnapshotReply* pMsg, SRpcMsg* pRpcMsg) {
+  memset(pRpcMsg, 0, sizeof(*pRpcMsg));
+  pRpcMsg->msgType = pMsg->msgType;
+  pRpcMsg->contLen = pMsg->bytes;
+  pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen);
+  syncPreSnapshotReplySerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen);
+}
+
+void syncPreSnapshotReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPreSnapshotReply* pMsg) {
+  syncPreSnapshotReplyDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg);
+}
+
+SyncPreSnapshotReply* syncPreSnapshotReplyFromRpcMsg2(const SRpcMsg* pRpcMsg) {
+  SyncPreSnapshotReply* pMsg = syncPreSnapshotReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen);
+  ASSERT(pMsg != NULL);
+  return pMsg;
+}
+
+cJSON* syncPreSnapshotReply2Json(const SyncPreSnapshotReply* pMsg) {
+  char   u64buf[128] = {0};
+  cJSON* pRoot = cJSON_CreateObject();
+
+  if (pMsg != NULL) {
+    cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
+    cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId);
+    cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
+
+    cJSON* pSrcId = cJSON_CreateObject();
+    snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr);
+    cJSON_AddStringToObject(pSrcId, "addr", u64buf);
+    {
+      uint64_t u64 = pMsg->srcId.addr;
+      cJSON*   pTmp = pSrcId;
+      char     host[128] = {0};
+      uint16_t port;
+      syncUtilU642Addr(u64, host, sizeof(host), &port);
+      cJSON_AddStringToObject(pTmp, "addr_host", host);
+      cJSON_AddNumberToObject(pTmp, "addr_port", port);
+    }
+    cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId);
+    cJSON_AddItemToObject(pRoot, "srcId", pSrcId);
+
+    cJSON* pDestId = cJSON_CreateObject();
+    snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr);
+    cJSON_AddStringToObject(pDestId, "addr", u64buf);
+    {
+      uint64_t u64 = pMsg->destId.addr;
+      cJSON*   pTmp = pDestId;
+      char     host[128] = {0};
+      uint16_t port;
+      syncUtilU642Addr(u64, host, sizeof(host), &port);
+      cJSON_AddStringToObject(pTmp, "addr_host", host);
+      cJSON_AddNumberToObject(pTmp, "addr_port", port);
+    }
+    cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId);
+    cJSON_AddItemToObject(pRoot, "destId", pDestId);
+
+    snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term);
+    cJSON_AddStringToObject(pRoot, "term", u64buf);
+
+    snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->snapStart);
+    cJSON_AddStringToObject(pRoot, "snap-start", u64buf);
+  }
+
+  cJSON* pJson = cJSON_CreateObject();
+  cJSON_AddItemToObject(pJson, "SyncPreSnapshotReply", pRoot);
+  return pJson;
+}
+
+char* syncPreSnapshotReply2Str(const SyncPreSnapshotReply* pMsg) {
+  cJSON* pJson = syncPreSnapshotReply2Json(pMsg);
+  char*  serialized = cJSON_Print(pJson);
+  cJSON_Delete(pJson);
+  return serialized;
+}
+
+void syncPreSnapshotReplyPrint(const SyncPreSnapshotReply* pMsg) {
+  char* serialized = syncPreSnapshotReply2Str(pMsg);
+  printf("syncPreSnapshotReplyPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized);
+  fflush(NULL);
+  taosMemoryFree(serialized);
+}
+
+void syncPreSnapshotReplyPrint2(char* s, const SyncPreSnapshotReply* pMsg) {
+  char* serialized = syncPreSnapshotReply2Str(pMsg);
+  printf("syncPreSnapshotReplyPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized);
+  fflush(NULL);
+  taosMemoryFree(serialized);
+}
+
+void syncPreSnapshotReplyLog(const SyncPreSnapshotReply* pMsg) {
+  char* serialized = syncPreSnapshotReply2Str(pMsg);
+  sTrace("syncPreSnapshotReplyLog | len:%d | %s", (int32_t)strlen(serialized), serialized);
+  taosMemoryFree(serialized);
+}
+
+void syncPreSnapshotReplyLog2(char* s, const SyncPreSnapshotReply* pMsg) {
+  if (gRaftDetailLog) {
+    char* serialized = syncPreSnapshotReply2Str(pMsg);
+    sTrace("syncPreSnapshotReplyLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized);
+    taosMemoryFree(serialized);
+  }
+}
+
 // ---- message process SyncApplyMsg----
 SyncApplyMsg* syncApplyMsgBuild(uint32_t dataLen) {
   uint32_t      bytes = sizeof(SyncApplyMsg) + dataLen;
@@ -2576,8 +2873,8 @@ cJSON* syncSnapshotSend2Json(const SyncSnapshotSend* pMsg) {
     snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term);
     cJSON_AddStringToObject(pRoot, "term", u64buf);
 
-    snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm);
+    snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->startTime);
-    cJSON_AddStringToObject(pRoot, "privateTerm", u64buf);
+    cJSON_AddStringToObject(pRoot, "startTime", u64buf);
 
     snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->beginIndex);
     cJSON_AddStringToObject(pRoot, "beginIndex", u64buf);
@@ -2751,8 +3048,8 @@ cJSON* syncSnapshotRsp2Json(const SyncSnapshotRsp* pMsg) {
     snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term);
     cJSON_AddStringToObject(pRoot, "term", u64buf);
 
-    snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm);
+    snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->startTime);
-    cJSON_AddStringToObject(pRoot, "privateTerm", u64buf);
+    cJSON_AddStringToObject(pRoot, "startTime", u64buf);
 
     snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->lastIndex);
     cJSON_AddStringToObject(pRoot, "lastIndex", u64buf);
@@ -2762,6 +3059,9 @@ cJSON* syncSnapshotRsp2Json(const SyncSnapshotRsp* pMsg) {
 
     cJSON_AddNumberToObject(pRoot, "ack", pMsg->ack);
     cJSON_AddNumberToObject(pRoot, "code", pMsg->code);
+
+    snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->snapBeginIndex);
+    cJSON_AddStringToObject(pRoot, "snap-begin", u64buf);
   }
 
   cJSON* pJson = cJSON_CreateObject();

source/libs/sync/src/syncReplication.c

@@ -62,8 +62,8 @@ int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId) {
     syncNodeEventLog(pSyncNode, logBuf);
 
     // start snapshot
-    int32_t code = syncNodeStartSnapshot(pSyncNode, pDestId);
+    // int32_t code = syncNodeStartSnapshot(pSyncNode, pDestId);
-    ASSERT(code == 0);
+
     return 0;
   }
 

source/libs/sync/src/syncSnapshot.c

@@ -41,8 +41,6 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI
     }
     memset(pSender, 0, sizeof(*pSender));
 
-    int64_t timeNow = taosGetTimestampMs();
-
     pSender->start = false;
     pSender->seq = SYNC_SNAPSHOT_SEQ_INVALID;
     pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID;
@@ -53,8 +51,7 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI
     pSender->pSyncNode = pSyncNode;
     pSender->replicaIndex = replicaIndex;
     pSender->term = pSyncNode->pRaftStore->currentTerm;
-    pSender->privateTerm = timeNow + 100;
+    pSender->startTime = 0;
-    pSender->startTime = timeNow;
     pSender->pSyncNode->pFsm->FpGetSnapshotInfo(pSender->pSyncNode->pFsm, &(pSender->snapshot));
     pSender->finish = false;
   } else {
@@ -88,88 +85,30 @@ void snapshotSenderDestroy(SSyncSnapshotSender *pSender) {
 
 bool snapshotSenderIsStart(SSyncSnapshotSender *pSender) { return pSender->start; }
 
-// begin send snapshot by param, snapshot, pReader
+int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) {
-//
-// action:
-// 1. assert reader not start
-// 2. update state
-// 3. send first snapshot block
-int32_t snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshotParam snapshotParam, SSnapshot snapshot,
-                            void *pReader) {
   ASSERT(!snapshotSenderIsStart(pSender));
 
-  // init snapshot, parm, reader
-  ASSERT(pSender->pReader == NULL);
-  pSender->pReader = pReader;
-  pSender->snapshot = snapshot;
-  pSender->snapshotParam = snapshotParam;
-
-  // init current block
-  if (pSender->pCurrentBlock != NULL) {
-    taosMemoryFree(pSender->pCurrentBlock);
-  }
-  pSender->blockLen = 0;
-
-  // update term
-  pSender->term = pSender->pSyncNode->pRaftStore->currentTerm;
-  ++(pSender->privateTerm);  // increase private term
-
-  // update state
-  pSender->finish = false;
   pSender->start = true;
   pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN;
   pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID;
+  pSender->pReader = NULL;
+  pSender->pCurrentBlock = NULL;
+  pSender->blockLen = 0;
 
-  // init last config
+  pSender->snapshotParam.start = SYNC_INDEX_INVALID;
-  if (pSender->snapshot.lastConfigIndex != SYNC_INDEX_INVALID) {
+  pSender->snapshotParam.end = SYNC_INDEX_INVALID;
-    int32_t         code = 0;
-    SSyncRaftEntry *pEntry = NULL;
-    bool            getLastConfig = false;
 
-    code = pSender->pSyncNode->pLogStore->syncLogGetEntry(pSender->pSyncNode->pLogStore,
+  pSender->snapshot.data = NULL;
-                                                          pSender->snapshot.lastConfigIndex, &pEntry);
+  pSender->snapshotParam.end = SYNC_INDEX_INVALID;
-    if (code == 0 && pEntry != NULL) {
+  pSender->snapshot.lastApplyIndex = SYNC_INDEX_INVALID;
-      SRpcMsg rpcMsg;
+  pSender->snapshot.lastApplyTerm = SYNC_TERM_INVALID;
-      syncEntry2OriginalRpc(pEntry, &rpcMsg);
+  pSender->snapshot.lastConfigIndex = SYNC_INDEX_INVALID;
 
-      SSyncCfg lastConfig;
+  memset(&(pSender->lastConfig), 0, sizeof(pSender->lastConfig));
-      int32_t  ret = syncCfgFromStr(rpcMsg.pCont, &lastConfig);
+  pSender->sendingMS = 0;
-      ASSERT(ret == 0);
+  pSender->term = pSender->pSyncNode->pRaftStore->currentTerm;
-      pSender->lastConfig = lastConfig;
+  pSender->startTime = taosGetTimestampMs();
-      getLastConfig = true;
+  pSender->finish = false;
-
-      rpcFreeCont(rpcMsg.pCont);
-      syncEntryDestory(pEntry);
-    } else {
-      if (pSender->snapshot.lastConfigIndex == pSender->pSyncNode->pRaftCfg->lastConfigIndex) {
-        sTrace("vgId:%d, sync sender get cfg from local", pSender->pSyncNode->vgId);
-        pSender->lastConfig = pSender->pSyncNode->pRaftCfg->cfg;
-        getLastConfig = true;
-      }
-    }
-
-    // last config not found in wal, update to -1
-    if (!getLastConfig) {
-      SyncIndex oldLastConfigIndex = pSender->snapshot.lastConfigIndex;
-      SyncIndex newLastConfigIndex = SYNC_INDEX_INVALID;
-      pSender->snapshot.lastConfigIndex = SYNC_INDEX_INVALID;
-      memset(&(pSender->lastConfig), 0, sizeof(SSyncCfg));
-
-      // event log
-      do {
-        char logBuf[128];
-        snprintf(logBuf, sizeof(logBuf), "snapshot sender update lcindex from %" PRId64 " to %" PRId64,
-                 oldLastConfigIndex, newLastConfigIndex);
-        char *eventLog = snapshotSender2SimpleStr(pSender, logBuf);
-        syncNodeEventLog(pSender->pSyncNode, eventLog);
-        taosMemoryFree(eventLog);
-      } while (0);
-    }
-
-  } else {
-    // no last config
-    memset(&(pSender->lastConfig), 0, sizeof(SSyncCfg));
-  }
 
   // build begin msg
   SyncSnapshotSend *pMsg = syncSnapshotSendBuild(0, pSender->pSyncNode->vgId);
@@ -181,8 +120,8 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshotParam snapsho
   pMsg->lastTerm = pSender->snapshot.lastApplyTerm;
   pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex;
   pMsg->lastConfig = pSender->lastConfig;
-  pMsg->seq = pSender->seq;  // SYNC_SNAPSHOT_SEQ_BEGIN
+  pMsg->startTime = pSender->startTime;
-  pMsg->privateTerm = pSender->privateTerm;
+  pMsg->seq = SYNC_SNAPSHOT_SEQ_PRE_SNAPSHOT;
 
   // send msg
   SRpcMsg rpcMsg;
@@ -201,6 +140,10 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshotParam snapsho
 }
 
 int32_t snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish) {
+  // update flag
+  pSender->start = false;
+  pSender->finish = finish;
+
   // close reader
   if (pSender->pReader != NULL) {
     int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader);
@@ -215,12 +158,6 @@ int32_t snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish) {
     pSender->blockLen = 0;
   }
 
-  // update flag
-  pSender->start = false;
-  pSender->finish = finish;
-
-  // do not update term, maybe print
-
   // event log
   do {
     char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender stop");
@@ -263,7 +200,9 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) {
   pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex;
   pMsg->lastConfig = pSender->lastConfig;
   pMsg->seq = pSender->seq;
-  pMsg->privateTerm = pSender->privateTerm;
+
+  // pMsg->privateTerm = pSender->privateTerm;
+
   memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen);
 
   // send msg
@@ -302,7 +241,9 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) {
     pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex;
     pMsg->lastConfig = pSender->lastConfig;
     pMsg->seq = pSender->seq;
-    pMsg->privateTerm = pSender->privateTerm;
+
+    //  pMsg->privateTerm = pSender->privateTerm;
+
     memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen);
 
     // send msg
@@ -367,8 +308,10 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) {
     cJSON_AddNumberToObject(pRoot, "replicaIndex", pSender->replicaIndex);
     snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->term);
     cJSON_AddStringToObject(pRoot, "term", u64buf);
-    snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->privateTerm);
+
-    cJSON_AddStringToObject(pRoot, "privateTerm", u64buf);
+    //  snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->privateTerm);
+    //  cJSON_AddStringToObject(pRoot, "privateTerm", u64buf);
+
     cJSON_AddNumberToObject(pRoot, "finish", pSender->finish);
   }
 
@@ -395,30 +338,38 @@ char *snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event) {
 
   snprintf(s, len,
            "%s {%p s-param:%" PRId64 " e-param:%" PRId64 " laindex:%" PRId64 " laterm:%" PRIu64 " lcindex:%" PRId64
-           " seq:%d ack:%d finish:%d pterm:%" PRIu64
+           " seq:%d ack:%d finish:%d replica-index:%d %s:%d}",
-           " "
-           "replica-index:%d %s:%d}",
            event, pSender, pSender->snapshotParam.start, pSender->snapshotParam.end, pSender->snapshot.lastApplyIndex,
            pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->seq, pSender->ack,
-           pSender->finish, pSender->privateTerm, pSender->replicaIndex, host, port);
+           pSender->finish, pSender->replicaIndex, host, port);
 
   return s;
 }
 
 int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) {
-  // calculate <start, end> index
+  syncNodeEventLog(pSyncNode, "starting snapshot ...");
-
-  syncNodeEventLog(pSyncNode, "start snapshot ...");
 
   SSyncSnapshotSender *pSender = syncNodeGetSnapshotSender(pSyncNode, pDestId);
   if (pSender == NULL) {
-    // create sender
+    syncNodeErrorLog(pSyncNode, "start snapshot error, sender is null");
-  } else {
+    return -1;
-    // if <start, end> is same
-    // return 0;
   }
 
-  // send begin msg
+  int32_t code = 0;
+
+  if (snapshotSenderIsStart(pSender)) {
+    code = snapshotSenderStop(pSender, false);
+    if (code != 0) {
+      syncNodeErrorLog(pSyncNode, "snapshot sender stop error");
+      return -1;
+    }
+  }
+
+  code = snapshotSenderStart(pSender);
+  if (code != 0) {
+    syncNodeErrorLog(pSyncNode, "snapshot sender start error");
+    return -1;
+  }
 
   return 0;
 }
@@ -440,7 +391,6 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId from
     pReceiver->pSyncNode = pSyncNode;
     pReceiver->fromId = fromId;
     pReceiver->term = pSyncNode->pRaftStore->currentTerm;
-    pReceiver->privateTerm = 0;
     pReceiver->snapshot.data = NULL;
     pReceiver->snapshot.lastApplyIndex = SYNC_INDEX_INVALID;
     pReceiver->snapshot.lastApplyTerm = 0;
@@ -474,19 +424,8 @@ bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver) { return pReceive
 // receive first snapshot data
 // write first block data
 static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg) {
-  // update state
-  pReceiver->term = pReceiver->pSyncNode->pRaftStore->currentTerm;
-  pReceiver->privateTerm = pBeginMsg->privateTerm;
-  pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN;
-  pReceiver->fromId = pBeginMsg->srcId;
   pReceiver->start = true;
-
+  pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN;
-  // update snapshot
-  pReceiver->snapshot.lastApplyIndex = pBeginMsg->lastIndex;
-  pReceiver->snapshot.lastApplyTerm = pBeginMsg->lastTerm;
-  pReceiver->snapshot.lastConfigIndex = pBeginMsg->lastConfigIndex;
-  pReceiver->snapshotParam.start = pBeginMsg->beginIndex;
-  pReceiver->snapshotParam.end = pBeginMsg->lastIndex;
 
   // start writer
   ASSERT(pReceiver->pWriter == NULL);
@@ -494,6 +433,19 @@ static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncSnapsh
                                                                  &(pReceiver->snapshotParam), &(pReceiver->pWriter));
   ASSERT(ret == 0);
 
+  pReceiver->term = pReceiver->pSyncNode->pRaftStore->currentTerm;
+  pReceiver->snapshotParam.start = pBeginMsg->beginIndex;
+  pReceiver->snapshotParam.end = pBeginMsg->lastIndex;
+
+  pReceiver->fromId = pBeginMsg->srcId;
+
+  // update snapshot
+  pReceiver->snapshot.lastApplyIndex = pBeginMsg->lastIndex;
+  pReceiver->snapshot.lastApplyTerm = pBeginMsg->lastTerm;
+  pReceiver->snapshot.lastConfigIndex = pBeginMsg->lastConfigIndex;
+
+  pReceiver->startTime = pBeginMsg->startTime;
+
   // event log
   do {
     char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver start");
@@ -523,22 +475,9 @@ void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver) {
 }
 
 // if receiver receive msg from seq = SYNC_SNAPSHOT_SEQ_BEGIN, start receiver
-// if already start, force close, start again
 int32_t snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg) {
-  if (!snapshotReceiverIsStart(pReceiver)) {
+  ASSERT(!snapshotReceiverIsStart(pReceiver));
-    // first start
+  snapshotReceiverDoStart(pReceiver, pBeginMsg);
-    snapshotReceiverDoStart(pReceiver, pBeginMsg);
-
-  } else {
-    // already start
-    sInfo("vgId:%d, snapshot recv, receiver already start", pReceiver->pSyncNode->vgId);
-
-    // force close, abandon incomplete data
-    snapshotReceiverForceStop(pReceiver);
-
-    // start again
-    snapshotReceiverDoStart(pReceiver, pBeginMsg);
-  }
 
   return 0;
 }
@@ -698,8 +637,8 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) {
     snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->term);
     cJSON_AddStringToObject(pRoot, "term", u64buf);
 
-    snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->privateTerm);
+    snprintf(u64buf, sizeof(u64buf), "%" PRId64, pReceiver->startTime);
-    cJSON_AddStringToObject(pRoot, "privateTerm", u64buf);
+    cJSON_AddStringToObject(pRoot, "startTime", u64buf);
   }
 
   cJSON *pJson = cJSON_CreateObject();
@@ -724,38 +663,204 @@ char *snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event)
   syncUtilU642Addr(fromId.addr, host, sizeof(host), &port);
 
   snprintf(s, len,
-           "%s {%p start:%d ack:%d term:%" PRIu64 " pterm:%" PRIu64 " from:%s:%d s-param:%" PRId64 " e-param:%" PRId64
+           "%s {%p start:%d ack:%d term:%" PRIu64 " start-time:%" PRId64 " from:%s:%d s-param:%" PRId64
-           " laindex:%" PRId64 " laterm:%" PRIu64
+           " e-param:%" PRId64 " laindex:%" PRId64 " laterm:%" PRIu64
            " "
            "lcindex:%" PRId64 "}",
-           event, pReceiver, pReceiver->start, pReceiver->ack, pReceiver->term, pReceiver->privateTerm, host, port,
+           event, pReceiver, pReceiver->start, pReceiver->ack, pReceiver->term, pReceiver->startTime, host, port,
            pReceiver->snapshotParam.start, pReceiver->snapshotParam.end, pReceiver->snapshot.lastApplyIndex,
            pReceiver->snapshot.lastApplyTerm, pReceiver->snapshot.lastConfigIndex);
 
   return s;
 }
 
+SyncIndex syncNodeGetSnapBeginIndex(SSyncNode *ths) {
+  SyncIndex snapStart = SYNC_INDEX_INVALID;
+
+  if (syncNodeIsMnode(ths)) {
+    snapStart = SYNC_INDEX_BEGIN;
+
+  } else {
+    SSyncLogStoreData *pData = ths->pLogStore->data;
+    SWal              *pWal = pData->pWal;
+
+    bool    isEmpty = ths->pLogStore->syncLogIsEmpty(ths->pLogStore);
+    int64_t walCommitVer = walGetCommittedVer(pWal);
+
+    if (!isEmpty && ths->commitIndex != walCommitVer) {
+      char logBuf[128];
+      snprintf(logBuf, sizeof(logBuf), "commit not same, wal-commit:%" PRId64 ", commit:%" PRId64 ", ignore",
+               walCommitVer, ths->commitIndex);
+      syncNodeErrorLog(ths, logBuf);
+
+      snapStart = walCommitVer + 1;
+    } else {
+      snapStart = ths->commitIndex + 1;
+    }
+  }
+
+  return snapStart;
+}
+
+static int32_t syncNodeOnSnapshotPre(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
+  SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver;
+
+  if (snapshotReceiverIsStart(pReceiver)) {
+    // already start
+
+    if (pMsg->startTime > pReceiver->startTime) {
+      goto _START_RECEIVER;
+
+    } else if (pMsg->startTime == pReceiver->startTime) {
+      goto _SEND_REPLY;
+
+    } else {
+      // ignore
+      return 0;
+    }
+
+  } else {
+    // start new
+    goto _START_RECEIVER;
+  }
+
+_START_RECEIVER:
+  if (taosGetTimestampMs() - pMsg->startTime > SNAPSHOT_MAX_CLOCK_SKEW_MS) {
+    syncNodeErrorLog(pSyncNode, "snapshot receiver time skew too much");
+    return -1;
+  } else {
+    // waiting for clock match
+    while (taosGetTimestampMs() > pMsg->startTime) {
+      taosMsleep(10);
+    }
+
+    snapshotReceiverStart(pReceiver, pMsg);  // set start-time same with sender
+  }
+
+_SEND_REPLY:
+    // build msg
+    ;  // make complier happy
+  SyncSnapshotRsp *pRspMsg = syncSnapshotRspBuild(pSyncNode->vgId);
+  pRspMsg->srcId = pSyncNode->myRaftId;
+  pRspMsg->destId = pMsg->srcId;
+  pRspMsg->term = pSyncNode->pRaftStore->currentTerm;
+  pRspMsg->lastIndex = pMsg->lastIndex;
+  pRspMsg->lastTerm = pMsg->lastTerm;
+  pRspMsg->startTime = pReceiver->startTime;
+  pRspMsg->ack = pMsg->seq;  // receiver maybe already closed
+  pRspMsg->code = 0;
+  pRspMsg->snapBeginIndex = syncNodeGetSnapBeginIndex(pSyncNode);
+
+  // send msg
+  SRpcMsg rpcMsg;
+  syncSnapshotRsp2RpcMsg(pRspMsg, &rpcMsg);
+  syncNodeSendMsgById(&(pRspMsg->destId), pSyncNode, &rpcMsg);
+  syncSnapshotRspDestroy(pRspMsg);
+
+  return 0;
+}
+
+static int32_t syncNodeOnSnapshotBegin(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
+  // condition 1
+  SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver;
+
+  if (snapshotReceiverIsStart(pReceiver)) {
+    if (pMsg->startTime > pReceiver->startTime) {
+      snapshotReceiverStop(pReceiver);
+
+    } else if (pMsg->startTime == pReceiver->startTime) {
+      return 0;
+    } else {
+      // ignore
+      syncNodeEventLog(pSyncNode, "msg ignore");
+      return 0;
+    }
+  }
+
+_START_RECEIVER:
+  if (taosGetTimestampMs() - pMsg->startTime > SNAPSHOT_MAX_CLOCK_SKEW_MS) {
+    syncNodeErrorLog(pSyncNode, "snapshot receiver time skew too much");
+    return -1;
+  } else {
+    // waiting for clock match
+    while (taosGetTimestampMs() > pMsg->startTime) {
+      taosMsleep(10);
+    }
+
+    snapshotReceiverStart(pReceiver, pMsg);
+
+    // build msg
+    SyncSnapshotRsp *pRspMsg = syncSnapshotRspBuild(pSyncNode->vgId);
+    pRspMsg->srcId = pSyncNode->myRaftId;
+    pRspMsg->destId = pMsg->srcId;
+    pRspMsg->term = pSyncNode->pRaftStore->currentTerm;
+    pRspMsg->lastIndex = pMsg->lastIndex;
+    pRspMsg->lastTerm = pMsg->lastTerm;
+    pRspMsg->ack = pReceiver->ack;  // receiver maybe already closed
+    pRspMsg->code = 0;
+
+    // send msg
+    SRpcMsg rpcMsg;
+    syncSnapshotRsp2RpcMsg(pRspMsg, &rpcMsg);
+    syncNodeSendMsgById(&(pRspMsg->destId), pSyncNode, &rpcMsg);
+    syncSnapshotRspDestroy(pRspMsg);
+  }
+
+  return 0;
+}
+
+static int32_t syncNodeOnSnapshotTransfer(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { return 0; }
+
+static int32_t syncNodeOnSnapshotEnd(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { return 0; }
+
 // receiver on message
 //
-// condition 1, recv SYNC_SNAPSHOT_SEQ_BEGIN, start receiver, update privateTerm
+// condition 1, recv SYNC_SNAPSHOT_SEQ_PRE_SNAPSHOT
-// condition 2, recv SYNC_SNAPSHOT_SEQ_END, finish receiver(apply snapshot data, update commit index, maybe reconfig)
+//              if receiver already start
-// condition 3, recv SYNC_SNAPSHOT_SEQ_FORCE_CLOSE, force close
+//                    if sender.start-time > receiver.start-time, restart receiver(reply snapshot start)
-// condition 4, got data, update ack
+//                    if sender.start-time = receiver.start-time, maybe duplicate msg
+//                    if sender.start-time < receiver.start-time, ignore
+//              else
+//                    waiting for clock match
+//                    start receiver(reply snapshot start)
+//
+// condition 2, recv SYNC_SNAPSHOT_SEQ_BEGIN
+//              a. create writer with <begin, end>
+//
+// condition 3, recv SYNC_SNAPSHOT_SEQ_END, finish receiver(apply snapshot data, update commit index, maybe reconfig)
+//
+// condition 4, recv SYNC_SNAPSHOT_SEQ_FORCE_CLOSE, force close
+//
+// condition 5, got data, update ack
 //
 int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
-  // get receiver
+  // if already drop replica, do not process
-  SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver;
+  if (!syncNodeInRaftGroup(pSyncNode, &(pMsg->srcId))) {
-  bool                   needRsp = false;
+    syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "not in my config");
+    return 0;
+  }
+
+  if (pMsg->term < pSyncNode->pRaftStore->currentTerm) {
+    syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "reject, small term");
+    return 0;
+  }
+
+  if (pMsg->term > pSyncNode->pRaftStore->currentTerm) {
+    syncNodeStepDown(pSyncNode, pMsg->term);
+  }
+  syncNodeResetElectTimer(pSyncNode);
+
   int32_t                code = 0;
+  SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver;
 
   // state, term, seq/ack
   if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) {
     if (pMsg->term == pSyncNode->pRaftStore->currentTerm) {
-      if (pMsg->seq == SYNC_SNAPSHOT_SEQ_BEGIN) {
+      if (pMsg->seq == SYNC_SNAPSHOT_SEQ_PRE_SNAPSHOT) {
-        // condition 1
+        syncNodeOnSnapshotPre(pSyncNode, pMsg);
-        // begin, no data
+
-        snapshotReceiverStart(pReceiver, pMsg);
+      } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_BEGIN) {
-        needRsp = true;
+        syncNodeOnSnapshotBegin(pSyncNode, pMsg);
 
       } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) {
         // condition 2
@@ -764,7 +869,7 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
         if (code == 0) {
           snapshotReceiverStop(pReceiver);
         }
-        needRsp = true;
+        bool needRsp = true;
 
         // maybe update lastconfig
         if (pMsg->lastConfigIndex >= SYNC_INDEX_BEGIN) {
@@ -782,7 +887,7 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
         // condition 3
         // force close
         snapshotReceiverForceStop(pReceiver);
-        needRsp = false;
+        bool needRsp = false;
 
       } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) {
         // condition 4
@@ -790,7 +895,7 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
         if (pMsg->seq == pReceiver->ack + 1) {
           snapshotReceiverGotData(pReceiver, pMsg);
         }
-        needRsp = true;
+        bool needRsp = true;
 
       } else {
         // error log
@@ -805,26 +910,6 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
         return -1;
       }
 
-      // send ack
-      if (needRsp) {
-        // build msg
-        SyncSnapshotRsp *pRspMsg = syncSnapshotRspBuild(pSyncNode->vgId);
-        pRspMsg->srcId = pSyncNode->myRaftId;
-        pRspMsg->destId = pMsg->srcId;
-        pRspMsg->term = pSyncNode->pRaftStore->currentTerm;
-        pRspMsg->lastIndex = pMsg->lastIndex;
-        pRspMsg->lastTerm = pMsg->lastTerm;
-        pRspMsg->ack = pReceiver->ack;  // receiver maybe already closed
-        pRspMsg->code = 0;
-        pRspMsg->privateTerm = pReceiver->privateTerm;  // receiver maybe already closed
-
-        // send msg
-        SRpcMsg rpcMsg;
-        syncSnapshotRsp2RpcMsg(pRspMsg, &rpcMsg);
-        syncNodeSendMsgById(&(pRspMsg->destId), pSyncNode, &rpcMsg);
-        syncSnapshotRspDestroy(pRspMsg);
-      }
-
     } else {
       // error log
       do {
@@ -849,6 +934,52 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
   return 0;
 }
 
+int32_t syncNodeOnSnapshotReplyPre(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) {
+  // get sender
+  SSyncSnapshotSender *pSender = syncNodeGetSnapshotSender(pSyncNode, &(pMsg->srcId));
+  ASSERT(pSender != NULL);
+
+  SSnapshot snapshot;
+  pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
+
+  // prepare <begin, end>
+  pSender->snapshotParam.start = pMsg->snapBeginIndex;
+  pSender->snapshotParam.end = snapshot.lastApplyIndex;
+
+  if (pMsg->snapBeginIndex > snapshot.lastApplyIndex) {
+    syncNodeErrorLog(pSyncNode, "snapshot last index too small");
+    return -1;
+  }
+
+  // start reader
+  int32_t code = pSyncNode->pFsm->FpSnapshotStartRead(pSyncNode->pFsm, &(pSender->snapshotParam), &(pSender->pReader));
+  if (code != 0) {
+    syncNodeErrorLog(pSyncNode, "create snapshot reader error");
+    return -1;
+  }
+
+  // build begin msg
+  SyncSnapshotSend *pSendMsg = syncSnapshotSendBuild(0, pSender->pSyncNode->vgId);
+  pSendMsg->srcId = pSender->pSyncNode->myRaftId;
+  pSendMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex];
+  pSendMsg->term = pSender->pSyncNode->pRaftStore->currentTerm;
+  pSendMsg->beginIndex = pSender->snapshotParam.start;
+  pSendMsg->lastIndex = pSender->snapshot.lastApplyIndex;
+  pSendMsg->lastTerm = pSender->snapshot.lastApplyTerm;
+  pSendMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex;
+  pSendMsg->lastConfig = pSender->lastConfig;
+  pSendMsg->startTime = pSender->startTime;
+  pSendMsg->seq = SYNC_SNAPSHOT_SEQ_BEGIN;
+
+  // send msg
+  SRpcMsg rpcMsg;
+  syncSnapshotSend2RpcMsg(pSendMsg, &rpcMsg);
+  syncNodeSendMsgById(&(pSendMsg->destId), pSender->pSyncNode, &rpcMsg);
+  syncSnapshotSendDestroy(pSendMsg);
+
+  return 0;
+}
+
 // sender on message
 //
 // condition 1 sender receives SYNC_SNAPSHOT_SEQ_END, close sender
@@ -857,8 +988,8 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
 //
 int32_t syncNodeOnSnapshotReply(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) {
   // if already drop replica, do not process
-  if (!syncNodeInRaftGroup(pSyncNode, &(pMsg->srcId)) && pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
+  if (!syncNodeInRaftGroup(pSyncNode, &(pMsg->srcId))) {
-    sError("vgId:%d, recv sync-snapshot-rsp, maybe replica already dropped", pSyncNode->vgId);
+    syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "maybe replica already dropped");
     return -1;
   }
 
@@ -866,17 +997,26 @@ int32_t syncNodeOnSnapshotReply(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) {
   SSyncSnapshotSender *pSender = syncNodeGetSnapshotSender(pSyncNode, &(pMsg->srcId));
   ASSERT(pSender != NULL);
 
+  if (pMsg->startTime != pSender->startTime) {
+    syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "sender/receiver start time not match");
+    return -1;
+  }
+
   // state, term, seq/ack
   if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
     if (pMsg->term == pSyncNode->pRaftStore->currentTerm) {
-      // condition 1
+      // prepare <begin, end>, send begin msg
+      if (pMsg->ack == SYNC_SNAPSHOT_SEQ_PRE_SNAPSHOT) {
+        syncNodeOnSnapshotReplyPre(pSyncNode, pMsg);
+        return 0;
+      }
+
       // receive ack is finish, close sender
       if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) {
         snapshotSenderStop(pSender, true);
         return 0;
       }
 
-      // condition 2
       // send next msg
       if (pMsg->ack == pSender->seq) {
         // update sender ack
@@ -922,3 +1062,63 @@ int32_t syncNodeOnSnapshotReply(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) {
 
   return 0;
 }
+
+int32_t syncNodeOnPreSnapshot(SSyncNode *ths, SyncPreSnapshot *pMsg) {
+  syncLogRecvSyncPreSnapshot(ths, pMsg, "");
+
+  SyncPreSnapshotReply *pMsgReply = syncPreSnapshotReplyBuild(ths->vgId);
+  pMsgReply->srcId = ths->myRaftId;
+  pMsgReply->destId = pMsg->srcId;
+  pMsgReply->term = ths->pRaftStore->currentTerm;
+
+  SSyncLogStoreData *pData = ths->pLogStore->data;
+  SWal              *pWal = pData->pWal;
+
+  if (syncNodeIsMnode(ths)) {
+    pMsgReply->snapStart = SYNC_INDEX_BEGIN;
+
+  } else {
+    bool    isEmpty = ths->pLogStore->syncLogIsEmpty(ths->pLogStore);
+    int64_t walCommitVer = walGetCommittedVer(pWal);
+
+    if (!isEmpty && ths->commitIndex != walCommitVer) {
+      char logBuf[128];
+      snprintf(logBuf, sizeof(logBuf), "commit not same, wal-commit:%" PRId64 ", commit:%" PRId64 ", ignore",
+               walCommitVer, ths->commitIndex);
+      syncNodeErrorLog(ths, logBuf);
+
+      goto _IGNORE;
+    }
+
+    pMsgReply->snapStart = ths->commitIndex + 1;
+
+    // make local log clean
+    int32_t code = ths->pLogStore->syncLogTruncate(ths->pLogStore, pMsgReply->snapStart);
+    if (code != 0) {
+      syncNodeErrorLog(ths, "truncate wal error");
+      goto _IGNORE;
+    }
+  }
+
+  // can not write behind _RESPONSE
+  SRpcMsg rpcMsg;
+
+_RESPONSE:
+  syncPreSnapshotReply2RpcMsg(pMsgReply, &rpcMsg);
+  syncNodeSendMsgById(&pMsgReply->destId, ths, &rpcMsg);
+
+  syncPreSnapshotReplyDestroy(pMsgReply);
+  return 0;
+
+_IGNORE:
+  syncPreSnapshotReplyDestroy(pMsgReply);
+  return 0;
+}
+
+int32_t syncNodeOnPreSnapshotReply(SSyncNode *ths, SyncPreSnapshotReply *pMsg) {
+  syncLogRecvSyncPreSnapshotReply(ths, pMsg, "");
+
+  // start snapshot
+
+  return 0;
+}

source/libs/sync/test/CMakeLists.txt

@@ -60,6 +60,8 @@ add_executable(syncRaftCfgIndexTest "")
 add_executable(syncHeartbeatTest "")
 add_executable(syncHeartbeatReplyTest "")
 add_executable(syncLocalCmdTest "")
+add_executable(syncPreSnapshotTest "")
+add_executable(syncPreSnapshotReplyTest "")
 
 
 target_sources(syncTest
@@ -310,6 +312,14 @@ target_sources(syncLocalCmdTest
     PRIVATE
     "syncLocalCmdTest.cpp"
 )
+target_sources(syncPreSnapshotTest
+    PRIVATE
+    "syncPreSnapshotTest.cpp"
+)
+target_sources(syncPreSnapshotReplyTest
+    PRIVATE
+    "syncPreSnapshotReplyTest.cpp"
+)
 
 
 target_include_directories(syncTest
@@ -622,6 +632,16 @@ target_include_directories(syncLocalCmdTest
     "${TD_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
+target_include_directories(syncPreSnapshotTest
+    PUBLIC
+    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
+target_include_directories(syncPreSnapshotReplyTest
+    PUBLIC
+    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
 
 
 target_link_libraries(syncTest
@@ -872,6 +892,14 @@ target_link_libraries(syncLocalCmdTest
     sync
     gtest_main
 )
+target_link_libraries(syncPreSnapshotTest
+    sync
+    gtest_main
+)
+target_link_libraries(syncPreSnapshotReplyTest
+    sync
+    gtest_main
+)
 
 
 enable_testing()

source/libs/sync/test/syncConfigChangeSnapshotTest.cpp

@@ -157,6 +157,7 @@ SSyncFSM* createFsm() {
   SSyncFSM* pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM));
   memset(pFsm, 0, sizeof(*pFsm));
 
+#if 0
   pFsm->FpCommitCb = CommitCb;
   pFsm->FpPreCommitCb = PreCommitCb;
   pFsm->FpRollBackCb = RollBackCb;
@@ -171,6 +172,7 @@ SSyncFSM* createFsm() {
   pFsm->FpSnapshotDoWrite = SnapshotDoWrite;
 
   pFsm->FpReConfigCb = ReConfigCb;
+#endif
 
   return pFsm;
 }

source/libs/sync/test/syncConfigChangeTest.cpp

@@ -33,7 +33,7 @@ void init() {
 
 void cleanup() { walCleanUp(); }
 
-void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {
+void CommitCb(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, const SFsmCbMeta cbMeta) {
   SyncIndex beginIndex = SYNC_INDEX_INVALID;
   if (pFsm->FpGetSnapshotInfo != NULL) {
     SSnapshot snapshot;
@@ -52,7 +52,7 @@ void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {
   }
 }
 
-void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {
+void PreCommitCb(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {
   char logBuf[256] = {0};
   snprintf(logBuf, sizeof(logBuf),
            "==callback== ==PreCommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s flag:%" PRIu64
@@ -61,7 +61,7 @@ void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta)
   syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg);
 }
 
-void RollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {
+void RollBackCb(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {
   char logBuf[256];
   snprintf(logBuf, sizeof(logBuf),
            "==callback== ==RollBackCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s flag:%" PRIu64 "\n",
@@ -69,7 +69,7 @@ void RollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {
   syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg);
 }
 
-int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) {
+int32_t GetSnapshotCb(const struct SSyncFSM* pFsm, SSnapshot* pSnapshot) {
   pSnapshot->data = NULL;
   pSnapshot->lastApplyIndex = gSnapshotLastApplyIndex;
   pSnapshot->lastApplyTerm = 100;
@@ -87,6 +87,7 @@ SSyncFSM* createFsm() {
   SSyncFSM* pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM));
   memset(pFsm, 0, sizeof(*pFsm));
 
+#if 0
   pFsm->FpCommitCb = CommitCb;
   pFsm->FpPreCommitCb = PreCommitCb;
   pFsm->FpRollBackCb = RollBackCb;
@@ -95,6 +96,7 @@ SSyncFSM* createFsm() {
   pFsm->FpRestoreFinishCb = RestoreFinishCb;
 
   pFsm->FpReConfigCb = ReConfigCb;
+#endif
 
   return pFsm;
 }

source/libs/sync/test/syncEnvTest.cpp

@@ -26,12 +26,12 @@ int main() {
   assert(ret == 0);
 
   for (int i = 0; i < 5; ++i) {
-    ret = syncEnvStartTimer();
+    //ret = syncEnvStartTimer();
     assert(ret == 0);
 
     taosMsleep(5000);
 
-    ret = syncEnvStopTimer();
+    //ret = syncEnvStopTimer();
     assert(ret == 0);
 
     taosMsleep(5000);

source/libs/sync/test/syncPreSnapshotReplyTest.cpp

@@ -0,0 +1,99 @@
+
+#include <gtest/gtest.h>
+#include <stdio.h>
+#include "syncIO.h"
+#include "syncInt.h"
+#include "syncMessage.h"
+#include "syncUtil.h"
+
+void logTest() {
+  sTrace("--- sync log test: trace");
+  sDebug("--- sync log test: debug");
+  sInfo("--- sync log test: info");
+  sWarn("--- sync log test: warn");
+  sError("--- sync log test: error");
+  sFatal("--- sync log test: fatal");
+}
+
+SyncPreSnapshotReply *createMsg() {
+  SyncPreSnapshotReply *pMsg = syncPreSnapshotReplyBuild(789);
+  pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234);
+  pMsg->srcId.vgId = 100;
+  pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678);
+  pMsg->destId.vgId = 100;
+  pMsg->term = 9527;
+  pMsg->snapStart = 12306;
+  return pMsg;
+}
+
+void test1() {
+  SyncPreSnapshotReply *pMsg = createMsg();
+  syncPreSnapshotReplyLog2((char *)"test1:", pMsg);
+  syncPreSnapshotReplyDestroy(pMsg);
+}
+
+void test2() {
+  SyncPreSnapshotReply *pMsg = createMsg();
+  uint32_t       len = pMsg->bytes;
+  char *         serialized = (char *)taosMemoryMalloc(len);
+  syncPreSnapshotReplySerialize(pMsg, serialized, len);
+  SyncPreSnapshotReply *pMsg2 = syncPreSnapshotReplyBuild(789);
+  syncPreSnapshotReplyDeserialize(serialized, len, pMsg2);
+  syncPreSnapshotReplyLog2((char *)"test2: syncPreSnapshotReplySerialize -> syncPreSnapshotReplyDeserialize ", pMsg2);
+
+  taosMemoryFree(serialized);
+  syncPreSnapshotReplyDestroy(pMsg);
+  syncPreSnapshotReplyDestroy(pMsg2);
+}
+
+void test3() {
+  SyncPreSnapshotReply *pMsg = createMsg();
+  uint32_t       len;
+  char *         serialized = syncPreSnapshotReplySerialize2(pMsg, &len);
+  SyncPreSnapshotReply *pMsg2 = syncPreSnapshotReplyDeserialize2(serialized, len);
+  syncPreSnapshotReplyLog2((char *)"test3: syncPreSnapshotReplySerialize2 -> syncPreSnapshotReplyDeserialize2 ", pMsg2);
+
+  taosMemoryFree(serialized);
+  syncPreSnapshotReplyDestroy(pMsg);
+  syncPreSnapshotReplyDestroy(pMsg2);
+}
+
+void test4() {
+  SyncPreSnapshotReply *pMsg = createMsg();
+  SRpcMsg        rpcMsg;
+  syncPreSnapshotReply2RpcMsg(pMsg, &rpcMsg);
+  SyncPreSnapshotReply *pMsg2 = (SyncPreSnapshotReply *)taosMemoryMalloc(rpcMsg.contLen);
+  syncPreSnapshotReplyFromRpcMsg(&rpcMsg, pMsg2);
+  syncPreSnapshotReplyLog2((char *)"test4: syncPreSnapshotReply2RpcMsg -> syncPreSnapshotReplyFromRpcMsg ", pMsg2);
+
+  rpcFreeCont(rpcMsg.pCont);
+  syncPreSnapshotReplyDestroy(pMsg);
+  syncPreSnapshotReplyDestroy(pMsg2);
+}
+
+void test5() {
+  SyncPreSnapshotReply *pMsg = createMsg();
+  SRpcMsg        rpcMsg;
+  syncPreSnapshotReply2RpcMsg(pMsg, &rpcMsg);
+  SyncPreSnapshotReply *pMsg2 = syncPreSnapshotReplyFromRpcMsg2(&rpcMsg);
+  syncPreSnapshotReplyLog2((char *)"test5: syncPreSnapshotReply2RpcMsg -> syncPreSnapshotReplyFromRpcMsg2 ", pMsg2);
+
+  rpcFreeCont(rpcMsg.pCont);
+  syncPreSnapshotReplyDestroy(pMsg);
+  syncPreSnapshotReplyDestroy(pMsg2);
+}
+
+int main() {
+  tsAsyncLog = 0;
+  sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE;
+  gRaftDetailLog = true;
+  logTest();
+
+  test1();
+  test2();
+  test3();
+  test4();
+  test5();
+
+  return 0;
+}

source/libs/sync/test/syncPreSnapshotTest.cpp

@@ -0,0 +1,97 @@
+#include <gtest/gtest.h>
+#include <stdio.h>
+#include "syncIO.h"
+#include "syncInt.h"
+#include "syncMessage.h"
+#include "syncUtil.h"
+
+void logTest() {
+  sTrace("--- sync log test: trace");
+  sDebug("--- sync log test: debug");
+  sInfo("--- sync log test: info");
+  sWarn("--- sync log test: warn");
+  sError("--- sync log test: error");
+  sFatal("--- sync log test: fatal");
+}
+
+SyncPreSnapshot *createMsg() {
+  SyncPreSnapshot *pMsg = syncPreSnapshotBuild(789);
+  pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234);
+  pMsg->srcId.vgId = 100;
+  pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678);
+  pMsg->destId.vgId = 100;
+  pMsg->term = 9527;
+  return pMsg;
+}
+
+void test1() {
+  SyncPreSnapshot *pMsg = createMsg();
+  syncPreSnapshotLog2((char *)"test1:", pMsg);
+  syncPreSnapshotDestroy(pMsg);
+}
+
+void test2() {
+  SyncPreSnapshot *pMsg = createMsg();
+  uint32_t       len = pMsg->bytes;
+  char *         serialized = (char *)taosMemoryMalloc(len);
+  syncPreSnapshotSerialize(pMsg, serialized, len);
+  SyncPreSnapshot *pMsg2 = syncPreSnapshotBuild(789);
+  syncPreSnapshotDeserialize(serialized, len, pMsg2);
+  syncPreSnapshotLog2((char *)"test2: syncPreSnapshotSerialize -> syncPreSnapshotDeserialize ", pMsg2);
+
+  taosMemoryFree(serialized);
+  syncPreSnapshotDestroy(pMsg);
+  syncPreSnapshotDestroy(pMsg2);
+}
+
+void test3() {
+  SyncPreSnapshot *pMsg = createMsg();
+  uint32_t       len;
+  char *         serialized = syncPreSnapshotSerialize2(pMsg, &len);
+  SyncPreSnapshot *pMsg2 = syncPreSnapshotDeserialize2(serialized, len);
+  syncPreSnapshotLog2((char *)"test3: syncPreSnapshotSerialize2 -> syncPreSnapshotDeserialize2 ", pMsg2);
+
+  taosMemoryFree(serialized);
+  syncPreSnapshotDestroy(pMsg);
+  syncPreSnapshotDestroy(pMsg2);
+}
+
+void test4() {
+  SyncPreSnapshot *pMsg = createMsg();
+  SRpcMsg        rpcMsg;
+  syncPreSnapshot2RpcMsg(pMsg, &rpcMsg);
+  SyncPreSnapshot *pMsg2 = (SyncPreSnapshot *)taosMemoryMalloc(rpcMsg.contLen);
+  syncPreSnapshotFromRpcMsg(&rpcMsg, pMsg2);
+  syncPreSnapshotLog2((char *)"test4: syncPreSnapshot2RpcMsg -> syncPreSnapshotFromRpcMsg ", pMsg2);
+
+  rpcFreeCont(rpcMsg.pCont);
+  syncPreSnapshotDestroy(pMsg);
+  syncPreSnapshotDestroy(pMsg2);
+}
+
+void test5() {
+  SyncPreSnapshot *pMsg = createMsg();
+  SRpcMsg        rpcMsg;
+  syncPreSnapshot2RpcMsg(pMsg, &rpcMsg);
+  SyncPreSnapshot *pMsg2 = syncPreSnapshotFromRpcMsg2(&rpcMsg);
+  syncPreSnapshotLog2((char *)"test5: syncPreSnapshot2RpcMsg -> syncPreSnapshotFromRpcMsg2 ", pMsg2);
+
+  rpcFreeCont(rpcMsg.pCont);
+  syncPreSnapshotDestroy(pMsg);
+  syncPreSnapshotDestroy(pMsg2);
+}
+
+int main() {
+  tsAsyncLog = 0;
+  sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE;
+  gRaftDetailLog = true;
+  logTest();
+
+  test1();
+  test2();
+  test3();
+  test4();
+  test5();
+
+  return 0;
+}

source/libs/sync/test/syncRaftLogTest2.cpp

@@ -25,7 +25,7 @@ const char*    pWalPath = "./syncLogStoreTest_wal";
 SyncIndex gSnapshotLastApplyIndex;
 SyncIndex gSnapshotLastApplyTerm;
 
-int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) {
+int32_t GetSnapshotCb(const struct SSyncFSM* pFsm, SSnapshot* pSnapshot) {
   pSnapshot->data = NULL;
   pSnapshot->lastApplyIndex = gSnapshotLastApplyIndex;
   pSnapshot->lastApplyTerm = gSnapshotLastApplyTerm;

source/libs/sync/test/syncRaftLogTest3.cpp

@@ -27,7 +27,7 @@ const char*    pWalPath = "./syncLogStoreTest_wal";
 SyncIndex gSnapshotLastApplyIndex;
 SyncIndex gSnapshotLastApplyTerm;
 
-int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) {
+int32_t GetSnapshotCb(const struct SSyncFSM* pFsm, SSnapshot* pSnapshot) {
   pSnapshot->data = NULL;
   pSnapshot->lastApplyIndex = gSnapshotLastApplyIndex;
   pSnapshot->lastApplyTerm = gSnapshotLastApplyTerm;

source/libs/sync/test/syncReplicateTest.cpp

@@ -74,10 +74,14 @@ int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) {
 
 SSyncFSM* createFsm() {
   SSyncFSM* pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM));
+
+#if 0 
   pFsm->FpCommitCb = CommitCb;
   pFsm->FpPreCommitCb = PreCommitCb;
   pFsm->FpRollBackCb = RollBackCb;
   pFsm->FpGetSnapshotInfo = GetSnapshotCb;
+#endif
+
   return pFsm;
 }
 

source/libs/sync/test/syncSnapshotReceiverTest.cpp

@@ -37,9 +37,12 @@ SSyncSnapshotReceiver* createReceiver() {
   SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(*pSyncNode));
   pSyncNode->pRaftStore = (SRaftStore*)taosMemoryMalloc(sizeof(*(pSyncNode->pRaftStore)));
   pSyncNode->pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(*(pSyncNode->pFsm)));
+
+#if 0 
   pSyncNode->pFsm->FpSnapshotStartWrite = SnapshotStartWrite;
   pSyncNode->pFsm->FpSnapshotStopWrite = SnapshotStopWrite;
   pSyncNode->pFsm->FpSnapshotDoWrite = SnapshotDoWrite;
+#endif
 
   SRaftId id;
   id.addr = syncUtilAddr2U64("1.2.3.4", 99);
@@ -50,7 +53,6 @@ SSyncSnapshotReceiver* createReceiver() {
   pReceiver->ack = 20;
   pReceiver->pWriter = (void*)0x11;
   pReceiver->term = 66;
-  pReceiver->privateTerm = 99;
 
   return pReceiver;
 }

source/libs/sync/test/syncSnapshotRspTest.cpp

@@ -21,7 +21,7 @@ SyncSnapshotRsp *createMsg() {
   pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678);
   pMsg->destId.vgId = 100;
   pMsg->term = 11;
-  pMsg->privateTerm = 99;
+  pMsg->startTime = 99;
   pMsg->lastIndex = 22;
   pMsg->lastTerm = 33;
   pMsg->ack = 44;

source/libs/sync/test/syncSnapshotSendTest.cpp

@@ -21,7 +21,6 @@ SyncSnapshotSend *createMsg() {
   pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678);
   pMsg->destId.vgId = 100;
   pMsg->term = 11;
-  pMsg->privateTerm = 99;
   pMsg->lastIndex = 22;
   pMsg->lastTerm = 33;
 

source/libs/sync/test/syncSnapshotSenderTest.cpp

@@ -37,10 +37,13 @@ SSyncSnapshotSender* createSender() {
   SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(*pSyncNode));
   pSyncNode->pRaftStore = (SRaftStore*)taosMemoryMalloc(sizeof(*(pSyncNode->pRaftStore)));
   pSyncNode->pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(*(pSyncNode->pFsm)));
+
+ #if 0 
   pSyncNode->pFsm->FpSnapshotStartRead = SnapshotStartRead;
   pSyncNode->pFsm->FpSnapshotStopRead = SnapshotStopRead;
   pSyncNode->pFsm->FpSnapshotDoRead = SnapshotDoRead;
   pSyncNode->pFsm->FpGetSnapshotInfo = GetSnapshot;
+#endif
 
   SSyncSnapshotSender* pSender = snapshotSenderCreate(pSyncNode, 2);
   pSender->start = true;
@@ -55,7 +58,8 @@ SSyncSnapshotSender* createSender() {
   pSender->snapshot.lastApplyTerm = 88;
   pSender->sendingMS = 77;
   pSender->term = 66;
-  pSender->privateTerm = 99;
+  
+  //pSender->privateTerm = 99;
 
   return pSender;
 }

source/libs/sync/test/syncSnapshotTest.cpp

@@ -33,7 +33,7 @@ SyncIndex  snapshotLastApplyIndex = SYNC_INDEX_INVALID;
 const char *pDir = "./syncSnapshotTest";
 const char *pWalDir = "./syncSnapshotTest_wal";
 
-void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
+void CommitCb(const struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
   SyncIndex beginIndex = SYNC_INDEX_INVALID;
   if (pFsm->FpGetSnapshotInfo != NULL) {
     SSnapshot snapshot;
@@ -52,7 +52,7 @@ void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
   }
 }
 
-void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
+void PreCommitCb(const struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
   char logBuf[256];
   snprintf(logBuf, sizeof(logBuf),
            "==callback== ==PreCommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s \n", pFsm,
@@ -60,7 +60,7 @@ void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta)
   syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
 }
 
-void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
+void RollBackCb(const struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
   char logBuf[256];
   snprintf(logBuf, sizeof(logBuf),
            "==callback== ==RollBackCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s \n", pFsm,
@@ -68,7 +68,7 @@ void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
   syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
 }
 
-int32_t GetSnapshotCb(struct SSyncFSM *pFsm, SSnapshot *pSnapshot) {
+int32_t GetSnapshotCb(const struct SSyncFSM *pFsm, SSnapshot *pSnapshot) {
   pSnapshot->data = NULL;
   pSnapshot->lastApplyIndex = snapshotLastApplyIndex;
   pSnapshot->lastApplyTerm = 100;
@@ -78,10 +78,14 @@ int32_t GetSnapshotCb(struct SSyncFSM *pFsm, SSnapshot *pSnapshot) {
 void initFsm() {
   pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM));
   memset(pFsm, 0, sizeof(*pFsm));
+
+#if 0 
   pFsm->FpCommitCb = CommitCb;
   pFsm->FpPreCommitCb = PreCommitCb;
   pFsm->FpRollBackCb = RollBackCb;
   pFsm->FpGetSnapshotInfo = GetSnapshotCb;
+#endif
+
 }
 
 SSyncNode *syncNodeInit() {

source/libs/sync/test/syncTestTool.cpp

@@ -177,6 +177,7 @@ SSyncFSM* createFsm() {
   SSyncFSM* pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM));
   memset(pFsm, 0, sizeof(*pFsm));
 
+#if 0
   pFsm->FpCommitCb = CommitCb;
   pFsm->FpPreCommitCb = PreCommitCb;
   pFsm->FpRollBackCb = RollBackCb;
@@ -193,6 +194,7 @@ SSyncFSM* createFsm() {
   pFsm->FpSnapshotDoWrite = SnapshotDoWrite;
 
   pFsm->FpLeaderTransferCb = LeaderTransferCb;
+#endif
 
   return pFsm;
 }

source/libs/sync/test/syncWriteTest.cpp

@@ -57,9 +57,13 @@ void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
 
 void initFsm() {
   pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM));
+
+#if 0
   pFsm->FpCommitCb = CommitCb;
   pFsm->FpPreCommitCb = PreCommitCb;
   pFsm->FpRollBackCb = RollBackCb;
+#endif
+
 }
 
 SSyncNode *syncNodeInit() {

source/libs/transport/src/transCli.c

@@ -718,8 +718,10 @@ static void cliSendCb(uv_write_t* req, int status) {
   if (status == 0) {
     tTrace("%s conn %p data already was written out", CONN_GET_INST_LABEL(pConn), pConn);
   } else {
-    tError("%s conn %p failed to write:%s", CONN_GET_INST_LABEL(pConn), pConn, uv_err_name(status));
+    if (!uv_is_closing((uv_handle_t*)&pConn->stream)) {
-    cliHandleExcept(pConn);
+      tError("%s conn %p failed to write:%s", CONN_GET_INST_LABEL(pConn), pConn, uv_err_name(status));
+      cliHandleExcept(pConn);
+    }
     return;
   }
   if (cliHandleNoResp(pConn) == true) {

source/libs/transport/src/transSvr.c

@@ -363,9 +363,11 @@ void uvOnSendCb(uv_write_t* req, int status) {
     }
     transUnrefSrvHandle(conn);
   } else {
-    tError("conn %p failed to write data, %s", conn, uv_err_name(status));
+    if (!uv_is_closing((uv_handle_t*)(conn->pTcp))) {
-    conn->broken = true;
+      tError("conn %p failed to write data, %s", conn, uv_err_name(status));
-    transUnrefSrvHandle(conn);
+      conn->broken = true;
+      transUnrefSrvHandle(conn);
+    }
   }
 }
 static void uvOnPipeWriteCb(uv_write_t* req, int status) {

source/util/src/terror.c

@@ -340,7 +340,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_CACHE_LAST_ROW,        "TSDB no cache last ro
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_NOT_EXIST,          "Table not exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_ALREADY_EXIST,        "Stable already exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_NOT_EXIST,            "Stable not exists")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECREATED,          "Table re-created")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER, "Table schema is old")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR,       "TDB env open error")
 
 // query

tests/develop-test/5-taos-tools/taosbenchmark/auto_create_table_json.py

@@ -25,7 +25,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor())
 

tests/develop-test/5-taos-tools/taosbenchmark/commandline.py

@@ -27,7 +27,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 

tests/develop-test/5-taos-tools/taosbenchmark/custom_col_tag.py

@@ -24,7 +24,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 

tests/develop-test/5-taos-tools/taosbenchmark/default_json.py

@@ -23,7 +23,8 @@ class TDTestCase:
         [TD-11510] taosBenchmark test cases
         """
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 

tests/develop-test/5-taos-tools/taosbenchmark/demo.py

@@ -27,7 +27,8 @@ class TDTestCase:
         """
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 

tests/develop-test/5-taos-tools/taosbenchmark/insert_alltypes_json.py

@@ -24,7 +24,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 

tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py

@@ -24,7 +24,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 

tests/develop-test/5-taos-tools/taosbenchmark/json_tag.py

@@ -24,7 +24,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 

tests/develop-test/5-taos-tools/taosbenchmark/limit_offset_json.py

@@ -24,7 +24,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 

tests/develop-test/5-taos-tools/taosbenchmark/query_json.py

@@ -29,7 +29,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 

tests/develop-test/5-taos-tools/taosbenchmark/sample_csv_json.py

@@ -24,7 +24,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 

tests/develop-test/5-taos-tools/taosbenchmark/sml_interlace.py

@@ -24,7 +24,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 

tests/develop-test/5-taos-tools/taosbenchmark/sml_json_alltypes.py

@@ -24,7 +24,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 

tests/develop-test/5-taos-tools/taosbenchmark/sml_telnet_alltypes.py

@@ -24,7 +24,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 

tests/develop-test/5-taos-tools/taosbenchmark/taosadapter_json.py

@@ -25,7 +25,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 

tests/develop-test/5-taos-tools/taosbenchmark/telnet_tcp.py

@@ -25,7 +25,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 

tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py

@@ -27,7 +27,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
         self.tmpdir = "tmp"
@@ -52,7 +53,7 @@ class TDTestCase:
         return paths[0]
 
     def run(self):
-        tdSql.prepare()
+        tdSql.prepare(replica=f"{self.replicaVar}")
 
         tdSql.execute("drop database if exists db")
         tdSql.execute("create database db  days 11 keep 3649 blocks 8 ")

tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBigInt.py

@@ -27,7 +27,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
         self.tmpdir = "tmp"
@@ -50,7 +51,7 @@ class TDTestCase:
         return buildPath
 
     def run(self):
-        tdSql.prepare()
+        tdSql.prepare(replica=f"{self.replicaVar}")
 
         tdSql.execute("drop database if exists db")
         tdSql.execute("create database db  days 11 keep 3649 blocks 8 ")

tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBinary.py

@@ -27,7 +27,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
         self.tmpdir = "tmp"
@@ -50,7 +51,7 @@ class TDTestCase:
         return buildPath
 
     def run(self):
-        tdSql.prepare()
+        tdSql.prepare(replica=f"{self.replicaVar}")
 
         tdSql.execute("drop database if exists db")
         tdSql.execute("create database db  days 11 keep 3649 blocks 8 ")

tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBool.py

@@ -27,7 +27,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
         self.tmpdir = "tmp"
@@ -50,7 +51,7 @@ class TDTestCase:
         return buildPath
 
     def run(self):
-        tdSql.prepare()
+        tdSql.prepare(replica=f"{self.replicaVar}")
 
         tdSql.execute("drop database if exists db")
         tdSql.execute("create database db  days 11 keep 3649 blocks 8 ")

tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeDouble.py

@@ -28,7 +28,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
         self.tmpdir = "tmp"
@@ -51,7 +52,7 @@ class TDTestCase:
         return buildPath
 
     def run(self):
-        tdSql.prepare()
+        tdSql.prepare(replica=f"{self.replicaVar}")
 
         tdSql.execute("drop database if exists db")
         tdSql.execute("create database db  days 11 keep 3649 blocks 8 ")

tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeFloat.py

@@ -28,7 +28,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
         self.tmpdir = "tmp"
@@ -51,7 +52,7 @@ class TDTestCase:
         return buildPath
 
     def run(self):
-        tdSql.prepare()
+        tdSql.prepare(replica=f"{self.replicaVar}")
 
         tdSql.execute("drop database if exists db")
         tdSql.execute("create database db  days 11 keep 3649 blocks 8 ")

tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeInt.py

@@ -27,7 +27,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
         self.tmpdir = "tmp"
@@ -50,7 +51,7 @@ class TDTestCase:
         return buildPath
 
     def run(self):
-        tdSql.prepare()
+        tdSql.prepare(replica=f"{self.replicaVar}")
 
         tdSql.execute("drop database if exists db")
         tdSql.execute("create database db  days 11 keep 3649 blocks 8 ")

tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeJson.py

@@ -27,7 +27,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
         self.tmpdir = "tmp"
@@ -50,7 +51,7 @@ class TDTestCase:
         return buildPath
 
     def run(self):
-        tdSql.prepare()
+        tdSql.prepare(replica=f"{self.replicaVar}")
 
         tdSql.execute("drop database if exists db")
         tdSql.execute("create database db  days 11 keep 3649 blocks 8 ")

tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeSmallInt.py

@@ -27,7 +27,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
         self.tmpdir = "tmp"
@@ -49,7 +50,7 @@ class TDTestCase:
         return buildPath
 
     def run(self):
-        tdSql.prepare()
+        tdSql.prepare(replica=f"{self.replicaVar}")
 
         tdSql.execute("drop database if exists db")
         tdSql.execute("create database db  days 11 keep 3649 blocks 8 ")

tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeTinyInt.py

@@ -27,7 +27,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
         self.tmpdir = "tmp"
@@ -49,7 +50,7 @@ class TDTestCase:
         return buildPath
 
     def run(self):
-        tdSql.prepare()
+        tdSql.prepare(replica=f"{self.replicaVar}")
 
         tdSql.execute("drop database if exists db")
         tdSql.execute("create database db  days 11 keep 3649 blocks 8 ")

tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedBigInt.py

@@ -27,7 +27,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
         self.tmpdir = "tmp"
@@ -50,7 +51,7 @@ class TDTestCase:
         return buildPath
 
     def run(self):
-        tdSql.prepare()
+        tdSql.prepare(replica=f"{self.replicaVar}")
 
         tdSql.execute("drop database if exists db")
         tdSql.execute("create database db  days 11 keep 3649 blocks 8 ")

tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedInt.py

@@ -27,7 +27,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
         self.tmpdir = "tmp"
@@ -50,7 +51,7 @@ class TDTestCase:
         return buildPath
 
     def run(self):
-        tdSql.prepare()
+        tdSql.prepare(replica=f"{self.replicaVar}")
 
         tdSql.execute("drop database if exists db")
         tdSql.execute("create database db  days 11 keep 3649 blocks 8 ")

tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedSmallInt.py

@@ -27,7 +27,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
         self.tmpdir = "tmp"
@@ -50,7 +51,7 @@ class TDTestCase:
         return buildPath
 
     def run(self):
-        tdSql.prepare()
+        tdSql.prepare(replica=f"{self.replicaVar}")
 
         tdSql.execute("drop database if exists db")
         tdSql.execute("create database db  days 11 keep 3649 blocks 8 ")

tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedTinyInt.py

@@ -27,7 +27,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
         self.tmpdir = "tmp"
@@ -50,7 +51,7 @@ class TDTestCase:
         return buildPath
 
     def run(self):
-        tdSql.prepare()
+        tdSql.prepare(replica=f"{self.replicaVar}")
 
         tdSql.execute("drop database if exists db")
         tdSql.execute("create database db  days 11 keep 3649 blocks 8 ")

tests/system-test/0-others/cachemodel.py

@@ -15,6 +15,7 @@ class TDTestCase:
     "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143,
     "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143}
     def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug(f"start to excute {__file__}")
         tdSql.init(conn.cursor(), True)
 

tests/system-test/0-others/compatibility.py

@@ -23,6 +23,7 @@ class TDTestCase:
         return
 
     def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug(f"start to excute {__file__}")
         tdSql.init(conn.cursor())
 

tests/system-test/0-others/fsync.py

@@ -27,6 +27,7 @@ TS_TYPE_COL = [ TS_COL, ]
 class TDTestCase:
 
     def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug(f"start to excute {__file__}")
         tdSql.init(conn.cursor())
 

tests/system-test/0-others/show.py

@@ -21,6 +21,7 @@ from util.sqlset import *
 
 class TDTestCase:
     def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor())
         self.setsql = TDSetSql()

tests/system-test/0-others/sysinfo.py

@@ -21,6 +21,7 @@ from util.common import *
 
 class TDTestCase:
     def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor())
         self.dbname = 'db'

tests/system-test/0-others/taosShell.py

@@ -109,6 +109,7 @@ class TDTestCase:
     print ("===================: ", updatecfgDict)
 
     def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug(f"start to excute {__file__}")
         tdSql.init(conn.cursor())
 

tests/system-test/0-others/taosShellError.py

@@ -111,6 +111,7 @@ class TDTestCase:
     print ("===================: ", updatecfgDict)
 
     def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug(f"start to excute {__file__}")
         tdSql.init(conn.cursor())
 

tests/system-test/0-others/taosShellNetChk.py

@@ -111,6 +111,7 @@ class TDTestCase:
     print ("===================: ", updatecfgDict)
 
     def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug(f"start to excute {__file__}")
         tdSql.init(conn.cursor())
 

tests/system-test/0-others/taosdMonitor.py

@@ -285,6 +285,7 @@ class TDTestCase:
     print ("===================: ", updatecfgDict)
 
     def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug(f"start to excute {__file__}")
         tdSql.init(conn.cursor())
 

tests/system-test/0-others/taosdShell.py

@@ -45,6 +45,7 @@ class TDTestCase:
     # print ("===================: ", updatecfgDict)
 
     def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug(f"start to excute {__file__}")
         tdSql.init(conn.cursor())