From 93ff8b4c7ee442f40d891844c888c6a2cd481e98 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 1 Feb 2021 09:58:22 +0800
Subject: [PATCH 01/13] fix compile error

---
 src/tsdb/inc/tsdbFile.h | 2 +-
 src/tsdb/src/tsdbFile.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/tsdb/inc/tsdbFile.h b/src/tsdb/inc/tsdbFile.h
index cb5c78ca84..3020ecd216 100644
--- a/src/tsdb/inc/tsdbFile.h
+++ b/src/tsdb/inc/tsdbFile.h
@@ -50,7 +50,7 @@ typedef struct {
 } SMFile;
 
 void  tsdbInitMFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver);
-void  tsdbInitMFileEx(SMFile* pMFile, SMFile* pOMFile);
+void  tsdbInitMFileEx(SMFile* pMFile, const SMFile* pOMFile);
 int   tsdbEncodeSMFile(void** buf, SMFile* pMFile);
 void* tsdbDecodeSMFile(void* buf, SMFile* pMFile);
 int   tsdbEncodeSMFileEx(void** buf, SMFile* pMFile);
diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c
index 9a53bf4577..e0e09b5deb 100644
--- a/src/tsdb/src/tsdbFile.c
+++ b/src/tsdb/src/tsdbFile.c
@@ -42,7 +42,7 @@ void tsdbInitMFile(SMFile *pMFile, SDiskID did, int vid, uint32_t ver) {
   tfsInitFile(TSDB_FILE_F(pMFile), did.level, did.id, fname);
 }
 
-void tsdbInitMFileEx(SMFile *pMFile, SMFile *pOMFile) {
+void tsdbInitMFileEx(SMFile *pMFile, const SMFile *pOMFile) {
   *pMFile = *pOMFile;
   TSDB_FILE_SET_CLOSED(pMFile);
 }

From 7ffafb5f169d03a641e9090ed38e3f7fcba4326c Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 1 Feb 2021 13:59:42 +0800
Subject: [PATCH 02/13] refact

---
 src/tsdb/src/tsdbCommit.c | 58 +++++++++++++++++++++++++++++----------
 src/tsdb/src/tsdbFS.c     |  8 ++++++
 2 files changed, 51 insertions(+), 15 deletions(-)

diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c
index 6c5d3ae015..13c0d568a5 100644
--- a/src/tsdb/src/tsdbCommit.c
+++ b/src/tsdb/src/tsdbCommit.c
@@ -52,7 +52,7 @@ static int  tsdbCommitMeta(STsdbRepo *pRepo);
 static int  tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen);
 static int  tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid);
 static int  tsdbCommitTSData(STsdbRepo *pRepo);
-static int  tsdbStartCommit(STsdbRepo *pRepo);
+static void tsdbStartCommit(STsdbRepo *pRepo);
 static void tsdbEndCommit(STsdbRepo *pRepo, int eno);
 static int  tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid);
 static int  tsdbCreateCommitIters(SCommitH *pCommith);
@@ -84,10 +84,7 @@ static int  tsdbApplyRtn(STsdbRepo *pRepo);
 static int  tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn);
 
 void *tsdbCommitData(STsdbRepo *pRepo) {
-  if (tsdbStartCommit(pRepo) < 0) {
-    tsdbError("vgId:%d failed to commit data while startting to commit since %s", REPO_ID(pRepo), tstrerror(terrno));
-    goto _err;
-  }
+  tsdbStartCommit(pRepo);
 
   // Commit to update meta file
   if (tsdbCommitMeta(pRepo) < 0) {
@@ -138,11 +135,15 @@ static int tsdbCommitMeta(STsdbRepo *pRepo) {
       tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)));
 
       if (tsdbCreateMFile(&mf, true) < 0) {
+        tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno));
         return -1;
       }
+
+      tsdbInfo("vgId:%d meta file %s is created to commit", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf));
     } else {
       tsdbInitMFileEx(&mf, pOMFile);
       if (tsdbOpenMFile(&mf, O_WRONLY) < 0) {
+        tsdbError("vgId:%d failed to open META file since %s", REPO_ID(pRepo), tstrerror(terrno));
         return -1;
       }
     }
@@ -154,12 +155,20 @@ static int tsdbCommitMeta(STsdbRepo *pRepo) {
     if (pAct->act == TSDB_UPDATE_META) {
       pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj));
       if (tsdbUpdateMetaRecord(pfs, &mf, pAct->uid, (void *)(pCont->cont), pCont->len) < 0) {
+        tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
+                  tstrerror(terrno));
         tsdbCloseMFile(&mf);
+        tsdbApplyMFileChange(&mf, pOMFile);
+        // TODO: need to reload metaCache
         return -1;
       }
     } else if (pAct->act == TSDB_DROP_META) {
       if (tsdbDropMetaRecord(pfs, &mf, pAct->uid) < 0) {
+        tsdbError("vgId:%d failed to drop META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
+                  tstrerror(terrno));
         tsdbCloseMFile(&mf);
+        tsdbApplyMFileChange(&mf, pOMFile);
+        // TODO: need to reload metaCache
         return -1;
       }
     } else {
@@ -168,6 +177,9 @@ static int tsdbCommitMeta(STsdbRepo *pRepo) {
   }
 
   if (tsdbUpdateMFileHeader(&mf) < 0) {
+    tsdbError("vgId:%d failed to update META file header since %s, revert it", REPO_ID(pRepo), tstrerror(terrno));
+    tsdbApplyMFileChange(&mf, pOMFile);
+    // TODO: need to reload metaCache
     return -1;
   }
 
@@ -238,7 +250,7 @@ static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void
   tsdbUpdateMFileMagic(pMFile, POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM)));
   SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)&uid, sizeof(uid));
   if (pRecord != NULL) {
-    pMFile->info.tombSize += pRecord->size;
+    pMFile->info.tombSize += (pRecord->size + sizeof(SKVRecord));
   } else {
     pMFile->info.nRecords++;
   }
@@ -253,7 +265,7 @@ static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) {
 
   SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)(&uid), sizeof(uid));
   if (pRecord == NULL) {
-    tsdbError("failed to drop KV store record with key %" PRIu64 " since not find", uid);
+    tsdbError("failed to drop META record with key %" PRIu64 " since not find", uid);
     return -1;
   }
 
@@ -264,11 +276,11 @@ static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) {
   void *pBuf = buf;
   tsdbEncodeKVRecord(&pBuf, &rInfo);
 
-  if (tsdbAppendMFile(pMFile, buf, POINTER_DISTANCE(pBuf, buf), NULL) < 0) {
+  if (tsdbAppendMFile(pMFile, buf, sizeof(SKVRecord), NULL) < 0) {
     return -1;
   }
 
-  pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t *)buf, (uint32_t)POINTER_DISTANCE(pBuf, buf));
+  pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t *)buf, sizeof(SKVRecord));
   pMFile->info.nDels++;
   pMFile->info.nRecords--;
   pMFile->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2);
@@ -300,7 +312,12 @@ static int tsdbCommitTSData(STsdbRepo *pRepo) {
   // Skip expired memory data and expired FSET
   tsdbSeekCommitIter(&commith, commith.rtn.minKey);
   while ((pSet = tsdbFSIterNext(&(commith.fsIter)))) {
-    if (pSet->fid >= commith.rtn.minFid) break;
+    if (pSet->fid >= commith.rtn.minFid) {
+      break;
+    } else {
+      tsdbInfo("vgId:%d FSET %d on level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid,
+               TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet));
+    }
   }
 
   // Loop to commit to each file
@@ -347,7 +364,7 @@ static int tsdbCommitTSData(STsdbRepo *pRepo) {
   return 0;
 }
 
-static int tsdbStartCommit(STsdbRepo *pRepo) {
+static void tsdbStartCommit(STsdbRepo *pRepo) {
   SMemTable *pMem = pRepo->imem;
 
   ASSERT(pMem->numOfRows > 0 || listNEles(pMem->actList) > 0);
@@ -358,7 +375,6 @@ static int tsdbStartCommit(STsdbRepo *pRepo) {
   tsdbStartFSTxn(pRepo, pMem->pointsAdd, pMem->storageAdd);
 
   pRepo->code = TSDB_CODE_SUCCESS;
-  return 0;
 }
 
 static void tsdbEndCommit(STsdbRepo *pRepo, int eno) {
@@ -1371,12 +1387,16 @@ static int tsdbApplyRtn(STsdbRepo *pRepo) {
   STsdbFS *  pfs = REPO_FS(pRepo);
   SDFileSet *pSet;
 
-  // Get retentioni snapshot
+  // Get retention snapshot
   tsdbGetRtnSnap(pRepo, &rtn);
 
   tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD);
   while ((pSet = tsdbFSIterNext(&fsiter))) {
-    if (pSet->fid < rtn.minFid) continue;
+    if (pSet->fid < rtn.minFid) {
+      tsdbInfo("vgId:%d FSET %d at level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid,
+               TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet));
+      continue;
+    }
 
     if (tsdbApplyRtnOnFSet(pRepo, pSet, &rtn) < 0) {
       return -1;
@@ -1390,10 +1410,13 @@ static int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn) {
   SDiskID   did;
   SDFileSet nSet;
   STsdbFS * pfs = REPO_FS(pRepo);
+  int       level;
 
   ASSERT(pSet->fid >= pRtn->minFid);
 
-  tfsAllocDisk(tsdbGetFidLevel(pSet->fid, pRtn), &(did.level), &(did.id));
+  level = tsdbGetFidLevel(pSet->fid, pRtn);
+
+  tfsAllocDisk(level, &(did.level), &(did.id));
   if (did.level == TFS_UNDECIDED_LEVEL) {
     terrno = TSDB_CODE_TDB_NO_AVAIL_DISK;
     return -1;
@@ -1404,12 +1427,17 @@ static int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn) {
     tsdbInitDFileSet(&nSet, did, REPO_ID(pRepo), pSet->fid, FS_TXN_VERSION(pfs));
 
     if (tsdbCopyDFileSet(pSet, &nSet) < 0) {
+      tsdbError("vgId:%d failed to copy FSET %d from level %d to level %d since %s", REPO_ID(pRepo), pSet->fid,
+                TSDB_FSET_LEVEL(pSet), did.level, tstrerror(terrno));
       return -1;
     }
 
     if (tsdbUpdateDFileSet(pfs, &nSet) < 0) {
       return -1;
     }
+
+    tsdbInfo("vgId:%d FSET %d is copied from level %d disk id %d to level %d disk id %d", REPO_ID(pRepo), pSet->fid,
+             TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet) did.level, did.id);
   } else {
     // On a correct level
     if (tsdbUpdateDFileSet(pfs, pSet) < 0) {
diff --git a/src/tsdb/src/tsdbFS.c b/src/tsdb/src/tsdbFS.c
index 9196fecfcd..5be44c79a0 100644
--- a/src/tsdb/src/tsdbFS.c
+++ b/src/tsdb/src/tsdbFS.c
@@ -699,6 +699,8 @@ int tsdbLoadMetaCache(STsdbRepo *pRepo, bool recoverMeta) {
   int64_t   maxBufSize = 0;
   SMFInfo   minfo;
 
+  taosHashEmpty(pfs->metaCache);
+
   // No meta file, just return
   if (pfs->cstatus->pmf == NULL) return 0;
 
@@ -716,6 +718,12 @@ int tsdbLoadMetaCache(STsdbRepo *pRepo, bool recoverMeta) {
   while (true) {
     int64_t tsize = tsdbReadMFile(pMFile, tbuf, sizeof(SKVRecord));
     if (tsize == 0) break;
+
+    if (tsize < 0) {
+      tsdbError("vgId:%d failed to read META file since %s", REPO_ID(pRepo), tstrerror(terrno));
+      return -1;
+    }
+
     if (tsize < sizeof(SKVRecord)) {
       tsdbError("vgId:%d failed to read %" PRIzu " bytes from file %s", REPO_ID(pRepo), sizeof(SKVRecord),
                 TSDB_FILE_FULL_NAME(pMFile));

From 89d9be4d78f6044b7e5fb7a25c81db2a8ec47e7e Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 1 Feb 2021 14:26:46 +0800
Subject: [PATCH 03/13] fix compile issue

---
 src/tsdb/src/tsdbCommit.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c
index 13c0d568a5..c09b492f29 100644
--- a/src/tsdb/src/tsdbCommit.c
+++ b/src/tsdb/src/tsdbCommit.c
@@ -1437,7 +1437,7 @@ static int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn) {
     }
 
     tsdbInfo("vgId:%d FSET %d is copied from level %d disk id %d to level %d disk id %d", REPO_ID(pRepo), pSet->fid,
-             TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet) did.level, did.id);
+             TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet), did.level, did.id);
   } else {
     // On a correct level
     if (tsdbUpdateDFileSet(pfs, pSet) < 0) {

From 291aaff1c69ff0f4ed8a7f828dc0f7dee897d305 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 1 Feb 2021 16:17:10 +0800
Subject: [PATCH 04/13] fix two review bugs

---
 src/tsdb/src/tsdbCommit.c | 45 ++++++++++++++++++++++++++++++++-------
 1 file changed, 37 insertions(+), 8 deletions(-)

diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c
index c09b492f29..35c4812c27 100644
--- a/src/tsdb/src/tsdbCommit.c
+++ b/src/tsdb/src/tsdbCommit.c
@@ -312,11 +312,11 @@ static int tsdbCommitTSData(STsdbRepo *pRepo) {
   // Skip expired memory data and expired FSET
   tsdbSeekCommitIter(&commith, commith.rtn.minKey);
   while ((pSet = tsdbFSIterNext(&(commith.fsIter)))) {
-    if (pSet->fid >= commith.rtn.minFid) {
-      break;
-    } else {
+    if (pSet->fid < commith.rtn.minFid) {
       tsdbInfo("vgId:%d FSET %d on level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid,
                TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet));
+    } else {
+      break;
     }
   }
 
@@ -427,14 +427,18 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) {
     if (pIter->pTable == NULL) continue;
 
     if (tsdbCommitToTable(pCommith, tid) < 0) {
-      // TODO: revert the file change
       tsdbCloseCommitFile(pCommith, true);
+      // revert the file change
+      tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet);
       return -1;
     }
   }
 
   if (tsdbWriteBlockIdx(pCommith) < 0) {
+    tsdbError("vgId:%d failed to write SBlockIdx part to FSET %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
     tsdbCloseCommitFile(pCommith, true);
+    // revert the file change
+    tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet);
     return -1;
   }
 
@@ -688,7 +692,11 @@ static int tsdbCommitToTable(SCommitH *pCommith, int tid) {
 
   TSDB_RUNLOCK_TABLE(pIter->pTable);
 
-  if (tsdbWriteBlockInfo(pCommith) < 0) return -1;
+  if (tsdbWriteBlockInfo(pCommith) < 0) {
+    tsdbError("vgId:%d failed to write SBlockInfo part into file %s since %s", REPO_ID(pRepo),
+              TSDB_FILE_FULL_NAME(TSDB_COMMIT_HEAD_FILE(pCommith)), tstrerror(terrno));
+    return -1;
+  }
 
   return 0;
 }
@@ -940,6 +948,8 @@ static int tsdbWriteBlockIdx(SCommitH *pCommih) {
 
   if (nidx <= 0) {
     // All data are deleted
+    pHeadf->info.offset = 0;
+    pHeadf->info.len = 0;
     return 0;
   }
 
@@ -1241,7 +1251,6 @@ static void tsdbResetCommitFile(SCommitH *pCommith) {
 }
 
 static void tsdbResetCommitTable(SCommitH *pCommith) {
-  tdResetDataCols(pCommith->pDataCols);
   taosArrayClear(pCommith->aSubBlk);
   taosArrayClear(pCommith->aSupBlk);
   pCommith->pTable = NULL;
@@ -1270,6 +1279,9 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
       tsdbCloseAndUnsetFSet(&(pCommith->readh));
       return -1;
     }
+
+    tsdbDebug("vgId:%d FSET %d at level %d disk id %d is opened to read to commit", REPO_ID(pRepo), TSDB_FSET_FID(pSet),
+              TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet));
   } else {
     pCommith->isRFileSet = false;
   }
@@ -1280,6 +1292,8 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
     tsdbInitDFileSet(pWSet, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo)));
 
     if (tsdbCreateDFileSet(pWSet, true) < 0) {
+      tsdbError("vgId:%d failed to create FSET %d at level %d disk id %d since %s", REPO_ID(pRepo),
+                TSDB_FSET_FID(pWSet), TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet), tstrerror(terrno));
       if (pCommith->isRFileSet) {
         tsdbCloseAndUnsetFSet(&(pCommith->readh));
       }
@@ -1288,6 +1302,9 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
 
     pCommith->isDFileSame = false;
     pCommith->isLFileSame = false;
+
+    tsdbDebug("vgId:%d FSET %d at level %d disk id %d is created to commit", REPO_ID(pRepo), TSDB_FSET_FID(pWSet),
+              TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet));
   } else {
     did.level = TSDB_FSET_LEVEL(pSet);
     did.id = TSDB_FSET_ID(pSet);
@@ -1299,6 +1316,9 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
     SDFile *pWHeadf = TSDB_COMMIT_HEAD_FILE(pCommith);
     tsdbInitDFile(pWHeadf, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_HEAD);
     if (tsdbCreateDFile(pWHeadf, true) < 0) {
+      tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWHeadf),
+                tstrerror(terrno));
+
       if (pCommith->isRFileSet) {
         tsdbCloseAndUnsetFSet(&(pCommith->readh));
         return -1;
@@ -1310,7 +1330,10 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
     SDFile *pWDataf = TSDB_COMMIT_DATA_FILE(pCommith);
     tsdbInitDFileEx(pWDataf, pRDataf);
     if (tsdbOpenDFile(pWDataf, O_WRONLY) < 0) {
-      tsdbCloseDFile(pWHeadf);
+      tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWDataf),
+                tstrerror(terrno));
+
+      tsdbCloseDFileSet(pWSet);
       tsdbRemoveDFile(pWHeadf);
       if (pCommith->isRFileSet) {
         tsdbCloseAndUnsetFSet(&(pCommith->readh));
@@ -1327,6 +1350,9 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
       pCommith->isLFileSame = true;
 
       if (tsdbOpenDFile(pWLastf, O_WRONLY) < 0) {
+        tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf),
+                  tstrerror(terrno));
+
         tsdbCloseDFileSet(pWSet);
         tsdbRemoveDFile(pWHeadf);
         if (pCommith->isRFileSet) {
@@ -1339,6 +1365,9 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
       pCommith->isLFileSame = false;
 
       if (tsdbCreateDFile(pWLastf, true) < 0) {
+        tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf),
+                  tstrerror(terrno));
+
         tsdbCloseDFileSet(pWSet);
         tsdbRemoveDFile(pWHeadf);
         if (pCommith->isRFileSet) {
@@ -1374,7 +1403,7 @@ static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *p
     if (pBlock->last) {
       if (pCommith->isLFileSame && mergeRows < pCfg->minRowsPerFileBlock) return true;
     } else {
-      if (mergeRows < pCfg->maxRowsPerFileBlock) return true;
+      if (pCommith->isDFileSame && mergeRows <= pCfg->maxRowsPerFileBlock) return true;
     }
   }
 

From e353e3b9463ebb611b8f5959d997673841d4a0cc Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 1 Feb 2021 16:19:22 +0800
Subject: [PATCH 05/13] fsync for file copy

---
 src/os/src/detail/osFile.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c
index 42f2ff3afe..0b7b5ca487 100644
--- a/src/os/src/detail/osFile.c
+++ b/src/os/src/detail/osFile.c
@@ -142,6 +142,8 @@ int64_t taosCopy(char *from, char *to) {
     if (bytes < sizeof(buffer)) break;
   }
 
+  fsync(fidto);
+
   close(fidfrom);
   close(fidto);
   return size;

From 3b1f833dab0c6d7a2a866ea87d3e2a4e62e50ade Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 1 Feb 2021 16:42:21 +0800
Subject: [PATCH 06/13] fix compile error

---
 src/tsdb/src/tsdbCommit.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c
index 35c4812c27..e6c0693d8c 100644
--- a/src/tsdb/src/tsdbCommit.c
+++ b/src/tsdb/src/tsdbCommit.c
@@ -693,7 +693,7 @@ static int tsdbCommitToTable(SCommitH *pCommith, int tid) {
   TSDB_RUNLOCK_TABLE(pIter->pTable);
 
   if (tsdbWriteBlockInfo(pCommith) < 0) {
-    tsdbError("vgId:%d failed to write SBlockInfo part into file %s since %s", REPO_ID(pRepo),
+    tsdbError("vgId:%d failed to write SBlockInfo part into file %s since %s", TSDB_COMMIT_REPO_ID(pCommith),
               TSDB_FILE_FULL_NAME(TSDB_COMMIT_HEAD_FILE(pCommith)), tstrerror(terrno));
     return -1;
   }

From 9c80a9b4d6df0b88529bda6944d01e9f1ef3d564 Mon Sep 17 00:00:00 2001
From: liuyq-617 <yqliu@taosdata.com>
Date: Mon, 1 Feb 2021 17:16:43 +0800
Subject: [PATCH 07/13] [TD-2883]<test>add jenkins server

---
 Jenkinsfile                      |  91 ++++++++++++++++++++++----
 tests/pytest/pytest_1.sh         | 105 +-----------------------------
 tests/pytest/pytest_3.sh         | 108 +++++++++++++++++++++++++++++++
 tests/script/jenkins/basic_1.txt |  72 +--------------------
 tests/script/jenkins/basic_2.txt |  30 +--------
 tests/script/jenkins/basic_3.txt |  12 ----
 tests/script/jenkins/basic_5.txt |  11 ++++
 tests/script/jenkins/basic_6.txt |  27 ++++++++
 tests/script/jenkins/basic_7.txt |  68 +++++++++++++++++++
 tests/test-all.sh                |  17 ++++-
 10 files changed, 310 insertions(+), 231 deletions(-)
 create mode 100755 tests/pytest/pytest_3.sh
 create mode 100644 tests/script/jenkins/basic_5.txt
 create mode 100644 tests/script/jenkins/basic_6.txt
 create mode 100644 tests/script/jenkins/basic_7.txt

diff --git a/Jenkinsfile b/Jenkinsfile
index 2ed90b2ade..536cbe73a2 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -80,13 +80,14 @@ pipeline {
               changeRequest()
           }
       parallel {
-        stage('python_1') {
+        stage('python_1_s1') {
           agent{label 'p1'}
           steps {
             
             pre_test()
-            timeout(time: 90, unit: 'MINUTES'){
+            timeout(time: 45, unit: 'MINUTES'){
               sh '''
+              date
               cd ${WKC}/tests
               find pytest -name '*'sql|xargs rm -rf
               ./test-all.sh p1
@@ -95,23 +96,35 @@ pipeline {
             
           }
         }
-        stage('python_2') {
+        stage('python_2_s5') {
           agent{label 'p2'}
           steps {
             
             pre_test()
+            timeout(time: 45, unit: 'MINUTES'){
             sh '''
+            date
             cd ${WKC}/tests
             find pytest -name '*'sql|xargs rm -rf
             ./test-all.sh p2
             date'''
-            sh '''
-            cd ${WKC}/tests
-            ./test-all.sh b4fq
-            '''
+            }
           }
         }
-        stage('test_b1') {
+        stage('python_3_s6') {
+          agent{label 'p3'}
+          steps {     
+            timeout(time: 45, unit: 'MINUTES'){       
+              pre_test()
+              sh '''
+              date
+              cd ${WKC}/tests
+              ./test-all.sh p3
+              date'''
+            }
+          }
+        }
+        stage('test_b1_s2') {
           agent{label 'b1'}
           steps {     
             timeout(time: 90, unit: 'MINUTES'){       
@@ -124,7 +137,7 @@ pipeline {
           }
         }
 
-        stage('test_crash_gen') {
+        stage('test_crash_gen_s3') {
           agent{label "b2"}
           steps {
             pre_test()
@@ -140,7 +153,7 @@ pipeline {
                 ./handle_crash_gen_val_log.sh
                 '''
             }
-            timeout(time: 90, unit: 'MINUTES'){
+            timeout(time: 45, unit: 'MINUTES'){
               sh '''
               date
               cd ${WKC}/tests
@@ -151,7 +164,7 @@ pipeline {
           }
         }
 
-        stage('test_valgrind') {
+        stage('test_valgrind_s4') {
           agent{label "b3"}
 
           steps {
@@ -163,7 +176,7 @@ pipeline {
                 ./handle_val_log.sh
                 '''
             }     
-            timeout(time: 90, unit: 'MINUTES'){      
+            timeout(time: 45, unit: 'MINUTES'){      
               sh '''
               date
               cd ${WKC}/tests
@@ -172,8 +185,58 @@ pipeline {
             }
           }
         }
-   
-        
+        stage('test_b4_s7') {
+          agent{label 'b4'}
+          steps {     
+            timeout(time: 45, unit: 'MINUTES'){       
+              pre_test()
+              sh '''
+              date
+              cd ${WKC}/tests
+              ./test-all.sh b4fq
+              date'''
+            }
+          }
+        }
+        stage('test_b5_s8') {
+          agent{label 'b5'}
+          steps {     
+            timeout(time: 45, unit: 'MINUTES'){       
+              pre_test()
+              sh '''
+              date
+              cd ${WKC}/tests
+              ./test-all.sh b5fq
+              date'''
+            }
+          }
+        }
+        stage('test_b6_s9') {
+          agent{label 'b6'}
+          steps {     
+            timeout(time: 45, unit: 'MINUTES'){       
+              pre_test()
+              sh '''
+              date
+              cd ${WKC}/tests
+              ./test-all.sh b6fq
+              date'''
+            }
+          }
+        }
+        stage('test_b7_s10') {
+          agent{label 'b7'}
+          steps {     
+            timeout(time: 45, unit: 'MINUTES'){       
+              pre_test()
+              sh '''
+              date
+              cd ${WKC}/tests
+              ./test-all.sh b7fq
+              date'''
+            }
+          }
+        }        
     }
   }
   }
diff --git a/tests/pytest/pytest_1.sh b/tests/pytest/pytest_1.sh
index ad26c48004..437e105e95 100755
--- a/tests/pytest/pytest_1.sh
+++ b/tests/pytest/pytest_1.sh
@@ -15,7 +15,6 @@ python3 ./test.py -f insert/nchar.py
 #python3 ./test.py -f insert/nchar-boundary.py
 python3 ./test.py -f insert/nchar-unicode.py
 python3 ./test.py -f insert/multi.py
-python3 ./test.py -f insert/randomNullCommit.py
 python3 insert/retentionpolicy.py
 python3 ./test.py -f insert/alterTableAndInsert.py
 python3 ./test.py -f insert/insertIntoTwoTables.py
@@ -135,106 +134,4 @@ python3 ./test.py -f import_merge/importTORestart.py
 python3 ./test.py -f import_merge/importTPORestart.py
 python3 ./test.py -f import_merge/importTRestart.py
 python3 ./test.py -f import_merge/importInsertThenImport.py
-python3 ./test.py -f import_merge/importCSV.py
-# user
-python3 ./test.py -f user/user_create.py
-python3 ./test.py -f user/pass_len.py
-
-# stable
-python3 ./test.py -f stable/query_after_reset.py
-
-#query
-python3 ./test.py -f query/filter.py
-python3 ./test.py -f query/filterCombo.py
-python3 ./test.py -f query/queryNormal.py
-python3 ./test.py -f query/queryError.py
-python3 ./test.py -f query/filterAllIntTypes.py
-python3 ./test.py -f query/filterFloatAndDouble.py
-python3 ./test.py -f query/filterOtherTypes.py
-python3 ./test.py -f query/querySort.py
-python3 ./test.py -f query/queryJoin.py
-python3 ./test.py -f query/select_last_crash.py
-python3 ./test.py -f query/queryNullValueTest.py
-python3 ./test.py -f query/queryInsertValue.py
-python3 ./test.py -f query/queryConnection.py
-python3 ./test.py -f query/queryCountCSVData.py
-python3 ./test.py -f query/natualInterval.py
-python3 ./test.py -f query/bug1471.py
-#python3 ./test.py -f query/dataLossTest.py
-python3 ./test.py -f query/bug1874.py
-python3 ./test.py -f query/bug1875.py
-python3 ./test.py -f query/bug1876.py
-python3 ./test.py -f query/bug2218.py
-python3 ./test.py -f query/bug2117.py
-python3 ./test.py -f query/bug2118.py
-python3 ./test.py -f query/bug2143.py
-python3 ./test.py -f query/sliding.py
-python3 ./test.py -f query/unionAllTest.py
-python3 ./test.py -f query/bug2281.py
-python3 ./test.py -f query/bug2119.py
-python3 ./test.py -f query/isNullTest.py
-python3 ./test.py -f query/queryWithTaosdKilled.py
-python3 ./test.py -f query/floatCompare.py
-
-#stream
-python3 ./test.py -f stream/metric_1.py
-python3 ./test.py -f stream/metric_n.py
-python3 ./test.py -f stream/new.py
-python3 ./test.py -f stream/stream1.py
-python3 ./test.py -f stream/stream2.py
-#python3 ./test.py -f stream/parser.py
-python3 ./test.py -f stream/history.py
-python3 ./test.py -f stream/sys.py
-python3 ./test.py -f stream/table_1.py
-python3 ./test.py -f stream/table_n.py
-
-#alter table
-python3 ./test.py -f alter/alter_table_crash.py
-
-# client
-python3 ./test.py -f client/client.py
-python3 ./test.py -f client/version.py
-python3 ./test.py -f client/alterDatabase.py
-python3 ./test.py -f client/noConnectionErrorTest.py
-
-# Misc
-python3 testCompress.py
-python3 testNoCompress.py
-python3 testMinTablesPerVnode.py
-
-# functions
-python3 ./test.py -f functions/function_avg.py -r 1
-python3 ./test.py -f functions/function_bottom.py -r 1
-python3 ./test.py -f functions/function_count.py -r 1
-python3 ./test.py -f functions/function_diff.py -r 1
-python3 ./test.py -f functions/function_first.py -r 1
-python3 ./test.py -f functions/function_last.py -r 1
-python3 ./test.py -f functions/function_last_row.py -r 1
-python3 ./test.py -f functions/function_leastsquares.py -r 1
-python3 ./test.py -f functions/function_max.py -r 1
-python3 ./test.py -f functions/function_min.py -r 1
-python3 ./test.py -f functions/function_operations.py -r 1 
-python3 ./test.py -f functions/function_percentile.py -r 1
-python3 ./test.py -f functions/function_spread.py -r 1
-python3 ./test.py -f functions/function_stddev.py -r 1
-python3 ./test.py -f functions/function_sum.py -r 1
-python3 ./test.py -f functions/function_top.py -r 1
-python3 ./test.py -f functions/function_twa.py -r 1
-python3 ./test.py -f functions/function_twa_test2.py
-python3 queryCount.py
-python3 ./test.py -f query/queryGroupbyWithInterval.py
-python3 client/twoClients.py
-python3 test.py -f query/queryInterval.py
-python3 test.py -f query/queryFillTest.py
-
-# tools
-python3 test.py -f tools/taosdemoTest.py
-python3 test.py -f tools/taosdumpTest.py
-python3 test.py -f tools/lowaTest.py
-python3 test.py -f tools/taosdemoTest2.py
-
-# subscribe
-python3 test.py -f subscribe/singlemeter.py
-#python3 test.py -f subscribe/stability.py  
-python3 test.py -f subscribe/supertable.py
-
+python3 ./test.py -f import_merge/importCSV.py
\ No newline at end of file
diff --git a/tests/pytest/pytest_3.sh b/tests/pytest/pytest_3.sh
new file mode 100755
index 0000000000..a64c144ffd
--- /dev/null
+++ b/tests/pytest/pytest_3.sh
@@ -0,0 +1,108 @@
+#!/bin/bash
+ulimit -c unlimited
+
+
+python3 ./test.py -f insert/randomNullCommit.py
+
+# user
+python3 ./test.py -f user/user_create.py
+python3 ./test.py -f user/pass_len.py
+
+# stable
+python3 ./test.py -f stable/query_after_reset.py
+
+#query
+python3 ./test.py -f query/filter.py
+python3 ./test.py -f query/filterCombo.py
+python3 ./test.py -f query/queryNormal.py
+python3 ./test.py -f query/queryError.py
+python3 ./test.py -f query/filterAllIntTypes.py
+python3 ./test.py -f query/filterFloatAndDouble.py
+python3 ./test.py -f query/filterOtherTypes.py
+python3 ./test.py -f query/querySort.py
+python3 ./test.py -f query/queryJoin.py
+python3 ./test.py -f query/select_last_crash.py
+python3 ./test.py -f query/queryNullValueTest.py
+python3 ./test.py -f query/queryInsertValue.py
+python3 ./test.py -f query/queryConnection.py
+python3 ./test.py -f query/queryCountCSVData.py
+python3 ./test.py -f query/natualInterval.py
+python3 ./test.py -f query/bug1471.py
+#python3 ./test.py -f query/dataLossTest.py
+python3 ./test.py -f query/bug1874.py
+python3 ./test.py -f query/bug1875.py
+python3 ./test.py -f query/bug1876.py
+python3 ./test.py -f query/bug2218.py
+python3 ./test.py -f query/bug2117.py
+python3 ./test.py -f query/bug2118.py
+python3 ./test.py -f query/bug2143.py
+python3 ./test.py -f query/sliding.py
+python3 ./test.py -f query/unionAllTest.py
+python3 ./test.py -f query/bug2281.py
+python3 ./test.py -f query/bug2119.py
+python3 ./test.py -f query/isNullTest.py
+python3 ./test.py -f query/queryWithTaosdKilled.py
+python3 ./test.py -f query/floatCompare.py
+
+#stream
+python3 ./test.py -f stream/metric_1.py
+python3 ./test.py -f stream/metric_n.py
+python3 ./test.py -f stream/new.py
+python3 ./test.py -f stream/stream1.py
+python3 ./test.py -f stream/stream2.py
+#python3 ./test.py -f stream/parser.py
+python3 ./test.py -f stream/history.py
+python3 ./test.py -f stream/sys.py
+python3 ./test.py -f stream/table_1.py
+python3 ./test.py -f stream/table_n.py
+
+#alter table
+python3 ./test.py -f alter/alter_table_crash.py
+
+# client
+python3 ./test.py -f client/client.py
+python3 ./test.py -f client/version.py
+python3 ./test.py -f client/alterDatabase.py
+python3 ./test.py -f client/noConnectionErrorTest.py
+
+# Misc
+python3 testCompress.py
+python3 testNoCompress.py
+python3 testMinTablesPerVnode.py
+
+# functions
+python3 ./test.py -f functions/function_avg.py -r 1
+python3 ./test.py -f functions/function_bottom.py -r 1
+python3 ./test.py -f functions/function_count.py -r 1
+python3 ./test.py -f functions/function_diff.py -r 1
+python3 ./test.py -f functions/function_first.py -r 1
+python3 ./test.py -f functions/function_last.py -r 1
+python3 ./test.py -f functions/function_last_row.py -r 1
+python3 ./test.py -f functions/function_leastsquares.py -r 1
+python3 ./test.py -f functions/function_max.py -r 1
+python3 ./test.py -f functions/function_min.py -r 1
+python3 ./test.py -f functions/function_operations.py -r 1 
+python3 ./test.py -f functions/function_percentile.py -r 1
+python3 ./test.py -f functions/function_spread.py -r 1
+python3 ./test.py -f functions/function_stddev.py -r 1
+python3 ./test.py -f functions/function_sum.py -r 1
+python3 ./test.py -f functions/function_top.py -r 1
+python3 ./test.py -f functions/function_twa.py -r 1
+python3 ./test.py -f functions/function_twa_test2.py
+python3 queryCount.py
+python3 ./test.py -f query/queryGroupbyWithInterval.py
+python3 client/twoClients.py
+python3 test.py -f query/queryInterval.py
+python3 test.py -f query/queryFillTest.py
+
+# tools
+python3 test.py -f tools/taosdemoTest.py
+python3 test.py -f tools/taosdumpTest.py
+python3 test.py -f tools/lowaTest.py
+python3 test.py -f tools/taosdemoTest2.py
+
+# subscribe
+python3 test.py -f subscribe/singlemeter.py
+#python3 test.py -f subscribe/stability.py  
+python3 test.py -f subscribe/supertable.py
+
diff --git a/tests/script/jenkins/basic_1.txt b/tests/script/jenkins/basic_1.txt
index 7f7ef18368..305971c15f 100644
--- a/tests/script/jenkins/basic_1.txt
+++ b/tests/script/jenkins/basic_1.txt
@@ -1,44 +1,3 @@
-
-./test.sh -f general/compute/avg.sim
-./test.sh -f general/compute/bottom.sim
-./test.sh -f general/compute/count.sim
-./test.sh -f general/compute/diff.sim
-./test.sh -f general/compute/diff2.sim
-./test.sh -f general/compute/first.sim
-./test.sh -f general/compute/interval.sim
-./test.sh -f general/compute/last.sim
-./test.sh -f general/compute/leastsquare.sim
-./test.sh -f general/compute/max.sim
-./test.sh -f general/compute/min.sim
-./test.sh -f general/compute/null.sim
-./test.sh -f general/compute/percentile.sim
-./test.sh -f general/compute/stddev.sim
-./test.sh -f general/compute/sum.sim
-./test.sh -f general/compute/top.sim
-
-./test.sh -f general/db/alter_option.sim
-./test.sh -f general/db/alter_tables_d2.sim
-./test.sh -f general/db/alter_tables_v1.sim
-./test.sh -f general/db/alter_tables_v4.sim
-./test.sh -f general/db/alter_vgroups.sim
-./test.sh -f general/db/basic.sim
-./test.sh -f general/db/basic1.sim
-./test.sh -f general/db/basic2.sim
-./test.sh -f general/db/basic3.sim
-./test.sh -f general/db/basic4.sim
-./test.sh -f general/db/basic5.sim
-./test.sh -f general/db/delete_reuse1.sim
-./test.sh -f general/db/delete_reuse2.sim
-./test.sh -f general/db/delete_reusevnode.sim
-./test.sh -f general/db/delete_reusevnode2.sim
-./test.sh -f general/db/delete_writing1.sim
-./test.sh -f general/db/delete_writing2.sim
-./test.sh -f general/db/delete.sim
-./test.sh -f general/db/len.sim
-./test.sh -f general/db/repeat.sim
-./test.sh -f general/db/tables.sim
-./test.sh -f general/db/vnodes.sim
-
 ./test.sh -f general/field/2.sim
 ./test.sh -f general/field/3.sim
 ./test.sh -f general/field/4.sim
@@ -128,33 +87,4 @@
 ./test.sh -f general/parser/union.sim
 ./test.sh -f general/parser/topbot.sim
 ./test.sh -f general/db/nosuchfile.sim
-./test.sh -f general/parser/function.sim
-
-./test.sh -f general/table/autocreate.sim
-./test.sh -f general/table/basic1.sim
-./test.sh -f general/table/basic2.sim
-./test.sh -f general/table/basic3.sim
-./test.sh -f general/table/bigint.sim
-./test.sh -f general/table/binary.sim
-./test.sh -f general/table/bool.sim
-./test.sh -f general/table/column_name.sim
-./test.sh -f general/table/column_num.sim
-./test.sh -f general/table/column_value.sim
-./test.sh -f general/table/column2.sim
-./test.sh -f general/table/date.sim
-./test.sh -f general/table/db.table.sim
-./test.sh -f general/table/delete_reuse1.sim
-./test.sh -f general/table/delete_reuse2.sim
-./test.sh -f general/table/delete_writing.sim
-./test.sh -f general/table/describe.sim
-./test.sh -f general/table/double.sim
-./test.sh -f general/table/fill.sim
-./test.sh -f general/table/float.sim
-./test.sh -f general/table/int.sim
-./test.sh -f general/table/limit.sim
-./test.sh -f general/table/smallint.sim
-./test.sh -f general/table/table_len.sim
-./test.sh -f general/table/table.sim
-./test.sh -f general/table/tinyint.sim
-./test.sh -f general/table/vgroup.sim
-./test.sh -f general/table/createmulti.sim
\ No newline at end of file
+./test.sh -f general/parser/function.sim
\ No newline at end of file
diff --git a/tests/script/jenkins/basic_2.txt b/tests/script/jenkins/basic_2.txt
index 9b3b3f9b18..20e574711a 100644
--- a/tests/script/jenkins/basic_2.txt
+++ b/tests/script/jenkins/basic_2.txt
@@ -72,32 +72,4 @@ cd ../../../debug; make
 ./test.sh -f unique/cluster/cache.sim
 ./test.sh -f unique/cluster/vgroup100.sim
 
-./test.sh -f unique/column/replica3.sim
-
-./test.sh -f unique/db/commit.sim
-./test.sh -f unique/db/delete.sim
-./test.sh -f unique/db/delete_part.sim
-./test.sh -f unique/db/replica_add12.sim
-./test.sh -f unique/db/replica_add13.sim
-./test.sh -f unique/db/replica_add23.sim
-./test.sh -f unique/db/replica_reduce21.sim
-./test.sh -f unique/db/replica_reduce32.sim
-./test.sh -f unique/db/replica_reduce31.sim
-./test.sh -f unique/db/replica_part.sim
-
-./test.sh -f unique/vnode/many.sim
-./test.sh -f unique/vnode/replica2_basic2.sim
-./test.sh -f unique/vnode/replica2_repeat.sim
-./test.sh -f unique/vnode/replica3_basic.sim
-./test.sh -f unique/vnode/replica3_repeat.sim
-./test.sh -f unique/vnode/replica3_vgroup.sim
-
-./test.sh -f unique/dnode/monitor.sim
-./test.sh -f unique/dnode/monitor_bug.sim
-./test.sh -f unique/dnode/simple.sim
-./test.sh -f unique/dnode/data1.sim
-./test.sh -f unique/dnode/m2.sim
-./test.sh -f unique/dnode/m3.sim
-./test.sh -f unique/dnode/offline3.sim
-./test.sh -f general/wal/kill.sim
-./test.sh -f general/wal/maxtables.sim
\ No newline at end of file
+./test.sh -f unique/column/replica3.sim
\ No newline at end of file
diff --git a/tests/script/jenkins/basic_3.txt b/tests/script/jenkins/basic_3.txt
index 25bfde28f0..0215f0d6cf 100644
--- a/tests/script/jenkins/basic_3.txt
+++ b/tests/script/jenkins/basic_3.txt
@@ -65,15 +65,3 @@
 ./test.sh -f unique/mnode/mgmt33.sim
 ./test.sh -f unique/mnode/mgmt34.sim
 ./test.sh -f unique/mnode/mgmtr2.sim
-
-./test.sh -f general/stream/metrics_del.sim
-./test.sh -f general/stream/metrics_replica1_vnoden.sim
-./test.sh -f general/stream/restart_stream.sim
-./test.sh -f general/stream/stream_3.sim
-./test.sh -f general/stream/stream_restart.sim
-./test.sh -f general/stream/table_del.sim
-./test.sh -f general/stream/table_replica1_vnoden.sim
-
-./test.sh -f general/connection/test_old_data.sim
-./test.sh -f unique/dnode/datatrans_3node.sim 
-./test.sh -f unique/dnode/datatrans_3node_2.sim
\ No newline at end of file
diff --git a/tests/script/jenkins/basic_5.txt b/tests/script/jenkins/basic_5.txt
new file mode 100644
index 0000000000..66c2ce36b2
--- /dev/null
+++ b/tests/script/jenkins/basic_5.txt
@@ -0,0 +1,11 @@
+./test.sh -f general/stream/metrics_del.sim
+./test.sh -f general/stream/metrics_replica1_vnoden.sim
+./test.sh -f general/stream/restart_stream.sim
+./test.sh -f general/stream/stream_3.sim
+./test.sh -f general/stream/stream_restart.sim
+./test.sh -f general/stream/table_del.sim
+./test.sh -f general/stream/table_replica1_vnoden.sim
+
+./test.sh -f general/connection/test_old_data.sim
+./test.sh -f unique/dnode/datatrans_3node.sim 
+./test.sh -f unique/dnode/datatrans_3node_2.sim
\ No newline at end of file
diff --git a/tests/script/jenkins/basic_6.txt b/tests/script/jenkins/basic_6.txt
new file mode 100644
index 0000000000..49523353ad
--- /dev/null
+++ b/tests/script/jenkins/basic_6.txt
@@ -0,0 +1,27 @@
+./test.sh -f unique/db/commit.sim
+./test.sh -f unique/db/delete.sim
+./test.sh -f unique/db/delete_part.sim
+./test.sh -f unique/db/replica_add12.sim
+./test.sh -f unique/db/replica_add13.sim
+./test.sh -f unique/db/replica_add23.sim
+./test.sh -f unique/db/replica_reduce21.sim
+./test.sh -f unique/db/replica_reduce32.sim
+./test.sh -f unique/db/replica_reduce31.sim
+./test.sh -f unique/db/replica_part.sim
+
+./test.sh -f unique/vnode/many.sim
+./test.sh -f unique/vnode/replica2_basic2.sim
+./test.sh -f unique/vnode/replica2_repeat.sim
+./test.sh -f unique/vnode/replica3_basic.sim
+./test.sh -f unique/vnode/replica3_repeat.sim
+./test.sh -f unique/vnode/replica3_vgroup.sim
+
+./test.sh -f unique/dnode/monitor.sim
+./test.sh -f unique/dnode/monitor_bug.sim
+./test.sh -f unique/dnode/simple.sim
+./test.sh -f unique/dnode/data1.sim
+./test.sh -f unique/dnode/m2.sim
+./test.sh -f unique/dnode/m3.sim
+./test.sh -f unique/dnode/offline3.sim
+./test.sh -f general/wal/kill.sim
+./test.sh -f general/wal/maxtables.sim
\ No newline at end of file
diff --git a/tests/script/jenkins/basic_7.txt b/tests/script/jenkins/basic_7.txt
new file mode 100644
index 0000000000..160bec81e1
--- /dev/null
+++ b/tests/script/jenkins/basic_7.txt
@@ -0,0 +1,68 @@
+
+./test.sh -f general/compute/avg.sim
+./test.sh -f general/compute/bottom.sim
+./test.sh -f general/compute/count.sim
+./test.sh -f general/compute/diff.sim
+./test.sh -f general/compute/diff2.sim
+./test.sh -f general/compute/first.sim
+./test.sh -f general/compute/interval.sim
+./test.sh -f general/compute/last.sim
+./test.sh -f general/compute/leastsquare.sim
+./test.sh -f general/compute/max.sim
+./test.sh -f general/compute/min.sim
+./test.sh -f general/compute/null.sim
+./test.sh -f general/compute/percentile.sim
+./test.sh -f general/compute/stddev.sim
+./test.sh -f general/compute/sum.sim
+./test.sh -f general/compute/top.sim
+
+./test.sh -f general/db/alter_option.sim
+./test.sh -f general/db/alter_tables_d2.sim
+./test.sh -f general/db/alter_tables_v1.sim
+./test.sh -f general/db/alter_tables_v4.sim
+./test.sh -f general/db/alter_vgroups.sim
+./test.sh -f general/db/basic.sim
+./test.sh -f general/db/basic1.sim
+./test.sh -f general/db/basic2.sim
+./test.sh -f general/db/basic3.sim
+./test.sh -f general/db/basic4.sim
+./test.sh -f general/db/basic5.sim
+./test.sh -f general/db/delete_reuse1.sim
+./test.sh -f general/db/delete_reuse2.sim
+./test.sh -f general/db/delete_reusevnode.sim
+./test.sh -f general/db/delete_reusevnode2.sim
+./test.sh -f general/db/delete_writing1.sim
+./test.sh -f general/db/delete_writing2.sim
+./test.sh -f general/db/delete.sim
+./test.sh -f general/db/len.sim
+./test.sh -f general/db/repeat.sim
+./test.sh -f general/db/tables.sim
+./test.sh -f general/db/vnodes.sim
+./test.sh -f general/table/autocreate.sim
+./test.sh -f general/table/basic1.sim
+./test.sh -f general/table/basic2.sim
+./test.sh -f general/table/basic3.sim
+./test.sh -f general/table/bigint.sim
+./test.sh -f general/table/binary.sim
+./test.sh -f general/table/bool.sim
+./test.sh -f general/table/column_name.sim
+./test.sh -f general/table/column_num.sim
+./test.sh -f general/table/column_value.sim
+./test.sh -f general/table/column2.sim
+./test.sh -f general/table/date.sim
+./test.sh -f general/table/db.table.sim
+./test.sh -f general/table/delete_reuse1.sim
+./test.sh -f general/table/delete_reuse2.sim
+./test.sh -f general/table/delete_writing.sim
+./test.sh -f general/table/describe.sim
+./test.sh -f general/table/double.sim
+./test.sh -f general/table/fill.sim
+./test.sh -f general/table/float.sim
+./test.sh -f general/table/int.sim
+./test.sh -f general/table/limit.sim
+./test.sh -f general/table/smallint.sim
+./test.sh -f general/table/table_len.sim
+./test.sh -f general/table/table.sim
+./test.sh -f general/table/tinyint.sim
+./test.sh -f general/table/vgroup.sim
+./test.sh -f general/table/createmulti.sim
\ No newline at end of file
diff --git a/tests/test-all.sh b/tests/test-all.sh
index bcbf013096..13efbec7a2 100755
--- a/tests/test-all.sh
+++ b/tests/test-all.sh
@@ -155,7 +155,10 @@ if [ "$2" != "python" ]; then
   elif [ "$1" == "b1" ]; then
     echo "### run TSIM b1 test ###"
     runSimCaseOneByOne jenkins/basic_1.txt
-    # runSimCaseOneByOne jenkins/basic_4.txt
+    runSimCaseOneByOne jenkins/basic_4.txt
+    runSimCaseOneByOne jenkins/basic_5.txt
+    runSimCaseOneByOne jenkins/basic_6.txt
+    runSimCaseOneByOne jenkins/basic_7.txt
   elif [ "$1" == "b2" ]; then
     echo "### run TSIM b2 test ###"
     runSimCaseOneByOne jenkins/basic_2.txt
@@ -174,6 +177,15 @@ if [ "$2" != "python" ]; then
   elif [ "$1" == "b4fq" ]; then
     echo "### run TSIM b4 test ###"
     runSimCaseOneByOnefq jenkins/basic_4.txt
+  elif [ "$1" == "b5fq" ]; then
+    echo "### run TSIM b5 test ###"
+    runSimCaseOneByOnefq jenkins/basic_5.txt
+  elif [ "$1" == "b6fq" ]; then
+    echo "### run TSIM b6 test ###"
+    runSimCaseOneByOnefq jenkins/basic_6.txt
+  elif [ "$1" == "b7fq" ]; then
+    echo "### run TSIM b7 test ###"
+    runSimCaseOneByOnefq jenkins/basic_7.txt
   elif [ "$1" == "smoke" ] || [ -z "$1" ]; then
     echo "### run TSIM smoke test ###"
     runSimCaseOneByOne basicSuite.sim
@@ -242,6 +254,9 @@ if [ "$2" != "sim" ]; then
   elif [ "$1" == "p2" ]; then
     echo "### run Python_2 test ###"
     runPyCaseOneByOnefq pytest_2.sh
+  elif [ "$1" == "p3" ]; then
+    echo "### run Python_3 test ###"
+    runPyCaseOneByOnefq pytest_3.sh
   elif [ "$1" == "b2" ] || [ "$1" == "b3" ]; then
     exit $(($totalFailed + $totalPyFailed))
   elif [ "$1" == "smoke" ] || [ -z "$1" ]; then

From 7cc4864f39f9f056a3931397c1a812c9eca93fdb Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Mon, 1 Feb 2021 17:27:00 +0800
Subject: [PATCH 08/13] vnodeCfg/update: fix missing update cfg

---
 src/vnode/src/vnodeCfg.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/vnode/src/vnodeCfg.c b/src/vnode/src/vnodeCfg.c
index 9b7916c8bd..03f2b11eec 100644
--- a/src/vnode/src/vnodeCfg.c
+++ b/src/vnode/src/vnodeCfg.c
@@ -34,6 +34,7 @@ static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) {
   pVnode->tsdbCfg.maxRowsPerFileBlock = vnodeMsg->cfg.maxRowsPerFileBlock;
   pVnode->tsdbCfg.precision = vnodeMsg->cfg.precision;
   pVnode->tsdbCfg.compression = vnodeMsg->cfg.compression;
+  pVnode->tsdbCfg.update = vnodeMsg->cfg.update;
   pVnode->tsdbCfg.cacheLastRow = vnodeMsg->cfg.cacheLastRow;
   pVnode->walCfg.walLevel = vnodeMsg->cfg.walLevel;
   pVnode->walCfg.fsyncPeriod = vnodeMsg->cfg.fsyncPeriod;
@@ -227,6 +228,15 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) {
   }
   vnodeMsg.cfg.quorum = (int8_t)quorum->valueint;
 
+  cJSON *update = cJSON_GetObjectItem(root, "update");
+  if (!update || update->type != cJSON_Number) {
+    vError("vgId: %d, failed to read %s, update not found", pVnode->vgId, file);
+    vnodeMsg.cfg.update = 0;
+    vnodeMsg.cfg.vgCfgVersion = 0;
+  } else {
+    vnodeMsg.cfg.update = (int8_t)update->valueint;
+  }
+
   cJSON *cacheLastRow = cJSON_GetObjectItem(root, "cacheLastRow");
   if (!cacheLastRow || cacheLastRow->type != cJSON_Number) {
     vError("vgId: %d, failed to read %s, cacheLastRow not found", pVnode->vgId, file);
@@ -325,6 +335,7 @@ int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) {
   len += snprintf(content + len, maxLen - len, "  \"dbReplica\": %d,\n", pMsg->cfg.dbReplica);
   len += snprintf(content + len, maxLen - len, "  \"wals\": %d,\n", pMsg->cfg.wals);
   len += snprintf(content + len, maxLen - len, "  \"quorum\": %d,\n", pMsg->cfg.quorum);
+  len += snprintf(content + len, maxLen - len, "  \"update\": %d,\n", pMsg->cfg.update);
   len += snprintf(content + len, maxLen - len, "  \"cacheLastRow\": %d,\n", pMsg->cfg.cacheLastRow);
   len += snprintf(content + len, maxLen - len, "  \"nodeInfos\": [{\n");
   for (int32_t i = 0; i < pMsg->cfg.vgReplica; i++) {

From 9f0e5d2ef00249d19c7c8f6d23d9bbaccfe14aad Mon Sep 17 00:00:00 2001
From: liuyq-617 <yqliu@taosdata.com>
Date: Tue, 2 Feb 2021 09:26:58 +0800
Subject: [PATCH 09/13] rebalance

---
 tests/script/jenkins/basic_3.txt | 13 ++-----------
 tests/script/jenkins/basic_7.txt | 15 ++++++++++++++-
 2 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/tests/script/jenkins/basic_3.txt b/tests/script/jenkins/basic_3.txt
index 0215f0d6cf..f53b1b763a 100644
--- a/tests/script/jenkins/basic_3.txt
+++ b/tests/script/jenkins/basic_3.txt
@@ -19,7 +19,7 @@
 ./test.sh -f unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim
 ./test.sh -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim
 ./test.sh -f unique/arbitrator/dn3_mn2_killDnode.sim
-./test.sh -f unique/arbitrator/insert_duplicationTs.sim
+
 ./test.sh -f unique/arbitrator/offline_replica2_alterTable_online.sim
 ./test.sh -f unique/arbitrator/offline_replica2_alterTag_online.sim
 ./test.sh -f unique/arbitrator/offline_replica2_createTable_online.sim
@@ -55,13 +55,4 @@
 ./test.sh -f unique/stable/replica3_dnode6.sim
 ./test.sh -f unique/stable/replica3_vnode3.sim
 
-./test.sh -f unique/mnode/mgmt20.sim
-./test.sh -f unique/mnode/mgmt21.sim
-./test.sh -f unique/mnode/mgmt22.sim
-./test.sh -f unique/mnode/mgmt23.sim
-./test.sh -f unique/mnode/mgmt24.sim
-./test.sh -f unique/mnode/mgmt25.sim
-./test.sh -f unique/mnode/mgmt26.sim
-./test.sh -f unique/mnode/mgmt33.sim
-./test.sh -f unique/mnode/mgmt34.sim
-./test.sh -f unique/mnode/mgmtr2.sim
+
diff --git a/tests/script/jenkins/basic_7.txt b/tests/script/jenkins/basic_7.txt
index 160bec81e1..27d7d4ff97 100644
--- a/tests/script/jenkins/basic_7.txt
+++ b/tests/script/jenkins/basic_7.txt
@@ -65,4 +65,17 @@
 ./test.sh -f general/table/table.sim
 ./test.sh -f general/table/tinyint.sim
 ./test.sh -f general/table/vgroup.sim
-./test.sh -f general/table/createmulti.sim
\ No newline at end of file
+./test.sh -f general/table/createmulti.sim
+
+./test.sh -f unique/mnode/mgmt20.sim
+./test.sh -f unique/mnode/mgmt21.sim
+./test.sh -f unique/mnode/mgmt22.sim
+./test.sh -f unique/mnode/mgmt23.sim
+./test.sh -f unique/mnode/mgmt24.sim
+./test.sh -f unique/mnode/mgmt25.sim
+./test.sh -f unique/mnode/mgmt26.sim
+./test.sh -f unique/mnode/mgmt33.sim
+./test.sh -f unique/mnode/mgmt34.sim
+./test.sh -f unique/mnode/mgmtr2.sim
+
+./test.sh -f unique/arbitrator/insert_duplicationTs.sim
\ No newline at end of file

From 0d46d2b2ec404c53ee2d7b5efd87a6cb70509e72 Mon Sep 17 00:00:00 2001
From: liuyq-617 <yqliu@taosdata.com>
Date: Tue, 2 Feb 2021 09:34:08 +0800
Subject: [PATCH 10/13] rebalance

---
 tests/script/jenkins/basic_1.txt | 5 +----
 tests/script/jenkins/basic_6.txt | 7 ++++++-
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/tests/script/jenkins/basic_1.txt b/tests/script/jenkins/basic_1.txt
index 305971c15f..1724576734 100644
--- a/tests/script/jenkins/basic_1.txt
+++ b/tests/script/jenkins/basic_1.txt
@@ -22,10 +22,7 @@
 ./test.sh -f general/http/grafana_bug.sim
 ./test.sh -f general/http/grafana.sim
 
-./test.sh -f general/import/basic.sim
-./test.sh -f general/import/commit.sim
-./test.sh -f general/import/large.sim
-./test.sh -f general/import/replica1.sim
+
 
 ./test.sh -f general/insert/basic.sim
 ./test.sh -f general/insert/insert_drop.sim
diff --git a/tests/script/jenkins/basic_6.txt b/tests/script/jenkins/basic_6.txt
index 49523353ad..893346e6ca 100644
--- a/tests/script/jenkins/basic_6.txt
+++ b/tests/script/jenkins/basic_6.txt
@@ -24,4 +24,9 @@
 ./test.sh -f unique/dnode/m3.sim
 ./test.sh -f unique/dnode/offline3.sim
 ./test.sh -f general/wal/kill.sim
-./test.sh -f general/wal/maxtables.sim
\ No newline at end of file
+./test.sh -f general/wal/maxtables.sim
+
+./test.sh -f general/import/basic.sim
+./test.sh -f general/import/commit.sim
+./test.sh -f general/import/large.sim
+./test.sh -f general/import/replica1.sim
\ No newline at end of file

From 4bec4814fdb164391846281e6cbb8fe39c64c3a0 Mon Sep 17 00:00:00 2001
From: liuyq-617 <yqliu@taosdata.com>
Date: Tue, 2 Feb 2021 10:01:59 +0800
Subject: [PATCH 11/13] modify testcase suite

---
 tests/pytest/pytest_2.sh | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tests/pytest/pytest_2.sh b/tests/pytest/pytest_2.sh
index dde9f78953..4ec517a0bf 100755
--- a/tests/pytest/pytest_2.sh
+++ b/tests/pytest/pytest_2.sh
@@ -1,18 +1,18 @@
 
 
 # update
-#python3 ./test.py -f update/allow_update.py
+python3 ./test.py -f update/allow_update.py
 python3 ./test.py -f update/allow_update-0.py
 python3 ./test.py -f update/append_commit_data.py
 python3 ./test.py -f update/append_commit_last-0.py
 python3 ./test.py -f update/append_commit_last.py
-#python3 ./test.py -f update/merge_commit_data.py
-#python3 ./test.py -f update/merge_commit_data-0.py
-#python3 ./test.py -f update/merge_commit_data2.py
-#python3 ./test.py -f update/merge_commit_data2_update0.py
-#python3 ./test.py -f update/merge_commit_last-0.py
-#python3 ./test.py -f update/merge_commit_last.py
-#python3 ./test.py -f update/bug_td2279.py
+python3 ./test.py -f update/merge_commit_data.py
+python3 ./test.py -f update/merge_commit_data-0.py
+python3 ./test.py -f update/merge_commit_data2.py
+python3 ./test.py -f update/merge_commit_data2_update0.py
+python3 ./test.py -f update/merge_commit_last-0.py
+python3 ./test.py -f update/merge_commit_last.py
+python3 ./test.py -f update/bug_td2279.py
 
 # wal
 python3 ./test.py -f wal/addOldWalTest.py

From 3b156b610c1c955363ba5e5dbdd676749f682e9e Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 2 Feb 2021 13:17:07 +0800
Subject: [PATCH 12/13] fix a possible coredump

---
 src/tsdb/src/tsdbCommit.c | 2 ++
 src/tsdb/src/tsdbFS.c     | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c
index 78beb3e293..a777b11186 100644
--- a/src/tsdb/src/tsdbCommit.c
+++ b/src/tsdb/src/tsdbCommit.c
@@ -220,6 +220,8 @@ void tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn) {
   pRtn->minFid = (int)(TSDB_KEY_FID(minKey, pCfg->daysPerFile, pCfg->precision));
   pRtn->midFid = (int)(TSDB_KEY_FID(midKey, pCfg->daysPerFile, pCfg->precision));
   pRtn->maxFid = (int)(TSDB_KEY_FID(maxKey, pCfg->daysPerFile, pCfg->precision));
+  tsdbDebug("vgId:%d now:%" PRId64 " minKey:%" PRId64 " minFid:%d, midFid:%d, maxFid:%d", REPO_ID(pRepo), now, minKey,
+            pRtn->minFid, pRtn->midFid, pRtn->maxFid);
 }
 
 static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen) {
diff --git a/src/tsdb/src/tsdbFS.c b/src/tsdb/src/tsdbFS.c
index fc28d784f2..e681508337 100644
--- a/src/tsdb/src/tsdbFS.c
+++ b/src/tsdb/src/tsdbFS.c
@@ -848,7 +848,7 @@ static int tsdbScanRootDir(STsdbRepo *pRepo) {
       continue;
     }
 
-    if (tfsIsSameFile(pf, &(pfs->cstatus->pmf->f))) {
+    if (pfs->cstatus->pmf && tfsIsSameFile(pf, &(pfs->cstatus->pmf->f))) {
       continue;
     }
 

From a4189faca4a327170907561de5890651936c7a4b Mon Sep 17 00:00:00 2001
From: liuyq-617 <yqliu@taosdata.com>
Date: Tue, 2 Feb 2021 13:32:08 +0800
Subject: [PATCH 13/13] modify time in another way

---
 tests/pytest/insert/retentionpolicy.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/tests/pytest/insert/retentionpolicy.py b/tests/pytest/insert/retentionpolicy.py
index c69060b5ae..0188ee22c5 100644
--- a/tests/pytest/insert/retentionpolicy.py
+++ b/tests/pytest/insert/retentionpolicy.py
@@ -44,7 +44,8 @@ class TDTestRetetion:
             caller = inspect.getframeinfo(inspect.stack()[1][0])
             args = (caller.filename, caller.lineno, sql, self.queryRows, expectRows)
             os.system("sudo timedatectl set-ntp true")
-            time.sleep(40)
+            os.system("date -s '%s'"%(datetime.datetime.now()+datetime.timedelta(hours=1)))
+            time.sleep(5)
             tdLog.exit("%s(%d) failed: sql:%s, queryRows:%d != expect:%d" % args)
 
     def run(self):
@@ -63,7 +64,7 @@ class TDTestRetetion:
         tdLog.info("=============== step2")
         tdDnodes.stop(1)
         os.system("sudo timedatectl set-ntp false")
-        os.system("sudo date -s $(date -d \"${DATE} 2 days\" \"+%Y%m%d\")")
+        os.system("date -s '%s'"%(datetime.datetime.now()+datetime.timedelta(hours=48)))
         tdDnodes.start(1)
         cmd = 'insert into test values(now,5);'
         tdDnodes.stop(1)
@@ -79,7 +80,7 @@ class TDTestRetetion:
             self.checkRows(5,cmd)
         tdLog.info("=============== step3")
         tdDnodes.stop(1)
-        os.system("sudo date -s $(date -d \"${DATE} 2 days\" \"+%Y%m%d\")")
+        os.system("date -s '%s'"%(datetime.datetime.now()+datetime.timedelta(hours=48)))
         tdDnodes.start(1)
         tdLog.info(cmd)
         tdSql.execute(cmd)
@@ -110,7 +111,8 @@ class TDTestRetetion:
 
     def stop(self):
         os.system("sudo timedatectl set-ntp true")
-        time.sleep(40)
+        os.system("date -s '%s'"%(datetime.datetime.now()+datetime.timedelta(hours=1)))
+        time.sleep(5)
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)