From 93ff8b4c7ee442f40d891844c888c6a2cd481e98 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 1 Feb 2021 09:58:22 +0800 Subject: [PATCH 01/10] fix compile error --- src/tsdb/inc/tsdbFile.h | 2 +- src/tsdb/src/tsdbFile.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tsdb/inc/tsdbFile.h b/src/tsdb/inc/tsdbFile.h index cb5c78ca84..3020ecd216 100644 --- a/src/tsdb/inc/tsdbFile.h +++ b/src/tsdb/inc/tsdbFile.h @@ -50,7 +50,7 @@ typedef struct { } SMFile; void tsdbInitMFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver); -void tsdbInitMFileEx(SMFile* pMFile, SMFile* pOMFile); +void tsdbInitMFileEx(SMFile* pMFile, const SMFile* pOMFile); int tsdbEncodeSMFile(void** buf, SMFile* pMFile); void* tsdbDecodeSMFile(void* buf, SMFile* pMFile); int tsdbEncodeSMFileEx(void** buf, SMFile* pMFile); diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 9a53bf4577..e0e09b5deb 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -42,7 +42,7 @@ void tsdbInitMFile(SMFile *pMFile, SDiskID did, int vid, uint32_t ver) { tfsInitFile(TSDB_FILE_F(pMFile), did.level, did.id, fname); } -void tsdbInitMFileEx(SMFile *pMFile, SMFile *pOMFile) { +void tsdbInitMFileEx(SMFile *pMFile, const SMFile *pOMFile) { *pMFile = *pOMFile; TSDB_FILE_SET_CLOSED(pMFile); } From 7ffafb5f169d03a641e9090ed38e3f7fcba4326c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 1 Feb 2021 13:59:42 +0800 Subject: [PATCH 02/10] refact --- src/tsdb/src/tsdbCommit.c | 58 +++++++++++++++++++++++++++++---------- src/tsdb/src/tsdbFS.c | 8 ++++++ 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 6c5d3ae015..13c0d568a5 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -52,7 +52,7 @@ static int tsdbCommitMeta(STsdbRepo *pRepo); static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen); static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid); static int tsdbCommitTSData(STsdbRepo *pRepo); -static int tsdbStartCommit(STsdbRepo *pRepo); +static void tsdbStartCommit(STsdbRepo *pRepo); static void tsdbEndCommit(STsdbRepo *pRepo, int eno); static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid); static int tsdbCreateCommitIters(SCommitH *pCommith); @@ -84,10 +84,7 @@ static int tsdbApplyRtn(STsdbRepo *pRepo); static int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn); void *tsdbCommitData(STsdbRepo *pRepo) { - if (tsdbStartCommit(pRepo) < 0) { - tsdbError("vgId:%d failed to commit data while startting to commit since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; - } + tsdbStartCommit(pRepo); // Commit to update meta file if (tsdbCommitMeta(pRepo) < 0) { @@ -138,11 +135,15 @@ static int tsdbCommitMeta(STsdbRepo *pRepo) { tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo))); if (tsdbCreateMFile(&mf, true) < 0) { + tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; } + + tsdbInfo("vgId:%d meta file %s is created to commit", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf)); } else { tsdbInitMFileEx(&mf, pOMFile); if (tsdbOpenMFile(&mf, O_WRONLY) < 0) { + tsdbError("vgId:%d failed to open META file since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; } } @@ -154,12 +155,20 @@ static int tsdbCommitMeta(STsdbRepo *pRepo) { if (pAct->act == TSDB_UPDATE_META) { pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj)); if (tsdbUpdateMetaRecord(pfs, &mf, pAct->uid, (void *)(pCont->cont), pCont->len) < 0) { + tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, + tstrerror(terrno)); tsdbCloseMFile(&mf); + tsdbApplyMFileChange(&mf, pOMFile); + // TODO: need to reload metaCache return -1; } } else if (pAct->act == TSDB_DROP_META) { if (tsdbDropMetaRecord(pfs, &mf, pAct->uid) < 0) { + tsdbError("vgId:%d failed to drop META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, + tstrerror(terrno)); tsdbCloseMFile(&mf); + tsdbApplyMFileChange(&mf, pOMFile); + // TODO: need to reload metaCache return -1; } } else { @@ -168,6 +177,9 @@ static int tsdbCommitMeta(STsdbRepo *pRepo) { } if (tsdbUpdateMFileHeader(&mf) < 0) { + tsdbError("vgId:%d failed to update META file header since %s, revert it", REPO_ID(pRepo), tstrerror(terrno)); + tsdbApplyMFileChange(&mf, pOMFile); + // TODO: need to reload metaCache return -1; } @@ -238,7 +250,7 @@ static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void tsdbUpdateMFileMagic(pMFile, POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM))); SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)&uid, sizeof(uid)); if (pRecord != NULL) { - pMFile->info.tombSize += pRecord->size; + pMFile->info.tombSize += (pRecord->size + sizeof(SKVRecord)); } else { pMFile->info.nRecords++; } @@ -253,7 +265,7 @@ static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) { SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)(&uid), sizeof(uid)); if (pRecord == NULL) { - tsdbError("failed to drop KV store record with key %" PRIu64 " since not find", uid); + tsdbError("failed to drop META record with key %" PRIu64 " since not find", uid); return -1; } @@ -264,11 +276,11 @@ static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) { void *pBuf = buf; tsdbEncodeKVRecord(&pBuf, &rInfo); - if (tsdbAppendMFile(pMFile, buf, POINTER_DISTANCE(pBuf, buf), NULL) < 0) { + if (tsdbAppendMFile(pMFile, buf, sizeof(SKVRecord), NULL) < 0) { return -1; } - pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t *)buf, (uint32_t)POINTER_DISTANCE(pBuf, buf)); + pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t *)buf, sizeof(SKVRecord)); pMFile->info.nDels++; pMFile->info.nRecords--; pMFile->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2); @@ -300,7 +312,12 @@ static int tsdbCommitTSData(STsdbRepo *pRepo) { // Skip expired memory data and expired FSET tsdbSeekCommitIter(&commith, commith.rtn.minKey); while ((pSet = tsdbFSIterNext(&(commith.fsIter)))) { - if (pSet->fid >= commith.rtn.minFid) break; + if (pSet->fid >= commith.rtn.minFid) { + break; + } else { + tsdbInfo("vgId:%d FSET %d on level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); + } } // Loop to commit to each file @@ -347,7 +364,7 @@ static int tsdbCommitTSData(STsdbRepo *pRepo) { return 0; } -static int tsdbStartCommit(STsdbRepo *pRepo) { +static void tsdbStartCommit(STsdbRepo *pRepo) { SMemTable *pMem = pRepo->imem; ASSERT(pMem->numOfRows > 0 || listNEles(pMem->actList) > 0); @@ -358,7 +375,6 @@ static int tsdbStartCommit(STsdbRepo *pRepo) { tsdbStartFSTxn(pRepo, pMem->pointsAdd, pMem->storageAdd); pRepo->code = TSDB_CODE_SUCCESS; - return 0; } static void tsdbEndCommit(STsdbRepo *pRepo, int eno) { @@ -1371,12 +1387,16 @@ static int tsdbApplyRtn(STsdbRepo *pRepo) { STsdbFS * pfs = REPO_FS(pRepo); SDFileSet *pSet; - // Get retentioni snapshot + // Get retention snapshot tsdbGetRtnSnap(pRepo, &rtn); tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); while ((pSet = tsdbFSIterNext(&fsiter))) { - if (pSet->fid < rtn.minFid) continue; + if (pSet->fid < rtn.minFid) { + tsdbInfo("vgId:%d FSET %d at level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); + continue; + } if (tsdbApplyRtnOnFSet(pRepo, pSet, &rtn) < 0) { return -1; @@ -1390,10 +1410,13 @@ static int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn) { SDiskID did; SDFileSet nSet; STsdbFS * pfs = REPO_FS(pRepo); + int level; ASSERT(pSet->fid >= pRtn->minFid); - tfsAllocDisk(tsdbGetFidLevel(pSet->fid, pRtn), &(did.level), &(did.id)); + level = tsdbGetFidLevel(pSet->fid, pRtn); + + tfsAllocDisk(level, &(did.level), &(did.id)); if (did.level == TFS_UNDECIDED_LEVEL) { terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; return -1; @@ -1404,12 +1427,17 @@ static int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn) { tsdbInitDFileSet(&nSet, did, REPO_ID(pRepo), pSet->fid, FS_TXN_VERSION(pfs)); if (tsdbCopyDFileSet(pSet, &nSet) < 0) { + tsdbError("vgId:%d failed to copy FSET %d from level %d to level %d since %s", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), did.level, tstrerror(terrno)); return -1; } if (tsdbUpdateDFileSet(pfs, &nSet) < 0) { return -1; } + + tsdbInfo("vgId:%d FSET %d is copied from level %d disk id %d to level %d disk id %d", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet) did.level, did.id); } else { // On a correct level if (tsdbUpdateDFileSet(pfs, pSet) < 0) { diff --git a/src/tsdb/src/tsdbFS.c b/src/tsdb/src/tsdbFS.c index 9196fecfcd..5be44c79a0 100644 --- a/src/tsdb/src/tsdbFS.c +++ b/src/tsdb/src/tsdbFS.c @@ -699,6 +699,8 @@ int tsdbLoadMetaCache(STsdbRepo *pRepo, bool recoverMeta) { int64_t maxBufSize = 0; SMFInfo minfo; + taosHashEmpty(pfs->metaCache); + // No meta file, just return if (pfs->cstatus->pmf == NULL) return 0; @@ -716,6 +718,12 @@ int tsdbLoadMetaCache(STsdbRepo *pRepo, bool recoverMeta) { while (true) { int64_t tsize = tsdbReadMFile(pMFile, tbuf, sizeof(SKVRecord)); if (tsize == 0) break; + + if (tsize < 0) { + tsdbError("vgId:%d failed to read META file since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + if (tsize < sizeof(SKVRecord)) { tsdbError("vgId:%d failed to read %" PRIzu " bytes from file %s", REPO_ID(pRepo), sizeof(SKVRecord), TSDB_FILE_FULL_NAME(pMFile)); From 89d9be4d78f6044b7e5fb7a25c81db2a8ec47e7e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 1 Feb 2021 14:26:46 +0800 Subject: [PATCH 03/10] fix compile issue --- src/tsdb/src/tsdbCommit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 13c0d568a5..c09b492f29 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -1437,7 +1437,7 @@ static int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn) { } tsdbInfo("vgId:%d FSET %d is copied from level %d disk id %d to level %d disk id %d", REPO_ID(pRepo), pSet->fid, - TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet) did.level, did.id); + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet), did.level, did.id); } else { // On a correct level if (tsdbUpdateDFileSet(pfs, pSet) < 0) { From 291aaff1c69ff0f4ed8a7f828dc0f7dee897d305 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 1 Feb 2021 16:17:10 +0800 Subject: [PATCH 04/10] fix two review bugs --- src/tsdb/src/tsdbCommit.c | 45 ++++++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index c09b492f29..35c4812c27 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -312,11 +312,11 @@ static int tsdbCommitTSData(STsdbRepo *pRepo) { // Skip expired memory data and expired FSET tsdbSeekCommitIter(&commith, commith.rtn.minKey); while ((pSet = tsdbFSIterNext(&(commith.fsIter)))) { - if (pSet->fid >= commith.rtn.minFid) { - break; - } else { + if (pSet->fid < commith.rtn.minFid) { tsdbInfo("vgId:%d FSET %d on level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); + } else { + break; } } @@ -427,14 +427,18 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { if (pIter->pTable == NULL) continue; if (tsdbCommitToTable(pCommith, tid) < 0) { - // TODO: revert the file change tsdbCloseCommitFile(pCommith, true); + // revert the file change + tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); return -1; } } if (tsdbWriteBlockIdx(pCommith) < 0) { + tsdbError("vgId:%d failed to write SBlockIdx part to FSET %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); tsdbCloseCommitFile(pCommith, true); + // revert the file change + tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); return -1; } @@ -688,7 +692,11 @@ static int tsdbCommitToTable(SCommitH *pCommith, int tid) { TSDB_RUNLOCK_TABLE(pIter->pTable); - if (tsdbWriteBlockInfo(pCommith) < 0) return -1; + if (tsdbWriteBlockInfo(pCommith) < 0) { + tsdbError("vgId:%d failed to write SBlockInfo part into file %s since %s", REPO_ID(pRepo), + TSDB_FILE_FULL_NAME(TSDB_COMMIT_HEAD_FILE(pCommith)), tstrerror(terrno)); + return -1; + } return 0; } @@ -940,6 +948,8 @@ static int tsdbWriteBlockIdx(SCommitH *pCommih) { if (nidx <= 0) { // All data are deleted + pHeadf->info.offset = 0; + pHeadf->info.len = 0; return 0; } @@ -1241,7 +1251,6 @@ static void tsdbResetCommitFile(SCommitH *pCommith) { } static void tsdbResetCommitTable(SCommitH *pCommith) { - tdResetDataCols(pCommith->pDataCols); taosArrayClear(pCommith->aSubBlk); taosArrayClear(pCommith->aSupBlk); pCommith->pTable = NULL; @@ -1270,6 +1279,9 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid tsdbCloseAndUnsetFSet(&(pCommith->readh)); return -1; } + + tsdbDebug("vgId:%d FSET %d at level %d disk id %d is opened to read to commit", REPO_ID(pRepo), TSDB_FSET_FID(pSet), + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); } else { pCommith->isRFileSet = false; } @@ -1280,6 +1292,8 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid tsdbInitDFileSet(pWSet, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo))); if (tsdbCreateDFileSet(pWSet, true) < 0) { + tsdbError("vgId:%d failed to create FSET %d at level %d disk id %d since %s", REPO_ID(pRepo), + TSDB_FSET_FID(pWSet), TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet), tstrerror(terrno)); if (pCommith->isRFileSet) { tsdbCloseAndUnsetFSet(&(pCommith->readh)); } @@ -1288,6 +1302,9 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid pCommith->isDFileSame = false; pCommith->isLFileSame = false; + + tsdbDebug("vgId:%d FSET %d at level %d disk id %d is created to commit", REPO_ID(pRepo), TSDB_FSET_FID(pWSet), + TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet)); } else { did.level = TSDB_FSET_LEVEL(pSet); did.id = TSDB_FSET_ID(pSet); @@ -1299,6 +1316,9 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid SDFile *pWHeadf = TSDB_COMMIT_HEAD_FILE(pCommith); tsdbInitDFile(pWHeadf, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_HEAD); if (tsdbCreateDFile(pWHeadf, true) < 0) { + tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWHeadf), + tstrerror(terrno)); + if (pCommith->isRFileSet) { tsdbCloseAndUnsetFSet(&(pCommith->readh)); return -1; @@ -1310,7 +1330,10 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid SDFile *pWDataf = TSDB_COMMIT_DATA_FILE(pCommith); tsdbInitDFileEx(pWDataf, pRDataf); if (tsdbOpenDFile(pWDataf, O_WRONLY) < 0) { - tsdbCloseDFile(pWHeadf); + tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWDataf), + tstrerror(terrno)); + + tsdbCloseDFileSet(pWSet); tsdbRemoveDFile(pWHeadf); if (pCommith->isRFileSet) { tsdbCloseAndUnsetFSet(&(pCommith->readh)); @@ -1327,6 +1350,9 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid pCommith->isLFileSame = true; if (tsdbOpenDFile(pWLastf, O_WRONLY) < 0) { + tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), + tstrerror(terrno)); + tsdbCloseDFileSet(pWSet); tsdbRemoveDFile(pWHeadf); if (pCommith->isRFileSet) { @@ -1339,6 +1365,9 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid pCommith->isLFileSame = false; if (tsdbCreateDFile(pWLastf, true) < 0) { + tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), + tstrerror(terrno)); + tsdbCloseDFileSet(pWSet); tsdbRemoveDFile(pWHeadf); if (pCommith->isRFileSet) { @@ -1374,7 +1403,7 @@ static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *p if (pBlock->last) { if (pCommith->isLFileSame && mergeRows < pCfg->minRowsPerFileBlock) return true; } else { - if (mergeRows < pCfg->maxRowsPerFileBlock) return true; + if (pCommith->isDFileSame && mergeRows <= pCfg->maxRowsPerFileBlock) return true; } } From e353e3b9463ebb611b8f5959d997673841d4a0cc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 1 Feb 2021 16:19:22 +0800 Subject: [PATCH 05/10] fsync for file copy --- src/os/src/detail/osFile.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c index 42f2ff3afe..0b7b5ca487 100644 --- a/src/os/src/detail/osFile.c +++ b/src/os/src/detail/osFile.c @@ -142,6 +142,8 @@ int64_t taosCopy(char *from, char *to) { if (bytes < sizeof(buffer)) break; } + fsync(fidto); + close(fidfrom); close(fidto); return size; From 3b1f833dab0c6d7a2a866ea87d3e2a4e62e50ade Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 1 Feb 2021 16:42:21 +0800 Subject: [PATCH 06/10] fix compile error --- src/tsdb/src/tsdbCommit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 35c4812c27..e6c0693d8c 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -693,7 +693,7 @@ static int tsdbCommitToTable(SCommitH *pCommith, int tid) { TSDB_RUNLOCK_TABLE(pIter->pTable); if (tsdbWriteBlockInfo(pCommith) < 0) { - tsdbError("vgId:%d failed to write SBlockInfo part into file %s since %s", REPO_ID(pRepo), + tsdbError("vgId:%d failed to write SBlockInfo part into file %s since %s", TSDB_COMMIT_REPO_ID(pCommith), TSDB_FILE_FULL_NAME(TSDB_COMMIT_HEAD_FILE(pCommith)), tstrerror(terrno)); return -1; } From 7cc4864f39f9f056a3931397c1a812c9eca93fdb Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 1 Feb 2021 17:27:00 +0800 Subject: [PATCH 07/10] vnodeCfg/update: fix missing update cfg --- src/vnode/src/vnodeCfg.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/vnode/src/vnodeCfg.c b/src/vnode/src/vnodeCfg.c index 9b7916c8bd..03f2b11eec 100644 --- a/src/vnode/src/vnodeCfg.c +++ b/src/vnode/src/vnodeCfg.c @@ -34,6 +34,7 @@ static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) { pVnode->tsdbCfg.maxRowsPerFileBlock = vnodeMsg->cfg.maxRowsPerFileBlock; pVnode->tsdbCfg.precision = vnodeMsg->cfg.precision; pVnode->tsdbCfg.compression = vnodeMsg->cfg.compression; + pVnode->tsdbCfg.update = vnodeMsg->cfg.update; pVnode->tsdbCfg.cacheLastRow = vnodeMsg->cfg.cacheLastRow; pVnode->walCfg.walLevel = vnodeMsg->cfg.walLevel; pVnode->walCfg.fsyncPeriod = vnodeMsg->cfg.fsyncPeriod; @@ -227,6 +228,15 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { } vnodeMsg.cfg.quorum = (int8_t)quorum->valueint; + cJSON *update = cJSON_GetObjectItem(root, "update"); + if (!update || update->type != cJSON_Number) { + vError("vgId: %d, failed to read %s, update not found", pVnode->vgId, file); + vnodeMsg.cfg.update = 0; + vnodeMsg.cfg.vgCfgVersion = 0; + } else { + vnodeMsg.cfg.update = (int8_t)update->valueint; + } + cJSON *cacheLastRow = cJSON_GetObjectItem(root, "cacheLastRow"); if (!cacheLastRow || cacheLastRow->type != cJSON_Number) { vError("vgId: %d, failed to read %s, cacheLastRow not found", pVnode->vgId, file); @@ -325,6 +335,7 @@ int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) { len += snprintf(content + len, maxLen - len, " \"dbReplica\": %d,\n", pMsg->cfg.dbReplica); len += snprintf(content + len, maxLen - len, " \"wals\": %d,\n", pMsg->cfg.wals); len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pMsg->cfg.quorum); + len += snprintf(content + len, maxLen - len, " \"update\": %d,\n", pMsg->cfg.update); len += snprintf(content + len, maxLen - len, " \"cacheLastRow\": %d,\n", pMsg->cfg.cacheLastRow); len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); for (int32_t i = 0; i < pMsg->cfg.vgReplica; i++) { From 4bec4814fdb164391846281e6cbb8fe39c64c3a0 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 2 Feb 2021 10:01:59 +0800 Subject: [PATCH 08/10] modify testcase suite --- tests/pytest/pytest_2.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/pytest/pytest_2.sh b/tests/pytest/pytest_2.sh index dde9f78953..4ec517a0bf 100755 --- a/tests/pytest/pytest_2.sh +++ b/tests/pytest/pytest_2.sh @@ -1,18 +1,18 @@ # update -#python3 ./test.py -f update/allow_update.py +python3 ./test.py -f update/allow_update.py python3 ./test.py -f update/allow_update-0.py python3 ./test.py -f update/append_commit_data.py python3 ./test.py -f update/append_commit_last-0.py python3 ./test.py -f update/append_commit_last.py -#python3 ./test.py -f update/merge_commit_data.py -#python3 ./test.py -f update/merge_commit_data-0.py -#python3 ./test.py -f update/merge_commit_data2.py -#python3 ./test.py -f update/merge_commit_data2_update0.py -#python3 ./test.py -f update/merge_commit_last-0.py -#python3 ./test.py -f update/merge_commit_last.py -#python3 ./test.py -f update/bug_td2279.py +python3 ./test.py -f update/merge_commit_data.py +python3 ./test.py -f update/merge_commit_data-0.py +python3 ./test.py -f update/merge_commit_data2.py +python3 ./test.py -f update/merge_commit_data2_update0.py +python3 ./test.py -f update/merge_commit_last-0.py +python3 ./test.py -f update/merge_commit_last.py +python3 ./test.py -f update/bug_td2279.py # wal python3 ./test.py -f wal/addOldWalTest.py From 3b156b610c1c955363ba5e5dbdd676749f682e9e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 2 Feb 2021 13:17:07 +0800 Subject: [PATCH 09/10] fix a possible coredump --- src/tsdb/src/tsdbCommit.c | 2 ++ src/tsdb/src/tsdbFS.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 78beb3e293..a777b11186 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -220,6 +220,8 @@ void tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn) { pRtn->minFid = (int)(TSDB_KEY_FID(minKey, pCfg->daysPerFile, pCfg->precision)); pRtn->midFid = (int)(TSDB_KEY_FID(midKey, pCfg->daysPerFile, pCfg->precision)); pRtn->maxFid = (int)(TSDB_KEY_FID(maxKey, pCfg->daysPerFile, pCfg->precision)); + tsdbDebug("vgId:%d now:%" PRId64 " minKey:%" PRId64 " minFid:%d, midFid:%d, maxFid:%d", REPO_ID(pRepo), now, minKey, + pRtn->minFid, pRtn->midFid, pRtn->maxFid); } static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen) { diff --git a/src/tsdb/src/tsdbFS.c b/src/tsdb/src/tsdbFS.c index fc28d784f2..e681508337 100644 --- a/src/tsdb/src/tsdbFS.c +++ b/src/tsdb/src/tsdbFS.c @@ -848,7 +848,7 @@ static int tsdbScanRootDir(STsdbRepo *pRepo) { continue; } - if (tfsIsSameFile(pf, &(pfs->cstatus->pmf->f))) { + if (pfs->cstatus->pmf && tfsIsSameFile(pf, &(pfs->cstatus->pmf->f))) { continue; } From a4189faca4a327170907561de5890651936c7a4b Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 2 Feb 2021 13:32:08 +0800 Subject: [PATCH 10/10] modify time in another way --- tests/pytest/insert/retentionpolicy.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/pytest/insert/retentionpolicy.py b/tests/pytest/insert/retentionpolicy.py index c69060b5ae..0188ee22c5 100644 --- a/tests/pytest/insert/retentionpolicy.py +++ b/tests/pytest/insert/retentionpolicy.py @@ -44,7 +44,8 @@ class TDTestRetetion: caller = inspect.getframeinfo(inspect.stack()[1][0]) args = (caller.filename, caller.lineno, sql, self.queryRows, expectRows) os.system("sudo timedatectl set-ntp true") - time.sleep(40) + os.system("date -s '%s'"%(datetime.datetime.now()+datetime.timedelta(hours=1))) + time.sleep(5) tdLog.exit("%s(%d) failed: sql:%s, queryRows:%d != expect:%d" % args) def run(self): @@ -63,7 +64,7 @@ class TDTestRetetion: tdLog.info("=============== step2") tdDnodes.stop(1) os.system("sudo timedatectl set-ntp false") - os.system("sudo date -s $(date -d \"${DATE} 2 days\" \"+%Y%m%d\")") + os.system("date -s '%s'"%(datetime.datetime.now()+datetime.timedelta(hours=48))) tdDnodes.start(1) cmd = 'insert into test values(now,5);' tdDnodes.stop(1) @@ -79,7 +80,7 @@ class TDTestRetetion: self.checkRows(5,cmd) tdLog.info("=============== step3") tdDnodes.stop(1) - os.system("sudo date -s $(date -d \"${DATE} 2 days\" \"+%Y%m%d\")") + os.system("date -s '%s'"%(datetime.datetime.now()+datetime.timedelta(hours=48))) tdDnodes.start(1) tdLog.info(cmd) tdSql.execute(cmd) @@ -110,7 +111,8 @@ class TDTestRetetion: def stop(self): os.system("sudo timedatectl set-ntp true") - time.sleep(40) + os.system("date -s '%s'"%(datetime.datetime.now()+datetime.timedelta(hours=1))) + time.sleep(5) tdSql.close() tdLog.success("%s successfully executed" % __file__)