From 17e1c19b81f9f73b2d0c678b74a469743e475fcb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 7 Aug 2020 09:30:52 +0000 Subject: [PATCH 1/4] remove Idx file --- src/tsdb/inc/tsdbMain.h | 15 ------------ src/tsdb/src/tsdbFile.c | 4 ---- src/tsdb/src/tsdbMemTable.c | 5 ---- src/tsdb/src/tsdbRWHelper.c | 48 ------------------------------------- 4 files changed, 72 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 6119f7086c..16a2e17f1e 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -151,18 +151,10 @@ typedef struct { // ------------------ tsdbFile.c extern const char* tsdbFileSuffix[]; typedef enum { -#ifdef TSDB_IDX - TSDB_FILE_TYPE_IDX = 0, - TSDB_FILE_TYPE_HEAD, -#else TSDB_FILE_TYPE_HEAD = 0, -#endif TSDB_FILE_TYPE_DATA, TSDB_FILE_TYPE_LAST, TSDB_FILE_TYPE_MAX, -#ifdef TSDB_IDX - TSDB_FILE_TYPE_NIDX, -#endif TSDB_FILE_TYPE_NHEAD, TSDB_FILE_TYPE_NLAST } TSDB_FILE_TYPE; @@ -281,9 +273,6 @@ typedef struct { TSKEY minKey; TSKEY maxKey; SFileGroup fGroup; -#ifdef TSDB_IDX - SFile nIdxF; -#endif SFile nHeadF; SFile nLastF; } SHelperFile; @@ -497,10 +486,6 @@ void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TS #define helperState(h) (h)->state #define TSDB_NLAST_FILE_OPENED(h) ((h)->files.nLastF.fd > 0) #define helperFileId(h) ((h)->files.fGroup.fileId) -#ifdef TSDB_IDX -#define helperIdxF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_IDX])) -#define helperNewIdxF(h) (&((h)->files.nIdxF)) -#endif #define helperHeadF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_HEAD])) #define helperDataF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_DATA])) #define helperLastF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_LAST])) diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 8cec7c08e1..a5435ad872 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -20,11 +20,7 @@ #include "tutil.h" #define TAOS_RANDOM_FILE_FAIL_TEST -#ifdef TSDB_IDX -const char *tsdbFileSuffix[] = {".idx", ".head", ".data", ".last", "", ".i", ".h", ".l"}; -#else const char *tsdbFileSuffix[] = {".head", ".data", ".last", "", ".h", ".l"}; -#endif static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type); static void tsdbDestroyFile(SFile *pFile); diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 990db76b7e..0303c47146 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -683,11 +683,6 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe pthread_rwlock_wrlock(&(pFileH->fhlock)); -#ifdef TSDB_IDX - rename(helperNewIdxF(pHelper)->fname, helperIdxF(pHelper)->fname); - pGroup->files[TSDB_FILE_TYPE_IDX].info = helperNewIdxF(pHelper)->info; -#endif - rename(helperNewHeadF(pHelper)->fname, helperHeadF(pHelper)->fname); pGroup->files[TSDB_FILE_TYPE_HEAD].info = helperNewHeadF(pHelper)->info; diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index fb79747d30..4b5acd8fe4 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -110,31 +110,16 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { // Set the files pHelper->files.fGroup = *pGroup; if (helperType(pHelper) == TSDB_WRITE_HELPER) { -#ifdef TSDB_IDX - tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NIDX, helperNewIdxF(pHelper)->fname); -#endif tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NHEAD, helperNewHeadF(pHelper)->fname); tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NLAST, helperNewLastF(pHelper)->fname); } // Open the files -#ifdef TSDB_IDX - if (tsdbOpenFile(helperIdxF(pHelper), O_RDONLY) < 0) return -1; -#endif if (tsdbOpenFile(helperHeadF(pHelper), O_RDONLY) < 0) return -1; if (helperType(pHelper) == TSDB_WRITE_HELPER) { if (tsdbOpenFile(helperDataF(pHelper), O_RDWR) < 0) return -1; if (tsdbOpenFile(helperLastF(pHelper), O_RDWR) < 0) return -1; -#ifdef TSDB_IDX - // Create and open .i file - pFile = helperNewIdxF(pHelper); - if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) return -1; - pFile->info.size = TSDB_FILE_HEAD_SIZE; - pFile->info.magic = TSDB_FILE_INIT_MAGIC; - if (tsdbUpdateFileHeader(pFile) < 0) return -1; -#endif - // Create and open .h pFile = helperNewHeadF(pHelper); if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) return -1; @@ -164,11 +149,6 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { SFile *pFile = NULL; -#ifdef TSDB_IDX - pFile = helperIdxF(pHelper); - tsdbCloseFile(pFile); -#endif - pFile = helperHeadF(pHelper); tsdbCloseFile(pFile); @@ -199,18 +179,6 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { } if (helperType(pHelper) == TSDB_WRITE_HELPER) { -#ifdef TSDB_IDX - pFile = helperNewIdxF(pHelper); - if (pFile->fd > 0) { - if (!hasError) { - tsdbUpdateFileHeader(pFile); - fsync(pFile->fd); - } - tsdbCloseFile(pFile); - if (hasError) (void)remove(pFile->fname); - } -#endif - pFile = helperNewHeadF(pHelper); if (pFile->fd > 0) { if (!hasError) { @@ -412,10 +380,6 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { return -1; } -#ifdef TSDB_IDX - pFile = helperNewIdxF(pHelper); -#endif - if (taosTSizeof(pHelper->pWIdx) < pFile->info.len + sizeof(SCompIdx) + 12) { pHelper->pWIdx = taosTRealloc(pHelper->pWIdx, taosTSizeof(pHelper->pWIdx) == 0 ? 1024 : taosTSizeof(pHelper->pWIdx) * 2); if (pHelper->pWIdx == NULL) { @@ -435,11 +399,7 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); off_t offset = 0; -#ifdef TSDB_IDX - SFile *pFile = helperNewIdxF(pHelper); -#else SFile *pFile = helperNewHeadF(pHelper); -#endif pFile->info.len += sizeof(TSCKSUM); if (taosTSizeof(pHelper->pWIdx) < pFile->info.len) { @@ -474,11 +434,7 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { ASSERT(pHelper->state == TSDB_HELPER_FILE_SET_AND_OPEN); -#ifdef TSDB_IDX - SFile *pFile = helperIdxF(pHelper); -#else SFile *pFile = helperHeadF(pHelper); -#endif int fd = pFile->fd; if (!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) { @@ -1052,10 +1008,6 @@ static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { helperLastF(pHelper)->fd = -1; helperNewHeadF(pHelper)->fd = -1; helperNewLastF(pHelper)->fd = -1; -#ifdef TSDB_IDX - helperIdxF(pHelper)->fd = -1; - helperNewIdxF(pHelper)->fd = -1; -#endif } static int tsdbInitHelperFile(SRWHelper *pHelper) { From 222ebe3602bf0ac9fd7766a9d38b5a268b336a1b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 8 Aug 2020 17:14:57 +0800 Subject: [PATCH 2/4] add file size statistics --- src/tsdb/inc/tsdbMain.h | 14 +++++++++++--- src/tsdb/src/tsdbFile.c | 2 +- src/tsdb/src/tsdbMemTable.c | 4 ++-- src/tsdb/src/tsdbRWHelper.c | 36 ++++++++++++++++++++++++++++-------- 4 files changed, 42 insertions(+), 14 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 16a2e17f1e..f42c00f171 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -154,11 +154,19 @@ typedef enum { TSDB_FILE_TYPE_HEAD = 0, TSDB_FILE_TYPE_DATA, TSDB_FILE_TYPE_LAST, - TSDB_FILE_TYPE_MAX, + TSDB_FILE_TYPE_STAT, TSDB_FILE_TYPE_NHEAD, - TSDB_FILE_TYPE_NLAST + TSDB_FILE_TYPE_NDATA, + TSDB_FILE_TYPE_NLAST, + TSDB_FILE_TYPE_NSTAT } TSDB_FILE_TYPE; +#ifndef TDINTERNAL +#define TSDB_FILE_TYPE_MAX (TSDB_FILE_TYPE_LAST+1) +#else +#define TSDB_FILE_TYPE_MAX (TSDB_FILE_TYPE_STAT+1) +#endif + typedef struct { uint32_t magic; uint32_t len; @@ -497,7 +505,7 @@ int tsdbInitWriteHelper(SRWHelper* pHelper, STsdbRepo* pRepo); void tsdbDestroyHelper(SRWHelper* pHelper); void tsdbResetHelper(SRWHelper* pHelper); int tsdbSetAndOpenHelperFile(SRWHelper* pHelper, SFileGroup* pGroup); -int tsdbCloseHelperFile(SRWHelper* pHelper, bool hasError); +int tsdbCloseHelperFile(SRWHelper* pHelper, bool hasError, SFileGroup* pGroup); int tsdbSetHelperTable(SRWHelper* pHelper, STable* pTable, STsdbRepo* pRepo); int tsdbCommitTableData(SRWHelper* pHelper, SCommitIter* pCommitIter, SDataCols* pDataCols, TSKEY maxKey); int tsdbMoveLastBlockIfNeccessary(SRWHelper* pHelper); diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index a5435ad872..d30377ad9f 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -20,7 +20,7 @@ #include "tutil.h" #define TAOS_RANDOM_FILE_FAIL_TEST -const char *tsdbFileSuffix[] = {".head", ".data", ".last", "", ".h", ".l"}; +const char *tsdbFileSuffix[] = {".head", ".data", ".last", ".stat", ".h", ".d", ".l", ".s"}; static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type); static void tsdbDestroyFile(SFile *pFile); diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 0303c47146..04b8b64e06 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -679,7 +679,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe } taosTFree(dataDir); - tsdbCloseHelperFile(pHelper, 0); + tsdbCloseHelperFile(pHelper, 0, pGroup); pthread_rwlock_wrlock(&(pFileH->fhlock)); @@ -701,7 +701,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe _err: taosTFree(dataDir); - tsdbCloseHelperFile(pHelper, 1); + tsdbCloseHelperFile(pHelper, 1, NULL); return -1; } diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 4b5acd8fe4..a00ed8998a 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -91,7 +91,7 @@ void tsdbResetHelper(SRWHelper *pHelper) { tsdbResetHelperTableImpl(pHelper); // Reset the file part - tsdbCloseHelperFile(pHelper, false); + tsdbCloseHelperFile(pHelper, false, NULL); tsdbResetHelperFileImpl(pHelper); pHelper->state = TSDB_HELPER_CLEAR_STATE; @@ -120,6 +120,14 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { if (tsdbOpenFile(helperDataF(pHelper), O_RDWR) < 0) return -1; if (tsdbOpenFile(helperLastF(pHelper), O_RDWR) < 0) return -1; + // NOTE: For data file compatibility + if (helperDataF(pHelper)->info.size == TSDB_FILE_HEAD_SIZE) { + helperDataF(pHelper)->info.size = (uint64_t)lseek(helperDataF(pHelper)->fd, 0, SEEK_END); + } + if (helperLastF(pHelper)->info.size == TSDB_FILE_HEAD_SIZE) { + helperLastF(pHelper)->info.size = (uint64_t)lseek(helperLastF(pHelper)->fd, 0, SEEK_END); + } + // Create and open .h pFile = helperNewHeadF(pHelper); if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) return -1; @@ -146,7 +154,7 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { return 0; } -int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { +int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError, SFileGroup *pGroup) { SFile *pFile = NULL; pFile = helperHeadF(pHelper); @@ -157,10 +165,11 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { if (helperType(pHelper) == TSDB_WRITE_HELPER) { if (!hasError) { tsdbUpdateFileHeader(pFile); - fsync(pFile->fd); } else { - // TODO: shrink back to origin + ASSERT(pGroup != NULL); + taosFtruncate(pFile->fd, pGroup->files[TSDB_FILE_TYPE_DATA].info.size); } + fsync(pFile->fd); } tsdbCloseFile(pFile); } @@ -170,10 +179,11 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { if (helperType(pHelper) == TSDB_WRITE_HELPER && !TSDB_NLAST_FILE_OPENED(pHelper)) { if (!hasError) { tsdbUpdateFileHeader(pFile); - fsync(pFile->fd); } else { - // TODO: shrink back to origin + ASSERT(pGroup != NULL); + taosFtruncate(pFile->fd, pGroup->files[TSDB_FILE_TYPE_LAST].info.size); } + fsync(pFile->fd); } tsdbCloseFile(pFile); } @@ -390,6 +400,9 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { void *pBuf = POINTER_SHIFT(pHelper->pWIdx, pFile->info.len); pFile->info.len += tsdbEncodeSCompIdx(&pBuf, &(pHelper->curCompIdx)); + + pFile->info.size += pIdx->len; + ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); } return 0; @@ -420,7 +433,7 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { return -1; } - pFile->info.offset = offset; + ASSERT(offset == pFile->info.size); if (taosTWrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < pFile->info.len) { tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len, @@ -429,6 +442,10 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { return -1; } + pFile->info.offset = offset; + pFile->info.size += pFile->info.len; + ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); + return 0; } @@ -803,6 +820,9 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa (int)(pCompBlock->numOfRows), pCompBlock->len, pCompBlock->numOfCols, pCompBlock->keyFirst, pCompBlock->keyLast); + pFile->info.size += pCompBlock->len; + ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); + return 0; _err: @@ -1016,7 +1036,7 @@ static int tsdbInitHelperFile(SRWHelper *pHelper) { } static void tsdbDestroyHelperFile(SRWHelper *pHelper) { - tsdbCloseHelperFile(pHelper, false); + tsdbCloseHelperFile(pHelper, false, NULL); tsdbResetHelperFileImpl(pHelper); taosTZfree(pHelper->idxH.pIdxArray); taosTZfree(pHelper->pWIdx); From 77cfb02ea0dcf9380ed0cd239e9d1ebd96d600d4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 10 Aug 2020 09:59:15 +0800 Subject: [PATCH 3/4] add file size statistics --- src/tsdb/src/tsdbFile.c | 4 ++++ src/tsdb/src/tsdbRWHelper.c | 8 -------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index b913070e8a..b8173d41b3 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -409,6 +409,10 @@ static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) { pBuf = taosDecodeFixedU32(pBuf, &version); pBuf = tsdbDecodeSFileInfo(pBuf, &(pFile->info)); + if (pFile->info.size == TSDB_FILE_HEAD_SIZE) { + pFile->info.size = lseek(pFile->fd, 0, SEEK_END); + } + if (version != TSDB_FILE_VERSION) { tsdbError("vgId:%d file %s version %u is not the same as program version %u which may cause problem", REPO_ID(pRepo), pFile->fname, version, TSDB_FILE_VERSION); diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index d30a060c2b..5d0e9f16b2 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -120,14 +120,6 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { if (tsdbOpenFile(helperDataF(pHelper), O_RDWR) < 0) return -1; if (tsdbOpenFile(helperLastF(pHelper), O_RDWR) < 0) return -1; - // NOTE: For data file compatibility - if (helperDataF(pHelper)->info.size == TSDB_FILE_HEAD_SIZE) { - helperDataF(pHelper)->info.size = (uint64_t)lseek(helperDataF(pHelper)->fd, 0, SEEK_END); - } - if (helperLastF(pHelper)->info.size == TSDB_FILE_HEAD_SIZE) { - helperLastF(pHelper)->info.size = (uint64_t)lseek(helperLastF(pHelper)->fd, 0, SEEK_END); - } - // Create and open .h pFile = helperNewHeadF(pHelper); if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) return -1; From 1f66342c4b12cd121ca637939576534a8fd6f753 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 10 Aug 2020 11:12:16 +0800 Subject: [PATCH 4/4] remove part of assert --- src/tsdb/src/tsdbRWHelper.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 5d0e9f16b2..1e082e39be 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -394,7 +394,7 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { pFile->info.len += tsdbEncodeSCompIdx(&pBuf, &(pHelper->curCompIdx)); pFile->info.size += pIdx->len; - ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); + // ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); } return 0; @@ -436,7 +436,7 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { pFile->info.offset = offset; pFile->info.size += pFile->info.len; - ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); + // ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); return 0; } @@ -813,7 +813,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa pCompBlock->keyLast); pFile->info.size += pCompBlock->len; - ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); + // ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); return 0;