From 0dde6985dc23a2f4a95a3e8fdccd8e234f4eb7d6 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Thu, 19 Mar 2020 15:21:37 +0800 Subject: [PATCH] TD-34 --- src/vnode/tsdb/inc/tsdbFile.h | 23 +++++---- src/vnode/tsdb/src/tsdbFile.c | 75 +++++++++++++++++++----------- src/vnode/tsdb/tests/tsdbTests.cpp | 9 ++++ 3 files changed, 69 insertions(+), 38 deletions(-) diff --git a/src/vnode/tsdb/inc/tsdbFile.h b/src/vnode/tsdb/inc/tsdbFile.h index 7e83b84375..6f539cb838 100644 --- a/src/vnode/tsdb/inc/tsdbFile.h +++ b/src/vnode/tsdb/inc/tsdbFile.h @@ -24,9 +24,10 @@ extern "C" { #endif typedef enum { - TSDB_FILE_TYPE_HEAD, // .head file type - TSDB_FILE_TYPE_DATA, // .data file type - TSDB_FILE_TYPE_LAST // .last file type + TSDB_FILE_TYPE_HEAD = 0, // .head file type + TSDB_FILE_TYPE_DATA, // .data file type + TSDB_FILE_TYPE_LAST, // .last file type + TSDB_FILE_TYPE_MAX } TSDB_FILE_TYPE; extern const char *tsdbFileSuffix[]; @@ -37,15 +38,15 @@ typedef struct { } SFileInfo; typedef struct { - int64_t size; // total size of the file - int64_t tombSize; // unused file size + int8_t type; + char fname[128]; + int64_t size; // total size of the file + int64_t tombSize; // unused file size } SFile; typedef struct { int32_t fileId; - SFile fhead; - SFile fdata; - SFile flast; + SFile files[TSDB_FILE_TYPE_MAX]; } SFileGroup; // TSDB file handle @@ -57,14 +58,12 @@ typedef struct { SFileGroup fGroup[]; } STsdbFileH; -#define IS_VALID_TSDB_FILE_TYPE(type) ((type) >= TSDB_FILE_TYPE_HEAD && (type) <= TSDB_FILE_TYPE_LAST) +#define IS_VALID_TSDB_FILE_TYPE(type) ((type) >= TSDB_FILE_TYPE_HEAD && (type) < TSDB_FILE_TYPE_MAX) STsdbFileH *tsdbInitFile(char *dataDir, int32_t daysPerFile, int32_t keep, int32_t minRowsPerFBlock, int32_t maxRowsPerFBlock); void tsdbCloseFile(STsdbFileH *pFileH); - -char *tsdbGetFileName(char *dirName, char *fname, TSDB_FILE_TYPE type); - +int tsdbCreateFileGroup(char *dataDir, int fileId, SFileGroup *pFGroup, int maxTables); #ifdef __cplusplus } #endif diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c index ad9045567b..da81791da3 100644 --- a/src/vnode/tsdb/src/tsdbFile.c +++ b/src/vnode/tsdb/src/tsdbFile.c @@ -88,23 +88,26 @@ const char *tsdbFileSuffix[] = { ".last" // TSDB_FILE_TYPE_LAST }; -static int tsdbWriteFileHead(int fd) { +static int tsdbWriteFileHead(int fd, SFile *pFile) { char head[TSDB_FILE_HEAD_SIZE] = "\0"; + pFile->size += TSDB_FILE_HEAD_SIZE; + + // TODO: write version and File statistic to the head lseek(fd, 0, SEEK_SET); if (write(fd, head, TSDB_FILE_HEAD_SIZE) < 0) return -1; return 0; } -static int tsdbWriteHeadFileIdx(int fd, int maxTables) { +static int tsdbWriteHeadFileIdx(int fd, int maxTables, SFile *pFile) { int size = sizeof(SCompIdx) * maxTables; void *buf = calloc(1, size); if (buf == NULL) return -1; if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) { free(buf); - return NULL; + return -1; } if (write(fd, buf, size) < 0) { @@ -112,40 +115,70 @@ static int tsdbWriteHeadFileIdx(int fd, int maxTables) { return -1; } + pFile->size += size; + return 0; } -static int tsdbCreateFile(char *dataDir, int fileId, int8_t type, int maxTables) { - char fname[128] = "\0"; +static int tsdbGetFileName(char *dataDir, int fileId, int8_t type, char *fname) { + if (dataDir == NULL || fname == NULL || !IS_VALID_TSDB_FILE_TYPE(type)) return -1; + sprintf(fname, "%s/f%d%s", dataDir, fileId, tsdbFileSuffix[type]); - if (access(fname, F_OK) == 0) { + + return 0; +} + +/** + * Create a file and set the SFile object + */ +static int tsdbCreateFile(char *dataDir, int fileId, int8_t type, int maxTables, SFile *pFile) { + memset((void *)pFile, 0, sizeof(SFile)); + pFile->type = type; + + tsdbGetFileName(dataDir, fileId, type, pFile->fname); + if (access(pFile->fname, F_OK) == 0) { // File already exists return -1; } - int fd = open(fname, O_RDWR | O_CREAT, 0755); + int fd = open(pFile->fname, O_WRONLY | O_CREAT, 0755); if (fd < 0) return -1; - if (tsdbWriteFileHead(fd) < 0) { - close(fd); - return -1; - } - - if (type == TSDB_FILE_TYPE_LAST) { - if (tsdbWriteHeadFileIdx(fd, maxTables) < 0) { + if (type == TSDB_FILE_TYPE_HEAD) { + if (tsdbWriteHeadFileIdx(fd, maxTables, pFile) < 0) { close(fd); return -1; } } + if (tsdbWriteFileHead(fd, pFile) < 0) { + close(fd); + return -1; + } + close(fd); return 0; } +/** + * + */ + // Create a file group with fileId and return a SFileGroup object -static int tsdbCreateFileGroup(char *dataDir, int fileId, SFileGroup *pFGroup) { - // tsdbCreateFile() +int tsdbCreateFileGroup(char *dataDir, int fileId, SFileGroup *pFGroup, int maxTables) { + if (dataDir == NULL || pFGroup == NULL) return -1; + + memset((void *)pFGroup, 0, sizeof(SFileGroup)); + + for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + if (tsdbCreateFile(dataDir, fileId, type, maxTables, &(pFGroup->files[type])) < 0) { + // TODO: deal with the error here, remove the created files + return -1; + } + } + + pFGroup->fileId = fileId; return 0; } @@ -199,16 +232,6 @@ void tsdbCloseFile(STsdbFileH *pFileH) { // TODO } -char *tsdbGetFileName(char *dirName, char *fname, TSDB_FILE_TYPE type) { - if (!IS_VALID_TSDB_FILE_TYPE(type)) return NULL; - - char *fileName = (char *)malloc(strlen(dirName) + strlen(fname) + strlen(tsdbFileSuffix[type]) + 5); - if (fileName == NULL) return NULL; - - sprintf(fileName, "%s/%s%s", dirName, fname, tsdbFileSuffix[type]); - return fileName; -} - static void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, TSKEY *maxKey) { *minKey = fileId * daysPerFile * tsMsPerDay[precision]; diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp index 46ae3940d2..ed6a3bfcbb 100644 --- a/src/vnode/tsdb/tests/tsdbTests.cpp +++ b/src/vnode/tsdb/tests/tsdbTests.cpp @@ -3,6 +3,7 @@ #include "tsdb.h" #include "dataformat.h" +#include "tsdbFile.h" #include "tsdbMeta.h" TEST(TsdbTest, tableEncodeDecode) { @@ -106,4 +107,12 @@ TEST(TsdbTest, createRepo) { TEST(TsdbTest, openRepo) { tsdb_repo_t *pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0"); ASSERT_NE(pRepo, nullptr); +} + +TEST(TsdbTest, createFileGroup) { + SFileGroup fGroup; + + ASSERT_EQ(tsdbCreateFileGroup("/home/ubuntu/work/ttest/vnode0/data", 1820, &fGroup, 1000), 0); + + int k = 0; } \ No newline at end of file