From 055aed43e1389a934feebaeec4aae9bf19890e71 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 18 May 2020 01:40:08 +0000 Subject: [PATCH 1/7] TD-185 --- src/tsdb/inc/tsdbMain.h | 29 +++++--- src/tsdb/src/tsdbFile.c | 38 +--------- src/tsdb/src/tsdbMain.c | 10 ++- src/tsdb/src/tsdbMeta.c | 1 - src/tsdb/src/tsdbRWHelper.c | 136 ++++++++++++++++++++++++++++++++---- src/util/inc/tcoding.h | 10 +++ 6 files changed, 162 insertions(+), 62 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 0a1444c2f0..1885b57bc0 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -210,12 +210,17 @@ typedef enum { extern const char *tsdbFileSuffix[]; typedef struct { - int64_t size; // total size of the file - int64_t tombSize; // unused file size - int32_t totalBlocks; - int32_t totalSubBlocks; + uint32_t offset; + uint32_t len; + uint64_t size; // total size of the file + uint64_t tombSize; // unused file size + uint32_t totalBlocks; + uint32_t totalSubBlocks; } SFileInfo; +void *tsdbEncodeSFileInfo(void *buf, const SFileInfo *pInfo); +void *tsdbDecodeSFileInfo(void *buf, SFileInfo *pInfo); + typedef struct { int fd; char fname[128]; @@ -242,8 +247,7 @@ typedef struct { STsdbFileH *tsdbInitFileH(char *dataDir, STsdbCfg *pCfg); void tsdbCloseFileH(STsdbFileH *pFileH); -int tsdbCreateFile(char *dataDir, int fileId, const char *suffix, int maxTables, SFile *pFile, int writeHeader, - int toClose); +int tsdbCreateFile(char *dataDir, int fileId, const char *suffix, SFile *pFile); SFileGroup *tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables); int tsdbOpenFile(SFile *pFile, int oflag); int tsdbCloseFile(SFile *pFile); @@ -266,15 +270,18 @@ void tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid); SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter); typedef struct { - int32_t len; - int32_t offset; - int32_t padding; // For padding purpose - int32_t hasLast : 1; - int32_t numOfBlocks : 31; + uint32_t len; + uint32_t offset; + uint32_t padding; // For padding purpose + uint32_t hasLast : 2; + uint32_t numOfBlocks : 30; int64_t uid; TSKEY maxKey; } SCompIdx; /* sizeof(SCompIdx) = 28 */ +void *tsdbEncodeSCompIdx(void *buf, SCompIdx *pIdx); +void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx); + /** * if numOfSubBlocks == 0, then the SCompBlock is a sub-block * if numOfSubBlocks >= 1, then the SCompBlock is a super-block diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index ad98bb7b20..c9db9344c9 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -38,7 +38,6 @@ const char *tsdbFileSuffix[] = { static int compFGroupKey(const void *key, const void *fgroup); static int compFGroup(const void *arg1, const void *arg2); static int tsdbWriteFileHead(SFile *pFile); -static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables); static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid); STsdbFileH *tsdbInitFileH(char *dataDir, STsdbCfg *pCfg) { @@ -121,8 +120,7 @@ SFileGroup *tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int max if (pGroup == NULL) { // if not exists, create one pFGroup->fileId = fid; for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { - if (tsdbCreateFile(dataDir, fid, tsdbFileSuffix[type], maxTables, &(pFGroup->files[type]), - type == TSDB_FILE_TYPE_HEAD ? 1 : 0, 1) < 0) + if (tsdbCreateFile(dataDir, fid, tsdbFileSuffix[type], &(pFGroup->files[type])) < 0) goto _err; } @@ -298,29 +296,6 @@ static int tsdbWriteFileHead(SFile *pFile) { return 0; } -static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables) { - int size = sizeof(SCompIdx) * maxTables + sizeof(TSCKSUM); - void *buf = calloc(1, size); - if (buf == NULL) return -1; - - if (lseek(pFile->fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) { - free(buf); - return -1; - } - - taosCalcChecksumAppend(0, (uint8_t *)buf, size); - - if (write(pFile->fd, buf, size) < 0) { - free(buf); - return -1; - } - - pFile->info.size += size; - - free(buf); - return 0; -} - int tsdbGetFileName(char *dataDir, int fileId, const char *suffix, char *fname) { if (dataDir == NULL || fname == NULL) return -1; @@ -354,7 +329,7 @@ SFileGroup * tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid) { return pGroup; } -int tsdbCreateFile(char *dataDir, int fileId, const char *suffix, int maxTables, SFile *pFile, int writeHeader, int toClose) { +int tsdbCreateFile(char *dataDir, int fileId, const char *suffix, SFile *pFile) { memset((void *)pFile, 0, sizeof(SFile)); pFile->fd = -1; @@ -370,19 +345,12 @@ int tsdbCreateFile(char *dataDir, int fileId, const char *suffix, int maxTables, return -1; } - if (writeHeader) { - if (tsdbWriteHeadFileIdx(pFile, maxTables) < 0) { - tsdbCloseFile(pFile); - return -1; - } - } - if (tsdbWriteFileHead(pFile) < 0) { tsdbCloseFile(pFile); return -1; } - if (toClose) tsdbCloseFile(pFile); + tsdbCloseFile(pFile); return 0; } diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index c63de165d3..7254318d74 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -1148,8 +1148,16 @@ static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) { } static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables) { - // TODO int oldMaxTables = pRepo->config.maxTables; + if (oldMaxTables < pRepo->config.maxTables) { + // TODO + } + + STsdbMeta *pMeta = pRepo->tsdbMeta; + + pMeta->maxTables = maxTables; + pMeta->tables = realloc(pMeta->tables, maxTables * sizeof(STable *)); + tsdbTrace("vgId:%d, tsdb maxTables is changed from %d to %d!", pRepo->config.tsdbId, oldMaxTables, maxTables); } diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index aaea50267d..1b161a4ad7 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -154,7 +154,6 @@ STsdbMeta *tsdbInitMeta(char *rootDir, int32_t maxTables) { STsdbMeta *pMeta = (STsdbMeta *)malloc(sizeof(STsdbMeta)); if (pMeta == NULL) return NULL; - pMeta->maxTables = maxTables; pMeta->nTables = 0; pMeta->superList = NULL; pMeta->tables = (STable **)calloc(maxTables, sizeof(STable *)); diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 829d7ab3fd..350fde412c 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -18,6 +18,7 @@ #include "tchecksum.h" #include "tscompression.h" #include "talgo.h" +#include "tcoding.h" // Local function definitions // static int tsdbCheckHelperCfg(SHelperCfg *pCfg); @@ -211,13 +212,15 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { // Create and open .h if (tsdbOpenFile(&(pHelper->files.nHeadF), O_WRONLY | O_CREAT) < 0) return -1; - size_t tsize = TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM); - if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, tsize) < tsize) goto _err; + // size_t tsize = TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM); + if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) + goto _err; // Create and open .l file if should if (tsdbShouldCreateNewLast(pHelper)) { if (tsdbOpenFile(&(pHelper->files.nLastF), O_WRONLY | O_CREAT) < 0) goto _err; - if (tsendfile(pHelper->files.nLastF.fd, pHelper->files.lastF.fd, NULL, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) goto _err; + if (tsendfile(pHelper->files.nLastF.fd, pHelper->files.lastF.fd, NULL, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) + goto _err; } } else { if (tsdbOpenFile(&(pHelper->files.dataF), O_RDONLY) < 0) goto _err; @@ -426,13 +429,26 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { int tsdbWriteCompIdx(SRWHelper *pHelper) { ASSERT(TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER); - if (lseek(pHelper->files.nHeadF.fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1; + off_t offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END); + if (offset < 0) return -1; - ASSERT(tsizeof(pHelper->pCompIdx) == sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM)); - taosCalcChecksumAppend(0, (uint8_t *)pHelper->pCompIdx, tsizeof(pHelper->pCompIdx)); + SFile *pFile = &(pHelper->files.nHeadF); + pFile->info.offset = offset; - if (twrite(pHelper->files.nHeadF.fd, (void *)pHelper->pCompIdx, tsizeof(pHelper->pCompIdx)) < tsizeof(pHelper->pCompIdx)) - return -1; + void *buf = pHelper->blockBuffer; + for (uint32_t i = 0; i < pHelper->config.maxTables; i++) { + SCompIdx *pCompIdx = pHelper->pCompIdx + i; + if (pCompIdx->offset > 0) { + buf = taosEncodeVariant32(buf, i); + buf = tsdbEncodeSCompIdx(buf, pCompIdx); + } + } + + int tsize = (char *)buf - (char *)pHelper->blockBuffer + sizeof(TSCKSUM); + taosCalcChecksumAppend(0, (uint8_t *)pHelper->blockBuffer, tsize); + + if (twrite(pHelper->files.nHeadF.fd, (void *)pHelper->blockBuffer, tsize) < tsize) return -1; + pFile->info.len = tsize; return 0; } @@ -441,14 +457,34 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { if (!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) { // If not load from file, just load it in object - int fd = pHelper->files.headF.fd; + SFile *pFile = &(pHelper->files.headF); + int fd = pFile->fd; - if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1; - if (tread(fd, (void *)(pHelper->pCompIdx), tsizeof((void *)pHelper->pCompIdx)) < tsizeof(pHelper->pCompIdx)) return -1; - if (!taosCheckChecksumWhole((uint8_t *)(pHelper->pCompIdx), tsizeof((void *)pHelper->pCompIdx))) { - // TODO: File is broken, try to deal with it - return -1; + memset(pHelper->pCompIdx, 0, tsizeof(pHelper->pCompIdx)); + if (pFile->info.offset > 0) { + ASSERT(pFile->info.offset > TSDB_FILE_HEAD_SIZE); + + if (lseek(fd, pFile->info.offset, SEEK_SET) < 0) return -1; + if (tread(fd, (void *)(pHelper->blockBuffer), pFile->info.len) < pFile->info.len) + return -1; + if (!taosCheckChecksumWhole((uint8_t *)(pHelper->blockBuffer), pFile->info.len)) { + // TODO: File is broken, try to deal with it + return -1; + } + + // Decode it + void *ptr = pHelper->blockBuffer; + while ((char *)ptr - (char *)pHelper->blockBuffer >= pFile->info.len - sizeof(TSCKSUM)) { + uint32_t tid = 0; + if ((ptr = taosDecodeVariant32(ptr, &tid)) == NULL) return -1; + ASSERT(tid > 0 && tid < pHelper->config.maxTables); + + if ((ptr = tsdbDecodeSCompIdx(ptr, pHelper->pCompIdx + tid)) == NULL) return -1; + + ASSERT((char *)ptr - (char *)pHelper->blockBuffer <= pFile->info.len - sizeof(TSCKSUM)); + } } + } helperSetState(pHelper, TSDB_HELPER_IDX_LOAD); @@ -1168,4 +1204,76 @@ static int tsdbGetRowsInRange(SDataCols *pDataCols, TSKEY minKey, TSKEY maxKey) if ((TSKEY *)ptr2 - (TSKEY *)ptr1 < 0) return 0; return ((TSKEY *)ptr2 - (TSKEY *)ptr1) + 1; +} + +void *tsdbEncodeSCompIdx(void *buf, SCompIdx *pIdx) { + buf = taosEncodeVariant32(buf, pIdx->len); + buf = taosEncodeVariant32(buf, pIdx->offset); + buf = taosEncodeFixed8(buf, pIdx->hasLast); + buf = taosEncodeVariant32(buf, pIdx->numOfBlocks); + buf = taosEncodeFixed64(buf, pIdx->uid); + buf = taosEncodeFixed64(buf, pIdx->maxKey); + + return buf; +} + +void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) { + uint8_t hasLast = 0; + uint32_t numOfBlocks = 0; + uint64_t value = 0; + + if ((buf = taosDecodeVariant32(buf, &(pIdx->len))) == NULL) return NULL; + if ((buf = taosDecodeVariant32(buf, &(pIdx->offset))) == NULL) return NULL; + if ((buf = taosDecodeFixed8(buf, &(hasLast))) == NULL) return NULL; + pIdx->hasLast = hasLast; + if ((buf = taosDecodeVariant32(buf, &(numOfBlocks))) == NULL) return NULL; + pIdx->numOfBlocks = numOfBlocks; + if ((buf = taosDecodeFixed64(buf, &value)) == NULL) return NULL; + pIdx->uid = (int64_t)value; + if ((buf = taosDecodeFixed64(buf, &value)) == NULL) return NULL; + pIdx->maxKey = (TSKEY)value; + + return buf; +} + +int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) { + char buf[TSDB_FILE_HEAD_SIZE] = "\0"; + + void *pBuf = (void *)buf; + pBuf = taosEncodeFixed32(pBuf, version); + pBuf = tsdbEncodeSFileInfo(pBuf, &(pFile->info)); + int tsize = (char *)pBuf - buf + sizeof(TSCKSUM); + + ASSERT(tsize <= TSDB_FILE_HEAD_SIZE); + + taosCalcChecksumAppend(0, (uint8_t *)buf, tsize); + + if (lseek(pFile->fd, 0, SEEK_SET) < 0) return -1; + if (twrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) return -1; + + return 0; +} + + + +void *tsdbEncodeSFileInfo(void *buf, const SFileInfo *pInfo) { + buf = taosEncodeFixed32(buf, pInfo->offset); + buf = taosEncodeFixed32(buf, pInfo->len); + buf = taosEncodeFixed64(buf, pInfo->size); + buf = taosEncodeFixed64(buf, pInfo->tombSize); + buf = taosEncodeFixed32(buf, pInfo->totalBlocks); + buf = taosEncodeFixed32(buf, pInfo->totalSubBlocks); + + return buf; +} + +void *tsdbDecodeSFileInfo(void *buf, SFileInfo *pInfo) { + buf = taosDecodeFixed32(buf, &(pInfo->offset)); + buf = taosDecodeFixed32(buf, &(pInfo->len)); + buf = taosDecodeFixed64(buf, &(pInfo->size)); + buf = taosDecodeFixed64(buf, &(pInfo->tombSize)); + buf = taosDecodeFixed32(buf, &(pInfo->totalBlocks)); + buf = taosDecodeFixed32(buf, &(pInfo->totalSubBlocks)); + + return buf; } \ No newline at end of file diff --git a/src/util/inc/tcoding.h b/src/util/inc/tcoding.h index 9f64f127e1..b4f7f596c5 100644 --- a/src/util/inc/tcoding.h +++ b/src/util/inc/tcoding.h @@ -29,6 +29,11 @@ extern "C" { static const int32_t TNUMBER = 1; #define IS_LITTLE_ENDIAN() (*(uint8_t *)(&TNUMBER) != 0) +static FORCE_INLINE void *taosEncodeFixed8(void *buf, uint8_t value) { + ((uint8_t *)buf)[0] = value; + return POINTER_SHIFT(buf, sizeof(value)); +} + static FORCE_INLINE void *taosEncodeFixed16(void *buf, uint16_t value) { if (IS_LITTLE_ENDIAN()) { memcpy(buf, &value, sizeof(value)); @@ -70,6 +75,11 @@ static FORCE_INLINE void *taosEncodeFixed64(void *buf, uint64_t value) { return POINTER_SHIFT(buf, sizeof(value)); } +static FORCE_INLINE void *taosDecodeFixed8(void *buf, uint8_t *value) { + *value = ((uint8_t *)buf)[0]; + return POINTER_SHIFT(buf, sizeof(*value)); +} + static FORCE_INLINE void *taosDecodeFixed16(void *buf, uint16_t *value) { if (IS_LITTLE_ENDIAN()) { memcpy(value, buf, sizeof(*value)); From 1c0cb588ca6c98108a6722561b830ac66d674b04 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 18 May 2020 05:41:49 +0000 Subject: [PATCH 2/7] change uid from int64_t to uint64_t in tsdb --- src/inc/tsdb.h | 12 ++++++------ src/tsdb/inc/tsdbMain.h | 24 ++++++++++++------------ src/tsdb/src/tsdbMain.c | 6 +++--- src/tsdb/src/tsdbMeta.c | 6 +++--- src/tsdb/src/tsdbMetaFile.c | 12 ++++++------ src/tsdb/src/tsdbRead.c | 7 ++++--- src/tsdb/tests/tsdbTests.cpp | 2 +- 7 files changed, 35 insertions(+), 34 deletions(-) diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 0b19155626..050b65153e 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -77,8 +77,8 @@ int32_t tsdbConfigRepo(TsdbRepoT *repo, STsdbCfg *pCfg); // --------- TSDB TABLE DEFINITION typedef struct { - int64_t uid; // the unique table ID - int32_t tid; // the table ID in the repository. + uint64_t uid; // the unique table ID + int32_t tid; // the table ID in the repository. } STableId; // --------- TSDB TABLE configuration @@ -88,14 +88,14 @@ typedef struct { STableId tableId; int32_t sversion; char * sname; // super table name - int64_t superUid; + uint64_t superUid; STSchema * schema; STSchema * tagSchema; SDataRow tagValues; } STableCfg; -int tsdbInitTableCfg(STableCfg *config, ETableType type, int64_t uid, int32_t tid); -int tsdbTableSetSuperUid(STableCfg *config, int64_t uid); +int tsdbInitTableCfg(STableCfg *config, ETableType type, uint64_t uid, int32_t tid); +int tsdbTableSetSuperUid(STableCfg *config, uint64_t uid); int tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup); int tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup); int tsdbTableSetTagValue(STableCfg *config, SDataRow row, bool dup); @@ -109,7 +109,7 @@ char* tsdbGetTableName(TsdbRepoT *repo, const STableId* id, int16_t* bytes); int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg); int tsdbDropTable(TsdbRepoT *pRepo, STableId tableId); int tsdbAlterTable(TsdbRepoT *repo, STableCfg *pCfg); -TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, int64_t uid); +TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, uint64_t uid); uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, int32_t *size); diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index bd68bd6151..e7537f6969 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -63,9 +63,9 @@ typedef struct { } SMetaFile; SMetaFile *tsdbInitMetaFile(char *rootDir, int32_t maxTables, iterFunc iFunc, afterFunc aFunc, void *appH); -int32_t tsdbInsertMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t contLen); -int32_t tsdbDeleteMetaRecord(SMetaFile *mfh, int64_t uid); -int32_t tsdbUpdateMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t contLen); +int32_t tsdbInsertMetaRecord(SMetaFile *mfh, uint64_t uid, void *cont, int32_t contLen); +int32_t tsdbDeleteMetaRecord(SMetaFile *mfh, uint64_t uid); +int32_t tsdbUpdateMetaRecord(SMetaFile *mfh, uint64_t uid, void *cont, int32_t contLen); void tsdbCloseMetaFile(SMetaFile *mfh); // ------------------------------ TSDB META INTERFACES ------------------------------ @@ -82,7 +82,7 @@ typedef struct { typedef struct STable { int8_t type; STableId tableId; - int64_t superUid; // Super table UID + uint64_t superUid; // Super table UID int32_t sversion; STSchema * schema; STSchema * tagSchema; @@ -153,7 +153,7 @@ STsdbMeta *tsdbGetMeta(TsdbRepoT *pRepo); STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId); // int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable); -STable *tsdbGetTableByUid(STsdbMeta *pMeta, int64_t uid); +STable *tsdbGetTableByUid(STsdbMeta *pMeta, uint64_t uid); char *getTSTupleKey(const void * data); typedef struct { @@ -275,8 +275,8 @@ typedef struct { uint32_t padding; // For padding purpose uint32_t hasLast : 2; uint32_t numOfBlocks : 30; - int64_t uid; - TSKEY maxKey; + uint64_t uid; + TSKEY maxKey; } SCompIdx; /* sizeof(SCompIdx) = 28 */ void *tsdbEncodeSCompIdx(void *buf, SCompIdx *pIdx); @@ -311,7 +311,7 @@ typedef struct { typedef struct { int32_t delimiter; // For recovery usage int32_t checksum; // TODO: decide if checksum logic in this file or make it one API - int64_t uid; + uint64_t uid; SCompBlock blocks[]; } SCompInfo; @@ -345,7 +345,7 @@ typedef struct { typedef struct { int32_t delimiter; // For recovery usage int32_t numOfCols; // For recovery usage - int64_t uid; // For recovery usage + uint64_t uid; // For recovery usage SCompCol cols[]; } SCompData; @@ -441,9 +441,9 @@ typedef struct { } SHelperFile; typedef struct { - int64_t uid; - int32_t tid; - int32_t sversion; + uint64_t uid; + int32_t tid; + int32_t sversion; } SHelperTable; typedef struct { diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 7254318d74..36cb937d33 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -392,7 +392,7 @@ int tsdbAlterTable(TsdbRepoT *pRepo, STableCfg *pCfg) { return 0; } -TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, int64_t uid) { +TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, uint64_t uid) { STsdbRepo *pRepo = (STsdbRepo *)repo; STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, uid); @@ -430,7 +430,7 @@ int32_t tsdbInsertData(TsdbRepoT *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg * p /** * Initialize a table configuration */ -int tsdbInitTableCfg(STableCfg *config, ETableType type, int64_t uid, int32_t tid) { +int tsdbInitTableCfg(STableCfg *config, ETableType type, uint64_t uid, int32_t tid) { if (config == NULL) return -1; if (type != TSDB_NORMAL_TABLE && type != TSDB_CHILD_TABLE) return -1; @@ -447,7 +447,7 @@ int tsdbInitTableCfg(STableCfg *config, ETableType type, int64_t uid, int32_t ti /** * Set the super table UID of the created table */ -int tsdbTableSetSuperUid(STableCfg *config, int64_t uid) { +int tsdbTableSetSuperUid(STableCfg *config, uint64_t uid) { if (config->type != TSDB_CHILD_TABLE) return -1; if (uid == TSDB_INVALID_SUPER_TABLE_ID) return -1; diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index f502c1babd..0a0a975fec 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -87,9 +87,9 @@ STable *tsdbDecodeTable(void *cont, int contLen) { memcpy(pTable->name->data, ptr, len); ptr = (char *)ptr + len; - T_READ_MEMBER(ptr, int64_t, pTable->tableId.uid); + T_READ_MEMBER(ptr, uint64_t, pTable->tableId.uid); T_READ_MEMBER(ptr, int32_t, pTable->tableId.tid); - T_READ_MEMBER(ptr, int64_t, pTable->superUid); + T_READ_MEMBER(ptr, uint64_t, pTable->superUid); T_READ_MEMBER(ptr, int32_t, pTable->sversion); if (pTable->type == TSDB_SUPER_TABLE) { @@ -455,7 +455,7 @@ static int32_t tsdbCheckTableCfg(STableCfg *pCfg) { return 0; } -STable *tsdbGetTableByUid(STsdbMeta *pMeta, int64_t uid) { +STable *tsdbGetTableByUid(STsdbMeta *pMeta, uint64_t uid) { void *ptr = taosHashGet(pMeta->map, (char *)(&uid), sizeof(uid)); if (ptr == NULL) return NULL; diff --git a/src/tsdb/src/tsdbMetaFile.c b/src/tsdb/src/tsdbMetaFile.c index f9d10ec579..19fcae94e3 100644 --- a/src/tsdb/src/tsdbMetaFile.c +++ b/src/tsdb/src/tsdbMetaFile.c @@ -23,9 +23,9 @@ #define TSDB_META_FILE_HEADER_SIZE 512 typedef struct { - int32_t offset; - int32_t size; - int64_t uid; + int32_t offset; + int32_t size; + uint64_t uid; } SRecordInfo; // static int32_t tsdbGetMetaFileName(char *rootDir, char *fname); @@ -76,7 +76,7 @@ SMetaFile *tsdbInitMetaFile(char *rootDir, int32_t maxTables, iterFunc iFunc, af return mfh; } -int32_t tsdbInsertMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t contLen) { +int32_t tsdbInsertMetaRecord(SMetaFile *mfh, uint64_t uid, void *cont, int32_t contLen) { if (taosHashGet(mfh->map, (char *)(&uid), sizeof(uid)) != NULL) { return -1; } @@ -112,7 +112,7 @@ int32_t tsdbInsertMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t co return 0; } -int32_t tsdbDeleteMetaRecord(SMetaFile *mfh, int64_t uid) { +int32_t tsdbDeleteMetaRecord(SMetaFile *mfh, uint64_t uid) { char *ptr = taosHashGet(mfh->map, (char *)(&uid), sizeof(uid)); if (ptr == NULL) return -1; @@ -139,7 +139,7 @@ int32_t tsdbDeleteMetaRecord(SMetaFile *mfh, int64_t uid) { return 0; } -int32_t tsdbUpdateMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t contLen) { +int32_t tsdbUpdateMetaRecord(SMetaFile *mfh, uint64_t uid, void *cont, int32_t contLen) { char *ptr = taosHashGet(mfh->map, (char *)(&uid), sizeof(uid)); if (ptr == NULL) return -1; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index a5cba70219..d6a070589d 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1516,8 +1516,9 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) return TSDB_CODE_SUCCESS; } -int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, int64_t uid, const char *pTagCond, size_t len, int16_t tagNameRelType, - const char* tbnameCond, STableGroupInfo *pGroupInfo, SColIndex *pColIndex, int32_t numOfCols) { +int32_t tsdbQuerySTableByTagCond(TsdbRepoT* tsdb, uint64_t uid, const char* pTagCond, size_t len, + int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo, + SColIndex* pColIndex, int32_t numOfCols) { STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); if (pTable == NULL) { uError("%p failed to get stable, uid:%" PRIu64, tsdb, uid); @@ -1589,7 +1590,7 @@ int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, int64_t uid, const char *pTagC return ret; } -int32_t tsdbGetOneTableGroup(TsdbRepoT* tsdb, int64_t uid, STableGroupInfo* pGroupInfo) { +int32_t tsdbGetOneTableGroup(TsdbRepoT* tsdb, uint64_t uid, STableGroupInfo* pGroupInfo) { STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); if (pTable == NULL) { return TSDB_CODE_INVALID_TABLE_ID; diff --git a/src/tsdb/tests/tsdbTests.cpp b/src/tsdb/tests/tsdbTests.cpp index c7ed6fcae1..e26caa4ae7 100644 --- a/src/tsdb/tests/tsdbTests.cpp +++ b/src/tsdb/tests/tsdbTests.cpp @@ -16,7 +16,7 @@ typedef struct { TsdbRepoT *pRepo; bool isAscend; int tid; - int64_t uid; + uint64_t uid; int sversion; TSKEY startTime; TSKEY interval; From 50e24fd9d137dcdca9bc735c836db8b2c28f1f9f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 18 May 2020 05:43:39 +0000 Subject: [PATCH 3/7] fix compile error --- src/inc/tsdb.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 050b65153e..729ed17053 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -278,9 +278,9 @@ SArray *tsdbGetTableList(TsdbQueryHandleT *pQueryHandle); * @param stableid. super table sid * @param pTagCond. tag query condition */ -int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, int64_t uid, const char *pTagCond, size_t len, - int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo *pGroupList, SColIndex *pColIndex, int32_t numOfCols); - +int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, uint64_t uid, const char *pTagCond, size_t len, + int16_t tagNameRelType, const char *tbnameCond, STableGroupInfo *pGroupList, + SColIndex *pColIndex, int32_t numOfCols); /** * create the table group result including only one table, used to handle the normal table query @@ -290,7 +290,7 @@ int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, int64_t uid, const char *pTagC * @param pGroupInfo the generated result * @return */ -int32_t tsdbGetOneTableGroup(TsdbRepoT *tsdb, int64_t uid, STableGroupInfo *pGroupInfo); +int32_t tsdbGetOneTableGroup(TsdbRepoT *tsdb, uint64_t uid, STableGroupInfo *pGroupInfo); /** * clean up the query handle From 520e97be3aeee8ca7feb11d94a5f47acb48e9c26 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 18 May 2020 08:32:54 +0000 Subject: [PATCH 4/7] fix maxtables problem --- src/tsdb/inc/tsdbMain.h | 3 ++- src/tsdb/src/tsdbFile.c | 31 +++++++++++++------------- src/tsdb/src/tsdbRWHelper.c | 43 +++++++++++++++++++------------------ 3 files changed, 39 insertions(+), 38 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index e7537f6969..8a0a9e1208 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -465,7 +465,7 @@ typedef struct { SCompData *pCompData; SDataCols *pDataCols[2]; - void *blockBuffer; // Buffer to hold the whole data block + void *pBuffer; // Buffer to hold the whole data block void *compBuffer; // Buffer for temperary compress/decompress purpose } SRWHelper; @@ -512,6 +512,7 @@ void tsdbFitRetention(STsdbRepo *pRepo); int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks); void tsdbAdjustCacheBlocks(STsdbCache *pCache); int32_t tsdbGetMetaFileName(char *rootDir, char *fname); +int tsdbUpdateFileHeader(SFile *pFile, uint32_t version); #ifdef __cplusplus } diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index c9db9344c9..50ef078a0c 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -37,7 +37,6 @@ const char *tsdbFileSuffix[] = { static int compFGroupKey(const void *key, const void *fgroup); static int compFGroup(const void *arg1, const void *arg2); -static int tsdbWriteFileHead(SFile *pFile); static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid); STsdbFileH *tsdbInitFileH(char *dataDir, STsdbCfg *pCfg) { @@ -83,11 +82,21 @@ void tsdbCloseFileH(STsdbFileH *pFileH) { } static int tsdbInitFile(char *dataDir, int fid, const char *suffix, SFile *pFile) { + uint32_t version; + char buf[512] = "\0"; + tsdbGetFileName(dataDir, fid, suffix, pFile->fname); if (access(pFile->fname, F_OK|R_OK|W_OK) < 0) return -1; pFile->fd = -1; - // TODO: recover the file info - // pFile->info = {0}; + if (tsdbOpenFile(pFile, O_RDONLY) < 0) return -1; + + if (tread(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) return -1; + if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) return -1; + + void *pBuf = buf; + pBuf = taosDecodeFixed32(pBuf, &version); + pBuf = tsdbDecodeSFileInfo(pBuf, &(pFile->info)); + return 0; } @@ -284,18 +293,6 @@ static int compFGroup(const void *arg1, const void *arg2) { return ((SFileGroup *)arg1)->fileId - ((SFileGroup *)arg2)->fileId; } -static int tsdbWriteFileHead(SFile *pFile) { - char head[TSDB_FILE_HEAD_SIZE] = "\0"; - - pFile->info.size += TSDB_FILE_HEAD_SIZE; - - // TODO: write version and File statistic to the head - lseek(pFile->fd, 0, SEEK_SET); - if (write(pFile->fd, head, TSDB_FILE_HEAD_SIZE) < 0) return -1; - - return 0; -} - int tsdbGetFileName(char *dataDir, int fileId, const char *suffix, char *fname) { if (dataDir == NULL || fname == NULL) return -1; @@ -345,7 +342,9 @@ int tsdbCreateFile(char *dataDir, int fileId, const char *suffix, SFile *pFile) return -1; } - if (tsdbWriteFileHead(pFile) < 0) { + pFile->info.size = TSDB_FILE_HEAD_SIZE; + + if (tsdbUpdateFileHeader(pFile, 0) < 0) { tsdbCloseFile(pFile); return -1; } diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 4b3c91bb02..79507aa15b 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -132,10 +132,10 @@ static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t // Init block part if (tsdbInitHelperBlock(pHelper) < 0) goto _err; - pHelper->blockBuffer = + pHelper->pBuffer = tmalloc(sizeof(SCompData) + (sizeof(SCompCol) + sizeof(TSCKSUM) + COMP_OVERFLOW_BYTES) * pHelper->config.maxCols + pHelper->config.maxRowSize * pHelper->config.maxRowsPerFileBlock + sizeof(TSCKSUM)); - if (pHelper->blockBuffer == NULL) goto _err; + if (pHelper->pBuffer == NULL) goto _err; return 0; @@ -155,7 +155,7 @@ int tsdbInitWriteHelper(SRWHelper *pHelper, STsdbRepo *pRepo) { void tsdbDestroyHelper(SRWHelper *pHelper) { if (pHelper) { - tzfree(pHelper->blockBuffer); + tzfree(pHelper->pBuffer); tzfree(pHelper->compBuffer); tsdbDestroyHelperFile(pHelper); tsdbDestroyHelperTable(pHelper); @@ -241,6 +241,7 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { pHelper->files.headF.fd = -1; } if (pHelper->files.dataF.fd > 0) { + if (!hasError) tsdbUpdateFileHeader(&(pHelper->files.dataF), 0); close(pHelper->files.dataF.fd); pHelper->files.dataF.fd = -1; } @@ -249,6 +250,7 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { pHelper->files.lastF.fd = -1; } if (pHelper->files.nHeadF.fd > 0) { + if (!hasError) tsdbUpdateFileHeader(&(pHelper->files.nHeadF), 0); close(pHelper->files.nHeadF.fd); pHelper->files.nHeadF.fd = -1; if (hasError) { @@ -260,6 +262,7 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { } if (pHelper->files.nLastF.fd > 0) { + if (!hasError) tsdbUpdateFileHeader(&(pHelper->files.nLastF), 0); close(pHelper->files.nLastF.fd); pHelper->files.nLastF.fd = -1; if (hasError) { @@ -419,7 +422,7 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { pIdx->offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END); pIdx->uid = pHelper->tableInfo.uid; if (pIdx->offset < 0) return -1; - ASSERT(pIdx->offset >= tsizeof(pHelper->pCompIdx)); + ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); if (twrite(pHelper->files.nHeadF.fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) return -1; } @@ -435,7 +438,8 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { SFile *pFile = &(pHelper->files.nHeadF); pFile->info.offset = offset; - void *buf = pHelper->blockBuffer; + // TODO: change the implementation of pHelper->pBuffer + void *buf = pHelper->pBuffer; for (uint32_t i = 0; i < pHelper->config.maxTables; i++) { SCompIdx *pCompIdx = pHelper->pCompIdx + i; if (pCompIdx->offset > 0) { @@ -444,10 +448,10 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { } } - int tsize = (char *)buf - (char *)pHelper->blockBuffer + sizeof(TSCKSUM); - taosCalcChecksumAppend(0, (uint8_t *)pHelper->blockBuffer, tsize); + int tsize = (char *)buf - (char *)pHelper->pBuffer + sizeof(TSCKSUM); + taosCalcChecksumAppend(0, (uint8_t *)pHelper->pBuffer, tsize); - if (twrite(pHelper->files.nHeadF.fd, (void *)pHelper->blockBuffer, tsize) < tsize) return -1; + if (twrite(pHelper->files.nHeadF.fd, (void *)pHelper->pBuffer, tsize) < tsize) return -1; pFile->info.len = tsize; return 0; } @@ -465,23 +469,23 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { ASSERT(pFile->info.offset > TSDB_FILE_HEAD_SIZE); if (lseek(fd, pFile->info.offset, SEEK_SET) < 0) return -1; - if (tread(fd, (void *)(pHelper->blockBuffer), pFile->info.len) < pFile->info.len) + if (tread(fd, (void *)(pHelper->pBuffer), pFile->info.len) < pFile->info.len) return -1; - if (!taosCheckChecksumWhole((uint8_t *)(pHelper->blockBuffer), pFile->info.len)) { + if (!taosCheckChecksumWhole((uint8_t *)(pHelper->pBuffer), pFile->info.len)) { // TODO: File is broken, try to deal with it return -1; } // Decode it - void *ptr = pHelper->blockBuffer; - while ((char *)ptr - (char *)pHelper->blockBuffer >= pFile->info.len - sizeof(TSCKSUM)) { + void *ptr = pHelper->pBuffer; + while ((char *)ptr - (char *)pHelper->pBuffer >= pFile->info.len - sizeof(TSCKSUM)) { uint32_t tid = 0; if ((ptr = taosDecodeVariant32(ptr, &tid)) == NULL) return -1; ASSERT(tid > 0 && tid < pHelper->config.maxTables); if ((ptr = tsdbDecodeSCompIdx(ptr, pHelper->pCompIdx + tid)) == NULL) return -1; - ASSERT((char *)ptr - (char *)pHelper->blockBuffer <= pFile->info.len - sizeof(TSCKSUM)); + ASSERT((char *)ptr - (char *)pHelper->pBuffer <= pFile->info.len - sizeof(TSCKSUM)); } } @@ -626,9 +630,9 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32 static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols) { ASSERT(pCompBlock->numOfSubBlocks <= 1); - ASSERT(tsizeof(pHelper->blockBuffer) >= pCompBlock->len); + ASSERT(tsizeof(pHelper->pBuffer) >= pCompBlock->len); - SCompData *pCompData = (SCompData *)pHelper->blockBuffer; + SCompData *pCompData = (SCompData *)pHelper->pBuffer; int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd; if (lseek(fd, pCompBlock->offset, SEEK_SET) < 0) goto _err; @@ -721,7 +725,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa ASSERT(rowsToWrite > 0 && rowsToWrite <= pDataCols->numOfPoints && rowsToWrite <= pHelper->config.maxRowsPerFileBlock); - SCompData *pCompData = (SCompData *)(pHelper->blockBuffer); + SCompData *pCompData = (SCompData *)(pHelper->pBuffer); int64_t offset = 0; offset = lseek(pFile->fd, 0, SEEK_END); @@ -776,7 +780,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa } pCompCol->len = (*(tDataTypeDesc[pDataCol->type].compFunc))( - (char *)pDataCol->pData, tlen, rowsToWrite, tptr, tsizeof(pHelper->blockBuffer) - lsize, + (char *)pDataCol->pData, tlen, rowsToWrite, tptr, tsizeof(pHelper->pBuffer) - lsize, pHelper->config.compress, pHelper->compBuffer, tsizeof(pHelper->compBuffer)); } else { pCompCol->len = tlen; @@ -1242,11 +1246,8 @@ int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) { void *pBuf = (void *)buf; pBuf = taosEncodeFixed32(pBuf, version); pBuf = tsdbEncodeSFileInfo(pBuf, &(pFile->info)); - int tsize = (char *)pBuf - buf + sizeof(TSCKSUM); - ASSERT(tsize <= TSDB_FILE_HEAD_SIZE); - - taosCalcChecksumAppend(0, (uint8_t *)buf, tsize); + taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE); if (lseek(pFile->fd, 0, SEEK_SET) < 0) return -1; if (twrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) return -1; From 26614ff964163602c033f50bc4053bb27a0b863d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 18 May 2020 08:53:33 +0000 Subject: [PATCH 5/7] fix coredump --- src/tsdb/src/tsdbFile.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 50ef078a0c..dcea2737ff 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -97,6 +97,8 @@ static int tsdbInitFile(char *dataDir, int fid, const char *suffix, SFile *pFile pBuf = taosDecodeFixed32(pBuf, &version); pBuf = tsdbDecodeSFileInfo(pBuf, &(pFile->info)); + tsdbCloseFile(pFile); + return 0; } From 17139e19f54244c87fc23322df99b1b0bd64cae4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 18 May 2020 09:37:02 +0000 Subject: [PATCH 6/7] fix read error --- src/tsdb/src/tsdbRWHelper.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 79507aa15b..c5ef00c233 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -478,7 +478,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { // Decode it void *ptr = pHelper->pBuffer; - while ((char *)ptr - (char *)pHelper->pBuffer >= pFile->info.len - sizeof(TSCKSUM)) { + while (((char *)ptr - (char *)pHelper->pBuffer) < (pFile->info.len - sizeof(TSCKSUM))) { uint32_t tid = 0; if ((ptr = taosDecodeVariant32(ptr, &tid)) == NULL) return -1; ASSERT(tid > 0 && tid < pHelper->config.maxTables); @@ -487,6 +487,8 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { ASSERT((char *)ptr - (char *)pHelper->pBuffer <= pFile->info.len - sizeof(TSCKSUM)); } + + ASSERT(((char *)ptr - (char *)pHelper->pBuffer) == (pFile->info.len - sizeof(TSCKSUM))); } } From 62c1f2f7b565542965834adc7f47df6576f21d50 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 18 May 2020 11:15:04 +0000 Subject: [PATCH 7/7] TD-308 --- src/tsdb/src/tsdbMeta.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 0a0a975fec..5516231483 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -508,10 +508,7 @@ static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable) { ASSERT(tTable != NULL && tTable->type == TSDB_CHILD_TABLE); - pMeta->tables[tTable->tableId.tid] = NULL; - taosHashRemove(pMeta->map, (char *)(&(pTable->tableId.uid)), sizeof(pTable->tableId.uid)); - pMeta->nTables--; - tsdbFreeTable(tTable); + tsdbRemoveTableFromMeta(pMeta, tTable); } tSkipListDestroyIter(pIter); @@ -534,8 +531,8 @@ static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable) { pMeta->nTables--; } - tsdbFreeTable(pTable); taosHashRemove(pMeta->map, (char *)(&(pTable->tableId.uid)), sizeof(pTable->tableId.uid)); + tsdbFreeTable(pTable); return 0; }