From 253c7e64d6ed284e8f68bde04378f6b753f7f2c2 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Tue, 7 Apr 2020 18:46:09 +0800 Subject: [PATCH 01/29] first version of committing data --- src/common/inc/dataformat.h | 1 + src/common/src/dataformat.c | 3 +- src/vnode/tsdb/inc/tsdbMain.h | 160 ++++- src/vnode/tsdb/src/tsdbFile.c | 124 ++-- src/vnode/tsdb/src/tsdbMain.c | 510 ++++++-------- src/vnode/tsdb/src/tsdbRWHelper.c | 1057 ++++++++++++++++++++++++++++ src/vnode/tsdb/src/tsdbRead.c | 22 +- src/vnode/tsdb/tests/tsdbTests.cpp | 47 +- 8 files changed, 1492 insertions(+), 432 deletions(-) create mode 100644 src/vnode/tsdb/src/tsdbRWHelper.c diff --git a/src/common/inc/dataformat.h b/src/common/inc/dataformat.h index 04fa7dcc7d..231786ff73 100644 --- a/src/common/inc/dataformat.h +++ b/src/common/inc/dataformat.h @@ -136,6 +136,7 @@ void tdInitDataCols(SDataCols *pCols, STSchema *pSchema); void tdFreeDataCols(SDataCols *pCols); void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols); void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop); +int tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge); #ifdef __cplusplus } diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c index bff041df1b..1f5d83d9af 100644 --- a/src/common/src/dataformat.c +++ b/src/common/src/dataformat.c @@ -382,6 +382,7 @@ static int tdFLenFromSchema(STSchema *pSchema) { return ret; } -int tdMergeDataCols(SDataCols *target, SDataCols *source) { +int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) { + // TODO return 0; } \ No newline at end of file diff --git a/src/vnode/tsdb/inc/tsdbMain.h b/src/vnode/tsdb/inc/tsdbMain.h index 06f62ea6f7..54f5ab0cce 100644 --- a/src/vnode/tsdb/inc/tsdbMain.h +++ b/src/vnode/tsdb/inc/tsdbMain.h @@ -90,9 +90,9 @@ typedef struct { STable *superList; // super table list TODO: change it to list container - void *map; // table map of (uid ===> table) + void *map; // table map of (uid ===> table) - SMetaFile *mfh; // meta file handle + SMetaFile *mfh; // meta file handle int maxRowBytes; int maxCols; } STsdbMeta; @@ -118,14 +118,14 @@ STSchema * tsdbGetTableSchema(STsdbMeta *pMeta, STable *pTable); #define TSDB_TABLE_OF_ID(pHandle, id) ((pHandle)->pTables)[id] #define TSDB_GET_TABLE_OF_NAME(pHandle, name) /* TODO */ -STsdbMeta* tsdbGetMeta(tsdb_repo_t* pRepo); +STsdbMeta *tsdbGetMeta(tsdb_repo_t *pRepo); int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg); int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId); STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId); // int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable); STable *tsdbGetTableByUid(STsdbMeta *pMeta, int64_t uid); -char *getTupleKey(const void * data); +char * getTupleKey(const void *data); // ------------------------------ TSDB CACHE INTERFACES ------------------------------ #define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 * 1024 * 1024 /* 16M */ @@ -191,8 +191,8 @@ typedef struct { } SFileInfo; typedef struct { - int fd; - char fname[128]; + int fd; + char fname[128]; SFileInfo info; } SFile; @@ -216,11 +216,14 @@ typedef struct { STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles); void tsdbCloseFileH(STsdbFileH *pFileH); -int tsdbCreateFile(char *dataDir, int fileId, const char *suffix, int maxTables, SFile *pFile, int writeHeader, int toClose); -int tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables); +int tsdbCreateFile(char *dataDir, int fileId, const char *suffix, int maxTables, SFile *pFile, int writeHeader, + int toClose); +SFileGroup *tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables); int tsdbOpenFile(SFile *pFile, int oflag); -int tsdbCloseFile(SFile *pFile); SFileGroup *tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid); +int tsdbCloseFile(SFile *pFile); +SFileGroup *tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid); int tsdbRemoveFileGroup(STsdbFileH *pFile, int fid); +int tsdbGetFileName(char *dataDir, int fileId, const char *suffix, char *fname); #define TSDB_FGROUP_ITER_FORWARD 0 #define TSDB_FGROUP_ITER_BACKWARD 1 @@ -265,6 +268,8 @@ typedef struct { TSKEY keyLast; } SCompBlock; +// Maximum number of sub-blocks a super-block can have +#define TSDB_MAX_SUBBLOCKS 8 #define IS_SUPER_BLOCK(pBlock) ((pBlock)->numOfSubBlocks >= 1) #define IS_SUB_BLOCK(pBlock) ((pBlock)->numOfSubBlocks == 0) @@ -276,15 +281,15 @@ typedef struct { } SCompInfo; #define TSDB_COMPBLOCK_AT(pCompInfo, idx) ((pCompInfo)->blocks + (idx)) -#define TSDB_COMPBLOCK_GET_START_AND_SIZE(pCompInfo, pCompBlock, size)\ -do {\ - if (pCompBlock->numOfSubBlocks > 1) {\ - pCompBlock = pCompInfo->blocks + pCompBlock->offset;\ - size = pCompBlock->numOfSubBlocks;\ - } else {\ - size = 1;\ - }\ -} while (0) +#define TSDB_COMPBLOCK_GET_START_AND_SIZE(pCompInfo, pCompBlock, size) \ + do { \ + if (pCompBlock->numOfSubBlocks > 1) { \ + pCompBlock = pCompInfo->blocks + pCompBlock->offset; \ + size = pCompBlock->numOfSubBlocks; \ + } else { \ + size = 1; \ + } \ + } while (0) // TODO: take pre-calculation into account typedef struct { @@ -302,18 +307,11 @@ typedef struct { SCompCol cols[]; } SCompData; -STsdbFileH* tsdbGetFile(tsdb_repo_t* pRepo); - -int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInfo, int idx, int isLast, SDataCols *pCols); - -int tsdbLoadCompIdx(SFileGroup *pGroup, void *buf, int maxTables); -int tsdbLoadCompBlocks(SFileGroup *pGroup, SCompIdx *pIdx, void *buf); -int tsdbLoadCompCols(SFile *pFile, SCompBlock *pBlock, void *buf); -int tsdbLoadColData(SFile *pFile, SCompCol *pCol, int64_t blockBaseOffset, void *buf); -int tsdbLoadDataBlock(SFile *pFile, SCompBlock *pStartBlock, int numOfBlocks, SDataCols *pCols, SCompData *pCompData); +STsdbFileH *tsdbGetFile(tsdb_repo_t *pRepo); +int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInfo, int idx, int isLast, + SDataCols *pCols); SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid); - void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, TSKEY *maxKey); // TSDB repository definition @@ -348,6 +346,112 @@ typedef struct _tsdb_repo { } STsdbRepo; +typedef enum { TSDB_WRITE_HELPER, TSDB_READ_HELPER } tsdb_rw_helper_t; + +typedef struct { + tsdb_rw_helper_t type; // helper type + int maxTables; + int maxRowSize; + int maxRows; + int maxCols; + int minRowsPerFileBlock; + int maxRowsPerFileBlock; + int8_t compress; +} SHelperCfg; + +typedef struct { + int fid; + TSKEY minKey; + TSKEY maxKey; + // For read/write purpose + SFile headF; + SFile dataF; + SFile lastF; + // For write purpose only + SFile nHeadF; + SFile nLastF; +} SHelperFile; + +typedef struct { + int64_t uid; + int32_t tid; + int32_t sversion; +} SHelperTable; + +typedef struct { + // Global configuration + SHelperCfg config; + + SHelperFile files; + + SHelperTable tableInfo; + + // ---------- For read purpose + int8_t state; // current loading state + + SCompIdx *pCompIdx; + size_t compIdxSize; + + SCompInfo *pCompInfo; + size_t compInfoSize; + int blockIter; // For write purpose + + SCompData *pCompData; + size_t compDataSize; + + SDataCols *pDataCols[2]; + + // ---------- For read purpose + bool hasLast; + + int newBlocks; + SCompIdx *pWCompIdx; + size_t wCompIdxSize; + + SCompInfo *pWCompInfo; + size_t wCompInfoSize; + + SCompData *pWCompData; + size_t wCompDataSize; +} SRWHelper; + +// --------- Helper state +#define TSDB_HELPER_CLEAR_STATE 0x0 // Clear state +#define TSDB_HELPER_FILE_SET 0x1 // File is set +#define TSDB_HELPER_FILE_OPEN 0x2 // File is opened + +#define TSDB_HELPER_IDX_LOAD 0x4 // SCompIdx part is loaded +#define TSDB_HELPER_INFO_LOAD 0x8 // SCompInfo part is loaded +#define TSDB_HELPER_FILE_DATA_LOAD 0x10 // SCompData part is loaded + +#define TSDB_HELPER_TYPE(h) ((h)->config.type) + +#define helperSetState(h, s) (((h)->state) |= (s)) +#define helperClearState(h, s) ((h)->state &= (~(s))) +#define helperHasState(h, s) ((((h)->state) & (s)) == (s)) + +int tsdbInitHelper(SRWHelper *pHelper, SHelperCfg *pCfg); +void tsdbDestroyHelper(SRWHelper *pHelper); +void tsdbClearHelper(SRWHelper *pHelper); + +// --------- For set operations +int tsdbSetHelperFile(SRWHelper *pHelper, SFileGroup *pGroup); +int tsdbOpenHelperFile(SRWHelper *pHelper); +void tsdbSetHelperTable(SRWHelper *pHelper, SHelperTable *pHelperTable, STSchema *pSchema); +int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError); + +// --------- For read operations +int tsdbLoadCompIdx(SRWHelper *pHelper, void *target); +int tsdbLoadCompInfo(SRWHelper *pHelper, void *target); +int tsdbLoadCompData(SRWHelper *pHelper, int blkIdx, void *target); +int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int32_t *colIds, int numOfColIds); +int tsdbLoadBlockData(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols); + +// --------- For write operations +int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols); +int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper); +int tsdbWriteCompInfo(SRWHelper *pHelper); +int tsdbWriteCompIdx(SRWHelper *pHelper); #ifdef __cplusplus } diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c index bd6699eb84..b6da572cfb 100644 --- a/src/vnode/tsdb/src/tsdbFile.c +++ b/src/vnode/tsdb/src/tsdbFile.c @@ -21,6 +21,7 @@ #include #include #include +#include #include "tutil.h" #include "tsdbMain.h" @@ -33,7 +34,6 @@ const char *tsdbFileSuffix[] = { static int compFGroupKey(const void *key, const void *fgroup); static int compFGroup(const void *arg1, const void *arg2); -static int tsdbGetFileName(char *dataDir, int fileId, const char *suffix, char *fname); static int tsdbWriteFileHead(SFile *pFile); static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables); static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid); @@ -91,24 +91,36 @@ static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid) { return 0; } -int tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables) { - if (pFileH->numOfFGroups >= pFileH->maxFGroups) return -1; +/** + * Create the file group if the file group not exists. + * + * @return A pointer to + */ +SFileGroup *tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables) { + if (pFileH->numOfFGroups >= pFileH->maxFGroups) return NULL; SFileGroup fGroup; SFileGroup *pFGroup = &fGroup; - if (tsdbSearchFGroup(pFileH, fid) == NULL) { // if not exists, create one + + SFileGroup *pGroup = tsdbSearchFGroup(pFileH, fid); + if (pGroup == NULL) { // if not exists, create one pFGroup->fileId = fid; for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { - if (tsdbCreateFile(dataDir, fid, tsdbFileSuffix[type], maxTables, &(pFGroup->files[type]), type == TSDB_FILE_TYPE_HEAD ? 1 : 0, 1) < 0) { - // TODO: deal with the ERROR here, remove those creaed file - return -1; - } + if (tsdbCreateFile(dataDir, fid, tsdbFileSuffix[type], maxTables, &(pFGroup->files[type]), + type == TSDB_FILE_TYPE_HEAD ? 1 : 0, 1) < 0) + goto _err; } pFileH->fGroup[pFileH->numOfFGroups++] = fGroup; qsort((void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroup); + return tsdbSearchFGroup(pFileH, fid); } - return 0; + + return pGroup; + +_err: + // TODO: deal with the err here + return NULL; } int tsdbRemoveFileGroup(STsdbFileH *pFileH, int fid) { @@ -181,27 +193,27 @@ SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter) { return ret; } -int tsdbLoadDataBlock(SFile *pFile, SCompBlock *pStartBlock, int numOfBlocks, SDataCols *pCols, SCompData *pCompData) { - SCompBlock *pBlock = pStartBlock; - for (int i = 0; i < numOfBlocks; i++) { - if (tsdbLoadCompCols(pFile, pBlock, (void *)pCompData) < 0) return -1; - pCols->numOfPoints += (pCompData->cols[0].len / 8); - for (int iCol = 0; iCol < pBlock->numOfCols; iCol++) { - SCompCol *pCompCol = &(pCompData->cols[iCol]); - // pCols->numOfPoints += pBlock->numOfPoints; - int k = 0; - for (; k < pCols->numOfCols; k++) { - if (pCompCol->colId == pCols->cols[k].colId) break; - } +// int tsdbLoadDataBlock(SFile *pFile, SCompBlock *pStartBlock, int numOfBlocks, SDataCols *pCols, SCompData *pCompData) { +// SCompBlock *pBlock = pStartBlock; +// for (int i = 0; i < numOfBlocks; i++) { +// if (tsdbLoadCompCols(pFile, pBlock, (void *)pCompData) < 0) return -1; +// pCols->numOfPoints += (pCompData->cols[0].len / 8); +// for (int iCol = 0; iCol < pBlock->numOfCols; iCol++) { +// SCompCol *pCompCol = &(pCompData->cols[iCol]); +// // pCols->numOfPoints += pBlock->numOfPoints; +// int k = 0; +// for (; k < pCols->numOfCols; k++) { +// if (pCompCol->colId == pCols->cols[k].colId) break; +// } - if (tsdbLoadColData(pFile, pCompCol, pBlock->offset, - (void *)((char *)(pCols->cols[k].pData) + pCols->cols[k].len)) < 0) - return -1; - } - pStartBlock++; - } - return 0; -} +// if (tsdbLoadColData(pFile, pCompCol, pBlock->offset, +// (void *)((char *)(pCols->cols[k].pData) + pCols->cols[k].len)) < 0) +// return -1; +// } +// pStartBlock++; +// } +// return 0; +// } int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInfo, int idx, int isLast, SDataCols *pCols) { SCompBlock *pSuperBlock = TSDB_COMPBLOCK_AT(pCompInfo, idx); @@ -237,42 +249,42 @@ int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInf return 0; } -int tsdbLoadCompIdx(SFileGroup *pGroup, void *buf, int maxTables) { - SFile *pFile = &(pGroup->files[TSDB_FILE_TYPE_HEAD]); - if (lseek(pFile->fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1; +// int tsdbLoadCompIdx(SFileGroup *pGroup, void *buf, int maxTables) { +// SFile *pFile = &(pGroup->files[TSDB_FILE_TYPE_HEAD]); +// if (lseek(pFile->fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1; - if (read(pFile->fd, buf, sizeof(SCompIdx) * maxTables) < 0) return -1; - // TODO: need to check the correctness - return 0; -} +// if (read(pFile->fd, buf, sizeof(SCompIdx) * maxTables) < 0) return -1; +// // TODO: need to check the correctness +// return 0; +// } -int tsdbLoadCompBlocks(SFileGroup *pGroup, SCompIdx *pIdx, void *buf) { - SFile *pFile = &(pGroup->files[TSDB_FILE_TYPE_HEAD]); +// int tsdbLoadCompBlocks(SFileGroup *pGroup, SCompIdx *pIdx, void *buf) { +// SFile *pFile = &(pGroup->files[TSDB_FILE_TYPE_HEAD]); - if (lseek(pFile->fd, pIdx->offset, SEEK_SET) < 0) return -1; +// if (lseek(pFile->fd, pIdx->offset, SEEK_SET) < 0) return -1; - if (read(pFile->fd, buf, pIdx->len) < 0) return -1; +// if (read(pFile->fd, buf, pIdx->len) < 0) return -1; - // TODO: need to check the correctness +// // TODO: need to check the correctness - return 0; -} +// return 0; +// } -int tsdbLoadCompCols(SFile *pFile, SCompBlock *pBlock, void *buf) { - // assert(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1); +// int tsdbLoadCompCols(SFile *pFile, SCompBlock *pBlock, void *buf) { +// // assert(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1); - if (lseek(pFile->fd, pBlock->offset, SEEK_SET) < 0) return -1; - size_t size = sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols; - if (read(pFile->fd, buf, size) < 0) return -1; +// if (lseek(pFile->fd, pBlock->offset, SEEK_SET) < 0) return -1; +// size_t size = sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols; +// if (read(pFile->fd, buf, size) < 0) return -1; - return 0; -} +// return 0; +// } -int tsdbLoadColData(SFile *pFile, SCompCol *pCol, int64_t blockBaseOffset, void *buf) { - if (lseek(pFile->fd, blockBaseOffset + pCol->offset, SEEK_SET) < 0) return -1; - if (read(pFile->fd, buf, pCol->len) < 0) return -1; - return 0; -} +// int tsdbLoadColData(SFile *pFile, SCompCol *pCol, int64_t blockBaseOffset, void *buf) { +// if (lseek(pFile->fd, blockBaseOffset + pCol->offset, SEEK_SET) < 0) return -1; +// if (read(pFile->fd, buf, pCol->len) < 0) return -1; +// return 0; +// } static int compFGroupKey(const void *key, const void *fgroup) { int fid = *(int *)key; @@ -317,7 +329,7 @@ static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables) { return 0; } -static int tsdbGetFileName(char *dataDir, int fileId, const char *suffix, char *fname) { +int tsdbGetFileName(char *dataDir, int fileId, const char *suffix, char *fname) { if (dataDir == NULL || fname == NULL) return -1; sprintf(fname, "%s/f%d%s", dataDir, fileId, suffix); diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 1a8e50d0ee..b5d573764e 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -55,11 +55,11 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock); static int32_t tsdbRestoreCfg(STsdbRepo *pRepo, STsdbCfg *pCfg); static int32_t tsdbGetDataDirName(STsdbRepo *pRepo, char *fname); static void * tsdbCommitData(void *arg); -static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SDataCols *pCols); -static int tsdbHasDataInRange(SSkipListIterator *pIter, TSKEY minKey, TSKEY maxKey); +static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SRWHelper *pHelper, SDataCols *pDataCols); +static TSKEY tsdbNextIterKey(SSkipListIterator *pIter); static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey); -static int tsdbWriteBlockToFileImpl(SFile *pFile, SDataCols *pCols, int pointsToWrite, int64_t *offset, int32_t *len, - int64_t uid); +// static int tsdbWriteBlockToFileImpl(SFile *pFile, SDataCols *pCols, int pointsToWrite, int64_t *offset, int32_t *len, +// int64_t uid); #define TSDB_GET_TABLE_BY_ID(pRepo, sid) (((STSDBRepo *)pRepo)->pTableList)[sid] #define TSDB_GET_TABLE_BY_NAME(pRepo, name) @@ -751,6 +751,8 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) { } static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols) { + if (pIter == NULL) return 0; + int numOfRows = 0; do { @@ -811,7 +813,8 @@ static void *tsdbCommitData(void *arg) { STsdbRepo * pRepo = (STsdbRepo *)arg; STsdbMeta * pMeta = pRepo->tsdbMeta; STsdbCache *pCache = pRepo->tsdbCache; - STsdbCfg * pCfg = &(pRepo->config); + STsdbCfg * pCfg = &(pRepo->config); + SDataCols * pDataCols = NULL; if (pCache->imem == NULL) return NULL; // Create the iterator to read from cache @@ -821,24 +824,34 @@ static void *tsdbCommitData(void *arg) { return NULL; } - // Create a data column buffer for commit - SDataCols *pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock); - if (pDataCols == NULL) { - // TODO: deal with the error - return NULL; - } + // Create a write helper for commit data + SRWHelper whelper; + SHelperCfg hcfg = { + .type = TSDB_WRITE_HELPER, + .maxTables = pCfg->maxTables, + .maxRowSize = pMeta->maxRowBytes, + .maxRows = pCfg->maxRowsPerFileBlock, + .maxCols = pMeta->maxCols, + .minRowsPerFileBlock = pCfg->minRowsPerFileBlock, + .compress = 2 // TODO make it a configuration + }; + if (tsdbInitHelper(&whelper, &hcfg) < 0) goto _exit; + if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) goto _exit; int sfid = tsdbGetKeyFileId(pCache->imem->keyFirst, pCfg->daysPerFile, pCfg->precision); int efid = tsdbGetKeyFileId(pCache->imem->keyLast, pCfg->daysPerFile, pCfg->precision); + // Loop to commit to each file for (int fid = sfid; fid <= efid; fid++) { - if (tsdbCommitToFile(pRepo, fid, iters, pDataCols) < 0) { - // TODO: deal with the error here - // assert(0); + if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) { + ASSERT(false); + goto _exit; } } +_exit: tdFreeDataCols(pDataCols); + tsdbDestroyHelper(&whelper); tsdbDestroyTableIters(iters, pCfg->maxTables); tsdbLockRepo(arg); @@ -849,7 +862,7 @@ static void *tsdbCommitData(void *arg) { // TODO: free the skiplist for (int i = 0; i < pCfg->maxTables; i++) { STable *pTable = pMeta->tables[i]; - if (pTable && pTable->imem) { // Here has memory leak + if (pTable && pTable->imem) { // Here has memory leak pTable->imem = NULL; } } @@ -858,19 +871,12 @@ static void *tsdbCommitData(void *arg) { return NULL; } -static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SDataCols *pCols) { - int isNewLastFile = 0; +static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SRWHelper *pHelper, SDataCols *pDataCols) { STsdbMeta * pMeta = pRepo->tsdbMeta; STsdbFileH *pFileH = pRepo->tsdbFileH; STsdbCfg * pCfg = &pRepo->config; - SFile hFile, lFile; SFileGroup *pGroup = NULL; - SCompIdx * pIndices = NULL; - SCompInfo * pCompInfo = NULL; - // size_t compInfoSize = 0; - // SCompBlock compBlock; - // SCompBlock *pBlock = &compBlock; TSKEY minKey = 0, maxKey = 0; tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); @@ -879,334 +885,212 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters int hasDataToCommit = tsdbHasDataToCommit(iters, pCfg->maxTables, minKey, maxKey); if (!hasDataToCommit) return 0; // No data to commit, just return - // TODO: make it more flexible - pCompInfo = (SCompInfo *)malloc(sizeof(SCompInfo) + sizeof(SCompBlock) * 1000); - // Create and open files for commit tsdbGetDataDirName(pRepo, dataDir); - if (tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables) < 0) { /* TODO */ - } - pGroup = tsdbOpenFilesForCommit(pFileH, fid); - if (pGroup == NULL) { /* TODO */ - } - tsdbCreateFile(dataDir, fid, ".h", pCfg->maxTables, &hFile, 1, 1); - tsdbOpenFile(&hFile, O_RDWR); - if (0 /*pGroup->files[TSDB_FILE_TYPE_LAST].size > TSDB_MAX_LAST_FILE_SIZE*/) { - // TODO: make it not to write the last file every time - tsdbCreateFile(dataDir, fid, ".l", pCfg->maxTables, &lFile, 0, 0); - isNewLastFile = 1; - } + if ((pGroup = tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables)) == NULL) goto _err; - // Load the SCompIdx - pIndices = (SCompIdx *)malloc(sizeof(SCompIdx) * pCfg->maxTables); - if (pIndices == NULL) { /* TODO*/ - } - if (tsdbLoadCompIdx(pGroup, (void *)pIndices, pCfg->maxTables) < 0) { /* TODO */ - } + // Set the file to write/read + tsdbSetHelperFile(pHelper, pGroup); - lseek(hFile.fd, TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pCfg->maxTables, SEEK_SET); + // Open files for write/read + if (tsdbOpenHelperFile(pHelper) < 0) goto _err; // Loop to commit data in each table for (int tid = 0; tid < pCfg->maxTables; tid++) { STable * pTable = pMeta->tables[tid]; SSkipListIterator *pIter = iters[tid]; - SCompIdx * pIdx = &pIndices[tid]; - int nNewBlocks = 0; - - if (pTable == NULL || pIter == NULL) continue; - - /* If no new data to write for this table, just write the old data to new file - * if there are. - */ - if (!tsdbHasDataInRange(pIter, minKey, maxKey)) { - // has old data - if (pIdx->len > 0) { - goto _table_over; - // if (isNewLastFile && pIdx->hasLast) { - if (0) { - // need to move the last block to new file - if ((pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, pIdx->len)) == NULL) { /* TODO */ - } - if (tsdbLoadCompBlocks(pGroup, pIdx, (void *)pCompInfo) < 0) { /* TODO */ - } - - tdInitDataCols(pCols, tsdbGetTableSchema(pMeta, pTable)); - - SCompBlock *pTBlock = TSDB_COMPBLOCK_AT(pCompInfo, pIdx->numOfSuperBlocks); - int nBlocks = 0; - - TSDB_COMPBLOCK_GET_START_AND_SIZE(pCompInfo, pTBlock, nBlocks); - - SCompData tBlock; - int64_t toffset; - int32_t tlen; - tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_LAST], pTBlock, nBlocks, pCols, &tBlock); - - tsdbWriteBlockToFileImpl(&lFile, pCols, pCols->numOfPoints, &toffset, &tlen, pTable->tableId.uid); - pTBlock = TSDB_COMPBLOCK_AT(pCompInfo, pIdx->numOfSuperBlocks); - pTBlock->offset = toffset; - pTBlock->len = tlen; - pTBlock->numOfPoints = pCols->numOfPoints; - pTBlock->numOfSubBlocks = 1; - - pIdx->offset = lseek(hFile.fd, 0, SEEK_CUR); - if (nBlocks > 1) { - pIdx->len -= (sizeof(SCompBlock) * nBlocks); - } - write(hFile.fd, (void *)pCompInfo, pIdx->len); - } else { - pIdx->offset = lseek(hFile.fd, 0, SEEK_CUR); - sendfile(pGroup->files[TSDB_FILE_TYPE_HEAD].fd, hFile.fd, NULL, pIdx->len); - hFile.info.size += pIdx->len; - } - } - continue; - } - - pCompInfo->delimiter = TSDB_FILE_DELIMITER; - pCompInfo->checksum = 0; - pCompInfo->uid = pTable->tableId.uid; - - // Load SCompBlock part if neccessary - // int isCompBlockLoaded = 0; - if (0) { - // if (pIdx->offset > 0) { - if (pIdx->hasLast || tsdbHasDataInRange(pIter, minKey, pIdx->maxKey)) { - // has last block || cache key overlap with commit key - pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, pIdx->len + sizeof(SCompBlock) * 100); - if (tsdbLoadCompBlocks(pGroup, pIdx, (void *)pCompInfo) < 0) { /* TODO */ - } - // if (pCompInfo->uid == pTable->tableId.uid) isCompBlockLoaded = 1; - } else { - // TODO: No need to load the SCompBlock part, just sendfile the SCompBlock part - // and write those new blocks to it - } - } - - tdInitDataCols(pCols, tsdbGetTableSchema(pMeta, pTable)); + SHelperTable hTable = {.uid = pTable->tableId.uid, .tid = pTable->tableId.tid, .sversion = pTable->sversion}; + tsdbSetHelperTable(pHelper, &hTable, tsdbGetTableSchema(pMeta, pTable)); + tdInitDataCols(pDataCols, tsdbGetTableSchema(pMeta, pTable)); + // Loop to write the data in the cache to files, if no data to write, just break + // the loop int maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5; - while (1) { - tsdbReadRowsFromCache(pIter, maxKey, maxRowsToRead, pCols); - if (pCols->numOfPoints == 0) break; + while (true) { + int rowsRead = tsdbReadRowsFromCache(pIter, maxKey, maxRowsToRead, pDataCols); + ASSERT(rowsRead >= 0); + if (pDataCols->numOfPoints == 0) break; - int pointsWritten = pCols->numOfPoints; - // TODO: all write to the end of .data file - int64_t toffset = 0; - int32_t tlen = 0; - tsdbWriteBlockToFileImpl(&pGroup->files[TSDB_FILE_TYPE_DATA], pCols, pCols->numOfPoints, &toffset, &tlen, pTable->tableId.uid); + int rowsWritten = tsdbWriteDataBlock(pHelper, pDataCols); + if (rowsWritten < 0) goto _err; + assert(rowsWritten <= pDataCols->numOfPoints); - // Make the compBlock - SCompBlock *pTBlock = pCompInfo->blocks + nNewBlocks++; - pTBlock->offset = toffset; - pTBlock->len = tlen; - pTBlock->keyFirst = dataColsKeyFirst(pCols); - pTBlock->keyLast = dataColsKeyLast(pCols); - pTBlock->last = 0; - pTBlock->algorithm = 0; - pTBlock->numOfPoints = pCols->numOfPoints; - pTBlock->sversion = pTable->sversion; - pTBlock->numOfSubBlocks = 1; - pTBlock->numOfCols = pCols->numOfCols; - - if (dataColsKeyLast(pCols) > pIdx->maxKey) pIdx->maxKey = dataColsKeyLast(pCols); - - tdPopDataColsPoints(pCols, pointsWritten); - maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5 - pCols->numOfPoints; + tdPopDataColsPoints(pDataCols, rowsWritten); + maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5 - pDataCols->numOfPoints; } + // Move the last block to the new .l file if neccessary + if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) goto _err; -_table_over: // Write the SCompBlock part - pIdx->offset = lseek(hFile.fd, 0, SEEK_END); - if (pIdx->len > 0) { - int bytes = tsendfile(hFile.fd, pGroup->files[TSDB_FILE_TYPE_HEAD].fd, NULL, pIdx->len); - if (bytes < pIdx->len) { - printf("Failed to send file, reason: %s\n", strerror(errno)); - } - if (nNewBlocks > 0) { - write(hFile.fd, (void *)(pCompInfo->blocks), sizeof(SCompBlock) * nNewBlocks); - pIdx->len += (sizeof(SCompBlock) * nNewBlocks); - } - } else { - if (nNewBlocks > 0) { - write(hFile.fd, (void *)pCompInfo, sizeof(SCompInfo) + sizeof(SCompBlock) * nNewBlocks); - pIdx->len += sizeof(SCompInfo) + sizeof(SCompBlock) * nNewBlocks; - } - } - - pIdx->checksum = 0; - pIdx->numOfSuperBlocks += nNewBlocks; - pIdx->hasLast = 0; + if (tsdbWriteCompInfo(pHelper) < 0) goto _err; + } - // Write the SCompIdx part - if (lseek(hFile.fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) {/* TODO */} - if (write(hFile.fd, (void *)pIndices, sizeof(SCompIdx) * pCfg->maxTables) < 0) {/* TODO */} + if (tsdbWriteCompIdx(pHelper) < 0) goto _err; - // close the files - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { - tsdbCloseFile(&pGroup->files[type]); - } - tsdbCloseFile(&hFile); - if (isNewLastFile) tsdbCloseFile(&lFile); - // TODO: replace the .head and .last file - rename(hFile.fname, pGroup->files[TSDB_FILE_TYPE_HEAD].fname); - pGroup->files[TSDB_FILE_TYPE_HEAD].info = hFile.info; - if (isNewLastFile) { - rename(lFile.fname, pGroup->files[TSDB_FILE_TYPE_LAST].fname); - pGroup->files[TSDB_FILE_TYPE_LAST].info = lFile.info; - } - - if (pIndices) free(pIndices); - if (pCompInfo) free(pCompInfo); + tsdbCloseHelperFile(pHelper, 0); + // TODO: make it atomic with some methods + pGroup->files[TSDB_FILE_TYPE_HEAD] = pHelper->files.headF; + pGroup->files[TSDB_FILE_TYPE_DATA] = pHelper->files.dataF; + pGroup->files[TSDB_FILE_TYPE_LAST] = pHelper->files.lastF; return 0; -} -static int tsdbHasDataInRange(SSkipListIterator *pIter, TSKEY minKey, TSKEY maxKey) { - if (pIter == NULL) return 0; - - SSkipListNode *node = tSkipListIterGet(pIter); - if (node == NULL) return 0; - - SDataRow row = SL_GET_NODE_DATA(node); - if (dataRowKey(row) >= minKey && dataRowKey(row) <= maxKey) return 1; - - return 0; -} - -static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey) { - for (int i = 0; i < nIters; i++) { - SSkipListIterator *pIter = iters[i]; - if (tsdbHasDataInRange(pIter, minKey, maxKey)) return 1; - } - return 0; -} - -static int tsdbWriteBlockToFileImpl(SFile *pFile, SDataCols *pCols, int pointsToWrite, int64_t *offset, int32_t *len, int64_t uid) { - size_t size = sizeof(SCompData) + sizeof(SCompCol) * pCols->numOfCols; - SCompData *pCompData = (SCompData *)malloc(size); - if (pCompData == NULL) return -1; - - pCompData->delimiter = TSDB_FILE_DELIMITER; - pCompData->uid = uid; - pCompData->numOfCols = pCols->numOfCols; - - *offset = lseek(pFile->fd, 0, SEEK_END); - *len = size; - - int toffset = size; - for (int iCol = 0; iCol < pCols->numOfCols; iCol++) { - SCompCol *pCompCol = pCompData->cols + iCol; - SDataCol *pDataCol = pCols->cols + iCol; - - pCompCol->colId = pDataCol->colId; - pCompCol->type = pDataCol->type; - pCompCol->offset = toffset; - - // TODO: add compression - pCompCol->len = TYPE_BYTES[pCompCol->type] * pointsToWrite; - toffset += pCompCol->len; - } - - // Write the block - if (write(pFile->fd, (void *)pCompData, size) < 0) goto _err; - *len += size; - for (int iCol = 0; iCol < pCols->numOfCols; iCol++) { - SDataCol *pDataCol = pCols->cols + iCol; - SCompCol *pCompCol = pCompData->cols + iCol; - if (write(pFile->fd, pDataCol->pData, pCompCol->len) < 0) goto _err; - *len += pCompCol->len; - } - - if (pCompData == NULL) free((void *)pCompData); - return 0; - -_err: - if (pCompData == NULL) free((void *)pCompData); + _err: + tsdbCloseHelperFile(pHelper, 1); return -1; } -static int compareKeyBlock(const void *arg1, const void *arg2) { - TSKEY key = *(TSKEY *)arg1; - SCompBlock *pBlock = (SCompBlock *)arg2; +/** + * Return the next iterator key. + * + * @return the next key if iter has + * -1 if iter not + */ +static TSKEY tsdbNextIterKey(SSkipListIterator *pIter) { + if (pIter == NULL) return -1; - if (key < pBlock->keyFirst) { - return -1; - } else if (key > pBlock->keyLast) { - return 1; + SSkipListNode *node = tSkipListIterGet(pIter); + if (node == NULL) return -1; + + SDataRow row = SL_GET_NODE_DATA(node); + return dataRowKey(row); +} + +static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey) { + TSKEY nextKey; + for (int i = 0; i < nIters; i++) { + SSkipListIterator *pIter = iters[i]; + nextKey = tsdbNextIterKey(pIter); + if (nextKey > 0 && (nextKey >= minKey && nextKey <= maxKey)) return 1; } - return 0; } -int tsdbWriteBlockToFile(STsdbRepo *pRepo, SFileGroup *pGroup, SCompIdx *pIdx, SCompInfo *pCompInfo, SDataCols *pCols, SCompBlock *pCompBlock, SFile *lFile, int64_t uid) { - STsdbCfg * pCfg = &(pRepo->config); - SFile * pFile = NULL; - int numOfPointsToWrite = 0; - int64_t offset = 0; - int32_t len = 0; +// static int tsdbWriteBlockToFileImpl(SFile *pFile, SDataCols *pCols, int pointsToWrite, int64_t *offset, int32_t *len, int64_t uid) { +// size_t size = sizeof(SCompData) + sizeof(SCompCol) * pCols->numOfCols; +// SCompData *pCompData = (SCompData *)malloc(size); +// if (pCompData == NULL) return -1; - memset((void *)pCompBlock, 0, sizeof(SCompBlock)); +// pCompData->delimiter = TSDB_FILE_DELIMITER; +// pCompData->uid = uid; +// pCompData->numOfCols = pCols->numOfCols; - if (pCompInfo == NULL) { - // Just append the data block to .data or .l or .last file - numOfPointsToWrite = pCols->numOfPoints; - if (pCols->numOfPoints > pCfg->minRowsPerFileBlock) { // Write to .data file - pFile = &(pGroup->files[TSDB_FILE_TYPE_DATA]); - } else { // Write to .last or .l file - pCompBlock->last = 1; - if (lFile) { - pFile = lFile; - } else { - pFile = &(pGroup->files[TSDB_FILE_TYPE_LAST]); - } - } - tsdbWriteBlockToFileImpl(pFile, pCols, numOfPointsToWrite, &offset, &len, uid); - pCompBlock->offset = offset; - pCompBlock->len = len; - pCompBlock->algorithm = 2; // TODO : add to configuration - pCompBlock->sversion = pCols->sversion; - pCompBlock->numOfPoints = pCols->numOfPoints; - pCompBlock->numOfSubBlocks = 1; - pCompBlock->numOfCols = pCols->numOfCols; - pCompBlock->keyFirst = dataColsKeyFirst(pCols); - pCompBlock->keyLast = dataColsKeyLast(pCols); - } else { - // Need to merge the block to either the last block or the other block - TSKEY keyFirst = dataColsKeyFirst(pCols); - SCompBlock *pMergeBlock = NULL; +// *offset = lseek(pFile->fd, 0, SEEK_END); +// *len = size; - // Search the block to merge in - void *ptr = taosbsearch((void *)&keyFirst, (void *)(pCompInfo->blocks), sizeof(SCompBlock), pIdx->numOfSuperBlocks, - compareKeyBlock, TD_GE); - if (ptr == NULL) { - // No block greater or equal than the key, but there are data in the .last file, need to merge the last file block - // and merge the data - pMergeBlock = TSDB_COMPBLOCK_AT(pCompInfo, pIdx->numOfSuperBlocks - 1); - } else { - pMergeBlock = (SCompBlock *)ptr; - } +// int toffset = size; +// for (int iCol = 0; iCol < pCols->numOfCols; iCol++) { +// SCompCol *pCompCol = pCompData->cols + iCol; +// SDataCol *pDataCol = pCols->cols + iCol; + +// pCompCol->colId = pDataCol->colId; +// pCompCol->type = pDataCol->type; +// pCompCol->offset = toffset; - if (pMergeBlock->last) { - if (pMergeBlock->last + pCols->numOfPoints > pCfg->minRowsPerFileBlock) { - // Need to load the data from .last and combine data in pCols to write to .data file +// // TODO: add compression +// pCompCol->len = TYPE_BYTES[pCompCol->type] * pointsToWrite; +// toffset += pCompCol->len; +// } - } else { // Just append the block to .last or .l file - if (lFile) { - // read the block from .last file and merge with pCols, write to .l file +// // Write the block +// if (write(pFile->fd, (void *)pCompData, size) < 0) goto _err; +// *len += size; +// for (int iCol = 0; iCol < pCols->numOfCols; iCol++) { +// SDataCol *pDataCol = pCols->cols + iCol; +// SCompCol *pCompCol = pCompData->cols + iCol; +// if (write(pFile->fd, pDataCol->pData, pCompCol->len) < 0) goto _err; +// *len += pCompCol->len; +// } - } else { - // tsdbWriteBlockToFileImpl(); - } - } - } else { // The block need to merge in .data file +// if (pCompData == NULL) free((void *)pCompData); +// return 0; - } +// _err: +// if (pCompData == NULL) free((void *)pCompData); +// return -1; +// } - } +// static int compareKeyBlock(const void *arg1, const void *arg2) { +// TSKEY key = *(TSKEY *)arg1; +// SCompBlock *pBlock = (SCompBlock *)arg2; - return numOfPointsToWrite; -} +// if (key < pBlock->keyFirst) { +// return -1; +// } else if (key > pBlock->keyLast) { +// return 1; +// } + +// return 0; +// } + +// int tsdbWriteBlockToFile(STsdbRepo *pRepo, SFileGroup *pGroup, SCompIdx *pIdx, SCompInfo *pCompInfo, SDataCols *pCols, SCompBlock *pCompBlock, SFile *lFile, int64_t uid) { +// STsdbCfg * pCfg = &(pRepo->config); +// SFile * pFile = NULL; +// int numOfPointsToWrite = 0; +// int64_t offset = 0; +// int32_t len = 0; + +// memset((void *)pCompBlock, 0, sizeof(SCompBlock)); + +// if (pCompInfo == NULL) { +// // Just append the data block to .data or .l or .last file +// numOfPointsToWrite = pCols->numOfPoints; +// if (pCols->numOfPoints > pCfg->minRowsPerFileBlock) { // Write to .data file +// pFile = &(pGroup->files[TSDB_FILE_TYPE_DATA]); +// } else { // Write to .last or .l file +// pCompBlock->last = 1; +// if (lFile) { +// pFile = lFile; +// } else { +// pFile = &(pGroup->files[TSDB_FILE_TYPE_LAST]); +// } +// } +// tsdbWriteBlockToFileImpl(pFile, pCols, numOfPointsToWrite, &offset, &len, uid); +// pCompBlock->offset = offset; +// pCompBlock->len = len; +// pCompBlock->algorithm = 2; // TODO : add to configuration +// pCompBlock->sversion = pCols->sversion; +// pCompBlock->numOfPoints = pCols->numOfPoints; +// pCompBlock->numOfSubBlocks = 1; +// pCompBlock->numOfCols = pCols->numOfCols; +// pCompBlock->keyFirst = dataColsKeyFirst(pCols); +// pCompBlock->keyLast = dataColsKeyLast(pCols); +// } else { +// // Need to merge the block to either the last block or the other block +// TSKEY keyFirst = dataColsKeyFirst(pCols); +// SCompBlock *pMergeBlock = NULL; + +// // Search the block to merge in +// void *ptr = taosbsearch((void *)&keyFirst, (void *)(pCompInfo->blocks), sizeof(SCompBlock), pIdx->numOfSuperBlocks, +// compareKeyBlock, TD_GE); +// if (ptr == NULL) { +// // No block greater or equal than the key, but there are data in the .last file, need to merge the last file block +// // and merge the data +// pMergeBlock = TSDB_COMPBLOCK_AT(pCompInfo, pIdx->numOfSuperBlocks - 1); +// } else { +// pMergeBlock = (SCompBlock *)ptr; +// } + +// if (pMergeBlock->last) { +// if (pMergeBlock->last + pCols->numOfPoints > pCfg->minRowsPerFileBlock) { +// // Need to load the data from .last and combine data in pCols to write to .data file + +// } else { // Just append the block to .last or .l file +// if (lFile) { +// // read the block from .last file and merge with pCols, write to .l file + +// } else { +// // tsdbWriteBlockToFileImpl(); +// } +// } +// } else { // The block need to merge in .data file + +// } + +// } + +// return numOfPointsToWrite; +// } diff --git a/src/vnode/tsdb/src/tsdbRWHelper.c b/src/vnode/tsdb/src/tsdbRWHelper.c new file mode 100644 index 0000000000..79fe3c49f7 --- /dev/null +++ b/src/vnode/tsdb/src/tsdbRWHelper.c @@ -0,0 +1,1057 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "tsdbMain.h" + +#define adjustMem(ptr, size, expectedSize) \ + do { \ + if ((size) < (expectedSize)) { \ + (ptr) = realloc((void *)(ptr), (expectedSize)); \ + if ((ptr) == NULL) return -1; \ + (size) = (expectedSize); \ + } \ + } while (0) + +// Local function definitions +static int tsdbCheckHelperCfg(SHelperCfg *pCfg); +static void tsdbInitHelperFile(SHelperFile *pHFile); +static int tsdbInitHelperRead(SRWHelper *pHelper); +static int tsdbInitHelperWrite(SRWHelper *pHelper); +static void tsdbClearHelperFile(SHelperFile *pHFile); +static void tsdbDestroyHelperRead(SRWHelper *pHelper); +static void tsdbDestroyHelperWrite(SRWHelper *pHelper); +static void tsdbClearHelperRead(SRWHelper *pHelper); +static void tsdbClearHelperWrite(SRWHelper *pHelper); +static bool tsdbShouldCreateNewLast(SRWHelper *pHelper); +static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, int rowsToWrite, SCompBlock *pCompBlock, + bool isLast); +static int compareKeyBlock(const void *arg1, const void *arg2); +static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols); +static int nRowsLEThan(SDataCols *pDataCols, int maxKey); +static int tsdbGetRowsCanBeMergedWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols); + +int tsdbInitHelper(SRWHelper *pHelper, SHelperCfg *pCfg) { + if (pHelper == NULL || pCfg == NULL || tsdbCheckHelperCfg(pCfg) < 0) return -1; + + memset((void *)pHelper, 0, sizeof(*pHelper)); + + pHelper->config = *pCfg; + + tsdbInitHelperFile(&(pHelper->files)); + + if (tsdbInitHelperRead(pHelper) < 0) goto _err; + + if ((TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER) && tsdbInitHelperWrite(pHelper) < 0) goto _err; + + pHelper->state = TSDB_HELPER_CLEAR_STATE; + + return 0; + +_err: + tsdbDestroyHelper(pHelper); + return -1; +} + +void tsdbDestroyHelper(SRWHelper *pHelper) { + if (pHelper == NULL) return; + + tsdbClearHelperFile(&(pHelper->files)); + tsdbDestroyHelperRead(pHelper); + tsdbDestroyHelperWrite(pHelper); +} + +void tsdbClearHelper(SRWHelper *pHelper) { + if (pHelper == NULL) return; + tsdbClearHelperFile(&(pHelper->files)); + tsdbClearHelperRead(pHelper); + tsdbClearHelperWrite(pHelper); +} + +int tsdbSetHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { + // TODO: reset the helper object + + pHelper->files.fid = pGroup->fileId; + + pHelper->files.headF = pGroup->files[TSDB_FILE_TYPE_HEAD]; + pHelper->files.dataF = pGroup->files[TSDB_FILE_TYPE_DATA]; + pHelper->files.lastF = pGroup->files[TSDB_FILE_TYPE_LAST]; + + if (TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER) { + char *fnameDup = strdup(pHelper->files.headF.fname); + if (fnameDup == NULL) return -1; + char *dataDir = dirname(fnameDup); + + tsdbGetFileName(dataDir, pHelper->files.fid, ".h", pHelper->files.nHeadF.fname); + tsdbGetFileName(dataDir, pHelper->files.fid, ".l", pHelper->files.nLastF.fname); + free((void *)fnameDup); + } + return 0; +} + +int tsdbOpenHelperFile(SRWHelper *pHelper) { + // TODO: check if the file is set + {} + + if (TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER) { + if (tsdbOpenFile(&(pHelper->files.headF), O_RDONLY) < 0) goto _err; + if (tsdbOpenFile(&(pHelper->files.dataF), O_RDWR) < 0) goto _err; + if (tsdbOpenFile(&(pHelper->files.lastF), O_RDWR) < 0) goto _err; + // TODO: need to write head and compIdx part + if (tsdbOpenFile(&(pHelper->files.nHeadF), O_WRONLY | O_CREAT) < 0) goto _err; + if (tsdbShouldCreateNewLast(pHelper)) { + if (tsdbOpenFile(&(pHelper->files.nLastF), O_WRONLY | O_CREAT) < 0) goto _err; + } + } else { + if (tsdbOpenFile(&(pHelper->files.headF), O_RDONLY) < 0) goto _err; + if (tsdbOpenFile(&(pHelper->files.dataF), O_RDONLY) < 0) goto _err; + if (tsdbOpenFile(&(pHelper->files.lastF), O_RDONLY) < 0) goto _err; + } + + return 0; + +_err: + tsdbCloseHelperFile(pHelper, true); + return -1; +} + +int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { + if (pHelper->files.headF.fd > 0) { + close(pHelper->files.headF.fd); + pHelper->files.headF.fd = -1; + } + if (pHelper->files.dataF.fd > 0) { + close(pHelper->files.dataF.fd); + pHelper->files.dataF.fd = -1; + } + if (pHelper->files.lastF.fd > 0) { + close(pHelper->files.lastF.fd); + pHelper->files.lastF.fd = -1; + } + if (pHelper->files.nHeadF.fd > 0) { + close(pHelper->files.nHeadF.fd); + pHelper->files.nHeadF.fd = -1; + if (hasError) remove(pHelper->files.nHeadF.fname); + } + + if (pHelper->files.nLastF.fd > 0) { + close(pHelper->files.nLastF.fd); + pHelper->files.nLastF.fd = -1; + if (hasError) remove(pHelper->files.nLastF.fname); + } + return 0; +} + +void tsdbSetHelperTable(SRWHelper *pHelper, SHelperTable *pHelperTable, STSchema *pSchema) { + // TODO: check if it is available to set the table + + pHelper->tableInfo = *pHelperTable; + // TODO: Set the pDataCols according to schema + + // TODO: set state +} + +int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { + ASSERT(TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER); + ASSERT(helperHasState(pHelper, TSDB_HELPER_FILE_SET) && helperHasState(pHelper, TSDB_HELPER_FILE_OPEN)); + SCompBlock compBlock; + int rowsToWrite = 0; + TSKEY keyFirst = dataColsKeyFirst(pDataCols); + + // Load SCompIdx part if not loaded yet + if ((!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) && (tsdbLoadCompIdx(pHelper, NULL) < 0)) goto _err; + + // Load the SCompInfo part if neccessary + SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + if ((pIdx->offset > 0) && (pIdx->hasLast || dataColsKeyFirst(pDataCols) <= pIdx->maxKey)) { + if (tsdbLoadCompInfo(pHelper, NULL) < 0) goto _err; + } + + SCompIdx *pWIdx = pHelper->pWCompIdx + pHelper->tableInfo.tid; + + if (!pIdx->hasLast && keyFirst > pIdx->maxKey) { + // Just need to append as a super block + rowsToWrite = pDataCols->numOfPoints; + SFile *pWFile = NULL; + bool isLast = false; + + if (rowsToWrite > pHelper->config.minRowsPerFileBlock) { + pWFile = &(pHelper->files.dataF); + } else { + isLast = true; + pWFile = (pHelper->files.nLastF.fd > 0) ? &(pHelper->files.nLastF) : &(pHelper->files.nLastF); + } + + if (tsdbWriteBlockToFile(pHelper, pWFile, pDataCols, rowsToWrite, &compBlock, isLast) < 0) goto _err; + + // TODO: may need to reallocate the memory + pHelper->pCompInfo->blocks[pHelper->blockIter++] = compBlock; + + pIdx->hasLast = compBlock.last; + pIdx->numOfSuperBlocks++; + pIdx->maxKey = dataColsKeyLast(pDataCols); + // pIdx->len = ?????? + } else { // (pIdx->hasLast) OR (keyFirst <= pIdx->maxKey) + if (keyFirst > pIdx->maxKey) { + int blkIdx = pIdx->numOfSuperBlocks - 1; + ASSERT(pIdx->hasLast && pHelper->pCompInfo->blocks[blkIdx].last); + + // Need to merge with the last block + if (tsdbMergeDataWithBlock(pHelper, blkIdx, pDataCols) < 0) goto _err; + } else { + // Find the first block greater or equal to the block + SCompBlock *pCompBlock = taosbsearch((void *)(&keyFirst), (void *)(pHelper->pCompInfo->blocks), + pIdx->numOfSuperBlocks, sizeof(SCompBlock), compareKeyBlock, TD_GE); + if (pCompBlock == NULL) { + if (tsdbMergeDataWithBlock(pHelper, pIdx->numOfSuperBlocks-1, pDataCols) < 0) goto _err; + } else { + if (compareKeyBlock((void *)(&keyFirst), (void *)pCompBlock) == 0) { + SCompBlock *pNextBlock = NULL; + TSKEY keyLimit = (pNextBlock == NULL) ? INT_MAX : (pNextBlock->keyFirst - 1); + rowsToWrite = + MIN(nRowsLEThan(pDataCols, keyLimit), pHelper->config.maxRowsPerFileBlock - pCompBlock->numOfPoints); + + if (tsdbMergeDataWithBlock(pHelper, pCompBlock-pHelper->pCompInfo->blocks, pDataCols) < 0) goto _err; + } else { + // There options: 1. merge with previous block + // 2. commit as one block + // 3. merge with current block + int nRows1 = INT_MAX; + int nRows2 = nRowsLEThan(pDataCols, pCompBlock->keyFirst); + int nRows3 = MIN(nRowsLEThan(pDataCols, (pCompBlock + 1)->keyFirst), (pHelper->config.maxRowsPerFileBlock - pCompBlock->numOfPoints)); + + // TODO: find the block with max rows can merge + if (tsdbMergeDataWithBlock(pHelper, pCompBlock, pDataCols) < 0) goto _err; + } + } + } + } + + return rowsToWrite; + + _err: + return -1; +} + +int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { + // TODO + return 0; +} + +int tsdbWriteCompInfo(SRWHelper *pHelper) { + // TODO + return 0; +} + +int tsdbWriteCompIdx(SRWHelper *pHelper) { + // TODO + return 0; +} + +int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { + // TODO: check helper state + ASSERT(!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)); + + int fd = pHelper->files.headF.fd; + + if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1; + if (tread(fd, pHelper->pCompIdx, pHelper->compIdxSize) < pHelper->compIdxSize) return -1; + // TODO: check the checksum + + if (target) memcpy(target, pHelper->pCompIdx, pHelper->compIdxSize); + helperSetState(pHelper, TSDB_HELPER_IDX_LOAD); + + return 0; +} + +int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { + SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + + ASSERT(pIdx->offset > 0); + + int fd = pHelper->files.headF.fd; + + if (lseek(fd, pIdx->offset, SEEK_SET) < 0) return -1; + ASSERT(pIdx->len > 0); + + adjustMem(pHelper->pCompInfo, pHelper->compInfoSize, pIdx->len); + if (tread(fd, (void *)(pHelper->pCompInfo), pIdx->len) < 0) return -1; + // TODO: check the checksum + + // TODO: think about when target has no space for the content + if (target) memcpy(target, (void *)(pHelper->pCompInfo), pIdx->len); + + helperSetState(pHelper, TSDB_HELPER_INFO_LOAD); + + return 0; +} + +int tsdbLoadCompData(SRWHelper *pHelper, int blkIdx, void *target) { + // TODO + return 0; +} + +int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int32_t *colIds, int numOfColIds) { + // TODO + return 0; +} + +int tsdbLoadBlockData(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols) { + // TODO + return 0; +} + +static int tsdbCheckHelperCfg(SHelperCfg *pCfg) { + // TODO + return 0; +} + +static void tsdbInitHelperFile(SHelperFile *pHFile) { + pHFile->fid = -1; + pHFile->headF.fd = -1; + pHFile->dataF.fd = -1; + pHFile->lastF.fd = -1; + pHFile->nHeadF.fd = -1; + pHFile->nLastF.fd = -1; +} + +static void tsdbClearHelperFile(SHelperFile *pHFile) { + pHFile->fid = -1; + if (pHFile->headF.fd > 0) { + close(pHFile->headF.fd); + pHFile->headF.fd = -1; + } + if (pHFile->dataF.fd > 0) { + close(pHFile->dataF.fd); + pHFile->dataF.fd = -1; + } + if (pHFile->lastF.fd > 0) { + close(pHFile->lastF.fd); + pHFile->lastF.fd = -1; + } + if (pHFile->nHeadF.fd > 0) { + close(pHFile->nHeadF.fd); + pHFile->nHeadF.fd = -1; + } + if (pHFile->nLastF.fd > 0) { + close(pHFile->nLastF.fd); + pHFile->nLastF.fd = -1; + } + +} + +static int tsdbInitHelperRead(SRWHelper *pHelper) { + SHelperCfg *pCfg = &(pHelper->config); + + pHelper->compIdxSize = pCfg->maxTables * sizeof(SCompIdx); + if ((pHelper->pCompIdx = (SCompIdx *)malloc(pHelper->compIdxSize)) == NULL) return -1; + + return 0; +} + +static void tsdbDestroyHelperRead(SRWHelper *pHelper) { + tfree(pHelper->pCompIdx); + pHelper->compIdxSize = 0; + + tfree(pHelper->pCompInfo); + pHelper->compInfoSize = 0; + + tfree(pHelper->pCompData); + pHelper->compDataSize = 0; + + tdFreeDataCols(pHelper->pDataCols[0]); + tdFreeDataCols(pHelper->pDataCols[1]); +} + +static int tsdbInitHelperWrite(SRWHelper *pHelper) { + SHelperCfg *pCfg = &(pHelper->config); + + pHelper->wCompIdxSize = pCfg->maxTables * sizeof(SCompIdx); + if ((pHelper->pWCompIdx = (SCompIdx *)malloc(pHelper->wCompIdxSize)) == NULL) return -1; + + return 0; +} + +static void tsdbDestroyHelperWrite(SRWHelper *pHelper) { + tfree(pHelper->pWCompIdx); + pHelper->wCompIdxSize = 0; + + tfree(pHelper->pWCompInfo); + pHelper->wCompInfoSize = 0; + + tfree(pHelper->pWCompData); + pHelper->wCompDataSize = 0; +} + +static void tsdbClearHelperRead(SRWHelper *pHelper) { + // TODO +} + +static void tsdbClearHelperWrite(SRWHelper *pHelper) { + // TODO +} + +static bool tsdbShouldCreateNewLast(SRWHelper *pHelper) { + // TODO + return 0; +} + +static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, int rowsToWrite, SCompBlock *pCompBlock, + bool isLast) { + ASSERT(rowsToWrite > 0 && rowsToWrite <= pDataCols->numOfPoints); + + int64_t offset = lseek(pFile->fd, 0, SEEK_END); + if (offset < 0) goto _err; + + SCompData *pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * pDataCols->numOfCols); + if (pCompData == NULL) goto _err; + + int nColsNotAllNull = 0; + int32_t toffset; + for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) { + SDataCol *pDataCol = pDataCols->cols + ncol; + SCompCol *pCompCol = pCompData->cols + nColsNotAllNull; + + if (0) { + // TODO: all data are NULL + continue; + } + + pCompCol->colId = pDataCol->colId; + pCompCol->type = pDataCol->type; + pCompCol->len = pDataCol->len; + pCompCol->offset = toffset; + nColsNotAllNull++; + toffset += pCompCol->len; + } + + pCompData->delimiter = TSDB_FILE_DELIMITER; + pCompData->uid = pHelper->tableInfo.uid; + pCompData->numOfCols = nColsNotAllNull; + + size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * nColsNotAllNull; + if (twrite(pFile->fd, (void *)pCompData, tsize) < tsize) goto _err; + for (int i = 0; i < pDataCols->numOfCols; i++) { + SDataCol *pDataCol = pCompData->cols + i; + SCompCol *pCompCol = NULL; + if (twrite(pFile->fd, (void *)(pDataCol->pData), pCompCol->len) < pCompCol->len) goto _err; + } + + pCompBlock->last = isLast; + pCompBlock->offset = offset; + // pCOmpBlock->algorithm = ; + pCompBlock->numOfPoints = rowsToWrite; + pCompBlock->sversion = pHelper->tableInfo.sversion; + // pCompBlock->len = ; + // pCompBlock->numOfSubBlocks = ; + pCompBlock->numOfCols = nColsNotAllNull; + // pCompBlock->keyFirst = ; + // pCompBlock->keyLast = ; + + return 0; + + _err: + return -1; +} + +// static int compareKeyBlock(const void *arg1, const void *arg2); + +// /** +// * Init a read-write helper object for read or write usage. +// */ +// int tsdbInitHelper(SRWHelper *pHelper, int maxTables, tsdb_rwhelper_t type, int maxRowSize, int maxRows, +// int maxCols) { +// if (pHelper == NULL) return -1; + +// memset((void *)pHelper, 0, sizeof(SRWHelper)); +// for (int ftype = TSDB_RW_HEADF; ftype <= TSDB_RW_LF; ftype++) { +// pHelper->files[ftype] = -1; +// } + +// // Set type +// pHelper->type = type; + +// // Set global configuration +// pHelper->maxTables = maxTables; +// pHelper->maxRowSize = maxRowSize; +// pHelper->maxRows = maxRows; +// pHelper->maxCols = maxCols; + +// // Allocate SCompIdx part memory +// pHelper->compIdxSize = sizeof(SCompIdx) * maxTables; +// pHelper->pCompIdx = (SCompIdx *)malloc(pHelper->compIdxSize); +// if (pHelper->pCompIdx == NULL) goto _err; + +// pHelper->compDataSize = sizeof(SCompData) + sizeof(SCompCol) * maxCols; +// pHelper->pCompData = (SCompData *)malloc(pHelper->compDataSize); + +// pHelper->pDataCols = tdNewDataCols(maxRowSize, maxCols, maxRows); +// if (pHelper->pDataCols == NULL) goto _err; + +// return 0; + +// _err: +// tsdbDestroyHelper(pHelper); +// return -1; +// } + +// void tsdbResetHelper(SRWHelper *pHelper) { +// if (pHelper->headF.fd > 0) { +// close(pHelper->headF.fd); +// pHelper->headF.fd = -1; +// } +// if (pHelper->dataF.fd > 0) { +// close(pHelper->dataF.fd); +// pHelper->dataF.fd = -1; +// } +// if (pHelper->lastF.fd > 0) { +// close(pHelper->lastF.fd); +// pHelper->lastF.fd = -1; +// } +// if (pHelper->hF.fd > 0) { +// close(pHelper->hF.fd); +// pHelper->hF.fd = -1; +// } +// if (pHelper->lF.fd > 0) { +// close(pHelper->lF.fd); +// pHelper->lF.fd = -1; +// } + +// pHelper->state = 0; +// tdResetDataCols(pHelper->pDataCols); +// } + +// int tsdbDestroyHelper(SRWHelper *pHelper) { +// if (pHelper->headF.fd > 0) close(pHelper->headF.fd); +// if (pHelper->dataF.fd > 0) close(pHelper->dataF.fd); +// if (pHelper->lastF.fd > 0) close(pHelper->lastF.fd); +// if (pHelper->hF.fd > 0) close(pHelper->hF.fd); +// if (pHelper->lF.fd > 0) close(pHelper->lF.fd); + +// if (pHelper->pCompIdx) free(pHelper->pCompIdx); +// if (pHelper->pCompInfo) free(pHelper->pCompInfo); +// if (pHelper->pCompData) free(pHelper->pCompData); +// memset((void *)pHelper, 0, sizeof(SRWHelper)); +// return 0; +// } + +// int tsdbSetHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { +// if (pHelper->state != 0) return -1; + +// pHelper->fid = pGroup->fileId; + +// pHelper->headF = pGroup->files[TSDB_FILE_TYPE_HEAD]; +// pHelper->headF.fd = -1; +// pHelper->dataF = pGroup->files[TSDB_FILE_TYPE_DATA]; +// pHelper->dataF.fd = -1; +// pHelper->lastF = pGroup->files[TSDB_FILE_TYPE_LAST]; +// pHelper->lastF.fd = -1; + +// if (pHelper->mode == TSDB_WRITE_HELPER) { +// char *fnameCpy = strdup(pHelper->headF.fname); +// if (fnameCpy == NULL) return -1; +// char *dataDir = dirname(fnameCpy); + +// memset((void *)(&pHelper->hF), 0, sizeof(SFile)); +// memset((void *)(&pHelper->lF), 0, sizeof(SFile)); +// pHelper->hF.fd = -1; +// pHelper->lF.fd = -1; + +// tsdbGetFileName(dataDir, pHelper->fid, ".h", pHelper->hF.fname); +// tsdbGetFileName(dataDir, pHelper->fid, ".l", pHelper->lF.fname); +// free((char *)fnameCpy); +// } + +// TSDB_SET_RWHELPER_STATE(pHelper, TSDB_RWHELPER_FILE_SET); + +// return 0; +// } + +// static int tsdbNeedToCreateNewLastFile() { +// // TODO +// return 0; +// } + +// int tsdbCloseHelperFile(SRWHelper *pHelper, int hasErr) { +// int ret = 0; +// if (pHelper->headF.fd > 0) { +// close(pHelper->headF.fd); +// pHelper->headF.fd = -1; +// } +// if (pHelper->dataF.fd > 0) { +// close(pHelper->dataF.fd); +// pHelper->dataF.fd = -1; +// } +// if (pHelper->lastF.fd > 0) { +// close(pHelper->lastF.fd); +// pHelper->lastF.fd = -1; +// } +// if (pHelper->hF.fd > 0) { +// close(pHelper->hF.fd); +// pHelper->hF.fd = -1; +// if (hasErr) remove(pHelper->hF.fname); +// } +// if (pHelper->lF.fd > 0) { +// close(pHelper->lF.fd); +// pHelper->lF.fd = -1; +// if (hasErr) remove(pHelper->hF.fname); +// } +// return 0; +// } + +// int tsdbOpenHelperFile(SRWHelper *pHelper) { +// if (pHelper->state != TSDB_RWHELPER_FILE_SET) return -1; + +// if (pHelper->mode == TSDB_READ_HELPER) { // The read helper +// if (tsdbOpenFile(&pHelper->headF, O_RDONLY) < 0) goto _err; +// if (tsdbOpenFile(&pHelper->dataF, O_RDONLY) < 0) goto _err; +// if (tsdbOpenFile(&pHelper->lastF, O_RDONLY) < 0) goto _err; +// } else { +// if (tsdbOpenFile(&pHelper->headF, O_RDONLY) < 0) goto _err; +// if (tsdbOpenFile(&pHelper->dataF, O_RDWR) < 0) goto _err; +// if (tsdbOpenFile(&pHelper->lastF, O_RDWR) < 0) goto _err; +// // Open .h and .l file +// if (tsdbOpenFile(&pHelper->hF, O_WRONLY | O_CREAT) < 0) goto _err; +// if (tsdbNeedToCreateNewLastFile()) { +// if (tsdbOpenFile(&pHelper->lF, O_WRONLY | O_CREAT) < 0) goto _err; +// } +// } + +// TSDB_SET_RWHELPER_STATE(pHelper, TSDB_RWHELPER_FILE_OPENED); + +// return 0; +// _err: +// tsdbCloseHelperFile(pHelper, 1); +// return -1; +// } + +// int tsdbLoadCompIdx(SRWHelper *pHelper) { +// if (pHelper->state != (TSDB_RWHELPER_FILE_SET | TSDB_RWHELPER_FILE_OPENED)) return -1; + +// if (lseek(pHelper->headF.fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1; +// if (tread(pHelper->headF.fd, (void *)(pHelper->pCompIdx), pHelper->compIdxSize) < pHelper->compIdxSize) return -1; + +// TSDB_SET_RWHELPER_STATE(pHelper, TSDB_RWHELPER_COMPIDX_LOADED); + +// return 0; +// } + +// int tsdbSetHelperTable(SRWHelper *pHelper, int32_t tid, int64_t uid, STSchema *pSchema) { +// // TODO: add some check information +// pHelper->tid = tid; +// pHelper->uid = uid; + +// tdInitDataCols(pHelper->pDataCols, pSchema); + +// TSDB_SET_RWHELPER_STATE(pHelper, TSDB_RWHELPER_TABLE_SET); + +// return 0; +// } + +// int tsdbLoadCompBlocks(SRWHelper *pHelper) { +// SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tid; +// if (pIdx->offset <= 0) return 0; + +// if (lseek(pHelper->headF.fd, pIdx->offset, SEEK_SET) < 0) return -1; +// if (pHelper->compInfoSize < pIdx->len) { +// pHelper->pCompInfo = (SCompInfo *)realloc((void *)(pHelper->pCompInfo), pIdx->len); +// if (pHelper->pCompInfo == NULL) return -1; +// pHelper->compInfoSize = pIdx->len; +// } + +// if (tread(pHelper->headF.fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) return -1; + +// TSDB_SET_RWHELPER_STATE(pHelper, TSDB_RWHELPER_COMPBLOCK_LOADED); + +// return 0; +// } + +// int tsdbRWHelperSetBlockIdx(SRWHelper *pHelper, int blkIdx) { +// SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tid; +// if (blkIdx > pIdx->numOfSuperBlocks) return -1; + +// pHelper->blkIdx = blkIdx; + +// TSDB_SET_RWHELPER_STATE(pHelper, TSDB_RWHELPER_BLOCKIDX_SET); + +// return 0; +// } + +// int tsdbRWHelperLoadCompData(SRWHelper *pHelper) { +// SCompBlock *pBlock = pHelper->pCompInfo->blocks + pHelper->blkIdx; + +// if (pBlock->numOfSubBlocks == 1) { // Only one super block +// size_t size = sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols; +// if (size > pHelper->compDataSize) { +// pHelper->pCompData = (SCompData *)realloc((void *)pHelper->pCompData, size); +// if (pHelper->pCompData == NULL) return -1; +// pHelper->compDataSize = size; +// } + +// if (lseek(pHelper->dataF.fd, pBlock->offset, SEEK_SET) < 0) return -1; +// if (tread(pHelper->dataF.fd, (void *)(pHelper->pCompData), size) < size) return -1; +// } else { // TODO: More sub blocks +// } + +// TSDB_SET_RWHELPER_STATE(pHelper, TSDB_RWHELPER_COMPCOL_LOADED); + +// return 0; +// } + + +// static int compColIdCompCol(const void *arg1, const void *arg2) { +// int colId = *(int *)arg1; +// SCompCol *pCompCol = (SCompCol *)arg2; + +// return (int)(colId - pCompCol->colId); +// } + +// static int compColIdDataCol(const void *arg1, const void *arg2) { +// int colId = *(int *)arg1; +// SDataCol *pDataCol = (SDataCol *)arg2; + +// return (int)(colId - pDataCol->colId); +// } + +// int tsdbRWHelperLoadColData(SRWHelper *pHelper, int colId) { +// SCompBlock *pBlock = pHelper->pCompInfo->blocks + pHelper->blkIdx; + +// if (pBlock->numOfSubBlocks == 1) { // only one super block +// SCompCol *pCompCol = bsearch((void *)(&colId), (void *)(pHelper->pCompData->cols), pBlock->numOfCols, compColIdCompCol, compColIdCompCol); +// if (pCompCol == NULL) return 0; // No data to read from this block , but we still return 0 + +// SDataCol *pDataCol = bsearch((void *)(&colId), (void *)(pHelper->pDataCols->cols), pHelper->pDataCols->numOfCols, sizeof(SDataCol), compColIdDataCol); +// assert(pDataCol != NULL); + +// int fd = (pBlock->last) ? pHelper->lastF.fd : pHelper->dataF.fd; +// if (lseek(fd, pBlock->offset + pCompCol->offset, SEEK_SET) < 0) return -1; +// if (tread(fd, (void *)pDataCol->pData, pCompCol->len) < pCompCol->len) return -1; +// pDataCol->len = pCompCol->len; +// } else { +// // TODO: more than 1 blocks +// } +// return 0; +// } + +// int tsdbRWHelperLoadBlockData(SRWHelper *pHelper, int blkIdx) { +// SCompBlock *pBlock = pHelper->pCompInfo->blocks + pHelper->blkIdx; + +// if (pBlock->numOfSubBlocks == 1) { +// for (int i = 0; i < pHelper->pDataCols->numOfCols; i++) { +// if (tsdbRWHelperLoadBlockData(pHelper, pHelper->pDataCols->cols[i].colId) < 0) return -1; +// } +// } else { +// // TODO: more than 1 block of data +// } +// return 0; +// } + +// int tsdbRWHelperCopyCompBlockPart(SRWHelper *pHelper) { +// // TODO +// return 0; +// } + +// int tsdbRWHelperCopyDataBlockPart(SRWHelper *pHelper ) { +// // TODO +// return 0; +// } + +// int tsdbRWHelperWriteCompIdx(SRWHelper *pHelper) { +// // TODO +// if (lseek(pHelper->hF.fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1; +// if (twrite(pHelper->hF.fd, (void *)(pHelper->pCompIdx), pHelper->compIdxSize) < pHelper->compIdxSize) return -1; + +// return 0; +// } + +// /** +// * Load the data block from file +// * +// * @return 0 for success +// * -1 for failure +// */ +// int tsdbLoadDataBlock(SRWHelper *pHelper, int bldIdx) { +// SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tid; +// if (pIdx->) + +// return 0; +// } + +// /** +// * Append the block to a file, either .data +// */ +// int tsdbAppendBlockToFile(SRWHelper *pHelper, tsdb_rw_file_t toFile, SDataCols *pDataCols, SCompBlock *pCompBlock, bool isSuper) { +// SFile *pFile = pHelper->files + toFile; + +// int64_t offset = lseek(pFile->fd, 0, SEEK_END); +// if (*offset < 0) return -1; + +// SCompData *pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * pDataCols->numOfCols); +// if (pCompData == NULL) return -1; + + +// int numOfNotAllNullCols = 0; +// int32_t toffset = 0; +// for (int i = 0; i < pDataCols->numOfCols; i++) { +// SDataCol *pDataCol = pDataCols->cols + i; +// SCompCol *pCompCol = pCompData->cols + numOfNotAllNullCols; + +// if (0 /* All data in this column are NULL value */) { +// continue; +// } +// pCompCol->colId = pDataCol->colId; +// pCompCol->type = pDataCol->type; +// pCompCol->len = pDataCol->len; +// // pCompCol->offset = toffset; +// numOfNotAllNullCols++; +// // toffset += pDataCol->len; +// } + +// pCompData->delimiter = TSDB_FILE_DELIMITER; +// pCompData->numOfCols = numOfNotAllNullCols; +// pCompData->uid = pHelper->uid; + +// size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * numOfNotAllNullCols; +// if (twrite(pFile->fd, (void *)pCompData, tsize) < 0) return -1; +// for (int i = 0; i < numOfNotAllNullCols; i++) { +// SCompCol *pCompCol = pCompData->cols + i; +// SDataCol *pDataCol = NULL; // bsearch() +// tassert(pDataCol != NULL); +// if (twrite(pFile->fd, (void *)(pDataCol->pData), pDataCol->len) < pDataCol->len) return -1; +// } + +// pCompBlock->last = (toFile == TSDB_RW_DATAF) ? 0 : 1; +// pCompBlock->offset = offset; +// pCompBlock->algorithm = pHelper->compression; +// pCompBlock->numOfPoints = pDataCols->numOfPoints; +// pCompBlock->sversion = pHelper->sversion; +// // pCompBlock->len = ; +// pCompBlock->numOfSubBlocks = isSuper ? 1 : 0; +// pCompBlock->numOfCols = numOfNotAllNullCols; +// pCompBlock->keyFirst = dataColsKeyFirst(pDataCols); +// pCompBlock->keyLast = dataColsKeyLast(pDataCols); + +// return 0; +// } + +// /** +// * Write the whole or part of the cached data block to file. +// * +// * There are four options: +// * 1. Append the whole block as a SUPER-BLOCK at the end +// * 2. Append part/whole block as a SUPER-BLOCK and insert in the middle +// * 3. Append part/whole block as a SUB-BLOCK +// * 4. Merge part/whole block as a SUPER-BLOCK +// */ +// int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { +// tassert(pHelper->type == TSDB_WRITE_HELPER); + +// int rowsWritten = 0; +// SCompBlock compBlock; + +// SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tid; +// // if ((no old data) OR (no last block AND cached first key is larger than the max key)) +// if ((pIdx->offset == 0) || (pIdx->hasLast && dataColsKeyFirst(pDataCols) > pIdx->maxKey)) { +// // Append the whole block as a SUPER-BLOCK at the end +// if (pDataCols->numOfPoints >= pHelper->minRowPerFileBlock) { +// if (tsdbAppendBlockToFile(pHelper, TSDB_RW_DATAF, pDataCols, &compBlock, true) < 0) goto _err; +// } else { +// tsdb_rw_file_t ftype = (pHelper->files[TSDB_RW_LF].fd > 0) ? TSDB_RW_LF : TSDB_RW_LASTF; +// if (tsdbAppendBlockToFile(pHelper, ftype, pDataCols, &compBlock, true) < 0) goto _err; +// } +// // Copy the compBlock part to the end +// if (IS_COMPBLOCK_LOADED(pHelper)) { + +// } else { + +// } + +// pIdx->hasLast = compBlock.last; +// pIdx->len += sizeof(compBlock); +// pIdx->numOfSuperBlocks++; +// pIdx->maxKey = compBlock.keyLast; + +// rowsWritten = pDataCols->numOfPoints; +// } else { +// // Need to find a block to merge with +// int blkIdx = 0; +// // if (has last block AND cached Key is larger than the max Key) +// if (pIdx->hasLast && dataColsKeyFirst(pDataCols) > pIdx->maxKey) { +// blkIdx = pIdx->numOfSuperBlocks - 1; +// rowsWritten = tsdbMergeDataWithBlock(pHelper, pDataCols, blkIdx); +// if (rowsWritten < 0) goto _err; +// } else { +// ASSERT(IS_COMPBLOCK_LOADED(pHelper)); +// // SCompBlock *pMergeBlock = taosbsearch(); +// } +// } + +// return numOfPointsWritten; + +// _err: +// return -1; +// } + +static int compareKeyBlock(const void *arg1, const void *arg2) { + TSKEY key = *(TSKEY *)arg1; + SCompBlock *pBlock = (SCompBlock *)arg2; + + if (key < pBlock->keyFirst) { + return -1; + } else if (key > pBlock->keyLast) { + return 1; + } + + return 0; +} + +static int nRowsLEThan(SDataCols *pDataCols, int maxKey) { + return 0; +} + +static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols) { + int rowsWritten = 0; + TSKEY keyFirst = dataColsKeyFirst(pDataCols); + SCompBlock compBlock = {0}; + + SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + ASSERT(blkIdx < pIdx->numOfSuperBlocks); + + SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx; + ASSERT(pCompBlock->numOfSubBlocks >= 1); + + int rowsCanMerge = tsdbGetRowsCanBeMergedWithBlock(pHelper, blkIdx, pDataCols); + if (rowsCanMerge < 0) goto _err; + + ASSERT(rowsCanMerge > 0); + + if (pCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS && + ((!pCompBlock->last) || (pHelper->files.nLastF.fd < 0 && + pCompBlock->numOfPoints + rowsCanMerge < pHelper->config.minRowsPerFileBlock))) { + + SFile *pFile = NULL; + + if (!pCompBlock->last) { + pFile = &(pHelper->files.dataF); + } else { + pFile = &(pHelper->files.lastF); + } + + if (tsdbWriteBlockToFile(pHelper, pFile, pDataCols, rowsCanMerge, &compBlock, pCompBlock->last) < 0) goto _err; + + // TODO: Add the sub-block + if (pCompBlock->numOfSubBlocks == 1) { + pCompBlock->numOfSubBlocks += 2; + // pCompBlock->offset = ; + // pCompBlock->len = ; + } else { + pCompBlock->numOfSubBlocks++; + } + pCompBlock->numOfPoints += rowsCanMerge; + pCompBlock->keyFirst = MIN(pCompBlock->keyFirst, dataColsKeyFirst(pDataCols)); + pCompBlock->keyLast = MAX(pCompBlock->keyLast, dataColsKeyAt(pDataCols, rowsCanMerge - 1)); + + // Update the Idx + // pIdx->hasLast = ; + // pIdx->len =; + // pIdx->numOfSuperBlocks = ; + + rowsWritten = rowsCanMerge; + } else { + // Read-Merge-Write as a super block + if (tsdbLoadBlockData(pHelper, blkIdx, NULL) < 0) goto _err; + tdMergeDataCols(pHelper->pDataCols[0], pDataCols, rowsCanMerge); + + int isLast = 0; + SFile *pFile = NULL; + if (!pCompBlock->last || (pCompBlock->numOfPoints + rowsCanMerge >= pHelper->config.minRowsPerFileBlock)) { + pFile = &(pHelper->files.dataF); + } else { + isLast = 1; + if (pHelper->files.nLastF.fd > 0) { + pFile = &(pHelper->files.nLastF); + } else { + pFile = &(pHelper->files.lastF); + } + } + + if (tsdbWriteBlockToFile(pHelper, pFile, pHelper->pDataCols[0], pCompBlock->numOfPoints + rowsCanMerge, &compBlock, isLast) < 0) goto _err; + + *pCompBlock = compBlock; + + pIdx->maxKey = MAX(pIdx->maxKey, compBlock.keyLast); + // pIdx->hasLast = ; + // pIdx-> + } + + return rowsWritten; + + _err: + return -1; +} + +static int compTSKEY(const void *key1, const void *key2) { return ((TSKEY *)key1 - (TSKEY *)key2); } + +// Get the number of rows the data can be merged into the block +static int tsdbGetRowsCanBeMergedWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols) { + int rowsCanMerge = 0; + TSKEY keyFirst = dataColsKeyFirst(pDataCols); + + SCompIdx * pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx; + + ASSERT(blkIdx < pIdx->numOfSuperBlocks); + + TSKEY keyMax = (blkIdx < pIdx->numOfSuperBlocks + 1) ? (pCompBlock + 1)->keyFirst - 1 : pHelper->files.maxKey; + + if (keyFirst > pCompBlock->keyLast) { + void *ptr = taosbsearch((void *)(&keyMax), pDataCols->cols[0].pData, pDataCols->numOfPoints, sizeof(TSKEY), + compTSKEY, TD_LE); + ASSERT(ptr != NULL); + + rowsCanMerge = + MIN((TSKEY *)ptr - (TSKEY *)pDataCols->cols[0].pData, pHelper->config.minRowsPerFileBlock - pCompBlock->numOfPoints); + + } else { + int32_t colId[1] = {0}; + if (tsdbLoadBlockDataCols(pHelper, NULL, &colId, 1) < 0) goto _err; + + int iter1 = 0; // For pDataCols + int iter2 = 0; // For loaded data cols + + while (1) { + if (iter1 >= pDataCols->numOfPoints || iter2 >= pHelper->pDataCols[0]->numOfPoints) break; + if (pCompBlock->numOfPoints + rowsCanMerge >= pHelper->config.maxRowsPerFileBlock) break; + + TSKEY key1 = dataColsKeyAt(pDataCols, iter1); + TSKEY key2 = dataColsKeyAt(pHelper->pDataCols[0], iter2); + + if (key1 > keyMax) break; + + if (key1 < key2) { + iter1++; + } else if (key1 == key2) { + iter1++; + iter2++; + } else { + iter2++; + rowsCanMerge++; + } + } + } + + return rowsCanMerge; + +_err: + return -1; +} \ No newline at end of file diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index 30d0e94950..ed12299050 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -369,14 +369,14 @@ static int32_t getFileCompInfo(STableCheckInfo* pCheckInfo, SFileGroup* fileGrou fileGroup->files[TSDB_FILE_TYPE_HEAD].fd = open(fileGroup->files[TSDB_FILE_TYPE_HEAD].fname, O_RDONLY); } - tsdbLoadCompIdx(fileGroup, pCheckInfo->compIndex, 10000); // todo set dynamic max tables - SCompIdx* compIndex = &pCheckInfo->compIndex[pCheckInfo->tableId.tid]; + // tsdbLoadCompIdx(fileGroup, pCheckInfo->compIndex, 10000); // todo set dynamic max tables + // SCompIdx* compIndex = &pCheckInfo->compIndex[pCheckInfo->tableId.tid]; - if (compIndex->len == 0 || compIndex->numOfSuperBlocks == 0) { // no data block in this file, try next file + // if (compIndex->len == 0 || compIndex->numOfSuperBlocks == 0) { // no data block in this file, try next file - } else { - tsdbLoadCompBlocks(fileGroup, compIndex, pCheckInfo->pCompInfo); - } + // } else { + // tsdbLoadCompBlocks(fileGroup, compIndex, pCheckInfo->pCompInfo); + // } return TSDB_CODE_SUCCESS; } @@ -444,7 +444,7 @@ static bool doLoadDataFromFileBlock(STsdbQueryHandle *pQueryHandle) { pFile->fd = open(pFile->fname, O_RDONLY); } - tsdbLoadDataBlock(pFile, pBlock, 1, pCheckInfo->pDataCols, data); + // tsdbLoadDataBlock(pFile, pBlock, 1, pCheckInfo->pDataCols, data); return true; } @@ -810,10 +810,10 @@ static bool getQualifiedDataBlock(STsdbQueryHandle *pQueryHandle, STableCheckInf pFile->fd = open(pFile->fname, O_RDONLY); } - if (tsdbLoadDataBlock(pFile, &pCheckInfo->pCompInfo->blocks[cur->slot], 1, - pCheckInfo->pDataCols, data) == 0) { - blockLoaded = true; - } + // if (tsdbLoadDataBlock(pFile, &pCheckInfo->pCompInfo->blocks[cur->slot], 1, + // pCheckInfo->pDataCols, data) == 0) { + // blockLoaded = true; + // } // dError("QInfo:%p fileId:%d total numOfBlks:%d blockId:%d load into memory failed due to error in disk files", // GET_QINFO_ADDR(pQuery), pQuery->fileId, pQuery->numOfBlocks, blkIdx); diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp index 9ee49d6a70..77f944cea2 100644 --- a/src/vnode/tsdb/tests/tsdbTests.cpp +++ b/src/vnode/tsdb/tests/tsdbTests.cpp @@ -54,7 +54,8 @@ TEST(TsdbTest, createRepo) { // 1. Create a tsdb repository tsdbSetDefaultCfg(&config); - tsdb_repo_t *pRepo = tsdbCreateRepo("/home/ubuntu/work/ttest/vnode0", &config, NULL); + tsdbCreateRepo("/home/ubuntu/work/ttest/vnode0", &config, NULL); + tsdb_repo_t *pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL); ASSERT_NE(pRepo, nullptr); // 2. Create a normal table @@ -139,42 +140,42 @@ TEST(TsdbTest, createRepo) { } // TEST(TsdbTest, DISABLED_openRepo) { -TEST(TsdbTest, openRepo) { - tsdb_repo_t *repo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL); - ASSERT_NE(repo, nullptr); +// TEST(TsdbTest, openRepo) { +// tsdb_repo_t *repo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL); +// ASSERT_NE(repo, nullptr); - STsdbRepo *pRepo = (STsdbRepo *)repo; +// STsdbRepo *pRepo = (STsdbRepo *)repo; - SFileGroup *pGroup = tsdbSearchFGroup(pRepo->tsdbFileH, 1833); +// SFileGroup *pGroup = tsdbSearchFGroup(pRepo->tsdbFileH, 1833); - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { - tsdbOpenFile(&pGroup->files[type], O_RDONLY); - } +// for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { +// tsdbOpenFile(&pGroup->files[type], O_RDONLY); +// } - SCompIdx *pIdx = (SCompIdx *)calloc(pRepo->config.maxTables, sizeof(SCompIdx)); - tsdbLoadCompIdx(pGroup, (void *)pIdx, pRepo->config.maxTables); +// SCompIdx *pIdx = (SCompIdx *)calloc(pRepo->config.maxTables, sizeof(SCompIdx)); +// tsdbLoadCompIdx(pGroup, (void *)pIdx, pRepo->config.maxTables); - SCompInfo *pCompInfo = (SCompInfo *)malloc(sizeof(SCompInfo) + pIdx[1].len); +// SCompInfo *pCompInfo = (SCompInfo *)malloc(sizeof(SCompInfo) + pIdx[1].len); - tsdbLoadCompBlocks(pGroup, &pIdx[0], (void *)pCompInfo); +// tsdbLoadCompBlocks(pGroup, &pIdx[0], (void *)pCompInfo); - int blockIdx = 0; - SCompBlock *pBlock = &(pCompInfo->blocks[blockIdx]); +// int blockIdx = 0; +// SCompBlock *pBlock = &(pCompInfo->blocks[blockIdx]); - SCompData *pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols); +// SCompData *pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols); - tsdbLoadCompCols(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, (void *)pCompData); +// tsdbLoadCompCols(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, (void *)pCompData); - STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, pCompData->uid); - SDataCols *pDataCols = tdNewDataCols(tdMaxRowBytesFromSchema(pTable->schema), 5, 10); - tdInitDataCols(pDataCols, pTable->schema); +// STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, pCompData->uid); +// SDataCols *pDataCols = tdNewDataCols(tdMaxRowBytesFromSchema(pTable->schema), 5, 10); +// tdInitDataCols(pDataCols, pTable->schema); - tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, 1, pDataCols, pCompData); +// tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, 1, pDataCols, pCompData); - int k = 0; +// int k = 0; -} +// } TEST(TsdbTest, DISABLED_createFileGroup) { SFileGroup fGroup; From bed8890fe808c67f8fc4046dbb41658e99a851ef Mon Sep 17 00:00:00 2001 From: hzcheng Date: Wed, 8 Apr 2020 18:04:03 +0800 Subject: [PATCH 02/29] TD-100 --- src/vnode/tsdb/inc/tsdbMain.h | 35 +- src/vnode/tsdb/src/tsdbMain.c | 10 +- src/vnode/tsdb/src/tsdbRWHelper.c | 630 ++++++------------------------ 3 files changed, 135 insertions(+), 540 deletions(-) diff --git a/src/vnode/tsdb/inc/tsdbMain.h b/src/vnode/tsdb/inc/tsdbMain.h index 54f5ab0cce..949bb9d48b 100644 --- a/src/vnode/tsdb/inc/tsdbMain.h +++ b/src/vnode/tsdb/inc/tsdbMain.h @@ -385,10 +385,11 @@ typedef struct { SHelperFile files; SHelperTable tableInfo; + SCompIdx compIdx; // SCompIdx of current table - // ---------- For read purpose int8_t state; // current loading state + // Information in .head file SCompIdx *pCompIdx; size_t compIdxSize; @@ -396,33 +397,24 @@ typedef struct { size_t compInfoSize; int blockIter; // For write purpose + // Information in .data or .last file SCompData *pCompData; size_t compDataSize; SDataCols *pDataCols[2]; - // ---------- For read purpose - bool hasLast; - - int newBlocks; - SCompIdx *pWCompIdx; - size_t wCompIdxSize; - - SCompInfo *pWCompInfo; - size_t wCompInfoSize; - - SCompData *pWCompData; - size_t wCompDataSize; + // Compression buffer + void * cBuffer; + size_t cBufSize; } SRWHelper; // --------- Helper state -#define TSDB_HELPER_CLEAR_STATE 0x0 // Clear state -#define TSDB_HELPER_FILE_SET 0x1 // File is set -#define TSDB_HELPER_FILE_OPEN 0x2 // File is opened - -#define TSDB_HELPER_IDX_LOAD 0x4 // SCompIdx part is loaded -#define TSDB_HELPER_INFO_LOAD 0x8 // SCompInfo part is loaded -#define TSDB_HELPER_FILE_DATA_LOAD 0x10 // SCompData part is loaded +#define TSDB_HELPER_CLEAR_STATE 0x0 // Clear state +#define TSDB_HELPER_FILE_SET_AND_OPEN 0x1 // File is set +#define TSDB_HELPER_IDX_LOAD 0x2 // SCompIdx part is loaded +#define TSDB_HELPER_TABLE_SET 0x4 // Table is set +#define TSDB_HELPER_INFO_LOAD 0x8 // SCompInfo part is loaded +#define TSDB_HELPER_FILE_DATA_LOAD 0x10 // SCompData part is loaded #define TSDB_HELPER_TYPE(h) ((h)->config.type) @@ -435,8 +427,7 @@ void tsdbDestroyHelper(SRWHelper *pHelper); void tsdbClearHelper(SRWHelper *pHelper); // --------- For set operations -int tsdbSetHelperFile(SRWHelper *pHelper, SFileGroup *pGroup); -int tsdbOpenHelperFile(SRWHelper *pHelper); +int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup); void tsdbSetHelperTable(SRWHelper *pHelper, SHelperTable *pHelperTable, STSchema *pSchema); int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError); diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index b5d573764e..1596cd54d3 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -889,23 +889,20 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters tsdbGetDataDirName(pRepo, dataDir); if ((pGroup = tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables)) == NULL) goto _err; - // Set the file to write/read - tsdbSetHelperFile(pHelper, pGroup); - // Open files for write/read - if (tsdbOpenHelperFile(pHelper) < 0) goto _err; + if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) goto _err; // Loop to commit data in each table for (int tid = 0; tid < pCfg->maxTables; tid++) { STable * pTable = pMeta->tables[tid]; SSkipListIterator *pIter = iters[tid]; + // Set the helper and the buffer dataCols object to help to write this table SHelperTable hTable = {.uid = pTable->tableId.uid, .tid = pTable->tableId.tid, .sversion = pTable->sversion}; tsdbSetHelperTable(pHelper, &hTable, tsdbGetTableSchema(pMeta, pTable)); tdInitDataCols(pDataCols, tsdbGetTableSchema(pMeta, pTable)); - // Loop to write the data in the cache to files, if no data to write, just break - // the loop + // Loop to write the data in the cache to files. If no data to write, just break the loop int maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5; while (true) { int rowsRead = tsdbReadRowsFromCache(pIter, maxKey, maxRowsToRead, pDataCols); @@ -939,6 +936,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters return 0; _err: + ASSERT(false); tsdbCloseHelperFile(pHelper, 1); return -1; } diff --git a/src/vnode/tsdb/src/tsdbRWHelper.c b/src/vnode/tsdb/src/tsdbRWHelper.c index 79fe3c49f7..2e01d9efec 100644 --- a/src/vnode/tsdb/src/tsdbRWHelper.c +++ b/src/vnode/tsdb/src/tsdbRWHelper.c @@ -35,7 +35,7 @@ static void tsdbClearHelperRead(SRWHelper *pHelper); static void tsdbClearHelperWrite(SRWHelper *pHelper); static bool tsdbShouldCreateNewLast(SRWHelper *pHelper); static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, int rowsToWrite, SCompBlock *pCompBlock, - bool isLast); + bool isLast, bool isSuperBlock); static int compareKeyBlock(const void *arg1, const void *arg2); static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols); static int nRowsLEThan(SDataCols *pDataCols, int maxKey); @@ -78,17 +78,22 @@ void tsdbClearHelper(SRWHelper *pHelper) { tsdbClearHelperWrite(pHelper); } -int tsdbSetHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { - // TODO: reset the helper object +int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { + ASSERT(pHelper != NULL && pGroup != NULL); + // Clear the helper object + tsdbClearHelper(pHelper); + + ASSERT(pHelper->state == TSDB_HELPER_CLEAR_STATE); + + // Set the files pHelper->files.fid = pGroup->fileId; - pHelper->files.headF = pGroup->files[TSDB_FILE_TYPE_HEAD]; pHelper->files.dataF = pGroup->files[TSDB_FILE_TYPE_DATA]; pHelper->files.lastF = pGroup->files[TSDB_FILE_TYPE_LAST]; - if (TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER) { char *fnameDup = strdup(pHelper->files.headF.fname); + if (fnameDup == NULL) goto _err; if (fnameDup == NULL) return -1; char *dataDir = dirname(fnameDup); @@ -96,32 +101,33 @@ int tsdbSetHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { tsdbGetFileName(dataDir, pHelper->files.fid, ".l", pHelper->files.nLastF.fname); free((void *)fnameDup); } - return 0; -} - -int tsdbOpenHelperFile(SRWHelper *pHelper) { - // TODO: check if the file is set - {} + // Open the files + if (tsdbOpenFile(&(pHelper->files.headF), O_RDONLY) < 0) goto _err; if (TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER) { - if (tsdbOpenFile(&(pHelper->files.headF), O_RDONLY) < 0) goto _err; if (tsdbOpenFile(&(pHelper->files.dataF), O_RDWR) < 0) goto _err; if (tsdbOpenFile(&(pHelper->files.lastF), O_RDWR) < 0) goto _err; - // TODO: need to write head and compIdx part + + // Create and open .h if (tsdbOpenFile(&(pHelper->files.nHeadF), O_WRONLY | O_CREAT) < 0) goto _err; + size_t tsize = TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pHelper->config.maxTables; + if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, tsize) < tsize) goto _err; + + // Create and open .l file if should if (tsdbShouldCreateNewLast(pHelper)) { if (tsdbOpenFile(&(pHelper->files.nLastF), O_WRONLY | O_CREAT) < 0) goto _err; + if (tsendfile(pHelper->files.nLastF.fd, pHelper->files.lastF.fd, NULL, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) goto _err; } } else { - if (tsdbOpenFile(&(pHelper->files.headF), O_RDONLY) < 0) goto _err; if (tsdbOpenFile(&(pHelper->files.dataF), O_RDONLY) < 0) goto _err; if (tsdbOpenFile(&(pHelper->files.lastF), O_RDONLY) < 0) goto _err; } - return 0; + helperSetState(pHelper, TSDB_HELPER_FILE_SET_AND_OPEN); -_err: - tsdbCloseHelperFile(pHelper, true); + return tsdbLoadCompIdx(pHelper, NULL); + + _err: return -1; } @@ -153,31 +159,44 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { } void tsdbSetHelperTable(SRWHelper *pHelper, SHelperTable *pHelperTable, STSchema *pSchema) { - // TODO: check if it is available to set the table + ASSERT(helperHasState(pHelper, TSDB_HELPER_FILE_SET_AND_OPEN)); + + // Clear members and state used by previous table + pHelper->blockIter = 0; + pHelper->state &= (TSDB_HELPER_TABLE_SET - 1); pHelper->tableInfo = *pHelperTable; - // TODO: Set the pDataCols according to schema + tdInitDataCols(pHelper->pDataCols[0], pSchema); + tdInitDataCols(pHelper->pDataCols[1], pSchema); - // TODO: set state + pHelper->compIdx = pHelper->pCompIdx[pHelper->tableInfo.tid]; + + helperSetState(pHelper, TSDB_HELPER_TABLE_SET); } +/** + * Write part of of points from pDataCols to file + * + * @return: number of points written to file successfully + * -1 for failure + */ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { ASSERT(TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER); - ASSERT(helperHasState(pHelper, TSDB_HELPER_FILE_SET) && helperHasState(pHelper, TSDB_HELPER_FILE_OPEN)); + SCompBlock compBlock; int rowsToWrite = 0; TSKEY keyFirst = dataColsKeyFirst(pDataCols); - // Load SCompIdx part if not loaded yet - if ((!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) && (tsdbLoadCompIdx(pHelper, NULL) < 0)) goto _err; + ASSERT(helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)); + SCompIdx curIdx = pHelper->compIdx; // old table SCompIdx for sendfile usage + SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; // for change purpose // Load the SCompInfo part if neccessary - SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; - if ((pIdx->offset > 0) && (pIdx->hasLast || dataColsKeyFirst(pDataCols) <= pIdx->maxKey)) { - if (tsdbLoadCompInfo(pHelper, NULL) < 0) goto _err; - } - - SCompIdx *pWIdx = pHelper->pWCompIdx + pHelper->tableInfo.tid; + ASSERT(helperHasState(pHelper, TSDB_HELPER_TABLE_SET)); + if ((!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) && + ((pIdx->offset > 0) && (pIdx->hasLast || dataColsKeyFirst(pDataCols) <= pIdx->maxKey)) && + (tsdbLoadCompInfo(pHelper, NULL) < 0)) + goto _err; if (!pIdx->hasLast && keyFirst > pIdx->maxKey) { // Just need to append as a super block @@ -189,10 +208,10 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { pWFile = &(pHelper->files.dataF); } else { isLast = true; - pWFile = (pHelper->files.nLastF.fd > 0) ? &(pHelper->files.nLastF) : &(pHelper->files.nLastF); + pWFile = (pHelper->files.nLastF.fd > 0) ? &(pHelper->files.nLastF) : &(pHelper->files.lastF); } - if (tsdbWriteBlockToFile(pHelper, pWFile, pDataCols, rowsToWrite, &compBlock, isLast) < 0) goto _err; + if (tsdbWriteBlockToFile(pHelper, pWFile, pDataCols, rowsToWrite, &compBlock, isLast, true) < 0) goto _err; // TODO: may need to reallocate the memory pHelper->pCompInfo->blocks[pHelper->blockIter++] = compBlock; @@ -259,39 +278,44 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { } int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { - // TODO: check helper state - ASSERT(!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)); + ASSERT(pHelper->state = TSDB_HELPER_FILE_SET_AND_OPEN); - int fd = pHelper->files.headF.fd; + if (!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) { + // If not load from file, just load it in object + int fd = pHelper->files.headF.fd; - if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1; - if (tread(fd, pHelper->pCompIdx, pHelper->compIdxSize) < pHelper->compIdxSize) return -1; - // TODO: check the checksum - - if (target) memcpy(target, pHelper->pCompIdx, pHelper->compIdxSize); + if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1; + if (tread(fd, (void *)(pHelper->pCompIdx), pHelper->compIdxSize) < pHelper->compIdxSize) return -1; + // TODO: check the correctness of the part + } helperSetState(pHelper, TSDB_HELPER_IDX_LOAD); + // Copy the memory for outside usage + if (target) memcpy(target, pHelper->pCompIdx, pHelper->compIdxSize); + return 0; } int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { - SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + ASSERT(helperHasState(pHelper, TSDB_HELPER_TABLE_SET)); - ASSERT(pIdx->offset > 0); + SCompIdx curCompIdx = pHelper->compIdx; + + ASSERT(curCompIdx.offset > 0 && curCompIdx.len > 0); int fd = pHelper->files.headF.fd; - if (lseek(fd, pIdx->offset, SEEK_SET) < 0) return -1; - ASSERT(pIdx->len > 0); + if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { + if (lseek(fd, curCompIdx.offset, SEEK_SET) < 0) return -1; - adjustMem(pHelper->pCompInfo, pHelper->compInfoSize, pIdx->len); - if (tread(fd, (void *)(pHelper->pCompInfo), pIdx->len) < 0) return -1; - // TODO: check the checksum + adjustMem(pHelper->pCompInfo, pHelper->compInfoSize, curCompIdx.len); + if (tread(fd, (void *)(pHelper->pCompInfo), pHelper) < 0) return -1; + // TODO: check the checksum - // TODO: think about when target has no space for the content - if (target) memcpy(target, (void *)(pHelper->pCompInfo), pIdx->len); + helperSetState(pHelper, TSDB_HELPER_INFO_LOAD); + } - helperSetState(pHelper, TSDB_HELPER_INFO_LOAD); + if (target) memcpy(target, (void *)(pHelper->pCompInfo), curCompIdx.len); return 0; } @@ -376,21 +400,21 @@ static void tsdbDestroyHelperRead(SRWHelper *pHelper) { static int tsdbInitHelperWrite(SRWHelper *pHelper) { SHelperCfg *pCfg = &(pHelper->config); - pHelper->wCompIdxSize = pCfg->maxTables * sizeof(SCompIdx); - if ((pHelper->pWCompIdx = (SCompIdx *)malloc(pHelper->wCompIdxSize)) == NULL) return -1; + // pHelper->wCompIdxSize = pCfg->maxTables * sizeof(SCompIdx); + // if ((pHelper->pWCompIdx = (SCompIdx *)malloc(pHelper->wCompIdxSize)) == NULL) return -1; return 0; } static void tsdbDestroyHelperWrite(SRWHelper *pHelper) { - tfree(pHelper->pWCompIdx); - pHelper->wCompIdxSize = 0; + // tfree(pHelper->pWCompIdx); + // pHelper->wCompIdxSize = 0; - tfree(pHelper->pWCompInfo); - pHelper->wCompInfoSize = 0; + // tfree(pHelper->pWCompInfo); + // pHelper->wCompInfoSize = 0; - tfree(pHelper->pWCompData); - pHelper->wCompDataSize = 0; + // tfree(pHelper->pWCompData); + // pHelper->wCompDataSize = 0; } static void tsdbClearHelperRead(SRWHelper *pHelper) { @@ -407,17 +431,21 @@ static bool tsdbShouldCreateNewLast(SRWHelper *pHelper) { } static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, int rowsToWrite, SCompBlock *pCompBlock, - bool isLast) { - ASSERT(rowsToWrite > 0 && rowsToWrite <= pDataCols->numOfPoints); + bool isLast, bool isSuperBlock) { + ASSERT(rowsToWrite > 0 && rowsToWrite <= pDataCols->numOfPoints && + rowsToWrite <= pHelper->config.maxRowsPerFileBlock); - int64_t offset = lseek(pFile->fd, 0, SEEK_END); + SCompData *pCompData = NULL; + int64_t offset = 0; + + offset = lseek(pFile->fd, 0, SEEK_END); if (offset < 0) goto _err; - SCompData *pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * pDataCols->numOfCols); + pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * pDataCols->numOfCols); if (pCompData == NULL) goto _err; int nColsNotAllNull = 0; - int32_t toffset; + int32_t toffset = 0; for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) { SDataCol *pDataCol = pDataCols->cols + ncol; SCompCol *pCompCol = pCompData->cols + nColsNotAllNull; @@ -427,481 +455,59 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa continue; } + // Compress the data here + {} + pCompCol->colId = pDataCol->colId; pCompCol->type = pDataCol->type; - pCompCol->len = pDataCol->len; + pCompCol->len = TYPE_BYTES[pCompCol->type] * rowsToWrite; // TODO: change it pCompCol->offset = toffset; nColsNotAllNull++; + toffset += pCompCol->len; } + ASSERT(nColsNotAllNull > 0); + pCompData->delimiter = TSDB_FILE_DELIMITER; pCompData->uid = pHelper->tableInfo.uid; pCompData->numOfCols = nColsNotAllNull; size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * nColsNotAllNull; if (twrite(pFile->fd, (void *)pCompData, tsize) < tsize) goto _err; - for (int i = 0; i < pDataCols->numOfCols; i++) { - SDataCol *pDataCol = pCompData->cols + i; - SCompCol *pCompCol = NULL; - if (twrite(pFile->fd, (void *)(pDataCol->pData), pCompCol->len) < pCompCol->len) goto _err; + int nCompCol = 0; + for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) { + ASSERT(nCompCol < nColsNotAllNull); + + SDataCol *pDataCol = pDataCols->cols + ncol; + SCompCol *pCompCol = pCompData->cols + nCompCol; + + if (pDataCol->colId == pCompCol->colId) { + if (twrite(pFile->fd, (void *)(pDataCol->pData), pCompCol->len) < pCompCol->len) goto _err; + tsize += pCompCol->len; + nCompCol++; + } } pCompBlock->last = isLast; pCompBlock->offset = offset; - // pCOmpBlock->algorithm = ; + pCompBlock->algorithm = 2; // TODO pCompBlock->numOfPoints = rowsToWrite; pCompBlock->sversion = pHelper->tableInfo.sversion; - // pCompBlock->len = ; - // pCompBlock->numOfSubBlocks = ; + pCompBlock->len = (int32_t)tsize; + pCompBlock->numOfSubBlocks = isSuperBlock ? 1 : 0; pCompBlock->numOfCols = nColsNotAllNull; - // pCompBlock->keyFirst = ; - // pCompBlock->keyLast = ; + pCompBlock->keyFirst = dataColsKeyFirst(pDataCols); + pCompBlock->keyLast = dataColsKeyAt(pDataCols, rowsToWrite - 1); + tfree(pCompData); return 0; _err: + tfree(pCompData); return -1; } -// static int compareKeyBlock(const void *arg1, const void *arg2); - -// /** -// * Init a read-write helper object for read or write usage. -// */ -// int tsdbInitHelper(SRWHelper *pHelper, int maxTables, tsdb_rwhelper_t type, int maxRowSize, int maxRows, -// int maxCols) { -// if (pHelper == NULL) return -1; - -// memset((void *)pHelper, 0, sizeof(SRWHelper)); -// for (int ftype = TSDB_RW_HEADF; ftype <= TSDB_RW_LF; ftype++) { -// pHelper->files[ftype] = -1; -// } - -// // Set type -// pHelper->type = type; - -// // Set global configuration -// pHelper->maxTables = maxTables; -// pHelper->maxRowSize = maxRowSize; -// pHelper->maxRows = maxRows; -// pHelper->maxCols = maxCols; - -// // Allocate SCompIdx part memory -// pHelper->compIdxSize = sizeof(SCompIdx) * maxTables; -// pHelper->pCompIdx = (SCompIdx *)malloc(pHelper->compIdxSize); -// if (pHelper->pCompIdx == NULL) goto _err; - -// pHelper->compDataSize = sizeof(SCompData) + sizeof(SCompCol) * maxCols; -// pHelper->pCompData = (SCompData *)malloc(pHelper->compDataSize); - -// pHelper->pDataCols = tdNewDataCols(maxRowSize, maxCols, maxRows); -// if (pHelper->pDataCols == NULL) goto _err; - -// return 0; - -// _err: -// tsdbDestroyHelper(pHelper); -// return -1; -// } - -// void tsdbResetHelper(SRWHelper *pHelper) { -// if (pHelper->headF.fd > 0) { -// close(pHelper->headF.fd); -// pHelper->headF.fd = -1; -// } -// if (pHelper->dataF.fd > 0) { -// close(pHelper->dataF.fd); -// pHelper->dataF.fd = -1; -// } -// if (pHelper->lastF.fd > 0) { -// close(pHelper->lastF.fd); -// pHelper->lastF.fd = -1; -// } -// if (pHelper->hF.fd > 0) { -// close(pHelper->hF.fd); -// pHelper->hF.fd = -1; -// } -// if (pHelper->lF.fd > 0) { -// close(pHelper->lF.fd); -// pHelper->lF.fd = -1; -// } - -// pHelper->state = 0; -// tdResetDataCols(pHelper->pDataCols); -// } - -// int tsdbDestroyHelper(SRWHelper *pHelper) { -// if (pHelper->headF.fd > 0) close(pHelper->headF.fd); -// if (pHelper->dataF.fd > 0) close(pHelper->dataF.fd); -// if (pHelper->lastF.fd > 0) close(pHelper->lastF.fd); -// if (pHelper->hF.fd > 0) close(pHelper->hF.fd); -// if (pHelper->lF.fd > 0) close(pHelper->lF.fd); - -// if (pHelper->pCompIdx) free(pHelper->pCompIdx); -// if (pHelper->pCompInfo) free(pHelper->pCompInfo); -// if (pHelper->pCompData) free(pHelper->pCompData); -// memset((void *)pHelper, 0, sizeof(SRWHelper)); -// return 0; -// } - -// int tsdbSetHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { -// if (pHelper->state != 0) return -1; - -// pHelper->fid = pGroup->fileId; - -// pHelper->headF = pGroup->files[TSDB_FILE_TYPE_HEAD]; -// pHelper->headF.fd = -1; -// pHelper->dataF = pGroup->files[TSDB_FILE_TYPE_DATA]; -// pHelper->dataF.fd = -1; -// pHelper->lastF = pGroup->files[TSDB_FILE_TYPE_LAST]; -// pHelper->lastF.fd = -1; - -// if (pHelper->mode == TSDB_WRITE_HELPER) { -// char *fnameCpy = strdup(pHelper->headF.fname); -// if (fnameCpy == NULL) return -1; -// char *dataDir = dirname(fnameCpy); - -// memset((void *)(&pHelper->hF), 0, sizeof(SFile)); -// memset((void *)(&pHelper->lF), 0, sizeof(SFile)); -// pHelper->hF.fd = -1; -// pHelper->lF.fd = -1; - -// tsdbGetFileName(dataDir, pHelper->fid, ".h", pHelper->hF.fname); -// tsdbGetFileName(dataDir, pHelper->fid, ".l", pHelper->lF.fname); -// free((char *)fnameCpy); -// } - -// TSDB_SET_RWHELPER_STATE(pHelper, TSDB_RWHELPER_FILE_SET); - -// return 0; -// } - -// static int tsdbNeedToCreateNewLastFile() { -// // TODO -// return 0; -// } - -// int tsdbCloseHelperFile(SRWHelper *pHelper, int hasErr) { -// int ret = 0; -// if (pHelper->headF.fd > 0) { -// close(pHelper->headF.fd); -// pHelper->headF.fd = -1; -// } -// if (pHelper->dataF.fd > 0) { -// close(pHelper->dataF.fd); -// pHelper->dataF.fd = -1; -// } -// if (pHelper->lastF.fd > 0) { -// close(pHelper->lastF.fd); -// pHelper->lastF.fd = -1; -// } -// if (pHelper->hF.fd > 0) { -// close(pHelper->hF.fd); -// pHelper->hF.fd = -1; -// if (hasErr) remove(pHelper->hF.fname); -// } -// if (pHelper->lF.fd > 0) { -// close(pHelper->lF.fd); -// pHelper->lF.fd = -1; -// if (hasErr) remove(pHelper->hF.fname); -// } -// return 0; -// } - -// int tsdbOpenHelperFile(SRWHelper *pHelper) { -// if (pHelper->state != TSDB_RWHELPER_FILE_SET) return -1; - -// if (pHelper->mode == TSDB_READ_HELPER) { // The read helper -// if (tsdbOpenFile(&pHelper->headF, O_RDONLY) < 0) goto _err; -// if (tsdbOpenFile(&pHelper->dataF, O_RDONLY) < 0) goto _err; -// if (tsdbOpenFile(&pHelper->lastF, O_RDONLY) < 0) goto _err; -// } else { -// if (tsdbOpenFile(&pHelper->headF, O_RDONLY) < 0) goto _err; -// if (tsdbOpenFile(&pHelper->dataF, O_RDWR) < 0) goto _err; -// if (tsdbOpenFile(&pHelper->lastF, O_RDWR) < 0) goto _err; -// // Open .h and .l file -// if (tsdbOpenFile(&pHelper->hF, O_WRONLY | O_CREAT) < 0) goto _err; -// if (tsdbNeedToCreateNewLastFile()) { -// if (tsdbOpenFile(&pHelper->lF, O_WRONLY | O_CREAT) < 0) goto _err; -// } -// } - -// TSDB_SET_RWHELPER_STATE(pHelper, TSDB_RWHELPER_FILE_OPENED); - -// return 0; -// _err: -// tsdbCloseHelperFile(pHelper, 1); -// return -1; -// } - -// int tsdbLoadCompIdx(SRWHelper *pHelper) { -// if (pHelper->state != (TSDB_RWHELPER_FILE_SET | TSDB_RWHELPER_FILE_OPENED)) return -1; - -// if (lseek(pHelper->headF.fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1; -// if (tread(pHelper->headF.fd, (void *)(pHelper->pCompIdx), pHelper->compIdxSize) < pHelper->compIdxSize) return -1; - -// TSDB_SET_RWHELPER_STATE(pHelper, TSDB_RWHELPER_COMPIDX_LOADED); - -// return 0; -// } - -// int tsdbSetHelperTable(SRWHelper *pHelper, int32_t tid, int64_t uid, STSchema *pSchema) { -// // TODO: add some check information -// pHelper->tid = tid; -// pHelper->uid = uid; - -// tdInitDataCols(pHelper->pDataCols, pSchema); - -// TSDB_SET_RWHELPER_STATE(pHelper, TSDB_RWHELPER_TABLE_SET); - -// return 0; -// } - -// int tsdbLoadCompBlocks(SRWHelper *pHelper) { -// SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tid; -// if (pIdx->offset <= 0) return 0; - -// if (lseek(pHelper->headF.fd, pIdx->offset, SEEK_SET) < 0) return -1; -// if (pHelper->compInfoSize < pIdx->len) { -// pHelper->pCompInfo = (SCompInfo *)realloc((void *)(pHelper->pCompInfo), pIdx->len); -// if (pHelper->pCompInfo == NULL) return -1; -// pHelper->compInfoSize = pIdx->len; -// } - -// if (tread(pHelper->headF.fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) return -1; - -// TSDB_SET_RWHELPER_STATE(pHelper, TSDB_RWHELPER_COMPBLOCK_LOADED); - -// return 0; -// } - -// int tsdbRWHelperSetBlockIdx(SRWHelper *pHelper, int blkIdx) { -// SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tid; -// if (blkIdx > pIdx->numOfSuperBlocks) return -1; - -// pHelper->blkIdx = blkIdx; - -// TSDB_SET_RWHELPER_STATE(pHelper, TSDB_RWHELPER_BLOCKIDX_SET); - -// return 0; -// } - -// int tsdbRWHelperLoadCompData(SRWHelper *pHelper) { -// SCompBlock *pBlock = pHelper->pCompInfo->blocks + pHelper->blkIdx; - -// if (pBlock->numOfSubBlocks == 1) { // Only one super block -// size_t size = sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols; -// if (size > pHelper->compDataSize) { -// pHelper->pCompData = (SCompData *)realloc((void *)pHelper->pCompData, size); -// if (pHelper->pCompData == NULL) return -1; -// pHelper->compDataSize = size; -// } - -// if (lseek(pHelper->dataF.fd, pBlock->offset, SEEK_SET) < 0) return -1; -// if (tread(pHelper->dataF.fd, (void *)(pHelper->pCompData), size) < size) return -1; -// } else { // TODO: More sub blocks -// } - -// TSDB_SET_RWHELPER_STATE(pHelper, TSDB_RWHELPER_COMPCOL_LOADED); - -// return 0; -// } - - -// static int compColIdCompCol(const void *arg1, const void *arg2) { -// int colId = *(int *)arg1; -// SCompCol *pCompCol = (SCompCol *)arg2; - -// return (int)(colId - pCompCol->colId); -// } - -// static int compColIdDataCol(const void *arg1, const void *arg2) { -// int colId = *(int *)arg1; -// SDataCol *pDataCol = (SDataCol *)arg2; - -// return (int)(colId - pDataCol->colId); -// } - -// int tsdbRWHelperLoadColData(SRWHelper *pHelper, int colId) { -// SCompBlock *pBlock = pHelper->pCompInfo->blocks + pHelper->blkIdx; - -// if (pBlock->numOfSubBlocks == 1) { // only one super block -// SCompCol *pCompCol = bsearch((void *)(&colId), (void *)(pHelper->pCompData->cols), pBlock->numOfCols, compColIdCompCol, compColIdCompCol); -// if (pCompCol == NULL) return 0; // No data to read from this block , but we still return 0 - -// SDataCol *pDataCol = bsearch((void *)(&colId), (void *)(pHelper->pDataCols->cols), pHelper->pDataCols->numOfCols, sizeof(SDataCol), compColIdDataCol); -// assert(pDataCol != NULL); - -// int fd = (pBlock->last) ? pHelper->lastF.fd : pHelper->dataF.fd; -// if (lseek(fd, pBlock->offset + pCompCol->offset, SEEK_SET) < 0) return -1; -// if (tread(fd, (void *)pDataCol->pData, pCompCol->len) < pCompCol->len) return -1; -// pDataCol->len = pCompCol->len; -// } else { -// // TODO: more than 1 blocks -// } -// return 0; -// } - -// int tsdbRWHelperLoadBlockData(SRWHelper *pHelper, int blkIdx) { -// SCompBlock *pBlock = pHelper->pCompInfo->blocks + pHelper->blkIdx; - -// if (pBlock->numOfSubBlocks == 1) { -// for (int i = 0; i < pHelper->pDataCols->numOfCols; i++) { -// if (tsdbRWHelperLoadBlockData(pHelper, pHelper->pDataCols->cols[i].colId) < 0) return -1; -// } -// } else { -// // TODO: more than 1 block of data -// } -// return 0; -// } - -// int tsdbRWHelperCopyCompBlockPart(SRWHelper *pHelper) { -// // TODO -// return 0; -// } - -// int tsdbRWHelperCopyDataBlockPart(SRWHelper *pHelper ) { -// // TODO -// return 0; -// } - -// int tsdbRWHelperWriteCompIdx(SRWHelper *pHelper) { -// // TODO -// if (lseek(pHelper->hF.fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1; -// if (twrite(pHelper->hF.fd, (void *)(pHelper->pCompIdx), pHelper->compIdxSize) < pHelper->compIdxSize) return -1; - -// return 0; -// } - -// /** -// * Load the data block from file -// * -// * @return 0 for success -// * -1 for failure -// */ -// int tsdbLoadDataBlock(SRWHelper *pHelper, int bldIdx) { -// SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tid; -// if (pIdx->) - -// return 0; -// } - -// /** -// * Append the block to a file, either .data -// */ -// int tsdbAppendBlockToFile(SRWHelper *pHelper, tsdb_rw_file_t toFile, SDataCols *pDataCols, SCompBlock *pCompBlock, bool isSuper) { -// SFile *pFile = pHelper->files + toFile; - -// int64_t offset = lseek(pFile->fd, 0, SEEK_END); -// if (*offset < 0) return -1; - -// SCompData *pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * pDataCols->numOfCols); -// if (pCompData == NULL) return -1; - - -// int numOfNotAllNullCols = 0; -// int32_t toffset = 0; -// for (int i = 0; i < pDataCols->numOfCols; i++) { -// SDataCol *pDataCol = pDataCols->cols + i; -// SCompCol *pCompCol = pCompData->cols + numOfNotAllNullCols; - -// if (0 /* All data in this column are NULL value */) { -// continue; -// } -// pCompCol->colId = pDataCol->colId; -// pCompCol->type = pDataCol->type; -// pCompCol->len = pDataCol->len; -// // pCompCol->offset = toffset; -// numOfNotAllNullCols++; -// // toffset += pDataCol->len; -// } - -// pCompData->delimiter = TSDB_FILE_DELIMITER; -// pCompData->numOfCols = numOfNotAllNullCols; -// pCompData->uid = pHelper->uid; - -// size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * numOfNotAllNullCols; -// if (twrite(pFile->fd, (void *)pCompData, tsize) < 0) return -1; -// for (int i = 0; i < numOfNotAllNullCols; i++) { -// SCompCol *pCompCol = pCompData->cols + i; -// SDataCol *pDataCol = NULL; // bsearch() -// tassert(pDataCol != NULL); -// if (twrite(pFile->fd, (void *)(pDataCol->pData), pDataCol->len) < pDataCol->len) return -1; -// } - -// pCompBlock->last = (toFile == TSDB_RW_DATAF) ? 0 : 1; -// pCompBlock->offset = offset; -// pCompBlock->algorithm = pHelper->compression; -// pCompBlock->numOfPoints = pDataCols->numOfPoints; -// pCompBlock->sversion = pHelper->sversion; -// // pCompBlock->len = ; -// pCompBlock->numOfSubBlocks = isSuper ? 1 : 0; -// pCompBlock->numOfCols = numOfNotAllNullCols; -// pCompBlock->keyFirst = dataColsKeyFirst(pDataCols); -// pCompBlock->keyLast = dataColsKeyLast(pDataCols); - -// return 0; -// } - -// /** -// * Write the whole or part of the cached data block to file. -// * -// * There are four options: -// * 1. Append the whole block as a SUPER-BLOCK at the end -// * 2. Append part/whole block as a SUPER-BLOCK and insert in the middle -// * 3. Append part/whole block as a SUB-BLOCK -// * 4. Merge part/whole block as a SUPER-BLOCK -// */ -// int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { -// tassert(pHelper->type == TSDB_WRITE_HELPER); - -// int rowsWritten = 0; -// SCompBlock compBlock; - -// SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tid; -// // if ((no old data) OR (no last block AND cached first key is larger than the max key)) -// if ((pIdx->offset == 0) || (pIdx->hasLast && dataColsKeyFirst(pDataCols) > pIdx->maxKey)) { -// // Append the whole block as a SUPER-BLOCK at the end -// if (pDataCols->numOfPoints >= pHelper->minRowPerFileBlock) { -// if (tsdbAppendBlockToFile(pHelper, TSDB_RW_DATAF, pDataCols, &compBlock, true) < 0) goto _err; -// } else { -// tsdb_rw_file_t ftype = (pHelper->files[TSDB_RW_LF].fd > 0) ? TSDB_RW_LF : TSDB_RW_LASTF; -// if (tsdbAppendBlockToFile(pHelper, ftype, pDataCols, &compBlock, true) < 0) goto _err; -// } -// // Copy the compBlock part to the end -// if (IS_COMPBLOCK_LOADED(pHelper)) { - -// } else { - -// } - -// pIdx->hasLast = compBlock.last; -// pIdx->len += sizeof(compBlock); -// pIdx->numOfSuperBlocks++; -// pIdx->maxKey = compBlock.keyLast; - -// rowsWritten = pDataCols->numOfPoints; -// } else { -// // Need to find a block to merge with -// int blkIdx = 0; -// // if (has last block AND cached Key is larger than the max Key) -// if (pIdx->hasLast && dataColsKeyFirst(pDataCols) > pIdx->maxKey) { -// blkIdx = pIdx->numOfSuperBlocks - 1; -// rowsWritten = tsdbMergeDataWithBlock(pHelper, pDataCols, blkIdx); -// if (rowsWritten < 0) goto _err; -// } else { -// ASSERT(IS_COMPBLOCK_LOADED(pHelper)); -// // SCompBlock *pMergeBlock = taosbsearch(); -// } -// } - -// return numOfPointsWritten; - -// _err: -// return -1; -// } - static int compareKeyBlock(const void *arg1, const void *arg2) { TSKEY key = *(TSKEY *)arg1; SCompBlock *pBlock = (SCompBlock *)arg2; @@ -947,7 +553,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa pFile = &(pHelper->files.lastF); } - if (tsdbWriteBlockToFile(pHelper, pFile, pDataCols, rowsCanMerge, &compBlock, pCompBlock->last) < 0) goto _err; + if (tsdbWriteBlockToFile(pHelper, pFile, pDataCols, rowsCanMerge, &compBlock, pCompBlock->last, false) < 0) goto _err; // TODO: Add the sub-block if (pCompBlock->numOfSubBlocks == 1) { @@ -985,7 +591,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa } } - if (tsdbWriteBlockToFile(pHelper, pFile, pHelper->pDataCols[0], pCompBlock->numOfPoints + rowsCanMerge, &compBlock, isLast) < 0) goto _err; + if (tsdbWriteBlockToFile(pHelper, pFile, pHelper->pDataCols[0], pCompBlock->numOfPoints + rowsCanMerge, &compBlock, isLast, true) < 0) goto _err; *pCompBlock = compBlock; From 136f99ed5c762844468408df47e0a7f70c6f722f Mon Sep 17 00:00:00 2001 From: hzcheng Date: Thu, 9 Apr 2020 09:58:06 +0800 Subject: [PATCH 03/29] TD-100 --- src/vnode/tsdb/src/tsdbRWHelper.c | 96 ++++++++++++++++++------------- 1 file changed, 55 insertions(+), 41 deletions(-) diff --git a/src/vnode/tsdb/src/tsdbRWHelper.c b/src/vnode/tsdb/src/tsdbRWHelper.c index 2e01d9efec..d543b12831 100644 --- a/src/vnode/tsdb/src/tsdbRWHelper.c +++ b/src/vnode/tsdb/src/tsdbRWHelper.c @@ -188,18 +188,17 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { TSKEY keyFirst = dataColsKeyFirst(pDataCols); ASSERT(helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)); - SCompIdx curIdx = pHelper->compIdx; // old table SCompIdx for sendfile usage - SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; // for change purpose + // SCompIdx curIdx = pHelper->compIdx; // old table SCompIdx for sendfile usage + SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; // for change purpose // Load the SCompInfo part if neccessary ASSERT(helperHasState(pHelper, TSDB_HELPER_TABLE_SET)); if ((!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) && - ((pIdx->offset > 0) && (pIdx->hasLast || dataColsKeyFirst(pDataCols) <= pIdx->maxKey)) && - (tsdbLoadCompInfo(pHelper, NULL) < 0)) - goto _err; + ((pIdx->offset > 0) && (pIdx->hasLast || keyFirst <= pIdx->maxKey))) { + if (tsdbLoadCompInfo(pHelper, NULL) < 0) goto _err; + } - if (!pIdx->hasLast && keyFirst > pIdx->maxKey) { - // Just need to append as a super block + if (pIdx->offset == 0 || (!pIdx->hasLast && keyFirst > pIdx->maxKey)) { // Just append as a super block rowsToWrite = pDataCols->numOfPoints; SFile *pWFile = NULL; bool isLast = false; @@ -218,39 +217,48 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { pIdx->hasLast = compBlock.last; pIdx->numOfSuperBlocks++; - pIdx->maxKey = dataColsKeyLast(pDataCols); - // pIdx->len = ?????? - } else { // (pIdx->hasLast) OR (keyFirst <= pIdx->maxKey) - if (keyFirst > pIdx->maxKey) { - int blkIdx = pIdx->numOfSuperBlocks - 1; - ASSERT(pIdx->hasLast && pHelper->pCompInfo->blocks[blkIdx].last); + pIdx->maxKey = compBlock.keyLast; + ASSERT(compBlock.keyLast == dataColsKeyLast(pDataCols)); + pIdx->len += sizeof(SCompBlock); + } else { // (Has old data) AND ((has last block) OR (key overlap)), need to merge the block + SCompBlock *pCompBlock = taosbsearch((void *)(&keyFirst), (void *)(pHelper->pCompInfo->blocks), + pIdx->numOfSuperBlocks, sizeof(SCompBlock), compareKeyBlock, TD_GE); - // Need to merge with the last block - if (tsdbMergeDataWithBlock(pHelper, blkIdx, pDataCols) < 0) goto _err; - } else { - // Find the first block greater or equal to the block - SCompBlock *pCompBlock = taosbsearch((void *)(&keyFirst), (void *)(pHelper->pCompInfo->blocks), - pIdx->numOfSuperBlocks, sizeof(SCompBlock), compareKeyBlock, TD_GE); - if (pCompBlock == NULL) { - if (tsdbMergeDataWithBlock(pHelper, pIdx->numOfSuperBlocks-1, pDataCols) < 0) goto _err; - } else { - if (compareKeyBlock((void *)(&keyFirst), (void *)pCompBlock) == 0) { - SCompBlock *pNextBlock = NULL; - TSKEY keyLimit = (pNextBlock == NULL) ? INT_MAX : (pNextBlock->keyFirst - 1); - rowsToWrite = - MIN(nRowsLEThan(pDataCols, keyLimit), pHelper->config.maxRowsPerFileBlock - pCompBlock->numOfPoints); - - if (tsdbMergeDataWithBlock(pHelper, pCompBlock-pHelper->pCompInfo->blocks, pDataCols) < 0) goto _err; + int blkIdx = (pCompBlock == NULL) ? (pIdx->numOfSuperBlocks - 1) : (pCompBlock - pHelper->pCompInfo->blocks); + + if (pCompBlock == NULL) { // No key overlap, must has last block, just merge with the last block + ASSERT(pIdx->hasLast && pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].last); + rowsToWrite = tsdbMergeDataWithBlock(pHelper, blkIdx, pDataCols); + if (rowsToWrite < 0) goto _err; + } else { // Has key overlap + + if (compareKeyBlock((void *)(&keyFirst), (void *)pCompBlock) == 0) { // Key overlap with the block + // TSKEY keyLimit = + // (blkIdx == pIdx->numOfSuperBlocks - 1) ? INT_MAX : (pHelper->pCompInfo->blocks[blkIdx + 1].keyFirst - 1); + + rowsToWrite = tsdbMergeDataWithBlock(pHelper, blkIdx, pDataCols); + if (rowsToWrite < 0) goto _err; + + ASSERT(rowsToWrite == MIN(rows1, rows2)); + } else { // Either merge with the previous block or save as a super block in the middle + SCompBlock *prevBlock = (blkIdx == 0) ? NULL : (pCompBlock - 1); + + int rows1 = nRowsLEThan(pDataCols, pCompBlock->keyFirst); // rows write as a super block in the middle + int rows2 = (prevBlock) ? (pHelper->config.maxRowsPerFileBlock - prevBlock->numOfPoints) + : rows1; // rows can merge with the previous block + if (rows1 >= rows2) { + rowsToWrite = tsdbWriteBlockToFile(pHelper, &(pHelper->files.dataF), pDataCols, rows1, &compBlock, false, true); + if (rowsToWrite < 0) goto _err; + + ASSERT(rowsToWrite == rows1); + + // Add the super block to it + pIdx->len += sizeof(SCompBlock); + pIdx->numOfSuperBlocks++; } else { - // There options: 1. merge with previous block - // 2. commit as one block - // 3. merge with current block - int nRows1 = INT_MAX; - int nRows2 = nRowsLEThan(pDataCols, pCompBlock->keyFirst); - int nRows3 = MIN(nRowsLEThan(pDataCols, (pCompBlock + 1)->keyFirst), (pHelper->config.maxRowsPerFileBlock - pCompBlock->numOfPoints)); - - // TODO: find the block with max rows can merge - if (tsdbMergeDataWithBlock(pHelper, pCompBlock, pDataCols) < 0) goto _err; + rowsToWrite = tsdbMergeDataWithBlock(pHelper, blkIdx-1, pDataCols); + if (rowsToWrite < 0) goto _err; + ASSERT(rowsToWrite == rows2); } } } @@ -258,7 +266,7 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { return rowsToWrite; - _err: +_err: return -1; } @@ -309,7 +317,7 @@ int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { if (lseek(fd, curCompIdx.offset, SEEK_SET) < 0) return -1; adjustMem(pHelper->pCompInfo, pHelper->compInfoSize, curCompIdx.len); - if (tread(fd, (void *)(pHelper->pCompInfo), pHelper) < 0) return -1; + if (tread(fd, (void *)(pHelper->pCompInfo), pHelper->compIdx.len) < pHelper->compIdx.len) return -1; // TODO: check the checksum helperSetState(pHelper, TSDB_HELPER_INFO_LOAD); @@ -521,8 +529,14 @@ static int compareKeyBlock(const void *arg1, const void *arg2) { return 0; } +static FORCE_INLINE int compKeyFunc(const void *arg1, const void *arg2) { + return ((*(TSKEY *)arg1) - (*(TSKEY *)arg2)); +} + static int nRowsLEThan(SDataCols *pDataCols, int maxKey) { - return 0; + void *ptr = taosbsearch((void *)&maxKey, pDataCols->cols[0].pData, pDataCols->numOfPoints, sizeof(TSKEY), compKeyFunc, TD_LE); + if (ptr == NULL) return 0; + return ((TSKEY *)ptr - (TSKEY *)(pDataCols->cols[0].pData)) + 1; } static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols) { From b0fe9341f175eeef0c21de3a7f786073d64cc6cd Mon Sep 17 00:00:00 2001 From: hzcheng Date: Thu, 9 Apr 2020 14:41:18 +0800 Subject: [PATCH 04/29] TD-100 --- src/util/src/tutil.c | 2 + src/util/tests/taosbsearchTest.cpp | 414 +++++++++++++++++++++++++++++ src/vnode/tsdb/src/tsdbRWHelper.c | 85 ++++-- src/vnode/tsdb/src/tsdbRead.c | 14 +- src/vnode/tsdb/tests/tsdbTests.cpp | 4 +- 5 files changed, 491 insertions(+), 28 deletions(-) create mode 100644 src/util/tests/taosbsearchTest.cpp diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index cadb2e706c..ca4a11c685 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -630,6 +630,7 @@ void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, if (flags == TD_EQ) { return bsearch(key, base, nmemb, size, compar); } else if (flags == TD_GE) { + if (nmemb == 0) return NULL; if ((*compar)(key, elePtrAt(base, size, 0)) <= 0) return elePtrAt(base, size, 0); if ((*compar)(key, elePtrAt(base, size, nmemb - 1)) > 0) return NULL; @@ -655,6 +656,7 @@ void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, } } } else if (flags == TD_LE) { + if (nmemb == 0) return NULL; if ((*compar)(key, elePtrAt(base, size, nmemb - 1)) >= 0) return elePtrAt(base, size, nmemb - 1); if ((*compar)(key, elePtrAt(base, size, 0)) < 0) return NULL; diff --git a/src/util/tests/taosbsearchTest.cpp b/src/util/tests/taosbsearchTest.cpp new file mode 100644 index 0000000000..fb01d34c03 --- /dev/null +++ b/src/util/tests/taosbsearchTest.cpp @@ -0,0 +1,414 @@ +#include + +#include "tutil.h" + +static int compareFunc(const void *arg1, const void *arg2) { return (*(int *)arg1) - (*(int *)arg2); } + +TEST(testCase, taosbsearch_equal) { + // For equal test + int key = 3; + void *pRet = NULL; + + pRet = taosbsearch((void *)&key, NULL, 0, sizeof(int), compareFunc, TD_EQ); + ASSERT_EQ(pRet, nullptr); + + // 1 element + int array1[1] = {5}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_EQ); + ASSERT_EQ(pRet, nullptr); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_EQ); + ASSERT_EQ(pRet, nullptr); + + key = 5; + pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_EQ); + ASSERT_NE(pRet, nullptr); + ASSERT_EQ(*(int *)pRet, key); + + // 2 element + int array2[2] = {3, 6}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_EQ); + ASSERT_EQ(pRet, nullptr); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_EQ); + ASSERT_EQ(*(int *)pRet, key); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_EQ); + ASSERT_EQ(pRet, nullptr); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_EQ); + ASSERT_EQ(*(int *)pRet, 6); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_EQ); + ASSERT_EQ(pRet, nullptr); + + // 3 element + int array3[3] = {3, 6, 8}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_EQ); + ASSERT_EQ(pRet, nullptr); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_EQ); + ASSERT_EQ(*(int *)pRet, 3); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_EQ); + ASSERT_EQ(pRet, nullptr); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_EQ); + ASSERT_EQ(*(int *)pRet, 6); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_EQ); + ASSERT_EQ(pRet, nullptr); + + key = 8; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_EQ); + ASSERT_EQ(*(int *)pRet, 8); + + key = 9; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_EQ); + ASSERT_EQ(pRet, nullptr); +} + +TEST(testCase, taosbsearch_greater_or_equal) { + // For equal test + int key = 3; + void *pRet = NULL; + + pRet = taosbsearch((void *)&key, NULL, 0, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(pRet, nullptr); + + // 1 element + int array1[1] = {5}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 5); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(pRet, nullptr); + + key = 5; + pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_GE); + ASSERT_NE(pRet, nullptr); + ASSERT_EQ(*(int *)pRet, 5); + + // 2 element + int array2[2] = {3, 6}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 3); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 3); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 6); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 6); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(pRet, nullptr); + + // 3 element + int array3[3] = {3, 6, 8}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 3); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 3); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 6); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 6); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 8); + + key = 8; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 8); + + key = 9; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(pRet, nullptr); + + // 4 element + int array4[4] = {3, 6, 8, 11}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 3); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 3); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 6); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 6); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 8); + + key = 8; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 8); + + key = 9; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 11); + + key = 11; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 11); + + key = 13; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(pRet, nullptr); + + // 5 element + int array5[5] = {3, 6, 8, 11, 15}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 3); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 3); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 6); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 6); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 8); + + key = 8; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 8); + + key = 9; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 11); + + key = 11; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 11); + + key = 13; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 15); + + key = 15; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(*(int *)pRet, 15); + + key = 17; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE); + ASSERT_EQ(pRet, nullptr); +} + +TEST(testCase, taosbsearch_less_or_equal) { + // For equal test + int key = 3; + void *pRet = NULL; + + pRet = taosbsearch((void *)&key, NULL, 0, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(pRet, nullptr); + + // 1 element + int array1[1] = {5}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(pRet, nullptr); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 5); + + key = 5; + pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_LE); + ASSERT_NE(pRet, nullptr); + ASSERT_EQ(*(int *)pRet, 5); + + // 2 element + int array2[2] = {3, 6}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(pRet, nullptr); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 3); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 3); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 6); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 6); + + // 3 element + int array3[3] = {3, 6, 8}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(pRet, nullptr); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 3); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 3); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 6); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 6); + + key = 8; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 8); + + key = 9; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 8); + + // 4 element + int array4[4] = {3, 6, 8, 11}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(pRet, nullptr); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 3); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 3); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 6); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 6); + + key = 8; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 8); + + key = 9; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 8); + + key = 11; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 11); + + key = 13; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 11); + + // 5 element + int array5[5] = {3, 6, 8, 11, 15}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(pRet, nullptr); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 3); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 3); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 6); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 6); + + key = 8; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 8); + + key = 9; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 8); + + key = 11; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 11); + + key = 13; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 11); + + key = 15; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 15); + + key = 17; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE); + ASSERT_EQ(*(int *)pRet, 15); +} \ No newline at end of file diff --git a/src/vnode/tsdb/src/tsdbRWHelper.c b/src/vnode/tsdb/src/tsdbRWHelper.c index d543b12831..c5bd90a6eb 100644 --- a/src/vnode/tsdb/src/tsdbRWHelper.c +++ b/src/vnode/tsdb/src/tsdbRWHelper.c @@ -27,7 +27,7 @@ static int tsdbCheckHelperCfg(SHelperCfg *pCfg); static void tsdbInitHelperFile(SHelperFile *pHFile); static int tsdbInitHelperRead(SRWHelper *pHelper); -static int tsdbInitHelperWrite(SRWHelper *pHelper); +// static int tsdbInitHelperWrite(SRWHelper *pHelper); static void tsdbClearHelperFile(SHelperFile *pHFile); static void tsdbDestroyHelperRead(SRWHelper *pHelper); static void tsdbDestroyHelperWrite(SRWHelper *pHelper); @@ -40,6 +40,7 @@ static int compareKeyBlock(const void *arg1, const void *arg2); static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols); static int nRowsLEThan(SDataCols *pDataCols, int maxKey); static int tsdbGetRowsCanBeMergedWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols); +static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); int tsdbInitHelper(SRWHelper *pHelper, SHelperCfg *pCfg) { if (pHelper == NULL || pCfg == NULL || tsdbCheckHelperCfg(pCfg) < 0) return -1; @@ -52,7 +53,10 @@ int tsdbInitHelper(SRWHelper *pHelper, SHelperCfg *pCfg) { if (tsdbInitHelperRead(pHelper) < 0) goto _err; - if ((TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER) && tsdbInitHelperWrite(pHelper) < 0) goto _err; + pHelper->pDataCols[0] = tdNewDataCols(pCfg->maxRowSize, pCfg->maxCols, pCfg->maxRows); + pHelper->pDataCols[1] = tdNewDataCols(pCfg->maxRowSize, pCfg->maxCols, pCfg->maxRows); + + if ((pHelper->pDataCols[0] == NULL) || (pHelper->pDataCols[1] == NULL)) goto _err; pHelper->state = TSDB_HELPER_CLEAR_STATE; @@ -182,6 +186,7 @@ void tsdbSetHelperTable(SRWHelper *pHelper, SHelperTable *pHelperTable, STSchema */ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { ASSERT(TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER); + ASSERT(pDataCols->numOfPoints > 0); SCompBlock compBlock; int rowsToWrite = 0; @@ -193,8 +198,7 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { // Load the SCompInfo part if neccessary ASSERT(helperHasState(pHelper, TSDB_HELPER_TABLE_SET)); - if ((!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) && - ((pIdx->offset > 0) && (pIdx->hasLast || keyFirst <= pIdx->maxKey))) { + if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD) && (pIdx->offset > 0)) { if (tsdbLoadCompInfo(pHelper, NULL) < 0) goto _err; } @@ -212,14 +216,7 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { if (tsdbWriteBlockToFile(pHelper, pWFile, pDataCols, rowsToWrite, &compBlock, isLast, true) < 0) goto _err; - // TODO: may need to reallocate the memory - pHelper->pCompInfo->blocks[pHelper->blockIter++] = compBlock; - - pIdx->hasLast = compBlock.last; - pIdx->numOfSuperBlocks++; - pIdx->maxKey = compBlock.keyLast; - ASSERT(compBlock.keyLast == dataColsKeyLast(pDataCols)); - pIdx->len += sizeof(SCompBlock); + if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfSuperBlocks) < 0) goto _err; } else { // (Has old data) AND ((has last block) OR (key overlap)), need to merge the block SCompBlock *pCompBlock = taosbsearch((void *)(&keyFirst), (void *)(pHelper->pCompInfo->blocks), pIdx->numOfSuperBlocks, sizeof(SCompBlock), compareKeyBlock, TD_GE); @@ -405,14 +402,14 @@ static void tsdbDestroyHelperRead(SRWHelper *pHelper) { tdFreeDataCols(pHelper->pDataCols[1]); } -static int tsdbInitHelperWrite(SRWHelper *pHelper) { - SHelperCfg *pCfg = &(pHelper->config); +// static int tsdbInitHelperWrite(SRWHelper *pHelper) { +// SHelperCfg *pCfg = &(pHelper->config); - // pHelper->wCompIdxSize = pCfg->maxTables * sizeof(SCompIdx); - // if ((pHelper->pWCompIdx = (SCompIdx *)malloc(pHelper->wCompIdxSize)) == NULL) return -1; +// // pHelper->wCompIdxSize = pCfg->maxTables * sizeof(SCompIdx); +// // if ((pHelper->pWCompIdx = (SCompIdx *)malloc(pHelper->wCompIdxSize)) == NULL) return -1; - return 0; -} +// return 0; +// } static void tsdbDestroyHelperWrite(SRWHelper *pHelper) { // tfree(pHelper->pWCompIdx); @@ -644,7 +641,7 @@ static int tsdbGetRowsCanBeMergedWithBlock(SRWHelper *pHelper, int blkIdx, SData } else { int32_t colId[1] = {0}; - if (tsdbLoadBlockDataCols(pHelper, NULL, &colId, 1) < 0) goto _err; + if (tsdbLoadBlockDataCols(pHelper, NULL, colId, 1) < 0) goto _err; int iter1 = 0; // For pDataCols int iter2 = 0; // For loaded data cols @@ -674,4 +671,54 @@ static int tsdbGetRowsCanBeMergedWithBlock(SRWHelper *pHelper, int blkIdx, SData _err: return -1; +} + +static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx) { + SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + + ASSERT(blkIdx >=0 && blkIdx <= pIdx->numOfSuperBlocks); + ASSERT(pCompBlock->numOfSubBlocks == 1); + + // Adjust memory if no more room + size_t spaceNeed = sizeof(SCompBlock); + size_t spaceLeft = pHelper->compInfoSize - pIdx->len; + ASSERT(spaceLeft >= 0); + if (spaceLeft < spaceNeed) { + size_t tsize = pHelper->compInfoSize + sizeof(SCompBlock) * 16; + if (pHelper->compInfoSize == 0) tsize += sizeof(SCompInfo); + + pHelper->pCompInfo = (SCompInfo *)realloc((void *)(pHelper->pCompInfo), tsize); + if (pHelper->pCompInfo == NULL) goto _err; + } + + // Insert the block + if (blkIdx < pIdx->numOfSuperBlocks) { + SCompBlock *pTCompBlock = pHelper->pCompInfo->blocks + blkIdx; + memmove((void *)(pTCompBlock + 1), (void *)pTCompBlock, pIdx->len - sizeof(SCompInfo) - sizeof(SCompBlock) *blkIdx); + pTCompBlock++; + for (int i = 0; i < pIdx->numOfSuperBlocks - blkIdx; i++) { + pTCompBlock->offset++; + } + } + pHelper->pCompInfo->blocks[blkIdx] = *pCompBlock; + + pIdx->numOfSuperBlocks++; + pIdx->len++; + pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].keyLast; + pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].last; + + return 0; + + _err: + return -1; +} + +static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock) { + // TODO + return 0; +} + +static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx) { + // TODO + return 0; } \ No newline at end of file diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index 9d8780d0f4..bd13163bfe 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -740,14 +740,14 @@ static bool getQualifiedDataBlock(STsdbQueryHandle *pQueryHandle, STableCheckInf pFile->fd = open(pFile->fname, O_RDONLY); } - if (tsdbLoadDataBlock(pFile, pBlock, 1, pCheckInfo->pDataCols, data) == 0) { - SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo; - pBlockLoadInfo->fileGroup = pCheckInfo->pFileGroup; - pBlockLoadInfo->slot = pQueryHandle->cur.slot; - pBlockLoadInfo->sid = pCheckInfo->pTableObj->tableId.tid; + // if (tsdbLoadDataBlock(pFile, pBlock, 1, pCheckInfo->pDataCols, data) == 0) { + // SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo; + // pBlockLoadInfo->fileGroup = pCheckInfo->pFileGroup; + // pBlockLoadInfo->slot = pQueryHandle->cur.slot; + // pBlockLoadInfo->sid = pCheckInfo->pTableObj->tableId.tid; - blockLoaded = true; - } + // blockLoaded = true; + // } // dError("QInfo:%p fileId:%d total numOfBlks:%d blockId:%d load into memory failed due to error in disk files", // GET_QINFO_ADDR(pQuery), pQuery->fileId, pQuery->numOfBlocks, blkIdx); diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp index 77f944cea2..340212cf10 100644 --- a/src/vnode/tsdb/tests/tsdbTests.cpp +++ b/src/vnode/tsdb/tests/tsdbTests.cpp @@ -79,8 +79,8 @@ TEST(TsdbTest, createRepo) { tsdbCreateTable(pRepo, &tCfg); // // 3. Loop to write some simple data - int nRows = 1; - int rowsPerSubmit = 1; + int nRows = 10000000; + int rowsPerSubmit = 10; int64_t start_time = 1584081000000; SSubmitMsg *pMsg = (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + tdMaxRowBytesFromSchema(schema) * rowsPerSubmit); From 3dfe486d1c6ed753fd8eba7fc39269cf4b762f9f Mon Sep 17 00:00:00 2001 From: hzcheng Date: Thu, 9 Apr 2020 16:12:47 +0800 Subject: [PATCH 05/29] TD-100 --- src/vnode/tsdb/src/tsdbRWHelper.c | 139 ++++++++++++++++++++++++++---- 1 file changed, 124 insertions(+), 15 deletions(-) diff --git a/src/vnode/tsdb/src/tsdbRWHelper.c b/src/vnode/tsdb/src/tsdbRWHelper.c index c5bd90a6eb..7da3327343 100644 --- a/src/vnode/tsdb/src/tsdbRWHelper.c +++ b/src/vnode/tsdb/src/tsdbRWHelper.c @@ -673,6 +673,24 @@ _err: return -1; } +static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t spaceNeeded) { + SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + + size_t spaceLeft = pHelper->compInfoSize - pIdx->len; + ASSERT(spaceLeft >= 0); + if (spaceLeft < spaceNeeded) { + size_t tsize = pHelper->compInfoSize + sizeof(SCompBlock) * 16; + if (pHelper->compInfoSize == 0) tsize += sizeof(SCompInfo); + + pHelper->pCompInfo = (SCompInfo *)realloc((void *)(pHelper->pCompInfo), tsize); + if (pHelper->pCompInfo == NULL) return -1; + + pHelper->compInfoSize = tsize; + } + + return 0; +} + static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx) { SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; @@ -680,16 +698,7 @@ static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int ASSERT(pCompBlock->numOfSubBlocks == 1); // Adjust memory if no more room - size_t spaceNeed = sizeof(SCompBlock); - size_t spaceLeft = pHelper->compInfoSize - pIdx->len; - ASSERT(spaceLeft >= 0); - if (spaceLeft < spaceNeed) { - size_t tsize = pHelper->compInfoSize + sizeof(SCompBlock) * 16; - if (pHelper->compInfoSize == 0) tsize += sizeof(SCompInfo); - - pHelper->pCompInfo = (SCompInfo *)realloc((void *)(pHelper->pCompInfo), tsize); - if (pHelper->pCompInfo == NULL) goto _err; - } + if (tsdbAdjustInfoSizeIfNeeded(pHelper, sizeof(SCompBlock)) < 0) goto _err; // Insert the block if (blkIdx < pIdx->numOfSuperBlocks) { @@ -697,13 +706,13 @@ static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int memmove((void *)(pTCompBlock + 1), (void *)pTCompBlock, pIdx->len - sizeof(SCompInfo) - sizeof(SCompBlock) *blkIdx); pTCompBlock++; for (int i = 0; i < pIdx->numOfSuperBlocks - blkIdx; i++) { - pTCompBlock->offset++; + if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += sizeof(SCompBlock); } } pHelper->pCompInfo->blocks[blkIdx] = *pCompBlock; pIdx->numOfSuperBlocks++; - pIdx->len++; + pIdx->len += sizeof(SCompBlock); pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].keyLast; pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].last; @@ -713,12 +722,112 @@ static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int return -1; } -static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock) { - // TODO +static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded) { + ASSERT(pCompBlock->numOfSubBlocks == 0); + + SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + SCompBlock *pSCompBlock = pHelper->pCompInfo->blocks + blkIdx; + ASSERT(pSCompBlock->numOfSubBlocks >= 1 && pSCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS); + + size_t spaceNeeded = (pSCompBlock->numOfSubBlocks == 1) ? sizeof(SCompBlock) * 2 : sizeof(SCompBlock); + if (tsdbAdjustInfoSizeIfNeeded(pHelper, spaceNeeded) < 0) goto _err; + + // Add the sub-block + if (pSCompBlock->numOfSubBlocks > 1) { + size_t tsize = pIdx->len - (pSCompBlock->offset + pSCompBlock->len); + if (tsize > 0) { + memmove((void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len + sizeof(SCompBlock)), + (void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len), tsize); + + for (int i = blkIdx; i < pIdx->numOfSuperBlocks; i++) { + SCompBlock *pTCompBlock = &pHelper->pCompInfo->blocks[i]; + if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += sizeof(SCompBlock); + } + } + + + *(SCompBlock *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len) = *pCompBlock; + + pSCompBlock->numOfSubBlocks++; + pSCompBlock->len += sizeof(SCompBlock); + pIdx->len += sizeof(SCompBlock); + } else { // Need to create two sub-blocks + void *ptr = NULL; + for (int i = blkIdx - 1; i >= 0; i--) { + SCompBlock *pTCompBlock = pHelper->pCompInfo->blocks + i; + if (pTCompBlock->numOfSubBlocks > 1) { + ptr = (void *)((char *)(pHelper->pCompInfo) + pTCompBlock->offset + pTCompBlock->len); + break; + } + } + + if (ptr == NULL) + ptr = (void *)((char *)(pHelper->pCompInfo) + sizeof(SCompInfo) + sizeof(SCompBlock) * pIdx->numOfSuperBlocks); + + size_t tsize = pIdx->len - ((char *)ptr - (char *)(pHelper->pCompInfo)); + if (tsize > 0) { + memmove((void *)((char *)ptr + sizeof(SCompBlock) * 2), ptr, tsize); + for (int i = blkIdx + 1; i < pIdx->numOfSuperBlocks; i++) { + SCompBlock *pTCompBlock = pHelper->pCompInfo->blocks + i; + if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += (sizeof(SCompBlock) * 2); + } + } + + ((SCompBlock *)ptr)[0] = *pSCompBlock; + ((SCompBlock *)ptr)[0].numOfSubBlocks = 0; + + ((SCompBlock *)ptr)[1] = *pCompBlock; + + pSCompBlock->numOfSubBlocks = 2; + pSCompBlock->numOfPoints += rowsAdded; + pSCompBlock->offset = ((char *)ptr) - ((char *)pHelper->pCompInfo); + pSCompBlock->len = sizeof(SCompBlock) * 2; + pSCompBlock->keyFirst = MIN(((SCompBlock *)ptr)[0].keyFirst, ((SCompBlock *)ptr)[1].keyFirst); + pSCompBlock->keyLast = MAX(((SCompBlock *)ptr)[0].keyLast, ((SCompBlock *)ptr)[1].keyLast); + + pIdx->len += (sizeof(SCompBlock) * 2); + } + + pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].keyLast; + pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].last; + return 0; + +_err: + return -1; } static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx) { - // TODO + ASSERT(pCompBlock->numOfSubBlocks == 1); + + SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + + ASSERT(blkIdx >= 0 && blkIdx < pIdx->numOfSuperBlocks); + + SCompBlock *pSCompBlock = pHelper->pCompInfo->blocks + blkIdx; + + ASSERT(pSCompBlock->numOfSubBlocks >= 1); + + // Delete the sub blocks it has + if (pSCompBlock->numOfSubBlocks > 1) { + size_t tsize = pIdx->len - (pSCompBlock->offset + pSCompBlock->len); + if (tsize > 0) { + memmove((void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset), + (void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len), tsize); + } + + for (int i = blkIdx + 1; i < pIdx->numOfSuperBlocks; i++) { + SCompBlock *pTCompBlock = &pHelper->pCompInfo->blocks[i]; + if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset -= (sizeof(SCompBlock) * pSCompBlock->numOfSubBlocks); + } + + pIdx->len -= (sizeof(SCompBlock) * pSCompBlock->numOfSubBlocks); + } + + *pSCompBlock = *pCompBlock; + + pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].keyLast; + pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].last; + return 0; } \ No newline at end of file From b64daf73fd06efb424d492fb6375ac20ae83239c Mon Sep 17 00:00:00 2001 From: hzcheng Date: Thu, 9 Apr 2020 18:44:27 +0800 Subject: [PATCH 06/29] TD-100 --- src/vnode/tsdb/src/tsdbRWHelper.c | 32 +++++-------------------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/src/vnode/tsdb/src/tsdbRWHelper.c b/src/vnode/tsdb/src/tsdbRWHelper.c index 7da3327343..dc51b86c29 100644 --- a/src/vnode/tsdb/src/tsdbRWHelper.c +++ b/src/vnode/tsdb/src/tsdbRWHelper.c @@ -41,6 +41,8 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa static int nRowsLEThan(SDataCols *pDataCols, int maxKey); static int tsdbGetRowsCanBeMergedWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols); static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); +static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded); +static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); int tsdbInitHelper(SRWHelper *pHelper, SHelperCfg *pCfg) { if (pHelper == NULL || pCfg == NULL || tsdbCheckHelperCfg(pCfg) < 0) return -1; @@ -236,7 +238,6 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { rowsToWrite = tsdbMergeDataWithBlock(pHelper, blkIdx, pDataCols); if (rowsToWrite < 0) goto _err; - ASSERT(rowsToWrite == MIN(rows1, rows2)); } else { // Either merge with the previous block or save as a super block in the middle SCompBlock *prevBlock = (blkIdx == 0) ? NULL : (pCompBlock - 1); @@ -283,7 +284,7 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { } int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { - ASSERT(pHelper->state = TSDB_HELPER_FILE_SET_AND_OPEN); + ASSERT(pHelper->state == TSDB_HELPER_FILE_SET_AND_OPEN); if (!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) { // If not load from file, just load it in object @@ -538,7 +539,6 @@ static int nRowsLEThan(SDataCols *pDataCols, int maxKey) { static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols) { int rowsWritten = 0; - TSKEY keyFirst = dataColsKeyFirst(pDataCols); SCompBlock compBlock = {0}; SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; @@ -566,24 +566,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa if (tsdbWriteBlockToFile(pHelper, pFile, pDataCols, rowsCanMerge, &compBlock, pCompBlock->last, false) < 0) goto _err; - // TODO: Add the sub-block - if (pCompBlock->numOfSubBlocks == 1) { - pCompBlock->numOfSubBlocks += 2; - // pCompBlock->offset = ; - // pCompBlock->len = ; - } else { - pCompBlock->numOfSubBlocks++; - } - pCompBlock->numOfPoints += rowsCanMerge; - pCompBlock->keyFirst = MIN(pCompBlock->keyFirst, dataColsKeyFirst(pDataCols)); - pCompBlock->keyLast = MAX(pCompBlock->keyLast, dataColsKeyAt(pDataCols, rowsCanMerge - 1)); - - // Update the Idx - // pIdx->hasLast = ; - // pIdx->len =; - // pIdx->numOfSuperBlocks = ; - - rowsWritten = rowsCanMerge; + if (tsdbAddSubBlock(pHelper, &compBlock, blkIdx, rowsCanMerge) < 0) goto _err; } else { // Read-Merge-Write as a super block if (tsdbLoadBlockData(pHelper, blkIdx, NULL) < 0) goto _err; @@ -603,12 +586,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa } if (tsdbWriteBlockToFile(pHelper, pFile, pHelper->pDataCols[0], pCompBlock->numOfPoints + rowsCanMerge, &compBlock, isLast, true) < 0) goto _err; - - *pCompBlock = compBlock; - - pIdx->maxKey = MAX(pIdx->maxKey, compBlock.keyLast); - // pIdx->hasLast = ; - // pIdx-> + if (tsdbUpdateSuperBlock(pHelper, &compBlock, blkIdx) < 0) goto _err; } return rowsWritten; From ac209abb50a89e8349599c9f38c3c692bd0d04f7 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Fri, 10 Apr 2020 23:12:53 +0800 Subject: [PATCH 07/29] TD-100 --- src/vnode/tsdb/inc/tsdbMain.h | 31 ++--- src/vnode/tsdb/src/tsdbRWHelper.c | 212 ++++++++++++++++++++++++++---- 2 files changed, 206 insertions(+), 37 deletions(-) diff --git a/src/vnode/tsdb/inc/tsdbMain.h b/src/vnode/tsdb/inc/tsdbMain.h index 949bb9d48b..beb9ba9868 100644 --- a/src/vnode/tsdb/inc/tsdbMain.h +++ b/src/vnode/tsdb/inc/tsdbMain.h @@ -382,27 +382,28 @@ typedef struct { // Global configuration SHelperCfg config; + int8_t state; + + // For file set usage SHelperFile files; + SCompIdx * pCompIdx; + size_t compIdxSize; + // For table set usage SHelperTable tableInfo; - SCompIdx compIdx; // SCompIdx of current table + SCompInfo * pCompInfo; + size_t compInfoSize; + bool hasOldLastBlock; - int8_t state; // current loading state - - // Information in .head file - SCompIdx *pCompIdx; - size_t compIdxSize; - - SCompInfo *pCompInfo; - size_t compInfoSize; - int blockIter; // For write purpose - - // Information in .data or .last file + // For block set usage SCompData *pCompData; size_t compDataSize; - SDataCols *pDataCols[2]; + // ------ Perhaps no usage + SCompIdx compIdx; // SCompIdx of current table + int blockIter; // For write purpose + // Compression buffer void * cBuffer; size_t cBufSize; @@ -434,8 +435,8 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError); // --------- For read operations int tsdbLoadCompIdx(SRWHelper *pHelper, void *target); int tsdbLoadCompInfo(SRWHelper *pHelper, void *target); -int tsdbLoadCompData(SRWHelper *pHelper, int blkIdx, void *target); -int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int32_t *colIds, int numOfColIds); +int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target); +int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx, int16_t *colIds, int numOfColIds); int tsdbLoadBlockData(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols); // --------- For write operations diff --git a/src/vnode/tsdb/src/tsdbRWHelper.c b/src/vnode/tsdb/src/tsdbRWHelper.c index dc51b86c29..fb46deb216 100644 --- a/src/vnode/tsdb/src/tsdbRWHelper.c +++ b/src/vnode/tsdb/src/tsdbRWHelper.c @@ -175,7 +175,12 @@ void tsdbSetHelperTable(SRWHelper *pHelper, SHelperTable *pHelperTable, STSchema tdInitDataCols(pHelper->pDataCols[0], pSchema); tdInitDataCols(pHelper->pDataCols[1], pSchema); - pHelper->compIdx = pHelper->pCompIdx[pHelper->tableInfo.tid]; + SCompIdx *pIdx = pHelper->pCompIdx + pHelperTable->tid; + if (pIdx->offset > 0 && pIdx->hasLast) { + pHelper->hasOldLastBlock = true; + } + + // pHelper->compIdx = pHelper->pCompIdx[pHelper->tableInfo.tid]; helperSetState(pHelper, TSDB_HELPER_TABLE_SET); } @@ -205,6 +210,7 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { } if (pIdx->offset == 0 || (!pIdx->hasLast && keyFirst > pIdx->maxKey)) { // Just append as a super block + ASSERT(pHelper->hasOldLastBlock == false); rowsToWrite = pDataCols->numOfPoints; SFile *pWFile = NULL; bool isLast = false; @@ -231,9 +237,8 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { if (rowsToWrite < 0) goto _err; } else { // Has key overlap - if (compareKeyBlock((void *)(&keyFirst), (void *)pCompBlock) == 0) { // Key overlap with the block - // TSKEY keyLimit = - // (blkIdx == pIdx->numOfSuperBlocks - 1) ? INT_MAX : (pHelper->pCompInfo->blocks[blkIdx + 1].keyFirst - 1); + if (compareKeyBlock((void *)(&keyFirst), (void *)pCompBlock) == 0) { + // Key overlap with the block, must merge with the block rowsToWrite = tsdbMergeDataWithBlock(pHelper, blkIdx, pDataCols); if (rowsToWrite < 0) goto _err; @@ -269,17 +274,61 @@ _err: } int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { - // TODO + SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + SCompBlock compBlock; + if ((pHelper->files.nHeadF.fd > 0) && (pHelper->hasOldLastBlock)) { + if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1; + + SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + pIdx->numOfSuperBlocks - 1; + ASSERT(pCompBlock->last); + + if (pCompBlock->numOfSubBlocks > 1) { + if (tsdbLoadBlockData(pHelper, pIdx->numOfSuperBlocks - 1, NULL) < 0) return -1; + if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.nLastF), pHelper->pDataCols[0], + pHelper->pDataCols[0]->numOfPoints, &compBlock, true, true) < 0) + return -1; + + if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfSuperBlocks - 1) < 0) return -1; + + } else { + if (lseek(pHelper->files.lastF.fd, pCompBlock->offset, SEEK_SET) < 0) return -1; + pCompBlock->offset = lseek(pHelper->files.nLastF.fd, 0, SEEK_END); + if (pCompBlock->offset < 0) return -1; + + if (tsendfile(pHelper->files.nLastF.fd, pHelper->files.lastF.fd, NULL, pCompBlock->len) < pCompBlock->len) + return -1; + } + + pHelper->hasOldLastBlock = false; + } + return 0; } int tsdbWriteCompInfo(SRWHelper *pHelper) { - // TODO + SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { + if (pIdx->offset > 0) { + pIdx->offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END); + if (pIdx->offset < 0) return -1; + + if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, pIdx->len) < pIdx->len) return -1; + } + } else { + pIdx->offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END); + if (pIdx->offset < 0) return -1; + + if (twrite(pHelper->files.nHeadF.fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) return -1; + } + return 0; } int tsdbWriteCompIdx(SRWHelper *pHelper) { - // TODO + if (lseek(pHelper->files.nHeadF.fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1; + + if (twrite(pHelper->files.nHeadF.fd, (void *)pHelper->pCompIdx, pHelper->compIdxSize) < pHelper->compIdxSize) + return -1; return 0; } @@ -326,13 +375,78 @@ int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { return 0; } -int tsdbLoadCompData(SRWHelper *pHelper, int blkIdx, void *target) { - // TODO +int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) { + ASSERT(pCompBlock->numOfSubBlocks <= 1); + int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd; + + if (lseek(fd, pCompBlock->offset, SEEK_SET) < 0) return -1; + + size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols; + adjustMem(pHelper->pCompData, pHelper->compDataSize, tsize); + + if (tread(fd, (void *)pHelper->pCompData, tsize) < tsize) return -1; + + ASSERT(pCompBlock->numOfCols == pHelper->pCompData->numOfCols); + + if (target) memcpy(target, pHelper->pCompData, tsize); + return 0; } -int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int32_t *colIds, int numOfColIds) { - // TODO +static int comparColIdCompCol(const void *arg1, const void *arg2) { + return (*(int16_t *)arg1) - ((SCompCol *)arg2)->colId; +} + +static int comparColIdDataCol(const void *arg1, const void *arg2) { + return (*(int16_t *)arg1) - ((SDataCol *)arg2)->colId; +} + +static int tsdbLoadSingleColumnData(int fd, SCompBlock *pCompBlock, SCompCol *pCompCol, void *buf) { + size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols; + if (lseek(fd, pCompBlock->offset + tsize + pCompCol->offset) < 0) return -1; + if (tread(fd, buf, pCompCol->len) < pCompCol->len) return -1; + + return 0; +} + +static int tsdbLoadSingleBlockDataCols(SRWHelper *pHelper, SCompBlock *pCompBlock, int16_t *colIds, int numOfColIds, + SDataCols *pDataCols) { + if (tsdbLoadCompData(pHelper, pCompBlock, NULL) < 0) return -1; + size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols; + int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd; + + void *ptr = NULL; + for (int i = 0; i < numOfColIds; i++) { + int16_t colId = colIds[i]; + + ptr = bsearch((void *)&colId, (void *)pHelper->pCompData->cols, pHelper->pCompData->numOfCols, sizeof(SCompCol), comparColIdCompCol); + if (ptr == NULL) continue; + SCompCol *pCompCol = (SCompCol *)ptr; + + ptr = bsearch((void *)&colId, (void *)(pDataCols->cols), pDataCols->numOfCols, sizeof(SDataCol), comparColIdDataCol); + ASSERT(ptr != NULL); + SDataCol *pDataCol = (SDataCol *)ptr; + + pDataCol->len = pCompCol->len; + if (tsdbLoadSingleColumnData(fd, pCompBlock, pCompCol, pDataCol->pData) < 0) return -1; + } + + return 0; +} + +// Load specific column data from file +int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx, int16_t *colIds, int numOfColIds) { + SCompIdx * pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx; + + ASSERT(pCompBlock->numOfSubBlocks >= 1); // Must be super block + if (pCompBlock->numOfSubBlocks == 1) { + + } + + + + return 0; } @@ -537,44 +651,74 @@ static int nRowsLEThan(SDataCols *pDataCols, int maxKey) { return ((TSKEY *)ptr - (TSKEY *)(pDataCols->cols[0].pData)) + 1; } +// Merge the data with a block static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols) { + // TODO: set pHelper->hasOldBlock int rowsWritten = 0; SCompBlock compBlock = {0}; + TSKEY keyFirst = dataColsKeyFirst(pDataCols); + SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; ASSERT(blkIdx < pIdx->numOfSuperBlocks); SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx; ASSERT(pCompBlock->numOfSubBlocks >= 1); + ASSERT(keyFirst >= pCompBlock->keyFirst); - int rowsCanMerge = tsdbGetRowsCanBeMergedWithBlock(pHelper, blkIdx, pDataCols); - if (rowsCanMerge < 0) goto _err; + // Start here + TSKEY keyLimit = + (blkIdx == pIdx->numOfSuperBlocks - 1) ? INT_MAX : pHelper->pCompInfo->blocks[blkIdx + 1].keyLast - 1; - ASSERT(rowsCanMerge > 0); + int rowsMustMerge = tsdbGetRowsInRange(pDataCols, 0, pCompBlock->keyLast); + int maxRowsCanMerge = + MIN(pHelper->config.maxRowsPerFileBlock - pCompBlock->numOfPoints, tsdbGetRowsInRange(pDataCols, keyLimit)); + + if (pCompBlock->numOfPoints + rowsMustMerge > pHelper->config.maxRowsPerFileBlock) { + // Need to load the block and split as two super block + } else { + } + + if (rowsMustMerge + pCompBlock->numOfPoints > pHelper->config.maxRowsPerFileBlock) { + // Load the block and merge as two super block + } + + if (rowsMustMerge > maxRowsCanMerge) { + ASSERT(pCompBlock->numOfPoints + rowsMustMerge > pHelper->config.maxRowsPerFileBlock); + + } else { + + } + + + int rowsToMerge = tsdbGetRowsCanBeMergedWithBlock(pHelper, blkIdx, pDataCols); + if (rowsToMerge < 0) goto _err; + + ASSERT(rowsToMerge > 0); if (pCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS && ((!pCompBlock->last) || (pHelper->files.nLastF.fd < 0 && - pCompBlock->numOfPoints + rowsCanMerge < pHelper->config.minRowsPerFileBlock))) { + pCompBlock->numOfPoints + rowsToMerge < pHelper->config.minRowsPerFileBlock))) { SFile *pFile = NULL; - if (!pCompBlock->last) { + if ((!pCompBlock->last) || (pCompBlock->numOfPoints + rowsToMerge >= pHelper->config.minRowsPerFileBlock)) { pFile = &(pHelper->files.dataF); } else { pFile = &(pHelper->files.lastF); } - if (tsdbWriteBlockToFile(pHelper, pFile, pDataCols, rowsCanMerge, &compBlock, pCompBlock->last, false) < 0) goto _err; + if (tsdbWriteBlockToFile(pHelper, pFile, pDataCols, rowsToMerge, &compBlock, pCompBlock->last, false) < 0) goto _err; - if (tsdbAddSubBlock(pHelper, &compBlock, blkIdx, rowsCanMerge) < 0) goto _err; + if (tsdbAddSubBlock(pHelper, &compBlock, blkIdx, rowsToMerge) < 0) goto _err; } else { // Read-Merge-Write as a super block if (tsdbLoadBlockData(pHelper, blkIdx, NULL) < 0) goto _err; - tdMergeDataCols(pHelper->pDataCols[0], pDataCols, rowsCanMerge); + tdMergeDataCols(pHelper->pDataCols[0], pDataCols, rowsToMerge); int isLast = 0; SFile *pFile = NULL; - if (!pCompBlock->last || (pCompBlock->numOfPoints + rowsCanMerge >= pHelper->config.minRowsPerFileBlock)) { + if (!pCompBlock->last || (pCompBlock->numOfPoints + rowsToMerge >= pHelper->config.minRowsPerFileBlock)) { pFile = &(pHelper->files.dataF); } else { isLast = 1; @@ -585,7 +729,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa } } - if (tsdbWriteBlockToFile(pHelper, pFile, pHelper->pDataCols[0], pCompBlock->numOfPoints + rowsCanMerge, &compBlock, isLast, true) < 0) goto _err; + if (tsdbWriteBlockToFile(pHelper, pFile, pHelper->pDataCols[0], pCompBlock->numOfPoints + rowsToMerge, &compBlock, isLast, true) < 0) goto _err; if (tsdbUpdateSuperBlock(pHelper, &compBlock, blkIdx) < 0) goto _err; } @@ -619,7 +763,7 @@ static int tsdbGetRowsCanBeMergedWithBlock(SRWHelper *pHelper, int blkIdx, SData } else { int32_t colId[1] = {0}; - if (tsdbLoadBlockDataCols(pHelper, NULL, colId, 1) < 0) goto _err; + if (tsdbLoadBlockDataCols(pHelper, NULL, blkIdx, colId, 1) < 0) goto _err; int iter1 = 0; // For pDataCols int iter2 = 0; // For loaded data cols @@ -808,4 +952,28 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].last; return 0; +} + +// Get the number of rows in range [minKey, maxKey] +static int tsdbGetRowsInRange(SDataCols *pDataCols, int minKey, int maxKey) { + if (pDataCols->numOfPoints == 0) return 0; + + ASSERT(minKey <= maxKey); + TSKEY keyFirst = dataColsKeyFirst(pDataCols); + TSKEY keyLast = dataColsKeyLast(pDataCols); + ASSERT(keyFirst <= keyLast); + + if (minKey > keyLast || maxKey < keyFirst) return 0; + + void *ptr1 = taosbsearch((void *)&minKey, (void *)pDataCols->cols[0].pData, pDataCols->numOfPoints, sizeof(TSKEY), + compTSKEY, TD_GE); + ASSERT(ptr1 != NULL); + + void *ptr2 = taosbsearch((void *)&maxKey, (void *)pDataCols->cols[0].pData, pDataCols->numOfPoints, sizeof(TSKEY), + compTSKEY, TD_LE); + ASSERT(ptr2 != NULL); + + if ((TSKEY *)ptr2 - (TSKEY *)ptr1 < 0) return 0; + + return (TSKEY *)ptr2 - (TSKEY *)ptr1; } \ No newline at end of file From d28fae1a45eac92e7141ca275223b8bf28a81228 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sat, 11 Apr 2020 10:20:19 +0800 Subject: [PATCH 08/29] TD-100 --- src/common/inc/dataformat.h | 2 +- src/common/src/dataformat.c | 62 +++++++++++++++++++++++++++++++ src/vnode/tsdb/src/tsdbRWHelper.c | 34 +++++++++++++---- 3 files changed, 90 insertions(+), 8 deletions(-) diff --git a/src/common/inc/dataformat.h b/src/common/inc/dataformat.h index 231786ff73..783e378eb7 100644 --- a/src/common/inc/dataformat.h +++ b/src/common/inc/dataformat.h @@ -111,7 +111,6 @@ typedef struct SDataCol { int len; int offset; void * pData; // Original data - void * pCData; // Compressed data } SDataCol; typedef struct { @@ -133,6 +132,7 @@ typedef struct { SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows); void tdResetDataCols(SDataCols *pCols); void tdInitDataCols(SDataCols *pCols, STSchema *pSchema); +SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData); void tdFreeDataCols(SDataCols *pCols); void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols); void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop); diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c index 1f5d83d9af..fd7dd5b91e 100644 --- a/src/common/src/dataformat.c +++ b/src/common/src/dataformat.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ #include "dataformat.h" +#include "tutil.h" static int tdFLenFromSchema(STSchema *pSchema); @@ -338,6 +339,27 @@ void tdFreeDataCols(SDataCols *pCols) { } } +SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { + SDataCols *pRet = tdNewDataCols(pDataCols->maxRowSize, pDataCols->maxCols, pDataCols->maxPoints); + if (pRet == NULL) return NULL; + + pRet->numOfCols = pDataCols->numOfCols; + pRet->sversion = pDataCols->sversion; + if (keepData) pRet->numOfPoints = pDataCols->numOfPoints; + + for (int i = 0; i < pDataCols->numOfCols; i++) { + pRet->cols[i].type = pDataCols->cols[i].type; + pRet->cols[i].colId = pDataCols->cols[i].colId; + pRet->cols[i].bytes = pDataCols->cols[i].bytes; + pRet->cols[i].len = pDataCols->cols[i].len; + pRet->cols[i].offset = pDataCols->cols[i].offset; + + if (keepData) memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pRet->cols[i].len); + } + + return pRet; +} + void tdResetDataCols(SDataCols *pCols) { pCols->numOfPoints = 0; for (int i = 0; i < pCols->maxCols; i++) { @@ -384,5 +406,45 @@ static int tdFLenFromSchema(STSchema *pSchema) { int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) { // TODO + ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfPoints); + + SDataCols *pTarget = tdDupDataCols(target, true); + if (pTarget == NULL) goto _err; + + int iter1 = 0; + int iter2 = 0; + while (true) { + if (iter1 >= pTarget->numOfPoints) { + // TODO: merge the source part + int rowsLeft = source->numOfPoints - iter2; + if (rowsLeft > 0) { + for (int i = 0; i < source->numOfCols; i++) { + ASSERT(target->cols[i].type == source->cols[i].type); + + memcpy((void *)((char *)(target->cols[i].pData) + TYPE_BYTES[target->cols[i].type] * target->numOfPoints), + (void *)((char *)(source->cols[i].pData) + TYPE_BYTES[source->cols[i].type] * iter2), + TYPE_BYTES[target->cols[i].type] * rowsLeft); + } + } + break; + } + + if (iter2 >= source->numOfPoints) { + // TODO: merge the pTemp part + int rowsLeft = pTarget->numOfPoints - iter1; + if (rowsLeft > 0) { + + } + break; + } + + + + + } + return 0; + + _err: + return -1; } \ No newline at end of file diff --git a/src/vnode/tsdb/src/tsdbRWHelper.c b/src/vnode/tsdb/src/tsdbRWHelper.c index fb46deb216..7f1c642ef4 100644 --- a/src/vnode/tsdb/src/tsdbRWHelper.c +++ b/src/vnode/tsdb/src/tsdbRWHelper.c @@ -43,6 +43,7 @@ static int tsdbGetRowsCanBeMergedWithBlock(SRWHelper *pHelper, int blkIdx, SData static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded); static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); +static int tsdbGetRowsInRange(SDataCols *pDataCols, int minKey, int maxKey); int tsdbInitHelper(SRWHelper *pHelper, SHelperCfg *pCfg) { if (pHelper == NULL || pCfg == NULL || tsdbCheckHelperCfg(pCfg) < 0) return -1; @@ -403,7 +404,7 @@ static int comparColIdDataCol(const void *arg1, const void *arg2) { static int tsdbLoadSingleColumnData(int fd, SCompBlock *pCompBlock, SCompCol *pCompCol, void *buf) { size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols; - if (lseek(fd, pCompBlock->offset + tsize + pCompCol->offset) < 0) return -1; + if (lseek(fd, pCompBlock->offset + tsize + pCompCol->offset, SEEK_SET) < 0) return -1; if (tread(fd, buf, pCompCol->len) < pCompCol->len) return -1; return 0; @@ -440,19 +441,38 @@ int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx, SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx; ASSERT(pCompBlock->numOfSubBlocks >= 1); // Must be super block - if (pCompBlock->numOfSubBlocks == 1) { + int numOfSubBlocks = pCompBlock->numOfSubBlocks; + SCompBlock *pStartBlock = + (numOfSubBlocks == 1) ? pCompBlock : (SCompBlock *)((char *)pHelper->pCompInfo->blocks + pCompBlock->offset); + + if (tsdbLoadSingleBlockDataCols(pHelper, pStartBlock, colIds, numOfColIds, pDataCols) < 0) return -1; + for (int i = 1; i < numOfSubBlocks; i++) { + pStartBlock++; + if (tsdbLoadSingleBlockDataCols(pHelper, pStartBlock, colIds, numOfColIds, pHelper->pDataCols[1]) < 0) return -1; + tdMergeDataCols(pDataCols, pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfPoints); } - - - return 0; } +// Load the whole block data int tsdbLoadBlockData(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols) { - // TODO + int16_t *colIds = (int16_t *)calloc(pDataCols->numOfCols, sizeof(int16_t)); + if (colIds == NULL) goto _err; + + for (int i = 0; i < pDataCols->numOfCols; i++) { + colIds[i] = pDataCols->cols[i].colId; + } + + if (tsdbLoadBlockDataCols(pHelper, pDataCols, blkIdx, colIds, pDataCols->numOfCols) < 0) goto _err; + + tfree(colIds); return 0; + +_err: + tfree(colIds); + return -1; } static int tsdbCheckHelperCfg(SHelperCfg *pCfg) { @@ -672,7 +692,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa int rowsMustMerge = tsdbGetRowsInRange(pDataCols, 0, pCompBlock->keyLast); int maxRowsCanMerge = - MIN(pHelper->config.maxRowsPerFileBlock - pCompBlock->numOfPoints, tsdbGetRowsInRange(pDataCols, keyLimit)); + MIN(pHelper->config.maxRowsPerFileBlock - pCompBlock->numOfPoints, tsdbGetRowsInRange(pDataCols, 0, keyLimit)); if (pCompBlock->numOfPoints + rowsMustMerge > pHelper->config.maxRowsPerFileBlock) { // Need to load the block and split as two super block From 66cf4a5bacc4077e239960cf0a0a0b6ebd623577 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Tue, 14 Apr 2020 09:33:14 +0800 Subject: [PATCH 09/29] TD-100 --- src/vnode/tsdb/src/tsdbMain.c | 3 ++- src/vnode/tsdb/src/tsdbRWHelper.c | 29 +++++++---------------------- 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index da602df424..6934392a0c 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -760,6 +760,8 @@ static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int max int numOfRows = 0; do { + if (numOfRows >= maxRowsToRead) break; + SSkipListNode *node = tSkipListIterGet(pIter); if (node == NULL) break; @@ -769,7 +771,6 @@ static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int max tdAppendDataRowToDataCol(row, pCols); numOfRows++; - if (numOfRows >= maxRowsToRead) break; } while (tSkipListIterNext(pIter)); return numOfRows; diff --git a/src/vnode/tsdb/src/tsdbRWHelper.c b/src/vnode/tsdb/src/tsdbRWHelper.c index 7f1c642ef4..de9d1008c1 100644 --- a/src/vnode/tsdb/src/tsdbRWHelper.c +++ b/src/vnode/tsdb/src/tsdbRWHelper.c @@ -243,27 +243,11 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { rowsToWrite = tsdbMergeDataWithBlock(pHelper, blkIdx, pDataCols); if (rowsToWrite < 0) goto _err; - - } else { // Either merge with the previous block or save as a super block in the middle - SCompBlock *prevBlock = (blkIdx == 0) ? NULL : (pCompBlock - 1); - - int rows1 = nRowsLEThan(pDataCols, pCompBlock->keyFirst); // rows write as a super block in the middle - int rows2 = (prevBlock) ? (pHelper->config.maxRowsPerFileBlock - prevBlock->numOfPoints) - : rows1; // rows can merge with the previous block - if (rows1 >= rows2) { - rowsToWrite = tsdbWriteBlockToFile(pHelper, &(pHelper->files.dataF), pDataCols, rows1, &compBlock, false, true); - if (rowsToWrite < 0) goto _err; - - ASSERT(rowsToWrite == rows1); - - // Add the super block to it - pIdx->len += sizeof(SCompBlock); - pIdx->numOfSuperBlocks++; - } else { - rowsToWrite = tsdbMergeDataWithBlock(pHelper, blkIdx-1, pDataCols); - if (rowsToWrite < 0) goto _err; - ASSERT(rowsToWrite == rows2); - } + } else { // Save as a super block in the middle + int rowsToWrite = tsdbGetRowsInRange(pDataCols, 0, pCompBlock->keyFirst-1); + ASSERT(rowsToWrite > 0); + if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.dataF), pDataCols, rowsToWrite, &compBlock, false, true) < 0) goto _err; + if (tsdbInsertSuperBlock(pHelper, pCompBlock, pCompBlock - pHelper->pCompInfo->blocks) < 0) goto _err; } } } @@ -671,7 +655,7 @@ static int nRowsLEThan(SDataCols *pDataCols, int maxKey) { return ((TSKEY *)ptr - (TSKEY *)(pDataCols->cols[0].pData)) + 1; } -// Merge the data with a block +// Merge the data with a block in file static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols) { // TODO: set pHelper->hasOldBlock int rowsWritten = 0; @@ -685,6 +669,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx; ASSERT(pCompBlock->numOfSubBlocks >= 1); ASSERT(keyFirst >= pCompBlock->keyFirst); + ASSERT(compareKeyBlock((void *)&keyFirst, (void *)pCompBlock) == 0); // Start here TSKEY keyLimit = From 69b818610b93ab5df2ec123f5c13526bb70ef509 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Tue, 14 Apr 2020 11:49:31 +0800 Subject: [PATCH 10/29] TD-100 --- src/common/src/dataformat.c | 7 +- src/vnode/tsdb/src/tsdbMain.c | 8 +- src/vnode/tsdb/src/tsdbRWHelper.c | 156 +++++++++++++++++------------- 3 files changed, 96 insertions(+), 75 deletions(-) diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c index fd7dd5b91e..a17ceff54b 100644 --- a/src/common/src/dataformat.c +++ b/src/common/src/dataformat.c @@ -433,18 +433,13 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) { // TODO: merge the pTemp part int rowsLeft = pTarget->numOfPoints - iter1; if (rowsLeft > 0) { - } break; } - - - - } return 0; - _err: +_err: return -1; } \ No newline at end of file diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 6934392a0c..33991aec52 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -755,6 +755,7 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) { } static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols) { + ASSERT(maxRowsToRead > 0); if (pIter == NULL) return 0; int numOfRows = 0; @@ -914,14 +915,19 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters ASSERT(rowsRead >= 0); if (pDataCols->numOfPoints == 0) break; + ASSERT(dataColsKeyFirst(pDataCols) >= minKey && dataColsKeyFirst(pDataCols) <= maxKey); + ASSERT(dataColsKeyLast(pDataCols) >= minKey && dataColsKeyLast(pDataCols) <= maxKey); + int rowsWritten = tsdbWriteDataBlock(pHelper, pDataCols); if (rowsWritten < 0) goto _err; - assert(rowsWritten <= pDataCols->numOfPoints); + ASSERT(rowsWritten <= pDataCols->numOfPoints); tdPopDataColsPoints(pDataCols, rowsWritten); maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5 - pDataCols->numOfPoints; } + ASSERT(pDataCols->numOfPoints == 0); + // Move the last block to the new .l file if neccessary if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) goto _err; diff --git a/src/vnode/tsdb/src/tsdbRWHelper.c b/src/vnode/tsdb/src/tsdbRWHelper.c index de9d1008c1..2f879bb37d 100644 --- a/src/vnode/tsdb/src/tsdbRWHelper.c +++ b/src/vnode/tsdb/src/tsdbRWHelper.c @@ -244,7 +244,7 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { rowsToWrite = tsdbMergeDataWithBlock(pHelper, blkIdx, pDataCols); if (rowsToWrite < 0) goto _err; } else { // Save as a super block in the middle - int rowsToWrite = tsdbGetRowsInRange(pDataCols, 0, pCompBlock->keyFirst-1); + rowsToWrite = tsdbGetRowsInRange(pDataCols, 0, pCompBlock->keyFirst-1); ASSERT(rowsToWrite > 0); if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.dataF), pDataCols, rowsToWrite, &compBlock, false, true) < 0) goto _err; if (tsdbInsertSuperBlock(pHelper, pCompBlock, pCompBlock - pHelper->pCompInfo->blocks) < 0) goto _err; @@ -440,22 +440,75 @@ int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx, return 0; } -// Load the whole block data -int tsdbLoadBlockData(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols) { - int16_t *colIds = (int16_t *)calloc(pDataCols->numOfCols, sizeof(int16_t)); - if (colIds == NULL) goto _err; +/** + * Interface to read the data of a sub-block OR the data of a super-block of which (numOfSubBlocks == 1) + */ +static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols) { + ASSERT(pCompBlock->numOfSubBlocks <= 1); - for (int i = 0; i < pDataCols->numOfCols; i++) { - colIds[i] = pDataCols->cols[i].colId; + SCompData *pCompData = (SCompData *)malloc(pCompBlock->len); + if (pCompData == NULL) return -1; + + int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd; + if (tread(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) goto _err; + + { // TODO : check the correctness of the part } - if (tsdbLoadBlockDataCols(pHelper, pDataCols, blkIdx, colIds, pDataCols->numOfCols) < 0) goto _err; + ASSERT(pCompBlock->numOfCols == pCompData->numOfCols); + + pDataCols->numOfPoints = pCompBlock->numOfPoints; + + size_t tlen = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols; + int ccol = 0, dcol = 0; + while (true) { + if (ccol >= pDataCols->numOfCols) { + // TODO: Fill rest NULL + break; + } + if (dcol >= pCompData->numOfCols) break; + + SCompCol *pCompCol = &(pCompData->cols[ccol]); + SDataCol *pDataCol = &(pDataCols->cols[dcol]); + + if (pCompCol->colId == pDataCol->colId) { + // TODO: uncompress + memcpy(pDataCol->pData, (void *)(((char *)pCompData) + tlen + pCompCol->offset), pCompCol->len); + ccol++; + dcol++; + } else if (pCompCol->colId > pDataCol->colId) { + // TODO: Fill NULL + dcol++; + } else { + ccol++; + } + } + + tfree(pCompData); + return 0; + +_err: + tfree(pCompData); + return -1; +} + +// Load the whole block data +int tsdbLoadBlockData(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols) { + SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx; + + int numOfSubBlock = pCompBlock->numOfSubBlocks; + if (numOfSubBlock > 1) pCompBlock = (SCompBlock *)((char *)pHelper->pCompInfo + pCompBlock->offset); + + if (tsdbLoadBlockDataImpl(pHelper, pCompBlock, pHelper->pDataCols[0]) < 0) goto _err; + for (int i = 1; i < numOfSubBlock; i++) { + pCompBlock++; + if (tsdbLoadBlockDataImpl(pHelper, pCompBlock, pHelper->pDataCols[1]) < 0) goto _err; + if (tdMergeDataCols(pHelper->pDataCols[0], pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfPoints) < 0) goto _err; + } - tfree(colIds); return 0; _err: - tfree(colIds); return -1; } @@ -671,71 +724,38 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa ASSERT(keyFirst >= pCompBlock->keyFirst); ASSERT(compareKeyBlock((void *)&keyFirst, (void *)pCompBlock) == 0); - // Start here - TSKEY keyLimit = - (blkIdx == pIdx->numOfSuperBlocks - 1) ? INT_MAX : pHelper->pCompInfo->blocks[blkIdx + 1].keyLast - 1; + if (keyFirst > pCompBlock->keyLast) { // Merge the last block by append + ASSERT(pCompBlock->last && pCompBlock->numOfPoints < pHelper->config.minRowsPerFileBlock); + int defaultRowsToWrite = pHelper->config.maxRowsPerFileBlock * 4 / 5; // TODO: make a interface - int rowsMustMerge = tsdbGetRowsInRange(pDataCols, 0, pCompBlock->keyLast); - int maxRowsCanMerge = - MIN(pHelper->config.maxRowsPerFileBlock - pCompBlock->numOfPoints, tsdbGetRowsInRange(pDataCols, 0, keyLimit)); - - if (pCompBlock->numOfPoints + rowsMustMerge > pHelper->config.maxRowsPerFileBlock) { - // Need to load the block and split as two super block - } else { - } - - if (rowsMustMerge + pCompBlock->numOfPoints > pHelper->config.maxRowsPerFileBlock) { - // Load the block and merge as two super block - } - - if (rowsMustMerge > maxRowsCanMerge) { - ASSERT(pCompBlock->numOfPoints + rowsMustMerge > pHelper->config.maxRowsPerFileBlock); - - } else { - - } - - - int rowsToMerge = tsdbGetRowsCanBeMergedWithBlock(pHelper, blkIdx, pDataCols); - if (rowsToMerge < 0) goto _err; - - ASSERT(rowsToMerge > 0); - - if (pCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS && - ((!pCompBlock->last) || (pHelper->files.nLastF.fd < 0 && - pCompBlock->numOfPoints + rowsToMerge < pHelper->config.minRowsPerFileBlock))) { - - SFile *pFile = NULL; - - if ((!pCompBlock->last) || (pCompBlock->numOfPoints + rowsToMerge >= pHelper->config.minRowsPerFileBlock)) { - pFile = &(pHelper->files.dataF); + rowsWritten = MIN((defaultRowsToWrite - pCompBlock->numOfPoints), pDataCols->numOfPoints); + if (rowsWritten + pCompBlock->numOfPoints >= pHelper->config.minRowsPerFileBlock) { + // Need to write to .data file + if (tsdbLoadBlockData(pHelper, blkIdx, NULL) < 0) goto _err; + // tdMergeDataCols(); + // if (tsdbWriteBlockToFile(pHelper, &pHelper->files.dataF, NULL, rowsWritten + pCompBlock->numOfPoints, &compBlock, + // false, true) < 0) + // goto _err; + if (tsdbUpdateSuperBlock(pHelper, &compBlock, blkIdx) < 0) goto _err; } else { - pFile = &(pHelper->files.lastF); - } - - if (tsdbWriteBlockToFile(pHelper, pFile, pDataCols, rowsToMerge, &compBlock, pCompBlock->last, false) < 0) goto _err; - - if (tsdbAddSubBlock(pHelper, &compBlock, blkIdx, rowsToMerge) < 0) goto _err; - } else { - // Read-Merge-Write as a super block - if (tsdbLoadBlockData(pHelper, blkIdx, NULL) < 0) goto _err; - tdMergeDataCols(pHelper->pDataCols[0], pDataCols, rowsToMerge); - - int isLast = 0; - SFile *pFile = NULL; - if (!pCompBlock->last || (pCompBlock->numOfPoints + rowsToMerge >= pHelper->config.minRowsPerFileBlock)) { - pFile = &(pHelper->files.dataF); - } else { - isLast = 1; + // Need still write the .last or .l file if (pHelper->files.nLastF.fd > 0) { - pFile = &(pHelper->files.nLastF); + if (tsdbLoadBlockData(pHelper, blkIdx, NULL) < 0) goto _err; + tdMergeDataCols(NULL, pDataCols, rowsWritten); + if (tsdbWriteBlockToFile(pHelper, &pHelper->files.nLastF, NULL, rowsWritten + pCompBlock->numOfPoints, + &compBlock, false, true) < 0) + goto _err; + if (tsdbUpdateSuperBlock(pHelper, &compBlock, blkIdx) < 0) goto _err; } else { - pFile = &(pHelper->files.lastF); + // Write to .last file and append as a sub-block + if (tsdbWriteBlockToFile(pHelper, &pHelper->files.lastF, pDataCols, rowsWritten, &compBlock, true, false) < 0) + goto _err; + if (tsdbAddSubBlock(pHelper, &compBlock, blkIdx, rowsWritten) < 0) goto _err; } } - if (tsdbWriteBlockToFile(pHelper, pFile, pHelper->pDataCols[0], pCompBlock->numOfPoints + rowsToMerge, &compBlock, isLast, true) < 0) goto _err; - if (tsdbUpdateSuperBlock(pHelper, &compBlock, blkIdx) < 0) goto _err; + } else { // Must merge with the block + } return rowsWritten; From 1accde3c042b5815060170fbb990a993132c8ec5 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Tue, 14 Apr 2020 15:17:14 +0800 Subject: [PATCH 11/29] TD-100 --- src/vnode/tsdb/inc/tsdbMain.h | 2 +- src/vnode/tsdb/src/tsdbRWHelper.c | 200 +++++++++++++++++++++--------- 2 files changed, 144 insertions(+), 58 deletions(-) diff --git a/src/vnode/tsdb/inc/tsdbMain.h b/src/vnode/tsdb/inc/tsdbMain.h index beb9ba9868..979e1c2865 100644 --- a/src/vnode/tsdb/inc/tsdbMain.h +++ b/src/vnode/tsdb/inc/tsdbMain.h @@ -437,7 +437,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target); int tsdbLoadCompInfo(SRWHelper *pHelper, void *target); int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target); int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx, int16_t *colIds, int numOfColIds); -int tsdbLoadBlockData(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols); +int tsdbLoadBlockData(SRWHelper *pHelper, int blkIdx, SDataCols *target); // --------- For write operations int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols); diff --git a/src/vnode/tsdb/src/tsdbRWHelper.c b/src/vnode/tsdb/src/tsdbRWHelper.c index 2f879bb37d..d67fd03964 100644 --- a/src/vnode/tsdb/src/tsdbRWHelper.c +++ b/src/vnode/tsdb/src/tsdbRWHelper.c @@ -38,8 +38,8 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa bool isLast, bool isSuperBlock); static int compareKeyBlock(const void *arg1, const void *arg2); static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols); -static int nRowsLEThan(SDataCols *pDataCols, int maxKey); -static int tsdbGetRowsCanBeMergedWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols); +// static int nRowsLEThan(SDataCols *pDataCols, int maxKey); +// static int tsdbGetRowsCanBeMergedWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols); static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded); static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); @@ -397,7 +397,6 @@ static int tsdbLoadSingleColumnData(int fd, SCompBlock *pCompBlock, SCompCol *pC static int tsdbLoadSingleBlockDataCols(SRWHelper *pHelper, SCompBlock *pCompBlock, int16_t *colIds, int numOfColIds, SDataCols *pDataCols) { if (tsdbLoadCompData(pHelper, pCompBlock, NULL) < 0) return -1; - size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols; int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd; void *ptr = NULL; @@ -421,7 +420,6 @@ static int tsdbLoadSingleBlockDataCols(SRWHelper *pHelper, SCompBlock *pCompBloc // Load specific column data from file int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx, int16_t *colIds, int numOfColIds) { - SCompIdx * pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx; ASSERT(pCompBlock->numOfSubBlocks >= 1); // Must be super block @@ -493,7 +491,7 @@ _err: } // Load the whole block data -int tsdbLoadBlockData(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols) { +int tsdbLoadBlockData(SRWHelper *pHelper, int blkIdx, SDataCols *target) { SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx; int numOfSubBlock = pCompBlock->numOfSubBlocks; @@ -506,6 +504,8 @@ int tsdbLoadBlockData(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols) { if (tdMergeDataCols(pHelper->pDataCols[0], pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfPoints) < 0) goto _err; } + // if (target) TODO + return 0; _err: @@ -702,11 +702,11 @@ static FORCE_INLINE int compKeyFunc(const void *arg1, const void *arg2) { return ((*(TSKEY *)arg1) - (*(TSKEY *)arg2)); } -static int nRowsLEThan(SDataCols *pDataCols, int maxKey) { - void *ptr = taosbsearch((void *)&maxKey, pDataCols->cols[0].pData, pDataCols->numOfPoints, sizeof(TSKEY), compKeyFunc, TD_LE); - if (ptr == NULL) return 0; - return ((TSKEY *)ptr - (TSKEY *)(pDataCols->cols[0].pData)) + 1; -} +// static int nRowsLEThan(SDataCols *pDataCols, int maxKey) { +// void *ptr = taosbsearch((void *)&maxKey, pDataCols->cols[0].pData, pDataCols->numOfPoints, sizeof(TSKEY), compKeyFunc, TD_LE); +// if (ptr == NULL) return 0; +// return ((TSKEY *)ptr - (TSKEY *)(pDataCols->cols[0].pData)) + 1; +// } // Merge the data with a block in file static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols) { @@ -732,18 +732,18 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa if (rowsWritten + pCompBlock->numOfPoints >= pHelper->config.minRowsPerFileBlock) { // Need to write to .data file if (tsdbLoadBlockData(pHelper, blkIdx, NULL) < 0) goto _err; - // tdMergeDataCols(); - // if (tsdbWriteBlockToFile(pHelper, &pHelper->files.dataF, NULL, rowsWritten + pCompBlock->numOfPoints, &compBlock, - // false, true) < 0) - // goto _err; + if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, rowsWritten) < 0) goto _err; + if (tsdbWriteBlockToFile(pHelper, &pHelper->files.dataF, pHelper->pDataCols[0], + rowsWritten + pCompBlock->numOfPoints, &compBlock, false, true) < 0) + goto _err; if (tsdbUpdateSuperBlock(pHelper, &compBlock, blkIdx) < 0) goto _err; } else { // Need still write the .last or .l file if (pHelper->files.nLastF.fd > 0) { if (tsdbLoadBlockData(pHelper, blkIdx, NULL) < 0) goto _err; - tdMergeDataCols(NULL, pDataCols, rowsWritten); - if (tsdbWriteBlockToFile(pHelper, &pHelper->files.nLastF, NULL, rowsWritten + pCompBlock->numOfPoints, - &compBlock, false, true) < 0) + if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, rowsWritten) < 0) goto _err; + if (tsdbWriteBlockToFile(pHelper, &pHelper->files.nLastF, pHelper->pDataCols[0], + rowsWritten + pCompBlock->numOfPoints, &compBlock, false, true) < 0) goto _err; if (tsdbUpdateSuperBlock(pHelper, &compBlock, blkIdx) < 0) goto _err; } else { @@ -753,9 +753,95 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa if (tsdbAddSubBlock(pHelper, &compBlock, blkIdx, rowsWritten) < 0) goto _err; } } + } else { + // TODO: key overlap, must merge with the block + ASSERT(keyFirst <= pCompBlock->keyLast); - } else { // Must merge with the block + TSKEY keyLimit = + (blkIdx == pIdx->numOfSuperBlocks - 1) ? INT64_MAX : pHelper->pCompInfo->blocks[blkIdx + 1].keyFirst - 1; + int rows1 = tsdbGetRowsInRange(pDataCols, pCompBlock->keyFirst, + pCompBlock->keyLast); // number of rows must merge in this block + int rows2 = + pHelper->config.maxRowsPerFileBlock - pCompBlock->numOfPoints; // max nuber of rows the block can have more + int rows3 = tsdbGetRowsInRange(pDataCols, pCompBlock->keyFirst, + keyLimit); // number of rows between this block and the next block + + ASSERT(rows3 >= rows1); + + if ((rows2 >= rows1) && ((!pCompBlock->last) || (pHelper->files.nLastF.fd < 0))) { + rowsWritten = rows1; + bool isLast = false; + SFile *pFile = NULL; + + if (pCompBlock->last) { + isLast = true; + pFile = &(pHelper->files.lastF); + } else { + pFile = &(pHelper->files.dataF); + } + + if (tsdbWriteBlockToFile(pHelper, pFile, pDataCols, rows1, &compBlock, isLast, false) < 0) goto _err; + if (tsdbAddSubBlock(pHelper, &compBlock, blkIdx, rowsWritten) < 0) goto _err; + } else { + // Need to read the data block and merge with pCompDataCol to write as super block + + // Read + if (tsdbLoadBlockData(pHelper, blkIdx, NULL) < 0) goto _err; + rowsWritten = rows3; + + int iter1 = 0; // iter over pHelper->pDataCols[0] + int iter2 = 0; // iter over pDataCols + tdResetDataCols(pHelper->pDataCols[1]); + while (true) { + if (iter1 >= pHelper->pDataCols[0]->numOfPoints && iter2 >= rows3) { + if (pHelper->pDataCols[1]->numOfPoints > 0) { + if (tsdbWriteBlockToFile(pHelper, &pHelper->files.dataF, pHelper->pDataCols[1], + pHelper->pDataCols[1]->numOfPoints, &compBlock, false, true) < 0) + goto _err; + // TODO: the blkIdx here is not correct + tsdbAddSubBlock(pHelper, &compBlock, blkIdx, pHelper->pDataCols[1]->numOfPoints); + } + } + + TSKEY key1 = iter1 >= pHelper->pDataCols[0]->numOfPoints + ? INT64_MAX + : ((int64_t *)(pHelper->pDataCols[0]->cols[0].pData))[iter1]; + TSKEY key2 = iter2 >= rowsWritten ? INT64_MAX : ((int64_t *)(pDataCols->cols[0].pData))[iter2]; + + if (key1 < key2) { + for (int i = 0; i < pDataCols->numOfCols; i++) { + SDataCol *pDataCol = pHelper->pDataCols[1]->cols + i; + memcpy(((char *)pDataCol->pData + TYPE_BYTES[pDataCol->type] * pHelper->pDataCols[1]->numOfPoints), + ((char *)pHelper->pDataCols[0]->cols[i].pData + TYPE_BYTES[pDataCol->type] * iter1), + TYPE_BYTES[pDataCol->type]); + } + pHelper->pDataCols[1]->numOfPoints++; + iter1++; + } else if (key1 == key2) { + // TODO: think about duplicate key cases + ASSERT(false); + } else { + for (int i = 0; i < pDataCols->numOfCols; i++) { + SDataCol *pDataCol = pHelper->pDataCols[1]->cols + i; + memcpy(((char *)pDataCol->pData + TYPE_BYTES[pDataCol->type] * pHelper->pDataCols[1]->numOfPoints), + ((char *)pDataCols->cols[i].pData + + TYPE_BYTES[pDataCol->type] * iter2), + TYPE_BYTES[pDataCol->type]); + } + pHelper->pDataCols[1]->numOfPoints++; + iter2++; + } + + if (pHelper->pDataCols[0]->numOfPoints >= pHelper->config.maxRowsPerFileBlock * 4 / 5) { + if (tsdbWriteBlockToFile(pHelper, &pHelper->files.dataF, pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfPoints, &compBlock, false, true) < 0) goto _err; + // TODO: blkIdx here is not correct, fix it + tsdbInsertSuperBlock(pHelper, &compBlock, blkIdx); + + tdResetDataCols(pHelper->pDataCols[1]); + } + } + } } return rowsWritten; @@ -767,58 +853,58 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa static int compTSKEY(const void *key1, const void *key2) { return ((TSKEY *)key1 - (TSKEY *)key2); } // Get the number of rows the data can be merged into the block -static int tsdbGetRowsCanBeMergedWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols) { - int rowsCanMerge = 0; - TSKEY keyFirst = dataColsKeyFirst(pDataCols); +// static int tsdbGetRowsCanBeMergedWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols) { +// int rowsCanMerge = 0; +// TSKEY keyFirst = dataColsKeyFirst(pDataCols); - SCompIdx * pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; - SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx; +// SCompIdx * pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; +// SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx; - ASSERT(blkIdx < pIdx->numOfSuperBlocks); +// ASSERT(blkIdx < pIdx->numOfSuperBlocks); - TSKEY keyMax = (blkIdx < pIdx->numOfSuperBlocks + 1) ? (pCompBlock + 1)->keyFirst - 1 : pHelper->files.maxKey; +// TSKEY keyMax = (blkIdx < pIdx->numOfSuperBlocks + 1) ? (pCompBlock + 1)->keyFirst - 1 : pHelper->files.maxKey; - if (keyFirst > pCompBlock->keyLast) { - void *ptr = taosbsearch((void *)(&keyMax), pDataCols->cols[0].pData, pDataCols->numOfPoints, sizeof(TSKEY), - compTSKEY, TD_LE); - ASSERT(ptr != NULL); +// if (keyFirst > pCompBlock->keyLast) { +// void *ptr = taosbsearch((void *)(&keyMax), pDataCols->cols[0].pData, pDataCols->numOfPoints, sizeof(TSKEY), +// compTSKEY, TD_LE); +// ASSERT(ptr != NULL); - rowsCanMerge = - MIN((TSKEY *)ptr - (TSKEY *)pDataCols->cols[0].pData, pHelper->config.minRowsPerFileBlock - pCompBlock->numOfPoints); +// rowsCanMerge = +// MIN((TSKEY *)ptr - (TSKEY *)pDataCols->cols[0].pData, pHelper->config.minRowsPerFileBlock - pCompBlock->numOfPoints); - } else { - int32_t colId[1] = {0}; - if (tsdbLoadBlockDataCols(pHelper, NULL, blkIdx, colId, 1) < 0) goto _err; +// } else { +// int32_t colId[1] = {0}; +// if (tsdbLoadBlockDataCols(pHelper, NULL, blkIdx, colId, 1) < 0) goto _err; - int iter1 = 0; // For pDataCols - int iter2 = 0; // For loaded data cols +// int iter1 = 0; // For pDataCols +// int iter2 = 0; // For loaded data cols - while (1) { - if (iter1 >= pDataCols->numOfPoints || iter2 >= pHelper->pDataCols[0]->numOfPoints) break; - if (pCompBlock->numOfPoints + rowsCanMerge >= pHelper->config.maxRowsPerFileBlock) break; +// while (1) { +// if (iter1 >= pDataCols->numOfPoints || iter2 >= pHelper->pDataCols[0]->numOfPoints) break; +// if (pCompBlock->numOfPoints + rowsCanMerge >= pHelper->config.maxRowsPerFileBlock) break; - TSKEY key1 = dataColsKeyAt(pDataCols, iter1); - TSKEY key2 = dataColsKeyAt(pHelper->pDataCols[0], iter2); +// TSKEY key1 = dataColsKeyAt(pDataCols, iter1); +// TSKEY key2 = dataColsKeyAt(pHelper->pDataCols[0], iter2); - if (key1 > keyMax) break; +// if (key1 > keyMax) break; - if (key1 < key2) { - iter1++; - } else if (key1 == key2) { - iter1++; - iter2++; - } else { - iter2++; - rowsCanMerge++; - } - } - } +// if (key1 < key2) { +// iter1++; +// } else if (key1 == key2) { +// iter1++; +// iter2++; +// } else { +// iter2++; +// rowsCanMerge++; +// } +// } +// } - return rowsCanMerge; +// return rowsCanMerge; -_err: - return -1; -} +// _err: +// return -1; +// } static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t spaceNeeded) { SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; From bc5396e90f7b35e2c278f0404a9299941ba6ef7b Mon Sep 17 00:00:00 2001 From: hzcheng Date: Tue, 14 Apr 2020 18:56:12 +0800 Subject: [PATCH 12/29] TD-100 --- src/vnode/tsdb/inc/tsdbMain.h | 7 -- src/vnode/tsdb/src/tsdbMain.c | 145 ++--------------------------- src/vnode/tsdb/src/tsdbRWHelper.c | 16 ++-- src/vnode/tsdb/src/tsdbRead.c | 18 ++-- src/vnode/tsdb/tests/tsdbTests.cpp | 22 ++--- 5 files changed, 37 insertions(+), 171 deletions(-) diff --git a/src/vnode/tsdb/inc/tsdbMain.h b/src/vnode/tsdb/inc/tsdbMain.h index 610f377d72..6c418d2c0a 100644 --- a/src/vnode/tsdb/inc/tsdbMain.h +++ b/src/vnode/tsdb/inc/tsdbMain.h @@ -401,13 +401,6 @@ typedef struct { size_t compDataSize; SDataCols *pDataCols[2]; - // ------ Perhaps no usage - SCompIdx compIdx; // SCompIdx of current table - int blockIter; // For write purpose - - // Compression buffer - void * cBuffer; - size_t cBufSize; } SRWHelper; // --------- Helper state diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 52d21d5bd1..292d2eabb9 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -801,19 +801,16 @@ static SSkipListIterator **tsdbCreateTableIters(STsdbMeta *pMeta, int maxTables) if (pTable == NULL || pTable->imem == NULL) continue; iters[tid] = tSkipListCreateIter(pTable->imem->pData); - if (iters[tid] == NULL) { - tsdbDestroyTableIters(iters, maxTables); - return NULL; - } + if (iters[tid] == NULL) goto _err; - if (!tSkipListIterNext(iters[tid])) { - // No data in this iterator - tSkipListDestroyIter(iters[tid]); - iters[tid] = NULL; - } + if (!tSkipListIterNext(iters[tid])) goto _err; } return iters; + + _err: + tsdbDestroyTableIters(iters, maxTables); + return NULL; } static void tsdbFreeMemTable(SMemTable *pMemTable) { @@ -832,6 +829,7 @@ static void *tsdbCommitData(void *arg) { STsdbCache *pCache = pRepo->tsdbCache; STsdbCfg * pCfg = &(pRepo->config); SDataCols * pDataCols = NULL; + SRWHelper whelper = {0}; if (pCache->imem == NULL) return NULL; // Create the iterator to read from cache @@ -842,7 +840,6 @@ static void *tsdbCommitData(void *arg) { } // Create a write helper for commit data - SRWHelper whelper; SHelperCfg hcfg = { .type = TSDB_WRITE_HELPER, .maxTables = pCfg->maxTables, @@ -989,130 +986,4 @@ static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minK if (nextKey > 0 && (nextKey >= minKey && nextKey <= maxKey)) return 1; } return 0; -} - -// static int tsdbWriteBlockToFileImpl(SFile *pFile, SDataCols *pCols, int pointsToWrite, int64_t *offset, int32_t *len, int64_t uid) { -// size_t size = sizeof(SCompData) + sizeof(SCompCol) * pCols->numOfCols; -// SCompData *pCompData = (SCompData *)malloc(size); -// if (pCompData == NULL) return -1; - -// pCompData->delimiter = TSDB_FILE_DELIMITER; -// pCompData->uid = uid; -// pCompData->numOfCols = pCols->numOfCols; - -// *offset = lseek(pFile->fd, 0, SEEK_END); -// *len = size; - -// int toffset = size; -// for (int iCol = 0; iCol < pCols->numOfCols; iCol++) { -// SCompCol *pCompCol = pCompData->cols + iCol; -// SDataCol *pDataCol = pCols->cols + iCol; - -// pCompCol->colId = pDataCol->colId; -// pCompCol->type = pDataCol->type; -// pCompCol->offset = toffset; - -// // TODO: add compression -// pCompCol->len = TYPE_BYTES[pCompCol->type] * pointsToWrite; -// toffset += pCompCol->len; -// } - -// // Write the block -// if (write(pFile->fd, (void *)pCompData, size) < 0) goto _err; -// *len += size; -// for (int iCol = 0; iCol < pCols->numOfCols; iCol++) { -// SDataCol *pDataCol = pCols->cols + iCol; -// SCompCol *pCompCol = pCompData->cols + iCol; -// if (write(pFile->fd, pDataCol->pData, pCompCol->len) < 0) goto _err; -// *len += pCompCol->len; -// } - -// if (pCompData == NULL) free((void *)pCompData); -// return 0; - -// _err: -// if (pCompData == NULL) free((void *)pCompData); -// return -1; -// } - -// static int compareKeyBlock(const void *arg1, const void *arg2) { -// TSKEY key = *(TSKEY *)arg1; -// SCompBlock *pBlock = (SCompBlock *)arg2; - -// if (key < pBlock->keyFirst) { -// return -1; -// } else if (key > pBlock->keyLast) { -// return 1; -// } - -// return 0; -// } - -// int tsdbWriteBlockToFile(STsdbRepo *pRepo, SFileGroup *pGroup, SCompIdx *pIdx, SCompInfo *pCompInfo, SDataCols *pCols, SCompBlock *pCompBlock, SFile *lFile, int64_t uid) { -// STsdbCfg * pCfg = &(pRepo->config); -// SFile * pFile = NULL; -// int numOfPointsToWrite = 0; -// int64_t offset = 0; -// int32_t len = 0; - -// memset((void *)pCompBlock, 0, sizeof(SCompBlock)); - -// if (pCompInfo == NULL) { -// // Just append the data block to .data or .l or .last file -// numOfPointsToWrite = pCols->numOfPoints; -// if (pCols->numOfPoints > pCfg->minRowsPerFileBlock) { // Write to .data file -// pFile = &(pGroup->files[TSDB_FILE_TYPE_DATA]); -// } else { // Write to .last or .l file -// pCompBlock->last = 1; -// if (lFile) { -// pFile = lFile; -// } else { -// pFile = &(pGroup->files[TSDB_FILE_TYPE_LAST]); -// } -// } -// tsdbWriteBlockToFileImpl(pFile, pCols, numOfPointsToWrite, &offset, &len, uid); -// pCompBlock->offset = offset; -// pCompBlock->len = len; -// pCompBlock->algorithm = 2; // TODO : add to configuration -// pCompBlock->sversion = pCols->sversion; -// pCompBlock->numOfPoints = pCols->numOfPoints; -// pCompBlock->numOfSubBlocks = 1; -// pCompBlock->numOfCols = pCols->numOfCols; -// pCompBlock->keyFirst = dataColsKeyFirst(pCols); -// pCompBlock->keyLast = dataColsKeyLast(pCols); -// } else { -// // Need to merge the block to either the last block or the other block -// TSKEY keyFirst = dataColsKeyFirst(pCols); -// SCompBlock *pMergeBlock = NULL; - -// // Search the block to merge in -// void *ptr = taosbsearch((void *)&keyFirst, (void *)(pCompInfo->blocks), sizeof(SCompBlock), pIdx->numOfSuperBlocks, -// compareKeyBlock, TD_GE); -// if (ptr == NULL) { -// // No block greater or equal than the key, but there are data in the .last file, need to merge the last file block -// // and merge the data -// pMergeBlock = TSDB_COMPBLOCK_AT(pCompInfo, pIdx->numOfSuperBlocks - 1); -// } else { -// pMergeBlock = (SCompBlock *)ptr; -// } - -// if (pMergeBlock->last) { -// if (pMergeBlock->last + pCols->numOfPoints > pCfg->minRowsPerFileBlock) { -// // Need to load the data from .last and combine data in pCols to write to .data file - -// } else { // Just append the block to .last or .l file -// if (lFile) { -// // read the block from .last file and merge with pCols, write to .l file - -// } else { -// // tsdbWriteBlockToFileImpl(); -// } -// } -// } else { // The block need to merge in .data file - -// } - -// } - -// return numOfPointsToWrite; -// } +} \ No newline at end of file diff --git a/src/vnode/tsdb/src/tsdbRWHelper.c b/src/vnode/tsdb/src/tsdbRWHelper.c index d67fd03964..f41d045ae5 100644 --- a/src/vnode/tsdb/src/tsdbRWHelper.c +++ b/src/vnode/tsdb/src/tsdbRWHelper.c @@ -169,7 +169,7 @@ void tsdbSetHelperTable(SRWHelper *pHelper, SHelperTable *pHelperTable, STSchema ASSERT(helperHasState(pHelper, TSDB_HELPER_FILE_SET_AND_OPEN)); // Clear members and state used by previous table - pHelper->blockIter = 0; + // pHelper->blockIter = 0; pHelper->state &= (TSDB_HELPER_TABLE_SET - 1); pHelper->tableInfo = *pHelperTable; @@ -339,23 +339,25 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { ASSERT(helperHasState(pHelper, TSDB_HELPER_TABLE_SET)); - SCompIdx curCompIdx = pHelper->compIdx; + SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; - ASSERT(curCompIdx.offset > 0 && curCompIdx.len > 0); + // SCompIdx curCompIdx = pHelper->compIdx; + + ASSERT(pIdx->offset > 0 && pIdx->len > 0); int fd = pHelper->files.headF.fd; if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { - if (lseek(fd, curCompIdx.offset, SEEK_SET) < 0) return -1; + if (lseek(fd, pIdx->offset, SEEK_SET) < 0) return -1; - adjustMem(pHelper->pCompInfo, pHelper->compInfoSize, curCompIdx.len); - if (tread(fd, (void *)(pHelper->pCompInfo), pHelper->compIdx.len) < pHelper->compIdx.len) return -1; + adjustMem(pHelper->pCompInfo, pHelper->compInfoSize, pIdx->len); + if (tread(fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) return -1; // TODO: check the checksum helperSetState(pHelper, TSDB_HELPER_INFO_LOAD); } - if (target) memcpy(target, (void *)(pHelper->pCompInfo), curCompIdx.len); + if (target) memcpy(target, (void *)(pHelper->pCompInfo), pIdx->len); return 0; } diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index bf45c2d0af..cd63993f7d 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -302,7 +302,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo } // load all the comp offset value for all tables in this file - tsdbLoadCompIdx(fileGroup, pQueryHandle->compIndex, 10000); // todo set dynamic max tables + // tsdbLoadCompIdx(fileGroup, pQueryHandle->compIndex, 10000); // todo set dynamic max tables *numOfBlocks = 0; size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); @@ -324,7 +324,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo pCheckInfo->compSize = compIndex->len; } - tsdbLoadCompBlocks(fileGroup, compIndex, pCheckInfo->pCompInfo); + // tsdbLoadCompBlocks(fileGroup, compIndex, pCheckInfo->pCompInfo); SCompInfo* pCompInfo = pCheckInfo->pCompInfo; @@ -421,15 +421,15 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo pFile->fd = open(pFile->fname, O_RDONLY); } - if (tsdbLoadDataBlock(pFile, pBlock, 1, pCheckInfo->pDataCols, data) == 0) { - SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo; + // if (tsdbLoadDataBlock(pFile, pBlock, 1, pCheckInfo->pDataCols, data) == 0) { + // SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo; - pBlockLoadInfo->fileGroup = pQueryHandle->pFileGroup; - pBlockLoadInfo->slot = pQueryHandle->cur.slot; - pBlockLoadInfo->sid = pCheckInfo->pTableObj->tableId.tid; + // pBlockLoadInfo->fileGroup = pQueryHandle->pFileGroup; + // pBlockLoadInfo->slot = pQueryHandle->cur.slot; + // pBlockLoadInfo->sid = pCheckInfo->pTableObj->tableId.tid; - blockLoaded = true; - } + // blockLoaded = true; + // } taosArrayDestroy(sa); tfree(data); diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp index 685b2cce95..28d511ae2b 100644 --- a/src/vnode/tsdb/tests/tsdbTests.cpp +++ b/src/vnode/tsdb/tests/tsdbTests.cpp @@ -140,12 +140,12 @@ TEST(TsdbTest, DISABLED_createRepo) { // TEST(TsdbTest, DISABLED_openRepo) { TEST(TsdbTest, openRepo) { - tsdb_repo_t *repo = tsdbOpenRepo("/home/ubuntu/work/build/test/data/vnode/vnode2/tsdb", NULL); - ASSERT_NE(repo, nullptr); + // tsdb_repo_t *repo = tsdbOpenRepo("/home/ubuntu/work/build/test/data/vnode/vnode2/tsdb", NULL); + // ASSERT_NE(repo, nullptr); -// STsdbRepo *pRepo = (STsdbRepo *)repo; + // STsdbRepo *pRepo = (STsdbRepo *)repo; - SFileGroup *pGroup = tsdbSearchFGroup(pRepo->tsdbFileH, 1655); + // SFileGroup *pGroup = tsdbSearchFGroup(pRepo->tsdbFileH, 1655); // for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { // tsdbOpenFile(&pGroup->files[type], O_RDONLY); @@ -156,7 +156,7 @@ TEST(TsdbTest, openRepo) { // SCompInfo *pCompInfo = (SCompInfo *)malloc(sizeof(SCompInfo) + pIdx[1].len); - tsdbLoadCompBlocks(pGroup, &pIdx[1], (void *)pCompInfo); + // tsdbLoadCompBlocks(pGroup, &pIdx[1], (void *)pCompInfo); // int blockIdx = 0; // SCompBlock *pBlock = &(pCompInfo->blocks[blockIdx]); @@ -165,20 +165,20 @@ TEST(TsdbTest, openRepo) { // tsdbLoadCompCols(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, (void *)pCompData); - STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, pCompData->uid); - SDataCols *pDataCols = tdNewDataCols(tdMaxRowBytesFromSchema(tsdbGetTableSchema(pRepo->tsdbMeta, pTable)), 5, 10); - tdInitDataCols(pDataCols, tsdbGetTableSchema(pRepo->tsdbMeta, pTable)); + // STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, pCompData->uid); + // SDataCols *pDataCols = tdNewDataCols(tdMaxRowBytesFromSchema(tsdbGetTableSchema(pRepo->tsdbMeta, pTable)), 5, 10); + // tdInitDataCols(pDataCols, tsdbGetTableSchema(pRepo->tsdbMeta, pTable)); // tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, 1, pDataCols, pCompData); - tdResetDataCols(pDataCols); + // tdResetDataCols(pDataCols); - tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock + 1, 1, pDataCols, pCompData); + // tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock + 1, 1, pDataCols, pCompData); // int k = 0; -// } +} TEST(TsdbTest, DISABLED_createFileGroup) { SFileGroup fGroup; From f15535afc369d9e4a4bd84c4863db579544ac96c Mon Sep 17 00:00:00 2001 From: hzcheng Date: Wed, 15 Apr 2020 18:28:12 +0800 Subject: [PATCH 13/29] TD-100 --- src/vnode/tsdb/inc/tsdbMain.h | 2 +- src/vnode/tsdb/src/tsdbRWHelper.c | 184 +++++++++++++++--------------- 2 files changed, 96 insertions(+), 90 deletions(-) diff --git a/src/vnode/tsdb/inc/tsdbMain.h b/src/vnode/tsdb/inc/tsdbMain.h index 6c418d2c0a..97a1a0208f 100644 --- a/src/vnode/tsdb/inc/tsdbMain.h +++ b/src/vnode/tsdb/inc/tsdbMain.h @@ -419,7 +419,7 @@ typedef struct { int tsdbInitHelper(SRWHelper *pHelper, SHelperCfg *pCfg); void tsdbDestroyHelper(SRWHelper *pHelper); -void tsdbClearHelper(SRWHelper *pHelper); +void tsdbResetHelper(SRWHelper *pHelper); // --------- For set operations int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup); diff --git a/src/vnode/tsdb/src/tsdbRWHelper.c b/src/vnode/tsdb/src/tsdbRWHelper.c index f41d045ae5..8f5826821b 100644 --- a/src/vnode/tsdb/src/tsdbRWHelper.c +++ b/src/vnode/tsdb/src/tsdbRWHelper.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ #include "tsdbMain.h" +#include "tchecksum.h" #define adjustMem(ptr, size, expectedSize) \ do { \ @@ -25,17 +26,11 @@ // Local function definitions static int tsdbCheckHelperCfg(SHelperCfg *pCfg); -static void tsdbInitHelperFile(SHelperFile *pHFile); -static int tsdbInitHelperRead(SRWHelper *pHelper); -// static int tsdbInitHelperWrite(SRWHelper *pHelper); +static int tsdbInitHelperFile(SRWHelper *pHelper); static void tsdbClearHelperFile(SHelperFile *pHFile); -static void tsdbDestroyHelperRead(SRWHelper *pHelper); -static void tsdbDestroyHelperWrite(SRWHelper *pHelper); -static void tsdbClearHelperRead(SRWHelper *pHelper); -static void tsdbClearHelperWrite(SRWHelper *pHelper); static bool tsdbShouldCreateNewLast(SRWHelper *pHelper); -static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, int rowsToWrite, SCompBlock *pCompBlock, - bool isLast, bool isSuperBlock); +static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, int rowsToWrite, + SCompBlock *pCompBlock, bool isLast, bool isSuperBlock); static int compareKeyBlock(const void *arg1, const void *arg2); static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols); // static int nRowsLEThan(SDataCols *pDataCols, int maxKey); @@ -45,24 +40,83 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); static int tsdbGetRowsInRange(SDataCols *pDataCols, int minKey, int maxKey); +// ---------- Operations on Helper File part +static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { + memset((void *)&pHelper->files, 0, sizeof(pHelper->files)); + pHelper->files.fid = -1; + pHelper->files.headF.fd = -1; + pHelper->files.dataF.fd = -1; + pHelper->files.lastF.fd = -1; + pHelper->files.nHeadF.fd = -1; + pHelper->files.nLastF.fd = -1; +} + +static int tsdbInitHelperFile(SRWHelper *pHelper) { + pHelper->compIdxSize = sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM); + pHelper->pCompIdx = (SCompIdx *)malloc(pHelper->compIdxSize); + if (pHelper->pCompIdx == NULL) return -1; + + tsdbResetHelperFileImpl(pHelper); + return 0; +} + +static void tsdbDestroyHelperFile(SRWHelper *pHelper) { + tsdbCloseHelperFile(pHelper, false); + tfree(pHelper->pCompIdx); +} + +// ---------- Operations on Helper Table part +static void tsdbResetHelperTableImpl(SRWHelper *pHelper) { + memset((void *)&pHelper->tableInfo, 0, sizeof(SHelperTable)); + pHelper->hasOldLastBlock = false; +} + +static void tsdbInitHelperTable(SRWHelper *pHelper) { + tsdbResetHelperTableImpl(pHelper); +} + +static void tsdbDestroyHelperTable(SRWHelper *pHelper) { return; } + +// ---------- Operations on Helper Block part +static void tsdbResetHelperBlockImpl(SRWHelper *pHelper) { + tdResetDataCols(pHelper->pDataCols[0]); + tdResetDataCols(pHelper->pDataCols[1]); +} + +static int tsdbInitHelperBlock(SRWHelper *pHelper) { + pHelper->pDataCols[0] = tdNewDataCols(pHelper->config.maxRowSize, pHelper->config.maxCols, pHelper->config.maxRows); + pHelper->pDataCols[1] = tdNewDataCols(pHelper->config.maxRowSize, pHelper->config.maxCols, pHelper->config.maxRows); + if (pHelper->pDataCols[0] == NULL || pHelper->pDataCols[1] == NULL) return -1; + + tsdbResetHelperBlockImpl(pHelper); + + return 0; +} + +static void tsdbDestroyHelperBlock(SRWHelper *pHelper) { + tdFreeDataCols(pHelper->pDataCols[0]); + tdFreeDataCols(pHelper->pDataCols[1]); +} + +// ------------------------------------------ OPERATIONS FOR OUTSIDE ------------------------------------------ int tsdbInitHelper(SRWHelper *pHelper, SHelperCfg *pCfg) { if (pHelper == NULL || pCfg == NULL || tsdbCheckHelperCfg(pCfg) < 0) return -1; memset((void *)pHelper, 0, sizeof(*pHelper)); + // Init global configuration pHelper->config = *pCfg; - - tsdbInitHelperFile(&(pHelper->files)); - - if (tsdbInitHelperRead(pHelper) < 0) goto _err; - - pHelper->pDataCols[0] = tdNewDataCols(pCfg->maxRowSize, pCfg->maxCols, pCfg->maxRows); - pHelper->pDataCols[1] = tdNewDataCols(pCfg->maxRowSize, pCfg->maxCols, pCfg->maxRows); - - if ((pHelper->pDataCols[0] == NULL) || (pHelper->pDataCols[1] == NULL)) goto _err; - pHelper->state = TSDB_HELPER_CLEAR_STATE; + // Init file part + if (tsdbInitHelperFile(pHelper) < 0) goto _err; + + // Init table part + tsdbInitHelperTable(pHelper); + + // Init block part + if (tsdbInitHelperBlock(pHelper) < 0) goto _err; + return 0; _err: @@ -71,25 +125,36 @@ _err: } void tsdbDestroyHelper(SRWHelper *pHelper) { - if (pHelper == NULL) return; - - tsdbClearHelperFile(&(pHelper->files)); - tsdbDestroyHelperRead(pHelper); - tsdbDestroyHelperWrite(pHelper); + if (pHelper) { + tsdbDestroyHelperFile(pHelper); + tsdbDestroyHelperTable(pHelper); + tsdbDestroyHelperBlock(pHelper); + memset((void *)pHelper, 0, sizeof(*pHelper)); + } } -void tsdbClearHelper(SRWHelper *pHelper) { - if (pHelper == NULL) return; - tsdbClearHelperFile(&(pHelper->files)); - tsdbClearHelperRead(pHelper); - tsdbClearHelperWrite(pHelper); +void tsdbResetHelper(SRWHelper *pHelper) { + if (pHelper) { + // Reset the block part + tsdbResetHelperBlockImpl(pHelper); + + // Reset the table part + tsdbResetHelperTableImpl(pHelper); + + // Reset the file part + tsdbCloseHelperFile(pHelper, false); + tsdbResetHelperFileImpl(pHelper); + + pHelper->state = TSDB_HELPER_CLEAR_STATE; + } } +// ------------ Operations for read/write purpose int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { ASSERT(pHelper != NULL && pGroup != NULL); // Clear the helper object - tsdbClearHelper(pHelper); + tsdbResetHelper(pHelper); ASSERT(pHelper->state == TSDB_HELPER_CLEAR_STATE); @@ -519,14 +584,6 @@ static int tsdbCheckHelperCfg(SHelperCfg *pCfg) { return 0; } -static void tsdbInitHelperFile(SHelperFile *pHFile) { - pHFile->fid = -1; - pHFile->headF.fd = -1; - pHFile->dataF.fd = -1; - pHFile->lastF.fd = -1; - pHFile->nHeadF.fd = -1; - pHFile->nLastF.fd = -1; -} static void tsdbClearHelperFile(SHelperFile *pHFile) { pHFile->fid = -1; @@ -553,57 +610,6 @@ static void tsdbClearHelperFile(SHelperFile *pHFile) { } -static int tsdbInitHelperRead(SRWHelper *pHelper) { - SHelperCfg *pCfg = &(pHelper->config); - - pHelper->compIdxSize = pCfg->maxTables * sizeof(SCompIdx); - if ((pHelper->pCompIdx = (SCompIdx *)malloc(pHelper->compIdxSize)) == NULL) return -1; - - return 0; -} - -static void tsdbDestroyHelperRead(SRWHelper *pHelper) { - tfree(pHelper->pCompIdx); - pHelper->compIdxSize = 0; - - tfree(pHelper->pCompInfo); - pHelper->compInfoSize = 0; - - tfree(pHelper->pCompData); - pHelper->compDataSize = 0; - - tdFreeDataCols(pHelper->pDataCols[0]); - tdFreeDataCols(pHelper->pDataCols[1]); -} - -// static int tsdbInitHelperWrite(SRWHelper *pHelper) { -// SHelperCfg *pCfg = &(pHelper->config); - -// // pHelper->wCompIdxSize = pCfg->maxTables * sizeof(SCompIdx); -// // if ((pHelper->pWCompIdx = (SCompIdx *)malloc(pHelper->wCompIdxSize)) == NULL) return -1; - -// return 0; -// } - -static void tsdbDestroyHelperWrite(SRWHelper *pHelper) { - // tfree(pHelper->pWCompIdx); - // pHelper->wCompIdxSize = 0; - - // tfree(pHelper->pWCompInfo); - // pHelper->wCompInfoSize = 0; - - // tfree(pHelper->pWCompData); - // pHelper->wCompDataSize = 0; -} - -static void tsdbClearHelperRead(SRWHelper *pHelper) { - // TODO -} - -static void tsdbClearHelperWrite(SRWHelper *pHelper) { - // TODO -} - static bool tsdbShouldCreateNewLast(SRWHelper *pHelper) { // TODO return 0; From a85c04c0fb1c59d49bc99accfad62d59786d65ec Mon Sep 17 00:00:00 2001 From: hzcheng Date: Wed, 15 Apr 2020 23:48:44 +0800 Subject: [PATCH 14/29] TD-100 --- src/util/inc/tutil.h | 7 +++++++ src/util/src/tutil.c | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h index c4c802a688..15f5ce6fa3 100644 --- a/src/util/inc/tutil.h +++ b/src/util/inc/tutil.h @@ -178,6 +178,13 @@ void taosDumpMemoryLeak(); void *taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, int (*compar)(const void *, const void *), int flags); +void * tmalloc(size_t size); +void * tcalloc(size_t nmemb, size_t size); +size_t tsizeof(void *ptr); +void tmemset(void *ptr, int c); +void * trealloc(void *ptr, size_t size); +void tzfree(void *ptr); + #ifdef TAOS_MEM_CHECK void * taos_malloc(size_t size, const char *file, uint32_t line); diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index aa144600a0..63e174ab06 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -689,3 +689,42 @@ void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, return NULL; } + +void *tmalloc(size_t size) { + if (size <= 0) return NULL; + + void *ret = malloc(size + sizeof(size_t)); + if (ret == NULL) return NULL; + + *(size_t *)ret = size; + + return (void *)((char *)ret + sizeof(size_t)); +} + +void *tcalloc(size_t nmemb, size_t size) { + size_t tsize = nmemb * size; + void * ret = tmalloc(tsize); + if (ret == NULL) return NULL; + + tmemset(ret, 0); + return ret; +} + +size_t tsizeof(void *ptr) { return *(size_t *)((char *)ptr - sizeof(size_t)); } + +void tmemset(void *ptr, int c) { memset(ptr, c, tsizeof(ptr)); } + +void * trealloc(void *ptr, size_t size) { + if (size <= tsizeof(ptr)) return ptr; + + void * tptr = (void *)((char *)ptr - sizeof(size_t)); + size_t tsize = size + sizeof(size_t); + tptr = realloc(tptr, tsize); + if (tptr == NULL) return NULL; + + *(size_t *)tptr = size; + + return (void *)((char *)tptr + sizeof(size_t)); +} + +void tzfree(void *ptr) { free((void *)((char *)ptr - sizeof(size_t))); } \ No newline at end of file From 3f5b1415beae6a99bd0fa4595a68b46d4723eb3c Mon Sep 17 00:00:00 2001 From: hzcheng Date: Thu, 16 Apr 2020 09:49:20 +0800 Subject: [PATCH 15/29] TD-100 --- src/util/src/tutil.c | 10 +++- src/vnode/tsdb/inc/tsdbMain.h | 6 +-- src/vnode/tsdb/src/tsdbRWHelper.c | 90 +++++++------------------------ 3 files changed, 29 insertions(+), 77 deletions(-) diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 63e174ab06..106663aadd 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -710,11 +710,13 @@ void *tcalloc(size_t nmemb, size_t size) { return ret; } -size_t tsizeof(void *ptr) { return *(size_t *)((char *)ptr - sizeof(size_t)); } +size_t tsizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; } void tmemset(void *ptr, int c) { memset(ptr, c, tsizeof(ptr)); } void * trealloc(void *ptr, size_t size) { + if (ptr == NULL) return tmalloc(size); + if (size <= tsizeof(ptr)) return ptr; void * tptr = (void *)((char *)ptr - sizeof(size_t)); @@ -727,4 +729,8 @@ void * trealloc(void *ptr, size_t size) { return (void *)((char *)tptr + sizeof(size_t)); } -void tzfree(void *ptr) { free((void *)((char *)ptr - sizeof(size_t))); } \ No newline at end of file +void tzfree(void *ptr) { + if (ptr) { + free((void *)((char *)ptr - sizeof(size_t))); + } +} \ No newline at end of file diff --git a/src/vnode/tsdb/inc/tsdbMain.h b/src/vnode/tsdb/inc/tsdbMain.h index 97a1a0208f..f6bc44def3 100644 --- a/src/vnode/tsdb/inc/tsdbMain.h +++ b/src/vnode/tsdb/inc/tsdbMain.h @@ -388,17 +388,17 @@ typedef struct { // For file set usage SHelperFile files; SCompIdx * pCompIdx; - size_t compIdxSize; + // size_t compIdxSize; // For table set usage SHelperTable tableInfo; SCompInfo * pCompInfo; - size_t compInfoSize; + // size_t compInfoSize; bool hasOldLastBlock; // For block set usage SCompData *pCompData; - size_t compDataSize; + // size_t compDataSize; SDataCols *pDataCols[2]; } SRWHelper; diff --git a/src/vnode/tsdb/src/tsdbRWHelper.c b/src/vnode/tsdb/src/tsdbRWHelper.c index 8f5826821b..540cef5719 100644 --- a/src/vnode/tsdb/src/tsdbRWHelper.c +++ b/src/vnode/tsdb/src/tsdbRWHelper.c @@ -52,8 +52,9 @@ static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { } static int tsdbInitHelperFile(SRWHelper *pHelper) { - pHelper->compIdxSize = sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM); - pHelper->pCompIdx = (SCompIdx *)malloc(pHelper->compIdxSize); + // pHelper->compIdxSize = sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM); + size_t tsize = sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM); + pHelper->pCompIdx = (SCompIdx *)tmalloc(tsize); if (pHelper->pCompIdx == NULL) return -1; tsdbResetHelperFileImpl(pHelper); @@ -62,7 +63,7 @@ static int tsdbInitHelperFile(SRWHelper *pHelper) { static void tsdbDestroyHelperFile(SRWHelper *pHelper) { tsdbCloseHelperFile(pHelper, false); - tfree(pHelper->pCompIdx); + tzfree(pHelper->pCompIdx); } // ---------- Operations on Helper Table part @@ -75,7 +76,7 @@ static void tsdbInitHelperTable(SRWHelper *pHelper) { tsdbResetHelperTableImpl(pHelper); } -static void tsdbDestroyHelperTable(SRWHelper *pHelper) { return; } +static void tsdbDestroyHelperTable(SRWHelper *pHelper) { tzfree((void *)pHelper->pCompInfo); } // ---------- Operations on Helper Block part static void tsdbResetHelperBlockImpl(SRWHelper *pHelper) { @@ -94,6 +95,7 @@ static int tsdbInitHelperBlock(SRWHelper *pHelper) { } static void tsdbDestroyHelperBlock(SRWHelper *pHelper) { + tzfree(pHelper->pCompData); tdFreeDataCols(pHelper->pDataCols[0]); tdFreeDataCols(pHelper->pDataCols[1]); } @@ -377,7 +379,7 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { int tsdbWriteCompIdx(SRWHelper *pHelper) { if (lseek(pHelper->files.nHeadF.fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1; - if (twrite(pHelper->files.nHeadF.fd, (void *)pHelper->pCompIdx, pHelper->compIdxSize) < pHelper->compIdxSize) + if (twrite(pHelper->files.nHeadF.fd, (void *)pHelper->pCompIdx, tsizeof(pHelper->pCompIdx)) < tsizeof(pHelper->pCompIdx)) return -1; return 0; } @@ -390,13 +392,13 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { int fd = pHelper->files.headF.fd; if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1; - if (tread(fd, (void *)(pHelper->pCompIdx), pHelper->compIdxSize) < pHelper->compIdxSize) return -1; + if (tread(fd, (void *)(pHelper->pCompIdx), tsizeof(pHelper->pCompIdx)) < tsizeof(pHelper->pCompIdx)) return -1; // TODO: check the correctness of the part } helperSetState(pHelper, TSDB_HELPER_IDX_LOAD); // Copy the memory for outside usage - if (target) memcpy(target, pHelper->pCompIdx, pHelper->compIdxSize); + if (target) memcpy(target, pHelper->pCompIdx, tsizeof(pHelper->pCompIdx)); return 0; } @@ -415,7 +417,8 @@ int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { if (lseek(fd, pIdx->offset, SEEK_SET) < 0) return -1; - adjustMem(pHelper->pCompInfo, pHelper->compInfoSize, pIdx->len); + // adjustMem(pHelper->pCompInfo, pHelper->compInfoSize, pIdx->len); + pHelper->pCompInfo = trealloc((void *)pHelper->pCompInfo, pIdx->len); if (tread(fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) return -1; // TODO: check the checksum @@ -433,9 +436,9 @@ int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) { if (lseek(fd, pCompBlock->offset, SEEK_SET) < 0) return -1; - size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols; - adjustMem(pHelper->pCompData, pHelper->compDataSize, tsize); - + size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols + sizeof(TSCKSUM); + pHelper->pCompData = trealloc((void *)pHelper->pCompData, tsize); + if (pHelper->pCompData == NULL) return -1; if (tread(fd, (void *)pHelper->pCompData, tsize) < tsize) return -1; ASSERT(pCompBlock->numOfCols == pHelper->pCompData->numOfCols); @@ -860,73 +863,16 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa static int compTSKEY(const void *key1, const void *key2) { return ((TSKEY *)key1 - (TSKEY *)key2); } -// Get the number of rows the data can be merged into the block -// static int tsdbGetRowsCanBeMergedWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols) { -// int rowsCanMerge = 0; -// TSKEY keyFirst = dataColsKeyFirst(pDataCols); - -// SCompIdx * pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; -// SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx; - -// ASSERT(blkIdx < pIdx->numOfSuperBlocks); - -// TSKEY keyMax = (blkIdx < pIdx->numOfSuperBlocks + 1) ? (pCompBlock + 1)->keyFirst - 1 : pHelper->files.maxKey; - -// if (keyFirst > pCompBlock->keyLast) { -// void *ptr = taosbsearch((void *)(&keyMax), pDataCols->cols[0].pData, pDataCols->numOfPoints, sizeof(TSKEY), -// compTSKEY, TD_LE); -// ASSERT(ptr != NULL); - -// rowsCanMerge = -// MIN((TSKEY *)ptr - (TSKEY *)pDataCols->cols[0].pData, pHelper->config.minRowsPerFileBlock - pCompBlock->numOfPoints); - -// } else { -// int32_t colId[1] = {0}; -// if (tsdbLoadBlockDataCols(pHelper, NULL, blkIdx, colId, 1) < 0) goto _err; - -// int iter1 = 0; // For pDataCols -// int iter2 = 0; // For loaded data cols - -// while (1) { -// if (iter1 >= pDataCols->numOfPoints || iter2 >= pHelper->pDataCols[0]->numOfPoints) break; -// if (pCompBlock->numOfPoints + rowsCanMerge >= pHelper->config.maxRowsPerFileBlock) break; - -// TSKEY key1 = dataColsKeyAt(pDataCols, iter1); -// TSKEY key2 = dataColsKeyAt(pHelper->pDataCols[0], iter2); - -// if (key1 > keyMax) break; - -// if (key1 < key2) { -// iter1++; -// } else if (key1 == key2) { -// iter1++; -// iter2++; -// } else { -// iter2++; -// rowsCanMerge++; -// } -// } -// } - -// return rowsCanMerge; - -// _err: -// return -1; -// } - static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t spaceNeeded) { SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; - size_t spaceLeft = pHelper->compInfoSize - pIdx->len; + size_t spaceLeft = tsizeof((void *)pHelper->pCompInfo) - pIdx->len; ASSERT(spaceLeft >= 0); if (spaceLeft < spaceNeeded) { - size_t tsize = pHelper->compInfoSize + sizeof(SCompBlock) * 16; - if (pHelper->compInfoSize == 0) tsize += sizeof(SCompInfo); + size_t tsize = tsizeof(pHelper->pCompInfo) + sizeof(SCompBlock) * 16; + if (tsizeof(pHelper->pCompInfo) == 0) tsize += sizeof(SCompInfo); - pHelper->pCompInfo = (SCompInfo *)realloc((void *)(pHelper->pCompInfo), tsize); - if (pHelper->pCompInfo == NULL) return -1; - - pHelper->compInfoSize = tsize; + pHelper->pCompInfo = (SCompInfo *)trealloc(pHelper->pCompInfo, tsize); } return 0; From 53f1d0a279999d3bcffbc1075a812bbd023a923e Mon Sep 17 00:00:00 2001 From: hzcheng Date: Thu, 16 Apr 2020 10:05:00 +0800 Subject: [PATCH 16/29] TD-100 --- src/{vnode => }/tsdb/src/tsdbRWHelper.c | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/{vnode => }/tsdb/src/tsdbRWHelper.c (100%) diff --git a/src/vnode/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c similarity index 100% rename from src/vnode/tsdb/src/tsdbRWHelper.c rename to src/tsdb/src/tsdbRWHelper.c From d50595644045129980897e93ef44ecf2e0131956 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Thu, 16 Apr 2020 15:56:17 +0800 Subject: [PATCH 17/29] TD-100 --- src/tsdb/inc/tsdbMain.h | 18 ++-- src/tsdb/src/tsdbFile.c | 5 +- src/tsdb/src/tsdbMain.c | 7 +- src/tsdb/src/tsdbRWHelper.c | 175 +++++++++++++++++++++-------------- src/tsdb/tests/tsdbTests.cpp | 141 ++++++++++++++-------------- 5 files changed, 193 insertions(+), 153 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index f6bc44def3..c7743e733d 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -351,13 +351,14 @@ typedef enum { TSDB_WRITE_HELPER, TSDB_READ_HELPER } tsdb_rw_helper_t; typedef struct { tsdb_rw_helper_t type; // helper type - int maxTables; - int maxRowSize; - int maxRows; - int maxCols; - int minRowsPerFileBlock; - int maxRowsPerFileBlock; - int8_t compress; + + int maxTables; + int maxRowSize; + int maxRows; + int maxCols; + int minRowsPerFileBlock; + int maxRowsPerFileBlock; + int8_t compress; } SHelperCfg; typedef struct { @@ -388,17 +389,14 @@ typedef struct { // For file set usage SHelperFile files; SCompIdx * pCompIdx; - // size_t compIdxSize; // For table set usage SHelperTable tableInfo; SCompInfo * pCompInfo; - // size_t compInfoSize; bool hasOldLastBlock; // For block set usage SCompData *pCompData; - // size_t compDataSize; SDataCols *pDataCols[2]; } SRWHelper; diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index b4d7e16967..afe0582c1b 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -25,6 +25,7 @@ #include "tutil.h" #include "tsdbMain.h" +#include "tchecksum.h" const char *tsdbFileSuffix[] = { ".head", // TSDB_FILE_TYPE_HEAD @@ -310,7 +311,7 @@ static int tsdbWriteFileHead(SFile *pFile) { } static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables) { - int size = sizeof(SCompIdx) * maxTables; + int size = sizeof(SCompIdx) * maxTables + sizeof(TSCKSUM); void *buf = calloc(1, size); if (buf == NULL) return -1; @@ -319,6 +320,8 @@ static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables) { return -1; } + taosCalcChecksumAppend(0, (uint8_t *)buf, size); + if (write(pFile->fd, buf, size) < 0) { free(buf); return -1; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 292d2eabb9..55f80a8d22 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -56,7 +56,8 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock); static int32_t tsdbRestoreCfg(STsdbRepo *pRepo, STsdbCfg *pCfg); static int32_t tsdbGetDataDirName(STsdbRepo *pRepo, char *fname); static void * tsdbCommitData(void *arg); -static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SRWHelper *pHelper, SDataCols *pDataCols); +static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SRWHelper *pHelper, + SDataCols *pDataCols); static TSKEY tsdbNextIterKey(SSkipListIterator *pIter); static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey); // static int tsdbWriteBlockToFileImpl(SFile *pFile, SDataCols *pCols, int pointsToWrite, int64_t *offset, int32_t *len, @@ -847,6 +848,7 @@ static void *tsdbCommitData(void *arg) { .maxRows = pCfg->maxRowsPerFileBlock, .maxCols = pMeta->maxCols, .minRowsPerFileBlock = pCfg->minRowsPerFileBlock, + .maxRowsPerFileBlock = pCfg->maxRowsPerFileBlock, .compress = 2 // TODO make it a configuration }; if (tsdbInitHelper(&whelper, &hcfg) < 0) goto _exit; @@ -911,6 +913,8 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters // Loop to commit data in each table for (int tid = 0; tid < pCfg->maxTables; tid++) { STable * pTable = pMeta->tables[tid]; + if (pTable == NULL) continue; + SSkipListIterator *pIter = iters[tid]; // Set the helper and the buffer dataCols object to help to write this table @@ -929,6 +933,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters ASSERT(dataColsKeyLast(pDataCols) >= minKey && dataColsKeyLast(pDataCols) <= maxKey); int rowsWritten = tsdbWriteDataBlock(pHelper, pDataCols); + ASSERT(rowsWritten != 0); if (rowsWritten < 0) goto _err; ASSERT(rowsWritten <= pDataCols->numOfPoints); diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 540cef5719..e24a3c5786 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -14,15 +14,7 @@ */ #include "tsdbMain.h" #include "tchecksum.h" - -#define adjustMem(ptr, size, expectedSize) \ - do { \ - if ((size) < (expectedSize)) { \ - (ptr) = realloc((void *)(ptr), (expectedSize)); \ - if ((ptr) == NULL) return -1; \ - (size) = (expectedSize); \ - } \ - } while (0) +#include "tscompression.h" // Local function definitions static int tsdbCheckHelperCfg(SHelperCfg *pCfg); @@ -33,12 +25,11 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pD SCompBlock *pCompBlock, bool isLast, bool isSuperBlock); static int compareKeyBlock(const void *arg1, const void *arg2); static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols); -// static int nRowsLEThan(SDataCols *pDataCols, int maxKey); -// static int tsdbGetRowsCanBeMergedWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols); static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded); static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); static int tsdbGetRowsInRange(SDataCols *pDataCols, int minKey, int maxKey); +static void tsdbResetHelperBlock(SRWHelper *pHelper); // ---------- Operations on Helper File part static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { @@ -72,6 +63,12 @@ static void tsdbResetHelperTableImpl(SRWHelper *pHelper) { pHelper->hasOldLastBlock = false; } +static void tsdbResetHelperTable(SRWHelper *pHelper) { + tsdbResetHelperBlock(pHelper); + tsdbResetHelperTableImpl(pHelper); + helperClearState(pHelper, TSDB_HELPER_TABLE_SET); +} + static void tsdbInitHelperTable(SRWHelper *pHelper) { tsdbResetHelperTableImpl(pHelper); } @@ -84,6 +81,10 @@ static void tsdbResetHelperBlockImpl(SRWHelper *pHelper) { tdResetDataCols(pHelper->pDataCols[1]); } +static void tsdbResetHelperBlock(SRWHelper *pHelper) { + // TODO +} + static int tsdbInitHelperBlock(SRWHelper *pHelper) { pHelper->pDataCols[0] = tdNewDataCols(pHelper->config.maxRowSize, pHelper->config.maxCols, pHelper->config.maxRows); pHelper->pDataCols[1] = tdNewDataCols(pHelper->config.maxRowSize, pHelper->config.maxCols, pHelper->config.maxRows); @@ -167,7 +168,6 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { pHelper->files.lastF = pGroup->files[TSDB_FILE_TYPE_LAST]; if (TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER) { char *fnameDup = strdup(pHelper->files.headF.fname); - if (fnameDup == NULL) goto _err; if (fnameDup == NULL) return -1; char *dataDir = dirname(fnameDup); @@ -183,8 +183,8 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { if (tsdbOpenFile(&(pHelper->files.lastF), O_RDWR) < 0) goto _err; // Create and open .h - if (tsdbOpenFile(&(pHelper->files.nHeadF), O_WRONLY | O_CREAT) < 0) goto _err; - size_t tsize = TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pHelper->config.maxTables; + if (tsdbOpenFile(&(pHelper->files.nHeadF), O_WRONLY | O_CREAT) < 0) return -1; + size_t tsize = TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM); if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, tsize) < tsize) goto _err; // Create and open .l file if should @@ -221,23 +221,33 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { if (pHelper->files.nHeadF.fd > 0) { close(pHelper->files.nHeadF.fd); pHelper->files.nHeadF.fd = -1; - if (hasError) remove(pHelper->files.nHeadF.fname); + if (hasError) { + remove(pHelper->files.nHeadF.fname); + } else { + rename(pHelper->files.nHeadF.fname, pHelper->files.headF.fname); + pHelper->files.headF.info = pHelper->files.nHeadF.info; + } } if (pHelper->files.nLastF.fd > 0) { close(pHelper->files.nLastF.fd); pHelper->files.nLastF.fd = -1; - if (hasError) remove(pHelper->files.nLastF.fname); + if (hasError) { + remove(pHelper->files.nLastF.fname); + } else { + rename(pHelper->files.nLastF.fname, pHelper->files.lastF.fname); + pHelper->files.lastF.info = pHelper->files.nLastF.info; + } } return 0; } void tsdbSetHelperTable(SRWHelper *pHelper, SHelperTable *pHelperTable, STSchema *pSchema) { - ASSERT(helperHasState(pHelper, TSDB_HELPER_FILE_SET_AND_OPEN)); + ASSERT(helperHasState(pHelper, TSDB_HELPER_FILE_SET_AND_OPEN | TSDB_HELPER_IDX_LOAD)); // Clear members and state used by previous table - // pHelper->blockIter = 0; - pHelper->state &= (TSDB_HELPER_TABLE_SET - 1); + tsdbResetHelperTable(pHelper); + ASSERT(pHelper->state == (TSDB_HELPER_FILE_SET_AND_OPEN | TSDB_HELPER_IDX_LOAD)); pHelper->tableInfo = *pHelperTable; tdInitDataCols(pHelper->pDataCols[0], pSchema); @@ -248,9 +258,8 @@ void tsdbSetHelperTable(SRWHelper *pHelper, SHelperTable *pHelperTable, STSchema pHelper->hasOldLastBlock = true; } - // pHelper->compIdx = pHelper->pCompIdx[pHelper->tableInfo.tid]; - helperSetState(pHelper, TSDB_HELPER_TABLE_SET); + ASSERT(pHelper->state == ((TSDB_HELPER_TABLE_SET << 1) - 1)); } /** @@ -268,7 +277,6 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { TSKEY keyFirst = dataColsKeyFirst(pDataCols); ASSERT(helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)); - // SCompIdx curIdx = pHelper->compIdx; // old table SCompIdx for sendfile usage SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; // for change purpose // Load the SCompInfo part if neccessary @@ -283,7 +291,7 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { SFile *pWFile = NULL; bool isLast = false; - if (rowsToWrite > pHelper->config.minRowsPerFileBlock) { + if (rowsToWrite >= pHelper->config.minRowsPerFileBlock) { pWFile = &(pHelper->files.dataF); } else { isLast = true; @@ -314,7 +322,7 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { rowsToWrite = tsdbGetRowsInRange(pDataCols, 0, pCompBlock->keyFirst-1); ASSERT(rowsToWrite > 0); if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.dataF), pDataCols, rowsToWrite, &compBlock, false, true) < 0) goto _err; - if (tsdbInsertSuperBlock(pHelper, pCompBlock, pCompBlock - pHelper->pCompInfo->blocks) < 0) goto _err; + if (tsdbInsertSuperBlock(pHelper, pCompBlock, blkIdx) < 0) goto _err; } } } @@ -367,6 +375,7 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, pIdx->len) < pIdx->len) return -1; } } else { + taosCalcChecksumAppend(0, (uint8_t *)pHelper->pCompInfo, pIdx->len); pIdx->offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END); if (pIdx->offset < 0) return -1; @@ -379,6 +388,8 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { int tsdbWriteCompIdx(SRWHelper *pHelper) { if (lseek(pHelper->files.nHeadF.fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1; + taosCalcChecksumAppend(0, (uint8_t *)pHelper->pCompIdx, tsizeof(pHelper->pCompIdx)); + if (twrite(pHelper->files.nHeadF.fd, (void *)pHelper->pCompIdx, tsizeof(pHelper->pCompIdx)) < tsizeof(pHelper->pCompIdx)) return -1; return 0; @@ -392,8 +403,11 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { int fd = pHelper->files.headF.fd; if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1; - if (tread(fd, (void *)(pHelper->pCompIdx), tsizeof(pHelper->pCompIdx)) < tsizeof(pHelper->pCompIdx)) return -1; - // TODO: check the correctness of the part + if (tread(fd, (void *)(pHelper->pCompIdx), tsizeof((void *)pHelper->pCompIdx)) < tsizeof(pHelper->pCompIdx)) return -1; + if (!taosCheckChecksumWhole((uint8_t *)(pHelper->pCompIdx), tsizeof((void *)pHelper->pCompIdx))) { + // TODO: File is broken, try to deal with it + return -1; + } } helperSetState(pHelper, TSDB_HELPER_IDX_LOAD); @@ -408,19 +422,16 @@ int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; - // SCompIdx curCompIdx = pHelper->compIdx; - - ASSERT(pIdx->offset > 0 && pIdx->len > 0); - int fd = pHelper->files.headF.fd; if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { - if (lseek(fd, pIdx->offset, SEEK_SET) < 0) return -1; + if (pIdx->offset > 0) { + if (lseek(fd, pIdx->offset, SEEK_SET) < 0) return -1; - // adjustMem(pHelper->pCompInfo, pHelper->compInfoSize, pIdx->len); - pHelper->pCompInfo = trealloc((void *)pHelper->pCompInfo, pIdx->len); - if (tread(fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) return -1; - // TODO: check the checksum + pHelper->pCompInfo = trealloc((void *)pHelper->pCompInfo, pIdx->len); + if (tread(fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) return -1; + if (!taosCheckChecksumWhole((uint8_t *)pHelper->pCompInfo, pIdx->len)) return -1; + } helperSetState(pHelper, TSDB_HELPER_INFO_LOAD); } @@ -519,16 +530,21 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd; if (tread(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) goto _err; + ASSERT(pCompData->numOfCols == pCompBlock->numOfCols); - { // TODO : check the correctness of the part + // TODO : check the checksum + size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols + sizeof(TSCKSUM); + if (!taosCheckChecksumWhole((uint8_t *)pCompData, tsize)) goto _err; + for (int i = 0; i < pCompData->numOfCols; i++) { + // TODO: check the data checksum + // if (!taosCheckChecksumWhole()) } ASSERT(pCompBlock->numOfCols == pCompData->numOfCols); pDataCols->numOfPoints = pCompBlock->numOfPoints; - size_t tlen = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols; - int ccol = 0, dcol = 0; + int ccol = 0, dcol = 0; while (true) { if (ccol >= pDataCols->numOfCols) { // TODO: Fill rest NULL @@ -541,7 +557,7 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa if (pCompCol->colId == pDataCol->colId) { // TODO: uncompress - memcpy(pDataCol->pData, (void *)(((char *)pCompData) + tlen + pCompCol->offset), pCompCol->len); + memcpy(pDataCol->pData, (void *)(((char *)pCompData) + tsize + pCompCol->offset), pCompCol->len); ccol++; dcol++; } else if (pCompCol->colId > pDataCol->colId) { @@ -567,8 +583,10 @@ int tsdbLoadBlockData(SRWHelper *pHelper, int blkIdx, SDataCols *target) { int numOfSubBlock = pCompBlock->numOfSubBlocks; if (numOfSubBlock > 1) pCompBlock = (SCompBlock *)((char *)pHelper->pCompInfo + pCompBlock->offset); + tdResetDataCols(pHelper->pDataCols[0]); if (tsdbLoadBlockDataImpl(pHelper, pCompBlock, pHelper->pDataCols[0]) < 0) goto _err; for (int i = 1; i < numOfSubBlock; i++) { + tdResetDataCols(pHelper->pDataCols[1]); pCompBlock++; if (tsdbLoadBlockDataImpl(pHelper, pCompBlock, pHelper->pDataCols[1]) < 0) goto _err; if (tdMergeDataCols(pHelper->pDataCols[0], pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfPoints) < 0) goto _err; @@ -614,8 +632,11 @@ static void tsdbClearHelperFile(SHelperFile *pHFile) { } static bool tsdbShouldCreateNewLast(SRWHelper *pHelper) { - // TODO - return 0; + ASSERT(pHelper->files.lastF.fd > 0); + struct stat st; + fstat(pHelper->files.lastF.fd, &st); + if (st.st_size > 32 * 1024 + TSDB_FILE_HEAD_SIZE) return true; + return false; } static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, int rowsToWrite, SCompBlock *pCompBlock, @@ -629,7 +650,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa offset = lseek(pFile->fd, 0, SEEK_END); if (offset < 0) goto _err; - pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * pDataCols->numOfCols); + pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * pDataCols->numOfCols + sizeof(TSCKSUM)); if (pCompData == NULL) goto _err; int nColsNotAllNull = 0; @@ -639,12 +660,14 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa SCompCol *pCompCol = pCompData->cols + nColsNotAllNull; if (0) { - // TODO: all data are NULL + // TODO: all data to commit are NULL continue; } // Compress the data here - {} + { + // TODO + } pCompCol->colId = pDataCol->colId; pCompCol->type = pDataCol->type; @@ -655,14 +678,17 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa toffset += pCompCol->len; } - ASSERT(nColsNotAllNull > 0); + ASSERT(nColsNotAllNull > 0 && nColsNotAllNull <= pDataCols->numOfCols); pCompData->delimiter = TSDB_FILE_DELIMITER; pCompData->uid = pHelper->tableInfo.uid; pCompData->numOfCols = nColsNotAllNull; - size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * nColsNotAllNull; + // Write SCompData + SCompCol part + size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * nColsNotAllNull + sizeof(TSCKSUM); + taosCalcChecksumAppend(0, (uint8_t *)pCompData, tsize); if (twrite(pFile->fd, (void *)pCompData, tsize) < tsize) goto _err; + // Write true data part int nCompCol = 0; for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) { ASSERT(nCompCol < nColsNotAllNull); @@ -679,7 +705,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa pCompBlock->last = isLast; pCompBlock->offset = offset; - pCompBlock->algorithm = 2; // TODO + pCompBlock->algorithm = 0; // TODO pCompBlock->numOfPoints = rowsToWrite; pCompBlock->sversion = pHelper->tableInfo.sversion; pCompBlock->len = (int32_t)tsize; @@ -713,18 +739,13 @@ static FORCE_INLINE int compKeyFunc(const void *arg1, const void *arg2) { return ((*(TSKEY *)arg1) - (*(TSKEY *)arg2)); } -// static int nRowsLEThan(SDataCols *pDataCols, int maxKey) { -// void *ptr = taosbsearch((void *)&maxKey, pDataCols->cols[0].pData, pDataCols->numOfPoints, sizeof(TSKEY), compKeyFunc, TD_LE); -// if (ptr == NULL) return 0; -// return ((TSKEY *)ptr - (TSKEY *)(pDataCols->cols[0].pData)) + 1; -// } - // Merge the data with a block in file static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols) { // TODO: set pHelper->hasOldBlock int rowsWritten = 0; SCompBlock compBlock = {0}; + ASSERT(pDataCols->numOfPoints > 0); TSKEY keyFirst = dataColsKeyFirst(pDataCols); SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; @@ -733,7 +754,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx; ASSERT(pCompBlock->numOfSubBlocks >= 1); ASSERT(keyFirst >= pCompBlock->keyFirst); - ASSERT(compareKeyBlock((void *)&keyFirst, (void *)pCompBlock) == 0); + // ASSERT(compareKeyBlock((void *)&keyFirst, (void *)pCompBlock) == 0); if (keyFirst > pCompBlock->keyLast) { // Merge the last block by append ASSERT(pCompBlock->last && pCompBlock->numOfPoints < pHelper->config.minRowsPerFileBlock); @@ -870,9 +891,13 @@ static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t spaceNeeded) { ASSERT(spaceLeft >= 0); if (spaceLeft < spaceNeeded) { size_t tsize = tsizeof(pHelper->pCompInfo) + sizeof(SCompBlock) * 16; - if (tsizeof(pHelper->pCompInfo) == 0) tsize += sizeof(SCompInfo); + if (tsizeof(pHelper->pCompInfo) == 0) { + pIdx->len = sizeof(SCompData) + sizeof(TSCKSUM); + tsize = tsize + sizeof(SCompInfo) + sizeof(TSCKSUM); + } pHelper->pCompInfo = (SCompInfo *)trealloc(pHelper->pCompInfo, tsize); + if (pHelper->pCompInfo == NULL) return -1; } return 0; @@ -881,20 +906,23 @@ static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t spaceNeeded) { static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx) { SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; - ASSERT(blkIdx >=0 && blkIdx <= pIdx->numOfSuperBlocks); + ASSERT(blkIdx >= 0 && blkIdx <= pIdx->numOfSuperBlocks); ASSERT(pCompBlock->numOfSubBlocks == 1); // Adjust memory if no more room - if (tsdbAdjustInfoSizeIfNeeded(pHelper, sizeof(SCompBlock)) < 0) goto _err; + if (tsdbAdjustInfoSizeIfNeeded(pHelper, sizeof(SCompBlock)) < 0) goto _err; - // Insert the block - if (blkIdx < pIdx->numOfSuperBlocks) { - SCompBlock *pTCompBlock = pHelper->pCompInfo->blocks + blkIdx; - memmove((void *)(pTCompBlock + 1), (void *)pTCompBlock, pIdx->len - sizeof(SCompInfo) - sizeof(SCompBlock) *blkIdx); - pTCompBlock++; - for (int i = 0; i < pIdx->numOfSuperBlocks - blkIdx; i++) { - if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += sizeof(SCompBlock); - } + // Change the offset + for (int i = 0; i < pIdx->numOfSuperBlocks; i++) { + SCompBlock *pTCompBlock = &pHelper->pCompInfo->blocks[i]; + if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += sizeof(SCompBlock); + } + + // Memmove if needed + int tsize = pIdx->len - (sizeof(SCompData) + sizeof(SCompCol) * blkIdx); + if (tsize > 0) { + memmove((void *)((char *)pHelper->pCompInfo + sizeof(SCompData) + sizeof(SCompBlock) * (blkIdx + 1)), + (void *)((char *)pHelper->pCompInfo + sizeof(SCompData) + sizeof(SCompBlock) * blkIdx), tsize); } pHelper->pCompInfo->blocks[blkIdx] = *pCompBlock; @@ -905,7 +933,7 @@ static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int return 0; - _err: +_err: return -1; } @@ -913,6 +941,8 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId ASSERT(pCompBlock->numOfSubBlocks == 0); SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + ASSERT(blkIdx >= 0 && blkIdx < pIdx->numOfSuperBlocks); + SCompBlock *pSCompBlock = pHelper->pCompInfo->blocks + blkIdx; ASSERT(pSCompBlock->numOfSubBlocks >= 1 && pSCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS); @@ -926,7 +956,7 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId memmove((void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len + sizeof(SCompBlock)), (void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len), tsize); - for (int i = blkIdx; i < pIdx->numOfSuperBlocks; i++) { + for (int i = blkIdx + 1; i < pIdx->numOfSuperBlocks; i++) { SCompBlock *pTCompBlock = &pHelper->pCompInfo->blocks[i]; if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += sizeof(SCompBlock); } @@ -936,11 +966,15 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId *(SCompBlock *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len) = *pCompBlock; pSCompBlock->numOfSubBlocks++; + ASSERT(pSCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS); pSCompBlock->len += sizeof(SCompBlock); + pSCompBlock->numOfPoints += rowsAdded; + pSCompBlock->keyFirst = MIN(pSCompBlock->keyFirst, pCompBlock->keyFirst); + pSCompBlock->keyLast = MAX(pSCompBlock->keyLast, pCompBlock->keyLast); pIdx->len += sizeof(SCompBlock); } else { // Need to create two sub-blocks void *ptr = NULL; - for (int i = blkIdx - 1; i >= 0; i--) { + for (int i = blkIdx + 1; i < pIdx->numOfSuperBlocks; i++) { SCompBlock *pTCompBlock = pHelper->pCompInfo->blocks + i; if (pTCompBlock->numOfSubBlocks > 1) { ptr = (void *)((char *)(pHelper->pCompInfo) + pTCompBlock->offset + pTCompBlock->len); @@ -948,8 +982,7 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId } } - if (ptr == NULL) - ptr = (void *)((char *)(pHelper->pCompInfo) + sizeof(SCompInfo) + sizeof(SCompBlock) * pIdx->numOfSuperBlocks); + if (ptr == NULL) ptr = (void *)((char *)(pHelper->pCompInfo) + pIdx->len - sizeof(TSCKSUM)); size_t tsize = pIdx->len - ((char *)ptr - (char *)(pHelper->pCompInfo)); if (tsize > 0) { @@ -1040,5 +1073,5 @@ static int tsdbGetRowsInRange(SDataCols *pDataCols, int minKey, int maxKey) { if ((TSKEY *)ptr2 - (TSKEY *)ptr1 < 0) return 0; - return (TSKEY *)ptr2 - (TSKEY *)ptr1; + return ((TSKEY *)ptr2 - (TSKEY *)ptr1) + 1; } \ No newline at end of file diff --git a/src/tsdb/tests/tsdbTests.cpp b/src/tsdb/tests/tsdbTests.cpp index 28d511ae2b..b54db6febb 100644 --- a/src/tsdb/tests/tsdbTests.cpp +++ b/src/tsdb/tests/tsdbTests.cpp @@ -48,98 +48,99 @@ TEST(TsdbTest, DISABLED_tableEncodeDecode) { ASSERT_EQ(memcmp(pTable->schema, tTable->schema, sizeof(STSchema) + sizeof(STColumn) * nCols), 0); } -TEST(TsdbTest, DISABLED_createRepo) { -// TEST(TsdbTest, createRepo) { - // STsdbCfg config; +// TEST(TsdbTest, DISABLED_createRepo) { +TEST(TsdbTest, createRepo) { + STsdbCfg config; - // // 1. Create a tsdb repository - // tsdbSetDefaultCfg(&config); - // tsdb_repo_t *pRepo = tsdbCreateRepo("/home/ubuntu/work/ttest/vnode0", &config, NULL); - // ASSERT_NE(pRepo, nullptr); + // 1. Create a tsdb repository + tsdbSetDefaultCfg(&config); + ASSERT_EQ(tsdbCreateRepo("/home/ubuntu/work/ttest/vnode0", &config, NULL), 0); - // // 2. Create a normal table - // STableCfg tCfg; - // ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_SUPER_TABLE, 987607499877672L, 0), -1); - // ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_NORMAL_TABLE, 987607499877672L, 0), 0); + tsdb_repo_t *pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL); + ASSERT_NE(pRepo, nullptr); - // int nCols = 5; - // STSchema *schema = tdNewSchema(nCols); + // 2. Create a normal table + STableCfg tCfg; + ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_SUPER_TABLE, 987607499877672L, 0), -1); + ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_NORMAL_TABLE, 987607499877672L, 0), 0); - // for (int i = 0; i < nCols; i++) { - // if (i == 0) { - // tdSchemaAppendCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1); - // } else { - // tdSchemaAppendCol(schema, TSDB_DATA_TYPE_INT, i, -1); - // } - // } + int nCols = 5; + STSchema *schema = tdNewSchema(nCols); - // tsdbTableSetSchema(&tCfg, schema, true); + for (int i = 0; i < nCols; i++) { + if (i == 0) { + tdSchemaAppendCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1); + } else { + tdSchemaAppendCol(schema, TSDB_DATA_TYPE_INT, i, -1); + } + } - // tsdbCreateTable(pRepo, &tCfg); + tsdbTableSetSchema(&tCfg, schema, true); - // // // 3. Loop to write some simple data - // int nRows = 1; - // int rowsPerSubmit = 1; - // int64_t start_time = 1584081000000; + tsdbCreateTable(pRepo, &tCfg); - // SSubmitMsg *pMsg = (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + tdMaxRowBytesFromSchema(schema) * rowsPerSubmit); + // // 3. Loop to write some simple data + int nRows = 10000000; + int rowsPerSubmit = 10; + int64_t start_time = 1584081000000; - // double stime = getCurTime(); + SSubmitMsg *pMsg = (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + tdMaxRowBytesFromSchema(schema) * rowsPerSubmit); - // for (int k = 0; k < nRows/rowsPerSubmit; k++) { - // memset((void *)pMsg, 0, sizeof(SSubmitMsg)); - // SSubmitBlk *pBlock = pMsg->blocks; - // pBlock->uid = 987607499877672L; - // pBlock->tid = 0; - // pBlock->sversion = 0; - // pBlock->len = 0; - // for (int i = 0; i < rowsPerSubmit; i++) { - // // start_time += 1000; - // start_time += 1000; - // SDataRow row = (SDataRow)(pBlock->data + pBlock->len); - // tdInitDataRow(row, schema); + double stime = getCurTime(); - // for (int j = 0; j < schemaNCols(schema); j++) { - // if (j == 0) { // Just for timestamp - // tdAppendColVal(row, (void *)(&start_time), schemaColAt(schema, j)); - // } else { // For int - // int val = 10; - // tdAppendColVal(row, (void *)(&val), schemaColAt(schema, j)); - // } - // } - // pBlock->len += dataRowLen(row); - // } - // pMsg->length = pMsg->length + sizeof(SSubmitBlk) + pBlock->len; - // pMsg->numOfBlocks = 1; + for (int k = 0; k < nRows/rowsPerSubmit; k++) { + memset((void *)pMsg, 0, sizeof(SSubmitMsg)); + SSubmitBlk *pBlock = pMsg->blocks; + pBlock->uid = 987607499877672L; + pBlock->tid = 0; + pBlock->sversion = 0; + pBlock->len = 0; + for (int i = 0; i < rowsPerSubmit; i++) { + // start_time += 1000; + start_time += 1000; + SDataRow row = (SDataRow)(pBlock->data + pBlock->len); + tdInitDataRow(row, schema); - // pBlock->len = htonl(pBlock->len); - // pBlock->numOfRows = htonl(pBlock->numOfRows); - // pBlock->uid = htobe64(pBlock->uid); - // pBlock->tid = htonl(pBlock->tid); + for (int j = 0; j < schemaNCols(schema); j++) { + if (j == 0) { // Just for timestamp + tdAppendColVal(row, (void *)(&start_time), schemaColAt(schema, j)); + } else { // For int + int val = 10; + tdAppendColVal(row, (void *)(&val), schemaColAt(schema, j)); + } + } + pBlock->len += dataRowLen(row); + } + pMsg->length = pMsg->length + sizeof(SSubmitBlk) + pBlock->len; + pMsg->numOfBlocks = 1; - // pBlock->sversion = htonl(pBlock->sversion); - // pBlock->padding = htonl(pBlock->padding); + pBlock->len = htonl(pBlock->len); + pBlock->numOfRows = htonl(pBlock->numOfRows); + pBlock->uid = htobe64(pBlock->uid); + pBlock->tid = htonl(pBlock->tid); - // pMsg->length = htonl(pMsg->length); - // pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); - // pMsg->compressed = htonl(pMsg->numOfBlocks); + pBlock->sversion = htonl(pBlock->sversion); + pBlock->padding = htonl(pBlock->padding); - // tsdbInsertData(pRepo, pMsg); - // } + pMsg->length = htonl(pMsg->length); + pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); + pMsg->compressed = htonl(pMsg->numOfBlocks); - // double etime = getCurTime(); + tsdbInsertData(pRepo, pMsg); + } - // void *ptr = malloc(150000); - // free(ptr); + double etime = getCurTime(); - // printf("Spent %f seconds to write %d records\n", etime - stime, nRows); + void *ptr = malloc(150000); + free(ptr); - // tsdbCloseRepo(pRepo); + printf("Spent %f seconds to write %d records\n", etime - stime, nRows); + tsdbCloseRepo(pRepo); } -// TEST(TsdbTest, DISABLED_openRepo) { -TEST(TsdbTest, openRepo) { +TEST(TsdbTest, DISABLED_openRepo) { +// TEST(TsdbTest, openRepo) { // tsdb_repo_t *repo = tsdbOpenRepo("/home/ubuntu/work/build/test/data/vnode/vnode2/tsdb", NULL); // ASSERT_NE(repo, nullptr); From 7c5633cf1ad2e4357c55aeda09f14ac1bd6a7906 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Thu, 16 Apr 2020 17:18:23 +0800 Subject: [PATCH 18/29] TD-100 --- src/tsdb/src/tsdbMain.c | 2 ++ src/tsdb/src/tsdbRWHelper.c | 15 +++++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 55f80a8d22..88a404135a 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -924,10 +924,12 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters // Loop to write the data in the cache to files. If no data to write, just break the loop int maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5; + int nLoop = 0; while (true) { int rowsRead = tsdbReadRowsFromCache(pIter, maxKey, maxRowsToRead, pDataCols); ASSERT(rowsRead >= 0); if (pDataCols->numOfPoints == 0) break; + nLoop++; ASSERT(dataColsKeyFirst(pDataCols) >= minKey && dataColsKeyFirst(pDataCols) <= maxKey); ASSERT(dataColsKeyLast(pDataCols) >= minKey && dataColsKeyLast(pDataCols) <= maxKey); diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index e24a3c5786..2656334567 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -281,9 +281,7 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { // Load the SCompInfo part if neccessary ASSERT(helperHasState(pHelper, TSDB_HELPER_TABLE_SET)); - if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD) && (pIdx->offset > 0)) { - if (tsdbLoadCompInfo(pHelper, NULL) < 0) goto _err; - } + if (tsdbLoadCompInfo(pHelper, NULL) < 0) goto _err; if (pIdx->offset == 0 || (!pIdx->hasLast && keyFirst > pIdx->maxKey)) { // Just append as a super block ASSERT(pHelper->hasOldLastBlock == false); @@ -370,6 +368,7 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { if (pIdx->offset > 0) { pIdx->offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END); + ASSERT(pIdx->offset > TSDB_FILE_HEAD_SIZE); if (pIdx->offset < 0) return -1; if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, pIdx->len) < pIdx->len) return -1; @@ -377,6 +376,7 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { } else { taosCalcChecksumAppend(0, (uint8_t *)pHelper->pCompInfo, pIdx->len); pIdx->offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END); + ASSERT(pIdx->offset > TSDB_FILE_HEAD_SIZE); if (pIdx->offset < 0) return -1; if (twrite(pHelper->files.nHeadF.fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) return -1; @@ -919,15 +919,18 @@ static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int } // Memmove if needed - int tsize = pIdx->len - (sizeof(SCompData) + sizeof(SCompCol) * blkIdx); + int tsize = pIdx->len - (sizeof(SCompInfo) + sizeof(SCompBlock) * blkIdx); if (tsize > 0) { - memmove((void *)((char *)pHelper->pCompInfo + sizeof(SCompData) + sizeof(SCompBlock) * (blkIdx + 1)), - (void *)((char *)pHelper->pCompInfo + sizeof(SCompData) + sizeof(SCompBlock) * blkIdx), tsize); + ASSERT(sizeof(SCompInfo) + sizeof(SCompBlock) * (blkIdx + 1) < tsizeof(pHelper->pCompInfo)); + ASSERT(sizeof(SCompInfo) + sizeof(SCompBlock) * (blkIdx + 1) + tsize <= tsizeof(pHelper->pCompInfo)); + memmove((void *)((char *)pHelper->pCompInfo + sizeof(SCompInfo) + sizeof(SCompBlock) * (blkIdx + 1)), + (void *)((char *)pHelper->pCompInfo + sizeof(SCompInfo) + sizeof(SCompBlock) * blkIdx), tsize); } pHelper->pCompInfo->blocks[blkIdx] = *pCompBlock; pIdx->numOfSuperBlocks++; pIdx->len += sizeof(SCompBlock); + ASSERT(pIdx->len <= tsizeof(pHelper->pCompInfo)); pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].keyLast; pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].last; From 01ffea909839c457585e6cd3bda6ff93fc314142 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Thu, 16 Apr 2020 18:08:53 +0800 Subject: [PATCH 19/29] TD-100 --- src/tsdb/src/tsdbRWHelper.c | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 2656334567..8b0025432b 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -884,18 +884,11 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa static int compTSKEY(const void *key1, const void *key2) { return ((TSKEY *)key1 - (TSKEY *)key2); } -static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t spaceNeeded) { +static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize) { SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; - size_t spaceLeft = tsizeof((void *)pHelper->pCompInfo) - pIdx->len; - ASSERT(spaceLeft >= 0); - if (spaceLeft < spaceNeeded) { - size_t tsize = tsizeof(pHelper->pCompInfo) + sizeof(SCompBlock) * 16; - if (tsizeof(pHelper->pCompInfo) == 0) { - pIdx->len = sizeof(SCompData) + sizeof(TSCKSUM); - tsize = tsize + sizeof(SCompInfo) + sizeof(TSCKSUM); - } - + if (tsizeof((void *)pHelper->pCompInfo) <= esize) { + size_t tsize = esize + sizeof(SCompBlock) * 16; pHelper->pCompInfo = (SCompInfo *)trealloc(pHelper->pCompInfo, tsize); if (pHelper->pCompInfo == NULL) return -1; } @@ -910,7 +903,8 @@ static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int ASSERT(pCompBlock->numOfSubBlocks == 1); // Adjust memory if no more room - if (tsdbAdjustInfoSizeIfNeeded(pHelper, sizeof(SCompBlock)) < 0) goto _err; + if (pIdx->len == 0) pIdx->len = sizeof(SCompData) + sizeof(TSCKSUM); + if (tsdbAdjustInfoSizeIfNeeded(pHelper, pIdx->len + sizeof(SCompInfo)) < 0) goto _err; // Change the offset for (int i = 0; i < pIdx->numOfSuperBlocks; i++) { @@ -949,7 +943,8 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId SCompBlock *pSCompBlock = pHelper->pCompInfo->blocks + blkIdx; ASSERT(pSCompBlock->numOfSubBlocks >= 1 && pSCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS); - size_t spaceNeeded = (pSCompBlock->numOfSubBlocks == 1) ? sizeof(SCompBlock) * 2 : sizeof(SCompBlock); + size_t spaceNeeded = + (pSCompBlock->numOfSubBlocks == 1) ? pIdx->len + sizeof(SCompBlock) * 2 : pIdx->len + sizeof(SCompBlock); if (tsdbAdjustInfoSizeIfNeeded(pHelper, spaceNeeded) < 0) goto _err; // Add the sub-block From 8518a5b2565ef9c835b7b436170b68ec76be0f03 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Thu, 16 Apr 2020 18:27:41 +0800 Subject: [PATCH 20/29] TD-100 --- src/tsdb/src/tsdbRWHelper.c | 2 +- src/util/tests/taosbsearchTest.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 8b0025432b..02216cc69b 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -15,6 +15,7 @@ #include "tsdbMain.h" #include "tchecksum.h" #include "tscompression.h" +#include "talgo.h" // Local function definitions static int tsdbCheckHelperCfg(SHelperCfg *pCfg); @@ -885,7 +886,6 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa static int compTSKEY(const void *key1, const void *key2) { return ((TSKEY *)key1 - (TSKEY *)key2); } static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize) { - SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; if (tsizeof((void *)pHelper->pCompInfo) <= esize) { size_t tsize = esize + sizeof(SCompBlock) * 16; diff --git a/src/util/tests/taosbsearchTest.cpp b/src/util/tests/taosbsearchTest.cpp index fb01d34c03..0b250c9ecc 100644 --- a/src/util/tests/taosbsearchTest.cpp +++ b/src/util/tests/taosbsearchTest.cpp @@ -1,6 +1,6 @@ #include -#include "tutil.h" +#include "talgo.h" static int compareFunc(const void *arg1, const void *arg2) { return (*(int *)arg1) - (*(int *)arg2); } From dfa8e5be9b8428774386bad4025966162db65479 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Fri, 17 Apr 2020 14:09:52 +0800 Subject: [PATCH 21/29] TD-100 --- src/inc/tsdb.h | 1 + src/tsdb/src/tsdbMain.c | 31 +++++++++++++++++++------------ src/vnode/src/vnodeMain.c | 1 + 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index f2d0b6b0c8..7bb6365865 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -46,6 +46,7 @@ typedef struct { // --------- TSDB REPOSITORY CONFIGURATION DEFINITION typedef struct { int8_t precision; + int8_t compression; int32_t tsdbId; int32_t maxTables; // maximum number of tables this repository can have int32_t daysPerFile; // day per file sharding policy diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 263c09f16c..fd01e8ce71 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -12,15 +12,16 @@ #include #include -// #include "taosdef.h" -// #include "disk.h" #include "os.h" #include "talgo.h" #include "tsdb.h" #include "tsdbMain.h" +#include "tscompression.h" #define TSDB_DEFAULT_PRECISION TSDB_PRECISION_MILLI // default precision #define IS_VALID_PRECISION(precision) (((precision) >= TSDB_PRECISION_MILLI) && ((precision) <= TSDB_PRECISION_NANO)) +#define TSDB_DEFAULT_COMPRESSION TWO_STAGE_COMP +#define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP)) #define TSDB_MIN_ID 0 #define TSDB_MAX_ID INT_MAX #define TSDB_MIN_TABLES 10 @@ -83,6 +84,7 @@ void tsdbSetDefaultCfg(STsdbCfg *pCfg) { pCfg->maxRowsPerFileBlock = -1; pCfg->keep = -1; pCfg->maxCacheSize = -1; + pCfg->compression = TWO_STAGE_COMP; } /** @@ -547,6 +549,13 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { if (!IS_VALID_PRECISION(pCfg->precision)) return -1; } + // Check compression + if (pCfg->compression == -1) { + pCfg->compression = TSDB_DEFAULT_COMPRESSION; + } else { + if (!IS_VALID_COMPRESSION(pCfg->compression)) return -1; + } + // Check tsdbId if (pCfg->tsdbId < 0) return -1; @@ -841,16 +850,14 @@ static void *tsdbCommitData(void *arg) { } // Create a write helper for commit data - SHelperCfg hcfg = { - .type = TSDB_WRITE_HELPER, - .maxTables = pCfg->maxTables, - .maxRowSize = pMeta->maxRowBytes, - .maxRows = pCfg->maxRowsPerFileBlock, - .maxCols = pMeta->maxCols, - .minRowsPerFileBlock = pCfg->minRowsPerFileBlock, - .maxRowsPerFileBlock = pCfg->maxRowsPerFileBlock, - .compress = 2 // TODO make it a configuration - }; + SHelperCfg hcfg = {.type = TSDB_WRITE_HELPER, + .maxTables = pCfg->maxTables, + .maxRowSize = pMeta->maxRowBytes, + .maxRows = pCfg->maxRowsPerFileBlock, + .maxCols = pMeta->maxCols, + .minRowsPerFileBlock = pCfg->minRowsPerFileBlock, + .maxRowsPerFileBlock = pCfg->maxRowsPerFileBlock, + .compress = pCfg->compression}; if (tsdbInitHelper(&whelper, &hcfg) < 0) goto _exit; if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) goto _exit; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 7ec9b0fef7..33c979aa2e 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -88,6 +88,7 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { STsdbCfg tsdbCfg = {0}; tsdbCfg.precision = pVnodeCfg->cfg.precision; + tsdbCfg.compression = -1; tsdbCfg.tsdbId = pVnodeCfg->cfg.vgId; tsdbCfg.maxTables = pVnodeCfg->cfg.maxSessions; tsdbCfg.daysPerFile = pVnodeCfg->cfg.daysPerFile; From 5380a36aaaa32ff6f9e96090e17d68c9da2f3d8d Mon Sep 17 00:00:00 2001 From: hzcheng Date: Fri, 17 Apr 2020 17:44:56 +0800 Subject: [PATCH 22/29] TD-100 --- src/common/src/dataformat.c | 50 +++++---- src/tsdb/src/tsdbRWHelper.c | 7 +- src/tsdb/tests/tsdbTests.cpp | 194 +++++++++++++++++++++++++---------- 3 files changed, 175 insertions(+), 76 deletions(-) diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c index 3db3f1d98a..b5451ae691 100644 --- a/src/common/src/dataformat.c +++ b/src/common/src/dataformat.c @@ -353,8 +353,9 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { pRet->cols[i].bytes = pDataCols->cols[i].bytes; pRet->cols[i].len = pDataCols->cols[i].len; pRet->cols[i].offset = pDataCols->cols[i].offset; + pRet->cols[i].pData = (void *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].pData) - (char *)(pDataCols->buf))); - if (keepData) memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pRet->cols[i].len); + if (keepData) memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pRet->cols[i].bytes * pDataCols->numOfPoints); } return pRet; @@ -410,36 +411,47 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) { SDataCols *pTarget = tdDupDataCols(target, true); if (pTarget == NULL) goto _err; + tdResetDataCols(target); int iter1 = 0; int iter2 = 0; while (true) { - if (iter1 >= pTarget->numOfPoints) { - // TODO: merge the source part - int rowsLeft = source->numOfPoints - iter2; - if (rowsLeft > 0) { - for (int i = 0; i < source->numOfCols; i++) { - ASSERT(target->cols[i].type == source->cols[i].type); + if (iter1 >= pTarget->numOfPoints && iter2 >= source->numOfPoints) break; - memcpy((void *)((char *)(target->cols[i].pData) + TYPE_BYTES[target->cols[i].type] * target->numOfPoints), - (void *)((char *)(source->cols[i].pData) + TYPE_BYTES[source->cols[i].type] * iter2), - TYPE_BYTES[target->cols[i].type] * rowsLeft); - } - } - break; - } + TSKEY key1 = (iter1 >= pTarget->numOfPoints) ? INT64_MAX : ((TSKEY *)(pTarget->cols[0].pData))[iter1]; + TSKEY key2 = (iter2 >= rowsToMerge) ? INT64_MAX : ((TSKEY *)(source->cols[0].pData))[iter2]; - if (iter2 >= source->numOfPoints) { - // TODO: merge the pTemp part - int rowsLeft = pTarget->numOfPoints - iter1; - if (rowsLeft > 0) { + if (key1 < key2) { // Copy from pTarget + for (int i = 0; i < pTarget->numOfCols; i++) { + ASSERT(target->cols[i].type == pTarget->cols[i].type); + memcpy((void *)((char *)(target->cols[i].pData) + TYPE_BYTES[target->cols[i].type] * target->numOfPoints), + (void *)((char *)(pTarget->cols[i].pData) + TYPE_BYTES[pTarget->cols[i].type] * iter1), + TYPE_BYTES[target->cols[i].type]); + target->cols[i].len += TYPE_BYTES[target->cols[i].type]; } - break; + + target->numOfPoints++; + iter1++; + } else if (key1 > key2) { // Copy from source + for (int i = 0; i < source->numOfCols; i++) { + ASSERT(target->cols[i].type == pTarget->cols[i].type); + memcpy((void *)((char *)(target->cols[i].pData) + TYPE_BYTES[target->cols[i].type] * target->numOfPoints), + (void *)((char *)(source->cols[i].pData) + TYPE_BYTES[source->cols[i].type] * iter2), + TYPE_BYTES[target->cols[i].type]); + target->cols[i].len += TYPE_BYTES[target->cols[i].type]; + } + + target->numOfPoints++; + iter2++; + } else { + assert(false); } } + tdFreeDataCols(pTarget); return 0; _err: + tdFreeDataCols(pTarget); return -1; } \ No newline at end of file diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 02216cc69b..ac96cff366 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -335,7 +335,7 @@ _err: int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; SCompBlock compBlock; - if ((pHelper->files.nHeadF.fd > 0) && (pHelper->hasOldLastBlock)) { + if ((pHelper->files.nLastF.fd > 0) && (pHelper->hasOldLastBlock)) { if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1; SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + pIdx->numOfSuperBlocks - 1; @@ -375,6 +375,8 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, pIdx->len) < pIdx->len) return -1; } } else { + pHelper->pCompInfo->delimiter = TSDB_FILE_DELIMITER; + pHelper->pCompInfo->uid = pHelper->tableInfo.uid; taosCalcChecksumAppend(0, (uint8_t *)pHelper->pCompInfo, pIdx->len); pIdx->offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END); ASSERT(pIdx->offset > TSDB_FILE_HEAD_SIZE); @@ -530,6 +532,7 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa if (pCompData == NULL) return -1; int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd; + if (lseek(fd, pCompBlock->offset, SEEK_SET) < 0) goto _err; if (tread(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) goto _err; ASSERT(pCompData->numOfCols == pCompBlock->numOfCols); @@ -947,6 +950,8 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId (pSCompBlock->numOfSubBlocks == 1) ? pIdx->len + sizeof(SCompBlock) * 2 : pIdx->len + sizeof(SCompBlock); if (tsdbAdjustInfoSizeIfNeeded(pHelper, spaceNeeded) < 0) goto _err; + pSCompBlock = pHelper->pCompInfo->blocks + blkIdx; + // Add the sub-block if (pSCompBlock->numOfSubBlocks > 1) { size_t tsize = pIdx->len - (pSCompBlock->offset + pSCompBlock->len); diff --git a/src/tsdb/tests/tsdbTests.cpp b/src/tsdb/tests/tsdbTests.cpp index b54db6febb..9d1bae1832 100644 --- a/src/tsdb/tests/tsdbTests.cpp +++ b/src/tsdb/tests/tsdbTests.cpp @@ -5,12 +5,84 @@ #include "dataformat.h" #include "tsdbMain.h" -double getCurTime() { +static double getCurTime() { struct timeval tv; gettimeofday(&tv, NULL); return tv.tv_sec + tv.tv_usec * 1E-6; } +typedef struct { + tsdb_repo_t *pRepo; + int tid; + int64_t uid; + int sversion; + TSKEY startTime; + TSKEY interval; + int totalRows; + int rowsPerSubmit; + STSchema * pSchema; +} SInsertInfo; + +static int insertData(SInsertInfo *pInfo) { + SSubmitMsg *pMsg = + (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + tdMaxRowBytesFromSchema(pInfo->pSchema) * pInfo->rowsPerSubmit); + if (pMsg == NULL) return -1; + TSKEY start_time = pInfo->startTime; + + // Loop to write data + double stime = getCurTime(); + + for (int k = 0; k < pInfo->totalRows/pInfo->rowsPerSubmit; k++) { + memset((void *)pMsg, 0, sizeof(SSubmitMsg)); + SSubmitBlk *pBlock = pMsg->blocks; + pBlock->uid = pInfo->uid; + pBlock->tid = pInfo->tid; + pBlock->sversion = pInfo->sversion; + pBlock->len = 0; + for (int i = 0; i < pInfo->rowsPerSubmit; i++) { + // start_time += 1000; + start_time += pInfo->interval; + SDataRow row = (SDataRow)(pBlock->data + pBlock->len); + tdInitDataRow(row, pInfo->pSchema); + + for (int j = 0; j < schemaNCols(pInfo->pSchema); j++) { + if (j == 0) { // Just for timestamp + tdAppendColVal(row, (void *)(&start_time), schemaColAt(pInfo->pSchema, j)); + } else { // For int + int val = 10; + tdAppendColVal(row, (void *)(&val), schemaColAt(pInfo->pSchema, j)); + } + } + pBlock->len += dataRowLen(row); + } + pMsg->length = pMsg->length + sizeof(SSubmitBlk) + pBlock->len; + pMsg->numOfBlocks = 1; + + pBlock->len = htonl(pBlock->len); + pBlock->numOfRows = htonl(pBlock->numOfRows); + pBlock->uid = htobe64(pBlock->uid); + pBlock->tid = htonl(pBlock->tid); + + pBlock->sversion = htonl(pBlock->sversion); + pBlock->padding = htonl(pBlock->padding); + + pMsg->length = htonl(pMsg->length); + pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); + pMsg->compressed = htonl(pMsg->numOfBlocks); + + if (tsdbInsertData(pInfo->pRepo, pMsg) < 0) { + tfree(pMsg); + return -1; + } + } + + double etime = getCurTime(); + + printf("Spent %f seconds to write %d records\n", etime - stime, pInfo->totalRows); + tfree(pMsg); + return 0; +} + TEST(TsdbTest, DISABLED_tableEncodeDecode) { // TEST(TsdbTest, tableEncodeDecode) { STable *pTable = (STable *)malloc(sizeof(STable)); @@ -51,6 +123,7 @@ TEST(TsdbTest, DISABLED_tableEncodeDecode) { // TEST(TsdbTest, DISABLED_createRepo) { TEST(TsdbTest, createRepo) { STsdbCfg config; + STsdbRepo *repo; // 1. Create a tsdb repository tsdbSetDefaultCfg(&config); @@ -79,64 +152,73 @@ TEST(TsdbTest, createRepo) { tsdbCreateTable(pRepo, &tCfg); - // // 3. Loop to write some simple data - int nRows = 10000000; - int rowsPerSubmit = 10; - int64_t start_time = 1584081000000; + // Insert Some Data + SInsertInfo iInfo = { + .pRepo = pRepo, + .tid = tCfg.tableId.tid, + .uid = tCfg.tableId.uid, + .sversion = tCfg.sversion, + .startTime = 1584081000000, + .interval = 1000, + .totalRows = 50, + .rowsPerSubmit = 1, + .pSchema = schema + }; - SSubmitMsg *pMsg = (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + tdMaxRowBytesFromSchema(schema) * rowsPerSubmit); - - double stime = getCurTime(); - - for (int k = 0; k < nRows/rowsPerSubmit; k++) { - memset((void *)pMsg, 0, sizeof(SSubmitMsg)); - SSubmitBlk *pBlock = pMsg->blocks; - pBlock->uid = 987607499877672L; - pBlock->tid = 0; - pBlock->sversion = 0; - pBlock->len = 0; - for (int i = 0; i < rowsPerSubmit; i++) { - // start_time += 1000; - start_time += 1000; - SDataRow row = (SDataRow)(pBlock->data + pBlock->len); - tdInitDataRow(row, schema); - - for (int j = 0; j < schemaNCols(schema); j++) { - if (j == 0) { // Just for timestamp - tdAppendColVal(row, (void *)(&start_time), schemaColAt(schema, j)); - } else { // For int - int val = 10; - tdAppendColVal(row, (void *)(&val), schemaColAt(schema, j)); - } - } - pBlock->len += dataRowLen(row); - } - pMsg->length = pMsg->length + sizeof(SSubmitBlk) + pBlock->len; - pMsg->numOfBlocks = 1; - - pBlock->len = htonl(pBlock->len); - pBlock->numOfRows = htonl(pBlock->numOfRows); - pBlock->uid = htobe64(pBlock->uid); - pBlock->tid = htonl(pBlock->tid); - - pBlock->sversion = htonl(pBlock->sversion); - pBlock->padding = htonl(pBlock->padding); - - pMsg->length = htonl(pMsg->length); - pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); - pMsg->compressed = htonl(pMsg->numOfBlocks); - - tsdbInsertData(pRepo, pMsg); - } - - double etime = getCurTime(); - - void *ptr = malloc(150000); - free(ptr); - - printf("Spent %f seconds to write %d records\n", etime - stime, nRows); + ASSERT_EQ(insertData(&iInfo), 0); + // Close the repository tsdbCloseRepo(pRepo); + + // Open the repository again + pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL); + repo = (STsdbRepo *)pRepo; + ASSERT_NE(pRepo, nullptr); + + // Insert more data + iInfo.startTime = iInfo.startTime + iInfo.interval * iInfo.totalRows; + iInfo.totalRows = 10; + iInfo.pRepo = pRepo; + ASSERT_EQ(insertData(&iInfo), 0); + + // Close the repository + tsdbCloseRepo(pRepo); + + // Open the repository again + pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL); + repo = (STsdbRepo *)pRepo; + ASSERT_NE(pRepo, nullptr); + + // Read from file + SRWHelper rhelper; + SHelperCfg helperCfg = { + .type = TSDB_READ_HELPER, + .maxTables = repo->config.maxTables, + .maxRowSize = repo->tsdbMeta->maxRowBytes, + .maxRows = repo->config.maxRowsPerFileBlock, + .maxCols = repo->tsdbMeta->maxCols, + .minRowsPerFileBlock = repo->config.minRowsPerFileBlock, + .maxRowsPerFileBlock = repo->config.maxRowsPerFileBlock, + .compress = repo->config.compression, + + }; + tsdbInitHelper(&rhelper, &helperCfg); + + SFileGroup *pFGroup = tsdbSearchFGroup(repo->tsdbFileH, 1833); + ASSERT_NE(pFGroup, nullptr); + ASSERT_GE(tsdbSetAndOpenHelperFile(&rhelper, pFGroup), 0); + + SHelperTable htable = { + .uid = tCfg.tableId.uid, + .tid = tCfg.tableId.tid, + .sversion = tCfg.sversion + }; + tsdbSetHelperTable(&rhelper, &htable, schema); + + ASSERT_EQ(tsdbLoadCompInfo(&rhelper, NULL), 0); + ASSERT_EQ(tsdbLoadBlockData(&rhelper, 0, NULL), 0); + + int k = 0; } TEST(TsdbTest, DISABLED_openRepo) { From 709f6d094e52a6cf58fbbb20ca786d6b3cece774 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sat, 18 Apr 2020 00:46:33 +0800 Subject: [PATCH 23/29] TD-100 --- src/common/inc/dataformat.h | 1 + src/common/src/dataformat.c | 98 ++++++++++++------ src/tsdb/inc/tsdbMain.h | 1 + src/tsdb/src/tsdbMain.c | 4 +- src/tsdb/src/tsdbRWHelper.c | 198 ++++++++++++++++++++---------------- 5 files changed, 184 insertions(+), 118 deletions(-) diff --git a/src/common/inc/dataformat.h b/src/common/inc/dataformat.h index 783e378eb7..17aa19cce7 100644 --- a/src/common/inc/dataformat.h +++ b/src/common/inc/dataformat.h @@ -137,6 +137,7 @@ void tdFreeDataCols(SDataCols *pCols); void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols); void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop); int tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge); +void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, SDataCols *src2, int *iter2, int tRows); #ifdef __cplusplus } diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c index b5451ae691..15dec2c71c 100644 --- a/src/common/src/dataformat.c +++ b/src/common/src/dataformat.c @@ -406,47 +406,48 @@ static int tdFLenFromSchema(STSchema *pSchema) { } int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) { - // TODO ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfPoints); SDataCols *pTarget = tdDupDataCols(target, true); if (pTarget == NULL) goto _err; - tdResetDataCols(target); + // tdResetDataCols(target); int iter1 = 0; int iter2 = 0; - while (true) { - if (iter1 >= pTarget->numOfPoints && iter2 >= source->numOfPoints) break; + tdMergeTwoDataCols(target,pTarget, &iter1, source, &iter2, pTarget->numOfPoints + rowsToMerge); + // while (true) { + // if (iter1 >= pTarget->numOfPoints && iter2 >= source->numOfPoints) break; - TSKEY key1 = (iter1 >= pTarget->numOfPoints) ? INT64_MAX : ((TSKEY *)(pTarget->cols[0].pData))[iter1]; - TSKEY key2 = (iter2 >= rowsToMerge) ? INT64_MAX : ((TSKEY *)(source->cols[0].pData))[iter2]; + // TSKEY key1 = (iter1 >= pTarget->numOfPoints) ? INT64_MAX : ((TSKEY *)(pTarget->cols[0].pData))[iter1]; + // TSKEY key2 = (iter2 >= rowsToMerge) ? INT64_MAX : ((TSKEY *)(source->cols[0].pData))[iter2]; - if (key1 < key2) { // Copy from pTarget - for (int i = 0; i < pTarget->numOfCols; i++) { - ASSERT(target->cols[i].type == pTarget->cols[i].type); - memcpy((void *)((char *)(target->cols[i].pData) + TYPE_BYTES[target->cols[i].type] * target->numOfPoints), - (void *)((char *)(pTarget->cols[i].pData) + TYPE_BYTES[pTarget->cols[i].type] * iter1), - TYPE_BYTES[target->cols[i].type]); - target->cols[i].len += TYPE_BYTES[target->cols[i].type]; - } + // if (key1 < key2) { // Copy from pTarget + // for (int i = 0; i < pTarget->numOfCols; i++) { + // ASSERT(target->cols[i].type == pTarget->cols[i].type); + // memcpy((void *)((char *)(target->cols[i].pData) + TYPE_BYTES[target->cols[i].type] * target->numOfPoints), + // (void *)((char *)(pTarget->cols[i].pData) + TYPE_BYTES[pTarget->cols[i].type] * iter1), + // TYPE_BYTES[target->cols[i].type]); + // target->cols[i].len += TYPE_BYTES[target->cols[i].type]; + // } - target->numOfPoints++; - iter1++; - } else if (key1 > key2) { // Copy from source - for (int i = 0; i < source->numOfCols; i++) { - ASSERT(target->cols[i].type == pTarget->cols[i].type); - memcpy((void *)((char *)(target->cols[i].pData) + TYPE_BYTES[target->cols[i].type] * target->numOfPoints), - (void *)((char *)(source->cols[i].pData) + TYPE_BYTES[source->cols[i].type] * iter2), - TYPE_BYTES[target->cols[i].type]); - target->cols[i].len += TYPE_BYTES[target->cols[i].type]; - } + // target->numOfPoints++; + // iter1++; + // } else if (key1 > key2) { // Copy from source + // for (int i = 0; i < source->numOfCols; i++) { + // ASSERT(target->cols[i].type == pTarget->cols[i].type); + // memcpy((void *)((char *)(target->cols[i].pData) + TYPE_BYTES[target->cols[i].type] * target->numOfPoints), + // (void *)((char *)(source->cols[i].pData) + TYPE_BYTES[source->cols[i].type] * iter2), + // TYPE_BYTES[target->cols[i].type]); + // target->cols[i].len += TYPE_BYTES[target->cols[i].type]; + // } - target->numOfPoints++; - iter2++; - } else { - assert(false); - } - } + // target->numOfPoints++; + // iter2++; + // } else { + // // TODO + // ASSERT(false); + // } + // } tdFreeDataCols(pTarget); return 0; @@ -454,4 +455,41 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) { _err: tdFreeDataCols(pTarget); return -1; +} + +void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, SDataCols *src2, int *iter2, int tRows) { + tdResetDataCols(target); + + while (target->numOfPoints < tRows) { + if (*iter1 >= src1->numOfPoints && *iter2 >= src2->numOfPoints) break; + + TSKEY key1 = (*iter1 >= src1->numOfPoints) ? INT64_MAX : ((TSKEY *)(src1->cols[0].pData))[*iter1]; + TSKEY key2 = (*iter2 >= src2->numOfPoints) ? INT64_MAX : ((TSKEY *)(src2->cols[0].pData))[*iter2]; + + if (key1 < key2) { + for (int i = 0; i < src1->numOfCols; i++) { + ASSERT(target->cols[i].type == src1->cols[i].type); + memcpy((void *)((char *)(target->cols[i].pData) + TYPE_BYTES[target->cols[i].type] * target->numOfPoints), + (void *)((char *)(src1->cols[i].pData) + TYPE_BYTES[target->cols[i].type] * (*iter1)), + TYPE_BYTES[target->cols[i].type]); + target->cols[i].len += TYPE_BYTES[target->cols[i].type]; + } + + target->numOfPoints++; + *iter1++; + } else if (key1 > key2) { + for (int i = 0; i < src2->numOfCols; i++) { + ASSERT(target->cols[i].type == src2->cols[i].type); + memcpy((void *)((char *)(target->cols[i].pData) + TYPE_BYTES[target->cols[i].type] * target->numOfPoints), + (void *)((char *)(src2->cols[i].pData) + TYPE_BYTES[src2->cols[i].type] * (*iter2)), + TYPE_BYTES[target->cols[i].type]); + target->cols[i].len += TYPE_BYTES[target->cols[i].type]; + } + + target->numOfPoints++; + *iter2++; + } else { + ASSERT(false); + } + } } \ No newline at end of file diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 06313f0786..8af2026977 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -440,6 +440,7 @@ typedef struct { #define helperSetState(h, s) (((h)->state) |= (s)) #define helperClearState(h, s) ((h)->state &= (~(s))) #define helperHasState(h, s) ((((h)->state) & (s)) == (s)) +#define blockAtIdx(h, idx) ((h)->pCompInfo->blocks + idx) int tsdbInitHelper(SRWHelper *pHelper, SHelperCfg *pCfg); void tsdbDestroyHelper(SRWHelper *pHelper); diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index fd01e8ce71..75a0c4d2a8 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -832,7 +832,6 @@ static void tsdbFreeMemTable(SMemTable *pMemTable) { // Commit to file static void *tsdbCommitData(void *arg) { - // TODO printf("Starting to commit....\n"); STsdbRepo * pRepo = (STsdbRepo *)arg; STsdbMeta * pMeta = pRepo->tsdbMeta; @@ -849,7 +848,7 @@ static void *tsdbCommitData(void *arg) { return NULL; } - // Create a write helper for commit data + // Create a write helper to commit data SHelperCfg hcfg = {.type = TSDB_WRITE_HELPER, .maxTables = pCfg->maxTables, .maxRowSize = pMeta->maxRowBytes, @@ -883,7 +882,6 @@ _exit: free(pCache->imem); pCache->imem = NULL; pRepo->commit = 0; - // TODO: free the skiplist for (int i = 0; i < pCfg->maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable && pTable->imem) { diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index ac96cff366..a5b93a5688 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -333,6 +333,7 @@ _err: } int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { + ASSERT(TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER); SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; SCompBlock compBlock; if ((pHelper->files.nLastF.fd > 0) && (pHelper->hasOldLastBlock)) { @@ -343,6 +344,8 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { if (pCompBlock->numOfSubBlocks > 1) { if (tsdbLoadBlockData(pHelper, pIdx->numOfSuperBlocks - 1, NULL) < 0) return -1; + ASSERT(pHelper->pDataCols[0]->numOfPoints > 0 && + pHelper->pDataCols[0]->numOfPoints < pHelper->config.minRowsPerFileBlock); if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.nLastF), pHelper->pDataCols[0], pHelper->pDataCols[0]->numOfPoints, &compBlock, true, true) < 0) return -1; @@ -369,18 +372,19 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { if (pIdx->offset > 0) { pIdx->offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END); - ASSERT(pIdx->offset > TSDB_FILE_HEAD_SIZE); if (pIdx->offset < 0) return -1; + ASSERT(pIdx->offset >= tsizeof(pHelper->pCompIdx)); if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, pIdx->len) < pIdx->len) return -1; } } else { pHelper->pCompInfo->delimiter = TSDB_FILE_DELIMITER; pHelper->pCompInfo->uid = pHelper->tableInfo.uid; + ASSERT((pIdx->len - sizeof(SCompInfo) - sizeof(TSCKSUM)) % sizeof(SCompBlock) == 0); taosCalcChecksumAppend(0, (uint8_t *)pHelper->pCompInfo, pIdx->len); pIdx->offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END); - ASSERT(pIdx->offset > TSDB_FILE_HEAD_SIZE); if (pIdx->offset < 0) return -1; + ASSERT(pIdx->offset >= tsizeof(pHelper->pCompIdx)); if (twrite(pHelper->files.nHeadF.fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) return -1; } @@ -389,8 +393,10 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { } int tsdbWriteCompIdx(SRWHelper *pHelper) { + ASSERT(TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER); if (lseek(pHelper->files.nHeadF.fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1; + ASSERT(tsizeof(pHelper->pCompIdx) == sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM)); taosCalcChecksumAppend(0, (uint8_t *)pHelper->pCompIdx, tsizeof(pHelper->pCompIdx)); if (twrite(pHelper->files.nHeadF.fd, (void *)pHelper->pCompIdx, tsizeof(pHelper->pCompIdx)) < tsizeof(pHelper->pCompIdx)) @@ -755,62 +761,68 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; ASSERT(blkIdx < pIdx->numOfSuperBlocks); - SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx; - ASSERT(pCompBlock->numOfSubBlocks >= 1); - ASSERT(keyFirst >= pCompBlock->keyFirst); + // SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx; + ASSERT(blockAtIdx(pHelper, blkIdx)->numOfSubBlocks >= 1); + ASSERT(keyFirst >= blockAtIdx(pHelper, blkIdx)->keyFirst); // ASSERT(compareKeyBlock((void *)&keyFirst, (void *)pCompBlock) == 0); - if (keyFirst > pCompBlock->keyLast) { // Merge the last block by append - ASSERT(pCompBlock->last && pCompBlock->numOfPoints < pHelper->config.minRowsPerFileBlock); + if (keyFirst > blockAtIdx(pHelper, blkIdx)->keyLast) { // Merge with the last block by append + ASSERT(blockAtIdx(pHelper, blkIdx)->numOfPoints < pHelper->config.minRowsPerFileBlock && blkIdx == pIdx->numOfSuperBlocks-1); int defaultRowsToWrite = pHelper->config.maxRowsPerFileBlock * 4 / 5; // TODO: make a interface - rowsWritten = MIN((defaultRowsToWrite - pCompBlock->numOfPoints), pDataCols->numOfPoints); - if (rowsWritten + pCompBlock->numOfPoints >= pHelper->config.minRowsPerFileBlock) { - // Need to write to .data file + rowsWritten = MIN((defaultRowsToWrite - blockAtIdx(pHelper, blkIdx)->numOfPoints), pDataCols->numOfPoints); + if ((blockAtIdx(pHelper, blkIdx)->numOfSubBlocks < TSDB_MAX_SUBBLOCKS) && + (blockAtIdx(pHelper, blkIdx)->numOfPoints + rowsWritten < pHelper->config.minRowsPerFileBlock) && (pHelper->files.nLastF.fd) > 0) { + if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.lastF), pDataCols, rowsWritten, &compBlock, true, false) < 0) + goto _err; + if (tsdbAddSubBlock(pHelper, &compBlock, blkIdx, rowsWritten) < 0) goto _err; + } else { + // Load if (tsdbLoadBlockData(pHelper, blkIdx, NULL) < 0) goto _err; + ASSERT(pHelper->pDataCols[0]->numOfPoints == blockAtIdx(pHelper, blkIdx)->numOfPoints); + // Merge if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, rowsWritten) < 0) goto _err; - if (tsdbWriteBlockToFile(pHelper, &pHelper->files.dataF, pHelper->pDataCols[0], - rowsWritten + pCompBlock->numOfPoints, &compBlock, false, true) < 0) + // Write + SFile *pWFile = NULL; + bool isLast = false; + if (pHelper->pDataCols[0]->numOfPoints >= pHelper->config.minRowsPerFileBlock) { + pWFile = &(pHelper->files.dataF); + } else { + isLast = true; + pWFile = (pHelper->files.nLastF.fd > 0) ? &(pHelper->files.nLastF) : &(pHelper->files.lastF); + } + if (tsdbWriteBlockToFile(pHelper, pWFile, pHelper->pDataCols[0], + pHelper->pDataCols[0]->numOfPoints, &compBlock, isLast, true) < 0) goto _err; if (tsdbUpdateSuperBlock(pHelper, &compBlock, blkIdx) < 0) goto _err; - } else { - // Need still write the .last or .l file - if (pHelper->files.nLastF.fd > 0) { - if (tsdbLoadBlockData(pHelper, blkIdx, NULL) < 0) goto _err; - if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, rowsWritten) < 0) goto _err; - if (tsdbWriteBlockToFile(pHelper, &pHelper->files.nLastF, pHelper->pDataCols[0], - rowsWritten + pCompBlock->numOfPoints, &compBlock, false, true) < 0) - goto _err; - if (tsdbUpdateSuperBlock(pHelper, &compBlock, blkIdx) < 0) goto _err; - } else { - // Write to .last file and append as a sub-block - if (tsdbWriteBlockToFile(pHelper, &pHelper->files.lastF, pDataCols, rowsWritten, &compBlock, true, false) < 0) - goto _err; - if (tsdbAddSubBlock(pHelper, &compBlock, blkIdx, rowsWritten) < 0) goto _err; - } } + + ASSERT(pHelper->hasOldLastBlock); + pHelper->hasOldLastBlock = false; } else { - // TODO: key overlap, must merge with the block - ASSERT(keyFirst <= pCompBlock->keyLast); + // Key must overlap with the block + ASSERT(keyFirst <= blockAtIdx(pHelper, blkIdx)->keyLast); TSKEY keyLimit = (blkIdx == pIdx->numOfSuperBlocks - 1) ? INT64_MAX : pHelper->pCompInfo->blocks[blkIdx + 1].keyFirst - 1; - int rows1 = tsdbGetRowsInRange(pDataCols, pCompBlock->keyFirst, - pCompBlock->keyLast); // number of rows must merge in this block - int rows2 = - pHelper->config.maxRowsPerFileBlock - pCompBlock->numOfPoints; // max nuber of rows the block can have more - int rows3 = tsdbGetRowsInRange(pDataCols, pCompBlock->keyFirst, - keyLimit); // number of rows between this block and the next block + // rows1: number of rows must merge in this block + int rows1 = tsdbGetRowsInRange(pDataCols, blockAtIdx(pHelper, blkIdx)->keyFirst, blockAtIdx(pHelper, blkIdx)->keyLast); + // rows2: max nuber of rows the block can have more + int rows2 = pHelper->config.maxRowsPerFileBlock - blockAtIdx(pHelper, blkIdx)->numOfPoints; + // rows3: number of rows between this block and the next block + int rows3 = tsdbGetRowsInRange(pDataCols, blockAtIdx(pHelper, blkIdx)->keyFirst, keyLimit); ASSERT(rows3 >= rows1); - if ((rows2 >= rows1) && ((!pCompBlock->last) || (pHelper->files.nLastF.fd < 0))) { + if ((rows2 >= rows1) && + (( blockAtIdx(pHelper, blkIdx)->last) || + ((rows1 + blockAtIdx(pHelper, blkIdx)->numOfPoints < pHelper->config.minRowsPerFileBlock) && (pHelper->files.nLastF.fd < 0)))) { rowsWritten = rows1; bool isLast = false; SFile *pFile = NULL; - if (pCompBlock->last) { + if (blockAtIdx(pHelper, blkIdx)->last) { isLast = true; pFile = &(pHelper->files.lastF); } else { @@ -819,63 +831,79 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa if (tsdbWriteBlockToFile(pHelper, pFile, pDataCols, rows1, &compBlock, isLast, false) < 0) goto _err; if (tsdbAddSubBlock(pHelper, &compBlock, blkIdx, rowsWritten) < 0) goto _err; - } else { - // Need to read the data block and merge with pCompDataCol to write as super block - - // Read + } else { // Load-Merge-Write + // Load if (tsdbLoadBlockData(pHelper, blkIdx, NULL) < 0) goto _err; + if (blockAtIdx(pHelper, blkIdx)->last) pHelper->hasOldLastBlock = false; + rowsWritten = rows3; int iter1 = 0; // iter over pHelper->pDataCols[0] int iter2 = 0; // iter over pDataCols - tdResetDataCols(pHelper->pDataCols[1]); + int round = 0; + // tdResetDataCols(pHelper->pDataCols[1]); while (true) { - if (iter1 >= pHelper->pDataCols[0]->numOfPoints && iter2 >= rows3) { - if (pHelper->pDataCols[1]->numOfPoints > 0) { - if (tsdbWriteBlockToFile(pHelper, &pHelper->files.dataF, pHelper->pDataCols[1], - pHelper->pDataCols[1]->numOfPoints, &compBlock, false, true) < 0) - goto _err; - // TODO: the blkIdx here is not correct - tsdbAddSubBlock(pHelper, &compBlock, blkIdx, pHelper->pDataCols[1]->numOfPoints); - } - } - - TSKEY key1 = iter1 >= pHelper->pDataCols[0]->numOfPoints - ? INT64_MAX - : ((int64_t *)(pHelper->pDataCols[0]->cols[0].pData))[iter1]; - TSKEY key2 = iter2 >= rowsWritten ? INT64_MAX : ((int64_t *)(pDataCols->cols[0].pData))[iter2]; - - if (key1 < key2) { - for (int i = 0; i < pDataCols->numOfCols; i++) { - SDataCol *pDataCol = pHelper->pDataCols[1]->cols + i; - memcpy(((char *)pDataCol->pData + TYPE_BYTES[pDataCol->type] * pHelper->pDataCols[1]->numOfPoints), - ((char *)pHelper->pDataCols[0]->cols[i].pData + TYPE_BYTES[pDataCol->type] * iter1), - TYPE_BYTES[pDataCol->type]); - } - pHelper->pDataCols[1]->numOfPoints++; - iter1++; - } else if (key1 == key2) { - // TODO: think about duplicate key cases - ASSERT(false); + if (iter1 >= pHelper->pDataCols[0]->numOfPoints && iter2 >= rows3) break; + tdMergeTwoDataCols(pHelper->pDataCols[1], pHelper->pDataCols[0], &iter1, pDataCols, &iter2, pHelper->config.maxRowsPerFileBlock * 4 / 5); + ASSERT(pHelper->pDataCols[1]->numOfPoints > 0); + if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.dataF), pHelper->pDataCols[1], + pHelper->pDataCols[1]->numOfPoints, &compBlock, false, true) < 0) + goto _err; + if (round == 0) { + tsdbUpdateSuperBlock(pHelper, &compBlock, blkIdx); } else { - for (int i = 0; i < pDataCols->numOfCols; i++) { - SDataCol *pDataCol = pHelper->pDataCols[1]->cols + i; - memcpy(((char *)pDataCol->pData + TYPE_BYTES[pDataCol->type] * pHelper->pDataCols[1]->numOfPoints), - ((char *)pDataCols->cols[i].pData + - TYPE_BYTES[pDataCol->type] * iter2), - TYPE_BYTES[pDataCol->type]); - } - pHelper->pDataCols[1]->numOfPoints++; - iter2++; - } - - if (pHelper->pDataCols[0]->numOfPoints >= pHelper->config.maxRowsPerFileBlock * 4 / 5) { - if (tsdbWriteBlockToFile(pHelper, &pHelper->files.dataF, pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfPoints, &compBlock, false, true) < 0) goto _err; - // TODO: blkIdx here is not correct, fix it tsdbInsertSuperBlock(pHelper, &compBlock, blkIdx); - - tdResetDataCols(pHelper->pDataCols[1]); } + round++; + blkIdx++; + // TODO: the blkIdx here is not correct + + // if (iter1 >= pHelper->pDataCols[0]->numOfPoints && iter2 >= rows3) { + // if (pHelper->pDataCols[1]->numOfPoints > 0) { + // if (tsdbWriteBlockToFile(pHelper, &pHelper->files.dataF, pHelper->pDataCols[1], + // pHelper->pDataCols[1]->numOfPoints, &compBlock, false, true) < 0) + // goto _err; + // // TODO: the blkIdx here is not correct + // tsdbAddSubBlock(pHelper, &compBlock, blkIdx, pHelper->pDataCols[1]->numOfPoints); + // } + // } + + // TSKEY key1 = iter1 >= pHelper->pDataCols[0]->numOfPoints + // ? INT64_MAX + // : ((int64_t *)(pHelper->pDataCols[0]->cols[0].pData))[iter1]; + // TSKEY key2 = iter2 >= rowsWritten ? INT64_MAX : ((int64_t *)(pDataCols->cols[0].pData))[iter2]; + + // if (key1 < key2) { + // for (int i = 0; i < pDataCols->numOfCols; i++) { + // SDataCol *pDataCol = pHelper->pDataCols[1]->cols + i; + // memcpy(((char *)pDataCol->pData + TYPE_BYTES[pDataCol->type] * pHelper->pDataCols[1]->numOfPoints), + // ((char *)pHelper->pDataCols[0]->cols[i].pData + TYPE_BYTES[pDataCol->type] * iter1), + // TYPE_BYTES[pDataCol->type]); + // } + // pHelper->pDataCols[1]->numOfPoints++; + // iter1++; + // } else if (key1 == key2) { + // // TODO: think about duplicate key cases + // ASSERT(false); + // } else { + // for (int i = 0; i < pDataCols->numOfCols; i++) { + // SDataCol *pDataCol = pHelper->pDataCols[1]->cols + i; + // memcpy(((char *)pDataCol->pData + TYPE_BYTES[pDataCol->type] * pHelper->pDataCols[1]->numOfPoints), + // ((char *)pDataCols->cols[i].pData + + // TYPE_BYTES[pDataCol->type] * iter2), + // TYPE_BYTES[pDataCol->type]); + // } + // pHelper->pDataCols[1]->numOfPoints++; + // iter2++; + // } + + // if (pHelper->pDataCols[0]->numOfPoints >= pHelper->config.maxRowsPerFileBlock * 4 / 5) { + // if (tsdbWriteBlockToFile(pHelper, &pHelper->files.dataF, pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfPoints, &compBlock, false, true) < 0) goto _err; + // // TODO: blkIdx here is not correct, fix it + // tsdbInsertSuperBlock(pHelper, &compBlock, blkIdx); + + // tdResetDataCols(pHelper->pDataCols[1]); + // } } } } From 8e1bd0743892b2b02ed0700ff26e24fcfd69b41d Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sat, 18 Apr 2020 11:45:41 +0800 Subject: [PATCH 24/29] TD-100 --- src/tsdb/src/tsdbRWHelper.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index a5b93a5688..8bb32cf179 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -715,7 +715,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa pCompBlock->last = isLast; pCompBlock->offset = offset; - pCompBlock->algorithm = 0; // TODO + pCompBlock->algorithm = pHelper->config.compress; pCompBlock->numOfPoints = rowsToWrite; pCompBlock->sversion = pHelper->tableInfo.sversion; pCompBlock->len = (int32_t)tsize; From 366b07fb969848a21d88db4fee1fa685ecde9f79 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sat, 18 Apr 2020 14:03:06 +0800 Subject: [PATCH 25/29] TD-100 --- src/common/src/dataformat.c | 37 ++-------------------------- src/tsdb/src/tsdbMain.c | 2 +- src/tsdb/src/tsdbRWHelper.c | 49 ++++++++++++++++++------------------- 3 files changed, 27 insertions(+), 61 deletions(-) diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c index 15dec2c71c..fb20892452 100644 --- a/src/common/src/dataformat.c +++ b/src/common/src/dataformat.c @@ -415,39 +415,6 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) { int iter1 = 0; int iter2 = 0; tdMergeTwoDataCols(target,pTarget, &iter1, source, &iter2, pTarget->numOfPoints + rowsToMerge); - // while (true) { - // if (iter1 >= pTarget->numOfPoints && iter2 >= source->numOfPoints) break; - - // TSKEY key1 = (iter1 >= pTarget->numOfPoints) ? INT64_MAX : ((TSKEY *)(pTarget->cols[0].pData))[iter1]; - // TSKEY key2 = (iter2 >= rowsToMerge) ? INT64_MAX : ((TSKEY *)(source->cols[0].pData))[iter2]; - - // if (key1 < key2) { // Copy from pTarget - // for (int i = 0; i < pTarget->numOfCols; i++) { - // ASSERT(target->cols[i].type == pTarget->cols[i].type); - // memcpy((void *)((char *)(target->cols[i].pData) + TYPE_BYTES[target->cols[i].type] * target->numOfPoints), - // (void *)((char *)(pTarget->cols[i].pData) + TYPE_BYTES[pTarget->cols[i].type] * iter1), - // TYPE_BYTES[target->cols[i].type]); - // target->cols[i].len += TYPE_BYTES[target->cols[i].type]; - // } - - // target->numOfPoints++; - // iter1++; - // } else if (key1 > key2) { // Copy from source - // for (int i = 0; i < source->numOfCols; i++) { - // ASSERT(target->cols[i].type == pTarget->cols[i].type); - // memcpy((void *)((char *)(target->cols[i].pData) + TYPE_BYTES[target->cols[i].type] * target->numOfPoints), - // (void *)((char *)(source->cols[i].pData) + TYPE_BYTES[source->cols[i].type] * iter2), - // TYPE_BYTES[target->cols[i].type]); - // target->cols[i].len += TYPE_BYTES[target->cols[i].type]; - // } - - // target->numOfPoints++; - // iter2++; - // } else { - // // TODO - // ASSERT(false); - // } - // } tdFreeDataCols(pTarget); return 0; @@ -476,7 +443,7 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, SDataCol } target->numOfPoints++; - *iter1++; + (*iter1)++; } else if (key1 > key2) { for (int i = 0; i < src2->numOfCols; i++) { ASSERT(target->cols[i].type == src2->cols[i].type); @@ -487,7 +454,7 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, SDataCol } target->numOfPoints++; - *iter2++; + (*iter2)++; } else { ASSERT(false); } diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 9816dbf135..d09c2bba7c 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -957,7 +957,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters int nLoop = 0; while (true) { int rowsRead = tsdbReadRowsFromCache(pIter, maxKey, maxRowsToRead, pDataCols); - ASSERT(rowsRead >= 0); + assert(rowsRead >= 0); if (pDataCols->numOfPoints == 0) break; nLoop++; diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 8bb32cf179..b3280e341c 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -20,7 +20,7 @@ // Local function definitions static int tsdbCheckHelperCfg(SHelperCfg *pCfg); static int tsdbInitHelperFile(SRWHelper *pHelper); -static void tsdbClearHelperFile(SHelperFile *pHFile); +// static void tsdbClearHelperFile(SHelperFile *pHFile); static bool tsdbShouldCreateNewLast(SRWHelper *pHelper); static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, int rowsToWrite, SCompBlock *pCompBlock, bool isLast, bool isSuperBlock); @@ -615,31 +615,30 @@ static int tsdbCheckHelperCfg(SHelperCfg *pCfg) { return 0; } +// static void tsdbClearHelperFile(SHelperFile *pHFile) { +// pHFile->fid = -1; +// if (pHFile->headF.fd > 0) { +// close(pHFile->headF.fd); +// pHFile->headF.fd = -1; +// } +// if (pHFile->dataF.fd > 0) { +// close(pHFile->dataF.fd); +// pHFile->dataF.fd = -1; +// } +// if (pHFile->lastF.fd > 0) { +// close(pHFile->lastF.fd); +// pHFile->lastF.fd = -1; +// } +// if (pHFile->nHeadF.fd > 0) { +// close(pHFile->nHeadF.fd); +// pHFile->nHeadF.fd = -1; +// } +// if (pHFile->nLastF.fd > 0) { +// close(pHFile->nLastF.fd); +// pHFile->nLastF.fd = -1; +// } -static void tsdbClearHelperFile(SHelperFile *pHFile) { - pHFile->fid = -1; - if (pHFile->headF.fd > 0) { - close(pHFile->headF.fd); - pHFile->headF.fd = -1; - } - if (pHFile->dataF.fd > 0) { - close(pHFile->dataF.fd); - pHFile->dataF.fd = -1; - } - if (pHFile->lastF.fd > 0) { - close(pHFile->lastF.fd); - pHFile->lastF.fd = -1; - } - if (pHFile->nHeadF.fd > 0) { - close(pHFile->nHeadF.fd); - pHFile->nHeadF.fd = -1; - } - if (pHFile->nLastF.fd > 0) { - close(pHFile->nLastF.fd); - pHFile->nLastF.fd = -1; - } - -} +// } static bool tsdbShouldCreateNewLast(SRWHelper *pHelper) { ASSERT(pHelper->files.lastF.fd > 0); From 7e8fc3b2ae93c34e5d46311988367dc02318ce82 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sat, 18 Apr 2020 16:05:07 +0800 Subject: [PATCH 26/29] TD-100 --- src/tsdb/inc/tsdbMain.h | 9 +++++--- src/tsdb/src/tsdbMain.c | 16 +++---------- src/tsdb/src/tsdbRWHelper.c | 44 ++++++++++++++++++++++++++---------- src/tsdb/tests/tsdbTests.cpp | 22 ++++-------------- 4 files changed, 45 insertions(+), 46 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 48181c078b..9d4b8b17f7 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -443,13 +443,16 @@ typedef struct { #define helperHasState(h, s) ((((h)->state) & (s)) == (s)) #define blockAtIdx(h, idx) ((h)->pCompInfo->blocks + idx) -int tsdbInitHelper(SRWHelper *pHelper, SHelperCfg *pCfg); +int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo); +int tsdbInitWriteHelper(SRWHelper *pHelper, STsdbRepo *pRepo); +// int tsdbInitHelper(SRWHelper *pHelper, SHelperCfg *pCfg); void tsdbDestroyHelper(SRWHelper *pHelper); void tsdbResetHelper(SRWHelper *pHelper); // --------- For set operations -int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup); -void tsdbSetHelperTable(SRWHelper *pHelper, SHelperTable *pHelperTable, STSchema *pSchema); +int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup); +// void tsdbSetHelperTable(SRWHelper *pHelper, SHelperTable *pHelperTable, STSchema *pSchema); +void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo); int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError); // --------- For read operations diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index d09c2bba7c..d5f703ab0f 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -400,6 +400,7 @@ int tsdbInitTableCfg(STableCfg *config, ETableType type, int64_t uid, int32_t ti config->superUid = TSDB_INVALID_SUPER_TABLE_ID; config->tableId.uid = uid; config->tableId.tid = tid; + config->name = strdup("test1"); return 0; } @@ -873,16 +874,7 @@ static void *tsdbCommitData(void *arg) { return NULL; } - // Create a write helper to commit data - SHelperCfg hcfg = {.type = TSDB_WRITE_HELPER, - .maxTables = pCfg->maxTables, - .maxRowSize = pMeta->maxRowBytes, - .maxRows = pCfg->maxRowsPerFileBlock, - .maxCols = pMeta->maxCols, - .minRowsPerFileBlock = pCfg->minRowsPerFileBlock, - .maxRowsPerFileBlock = pCfg->maxRowsPerFileBlock, - .compress = pCfg->compression}; - if (tsdbInitHelper(&whelper, &hcfg) < 0) goto _exit; + if (tsdbInitWriteHelper(&whelper, pRepo) < 0) goto _exit; if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) goto _exit; int sfid = tsdbGetKeyFileId(pCache->imem->keyFirst, pCfg->daysPerFile, pCfg->precision); @@ -898,7 +890,6 @@ static void *tsdbCommitData(void *arg) { _exit: tdFreeDataCols(pDataCols); - tsdbDestroyHelper(&whelper); tsdbDestroyTableIters(iters, pCfg->maxTables); tsdbLockRepo(arg); @@ -948,8 +939,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters SSkipListIterator *pIter = iters[tid]; // Set the helper and the buffer dataCols object to help to write this table - SHelperTable hTable = {.uid = pTable->tableId.uid, .tid = pTable->tableId.tid, .sversion = pTable->sversion}; - tsdbSetHelperTable(pHelper, &hTable, tsdbGetTableSchema(pMeta, pTable)); + tsdbSetHelperTable(pHelper, pTable, pRepo); tdInitDataCols(pDataCols, tsdbGetTableSchema(pMeta, pTable)); // Loop to write the data in the cache to files. If no data to write, just break the loop diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index b3280e341c..9a42241900 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -18,7 +18,7 @@ #include "talgo.h" // Local function definitions -static int tsdbCheckHelperCfg(SHelperCfg *pCfg); +// static int tsdbCheckHelperCfg(SHelperCfg *pCfg); static int tsdbInitHelperFile(SRWHelper *pHelper); // static void tsdbClearHelperFile(SHelperFile *pHFile); static bool tsdbShouldCreateNewLast(SRWHelper *pHelper); @@ -102,14 +102,21 @@ static void tsdbDestroyHelperBlock(SRWHelper *pHelper) { tdFreeDataCols(pHelper->pDataCols[1]); } -// ------------------------------------------ OPERATIONS FOR OUTSIDE ------------------------------------------ -int tsdbInitHelper(SRWHelper *pHelper, SHelperCfg *pCfg) { - if (pHelper == NULL || pCfg == NULL || tsdbCheckHelperCfg(pCfg) < 0) return -1; +static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t type) { + if (pHelper == NULL || pRepo == NULL) return -1; memset((void *)pHelper, 0, sizeof(*pHelper)); // Init global configuration - pHelper->config = *pCfg; + pHelper->config.type = type; + pHelper->config.maxTables = pRepo->config.maxTables; + pHelper->config.maxRowSize = pRepo->tsdbMeta->maxRowBytes; + pHelper->config.maxRows = pRepo->config.maxRowsPerFileBlock; + pHelper->config.maxCols = pRepo->tsdbMeta->maxCols; + pHelper->config.minRowsPerFileBlock = pRepo->config.minRowsPerFileBlock; + pHelper->config.maxRowsPerFileBlock = pRepo->config.maxRowsPerFileBlock; + pHelper->config.compress = pRepo->config.compression; + pHelper->state = TSDB_HELPER_CLEAR_STATE; // Init file part @@ -128,6 +135,15 @@ _err: return -1; } +// ------------------------------------------ OPERATIONS FOR OUTSIDE ------------------------------------------ +int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) { + return tsdbInitHelper(pHelper, pRepo, TSDB_READ_HELPER); +} + +int tsdbInitWriteHelper(SRWHelper *pHelper, STsdbRepo *pRepo) { + return tsdbInitHelper(pHelper, pRepo, TSDB_WRITE_HELPER); +} + void tsdbDestroyHelper(SRWHelper *pHelper) { if (pHelper) { tsdbDestroyHelperFile(pHelper); @@ -243,18 +259,22 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { return 0; } -void tsdbSetHelperTable(SRWHelper *pHelper, SHelperTable *pHelperTable, STSchema *pSchema) { +void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) { ASSERT(helperHasState(pHelper, TSDB_HELPER_FILE_SET_AND_OPEN | TSDB_HELPER_IDX_LOAD)); // Clear members and state used by previous table tsdbResetHelperTable(pHelper); ASSERT(pHelper->state == (TSDB_HELPER_FILE_SET_AND_OPEN | TSDB_HELPER_IDX_LOAD)); - pHelper->tableInfo = *pHelperTable; + pHelper->tableInfo.tid = pTable->tableId.tid; + pHelper->tableInfo.uid = pTable->tableId.uid; + pHelper->tableInfo.sversion = pTable->sversion; + STSchema *pSchema = tsdbGetTableSchema(pRepo->tsdbMeta, pTable); + tdInitDataCols(pHelper->pDataCols[0], pSchema); tdInitDataCols(pHelper->pDataCols[1], pSchema); - SCompIdx *pIdx = pHelper->pCompIdx + pHelperTable->tid; + SCompIdx *pIdx = pHelper->pCompIdx + pTable->tableId.tid; if (pIdx->offset > 0 && pIdx->hasLast) { pHelper->hasOldLastBlock = true; } @@ -610,10 +630,10 @@ _err: return -1; } -static int tsdbCheckHelperCfg(SHelperCfg *pCfg) { - // TODO - return 0; -} +// static int tsdbCheckHelperCfg(SHelperCfg *pCfg) { +// // TODO +// return 0; +// } // static void tsdbClearHelperFile(SHelperFile *pHFile) { // pHFile->fid = -1; diff --git a/src/tsdb/tests/tsdbTests.cpp b/src/tsdb/tests/tsdbTests.cpp index 9d1bae1832..a7d94f2362 100644 --- a/src/tsdb/tests/tsdbTests.cpp +++ b/src/tsdb/tests/tsdbTests.cpp @@ -191,29 +191,15 @@ TEST(TsdbTest, createRepo) { // Read from file SRWHelper rhelper; - SHelperCfg helperCfg = { - .type = TSDB_READ_HELPER, - .maxTables = repo->config.maxTables, - .maxRowSize = repo->tsdbMeta->maxRowBytes, - .maxRows = repo->config.maxRowsPerFileBlock, - .maxCols = repo->tsdbMeta->maxCols, - .minRowsPerFileBlock = repo->config.minRowsPerFileBlock, - .maxRowsPerFileBlock = repo->config.maxRowsPerFileBlock, - .compress = repo->config.compression, - - }; - tsdbInitHelper(&rhelper, &helperCfg); + tsdbInitReadHelper(&rhelper, repo); SFileGroup *pFGroup = tsdbSearchFGroup(repo->tsdbFileH, 1833); ASSERT_NE(pFGroup, nullptr); ASSERT_GE(tsdbSetAndOpenHelperFile(&rhelper, pFGroup), 0); - SHelperTable htable = { - .uid = tCfg.tableId.uid, - .tid = tCfg.tableId.tid, - .sversion = tCfg.sversion - }; - tsdbSetHelperTable(&rhelper, &htable, schema); + STable *pTable = tsdbGetTableByUid(repo->tsdbMeta, tCfg.tableId.uid); + ASSERT_NE(pTable, nullptr); + tsdbSetHelperTable(&rhelper, pTable, repo); ASSERT_EQ(tsdbLoadCompInfo(&rhelper, NULL), 0); ASSERT_EQ(tsdbLoadBlockData(&rhelper, 0, NULL), 0); From e165343c01d81192e027ab4b75e55672368da98c Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sat, 18 Apr 2020 17:57:31 +0800 Subject: [PATCH 27/29] TD-100 --- src/tsdb/inc/tsdbMain.h | 2 +- src/tsdb/src/tsdbRWHelper.c | 10 +++--- src/tsdb/src/tsdbRead.c | 60 +++++++++++++++++++----------------- src/tsdb/tests/tsdbTests.cpp | 2 +- 4 files changed, 39 insertions(+), 35 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 9d4b8b17f7..756a206af0 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -460,7 +460,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target); int tsdbLoadCompInfo(SRWHelper *pHelper, void *target); int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target); int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx, int16_t *colIds, int numOfColIds); -int tsdbLoadBlockData(SRWHelper *pHelper, int blkIdx, SDataCols *target); +int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *target); // --------- For write operations int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols); diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 9a42241900..da3c55bbf9 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -363,7 +363,7 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { ASSERT(pCompBlock->last); if (pCompBlock->numOfSubBlocks > 1) { - if (tsdbLoadBlockData(pHelper, pIdx->numOfSuperBlocks - 1, NULL) < 0) return -1; + if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, pIdx->numOfSuperBlocks - 1), NULL) < 0) return -1; ASSERT(pHelper->pDataCols[0]->numOfPoints > 0 && pHelper->pDataCols[0]->numOfPoints < pHelper->config.minRowsPerFileBlock); if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.nLastF), pHelper->pDataCols[0], @@ -607,8 +607,8 @@ _err: } // Load the whole block data -int tsdbLoadBlockData(SRWHelper *pHelper, int blkIdx, SDataCols *target) { - SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx; +int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *target) { + // SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx; int numOfSubBlock = pCompBlock->numOfSubBlocks; if (numOfSubBlock > 1) pCompBlock = (SCompBlock *)((char *)pHelper->pCompInfo + pCompBlock->offset); @@ -797,7 +797,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa if (tsdbAddSubBlock(pHelper, &compBlock, blkIdx, rowsWritten) < 0) goto _err; } else { // Load - if (tsdbLoadBlockData(pHelper, blkIdx, NULL) < 0) goto _err; + if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, blkIdx), NULL) < 0) goto _err; ASSERT(pHelper->pDataCols[0]->numOfPoints == blockAtIdx(pHelper, blkIdx)->numOfPoints); // Merge if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, rowsWritten) < 0) goto _err; @@ -852,7 +852,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa if (tsdbAddSubBlock(pHelper, &compBlock, blkIdx, rowsWritten) < 0) goto _err; } else { // Load-Merge-Write // Load - if (tsdbLoadBlockData(pHelper, blkIdx, NULL) < 0) goto _err; + if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, blkIdx), NULL) < 0) goto _err; if (blockAtIdx(pHelper, blkIdx)->last) pHelper->hasOldLastBlock = false; rowsWritten = rows3; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 6b3b7e1e4e..5ede599737 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -128,6 +128,7 @@ typedef struct STsdbQueryHandle { SFileGroup* pFileGroup; SFileGroupIter fileIter; SCompIdx* compIndex; + SRWHelper rhelper; } STsdbQueryHandle; static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { @@ -150,7 +151,8 @@ tsdb_query_handle_t* tsdbQueryTables(tsdb_repo_t* tsdb, STsdbQueryCond* pCond, S pQueryHandle->order = pCond->order; pQueryHandle->window = pCond->twindow; pQueryHandle->pTsdb = tsdb; - pQueryHandle->compIndex = calloc(10000, sizeof(SCompIdx)), + pQueryHandle->compIndex = calloc(10000, sizeof(SCompIdx)); + tsdbInitReadHelper(&pQueryHandle->rhelper, (STsdbRepo*) tsdb); pQueryHandle->loadDataAfterSeek = false; pQueryHandle->isFirstSlot = true; @@ -299,11 +301,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo SFileGroup* fileGroup = pQueryHandle->pFileGroup; assert(fileGroup->files[TSDB_FILE_TYPE_HEAD].fname > 0); - if (fileGroup->files[TSDB_FILE_TYPE_HEAD].fd == FD_INITIALIZER) { - fileGroup->files[TSDB_FILE_TYPE_HEAD].fd = open(fileGroup->files[TSDB_FILE_TYPE_HEAD].fname, O_RDONLY); - } else { - assert(FD_VALID(fileGroup->files[TSDB_FILE_TYPE_HEAD].fd)); - } + tsdbSetAndOpenHelperFile(&pQueryHandle->rhelper, fileGroup); // load all the comp offset value for all tables in this file // tsdbLoadCompIdx(fileGroup, pQueryHandle->compIndex, 10000); // todo set dynamic max tables @@ -314,7 +312,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo for (int32_t i = 0; i < numOfTables; ++i) { STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - SCompIdx* compIndex = &pQueryHandle->compIndex[pCheckInfo->tableId.tid]; + SCompIdx* compIndex = &pQueryHandle->rhelper.pCompIdx[pCheckInfo->tableId.tid]; if (compIndex->len == 0 || compIndex->numOfSuperBlocks == 0) { // no data block in this file, try next file continue;//no data blocks in the file belongs to pCheckInfo->pTable } else { @@ -329,7 +327,12 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo } // tsdbLoadCompBlocks(fileGroup, compIndex, pCheckInfo->pCompInfo); - + STable* pTable = tsdbGetTableByUid(tsdbGetMeta(pQueryHandle->pTsdb), pCheckInfo->tableId.uid); + assert(pTable != NULL); + + tsdbSetHelperTable(&pQueryHandle->rhelper, pTable, pQueryHandle->pTsdb); + + tsdbLoadCompInfo(&(pQueryHandle->rhelper), (void *)(pCheckInfo->pCompInfo)); SCompInfo* pCompInfo = pCheckInfo->pCompInfo; TSKEY s = MIN(pCheckInfo->lastKey, pQueryHandle->window.ekey); @@ -420,26 +423,26 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo tdInitDataCols(pCheckInfo->pDataCols, tsdbGetTableSchema(tsdbGetMeta(pQueryHandle->pTsdb), pCheckInfo->pTableObj)); - SFile* pFile = &pQueryHandle->pFileGroup->files[TSDB_FILE_TYPE_DATA]; - if (pFile->fd == FD_INITIALIZER) { - pFile->fd = open(pFile->fname, O_RDONLY); - } - - // if (tsdbLoadDataBlock(pFile, pBlock, 1, pCheckInfo->pDataCols, data) == 0) { - // SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo; - - // pBlockLoadInfo->fileGroup = pQueryHandle->pFileGroup; - // pBlockLoadInfo->slot = pQueryHandle->cur.slot; - // pBlockLoadInfo->sid = pCheckInfo->pTableObj->tableId.tid; - - // blockLoaded = true; + // SFile* pFile = &pQueryHandle->pFileGroup->files[TSDB_FILE_TYPE_DATA]; + // if (pFile->fd == FD_INITIALIZER) { + // pFile->fd = open(pFile->fname, O_RDONLY); // } + if (tsdbLoadBlockData(&(pQueryHandle->rhelper), pBlock, NULL) == 0) { + SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo; + + pBlockLoadInfo->fileGroup = pQueryHandle->pFileGroup; + pBlockLoadInfo->slot = pQueryHandle->cur.slot; + pBlockLoadInfo->sid = pCheckInfo->pTableObj->tableId.tid; + + blockLoaded = true; + } + taosArrayDestroy(sa); tfree(data); - TSKEY* d = (TSKEY*)pCheckInfo->pDataCols->cols[PRIMARYKEY_TIMESTAMP_COL_INDEX].pData; - assert(d[0] == pBlock->keyFirst && d[pBlock->numOfPoints - 1] == pBlock->keyLast); + // TSKEY* d = (TSKEY*)pCheckInfo->pDataCols->cols[PRIMARYKEY_TIMESTAMP_COL_INDEX].pData; + // assert(d[0] == pBlock->keyFirst && d[pBlock->numOfPoints - 1] == pBlock->keyLast); return blockLoaded; } @@ -595,7 +598,7 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf } } - int32_t start = MIN(cur->pos, endPos); + // int32_t start = MIN(cur->pos, endPos); // move the data block in the front to data block if needed int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); @@ -607,9 +610,10 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, j); if (pCol->info.colId == colId) { - SDataCol* pDataCol = &pCols->cols[i]; - memmove(pCol->pData, pDataCol->pData + pCol->info.bytes * start, - pQueryHandle->realNumOfRows * pCol->info.bytes); + // SDataCol* pDataCol = &pCols->cols[i]; + pCol->pData = pQueryHandle->rhelper.pDataCols[0]->cols[i].pData; + // memmove(pCol->pData, pDataCol->pData + pCol->info.bytes * start, + // pQueryHandle->realNumOfRows * pCol->info.bytes); break; } } @@ -1530,7 +1534,7 @@ void tsdbCleanupQueryHandle(tsdb_query_handle_t queryHandle) { size_t cols = taosArrayGetSize(pQueryHandle->pColumns); for (int32_t i = 0; i < cols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - tfree(pColInfo->pData); + // tfree(pColInfo->pData); } taosArrayDestroy(pQueryHandle->pColumns); diff --git a/src/tsdb/tests/tsdbTests.cpp b/src/tsdb/tests/tsdbTests.cpp index a7d94f2362..85fca7d94f 100644 --- a/src/tsdb/tests/tsdbTests.cpp +++ b/src/tsdb/tests/tsdbTests.cpp @@ -202,7 +202,7 @@ TEST(TsdbTest, createRepo) { tsdbSetHelperTable(&rhelper, pTable, repo); ASSERT_EQ(tsdbLoadCompInfo(&rhelper, NULL), 0); - ASSERT_EQ(tsdbLoadBlockData(&rhelper, 0, NULL), 0); + ASSERT_EQ(tsdbLoadBlockData(&rhelper, blockAtIdx(&rhelper, 0), NULL), 0); int k = 0; } From 0d2cc5b6f3819e8b74a4abe7bdef8c9ff85cb8d0 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sat, 18 Apr 2020 18:06:53 +0800 Subject: [PATCH 28/29] TD-100 --- src/tsdb/src/tsdbRead.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 5ede599737..1783e26f53 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1531,14 +1531,15 @@ void tsdbCleanupQueryHandle(tsdb_query_handle_t queryHandle) { taosArrayDestroy(pQueryHandle->pTableCheckInfo); tfree(pQueryHandle->compIndex); - size_t cols = taosArrayGetSize(pQueryHandle->pColumns); - for (int32_t i = 0; i < cols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - // tfree(pColInfo->pData); - } + // size_t cols = taosArrayGetSize(pQueryHandle->pColumns); + // for (int32_t i = 0; i < cols; ++i) { + // SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + // // tfree(pColInfo->pData); + // } taosArrayDestroy(pQueryHandle->pColumns); tfree(pQueryHandle->pDataBlockInfo); + tsdbDestroyHelper(&pQueryHandle->rhelper); tfree(pQueryHandle); } From 3a6ef2530761798f7eedff9ae56ed1d777269b9d Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sat, 18 Apr 2020 21:07:13 +0800 Subject: [PATCH 29/29] TD-100 --- src/tsdb/src/tsdbMain.c | 2 ++ src/tsdb/src/tsdbRWHelper.c | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index d5f703ab0f..7375a35796 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -867,6 +867,8 @@ static void *tsdbCommitData(void *arg) { SRWHelper whelper = {0}; if (pCache->imem == NULL) return NULL; + pRepo->appH.walCallBack(pRepo->appH.appH); + // Create the iterator to read from cache SSkipListIterator **iters = tsdbCreateTableIters(pMeta, pCfg->maxTables); if (iters == NULL) { diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index da3c55bbf9..9c35ebb40f 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -67,7 +67,7 @@ static void tsdbResetHelperTableImpl(SRWHelper *pHelper) { static void tsdbResetHelperTable(SRWHelper *pHelper) { tsdbResetHelperBlock(pHelper); tsdbResetHelperTableImpl(pHelper); - helperClearState(pHelper, TSDB_HELPER_TABLE_SET); + helperClearState(pHelper, (TSDB_HELPER_TABLE_SET|TSDB_HELPER_INFO_LOAD)); } static void tsdbInitHelperTable(SRWHelper *pHelper) { @@ -83,7 +83,8 @@ static void tsdbResetHelperBlockImpl(SRWHelper *pHelper) { } static void tsdbResetHelperBlock(SRWHelper *pHelper) { - // TODO + tsdbResetHelperBlockImpl(pHelper); + // helperClearState(pHelper, TSDB_HELPER_) } static int tsdbInitHelperBlock(SRWHelper *pHelper) { @@ -264,7 +265,7 @@ void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) { // Clear members and state used by previous table tsdbResetHelperTable(pHelper); - ASSERT(pHelper->state == (TSDB_HELPER_FILE_SET_AND_OPEN | TSDB_HELPER_IDX_LOAD)); + ASSERT(helperHasState(pHelper, (TSDB_HELPER_FILE_SET_AND_OPEN | TSDB_HELPER_IDX_LOAD))); pHelper->tableInfo.tid = pTable->tableId.tid; pHelper->tableInfo.uid = pTable->tableId.uid;