From d4c8fed317bf6b6bcaa34b1317d775a48bddec46 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 2 Mar 2022 17:46:00 +0800 Subject: [PATCH 01/17] Block-wise SMA extraction --- include/common/taosdef.h | 4 + source/dnode/vnode/inc/tsdb.h | 10 +- source/dnode/vnode/src/inc/tsdbFS.h | 2 - source/dnode/vnode/src/inc/tsdbFile.h | 47 ++++++- source/dnode/vnode/src/inc/tsdbReadImpl.h | 127 ++++++++++++++---- source/dnode/vnode/src/tsdb/tsdbCommit.c | 142 ++++++++++++++++++--- source/dnode/vnode/src/tsdb/tsdbCompact.c | 2 + source/dnode/vnode/src/tsdb/tsdbFS.c | 4 +- source/dnode/vnode/src/tsdb/tsdbFile.c | 43 ++++--- source/dnode/vnode/src/tsdb/tsdbMain.c | 7 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 8 +- source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 80 +++++++++--- source/util/src/terror.c | 1 + 13 files changed, 377 insertions(+), 100 deletions(-) diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 66cb0bb4ba..69c2618ac8 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -51,6 +51,10 @@ typedef enum { } ECheckItemType; typedef enum { TD_ROW_DISCARD_UPDATE = 0, TD_ROW_OVERWRITE_UPDATE = 1, TD_ROW_PARTIAL_UPDATE = 2 } TDUpdateConfig; +typedef enum { + TSDB_STATIS_OK = 0, // statis part exist and load successfully + TSDB_STATIS_NONE = 1, // statis part not exist +} ETsdbStatisStatus; extern char *qtypeStr[]; diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index 677ca9c336..6bc89ddd66 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -27,12 +27,12 @@ extern "C" { typedef struct SDataStatis { int16_t colId; - int64_t sum; - int64_t max; - int64_t min; int16_t maxIndex; int16_t minIndex; int16_t numOfNull; + int64_t sum; + int64_t max; + int64_t min; } SDataStatis; typedef struct STable { @@ -53,6 +53,8 @@ typedef struct STsdb STsdb; typedef struct STsdbCfg { int8_t precision; + int8_t update; + int8_t compression; uint64_t lruCacheSize; int32_t daysPerFile; int32_t minRowsPerFileBlock; @@ -60,8 +62,6 @@ typedef struct STsdbCfg { int32_t keep; int32_t keep1; int32_t keep2; - int8_t update; - int8_t compression; } STsdbCfg; // query condition to build multi-table data block iterator diff --git a/source/dnode/vnode/src/inc/tsdbFS.h b/source/dnode/vnode/src/inc/tsdbFS.h index af432aa9d9..dab697ce8b 100644 --- a/source/dnode/vnode/src/inc/tsdbFS.h +++ b/source/dnode/vnode/src/inc/tsdbFS.h @@ -18,8 +18,6 @@ #include "tsdbFile.h" -#define TSDB_FS_VERSION 0 - // ================== TSDB global config extern bool tsdbForceKeepFile; diff --git a/source/dnode/vnode/src/inc/tsdbFile.h b/source/dnode/vnode/src/inc/tsdbFile.h index 15c8d512d6..370de0ab0a 100644 --- a/source/dnode/vnode/src/inc/tsdbFile.h +++ b/source/dnode/vnode/src/inc/tsdbFile.h @@ -44,7 +44,37 @@ #define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK) #define TSDB_FILE_IS_BAD(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_BAD) -typedef enum { TSDB_FILE_HEAD = 0, TSDB_FILE_DATA, TSDB_FILE_LAST, TSDB_FILE_MAX, TSDB_FILE_META } TSDB_FILE_T; +typedef enum { + TSDB_FILE_HEAD = 0, // .head + TSDB_FILE_DATA, // .data + TSDB_FILE_LAST, // .last + TSDB_FILE_SMAD, // .smad(Block-wise SMA) + TSDB_FILE_SMAL, // .smal(Block-wise SMA) + TSDB_FILE_MAX, // + TSDB_FILE_TSMA, // .tsma.${sma_index_name}, Time-range-wise SMA + TSDB_FILE_RSMA, // .rsma.${sma_index_name}, Time-range-wise Rollup SMA + TSDB_FILE_META // meta +} TSDB_FILE_T; + +typedef enum { + TSDB_FS_VER_0 = 0, + TSDB_FS_VER_MAX, +} ETsdbFsVer; + +#define TSDB_LATEST_FVER TSDB_FS_VER_0 // latest version for DFile +#define TSDB_LATEST_SFS_VER TSDB_FS_VER_0 // latest version for 'current' file + +static FORCE_INLINE uint32_t tsdbGetDFSVersion(TSDB_FILE_T fType) { // latest version for DFile + switch (fType) { + case TSDB_FILE_HEAD: // .head + case TSDB_FILE_DATA: // .data + case TSDB_FILE_LAST: // .last + case TSDB_FILE_SMAD: // .smad(Block-wise SMA) + case TSDB_FILE_SMAL: // .smal(Block-wise SMA) + default: + return TSDB_LATEST_FVER; + } +} #if 0 // =============== SMFile @@ -169,6 +199,7 @@ static FORCE_INLINE int64_t tsdbReadMFile(SMFile* pMFile, void* buf, int64_t nby // =============== SDFile typedef struct { uint32_t magic; + uint32_t fver; uint32_t len; uint32_t totalBlocks; uint32_t totalSubBlocks; @@ -188,7 +219,7 @@ void tsdbInitDFile(STsdb *pRepo, SDFile* pDFile, SDiskID did, int fid, uint32_t void tsdbInitDFileEx(SDFile* pDFile, SDFile* pODFile); int tsdbEncodeSDFile(void** buf, SDFile* pDFile); void* tsdbDecodeSDFile(STsdb *pRepo, void* buf, SDFile* pDFile); -int tsdbCreateDFile(STsdb *pRepo, SDFile* pDFile, bool updateHeader); +int tsdbCreateDFile(STsdb *pRepo, SDFile* pDFile, bool updateHeader, TSDB_FILE_T fType); int tsdbUpdateDFileHeader(SDFile* pDFile); int tsdbLoadDFileHeader(SDFile* pDFile, SDFInfo* pInfo); int tsdbParseDFilename(const char* fname, int* vid, int* fid, TSDB_FILE_T* ftype, uint32_t* version); @@ -292,12 +323,18 @@ static FORCE_INLINE int tsdbCopyDFile(SDFile* pSrc, SDFile* pDest) { // =============== SDFileSet typedef struct { - int fid; - int state; - SDFile files[TSDB_FILE_MAX]; + int fid; + int8_t state; // -128~127 + uint8_t ver; // 0~255, DFileSet version + uint16_t reserve; + SDFile files[TSDB_FILE_MAX]; } SDFileSet; +#define TSDB_LATEST_FSET_VER 0 + #define TSDB_FSET_FID(s) ((s)->fid) +#define TSDB_FSET_STATE(s) ((s)->state) +#define TSDB_FSET_VER(s) ((s)->ver) #define TSDB_DFILE_IN_SET(s, t) ((s)->files + (t)) #define TSDB_FSET_LEVEL(s) TSDB_FILE_LEVEL(TSDB_DFILE_IN_SET(s, 0)) #define TSDB_FSET_ID(s) TSDB_FILE_ID(TSDB_DFILE_IN_SET(s, 0)) diff --git a/source/dnode/vnode/src/inc/tsdbReadImpl.h b/source/dnode/vnode/src/inc/tsdbReadImpl.h index 8ecb4e4a6e..60e61bc57f 100644 --- a/source/dnode/vnode/src/inc/tsdbReadImpl.h +++ b/source/dnode/vnode/src/inc/tsdbReadImpl.h @@ -36,6 +36,7 @@ typedef struct { TSKEY maxKey; } SBlockIdx; +#ifdef TD_REFACTOR_3 typedef struct { int64_t last : 1; int64_t offset : 63; @@ -48,23 +49,35 @@ typedef struct { TSKEY keyFirst; TSKEY keyLast; } SBlock; +#else + +typedef enum { + TSDB_SBLK_VER_0 = 0, + TSDB_SBLK_VER_MAX, +} ESBlockVer; + +#define SBlockVerLatest TSDB_SBLK_VER_0 typedef struct { - int64_t last : 1; - int64_t offset : 63; + int64_t offset; int32_t algorithm : 8; int32_t numOfRows : 24; - uint8_t reserve0; + uint8_t last : 1; + uint8_t blkVer : 7; uint8_t numOfSubBlocks; int16_t numOfCols; // not including timestamp column - uint32_t len : 32; // data block length + uint32_t len; // data block length uint32_t keyLen : 24; // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols - uint32_t reserve1 : 8; - uint64_t blkVer : 8; - uint64_t aggrOffset : 56; + uint32_t reserve : 8; + uint64_t aggrStat : 1; + uint64_t aggrOffset : 63; TSKEY keyFirst; TSKEY keyLast; -} SBlock_3; +} SBlockV0; + +#define SBlock SBlockV0 // latest SBlock definition + +#endif typedef struct { int32_t delimiter; // For recovery usage @@ -73,6 +86,7 @@ typedef struct { SBlock blocks[]; } SBlockInfo; +#ifdef TD_REFACTOR_3 typedef struct { int16_t colId; uint16_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM @@ -89,17 +103,50 @@ typedef struct { uint8_t offsetH; char padding[1]; } SBlockCol; +#else +typedef struct { + int16_t colId; + uint8_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM + uint8_t reserve : 7; + uint8_t type; + int32_t len; + uint32_t offset; +} SBlockColV0; + +#define SBlockCol SBlockColV0 // latest SBlockCol definition + +typedef struct { + int16_t colId; + int16_t maxIndex; + int16_t minIndex; + int16_t numOfNull; + int64_t sum; + int64_t max; + int64_t min; +} SAggrBlkColV0; + +#define SAggrBlkCol SAggrBlkColV0 // latest SAggrBlkCol definition + +#endif // Code here just for back-ward compatibility static FORCE_INLINE void tsdbSetBlockColOffset(SBlockCol *pBlockCol, uint32_t offset) { +#ifdef TD_REFACTOR_3 pBlockCol->offset = offset & ((((uint32_t)1) << 24) - 1); pBlockCol->offsetH = (uint8_t)(offset >> 24); +#else + pBlockCol->offset = offset; +#endif } static FORCE_INLINE uint32_t tsdbGetBlockColOffset(SBlockCol *pBlockCol) { +#ifdef TD_REFACTOR_3 uint32_t offset1 = pBlockCol->offset; uint32_t offset2 = pBlockCol->offsetH; return (offset1 | (offset2 << 24)); +#else + return pBlockCol->offset; +#endif } typedef struct { @@ -109,31 +156,57 @@ typedef struct { SBlockCol cols[]; } SBlockData; +typedef void SAggrBlkData; // SBlockCol cols[]; + struct SReadH { - STsdb * pRepo; - SDFileSet rSet; // FSET to read - SArray * aBlkIdx; // SBlockIdx array - STable * pTable; // table to read - SBlockIdx * pBlkIdx; // current reading table SBlockIdx - int cidx; - SBlockInfo *pBlkInfo; - SBlockData *pBlkData; // Block info - SDataCols * pDCols[2]; - void * pBuf; // buffer - void * pCBuf; // compression buffer + STsdb * pRepo; + SDFileSet rSet; // FSET to read + SArray * aBlkIdx; // SBlockIdx array + STable * pTable; // table to read + SBlockIdx * pBlkIdx; // current reading table SBlockIdx + int cidx; + SBlockInfo * pBlkInfo; + SBlockData * pBlkData; // Block info + SAggrBlkData *pAggrBlkData; // Aggregate Block info + SDataCols * pDCols[2]; + void * pBuf; // buffer + void * pCBuf; // compression buffer + void * pExBuf; // extra buffer }; -#define TSDB_READ_REPO(rh) ((rh)->pRepo) -#define TSDB_READ_REPO_ID(rh) REPO_ID(TSDB_READ_REPO(rh)) -#define TSDB_READ_FSET(rh) (&((rh)->rSet)) -#define TSDB_READ_TABLE(rh) ((rh)->pTable) +#define TSDB_READ_REPO(rh) ((rh)->pRepo) +#define TSDB_READ_REPO_ID(rh) REPO_ID(TSDB_READ_REPO(rh)) +#define TSDB_READ_FSET(rh) (&((rh)->rSet)) +#define TSDB_READ_TABLE(rh) ((rh)->pTable) #define TSDB_READ_HEAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_HEAD) #define TSDB_READ_DATA_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_DATA) #define TSDB_READ_LAST_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_LAST) -#define TSDB_READ_BUF(rh) ((rh)->pBuf) -#define TSDB_READ_COMP_BUF(rh) ((rh)->pCBuf) +#define TSDB_READ_SMAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_SMAD) +#define TSDB_READ_SMAL_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_SMAL) +#define TSDB_READ_BUF(rh) ((rh)->pBuf) +#define TSDB_READ_COMP_BUF(rh) ((rh)->pCBuf) +#define TSDB_READ_EXBUF(rh) ((rh)->pExBuf) -#define TSDB_BLOCK_STATIS_SIZE(ncols) (sizeof(SBlockData) + sizeof(SBlockCol) * (ncols) + sizeof(TSCKSUM)) +#define TSDB_BLOCK_STATIS_SIZE(ncols, blkVer) \ + (sizeof(SBlockData) + sizeof(SBlockColV##blkVer) * (ncols) + sizeof(TSCKSUM)) + +static FORCE_INLINE size_t tsdbBlockStatisSize(int nCols, uint32_t blkVer) { + switch (blkVer) { + case TSDB_SBLK_VER_0: + default: + return TSDB_BLOCK_STATIS_SIZE(nCols, 0); + } +} + +#define TSDB_BLOCK_AGGR_SIZE(ncols, blkVer) (sizeof(SAggrBlkColV##blkVer) * (ncols) + sizeof(TSCKSUM)) + +static FORCE_INLINE size_t tsdbBlockAggrSize(int nCols, uint32_t blkVer) { + switch (blkVer) { + case TSDB_SBLK_VER_0: + default: + return TSDB_BLOCK_AGGR_SIZE(nCols, 0); + } +} int tsdbInitReadH(SReadH *pReadh, STsdb *pRepo); void tsdbDestroyReadH(SReadH *pReadh); @@ -147,7 +220,7 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock); int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx); void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx); -void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols); +void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBlock *pBlock); static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) { void * pBuf = *ppBuf; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index f7e4d56fe2..2e7116969e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -50,8 +50,11 @@ typedef struct { #define TSDB_COMMIT_HEAD_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_HEAD) #define TSDB_COMMIT_DATA_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_DATA) #define TSDB_COMMIT_LAST_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_LAST) +#define TSDB_COMMIT_SMAD_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_SMAD) +#define TSDB_COMMIT_SMAL_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_SMAL) #define TSDB_COMMIT_BUF(ch) TSDB_READ_BUF(&((ch)->readh)) #define TSDB_COMMIT_COMP_BUF(ch) TSDB_READ_COMP_BUF(&((ch)->readh)) +#define TSDB_COMMIT_EXBUF(ch) TSDB_READ_EXBUF(&((ch)->readh)) #define TSDB_COMMIT_DEFAULT_ROWS(ch) TSDB_DEFAULT_BLOCK_ROWS(TSDB_COMMIT_REPO(ch)->config.maxRowsPerFileBlock) #define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch))) @@ -509,7 +512,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid // TSDB_FILE_HEAD SDFile *pWHeadf = TSDB_COMMIT_HEAD_FILE(pCommith); tsdbInitDFile(pRepo, pWHeadf, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_HEAD); - if (tsdbCreateDFile(pRepo, pWHeadf, true) < 0) { + if (tsdbCreateDFile(pRepo, pWHeadf, true, TSDB_FILE_HEAD) < 0) { tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWHeadf), tstrerror(terrno)); @@ -560,7 +563,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid tsdbInitDFile(pRepo, pWLastf, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_LAST); pCommith->isLFileSame = false; - if (tsdbCreateDFile(pRepo, pWLastf, true) < 0) { + if (tsdbCreateDFile(pRepo, pWLastf, true, TSDB_FILE_LAST) < 0) { tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), tstrerror(terrno)); @@ -572,6 +575,74 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid } } } + + // TSDB_FILE_SMAD + SDFile *pRSmadF = TSDB_READ_SMAD_FILE(&(pCommith->readh)); + SDFile *pWSmadF = TSDB_COMMIT_SMAD_FILE(pCommith); + + if (access(TSDB_FILE_FULL_NAME(pRSmadF), F_OK) != 0) { + tsdbDebug("vgId:%d create data file %s as not exist", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pRSmadF)); + tsdbInitDFile(pRepo, pWSmadF, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_SMAD); + + if (tsdbCreateDFile(pRepo, pWSmadF, true, TSDB_FILE_SMAD) < 0) { + tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWSmadF), + tstrerror(terrno)); + + tsdbCloseDFileSet(pWSet); + (void)tsdbRemoveDFile(pWHeadf); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + } else { + tsdbInitDFileEx(pWSmadF, pRSmadF); + if (tsdbOpenDFile(pWSmadF, O_RDWR) < 0) { + tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWSmadF), + tstrerror(terrno)); + + tsdbCloseDFileSet(pWSet); + tsdbRemoveDFile(pWHeadf); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + } + + // TSDB_FILE_SMAL + SDFile *pRSmalF = TSDB_READ_SMAL_FILE(&(pCommith->readh)); + SDFile *pWSmalF = TSDB_COMMIT_SMAL_FILE(pCommith); + + if ((pCommith->isLFileSame) && access(TSDB_FILE_FULL_NAME(pRSmalF), F_OK) == 0) { + tsdbInitDFileEx(pWSmalF, pRSmalF); + if (tsdbOpenDFile(pWSmalF, O_RDWR) < 0) { + tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWSmalF), + tstrerror(terrno)); + + tsdbCloseDFileSet(pWSet); + tsdbRemoveDFile(pWHeadf); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + } else { + tsdbDebug("vgId:%d create data file %s as not exist", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pRSmalF)); + tsdbInitDFile(pRepo, pWSmalF, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_SMAL); + + if (tsdbCreateDFile(pRepo, pWSmalF, true, TSDB_FILE_SMAL) < 0) { + tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWSmalF), + tstrerror(terrno)); + + tsdbCloseDFileSet(pWSet); + (void)tsdbRemoveDFile(pWHeadf); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + } } return 0; @@ -1131,41 +1202,57 @@ static int tsdbComparKeyBlock(const void *arg1, const void *arg2) { } } -int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock, bool isLast, - bool isSuper, void **ppBuf, void **ppCBuf) { - STsdbCfg *pCfg = REPO_CFG(pRepo); - SBlockData *pBlockData; - int64_t offset = 0; - int rowsToWrite = pDataCols->numOfRows; +int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDFileAggr, SDataCols *pDataCols, + SBlock *pBlock, bool isLast, bool isSuper, void **ppBuf, void **ppCBuf, void **ppExBuf) { + STsdbCfg * pCfg = REPO_CFG(pRepo); + SBlockData * pBlockData = NULL; + SAggrBlkData *pAggrBlkData = NULL; + int64_t offset = 0, offsetAggr = 0; + int rowsToWrite = pDataCols->numOfRows; ASSERT(rowsToWrite > 0 && rowsToWrite <= pCfg->maxRowsPerFileBlock); ASSERT((!isLast) || rowsToWrite < pCfg->minRowsPerFileBlock); // Make buffer space - if (tsdbMakeRoom(ppBuf, TSDB_BLOCK_STATIS_SIZE(pDataCols->numOfCols)) < 0) { + if (tsdbMakeRoom(ppBuf, tsdbBlockStatisSize(pDataCols->numOfCols, SBlockVerLatest)) < 0) { return -1; } pBlockData = (SBlockData *)(*ppBuf); + if (tsdbMakeRoom(ppExBuf, tsdbBlockAggrSize(pDataCols->numOfCols, SBlockVerLatest)) < 0) { + return -1; + } + pAggrBlkData = (SAggrBlkData *)(*ppExBuf); + // Get # of cols not all NULL(not including key column) int nColsNotAllNull = 0; - for (int ncol = 1; ncol < pDataCols->numOfCols; ncol++) { // ncol from 1, we skip the timestamp column - SDataCol *pDataCol = pDataCols->cols + ncol; - SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull; + for (int ncol = 1; ncol < pDataCols->numOfCols; ++ncol) { // ncol from 1, we skip the timestamp column + SDataCol * pDataCol = pDataCols->cols + ncol; + SBlockCol * pBlockCol = pBlockData->cols + nColsNotAllNull; + SAggrBlkCol *pAggrBlkCol = (SAggrBlkCol *)pAggrBlkData + nColsNotAllNull; if (isAllRowsNull(pDataCol)) { // all data to commit are NULL, just ignore it continue; } memset(pBlockCol, 0, sizeof(*pBlockCol)); + memset(pAggrBlkCol, 0, sizeof(*pAggrBlkCol)); pBlockCol->colId = pDataCol->colId; pBlockCol->type = pDataCol->type; + pAggrBlkCol->colId = pDataCol->colId; + if (tDataTypes[pDataCol->type].statisFunc) { +#if 0 (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pBlockCol->min), &(pBlockCol->max), &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex), &(pBlockCol->numOfNull)); - if (pBlockCol->numOfNull == 0) { +#endif + (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pAggrBlkCol->min), &(pAggrBlkCol->max), + &(pAggrBlkCol->sum), &(pAggrBlkCol->minIndex), &(pAggrBlkCol->maxIndex), + &(pAggrBlkCol->numOfNull)); + + if (pAggrBlkCol->numOfNull == 0) { TD_SET_COL_ROWS_NORM(pBlockCol); } else { TD_SET_COL_ROWS_MISC(pBlockCol); @@ -1181,13 +1268,14 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols * // Compress the data if neccessary int tcol = 0; // counter of not all NULL and written columns uint32_t toffset = 0; - int32_t tsize = TSDB_BLOCK_STATIS_SIZE(nColsNotAllNull); + int32_t tsize = (int32_t)tsdbBlockStatisSize(nColsNotAllNull, SBlockVerLatest); int32_t lsize = tsize; + uint32_t tsizeAggr = (uint32_t)tsdbBlockAggrSize(nColsNotAllNull, SBlockVerLatest); int32_t keyLen = 0; int32_t nBitmaps = (int32_t)TD_BITMAP_BYTES(rowsToWrite); int32_t tBitmaps = 0; - for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) { + for (int ncol = 0; ncol < pDataCols->numOfCols; ++ncol) { // All not NULL columns finish if (ncol != 0 && tcol >= nColsNotAllNull) break; @@ -1248,7 +1336,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols * if (ncol != 0) { tsdbSetBlockColOffset(pBlockCol, toffset); pBlockCol->len = flen; - tcol++; + ++tcol; } else { keyLen = flen; } @@ -1269,6 +1357,18 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols * return -1; } + uint32_t aggrStatus = nColsNotAllNull > 0 ? 1 : 0; + if (aggrStatus > 0) { + + taosCalcChecksumAppend(0, (uint8_t *)pAggrBlkData, tsizeAggr); + tsdbUpdateDFileMagic(pDFileAggr, POINTER_SHIFT(pAggrBlkData, tsizeAggr - sizeof(TSCKSUM))); + + // Write the whole block to file + if (tsdbAppendDFile(pDFileAggr, (void *)pAggrBlkData, tsizeAggr, &offsetAggr) < tsizeAggr) { + return -1; + } + } + // Update pBlock membership vairables pBlock->last = isLast; pBlock->offset = offset; @@ -1280,6 +1380,9 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols * pBlock->numOfCols = nColsNotAllNull; pBlock->keyFirst = dataColsKeyFirst(pDataCols); pBlock->keyLast = dataColsKeyLast(pDataCols); + pBlock->aggrStat = aggrStatus; + pBlock->blkVer = SBlockVerLatest; + pBlock->aggrOffset = (uint64_t)offsetAggr; tsdbDebug("vgId:%d uid:%" PRId64 " a block of data is written to file %s, offset %" PRId64 " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, @@ -1291,9 +1394,10 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols * static int tsdbWriteBlock(SCommitH *pCommith, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock, bool isLast, bool isSuper) { - return tsdbWriteBlockImpl(TSDB_COMMIT_REPO(pCommith), TSDB_COMMIT_TABLE(pCommith), pDFile, pDataCols, pBlock, isLast, - isSuper, (void **)(&(TSDB_COMMIT_BUF(pCommith))), - (void **)(&(TSDB_COMMIT_COMP_BUF(pCommith)))); + return tsdbWriteBlockImpl(TSDB_COMMIT_REPO(pCommith), TSDB_COMMIT_TABLE(pCommith), pDFile, + isLast ? TSDB_COMMIT_SMAL_FILE(pCommith) : TSDB_COMMIT_SMAD_FILE(pCommith), pDataCols, + pBlock, isLast, isSuper, (void **)(&(TSDB_COMMIT_BUF(pCommith))), + (void **)(&(TSDB_COMMIT_COMP_BUF(pCommith))), (void **)(&(TSDB_COMMIT_EXBUF(pCommith)))); } static int tsdbWriteBlockInfo(SCommitH *pCommih) { diff --git a/source/dnode/vnode/src/tsdb/tsdbCompact.c b/source/dnode/vnode/src/tsdb/tsdbCompact.c index 80b59d8756..e24d5974af 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCompact.c +++ b/source/dnode/vnode/src/tsdb/tsdbCompact.c @@ -38,6 +38,8 @@ typedef struct { #define TSDB_COMPACT_HEAD_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_HEAD) #define TSDB_COMPACT_DATA_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_DATA) #define TSDB_COMPACT_LAST_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_LAST) +#define TSDB_COMPACT_SMAD_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_SMAD) +#define TSDB_COMPACT_SMAL_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_SMAL) #define TSDB_COMPACT_BUF(pComph) TSDB_READ_BUF(&((pComph)->readh)) #define TSDB_COMPACT_COMP_BUF(pComph) TSDB_READ_COMP_BUF(&((pComph)->readh)) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 24c765d3e5..dcc9700d83 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -422,7 +422,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { return -1; } - fsheader.version = TSDB_FS_VERSION; + fsheader.version = TSDB_LATEST_SFS_VER; if (taosArrayGetSize(pStatus->df) == 0) { fsheader.len = 0; } else { @@ -697,7 +697,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { ptr = tsdbDecodeFSHeader(ptr, &fsheader); ptr = tsdbDecodeFSMeta(ptr, &(pStatus->meta)); - if (fsheader.version != TSDB_FS_VERSION) { + if (fsheader.version != TSDB_LATEST_SFS_VER) { // TODO: handle file version change } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 5a00fa20a0..b345853ac7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -19,6 +19,8 @@ static const char *TSDB_FNAME_SUFFIX[] = { "head", // TSDB_FILE_HEAD "data", // TSDB_FILE_DATA "last", // TSDB_FILE_LAST + "smad", // TSDB_FILE_SMAD + "smal", // TSDB_FILE_SMAL "", // TSDB_FILE_MAX "meta", // TSDB_FILE_META }; @@ -304,6 +306,7 @@ void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t memset(&(pDFile->info), 0, sizeof(pDFile->info)); pDFile->info.magic = TSDB_FILE_INIT_MAGIC; + pDFile->info.fver = tsdbGetDFSVersion(ftype); tsdbGetFilename(pRepo->vgId, fid, ver, ftype, fname); tfsInitFile(pRepo->pTfs, &(pDFile->f), did, fname); @@ -341,7 +344,7 @@ static int tsdbEncodeSDFileEx(void **buf, SDFile *pDFile) { } static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) { - char *aname; + char *aname = NULL; buf = tsdbDecodeDFInfo(buf, &(pDFile->info)); buf = taosDecodeString(buf, &aname); @@ -352,7 +355,7 @@ static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) { return buf; } -int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) { +int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader, TSDB_FILE_T fType) { ASSERT(pDFile->info.size == 0 && pDFile->info.magic == TSDB_FILE_INIT_MAGIC); pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); @@ -382,6 +385,7 @@ int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) { } pDFile->info.size += TSDB_FILE_HEAD_SIZE; + pDFile->info.fver = tsdbGetDFSVersion(fType); if (tsdbUpdateDFileHeader(pDFile) < 0) { tsdbCloseDFile(pDFile); @@ -493,6 +497,7 @@ static int tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo) { int tlen = 0; tlen += taosEncodeFixedU32(buf, pInfo->magic); + tlen += taosEncodeFixedU32(buf, pInfo->fver); tlen += taosEncodeFixedU32(buf, pInfo->len); tlen += taosEncodeFixedU32(buf, pInfo->totalBlocks); tlen += taosEncodeFixedU32(buf, pInfo->totalSubBlocks); @@ -505,6 +510,7 @@ static int tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo) { static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo) { buf = taosDecodeFixedU32(buf, &(pInfo->magic)); + buf = taosDecodeFixedU32(buf, &(pInfo->fver)); buf = taosDecodeFixedU32(buf, &(pInfo->len)); buf = taosDecodeFixedU32(buf, &(pInfo->totalBlocks)); buf = taosDecodeFixedU32(buf, &(pInfo->totalSubBlocks)); @@ -562,8 +568,10 @@ static int tsdbRollBackDFile(SDFile *pDFile) { // ============== Operations on SDFileSet void tsdbInitDFileSet(STsdb *pRepo, SDFileSet *pSet, SDiskID did, int fid, uint32_t ver) { - pSet->fid = fid; - pSet->state = 0; + TSDB_FSET_FID(pSet) = fid; + TSDB_FSET_VER(pSet) = TSDB_LATEST_FSET_VER; + TSDB_FSET_STATE(pSet) = 0; + pSet->reserve = 0; for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { SDFile *pDFile = TSDB_DFILE_IN_SET(pSet, ftype); @@ -572,7 +580,7 @@ void tsdbInitDFileSet(STsdb *pRepo, SDFileSet *pSet, SDiskID did, int fid, uint3 } void tsdbInitDFileSetEx(SDFileSet *pSet, SDFileSet *pOSet) { - pSet->fid = pOSet->fid; + TSDB_FSET_FID(pSet) = TSDB_FSET_FID(pOSet); for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { tsdbInitDFileEx(TSDB_DFILE_IN_SET(pSet, ftype), TSDB_DFILE_IN_SET(pOSet, ftype)); } @@ -581,7 +589,10 @@ void tsdbInitDFileSetEx(SDFileSet *pSet, SDFileSet *pOSet) { int tsdbEncodeDFileSet(void **buf, SDFileSet *pSet) { int tlen = 0; - tlen += taosEncodeFixedI32(buf, pSet->fid); + tlen += taosEncodeFixedI32(buf, TSDB_FSET_FID(pSet)); + // state not included + tlen += taosEncodeFixedU8(buf, TSDB_FSET_VER(pSet)); + tlen += taosEncodeFixedU16(buf, pSet->reserve); for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { tlen += tsdbEncodeSDFile(buf, TSDB_DFILE_IN_SET(pSet, ftype)); } @@ -590,11 +601,11 @@ int tsdbEncodeDFileSet(void **buf, SDFileSet *pSet) { } void *tsdbDecodeDFileSet(STsdb *pRepo, void *buf, SDFileSet *pSet) { - int32_t fid; + buf = taosDecodeFixedI32(buf, &(TSDB_FSET_FID(pSet))); + TSDB_FSET_STATE(pSet) = 0; + buf = taosDecodeFixedU8(buf, &(TSDB_FSET_VER(pSet))); + buf = taosDecodeFixedU16(buf, &(pSet->reserve)); - buf = taosDecodeFixedI32(buf, &(fid)); - pSet->state = 0; - pSet->fid = fid; for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { buf = tsdbDecodeSDFile(pRepo, buf, TSDB_DFILE_IN_SET(pSet, ftype)); } @@ -604,7 +615,9 @@ void *tsdbDecodeDFileSet(STsdb *pRepo, void *buf, SDFileSet *pSet) { int tsdbEncodeDFileSetEx(void **buf, SDFileSet *pSet) { int tlen = 0; - tlen += taosEncodeFixedI32(buf, pSet->fid); + tlen += taosEncodeFixedI32(buf, TSDB_FSET_FID(pSet)); + tlen += taosEncodeFixedU8(buf, TSDB_FSET_VER(pSet)); + tlen += taosEncodeFixedU16(buf, pSet->reserve); for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { tlen += tsdbEncodeSDFileEx(buf, TSDB_DFILE_IN_SET(pSet, ftype)); } @@ -613,10 +626,10 @@ int tsdbEncodeDFileSetEx(void **buf, SDFileSet *pSet) { } void *tsdbDecodeDFileSetEx(void *buf, SDFileSet *pSet) { - int32_t fid; + buf = taosDecodeFixedI32(buf, &(TSDB_FSET_FID(pSet))); + buf = taosDecodeFixedU8(buf, &(TSDB_FSET_VER(pSet))); + buf = taosDecodeFixedU16(buf, &(pSet->reserve)); - buf = taosDecodeFixedI32(buf, &(fid)); - pSet->fid = fid; for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { buf = tsdbDecodeSDFileEx(buf, TSDB_DFILE_IN_SET(pSet, ftype)); } @@ -637,7 +650,7 @@ int tsdbApplyDFileSetChange(SDFileSet *from, SDFileSet *to) { int tsdbCreateDFileSet(STsdb *pRepo, SDFileSet *pSet, bool updateHeader) { for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - if (tsdbCreateDFile(pRepo, TSDB_DFILE_IN_SET(pSet, ftype), updateHeader) < 0) { + if (tsdbCreateDFile(pRepo, TSDB_DFILE_IN_SET(pSet, ftype), updateHeader, ftype) < 0) { tsdbCloseDFileSet(pSet); tsdbRemoveDFileSet(pSet); return -1; diff --git a/source/dnode/vnode/src/tsdb/tsdbMain.c b/source/dnode/vnode/src/tsdb/tsdbMain.c index 812ec67ec4..2d8c470113 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMain.c +++ b/source/dnode/vnode/src/tsdb/tsdbMain.c @@ -821,9 +821,10 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea // file block with sub-blocks has no statistics data if (pBlock->numOfSubBlocks <= 1) { - tsdbLoadBlockStatis(pReadh, pBlock); - tsdbGetBlockStatis(pReadh, pBlockStatis, (int)numColumns); - loadStatisData = true; + if (tsdbLoadBlockStatis(pReadh, pBlock) == TSDB_STATIS_OK) { + tsdbGetBlockStatis(pReadh, pBlockStatis, (int)numColumns, pBlock); + loadStatisData = true; + } } for (int16_t i = 0; i < numColumns && numColumns > pTable->restoreColumnNum; ++i) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index c1cf776cb5..4bd0ee959e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3276,8 +3276,12 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT* pTsdbReadHandle, SDataStati } int64_t stime = taosGetTimestampUs(); - if (tsdbLoadBlockStatis(&pHandle->rhelper, pBlockInfo->compBlock) < 0) { + int statisStatus = tsdbLoadBlockStatis(&pHandle->rhelper, pBlockInfo->compBlock); + if (statisStatus < TSDB_STATIS_OK) { return terrno; + } else if (statisStatus > TSDB_STATIS_OK) { + *pBlockStatis = NULL; + return TSDB_CODE_SUCCESS; } int16_t* colIds = pHandle->defaultLoadColumn->pData; @@ -3288,7 +3292,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT* pTsdbReadHandle, SDataStati pHandle->statis[i].colId = colIds[i]; } - tsdbGetBlockStatis(&pHandle->rhelper, pHandle->statis, (int)numOfCols); + tsdbGetBlockStatis(&pHandle->rhelper, pHandle->statis, (int)numOfCols, pBlockInfo->compBlock); // always load the first primary timestamp column data SDataStatis* pPrimaryColStatis = &pHandle->statis[0]; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index 7e8e7866d7..53d7c81752 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -63,10 +63,12 @@ int tsdbInitReadH(SReadH *pReadh, STsdb *pRepo) { void tsdbDestroyReadH(SReadH *pReadh) { if (pReadh == NULL) return; + pReadh->pExBuf = taosTZfree(pReadh->pExBuf); pReadh->pCBuf = taosTZfree(pReadh->pCBuf); pReadh->pBuf = taosTZfree(pReadh->pBuf); pReadh->pDCols[0] = tdFreeDataCols(pReadh->pDCols[0]); pReadh->pDCols[1] = tdFreeDataCols(pReadh->pDCols[1]); + pReadh->pAggrBlkData = taosTZfree(pReadh->pAggrBlkData); pReadh->pBlkData = taosTZfree(pReadh->pBlkData); pReadh->pBlkInfo = taosTZfree(pReadh->pBlkInfo); pReadh->cidx = 0; @@ -305,39 +307,45 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) { ASSERT(pBlock->numOfSubBlocks <= 1); - SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh); + if (!pBlock->aggrStat) { + return TSDB_STATIS_NONE; + } - if (tsdbSeekDFile(pDFile, pBlock->offset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to load block statis part while seek file %s to offset %" PRId64 " since %s", - TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tstrerror(terrno)); + SDFile *pDFileAggr = pBlock->last ? TSDB_READ_SMAL_FILE(pReadh) : TSDB_READ_SMAD_FILE(pReadh); + + if (tsdbSeekDFile(pDFileAggr, pBlock->aggrOffset, SEEK_SET) < 0) { + tsdbError("vgId:%d failed to load block aggr part while seek file %s to offset %" PRIu64 " since %s", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), (uint64_t)pBlock->aggrOffset, + tstrerror(terrno)); return -1; } - size_t size = TSDB_BLOCK_STATIS_SIZE(pBlock->numOfCols); - if (tsdbMakeRoom((void **)(&(pReadh->pBlkData)), size) < 0) return -1; + size_t sizeAggr = tsdbBlockAggrSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer); + if (tsdbMakeRoom((void **)(&(pReadh->pAggrBlkData)), sizeAggr) < 0) return -1; - int64_t nread = tsdbReadDFile(pDFile, (void *)(pReadh->pBlkData), size); - if (nread < 0) { - tsdbError("vgId:%d failed to load block statis part while read file %s since %s, offset:%" PRId64 " len :%" PRIzu, - TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), (int64_t)pBlock->offset, size); + int64_t nreadAggr = tsdbReadDFile(pDFileAggr, (void *)(pReadh->pAggrBlkData), sizeAggr); + if (nreadAggr < 0) { + tsdbError("vgId:%d failed to load block aggr part while read file %s since %s, offset:%" PRIu64 " len :%" PRIzu, + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), tstrerror(terrno), + (uint64_t)pBlock->aggrOffset, sizeAggr); return -1; } - if (nread < size) { + if (nreadAggr < sizeAggr) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - tsdbError("vgId:%d block statis part in file %s is corrupted, offset:%" PRId64 " expected bytes:%" PRIzu + tsdbError("vgId:%d block aggr part in file %s is corrupted, offset:%" PRIu64 " expected bytes:%" PRIzu " read bytes: %" PRId64, - TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size, nread); + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), (uint64_t)pBlock->aggrOffset, sizeAggr, + nreadAggr); return -1; } - if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pBlkData), (uint32_t)size)) { + if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pAggrBlkData), (uint32_t)sizeAggr)) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - tsdbError("vgId:%d block statis part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%" PRIzu, - TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size); + tsdbError("vgId:%d block aggr part in file %s is corrupted since wrong checksum, offset:%" PRIu64 " len :%" PRIzu, + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), (uint64_t)pBlock->aggrOffset, sizeAggr); return -1; } - return 0; } @@ -375,7 +383,8 @@ void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx) { return buf; } -void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols) { +void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBlock *pBlock) { +#ifdef TD_REFACTOR_3 SBlockData *pBlockData = pReadh->pBlkData; for (int i = 0, j = 0; i < numOfCols;) { @@ -401,6 +410,36 @@ void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols) { j++; } } +#else + if (pBlock->aggrStat) { + SAggrBlkData *pAggrBlkData = pReadh->pAggrBlkData; + + for (int i = 0, j = 0; i < numOfCols;) { + if (j >= pBlock->numOfCols) { + pStatis[i].numOfNull = -1; + i++; + continue; + } + SAggrBlkCol *pAggrBlkCol = ((SAggrBlkCol *)(pAggrBlkData)) + j; + if (pStatis[i].colId == pAggrBlkCol->colId) { + pStatis[i].sum = pAggrBlkCol->sum; + pStatis[i].max = pAggrBlkCol->max; + pStatis[i].min = pAggrBlkCol->min; + pStatis[i].maxIndex = pAggrBlkCol->maxIndex; + pStatis[i].minIndex = pAggrBlkCol->minIndex; + pStatis[i].numOfNull = pAggrBlkCol->numOfNull; + i++; + j++; + } else if (pStatis[i].colId < pAggrBlkCol->colId) { + pStatis[i].numOfNull = -1; + i++; + } else { + j++; + } + } + } + +#endif } static void tsdbResetReadTable(SReadH *pReadh) { @@ -449,7 +488,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat return -1; } - int32_t tsize = TSDB_BLOCK_STATIS_SIZE(pBlock->numOfCols); + int32_t tsize = (int32_t)tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer); if (!taosCheckChecksumWhole((uint8_t *)TSDB_READ_BUF(pReadh), tsize)) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; tsdbError("vgId:%d block statis part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%d", @@ -686,7 +725,8 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlockCol->len) < 0) return -1; if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), tsize) < 0) return -1; - int64_t offset = pBlock->offset + TSDB_BLOCK_STATIS_SIZE(pBlock->numOfCols) + tsdbGetBlockColOffset(pBlockCol); + int64_t offset = pBlock->offset + tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer) + + tsdbGetBlockColOffset(pBlockCol); if (tsdbSeekDFile(pDFile, offset, SEEK_SET) < 0) { tsdbError("vgId:%d failed to load block column data while seek file %s to offset %" PRId64 " since %s", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), offset, tstrerror(terrno)); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index c8cafb93dd..176290ba38 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -345,6 +345,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO, "Invalid information t TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_AVAIL_DISK, "No available disk") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_MESSED_MSG, "TSDB messed message") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVLD_TAG_VAL, "TSDB invalid tag value") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_CACHE_LAST_ROW, "TSDB no cache last row data") // query TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, "Invalid handle") From 2fbddc9a81df01b7ad0c58f3269db18499a56635 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 2 Mar 2022 17:53:55 +0800 Subject: [PATCH 02/17] refactor the SBlock --- source/dnode/vnode/src/inc/tsdbReadImpl.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdbReadImpl.h b/source/dnode/vnode/src/inc/tsdbReadImpl.h index 60e61bc57f..6690190381 100644 --- a/source/dnode/vnode/src/inc/tsdbReadImpl.h +++ b/source/dnode/vnode/src/inc/tsdbReadImpl.h @@ -48,7 +48,8 @@ typedef struct { int16_t numOfCols; // not including timestamp column TSKEY keyFirst; TSKEY keyLast; -} SBlock; +} SBlockX; + #else typedef enum { @@ -59,9 +60,6 @@ typedef enum { #define SBlockVerLatest TSDB_SBLK_VER_0 typedef struct { - int64_t offset; - int32_t algorithm : 8; - int32_t numOfRows : 24; uint8_t last : 1; uint8_t blkVer : 7; uint8_t numOfSubBlocks; @@ -69,6 +67,9 @@ typedef struct { uint32_t len; // data block length uint32_t keyLen : 24; // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols uint32_t reserve : 8; + int32_t algorithm : 8; + int32_t numOfRows : 24; + int64_t offset; uint64_t aggrStat : 1; uint64_t aggrOffset : 63; TSKEY keyFirst; From 65439839f2ae7caa385c94e0874e236de8b70c9a Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 2 Mar 2022 18:17:59 +0800 Subject: [PATCH 03/17] add method tsdbLoadBlockOffset --- source/dnode/vnode/src/inc/tsdbReadImpl.h | 3 +- source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 38 +++++++++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdbReadImpl.h b/source/dnode/vnode/src/inc/tsdbReadImpl.h index 6690190381..171d003e3b 100644 --- a/source/dnode/vnode/src/inc/tsdbReadImpl.h +++ b/source/dnode/vnode/src/inc/tsdbReadImpl.h @@ -48,7 +48,7 @@ typedef struct { int16_t numOfCols; // not including timestamp column TSKEY keyFirst; TSKEY keyLast; -} SBlockX; +} SBlock; #else @@ -219,6 +219,7 @@ int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget); int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlockInfo); int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, int numOfColsIds); int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock); +int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock); int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx); void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx); void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBlock *pBlock); diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index 53d7c81752..2ca5e948a6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -349,6 +349,42 @@ int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) { return 0; } +int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock) { + ASSERT(pBlock->numOfSubBlocks <= 1); + SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh); + if (tsdbSeekDFile(pDFile, pBlock->offset, SEEK_SET) < 0) { + tsdbError("vgId:%d failed to load block statis part while seek file %s to offset %" PRId64 " since %s", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tstrerror(terrno)); + return -1; + } + + size_t size = tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer); + if (tsdbMakeRoom((void **)(&(pReadh->pBlkData)), size) < 0) return -1; + + int64_t nread = tsdbReadDFile(pDFile, (void *)(pReadh->pBlkData), size); + if (nread < 0) { + tsdbError("vgId:%d failed to load block statis part while read file %s since %s, offset:%" PRId64 " len :%" PRIzu, + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), (int64_t)pBlock->offset, size); + return -1; + } + + if (nread < size) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tsdbError("vgId:%d block statis part in file %s is corrupted, offset:%" PRId64 " expected bytes:%" PRIzu + " read bytes: %" PRId64, + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size, nread); + return -1; + } + + if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pBlkData), (uint32_t)size)) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tsdbError("vgId:%d block statis part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%" PRIzu, + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size); + return -1; + } + return 0; +} + int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx) { int tlen = 0; @@ -631,7 +667,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * tdResetDataCols(pDataCols); // If only load timestamp column, no need to load SBlockData part - if (numOfColIds > 1 && tsdbLoadBlockStatis(pReadh, pBlock) < 0) return -1; + if (numOfColIds > 1 && tsdbLoadBlockOffset(pReadh, pBlock) < 0) return -1; pDataCols->numOfRows = pBlock->numOfRows; From 740e8db4cfaa41750afd555bc4f81350aa25ccc7 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 2 Mar 2022 18:20:11 +0800 Subject: [PATCH 04/17] set method tsdbLoadBlockOffset static --- source/dnode/vnode/src/inc/tsdbReadImpl.h | 1 - source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdbReadImpl.h b/source/dnode/vnode/src/inc/tsdbReadImpl.h index 171d003e3b..16eb55967c 100644 --- a/source/dnode/vnode/src/inc/tsdbReadImpl.h +++ b/source/dnode/vnode/src/inc/tsdbReadImpl.h @@ -219,7 +219,6 @@ int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget); int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlockInfo); int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, int numOfColsIds); int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock); -int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock); int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx); void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx); void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBlock *pBlock); diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index 2ca5e948a6..40cdcba697 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -19,6 +19,7 @@ static void tsdbResetReadTable(SReadH *pReadh); static void tsdbResetReadFile(SReadH *pReadh); +static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock); static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols); static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows, int numOfBitmaps, int lenOfBitmaps, int maxPoints, char *buffer, @@ -349,7 +350,7 @@ int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) { return 0; } -int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock) { +static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock) { ASSERT(pBlock->numOfSubBlocks <= 1); SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh); if (tsdbSeekDFile(pDFile, pBlock->offset, SEEK_SET) < 0) { From 8ad0d705b8349bc2e125ebb0888c20a5aad11616 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 2 Mar 2022 18:29:38 +0800 Subject: [PATCH 05/17] refactor --- source/dnode/vnode/src/inc/tsdbFile.h | 2 +- source/dnode/vnode/src/tsdb/tsdbFile.c | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/inc/tsdbFile.h b/source/dnode/vnode/src/inc/tsdbFile.h index 370de0ab0a..bbeb6c6a0f 100644 --- a/source/dnode/vnode/src/inc/tsdbFile.h +++ b/source/dnode/vnode/src/inc/tsdbFile.h @@ -51,9 +51,9 @@ typedef enum { TSDB_FILE_SMAD, // .smad(Block-wise SMA) TSDB_FILE_SMAL, // .smal(Block-wise SMA) TSDB_FILE_MAX, // + TSDB_FILE_META, // meta TSDB_FILE_TSMA, // .tsma.${sma_index_name}, Time-range-wise SMA TSDB_FILE_RSMA, // .rsma.${sma_index_name}, Time-range-wise Rollup SMA - TSDB_FILE_META // meta } TSDB_FILE_T; typedef enum { diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index b345853ac7..4d70c429ff 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -23,6 +23,8 @@ static const char *TSDB_FNAME_SUFFIX[] = { "smal", // TSDB_FILE_SMAL "", // TSDB_FILE_MAX "meta", // TSDB_FILE_META + "tsma", // TSDB_FILE_TSMA + "rsma", // TSDB_FILE_RSMA }; static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname); From b9f9aca900a38db3062fd7f918c80982ee2eedef Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 2 Mar 2022 19:08:59 +0800 Subject: [PATCH 06/17] trigger CI From 336f4500b9b7b76886cf1363ced79c712597fc22 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 2 Mar 2022 19:31:11 +0800 Subject: [PATCH 07/17] minor change --- source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index 40cdcba697..f6827eaae1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -427,7 +427,7 @@ void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBl for (int i = 0, j = 0; i < numOfCols;) { if (j >= pBlockData->numOfCols) { pStatis[i].numOfNull = -1; - i++; + ++i; continue; } @@ -438,13 +438,13 @@ void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBl pStatis[i].maxIndex = pBlockData->cols[j].maxIndex; pStatis[i].minIndex = pBlockData->cols[j].minIndex; pStatis[i].numOfNull = pBlockData->cols[j].numOfNull; - i++; - j++; + ++i; + ++j; } else if (pStatis[i].colId < pBlockData->cols[j].colId) { pStatis[i].numOfNull = -1; - i++; + ++i; } else { - j++; + ++j; } } #else @@ -454,7 +454,7 @@ void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBl for (int i = 0, j = 0; i < numOfCols;) { if (j >= pBlock->numOfCols) { pStatis[i].numOfNull = -1; - i++; + ++i; continue; } SAggrBlkCol *pAggrBlkCol = ((SAggrBlkCol *)(pAggrBlkData)) + j; @@ -465,13 +465,13 @@ void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBl pStatis[i].maxIndex = pAggrBlkCol->maxIndex; pStatis[i].minIndex = pAggrBlkCol->minIndex; pStatis[i].numOfNull = pAggrBlkCol->numOfNull; - i++; - j++; + ++i; + ++j; } else if (pStatis[i].colId < pAggrBlkCol->colId) { pStatis[i].numOfNull = -1; - i++; + ++i; } else { - j++; + ++j; } } } From a2efee8ec94351fa592c7a42cb0b39891160b5a3 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 2 Mar 2022 19:38:40 +0800 Subject: [PATCH 08/17] trigger CI From ffeba1b1d4e4d469c00410ecac89f8faf82eca0a Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 4 Mar 2022 15:09:16 +0800 Subject: [PATCH 09/17] add STSma defintion --- cmake/stub_CMakeLists.txt.in | 20 ++-- include/common/tmsg.h | 129 ++++++++++++++++++++++ include/util/tdef.h | 1 + source/dnode/vnode/src/meta/metaBDBImpl.c | 1 + source/dnode/vnode/src/vnd/vnodeWrite.c | 27 +++-- source/dnode/vnode/test/CMakeLists.txt | 53 ++++++--- source/dnode/vnode/test/tsdbSmaTest.cpp | 76 +++++++++++++ 7 files changed, 268 insertions(+), 39 deletions(-) create mode 100644 source/dnode/vnode/test/tsdbSmaTest.cpp diff --git a/cmake/stub_CMakeLists.txt.in b/cmake/stub_CMakeLists.txt.in index e227e820d6..c768239006 100644 --- a/cmake/stub_CMakeLists.txt.in +++ b/cmake/stub_CMakeLists.txt.in @@ -1,12 +1,12 @@ # stub -ExternalProject_Add(stub - GIT_REPOSITORY https://github.com/coolxv/cpp-stub.git - GIT_SUBMODULES "src" - SOURCE_DIR "${CMAKE_CONTRIB_DIR}/cpp-stub" - BINARY_DIR "${CMAKE_CONTRIB_DIR}/cpp-stub/src" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" - TEST_COMMAND "" -) +#ExternalProject_Add(stub +# GIT_REPOSITORY https://github.com/coolxv/cpp-stub.git +# GIT_SUBMODULES "src" +# SOURCE_DIR "${CMAKE_CONTRIB_DIR}/cpp-stub" +# BINARY_DIR "${CMAKE_CONTRIB_DIR}/cpp-stub/src" +# CONFIGURE_COMMAND "" +# BUILD_COMMAND "" +# INSTALL_COMMAND "" +# TEST_COMMAND "" +#) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index ab0472a575..0dd7a513d6 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1834,6 +1834,135 @@ static FORCE_INLINE void* tDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW) return buf; } +typedef struct { + uint8_t version; // for compatibility + uint8_t intervalUnit; + uint8_t slidingUnit; + char indexName[TSDB_INDEX_NAME_LEN + 1]; + col_id_t numOfColIds; + uint16_t numOfFuncIds; + uint64_t tableUid; // super/common table uid + int64_t interval; + int64_t sliding; + col_id_t* colIds; // N.B. sorted column ids + uint16_t* funcIds; // N.B. sorted sma function ids +} STSma; // Time-range-wise SMA + +typedef struct { + uint32_t number; + STSma* tSma; +} STSmaWrapper; + +static FORCE_INLINE void tdDestroyTSma(STSma* pSma, bool releaseSelf) { + if (pSma) { + tfree(pSma->colIds); + tfree(pSma->funcIds); + if (releaseSelf) { + free(pSma); + } + } +} + +static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) { + int32_t tlen = 0; + + tlen += taosEncodeFixedU8(buf, pSma->version); + tlen += taosEncodeFixedU8(buf, pSma->intervalUnit); + tlen += taosEncodeFixedU8(buf, pSma->slidingUnit); + tlen += taosEncodeString(buf, pSma->indexName); + tlen += taosEncodeFixedU16(buf, pSma->numOfColIds); + tlen += taosEncodeFixedU16(buf, pSma->numOfFuncIds); + tlen += taosEncodeFixedU64(buf, pSma->tableUid); + tlen += taosEncodeFixedI64(buf, pSma->interval); + tlen += taosEncodeFixedI64(buf, pSma->sliding); + + for (uint16_t i = 0; i < pSma->numOfColIds; ++i) { + tlen += taosEncodeFixedU16(buf, *(pSma->colIds + i)); + } + for (uint16_t i = 0; i < pSma->numOfFuncIds; ++i) { + tlen += taosEncodeFixedU16(buf, *(pSma->funcIds + i)); + } + return tlen; +} + +static FORCE_INLINE int32_t tEncodeTSmaWrapper(void** buf, const STSmaWrapper* pSW) { + int32_t tlen = 0; + + tlen += taosEncodeFixedU32(buf, pSW->number); + for (uint32_t i = 0; i < pSW->number; ++i) { + tlen += tEncodeTSma(buf, pSW->tSma + i); + } + return tlen; +} + +static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) { + buf = taosDecodeFixedU8(buf, &pSma->version); + buf = taosDecodeFixedU8(buf, &pSma->intervalUnit); + buf = taosDecodeFixedU8(buf, &pSma->slidingUnit); + buf = taosDecodeStringTo(buf, pSma->indexName); + buf = taosDecodeFixedU16(buf, &pSma->numOfColIds); + buf = taosDecodeFixedU16(buf, &pSma->numOfFuncIds); + buf = taosDecodeFixedU64(buf, &pSma->tableUid); + buf = taosDecodeFixedI64(buf, &pSma->interval); + buf = taosDecodeFixedI64(buf, &pSma->sliding); + + if (pSma->numOfColIds > 0) { + pSma->colIds = (col_id_t*)calloc(pSma->numOfColIds, sizeof(STSma)); + if (pSma->colIds == NULL) { + return NULL; + } + for (uint16_t i = 0; i < pSma->numOfColIds; ++i) { + buf = taosDecodeFixedU16(buf, pSma->colIds + i); + } + } + + if (pSma->numOfFuncIds > 0) { + pSma->funcIds = (uint16_t*)calloc(pSma->numOfFuncIds, sizeof(STSma)); + if (pSma->funcIds == NULL) { + return NULL; + } + for (uint16_t i = 0; i < pSma->numOfFuncIds; ++i) { + buf = taosDecodeFixedU16(buf, pSma->funcIds + i); + } + } + + return buf; +} + +static FORCE_INLINE void* tDecodeTSmaWrapper(void* buf, STSmaWrapper* pSW) { + buf = taosDecodeFixedU32(buf, &pSW->number); + + pSW->tSma = (STSma*)calloc(pSW->number, sizeof(STSma)); + if (pSW->tSma == NULL) { + return NULL; + } + + for (uint32_t i = 0; i < pSW->number; ++i) { + if ((buf = tDecodeTSma(buf, pSW->tSma + i)) == NULL) { + for (uint32_t j = i; j >= 0; --i) { + tdDestroyTSma(pSW->tSma + j, false); + } + free(pSW->tSma); + return NULL; + } + } + return buf; +} + +// RSma: Time-range-wise Rollup SMA +typedef struct { + int64_t interval; + int32_t retention; // unit: day + uint16_t days; // unit: day + int8_t intervalUnit; +} SSmaParams; + +typedef struct { + STSma tsma; + float xFilesFactor; + SArray* smaParams; // SSmaParams +} SRSma; + typedef struct { int64_t uid; int32_t numOfRows; diff --git a/include/util/tdef.h b/include/util/tdef.h index a1b4fc11cf..9695c2e4c8 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -206,6 +206,7 @@ typedef enum ELogicConditionType { #define TSDB_FUNC_TYPE_AGGREGATE 2 #define TSDB_FUNC_MAX_RETRIEVE 1024 +#define TSDB_INDEX_NAME_LEN 32 #define TSDB_TYPE_STR_MAX_LEN 32 #define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index 3c464391d2..91a7a91f3a 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -38,6 +38,7 @@ struct SMetaDB { // DB DB *pTbDB; DB *pSchemaDB; + // IDX DB *pNameIdx; DB *pStbIdx; diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index 81eb09f48f..a2616307ff 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -41,10 +41,8 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { return 0; } -int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - SVCreateTbReq vCreateTbReq; - SVCreateTbBatchReq vCreateTbBatchReq; - void *ptr = NULL; +int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { + void *ptr = NULL; if (pVnode->config.streamMode == 0) { ptr = vnodeMalloc(pVnode, pMsg->contLen); @@ -64,7 +62,8 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } switch (pMsg->msgType) { - case TDMT_VND_CREATE_STB: + case TDMT_VND_CREATE_STB: { + SVCreateTbReq vCreateTbReq = {0}; tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbReq); if (metaCreateTable(pVnode->pMeta, &(vCreateTbReq)) < 0) { // TODO: handle error @@ -75,7 +74,9 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { free(vCreateTbReq.stbCfg.pTagSchema); free(vCreateTbReq.name); break; - case TDMT_VND_CREATE_TABLE: + } + case TDMT_VND_CREATE_TABLE: { + SVCreateTbBatchReq vCreateTbBatchReq = {0}; tDeserializeSVCreateTbBatchReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbBatchReq); for (int i = 0; i < taosArrayGetSize(vCreateTbBatchReq.pArray); i++) { SVCreateTbReq *pCreateTbReq = taosArrayGet(vCreateTbBatchReq.pArray, i); @@ -97,14 +98,16 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { vTrace("vgId:%d process create %" PRIzu " tables", pVnode->vgId, taosArrayGetSize(vCreateTbBatchReq.pArray)); taosArrayDestroy(vCreateTbBatchReq.pArray); break; - - case TDMT_VND_ALTER_STB: + } + case TDMT_VND_ALTER_STB: { + SVCreateTbReq vAlterTbReq = {0}; vTrace("vgId:%d, process alter stb req", pVnode->vgId); - tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbReq); - free(vCreateTbReq.stbCfg.pSchema); - free(vCreateTbReq.stbCfg.pTagSchema); - free(vCreateTbReq.name); + tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vAlterTbReq); + free(vAlterTbReq.stbCfg.pSchema); + free(vAlterTbReq.stbCfg.pTagSchema); + free(vAlterTbReq.name); break; + } case TDMT_VND_DROP_STB: vTrace("vgId:%d, process drop stb req", pVnode->vgId); break; diff --git a/source/dnode/vnode/test/CMakeLists.txt b/source/dnode/vnode/test/CMakeLists.txt index 6b468497d5..40d016300e 100644 --- a/source/dnode/vnode/test/CMakeLists.txt +++ b/source/dnode/vnode/test/CMakeLists.txt @@ -1,20 +1,39 @@ -add_executable(tqTest "") -target_sources(tqTest - PRIVATE - "tqMetaTest.cpp" -) -target_include_directories(tqTest - PUBLIC - "${CMAKE_SOURCE_DIR}/include/server/vnode/tq" - "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +MESSAGE(STATUS "vnode unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +# add_executable(tqTest "") +# target_sources(tqTest +# PRIVATE +# "tqMetaTest.cpp" +# ) +# target_include_directories(tqTest +# PUBLIC +# "${CMAKE_SOURCE_DIR}/include/server/vnode/tq" +# "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +# ) + +# target_link_libraries(tqTest +# tq +# gtest_main +# ) +# enable_testing() +# add_test( +# NAME tq_test +# COMMAND tqTest +# ) + +ADD_EXECUTABLE(tsdbSmaTest tsdbSmaTest.cpp) +TARGET_LINK_LIBRARIES( + tsdbSmaTest + PUBLIC os util common vnode gtest_main ) -target_link_libraries(tqTest - tq - gtest_main -) -enable_testing() -add_test( - NAME tq_test - COMMAND tqTest +TARGET_INCLUDE_DIRECTORIES( + tsdbSmaTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/common" + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../src/inc" + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp new file mode 100644 index 0000000000..90158b68b6 --- /dev/null +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +#include "tsdbDef.h" + + + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#if 0 +TEST(testCase, tSmaInsertTest) { + STSma tSma = {0}; + STSmaData* pSmaData = NULL; + STsdb tsdb = {0}; + + // init + tSma.intervalUnit = TD_TIME_UNIT_DAY; + tSma.interval = 1; + tSma.numOfFuncIds = 5; // sum/min/max/avg/last + + int32_t blockSize = tSma.numOfFuncIds * sizeof(int64_t); + int32_t numOfColIds = 3; + int32_t numOfSmaBlocks = 10; + + int32_t dataLen = numOfColIds * numOfSmaBlocks * blockSize; + + pSmaData = (STSmaData*)malloc(sizeof(STSmaData) + dataLen); + ASSERT_EQ(pSmaData != NULL, true); + pSmaData->tableUid = 3232329230; + pSmaData->numOfColIds = numOfColIds; + pSmaData->numOfSmaBlocks = numOfSmaBlocks; + pSmaData->dataLen = dataLen; + pSmaData->tsWindow.skey = 1640000000; + pSmaData->tsWindow.ekey = 1645788649; + pSmaData->colIds = (col_id_t*)malloc(sizeof(col_id_t) * numOfColIds); + ASSERT_EQ(pSmaData->colIds != NULL, true); + + for (int32_t i = 0; i < numOfColIds; ++i) { + *(pSmaData->colIds + i) = (i + PRIMARYKEY_TIMESTAMP_COL_ID); + } + + // execute + EXPECT_EQ(tsdbInsertTSmaData(&tsdb, &tSma, pSmaData), TSDB_CODE_SUCCESS); + + // release + tdDestroySmaData(pSmaData); +} +#endif + +#pragma GCC diagnostic pop \ No newline at end of file From 5443bb0daeadf98f99bd7b22b775409192f0da02 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 5 Mar 2022 09:29:55 +0800 Subject: [PATCH 10/17] add STSma schema encode/decode --- include/common/tmsg.h | 42 ++++++++----- source/dnode/vnode/CMakeLists.txt | 2 +- source/dnode/vnode/src/inc/tsdbCommit.h | 8 +++ source/dnode/vnode/src/inc/tsdbFS.h | 8 +++ source/dnode/vnode/src/inc/tsdbFile.h | 8 +++ source/dnode/vnode/src/inc/tsdbLog.h | 8 +++ source/dnode/vnode/src/inc/tsdbMemory.h | 7 +++ source/dnode/vnode/src/inc/tsdbReadImpl.h | 8 +++ source/dnode/vnode/src/meta/metaBDBImpl.c | 7 +++ source/dnode/vnode/test/CMakeLists.txt | 2 +- source/dnode/vnode/test/tsdbSmaTest.cpp | 74 +++++++++++++++++++++-- 11 files changed, 151 insertions(+), 23 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 0dd7a513d6..f3ba1ec17f 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1833,7 +1833,20 @@ static FORCE_INLINE void* tDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW) } return buf; } - +typedef enum { + TD_TIME_UNIT_UNKNOWN = -1, + TD_TIME_UNIT_YEAR = 0, + TD_TIME_UNIT_SEASON = 1, + TD_TIME_UNIT_MONTH = 2, + TD_TIME_UNIT_WEEK = 3, + TD_TIME_UNIT_DAY = 4, + TD_TIME_UNIT_HOUR = 5, + TD_TIME_UNIT_MINUTE = 6, + TD_TIME_UNIT_SEC = 7, + TD_TIME_UNIT_MILLISEC = 8, + TD_TIME_UNIT_MICROSEC = 9, + TD_TIME_UNIT_NANOSEC = 10 +} ETDTimeUnit; typedef struct { uint8_t version; // for compatibility uint8_t intervalUnit; @@ -1863,6 +1876,15 @@ static FORCE_INLINE void tdDestroyTSma(STSma* pSma, bool releaseSelf) { } } +static FORCE_INLINE void tdDestroyWrapper(STSmaWrapper* pSW) { + if (pSW && pSW->tSma) { + for (uint32_t i = 0; i < pSW->number; ++i) { + tdDestroyTSma(pSW->tSma + i, false); + } + tfree(pSW->tSma); + } +} + static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) { int32_t tlen = 0; @@ -1876,12 +1898,14 @@ static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) { tlen += taosEncodeFixedI64(buf, pSma->interval); tlen += taosEncodeFixedI64(buf, pSma->sliding); - for (uint16_t i = 0; i < pSma->numOfColIds; ++i) { + for (col_id_t i = 0; i < pSma->numOfColIds; ++i) { tlen += taosEncodeFixedU16(buf, *(pSma->colIds + i)); } + for (uint16_t i = 0; i < pSma->numOfFuncIds; ++i) { tlen += taosEncodeFixedU16(buf, *(pSma->funcIds + i)); } + return tlen; } @@ -1949,20 +1973,6 @@ static FORCE_INLINE void* tDecodeTSmaWrapper(void* buf, STSmaWrapper* pSW) { return buf; } -// RSma: Time-range-wise Rollup SMA -typedef struct { - int64_t interval; - int32_t retention; // unit: day - uint16_t days; // unit: day - int8_t intervalUnit; -} SSmaParams; - -typedef struct { - STSma tsma; - float xFilesFactor; - SArray* smaParams; // SSmaParams -} SRSma; - typedef struct { int64_t uid; int32_t numOfRows; diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 3c0d45df63..f026f8331b 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -54,5 +54,5 @@ elseif(${META_DB_IMPL} STREQUAL "TDB") endif() if(${BUILD_TEST}) - # add_subdirectory(test) + add_subdirectory(test) endif(${BUILD_TEST}) diff --git a/source/dnode/vnode/src/inc/tsdbCommit.h b/source/dnode/vnode/src/inc/tsdbCommit.h index 9c35d06880..699aeaa133 100644 --- a/source/dnode/vnode/src/inc/tsdbCommit.h +++ b/source/dnode/vnode/src/inc/tsdbCommit.h @@ -16,6 +16,10 @@ #ifndef _TD_TSDB_COMMIT_H_ #define _TD_TSDB_COMMIT_H_ +#ifdef __cplusplus +extern "C" { +#endif + typedef struct { int minFid; int midFid; @@ -66,4 +70,8 @@ int tsdbApplyRtn(STsdbRepo *pRepo); #endif +#ifdef __cplusplus +} +#endif + #endif /* _TD_TSDB_COMMIT_H_ */ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/tsdbFS.h b/source/dnode/vnode/src/inc/tsdbFS.h index dab697ce8b..71f35a9eca 100644 --- a/source/dnode/vnode/src/inc/tsdbFS.h +++ b/source/dnode/vnode/src/inc/tsdbFS.h @@ -18,6 +18,10 @@ #include "tsdbFile.h" +#ifdef __cplusplus +extern "C" { +#endif + // ================== TSDB global config extern bool tsdbForceKeepFile; @@ -111,4 +115,8 @@ static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) { return 0; } +#ifdef __cplusplus +} +#endif + #endif /* _TD_TSDB_FS_H_ */ diff --git a/source/dnode/vnode/src/inc/tsdbFile.h b/source/dnode/vnode/src/inc/tsdbFile.h index bbeb6c6a0f..1034ae015a 100644 --- a/source/dnode/vnode/src/inc/tsdbFile.h +++ b/source/dnode/vnode/src/inc/tsdbFile.h @@ -19,6 +19,10 @@ #include "tchecksum.h" #include "tfs.h" +#ifdef __cplusplus +extern "C" { +#endif + #define TSDB_FILE_HEAD_SIZE 512 #define TSDB_FILE_DELIMITER 0xF00AFA0F #define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF @@ -410,4 +414,8 @@ static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet* pSet) { return true; } +#ifdef __cplusplus +} +#endif + #endif /* _TS_TSDB_FILE_H_ */ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/tsdbLog.h b/source/dnode/vnode/src/inc/tsdbLog.h index bde9b338a2..2f3e11650d 100644 --- a/source/dnode/vnode/src/inc/tsdbLog.h +++ b/source/dnode/vnode/src/inc/tsdbLog.h @@ -18,6 +18,10 @@ #include "tlog.h" +#ifdef __cplusplus +extern "C" { +#endif + extern int32_t tsdbDebugFlag; #define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }} while(0) @@ -27,4 +31,8 @@ extern int32_t tsdbDebugFlag; #define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) #define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) +#ifdef __cplusplus +} +#endif + #endif /* _TD_TSDB_LOG_H_ */ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/tsdbMemory.h b/source/dnode/vnode/src/inc/tsdbMemory.h index 1fc4cd9e52..df4df0053f 100644 --- a/source/dnode/vnode/src/inc/tsdbMemory.h +++ b/source/dnode/vnode/src/inc/tsdbMemory.h @@ -16,6 +16,10 @@ #ifndef _TD_TSDB_MEMORY_H_ #define _TD_TSDB_MEMORY_H_ +#ifdef __cplusplus +extern "C" { +#endif + static void * taosTMalloc(size_t size); static void * taosTCalloc(size_t nmemb, size_t size); static void * taosTRealloc(void *ptr, size_t size); @@ -70,5 +74,8 @@ static FORCE_INLINE void* taosTZfree(void* ptr) { return NULL; } +#ifdef __cplusplus +} +#endif #endif /* _TD_TSDB_MEMORY_H_ */ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/tsdbReadImpl.h b/source/dnode/vnode/src/inc/tsdbReadImpl.h index 16eb55967c..cd24358b27 100644 --- a/source/dnode/vnode/src/inc/tsdbReadImpl.h +++ b/source/dnode/vnode/src/inc/tsdbReadImpl.h @@ -24,6 +24,10 @@ #include "tsdbMemory.h" #include "tcommon.h" +#ifdef __cplusplus +extern "C" { +#endif + typedef struct SReadH SReadH; typedef struct { @@ -244,4 +248,8 @@ static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) { return 0; } +#ifdef __cplusplus +} +#endif + #endif /*_TD_TSDB_READ_IMPL_H_*/ diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index 91a7a91f3a..28c5777f4b 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -38,6 +38,7 @@ struct SMetaDB { // DB DB *pTbDB; DB *pSchemaDB; + DB *pSmaDB; // IDX DB *pNameIdx; @@ -101,6 +102,11 @@ int metaOpenDB(SMeta *pMeta) { return -1; } + if (metaOpenBDBDb(&(pDB->pSmaDB), pDB->pEvn, "sma.db", false) < 0) { + metaCloseDB(pMeta); + return -1; + } + // Open Indices if (metaOpenBDBIdx(&(pDB->pNameIdx), pDB->pEvn, "name.index", pDB->pTbDB, &metaNameIdxCb, false) < 0) { metaCloseDB(pMeta); @@ -131,6 +137,7 @@ void metaCloseDB(SMeta *pMeta) { metaCloseBDBIdx(pMeta->pDB->pNtbIdx); metaCloseBDBIdx(pMeta->pDB->pStbIdx); metaCloseBDBIdx(pMeta->pDB->pNameIdx); + metaCloseBDBDb(pMeta->pDB->pSmaDB); metaCloseBDBDb(pMeta->pDB->pSchemaDB); metaCloseBDBDb(pMeta->pDB->pTbDB); metaCloseBDBEnv(pMeta->pDB->pEvn); diff --git a/source/dnode/vnode/test/CMakeLists.txt b/source/dnode/vnode/test/CMakeLists.txt index 40d016300e..af123a3133 100644 --- a/source/dnode/vnode/test/CMakeLists.txt +++ b/source/dnode/vnode/test/CMakeLists.txt @@ -36,4 +36,4 @@ TARGET_INCLUDE_DIRECTORIES( PUBLIC "${CMAKE_SOURCE_DIR}/include/common" PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../src/inc" PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) +) \ No newline at end of file diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index 90158b68b6..f5aa82cb2d 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -18,21 +18,85 @@ #include #include +#include + #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" -#include "tsdbDef.h" - - - -int main(int argc, char** argv) { +int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } +TEST(testCase, tSmaEncodeDecodeTest) { + // encode + STSma tSma = {0}; + tSma.version = 0; + tSma.intervalUnit = TD_TIME_UNIT_DAY; + tSma.interval = 1; + tSma.slidingUnit = TD_TIME_UNIT_HOUR; + tSma.sliding = 0; + tstrncpy(tSma.indexName, "sma_index_test", TSDB_INDEX_NAME_LEN); + tSma.tableUid = 1234567890; + tSma.numOfColIds = 2; + tSma.numOfFuncIds = 5; // sum/min/max/avg/last + tSma.colIds = (col_id_t *)calloc(tSma.numOfColIds, sizeof(col_id_t)); + tSma.funcIds = (uint16_t *)calloc(tSma.numOfFuncIds, sizeof(uint16_t)); + + for (int32_t i = 0; i < tSma.numOfColIds; ++i) { + *(tSma.colIds + i) = (i + PRIMARYKEY_TIMESTAMP_COL_ID); + } + for (int32_t i = 0; i < tSma.numOfFuncIds; ++i) { + *(tSma.funcIds + i) = (i + 2); + } + + STSmaWrapper tSmaWrapper = {.number = 1, .tSma = &tSma}; + uint32_t bufLen = tEncodeTSmaWrapper(NULL, &tSmaWrapper); + + void *buf = calloc(bufLen, 1); + assert(buf != NULL); + + STSmaWrapper *pSW = (STSmaWrapper *)buf; + uint32_t len = tEncodeTSmaWrapper(&buf, &tSmaWrapper); + + EXPECT_EQ(len, bufLen); + + // decode + STSmaWrapper dstTSmaWrapper = {0}; + void * result = tDecodeTSmaWrapper(pSW, &dstTSmaWrapper); + assert(result != NULL); + + EXPECT_EQ(tSmaWrapper.number, dstTSmaWrapper.number); + + for (int i = 0; i < tSmaWrapper.number; ++i) { + STSma *pSma = tSmaWrapper.tSma + i; + STSma *qSma = dstTSmaWrapper.tSma + i; + + EXPECT_EQ(pSma->version, qSma->version); + EXPECT_EQ(pSma->intervalUnit, qSma->intervalUnit); + EXPECT_EQ(pSma->slidingUnit, qSma->slidingUnit); + EXPECT_STRCASEEQ(pSma->indexName, qSma->indexName); + EXPECT_EQ(pSma->numOfColIds, qSma->numOfColIds); + EXPECT_EQ(pSma->numOfFuncIds, qSma->numOfFuncIds); + EXPECT_EQ(pSma->tableUid, qSma->tableUid); + EXPECT_EQ(pSma->interval, qSma->interval); + EXPECT_EQ(pSma->sliding, qSma->sliding); + for (uint32_t j = 0; j < pSma->numOfColIds; ++j) { + EXPECT_EQ(*(col_id_t *)(pSma->colIds + j), *(col_id_t *)(qSma->colIds + j)); + } + for (uint32_t j = 0; j < pSma->numOfFuncIds; ++j) { + EXPECT_EQ(*(uint16_t *)(pSma->funcIds + j), *(uint16_t *)(qSma->funcIds + j)); + } + } + + // resource release + tdDestroyTSma(&tSma, false); + tdDestroyWrapper(&dstTSmaWrapper); +} + #if 0 TEST(testCase, tSmaInsertTest) { STSma tSma = {0}; From 80dadafc87c900fe2ac6129585771dd259d6e65d Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 5 Mar 2022 16:20:40 +0800 Subject: [PATCH 11/17] restore --- cmake/stub_CMakeLists.txt.in | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cmake/stub_CMakeLists.txt.in b/cmake/stub_CMakeLists.txt.in index c768239006..e227e820d6 100644 --- a/cmake/stub_CMakeLists.txt.in +++ b/cmake/stub_CMakeLists.txt.in @@ -1,12 +1,12 @@ # stub -#ExternalProject_Add(stub -# GIT_REPOSITORY https://github.com/coolxv/cpp-stub.git -# GIT_SUBMODULES "src" -# SOURCE_DIR "${CMAKE_CONTRIB_DIR}/cpp-stub" -# BINARY_DIR "${CMAKE_CONTRIB_DIR}/cpp-stub/src" -# CONFIGURE_COMMAND "" -# BUILD_COMMAND "" -# INSTALL_COMMAND "" -# TEST_COMMAND "" -#) +ExternalProject_Add(stub + GIT_REPOSITORY https://github.com/coolxv/cpp-stub.git + GIT_SUBMODULES "src" + SOURCE_DIR "${CMAKE_CONTRIB_DIR}/cpp-stub" + BINARY_DIR "${CMAKE_CONTRIB_DIR}/cpp-stub/src" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) From b158160bc722b6b71fd6396484bc69d4018946ed Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 5 Mar 2022 16:27:47 +0800 Subject: [PATCH 12/17] code optimization --- include/common/tmsg.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index bd537a4a1a..8eb54c757c 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1942,6 +1942,8 @@ static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) { for (uint16_t i = 0; i < pSma->numOfColIds; ++i) { buf = taosDecodeFixedU16(buf, pSma->colIds + i); } + } else { + pSma->colIds = NULL; } if (pSma->numOfFuncIds > 0) { @@ -1952,6 +1954,8 @@ static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) { for (uint16_t i = 0; i < pSma->numOfFuncIds; ++i) { buf = taosDecodeFixedU16(buf, pSma->funcIds + i); } + } else { + pSma->funcIds = NULL; } return buf; From 228292387cfcddb9298d496a733c711449e810c8 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 6 Mar 2022 14:30:45 +0800 Subject: [PATCH 13/17] put/get sma schema from bdb --- include/common/tmsg.h | 2 +- source/dnode/vnode/inc/meta.h | 15 ++- source/dnode/vnode/src/inc/metaDef.h | 2 + source/dnode/vnode/src/meta/metaBDBImpl.c | 146 ++++++++++++++++++++++ source/dnode/vnode/src/meta/metaIdx.c | 17 +++ source/dnode/vnode/test/tsdbSmaTest.cpp | 89 ++++++++++++- 6 files changed, 265 insertions(+), 6 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 134b9c8ec9..d1e02af287 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1885,7 +1885,7 @@ static FORCE_INLINE void tdDestroyTSma(STSma* pSma, bool releaseSelf) { } } -static FORCE_INLINE void tdDestroyWrapper(STSmaWrapper* pSW) { +static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW) { if (pSW && pSW->tSma) { for (uint32_t i = 0; i < pSW->number; ++i) { tdDestroyTSma(pSW->tSma + i, false); diff --git a/source/dnode/vnode/inc/meta.h b/source/dnode/vnode/inc/meta.h index fd079b8f32..e5ad43a4ee 100644 --- a/source/dnode/vnode/inc/meta.h +++ b/source/dnode/vnode/inc/meta.h @@ -38,8 +38,10 @@ typedef struct SMetaCfg { typedef struct SMTbCursor SMTbCursor; typedef struct SMCtbCursor SMCtbCursor; +typedef struct SMSmaCursor SMSmaCursor; typedef SVCreateTbReq STbCfg; +typedef STSma SSmaCfg; // SMeta operations SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF); @@ -50,19 +52,24 @@ int metaDropTable(SMeta *pMeta, tb_uid_t uid); int metaCommit(SMeta *pMeta); // For Query -STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid); -STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid); +STbCfg * metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid); +STbCfg * metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid); SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline); -STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver); +STSchema * metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver); +SSmaCfg * metaGetSmaInfoByName(SMeta *pMeta, const char *indexName); SMTbCursor *metaOpenTbCursor(SMeta *pMeta); void metaCloseTbCursor(SMTbCursor *pTbCur); -char *metaTbCursorNext(SMTbCursor *pTbCur); +char * metaTbCursorNext(SMTbCursor *pTbCur); SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid); void metaCloseCtbCurosr(SMCtbCursor *pCtbCur); tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur); +SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid); +void metaCloseSmaCurosr(SMSmaCursor *pSmaCur); +const char * metaSmaCursorNext(SMSmaCursor *pSmaCur); + // Options void metaOptionsInit(SMetaCfg *pMetaCfg); void metaOptionsClear(SMetaCfg *pMetaCfg); diff --git a/source/dnode/vnode/src/inc/metaDef.h b/source/dnode/vnode/src/inc/metaDef.h index 71bfd91356..6b4c036b39 100644 --- a/source/dnode/vnode/src/inc/metaDef.h +++ b/source/dnode/vnode/src/inc/metaDef.h @@ -33,6 +33,8 @@ int metaOpenDB(SMeta* pMeta); void metaCloseDB(SMeta* pMeta); int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg); int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid); +int metaSaveSmaToDB(SMeta* pMeta, SSmaCfg* pTbCfg); +int metaRemoveSmaFromDb(SMeta* pMeta, const char* indexName); // SMetaCache int metaOpenCache(SMeta* pMeta); diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index 82b4d3ce12..c31f28d983 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -45,6 +45,7 @@ struct SMetaDB { DB *pStbIdx; DB *pNtbIdx; DB *pCtbIdx; + DB *pSmaIdx; // ENV DB_ENV *pEvn; }; @@ -63,6 +64,7 @@ static int metaNameIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT static int metaStbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); static int metaNtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); +static int metaSmaIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg); static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); static void metaClearTbCfg(STbCfg *pTbCfg); @@ -128,11 +130,17 @@ int metaOpenDB(SMeta *pMeta) { return -1; } + if (metaOpenBDBIdx(&(pDB->pSmaIdx), pDB->pEvn, "sma.index", pDB->pSmaDB, &metaSmaIdxCb, true) < 0) { + metaCloseDB(pMeta); + return -1; + } + return 0; } void metaCloseDB(SMeta *pMeta) { if (pMeta->pDB) { + metaCloseBDBIdx(pMeta->pDB->pSmaIdx); metaCloseBDBIdx(pMeta->pDB->pCtbIdx); metaCloseBDBIdx(pMeta->pDB->pNtbIdx); metaCloseBDBIdx(pMeta->pDB->pStbIdx); @@ -218,6 +226,49 @@ int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) { return 0; } +int metaSaveSmaToDB(SMeta *pMeta, SSmaCfg *pSmaCfg) { + char buf[512] = {0}; // TODO: may overflow + void *pBuf = NULL; + DBT key1 = {0}, value1 = {0}; + + { + // save sma info + pBuf = buf; + + key1.data = pSmaCfg->indexName; + key1.size = strlen(key1.data); + + tEncodeTSma(&pBuf, pSmaCfg); + + value1.data = buf; + value1.size = POINTER_DISTANCE(pBuf, buf); + value1.app_data = pSmaCfg; + } + + metaDBWLock(pMeta->pDB); + pMeta->pDB->pSmaDB->put(pMeta->pDB->pSmaDB, NULL, &key1, &value1, 0); + metaDBULock(pMeta->pDB); + + return 0; +} + +int metaRemoveSmaFromDb(SMeta *pMeta, const char *indexName) { + // TODO +#if 0 + DBT key = {0}; + + key.data = (void *)indexName; + key.size = strlen(indexName); + + metaDBWLock(pMeta->pDB); + // TODO: No guarantee of consistence. + // Use transaction or DB->sync() for some guarantee. + pMeta->pDB->pSmaDB->del(pMeta->pDB->pSmaDB, NULL, &key, 0); + metaDBULock(pMeta->pDB); +#endif + return 0; +} + /* ------------------------ STATIC METHODS ------------------------ */ static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW) { int tlen = 0; @@ -433,6 +484,16 @@ static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey } } +static int metaSmaIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey) { + SSmaCfg *pSmaCfg = (SSmaCfg *)(pValue->app_data); + + memset(pSKey, 0, sizeof(*pSKey)); + pSKey->data = &(pSmaCfg->tableUid); + pSKey->size = sizeof(pSmaCfg->tableUid); + + return 0; +} + static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) { int tsize = 0; @@ -548,6 +609,36 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { return pTbCfg; } +SSmaCfg *metaGetSmaInfoByName(SMeta *pMeta, const char *indexName) { + SSmaCfg *pCfg = NULL; + SMetaDB *pDB = pMeta->pDB; + DBT key = {0}; + DBT value = {0}; + int ret; + + // Set key/value + key.data = (void *)indexName; + key.size = strlen(indexName); + + // Query + metaDBRLock(pDB); + ret = pDB->pTbDB->get(pDB->pSmaDB, NULL, &key, &value, 0); + metaDBULock(pDB); + if (ret != 0) { + return NULL; + } + + // Decode + pCfg = (SSmaCfg *)malloc(sizeof(SSmaCfg)); + if (pCfg == NULL) { + return NULL; + } + + tDecodeTSma(value.data, pCfg); + + return pCfg; +} + SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) { uint32_t nCols; SSchemaWrapper *pSW = NULL; @@ -726,6 +817,61 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { } } +struct SMSmaCursor { + DBC *pCur; + tb_uid_t uid; +}; + +SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { + SMSmaCursor *pCur = NULL; + SMetaDB *pDB = pMeta->pDB; + int ret; + + pCur = (SMSmaCursor *)calloc(1, sizeof(*pCur)); + if (pCur == NULL) { + return NULL; + } + + pCur->uid = uid; + ret = pDB->pCtbIdx->cursor(pDB->pSmaIdx, NULL, &(pCur->pCur), 0); + if (ret != 0) { + free(pCur); + return NULL; + } + + return pCur; +} + +void metaCloseSmaCurosr(SMSmaCursor *pCur) { + if (pCur) { + if (pCur->pCur) { + pCur->pCur->close(pCur->pCur); + } + + free(pCur); + } +} + +const char* metaSmaCursorNext(SMSmaCursor *pCur) { + DBT skey = {0}; + DBT pkey = {0}; + DBT pval = {0}; + void *pBuf; + + // Set key + skey.data = &(pCur->uid); + skey.size = sizeof(pCur->uid); + + if (pCur->pCur->pget(pCur->pCur, &skey, &pkey, &pval, DB_NEXT) == 0) { + const char* indexName = (const char *)pkey.data; + assert(indexName != NULL); + return indexName; + } else { + return 0; + } +} + + static void metaDBWLock(SMetaDB *pDB) { #if IMPL_WITH_LOCK pthread_rwlock_wrlock(&(pDB->rwlock)); diff --git a/source/dnode/vnode/src/meta/metaIdx.c b/source/dnode/vnode/src/meta/metaIdx.c index d9abb4bb7b..2ca02a2b80 100644 --- a/source/dnode/vnode/src/meta/metaIdx.c +++ b/source/dnode/vnode/src/meta/metaIdx.c @@ -106,3 +106,20 @@ int metaRemoveTableFromIdx(SMeta *pMeta, tb_uid_t uid) { // TODO return 0; } + +int metaCreateSma(SMeta *pMeta, SSmaCfg *pSmaCfg) { + // Validate the tbOptions + // if (metaValidateTbCfg(pMeta, pTbCfg) < 0) { + // // TODO: handle error + // return -1; + // } + + // TODO: add atomicity + + if (metaSaveSmaToDB(pMeta, pSmaCfg) < 0) { + // TODO: handle error + return -1; + } + + return 0; +} diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index f5aa82cb2d..986986aa70 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -18,6 +18,7 @@ #include #include +#include #include #pragma GCC diagnostic push @@ -94,7 +95,93 @@ TEST(testCase, tSmaEncodeDecodeTest) { // resource release tdDestroyTSma(&tSma, false); - tdDestroyWrapper(&dstTSmaWrapper); + tdDestroyTSmaWrapper(&dstTSmaWrapper); +} + +TEST(testCase, tSma_DB_Put_Get_Del_Test) { + const char *smaIndexName1 = "sma_index_test_1"; + const char *smaIndexName2 = "sma_index_test_2"; + const char *smaTestDir = "./smaTest"; + const uint64_t tbUid = 1234567890; + // encode + STSma tSma = {0}; + tSma.version = 0; + tSma.intervalUnit = TD_TIME_UNIT_DAY; + tSma.interval = 1; + tSma.slidingUnit = TD_TIME_UNIT_HOUR; + tSma.sliding = 0; + tstrncpy(tSma.indexName, smaIndexName1, TSDB_INDEX_NAME_LEN); + tSma.tableUid = tbUid; + tSma.numOfColIds = 2; + tSma.numOfFuncIds = 5; // sum/min/max/avg/last + tSma.colIds = (col_id_t *)calloc(tSma.numOfColIds, sizeof(col_id_t)); + tSma.funcIds = (uint16_t *)calloc(tSma.numOfFuncIds, sizeof(uint16_t)); + + for (int32_t i = 0; i < tSma.numOfColIds; ++i) { + *(tSma.colIds + i) = (i + PRIMARYKEY_TIMESTAMP_COL_ID); + } + for (int32_t i = 0; i < tSma.numOfFuncIds; ++i) { + *(tSma.funcIds + i) = (i + 2); + } + + SMeta * pMeta = NULL; + SSmaCfg * pSmaCfg = &tSma; + const SMetaCfg *pMetaCfg = &defaultMetaOptions; + + taosRemoveDir(smaTestDir); + + pMeta = metaOpen(smaTestDir, pMetaCfg, NULL); + assert(pMeta != NULL); + // save index 1 + metaSaveSmaToDB(pMeta, pSmaCfg); + + tstrncpy(pSmaCfg->indexName, smaIndexName2, TSDB_INDEX_NAME_LEN); + pSmaCfg->version = 1; + pSmaCfg->intervalUnit = TD_TIME_UNIT_HOUR; + pSmaCfg->interval = 1; + pSmaCfg->slidingUnit = TD_TIME_UNIT_MINUTE; + pSmaCfg->sliding = 5; + + // save index 2 + metaSaveSmaToDB(pMeta, pSmaCfg); + + // get value by indexName + SSmaCfg *qSmaCfg = NULL; + qSmaCfg = metaGetSmaInfoByName(pMeta, smaIndexName1); + assert(qSmaCfg != NULL); + printf("name1 = %s\n", qSmaCfg->indexName); + EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName1); + EXPECT_EQ(qSmaCfg->tableUid, tSma.tableUid); + tdDestroyTSma(qSmaCfg, true); + + qSmaCfg = metaGetSmaInfoByName(pMeta, smaIndexName2); + assert(qSmaCfg != NULL); + printf("name2 = %s\n", qSmaCfg->indexName); + EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName2); + EXPECT_EQ(qSmaCfg->interval, tSma.interval); + tdDestroyTSma(qSmaCfg, true); + + // get value by table uid + SMSmaCursor *pSmaCur = metaOpenSmaCursor(pMeta, tbUid); + assert(pSmaCur != NULL); + uint32_t indexCnt = 0; + while (1) { + const char* indexName = metaSmaCursorNext(pSmaCur); + if (indexName == NULL) { + break; + } + printf("indexName = %s\n", indexName); + ++indexCnt; + } + EXPECT_EQ(indexCnt, 2); + metaCloseSmaCurosr(pSmaCur); + + // resource release + metaRemoveSmaFromDb(pMeta, smaIndexName1); + metaRemoveSmaFromDb(pMeta, smaIndexName2); + + tdDestroyTSma(&tSma, false); + metaClose(pMeta); } #if 0 From d412397251234e8d48e7f81f45a14254ec910fd4 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 6 Mar 2022 19:45:42 +0800 Subject: [PATCH 14/17] add/check tSma schema methods --- include/common/tmsg.h | 68 +- include/common/tmsgdef.h | 3 + include/common/trow.h | 6 +- source/dnode/vnode/inc/meta.h | 1 + source/dnode/vnode/inc/tsdb.h | 21 + source/dnode/vnode/src/inc/tsdbDef.h | 1 + source/dnode/vnode/src/inc/tsdbFS.h | 1 + source/dnode/vnode/src/inc/tsdbSma.h | 95 + source/dnode/vnode/src/meta/metaBDBImpl.c | 60 +- source/dnode/vnode/src/tsdb/tsdbSma.c | 2180 +++++++++++++++++++++ source/dnode/vnode/src/tsdb/tsdbWrite.c | 41 + source/dnode/vnode/src/vnd/vnodeWrite.c | 9 + source/dnode/vnode/test/tsdbSmaTest.cpp | 19 +- 13 files changed, 2482 insertions(+), 23 deletions(-) create mode 100644 source/dnode/vnode/src/inc/tsdbSma.h create mode 100644 source/dnode/vnode/src/tsdb/tsdbSma.c diff --git a/include/common/tmsg.h b/include/common/tmsg.h index d1e02af287..65860d4959 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1866,10 +1866,59 @@ typedef struct { uint64_t tableUid; // super/common table uid int64_t interval; int64_t sliding; - col_id_t* colIds; // N.B. sorted column ids - uint16_t* funcIds; // N.B. sorted sma function ids + col_id_t* colIds; // sorted column ids + uint16_t* funcIds; // sorted sma function ids } STSma; // Time-range-wise SMA +typedef struct { + int8_t msgType; // 0 create, 1 recreate + STSma tSma; + STimeWindow window; +} SCreateTSmaMsg; + +typedef struct { + STimeWindow window; + char indexName[TSDB_INDEX_NAME_LEN + 1]; +} SDropTSmaMsg; + +typedef struct { + STimeWindow tsWindow; // [skey, ekey] + uint64_t tableUid; // sub/common table uid + int32_t numOfBlocks; // number of sma blocks for each column, total number is numOfBlocks*numOfColId + int32_t dataLen; // total data length + col_id_t* colIds; // e.g. 2,4,9,10 + col_id_t numOfColIds; // e.g. 4 + char data[]; // the sma blocks +} STSmaData; + +// TODO: move to the final location afte schema of STSma/STSmaData defined +static FORCE_INLINE void tdDestroySmaData(STSmaData* pSmaData) { + if (pSmaData) { + if (pSmaData->colIds) { + tfree(pSmaData->colIds); + } + tfree(pSmaData); + } +} + +// RSma: Time-range-wise Rollup SMA +// TODO: refactor when rSma grammar defined finally => +typedef struct { + int64_t interval; + int32_t retention; // unit: day + uint16_t days; // unit: day + int8_t intervalUnit; +} SSmaParams; +// TODO: refactor when rSma grammar defined finally <= + +typedef struct { + // TODO: refactor to use the real schema => + STSma tsma; + float xFilesFactor; + SArray* smaParams; // SSmaParams + // TODO: refactor to use the real schema <= +} SRSma; + typedef struct { uint32_t number; STSma* tSma; @@ -1885,12 +1934,17 @@ static FORCE_INLINE void tdDestroyTSma(STSma* pSma, bool releaseSelf) { } } -static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW) { - if (pSW && pSW->tSma) { - for (uint32_t i = 0; i < pSW->number; ++i) { - tdDestroyTSma(pSW->tSma + i, false); +static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW, bool releaseSelf) { + if (pSW) { + if (pSW->tSma) { + for (uint32_t i = 0; i < pSW->number; ++i) { + tdDestroyTSma(pSW->tSma + i, false); + } + tfree(pSW->tSma); + } + if (releaseSelf) { + free(pSW); } - tfree(pSW->tSma); } } diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index cae186ba16..0f0c4729bc 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -184,6 +184,9 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp) TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqCVConsumeReq, SMqCVConsumeRsp) + TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL) // Requests handled by QNODE TD_NEW_MSG_SEG(TDMT_QND_MSG) diff --git a/include/common/trow.h b/include/common/trow.h index 49bc2f3515..ef30796d78 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -118,6 +118,8 @@ typedef struct { } SKvRow; typedef struct { + /// timestamp + TSKEY ts; union { /// union field for encode and decode uint32_t info; @@ -138,8 +140,6 @@ typedef struct { uint32_t len; /// row version uint64_t ver; - /// timestamp - TSKEY ts; /// the inline data, maybe a tuple or a k-v tuple char data[]; } STSRow; @@ -173,7 +173,7 @@ typedef struct { #define TD_ROW_DATA(r) ((r)->data) #define TD_ROW_LEN(r) ((r)->len) #define TD_ROW_KEY(r) ((r)->ts) -#define TD_ROW_KEY_ADDR(r) POINTER_SHIFT((r), 16) +#define TD_ROW_KEY_ADDR(r) (r) // N.B. If without STSchema, getExtendedRowSize() is used to get the rowMaxBytes and // (int32_t)ceil((double)nCols/TD_VTYPE_PARTS) should be added if TD_SUPPORT_BITMAP defined. diff --git a/source/dnode/vnode/inc/meta.h b/source/dnode/vnode/inc/meta.h index e5ad43a4ee..b20be691ef 100644 --- a/source/dnode/vnode/inc/meta.h +++ b/source/dnode/vnode/inc/meta.h @@ -57,6 +57,7 @@ STbCfg * metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid); SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline); STSchema * metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver); SSmaCfg * metaGetSmaInfoByName(SMeta *pMeta, const char *indexName); +STSmaWrapper * metaGetSmaInfoByUid(SMeta *pMeta, tb_uid_t uid); SMTbCursor *metaOpenTbCursor(SMeta *pMeta); void metaCloseTbCursor(SMTbCursor *pTbCur); diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index 6bc89ddd66..5513742c73 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -87,6 +87,27 @@ int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp); int tsdbPrepareCommit(STsdb *pTsdb); int tsdbCommit(STsdb *pTsdb); +/** + * @brief Insert tSma(Time-range-wise SMA) data from stream computing engine + * + * @param pTsdb + * @param param + * @param pData + * @return int32_t + */ +int32_t tsdbInsertTSmaData(STsdb *pTsdb, STSma *param, STSmaData *pData); + +/** + * @brief Insert RSma(Time-range-wise Rollup SMA) data. + * + * @param pTsdb + * @param param + * @param pData + * @return int32_t + */ +int32_t tsdbInsertRSmaData(STsdb *pTsdb, SRSma *param, STSmaData *pData); + + // STsdbCfg int tsdbOptionsInit(STsdbCfg *); void tsdbOptionsClear(STsdbCfg *); diff --git a/source/dnode/vnode/src/inc/tsdbDef.h b/source/dnode/vnode/src/inc/tsdbDef.h index 98c0de32a8..96a76ea7d4 100644 --- a/source/dnode/vnode/src/inc/tsdbDef.h +++ b/source/dnode/vnode/src/inc/tsdbDef.h @@ -35,6 +35,7 @@ #include "tsdbMemory.h" #include "tsdbOptions.h" #include "tsdbReadImpl.h" +#include "tsdbSma.h" #ifdef __cplusplus extern "C" { diff --git a/source/dnode/vnode/src/inc/tsdbFS.h b/source/dnode/vnode/src/inc/tsdbFS.h index 71f35a9eca..641255a294 100644 --- a/source/dnode/vnode/src/inc/tsdbFS.h +++ b/source/dnode/vnode/src/inc/tsdbFS.h @@ -42,6 +42,7 @@ typedef struct { typedef struct { STsdbFSMeta meta; // FS meta SArray * df; // data file array + SArray * smaf; // sma data file array } SFSStatus; typedef struct { diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/inc/tsdbSma.h new file mode 100644 index 0000000000..2a326eece8 --- /dev/null +++ b/source/dnode/vnode/src/inc/tsdbSma.h @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TSDB_SMA_H_ +#define _TD_TSDB_SMA_H_ + +// insert/update interface +int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData); +int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, SRSma *param, STSmaData *pData); + + +// query interface +// TODO: This is the basic params, and should wrap the params to a queryHandle. +int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData, STimeWindow *queryWin, int32_t nMaxResult); + +// management interface +int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void* result); +int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin); + + + + +// internal func +static FORCE_INLINE int32_t tsdbEncodeTSmaKey(uint64_t tableUid, col_id_t colId, TSKEY tsKey, void **pData) { + int32_t len = 0; + len += taosEncodeFixedU64(pData, tableUid); + len += taosEncodeFixedU16(pData, colId); + len += taosEncodeFixedI64(pData, tsKey); + return len; +} + +#if 0 + +typedef struct { + int minFid; + int midFid; + int maxFid; + TSKEY minKey; +} SRtn; + +typedef struct { + uint64_t uid; + int64_t offset; + int64_t size; +} SKVRecord; + +void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn); + +static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) { + if (key < 0) { + return (int)((key + 1) / tsTickPerDay[precision] / days - 1); + } else { + return (int)((key / tsTickPerDay[precision] / days)); + } +} + +static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) { + if (fid >= pRtn->maxFid) { + return 0; + } else if (fid >= pRtn->midFid) { + return 1; + } else if (fid >= pRtn->minFid) { + return 2; + } else { + return -1; + } +} + +#define TSDB_DEFAULT_BLOCK_ROWS(maxRows) ((maxRows)*4 / 5) + +int tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord); +void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord); +void *tsdbCommitData(STsdbRepo *pRepo); +int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn); +int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray *pSubA, void **ppBuf, SBlockIdx *pIdx); +int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf); +int tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock, + bool isLast, bool isSuper, void **ppBuf, void **ppCBuf); +int tsdbApplyRtn(STsdbRepo *pRepo); + +#endif + +#endif /* _TD_TSDB_SMA_H_ */ \ No newline at end of file diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index c31f28d983..03a2937679 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -833,6 +833,7 @@ SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { } pCur->uid = uid; + // TODO: lock? ret = pDB->pCtbIdx->cursor(pDB->pSmaIdx, NULL, &(pCur->pCur), 0); if (ret != 0) { free(pCur); @@ -852,25 +853,68 @@ void metaCloseSmaCurosr(SMSmaCursor *pCur) { } } -const char* metaSmaCursorNext(SMSmaCursor *pCur) { - DBT skey = {0}; - DBT pkey = {0}; - DBT pval = {0}; - void *pBuf; +const char *metaSmaCursorNext(SMSmaCursor *pCur) { + DBT skey = {0}; + DBT pkey = {0}; + DBT pval = {0}; // Set key skey.data = &(pCur->uid); skey.size = sizeof(pCur->uid); - + // TODO: lock? if (pCur->pCur->pget(pCur->pCur, &skey, &pkey, &pval, DB_NEXT) == 0) { - const char* indexName = (const char *)pkey.data; + const char *indexName = (const char *)pkey.data; assert(indexName != NULL); return indexName; } else { - return 0; + return NULL; } } +STSmaWrapper *metaGetSmaInfoByUid(SMeta *pMeta, tb_uid_t uid) { + STSmaWrapper *pSW = NULL; + + pSW = calloc(sizeof(*pSW), 1); + if (pSW == NULL) { + return NULL; + } + + SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid); + if (pCur == NULL) { + free(pSW); + return NULL; + } + + DBT skey = {.data = &(pCur->uid)}; + DBT pval = {.size = sizeof(pCur->uid)}; + void *pBuf = NULL; + + while (true) { + // TODO: lock? + if (pCur->pCur->pget(pCur->pCur, &skey, NULL, &pval, DB_NEXT) == 0) { + ++pSW->number; + STSma *tptr = (STSma *)realloc(pSW->tSma, pSW->number * sizeof(STSma)); + if (tptr == NULL) { + metaCloseSmaCurosr(pCur); + tdDestroyTSmaWrapper(pSW, true); + return NULL; + } + pSW->tSma = tptr; + pBuf = pval.data; + if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) { + metaCloseSmaCurosr(pCur); + tdDestroyTSmaWrapper(pSW, true); + return NULL; + } + continue; + } + break; + } + + metaCloseSmaCurosr(pCur); + + return pSW; +} static void metaDBWLock(SMetaDB *pDB) { #if IMPL_WITH_LOCK diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c new file mode 100644 index 0000000000..297c2783ec --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -0,0 +1,2180 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbDef.h" + +#define SMA_STORAGE_TSDB_DAYS 30 +#define SMA_STORAGE_SPLIT_HOURS 24 +#define SMA_KEY_LEN 18 // tableUid_colId_TSKEY 8+2+8 + +#define SMA_STORE_SINGLE_BLOCKS // store SMA data by single block or multiple blocks + +typedef enum { + SMA_STORAGE_LEVEL_TSDB = 0, // store TSma in dir e.g. vnode${N}/tsdb/.tsma + SMA_STORAGE_LEVEL_DFILESET = 1 // store TSma in file e.g. vnode${N}/tsdb/v2f1900.tsma.${sma_index_name} +} ESmaStorageLevel; + +typedef struct { + STsdb * pTsdb; + char * pDFile; // TODO: use the real DFile type, not char* + int32_t interval; // interval with the precision of DB + int32_t blockSize; // size of SMA block item + // TODO +} STSmaWriteH; + +typedef struct { + int32_t iter; +} SmaFsIter; +typedef struct { + STsdb * pTsdb; + char * pDFile; // TODO: use the real DFile type, not char* + int32_t interval; // interval with the precision of DB + int32_t blockSize; // size of SMA block item + int8_t storageLevel; + int8_t days; + SmaFsIter smaFsIter; + // TODO +} STSmaReadH; + +// declaration of static functions +static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData); +static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData); +static int32_t tsdbJudgeStorageLevel(int64_t interval, int8_t intervalUnit); +static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaData *pData, int32_t sectionDataLen, int32_t nBlocks); +static int32_t tsdbInsertTSmaBlocks(void *bTree, const char *smaKey, const char *pData, int32_t dataLen); +static int32_t tsdbTSmaDataSplit(STSmaWriteH *pSmaH, STSma *param, STSmaData *pData, int32_t days, int32_t nOffset, + int32_t fid, int32_t *nSmaBlocks); +static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision); +static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSma *param, STSmaData *pData, int32_t storageLevel, + int32_t fid); + +static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData); +static int32_t tsdbInitTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWindow *queryWin); +static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWindow *queryWin); + +/** + * @brief Judge the tSma storage level + * + * @param interval + * @param intervalUnit + * @return int32_t + */ +static int32_t tsdbJudgeStorageLevel(int64_t interval, int8_t intervalUnit) { + // TODO: configurable for SMA_STORAGE_SPLIT_HOURS? + switch (intervalUnit) { + case TD_TIME_UNIT_HOUR: + if (interval < SMA_STORAGE_SPLIT_HOURS) { + return SMA_STORAGE_LEVEL_DFILESET; + } + break; + case TD_TIME_UNIT_MINUTE: + if (interval < 60 * SMA_STORAGE_SPLIT_HOURS) { + return SMA_STORAGE_LEVEL_DFILESET; + } + break; + case TD_TIME_UNIT_SEC: + if (interval < 3600 * SMA_STORAGE_SPLIT_HOURS) { + return SMA_STORAGE_LEVEL_DFILESET; + } + break; + case TD_TIME_UNIT_MILLISEC: + if (interval < 3600 * 1e3 * SMA_STORAGE_SPLIT_HOURS) { + return SMA_STORAGE_LEVEL_DFILESET; + } + break; + case TD_TIME_UNIT_MICROSEC: + if (interval < 3600 * 1e6 * SMA_STORAGE_SPLIT_HOURS) { + return SMA_STORAGE_LEVEL_DFILESET; + } + break; + case TD_TIME_UNIT_NANOSEC: + if (interval < 3600 * 1e9 * SMA_STORAGE_SPLIT_HOURS) { + return SMA_STORAGE_LEVEL_DFILESET; + } + break; + default: + break; + } + return SMA_STORAGE_LEVEL_TSDB; +} + +/** + * @brief Insert TSma data blocks to B+Tree + * + * @param bTree + * @param smaKey + * @param pData + * @param dataLen + * @return int32_t + */ +static int32_t tsdbInsertTSmaBlocks(void *bTree, const char *smaKey, const char *pData, int32_t dataLen) { + // TODO: insert sma data blocks into B+Tree + printf("insert sma data blocks into B+Tree: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", dataLen %d\n", + *(uint64_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8), *(int64_t *)POINTER_SHIFT(smaKey, 10), dataLen); + return TSDB_CODE_SUCCESS; +} + +static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision) { + if (intervalUnit < TD_TIME_UNIT_MILLISEC) { + switch (intervalUnit) { + case TD_TIME_UNIT_YEAR: + case TD_TIME_UNIT_SEASON: + case TD_TIME_UNIT_MONTH: + case TD_TIME_UNIT_WEEK: + // illegal time unit + tsdbError("invalid interval unit: %d\n", intervalUnit); + TASSERT(0); + break; + case TD_TIME_UNIT_DAY: // the interval for tSma calculation must <= day + interval *= 86400 * 1e3; + break; + case TD_TIME_UNIT_HOUR: + interval *= 3600 * 1e3; + break; + case TD_TIME_UNIT_MINUTE: + interval *= 60 * 1e3; + break; + case TD_TIME_UNIT_SEC: + interval *= 1e3; + break; + default: + break; + } + } + + switch (intervalUnit) { + case TD_TIME_UNIT_MILLISEC: + if (TSDB_TIME_PRECISION_MILLI == precision) { + return interval; + } else if (TSDB_TIME_PRECISION_MICRO == precision) { + return interval * 1e3; + } else { // nano second + return interval * 1e6; + } + break; + case TD_TIME_UNIT_MICROSEC: + if (TSDB_TIME_PRECISION_MILLI == precision) { + return interval / 1e3; + } else if (TSDB_TIME_PRECISION_MICRO == precision) { + return interval; + } else { // nano second + return interval * 1e3; + } + break; + case TD_TIME_UNIT_NANOSEC: + if (TSDB_TIME_PRECISION_MILLI == precision) { + return interval / 1e6; + } else if (TSDB_TIME_PRECISION_MICRO == precision) { + return interval / 1e3; + } else { // nano second + return interval; + } + break; + default: + if (TSDB_TIME_PRECISION_MILLI == precision) { + return interval * 1e3; + } else if (TSDB_TIME_PRECISION_MICRO == precision) { + return interval * 1e6; + } else { // nano second + return interval * 1e9; + } + break; + } + return interval; +} + +/** + * @brief Split the TSma data blocks into expected size and insert into B+Tree. + * + * @param pSmaH + * @param pData + * @param nOffset The nOffset of blocks since fid changes. + * @param nBlocks The nBlocks with the same fid since nOffset. + * @return int32_t + */ +static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaData *pData, int32_t nOffset, int32_t nBlocks) { + STsdb *pTsdb = pSmaH->pTsdb; + + TASSERT(pData->colIds != NULL); + + tsdbDebug("tsdbInsertTSmaDataSection: nOffset %d, nBlocks %d", nOffset, nBlocks); + printf("tsdbInsertTSmaDataSection: nOffset %d, nBlocks %d\n", nOffset, nBlocks); + + int32_t colDataLen = pData->dataLen / pData->numOfColIds; + int32_t sectionDataLen = pSmaH->blockSize * nBlocks; + + for (col_id_t i = 0; i < pData->numOfColIds; ++i) { + // param: pointer of B+Tree, key, value, dataLen + void *bTree = pSmaH->pDFile; +#ifndef SMA_STORE_SINGLE_BLOCKS + // save tSma data blocks as a whole + char smaKey[SMA_KEY_LEN] = {0}; + void *pSmaKey = &smaKey; + tsdbEncodeTSmaKey(pData->tableUid, *(pData->colIds + i), pData->tsWindow.skey + nOffset * pSmaH->interval, + (void **)&pSmaKey); + if (tsdbInsertTSmaBlocks(bTree, smaKey, pData->data + i * colDataLen + nOffset * pSmaH->blockSize, sectionDataLen) < + 0) { + tsdbWarn("vgId:%d insert tSma blocks failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); + } +#else + // save tSma data blocks separately + for (int32_t n = 0; n < nBlocks; ++n) { + char smaKey[SMA_KEY_LEN] = {0}; + void *pSmaKey = &smaKey; + tsdbEncodeTSmaKey(pData->tableUid, *(pData->colIds + i), pData->tsWindow.skey + (nOffset + n) * pSmaH->interval, + (void **)&pSmaKey); + if (tsdbInsertTSmaBlocks(bTree, smaKey, pData->data + i * colDataLen + (nOffset + n) * pSmaH->blockSize, + pSmaH->blockSize) < 0) { + tsdbWarn("vgId:%d insert tSma blocks failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); + } + } +#endif + } + return TSDB_CODE_SUCCESS; +} + +static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData) { + pSmaH->pTsdb = pTsdb; + pSmaH->interval = tsdbGetIntervalByPrecision(param->interval, param->intervalUnit, REPO_CFG(pTsdb)->precision); + pSmaH->blockSize = param->numOfFuncIds * sizeof(int64_t); +} + +static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSma *param, STSmaData *pData, int32_t storageLevel, + int32_t fid) { + // TODO + pSmaH->pDFile = "tSma_interval_file_name"; + + return TSDB_CODE_SUCCESS; +} /** + * @brief Split the sma data blocks by fid. + * + * @param pSmaH + * @param param + * @param pData + * @param nOffset + * @param fid + * @param nSmaBlocks + * @return int32_t + */ +static int32_t tsdbTSmaDataSplit(STSmaWriteH *pSmaH, STSma *param, STSmaData *pData, int32_t days, int32_t nOffset, + int32_t fid, int32_t *nSmaBlocks) { + STsdbCfg *pCfg = REPO_CFG(pSmaH->pTsdb); + + // TODO: use binary search + for (int32_t n = nOffset + 1; n < pData->numOfBlocks; ++n) { + // TODO: The tsWindow.skey should use the precision of DB. + int32_t tFid = (int32_t)(TSDB_KEY_FID(pData->tsWindow.skey + pSmaH->interval * n, days, pCfg->precision)); + if (tFid > fid) { + *nSmaBlocks = n - nOffset; + break; + } + } + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Insert/Update Time-range-wise SMA data. + * - If interval < SMA_STORAGE_SPLIT_HOURS(e.g. 24), save the SMA data as a part of DFileSet to e.g. + * v3f1900.tsma.${sma_index_name}. The days is the same with that for TS data files. + * - If interval >= SMA_STORAGE_SPLIT_HOURS, save the SMA data to e.g. vnode3/tsma/v3f632.tsma.${sma_index_name}. The + * days is 30 times of the interval, and the minimum days is SMA_STORAGE_TSDB_DAYS(30d). + * - The destination file of one data block for some interval is determined by its start TS key. + * + * @param pTsdb + * @param param + * @param pData + * @return int32_t + */ +int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData) { + STsdbCfg * pCfg = REPO_CFG(pTsdb); + STSmaData * curData = pData; + STSmaWriteH tSmaH = {0}; + + tsdbInitTSmaWriteH(&tSmaH, pTsdb, param, pData); + + if (pData->numOfBlocks <= 0 || pData->numOfColIds <= 0 || pData->dataLen <= 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + // Step 1: Judge the storage level + int32_t storageLevel = tsdbJudgeStorageLevel(param->interval, param->intervalUnit); + int32_t daysPerFile = storageLevel == SMA_STORAGE_LEVEL_TSDB ? SMA_STORAGE_TSDB_DAYS : pCfg->daysPerFile; + + // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file + // - Set and open the DFile or the B+Tree file + + int32_t minFid = (int32_t)(TSDB_KEY_FID(pData->tsWindow.skey, daysPerFile, pCfg->precision)); + int32_t maxFid = (int32_t)(TSDB_KEY_FID(pData->tsWindow.ekey, daysPerFile, pCfg->precision)); + + if (minFid == maxFid) { + // Save all the TSma data to one file + // TODO: tsdbStartTSmaCommit(); + tsdbSetTSmaDataFile(&tSmaH, param, pData, storageLevel, minFid); + tsdbInsertTSmaDataSection(&tSmaH, pData, 0, pData->numOfBlocks); + // TODO:tsdbEndTSmaCommit(); + } else if (minFid < maxFid) { + // Split the TSma data and save to multiple files. As there is limit for the span, it can't span more than 2 files + // actually. + // TODO: tsdbStartTSmaCommit(); + int32_t tFid = minFid; + int32_t nOffset = 0; + int32_t nSmaBlocks = 0; + do { + tsdbTSmaDataSplit(&tSmaH, param, pData, daysPerFile, nOffset, tFid, &nSmaBlocks); + tsdbSetTSmaDataFile(&tSmaH, param, pData, storageLevel, tFid); + if (tsdbInsertTSmaDataSection(&tSmaH, pData, nOffset, nSmaBlocks) < 0) { + return terrno; + } + + ++tFid; + nOffset += nSmaBlocks; + + if (tFid == maxFid) { + tsdbSetTSmaDataFile(&tSmaH, param, pData, storageLevel, tFid); + tsdbInsertTSmaDataSection(&tSmaH, pData, nOffset, pData->numOfBlocks - nOffset); + break; + } + } while (true); + + // TODO:tsdbEndTSmaCommit(); + } else { + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, SRSma *param, STSmaData *pData, int32_t fid) { + // TODO + pSmaH->pDFile = "rSma_interval_file_name"; + + return TSDB_CODE_SUCCESS; +} + +int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, SRSma *param, STSmaData *pData) { + STsdbCfg * pCfg = REPO_CFG(pTsdb); + STSma * tParam = ¶m->tsma; + STSmaData * curData = pData; + STSmaWriteH tSmaH = {0}; + + tsdbInitTSmaWriteH(&tSmaH, pTsdb, tParam, pData); + + int32_t nSmaBlocks = pData->numOfBlocks; + int32_t colDataLen = pData->dataLen / nSmaBlocks; + + // Step 2.2: Storage of SMA_STORAGE_LEVEL_DFILESET + // TODO: Use the daysPerFile for rSma data, not for TS data. + // TODO: The lifecycle of rSma data should be processed like the TS data files. + int32_t minFid = (int32_t)(TSDB_KEY_FID(pData->tsWindow.skey, pCfg->daysPerFile, pCfg->precision)); + int32_t maxFid = (int32_t)(TSDB_KEY_FID(pData->tsWindow.ekey, pCfg->daysPerFile, pCfg->precision)); + + if (minFid == maxFid) { + // Save all the TSma data to one file + tsdbSetRSmaDataFile(&tSmaH, param, pData, minFid); + // TODO: tsdbStartTSmaCommit(); + tsdbInsertTSmaDataSection(&tSmaH, pData, colDataLen, nSmaBlocks); + // TODO:tsdbEndTSmaCommit(); + } else if (minFid < maxFid) { + // Split the TSma data and save to multiple files. As there is limit for the span, it can't span more than 2 files + // actually. + // TODO: tsdbStartTSmaCommit(); + int32_t tmpFid = 0; + int32_t step = 0; + for (int32_t n = 0; n < pData->numOfBlocks; ++n) { + } + tsdbInsertTSmaDataSection(&tSmaH, pData, colDataLen, nSmaBlocks); + // TODO:tsdbEndTSmaCommit(); + } else { + TASSERT(0); + return TSDB_CODE_INVALID_PARA; + } + // Step 4: finish + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Init of tSma ReadH + * + * @param pSmaH + * @param pTsdb + * @param param + * @param pData + * @return int32_t + */ +static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData) { + pSmaH->pTsdb = pTsdb; + pSmaH->interval = tsdbGetIntervalByPrecision(param->interval, param->intervalUnit, REPO_CFG(pTsdb)->precision); + pSmaH->blockSize = param->numOfFuncIds * sizeof(int64_t); +} + +/** + * @brief Init of tSma FS + * + * @param pReadH + * @param param + * @param queryWin + * @return int32_t + */ +static int32_t tsdbInitTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWindow *queryWin) { + int32_t storageLevel = tsdbJudgeStorageLevel(param->interval, param->intervalUnit); + int32_t daysPerFile = + storageLevel == SMA_STORAGE_LEVEL_TSDB ? SMA_STORAGE_TSDB_DAYS : REPO_CFG(pReadH->pTsdb)->daysPerFile; + pReadH->storageLevel = storageLevel; + pReadH->days = daysPerFile; + pReadH->smaFsIter.iter = 0; +} + +/** + * @brief Set and open tSma file if it has key locates in queryWin. + * + * @param pReadH + * @param param + * @param queryWin + * @return true + * @return false + */ +static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWindow *queryWin) { + SArray *smaFs = pReadH->pTsdb->fs->cstatus->smaf; + int32_t nSmaFs = taosArrayGetSize(smaFs); + + pReadH->pDFile = NULL; + + while (pReadH->smaFsIter.iter < nSmaFs) { + void *pSmaFile = taosArrayGet(smaFs, pReadH->smaFsIter.iter); + if (pSmaFile) { // match(indexName, queryWindow) + // TODO: select the file by index_name ... + pReadH->pDFile = pSmaFile; + ++pReadH->smaFsIter.iter; + break; + } + ++pReadH->smaFsIter.iter; + } + + if (pReadH->pDFile != NULL) { + tsdbDebug("vg%d: smaFile %s matched", REPO_ID(pReadH->pTsdb), "[pSmaFile dir]"); + return true; + } + + return false; +} + +/** + * @brief Return the data between queryWin and fill the pData. + * + * @param pTsdb + * @param param + * @param pData + * @param queryWin + * @param nMaxResult The query invoker should control the nMaxResult need to return to avoid OOM. + * @return int32_t + */ +int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData, STimeWindow *queryWin, int32_t nMaxResult) { + STSmaReadH tReadH = {0}; + tsdbInitTSmaReadH(&tReadH, pTsdb, param, pData); + + tsdbInitTSmaFile(&tReadH, param, queryWin); + + int32_t nResult = 0; + int64_t lastKey = 0; + + while (true) { + if (nResult >= nMaxResult) { + break; + } + + // set and open the file according to the STSma param + if (tsdbSetAndOpenTSmaFile(&tReadH, param, queryWin)) { + char bTree[100] = "\0"; + while (strncmp(bTree, "has more nodes", 100) == 0) { + if (nResult >= nMaxResult) { + break; + } + // tsdbGetDataFromBTree(bTree, queryWin, lastKey) + // fill the pData + ++nResult; + } + } + } + + // read data from file and fill the result + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Get the start TS key of the last data block of one interval/sliding. + * + * @param pTsdb + * @param param + * @param result + * @return int32_t + * 1) Return 0 and fill the result if the check procedure is normal; + * 2) Return -1 if error occurs during the check procedure. + */ +int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result) { + const char *procedure = ""; + if (strncmp(procedure, "get the start TS key of the last data block", 100) != 0) { + return -1; + } + // fill the result + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Remove the tSma data files related to param between pWin. + * + * @param pTsdb + * @param param + * @param pWin + * @return int32_t + */ +int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin) { + // for ("tSmaFiles of param-interval-sliding between pWin") { + // // remove the tSmaFile + // } + return TSDB_CODE_SUCCESS; +} + +#if 0 +#define TSDB_MAX_SUBBLOCKS 8 + +typedef struct { + STable *pTable; + SSkipListIterator *pIter; +} SCommitIter; + +typedef struct { + SRtn rtn; // retention snapshot + SFSIter fsIter; // tsdb file iterator + int niters; // memory iterators + SCommitIter *iters; + bool isRFileSet; // read and commit FSET + SReadH readh; + SDFileSet wSet; + bool isDFileSame; + bool isLFileSame; + TSKEY minKey; + TSKEY maxKey; + SArray *aBlkIdx; // SBlockIdx array + STable *pTable; + SArray *aSupBlk; // Table super-block array + SArray *aSubBlk; // table sub-block array + SDataCols *pDataCols; +} STSmaWriteH; + +#define TSDB_DEFAULT_BLOCK_ROWS(maxRows) ((maxRows)*4 / 5) + +#define TSDB_COMMIT_REPO(ch) TSDB_READ_REPO(&(ch->readh)) +#define TSDB_COMMIT_REPO_ID(ch) REPO_ID(TSDB_READ_REPO(&(ch->readh))) +#define TSDB_COMMIT_WRITE_FSET(ch) (&((ch)->wSet)) +#define TSDB_COMMIT_TABLE(ch) ((ch)->pTable) +#define TSDB_COMMIT_HEAD_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_HEAD) +#define TSDB_COMMIT_DATA_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_DATA) +#define TSDB_COMMIT_LAST_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_LAST) +#define TSDB_COMMIT_SMAD_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_SMAD) +#define TSDB_COMMIT_SMAL_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_SMAL) +#define TSDB_COMMIT_BUF(ch) TSDB_READ_BUF(&((ch)->readh)) +#define TSDB_COMMIT_COMP_BUF(ch) TSDB_READ_COMP_BUF(&((ch)->readh)) +#define TSDB_COMMIT_DEFAULT_ROWS(ch) TSDB_DEFAULT_BLOCK_ROWS(TSDB_COMMIT_REPO(ch)->config.maxRowsPerFileBlock) +#define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch))) + +static void tsdbStartCommit(STsdb *pRepo); +static void tsdbEndCommit(STsdb *pTsdb, int eno); +static int tsdbInitCommitH(SCommitH *pCommith, STsdb *pRepo); +static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key); +static int tsdbNextCommitFid(SCommitH *pCommith); +static void tsdbDestroyCommitH(SCommitH *pCommith); +static int tsdbCreateCommitIters(SCommitH *pCommith); +static void tsdbDestroyCommitIters(SCommitH *pCommith); +static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid); +static void tsdbResetCommitFile(SCommitH *pCommith); +static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid); +// static int tsdbCommitMeta(STsdbRepo *pRepo); +// static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool compact); +// static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid); +// static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile); +static int tsdbCommitToTable(SCommitH *pCommith, int tid); +static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable); +static int tsdbComparKeyBlock(const void *arg1, const void *arg2); +static int tsdbWriteBlockInfo(SCommitH *pCommih); +static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData); +static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx); +static int tsdbMoveBlock(SCommitH *pCommith, int bidx); +static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const SBlock *pSubBlocks, int nSubBlocks); +static int tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit, + bool isLastOneBlock); +static void tsdbResetCommitTable(SCommitH *pCommith); +static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError); +static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo); +static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, + TSKEY maxKey, int maxRows, int8_t update); +int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf); + +int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) { + SDiskID did; + SDFileSet nSet = {0}; + STsdbFS *pfs = REPO_FS(pRepo); + int level; + + ASSERT(pSet->fid >= pRtn->minFid); + + level = tsdbGetFidLevel(pSet->fid, pRtn); + + if (tfsAllocDisk(pRepo->pTfs, level, &did) < 0) { + terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; + return -1; + } + + if (did.level > TSDB_FSET_LEVEL(pSet)) { + // Need to move the FSET to higher level + tsdbInitDFileSet(pRepo, &nSet, did, pSet->fid, FS_TXN_VERSION(pfs)); + + if (tsdbCopyDFileSet(pSet, &nSet) < 0) { + tsdbError("vgId:%d failed to copy FSET %d from level %d to level %d since %s", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), did.level, tstrerror(terrno)); + return -1; + } + + if (tsdbUpdateDFileSet(pfs, &nSet) < 0) { + return -1; + } + + tsdbInfo("vgId:%d FSET %d is copied from level %d disk id %d to level %d disk id %d", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet), did.level, did.id); + } else { + // On a correct level + if (tsdbUpdateDFileSet(pfs, pSet) < 0) { + return -1; + } + } + + return 0; +} + +int tsdbPrepareCommit(STsdb *pTsdb) { + if (pTsdb->mem == NULL) return 0; + + ASSERT(pTsdb->imem == NULL); + + pTsdb->imem = pTsdb->mem; + pTsdb->mem = NULL; + return 0; +} + +int tsdbCommit(STsdb *pRepo) { + STsdbMemTable *pMem = pRepo->imem; + SCommitH commith = {0}; + SDFileSet *pSet = NULL; + int fid; + + if (pRepo->imem == NULL) return 0; + + tsdbStartCommit(pRepo); + // Resource initialization + if (tsdbInitCommitH(&commith, pRepo) < 0) { + return -1; + } + + // Skip expired memory data and expired FSET + tsdbSeekCommitIter(&commith, commith.rtn.minKey); + while ((pSet = tsdbFSIterNext(&(commith.fsIter)))) { + if (pSet->fid < commith.rtn.minFid) { + tsdbInfo("vgId:%d FSET %d on level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); + } else { + break; + } + } + + // Loop to commit to each file + fid = tsdbNextCommitFid(&(commith)); + while (true) { + // Loop over both on disk and memory + if (pSet == NULL && fid == TSDB_IVLD_FID) break; + + if (pSet && (fid == TSDB_IVLD_FID || pSet->fid < fid)) { + // Only has existing FSET but no memory data to commit in this + // existing FSET, only check if file in correct retention + if (tsdbApplyRtnOnFSet(pRepo, pSet, &(commith.rtn)) < 0) { + tsdbDestroyCommitH(&commith); + return -1; + } + + pSet = tsdbFSIterNext(&(commith.fsIter)); + } else { + // Has memory data to commit + SDFileSet *pCSet; + int cfid; + + if (pSet == NULL || pSet->fid > fid) { + // Commit to a new FSET with fid: fid + pCSet = NULL; + cfid = fid; + } else { + // Commit to an existing FSET + pCSet = pSet; + cfid = pSet->fid; + pSet = tsdbFSIterNext(&(commith.fsIter)); + } + + if (tsdbCommitToFile(&commith, pCSet, cfid) < 0) { + tsdbDestroyCommitH(&commith); + return -1; + } + + fid = tsdbNextCommitFid(&commith); + } + } + + tsdbDestroyCommitH(&commith); + tsdbEndCommit(pRepo, TSDB_CODE_SUCCESS); + + return 0; +} + +void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) { + STsdbCfg *pCfg = REPO_CFG(pRepo); + TSKEY minKey, midKey, maxKey, now; + + now = taosGetTimestamp(pCfg->precision); + minKey = now - pCfg->keep * tsTickPerDay[pCfg->precision]; + midKey = now - pCfg->keep2 * tsTickPerDay[pCfg->precision]; + maxKey = now - pCfg->keep1 * tsTickPerDay[pCfg->precision]; + + pRtn->minKey = minKey; + pRtn->minFid = (int)(TSDB_KEY_FID(minKey, pCfg->daysPerFile, pCfg->precision)); + pRtn->midFid = (int)(TSDB_KEY_FID(midKey, pCfg->daysPerFile, pCfg->precision)); + pRtn->maxFid = (int)(TSDB_KEY_FID(maxKey, pCfg->daysPerFile, pCfg->precision)); + tsdbDebug("vgId:%d now:%" PRId64 " minKey:%" PRId64 " minFid:%d, midFid:%d, maxFid:%d", REPO_ID(pRepo), now, minKey, + pRtn->minFid, pRtn->midFid, pRtn->maxFid); +} + +static void tsdbStartCommit(STsdb *pRepo) { + STsdbMemTable *pMem = pRepo->imem; + + tsdbInfo("vgId:%d start to commit", REPO_ID(pRepo)); + + tsdbStartFSTxn(pRepo, 0, 0); +} + +static void tsdbEndCommit(STsdb *pTsdb, int eno) { + tsdbEndFSTxn(pTsdb); + tsdbFreeMemTable(pTsdb, pTsdb->imem); + pTsdb->imem = NULL; + tsdbInfo("vgId:%d commit over, %s", REPO_ID(pTsdb), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed"); +} + +static int tsdbInitCommitH(SCommitH *pCommith, STsdb *pRepo) { + STsdbCfg *pCfg = REPO_CFG(pRepo); + + memset(pCommith, 0, sizeof(*pCommith)); + tsdbGetRtnSnap(pRepo, &(pCommith->rtn)); + + TSDB_FSET_SET_CLOSED(TSDB_COMMIT_WRITE_FSET(pCommith)); + + // Init read handle + if (tsdbInitReadH(&(pCommith->readh), pRepo) < 0) { + return -1; + } + + // Init file iterator + tsdbFSIterInit(&(pCommith->fsIter), REPO_FS(pRepo), TSDB_FS_ITER_FORWARD); + + if (tsdbCreateCommitIters(pCommith) < 0) { + tsdbDestroyCommitH(pCommith); + return -1; + } + + pCommith->aBlkIdx = taosArrayInit(1024, sizeof(SBlockIdx)); + if (pCommith->aBlkIdx == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyCommitH(pCommith); + return -1; + } + + pCommith->aSupBlk = taosArrayInit(1024, sizeof(SBlock)); + if (pCommith->aSupBlk == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyCommitH(pCommith); + return -1; + } + + pCommith->aSubBlk = taosArrayInit(1024, sizeof(SBlock)); + if (pCommith->aSubBlk == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyCommitH(pCommith); + return -1; + } + + pCommith->pDataCols = tdNewDataCols(0, pCfg->maxRowsPerFileBlock); + if (pCommith->pDataCols == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyCommitH(pCommith); + return -1; + } + + return 0; +} + +// Skip all keys until key (not included) +static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key) { + for (int i = 0; i < pCommith->niters; i++) { + SCommitIter *pIter = pCommith->iters + i; + if (pIter->pTable == NULL || pIter->pIter == NULL) continue; + + tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, key - 1, INT32_MAX, NULL, NULL, 0, true, NULL); + } +} + +static int tsdbNextCommitFid(SCommitH *pCommith) { + STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg *pCfg = REPO_CFG(pRepo); + int fid = TSDB_IVLD_FID; + + for (int i = 0; i < pCommith->niters; i++) { + SCommitIter *pIter = pCommith->iters + i; + // if (pIter->pTable == NULL || pIter->pIter == NULL) continue; + + TSKEY nextKey = tsdbNextIterKey(pIter->pIter); + if (nextKey == TSDB_DATA_TIMESTAMP_NULL) { + continue; + } else { + int tfid = (int)(TSDB_KEY_FID(nextKey, pCfg->daysPerFile, pCfg->precision)); + if (fid == TSDB_IVLD_FID || fid > tfid) { + fid = tfid; + } + } + } + + return fid; +} + +static void tsdbDestroyCommitH(SCommitH *pCommith) { + pCommith->pDataCols = tdFreeDataCols(pCommith->pDataCols); + pCommith->aSubBlk = taosArrayDestroy(pCommith->aSubBlk); + pCommith->aSupBlk = taosArrayDestroy(pCommith->aSupBlk); + pCommith->aBlkIdx = taosArrayDestroy(pCommith->aBlkIdx); + tsdbDestroyCommitIters(pCommith); + tsdbDestroyReadH(&(pCommith->readh)); + tsdbCloseDFileSet(TSDB_COMMIT_WRITE_FSET(pCommith)); +} + +static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { + STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg *pCfg = REPO_CFG(pRepo); + + ASSERT(pSet == NULL || pSet->fid == fid); + + tsdbResetCommitFile(pCommith); + tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, fid, &(pCommith->minKey), &(pCommith->maxKey)); + + // Set and open files + if (tsdbSetAndOpenCommitFile(pCommith, pSet, fid) < 0) { + return -1; + } + + // Loop to commit each table data + for (int tid = 0; tid < pCommith->niters; tid++) { + SCommitIter *pIter = pCommith->iters + tid; + + if (pIter->pTable == NULL) continue; + + if (tsdbCommitToTable(pCommith, tid) < 0) { + tsdbCloseCommitFile(pCommith, true); + // revert the file change + tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); + return -1; + } + } + + if (tsdbWriteBlockIdx(TSDB_COMMIT_HEAD_FILE(pCommith), pCommith->aBlkIdx, (void **)(&(TSDB_COMMIT_BUF(pCommith)))) < + 0) { + tsdbError("vgId:%d failed to write SBlockIdx part to FSET %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); + tsdbCloseCommitFile(pCommith, true); + // revert the file change + tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); + return -1; + } + + if (tsdbUpdateDFileSetHeader(&(pCommith->wSet)) < 0) { + tsdbError("vgId:%d failed to update FSET %d header since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); + tsdbCloseCommitFile(pCommith, true); + // revert the file change + tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); + return -1; + } + + // Close commit file + tsdbCloseCommitFile(pCommith, false); + + if (tsdbUpdateDFileSet(REPO_FS(pRepo), &(pCommith->wSet)) < 0) { + return -1; + } + + return 0; +} + +static int tsdbCreateCommitIters(SCommitH *pCommith) { + STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbMemTable *pMem = pRepo->imem; + SSkipListIterator *pSlIter; + SCommitIter *pCommitIter; + SSkipListNode *pNode; + STbData *pTbData; + + pCommith->niters = SL_SIZE(pMem->pSlIdx); + pCommith->iters = (SCommitIter *)calloc(pCommith->niters, sizeof(SCommitIter)); + if (pCommith->iters == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + // Loop to create iters for each skiplist + pSlIter = tSkipListCreateIter(pMem->pSlIdx); + if (pSlIter == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + for (int i = 0; i < pCommith->niters; i++) { + tSkipListIterNext(pSlIter); + pNode = tSkipListIterGet(pSlIter); + pTbData = (STbData *)pNode->pData; + + pCommitIter = pCommith->iters + i; + pCommitIter->pIter = tSkipListCreateIter(pTbData->pData); + tSkipListIterNext(pCommitIter->pIter); + + pCommitIter->pTable = (STable *)malloc(sizeof(STable)); + pCommitIter->pTable->uid = pTbData->uid; + pCommitIter->pTable->tid = pTbData->uid; + pCommitIter->pTable->pSchema = metaGetTbTSchema(pRepo->pMeta, pTbData->uid, 0); + } + + return 0; +} + +static void tsdbDestroyCommitIters(SCommitH *pCommith) { + if (pCommith->iters == NULL) return; + + for (int i = 1; i < pCommith->niters; i++) { + tSkipListDestroyIter(pCommith->iters[i].pIter); + tdFreeSchema(pCommith->iters[i].pTable->pSchema); + free(pCommith->iters[i].pTable); + } + + free(pCommith->iters); + pCommith->iters = NULL; + pCommith->niters = 0; +} + +static void tsdbResetCommitFile(SCommitH *pCommith) { + pCommith->isRFileSet = false; + pCommith->isDFileSame = false; + pCommith->isLFileSame = false; + taosArrayClear(pCommith->aBlkIdx); +} + +static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { + SDiskID did; + STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); + SDFileSet *pWSet = TSDB_COMMIT_WRITE_FSET(pCommith); + + if (tfsAllocDisk(pRepo->pTfs, tsdbGetFidLevel(fid, &(pCommith->rtn)), &did) < 0) { + terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; + return -1; + } + + // Open read FSET + if (pSet) { + if (tsdbSetAndOpenReadFSet(&(pCommith->readh), pSet) < 0) { + return -1; + } + + pCommith->isRFileSet = true; + + if (tsdbLoadBlockIdx(&(pCommith->readh)) < 0) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + + tsdbDebug("vgId:%d FSET %d at level %d disk id %d is opened to read to commit", REPO_ID(pRepo), TSDB_FSET_FID(pSet), + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); + } else { + pCommith->isRFileSet = false; + } + + // Set and open commit FSET + if (pSet == NULL || did.level > TSDB_FSET_LEVEL(pSet)) { + // Create a new FSET to write data + tsdbInitDFileSet(pRepo, pWSet, did, fid, FS_TXN_VERSION(REPO_FS(pRepo))); + + if (tsdbCreateDFileSet(pRepo, pWSet, true) < 0) { + tsdbError("vgId:%d failed to create FSET %d at level %d disk id %d since %s", REPO_ID(pRepo), + TSDB_FSET_FID(pWSet), TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet), tstrerror(terrno)); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + } + return -1; + } + + pCommith->isDFileSame = false; + pCommith->isLFileSame = false; + + tsdbDebug("vgId:%d FSET %d at level %d disk id %d is created to commit", REPO_ID(pRepo), TSDB_FSET_FID(pWSet), + TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet)); + } else { + did.level = TSDB_FSET_LEVEL(pSet); + did.id = TSDB_FSET_ID(pSet); + + pCommith->wSet.fid = fid; + pCommith->wSet.state = 0; + + // TSDB_FILE_HEAD + SDFile *pWHeadf = TSDB_COMMIT_HEAD_FILE(pCommith); + tsdbInitDFile(pRepo, pWHeadf, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_HEAD); + if (tsdbCreateDFile(pRepo, pWHeadf, true) < 0) { + tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWHeadf), + tstrerror(terrno)); + + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + + // TSDB_FILE_DATA + SDFile *pRDataf = TSDB_READ_DATA_FILE(&(pCommith->readh)); + SDFile *pWDataf = TSDB_COMMIT_DATA_FILE(pCommith); + tsdbInitDFileEx(pWDataf, pRDataf); + if (tsdbOpenDFile(pWDataf, O_WRONLY) < 0) { + tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWDataf), + tstrerror(terrno)); + + tsdbCloseDFileSet(pWSet); + tsdbRemoveDFile(pWHeadf); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + pCommith->isDFileSame = true; + + // TSDB_FILE_LAST + SDFile *pRLastf = TSDB_READ_LAST_FILE(&(pCommith->readh)); + SDFile *pWLastf = TSDB_COMMIT_LAST_FILE(pCommith); + if (pRLastf->info.size < 32 * 1024) { + tsdbInitDFileEx(pWLastf, pRLastf); + pCommith->isLFileSame = true; + + if (tsdbOpenDFile(pWLastf, O_WRONLY) < 0) { + tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), + tstrerror(terrno)); + + tsdbCloseDFileSet(pWSet); + tsdbRemoveDFile(pWHeadf); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + } else { + tsdbInitDFile(pRepo, pWLastf, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_LAST); + pCommith->isLFileSame = false; + + if (tsdbCreateDFile(pRepo, pWLastf, true) < 0) { + tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), + tstrerror(terrno)); + + tsdbCloseDFileSet(pWSet); + (void)tsdbRemoveDFile(pWHeadf); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + } + } + + return 0; +} + +// extern int32_t tsTsdbMetaCompactRatio; + +int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray *pSubA, void **ppBuf, + SBlockIdx *pIdx) { + size_t nSupBlocks; + size_t nSubBlocks; + uint32_t tlen; + SBlockInfo *pBlkInfo; + int64_t offset; + SBlock *pBlock; + + memset(pIdx, 0, sizeof(*pIdx)); + + nSupBlocks = taosArrayGetSize(pSupA); + nSubBlocks = (pSubA == NULL) ? 0 : taosArrayGetSize(pSubA); + + if (nSupBlocks <= 0) { + // No data (data all deleted) + return 0; + } + + tlen = (uint32_t)(sizeof(SBlockInfo) + sizeof(SBlock) * (nSupBlocks + nSubBlocks) + sizeof(TSCKSUM)); + if (tsdbMakeRoom(ppBuf, tlen) < 0) return -1; + pBlkInfo = *ppBuf; + + pBlkInfo->delimiter = TSDB_FILE_DELIMITER; + pBlkInfo->tid = TABLE_TID(pTable); + pBlkInfo->uid = TABLE_UID(pTable); + + memcpy((void *)(pBlkInfo->blocks), taosArrayGet(pSupA, 0), nSupBlocks * sizeof(SBlock)); + if (nSubBlocks > 0) { + memcpy((void *)(pBlkInfo->blocks + nSupBlocks), taosArrayGet(pSubA, 0), nSubBlocks * sizeof(SBlock)); + + for (int i = 0; i < nSupBlocks; i++) { + pBlock = pBlkInfo->blocks + i; + + if (pBlock->numOfSubBlocks > 1) { + pBlock->offset += (sizeof(SBlockInfo) + sizeof(SBlock) * nSupBlocks); + } + } + } + + taosCalcChecksumAppend(0, (uint8_t *)pBlkInfo, tlen); + + if (tsdbAppendDFile(pHeadf, (void *)pBlkInfo, tlen, &offset) < 0) { + return -1; + } + + tsdbUpdateDFileMagic(pHeadf, POINTER_SHIFT(pBlkInfo, tlen - sizeof(TSCKSUM))); + + // Set pIdx + pBlock = taosArrayGetLast(pSupA); + + pIdx->tid = TABLE_TID(pTable); + pIdx->uid = TABLE_UID(pTable); + pIdx->hasLast = pBlock->last ? 1 : 0; + pIdx->maxKey = pBlock->keyLast; + pIdx->numOfBlocks = (uint32_t)nSupBlocks; + pIdx->len = tlen; + pIdx->offset = (uint32_t)offset; + + return 0; +} + +int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { + SBlockIdx *pBlkIdx; + size_t nidx = taosArrayGetSize(pIdxA); + int tlen = 0, size; + int64_t offset; + + if (nidx <= 0) { + // All data are deleted + pHeadf->info.offset = 0; + pHeadf->info.len = 0; + return 0; + } + + for (size_t i = 0; i < nidx; i++) { + pBlkIdx = (SBlockIdx *)taosArrayGet(pIdxA, i); + + size = tsdbEncodeSBlockIdx(NULL, pBlkIdx); + if (tsdbMakeRoom(ppBuf, tlen + size) < 0) return -1; + + void *ptr = POINTER_SHIFT(*ppBuf, tlen); + tsdbEncodeSBlockIdx(&ptr, pBlkIdx); + + tlen += size; + } + + tlen += sizeof(TSCKSUM); + if (tsdbMakeRoom(ppBuf, tlen) < 0) return -1; + taosCalcChecksumAppend(0, (uint8_t *)(*ppBuf), tlen); + + if (tsdbAppendDFile(pHeadf, *ppBuf, tlen, &offset) < tlen) { + return -1; + } + + tsdbUpdateDFileMagic(pHeadf, POINTER_SHIFT(*ppBuf, tlen - sizeof(TSCKSUM))); + pHeadf->info.offset = (uint32_t)offset; + pHeadf->info.len = tlen; + + return 0; +} + +// // =================== Commit Meta Data +// static int tsdbInitCommitMetaFile(STsdbRepo *pRepo, SMFile* pMf, bool open) { +// STsdbFS * pfs = REPO_FS(pRepo); +// SMFile * pOMFile = pfs->cstatus->pmf; +// SDiskID did; + +// // Create/Open a meta file or open the existing file +// if (pOMFile == NULL) { +// // Create a new meta file +// did.level = TFS_PRIMARY_LEVEL; +// did.id = TFS_PRIMARY_ID; +// tsdbInitMFile(pMf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo))); + +// if (open && tsdbCreateMFile(pMf, true) < 0) { +// tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); +// return -1; +// } + +// tsdbInfo("vgId:%d meta file %s is created to commit", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMf)); +// } else { +// tsdbInitMFileEx(pMf, pOMFile); +// if (open && tsdbOpenMFile(pMf, O_WRONLY) < 0) { +// tsdbError("vgId:%d failed to open META file since %s", REPO_ID(pRepo), tstrerror(terrno)); +// return -1; +// } +// } + +// return 0; +// } + +// static int tsdbCommitMeta(STsdbRepo *pRepo) { +// STsdbFS * pfs = REPO_FS(pRepo); +// SMemTable *pMem = pRepo->imem; +// SMFile * pOMFile = pfs->cstatus->pmf; +// SMFile mf; +// SActObj * pAct = NULL; +// SActCont * pCont = NULL; +// SListNode *pNode = NULL; + +// ASSERT(pOMFile != NULL || listNEles(pMem->actList) > 0); + +// if (listNEles(pMem->actList) <= 0) { +// // no meta data to commit, just keep the old meta file +// tsdbUpdateMFile(pfs, pOMFile); +// if (tsTsdbMetaCompactRatio > 0) { +// if (tsdbInitCommitMetaFile(pRepo, &mf, false) < 0) { +// return -1; +// } +// int ret = tsdbCompactMetaFile(pRepo, pfs, &mf); +// if (ret < 0) tsdbError("compact meta file error"); + +// return ret; +// } +// return 0; +// } else { +// if (tsdbInitCommitMetaFile(pRepo, &mf, true) < 0) { +// return -1; +// } +// } + +// // Loop to write +// while ((pNode = tdListPopHead(pMem->actList)) != NULL) { +// pAct = (SActObj *)pNode->data; +// if (pAct->act == TSDB_UPDATE_META) { +// pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj)); +// if (tsdbUpdateMetaRecord(pfs, &mf, pAct->uid, (void *)(pCont->cont), pCont->len, false) < 0) { +// tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, +// tstrerror(terrno)); +// tsdbCloseMFile(&mf); +// (void)tsdbApplyMFileChange(&mf, pOMFile); +// // TODO: need to reload metaCache +// return -1; +// } +// } else if (pAct->act == TSDB_DROP_META) { +// if (tsdbDropMetaRecord(pfs, &mf, pAct->uid) < 0) { +// tsdbError("vgId:%d failed to drop META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, +// tstrerror(terrno)); +// tsdbCloseMFile(&mf); +// tsdbApplyMFileChange(&mf, pOMFile); +// // TODO: need to reload metaCache +// return -1; +// } +// } else { +// ASSERT(false); +// } +// } + +// if (tsdbUpdateMFileHeader(&mf) < 0) { +// tsdbError("vgId:%d failed to update META file header since %s, revert it", REPO_ID(pRepo), tstrerror(terrno)); +// tsdbApplyMFileChange(&mf, pOMFile); +// // TODO: need to reload metaCache +// return -1; +// } + +// TSDB_FILE_FSYNC(&mf); +// tsdbCloseMFile(&mf); +// tsdbUpdateMFile(pfs, &mf); + +// if (tsTsdbMetaCompactRatio > 0 && tsdbCompactMetaFile(pRepo, pfs, &mf) < 0) { +// tsdbError("compact meta file error"); +// } + +// return 0; +// } + +// int tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord) { +// int tlen = 0; +// tlen += taosEncodeFixedU64(buf, pRecord->uid); +// tlen += taosEncodeFixedI64(buf, pRecord->offset); +// tlen += taosEncodeFixedI64(buf, pRecord->size); + +// return tlen; +// } + +// void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord) { +// buf = taosDecodeFixedU64(buf, &(pRecord->uid)); +// buf = taosDecodeFixedI64(buf, &(pRecord->offset)); +// buf = taosDecodeFixedI64(buf, &(pRecord->size)); + +// return buf; +// } + +// static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool compact) { +// char buf[64] = "\0"; +// void * pBuf = buf; +// SKVRecord rInfo; +// int64_t offset; + +// // Seek to end of meta file +// offset = tsdbSeekMFile(pMFile, 0, SEEK_END); +// if (offset < 0) { +// return -1; +// } + +// rInfo.offset = offset; +// rInfo.uid = uid; +// rInfo.size = contLen; + +// int tlen = tsdbEncodeKVRecord((void **)(&pBuf), &rInfo); +// if (tsdbAppendMFile(pMFile, buf, tlen, NULL) < tlen) { +// return -1; +// } + +// if (tsdbAppendMFile(pMFile, cont, contLen, NULL) < contLen) { +// return -1; +// } + +// tsdbUpdateMFileMagic(pMFile, POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM))); + +// SHashObj* cache = compact ? pfs->metaCacheComp : pfs->metaCache; + +// pMFile->info.nRecords++; + +// SKVRecord *pRecord = taosHashGet(cache, (void *)&uid, sizeof(uid)); +// if (pRecord != NULL) { +// pMFile->info.tombSize += (pRecord->size + sizeof(SKVRecord)); +// } else { +// pMFile->info.nRecords++; +// } +// taosHashPut(cache, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo)); + +// return 0; +// } + +// static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) { +// SKVRecord rInfo = {0}; +// char buf[128] = "\0"; + +// SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)(&uid), sizeof(uid)); +// if (pRecord == NULL) { +// tsdbError("failed to drop META record with key %" PRIu64 " since not find", uid); +// return -1; +// } + +// rInfo.offset = -pRecord->offset; +// rInfo.uid = pRecord->uid; +// rInfo.size = pRecord->size; + +// void *pBuf = buf; +// tsdbEncodeKVRecord(&pBuf, &rInfo); + +// if (tsdbAppendMFile(pMFile, buf, sizeof(SKVRecord), NULL) < 0) { +// return -1; +// } + +// pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t *)buf, sizeof(SKVRecord)); +// pMFile->info.nDels++; +// pMFile->info.nRecords--; +// pMFile->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2); + +// taosHashRemove(pfs->metaCache, (void *)(&uid), sizeof(uid)); +// return 0; +// } + +// static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) { +// float delPercent = (float)(pMFile->info.nDels) / (float)(pMFile->info.nRecords); +// float tombPercent = (float)(pMFile->info.tombSize) / (float)(pMFile->info.size); +// float compactRatio = (float)(tsTsdbMetaCompactRatio)/100; + +// if (delPercent < compactRatio && tombPercent < compactRatio) { +// return 0; +// } + +// if (tsdbOpenMFile(pMFile, O_RDONLY) < 0) { +// tsdbError("open meta file %s compact fail", pMFile->f.rname); +// return -1; +// } + +// tsdbInfo("begin compact tsdb meta file, ratio:%d, nDels:%" PRId64 ",nRecords:%" PRId64 ",tombSize:%" PRId64 +// ",size:%" PRId64, +// tsTsdbMetaCompactRatio, pMFile->info.nDels,pMFile->info.nRecords,pMFile->info.tombSize,pMFile->info.size); + +// SMFile mf; +// SDiskID did; + +// // first create tmp meta file +// did.level = TFS_PRIMARY_LEVEL; +// did.id = TFS_PRIMARY_ID; +// tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)) + 1); + +// if (tsdbCreateMFile(&mf, true) < 0) { +// tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); +// return -1; +// } + +// tsdbInfo("vgId:%d meta file %s is created to compact meta data", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf)); + +// // second iterator metaCache +// int code = -1; +// int64_t maxBufSize = 1024; +// SKVRecord *pRecord; +// void *pBuf = NULL; + +// pBuf = malloc((size_t)maxBufSize); +// if (pBuf == NULL) { +// goto _err; +// } + +// // init Comp +// assert(pfs->metaCacheComp == NULL); +// pfs->metaCacheComp = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); +// if (pfs->metaCacheComp == NULL) { +// goto _err; +// } + +// pRecord = taosHashIterate(pfs->metaCache, NULL); +// while (pRecord) { +// if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) { +// tsdbError("vgId:%d failed to seek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), +// tstrerror(terrno)); +// goto _err; +// } +// if (pRecord->size > maxBufSize) { +// maxBufSize = pRecord->size; +// void* tmp = realloc(pBuf, (size_t)maxBufSize); +// if (tmp == NULL) { +// goto _err; +// } +// pBuf = tmp; +// } +// int nread = (int)tsdbReadMFile(pMFile, pBuf, pRecord->size); +// if (nread < 0) { +// tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), +// tstrerror(terrno)); +// goto _err; +// } + +// if (nread < pRecord->size) { +// tsdbError("vgId:%d failed to read file %s since file corrupted, expected read:%" PRId64 " actual read:%d", +// REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), pRecord->size, nread); +// goto _err; +// } + +// if (tsdbUpdateMetaRecord(pfs, &mf, pRecord->uid, pBuf, (int)pRecord->size, true) < 0) { +// tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pRecord->uid, +// tstrerror(terrno)); +// goto _err; +// } + +// pRecord = taosHashIterate(pfs->metaCache, pRecord); +// } +// code = 0; + +// _err: +// if (code == 0) TSDB_FILE_FSYNC(&mf); +// tsdbCloseMFile(&mf); +// tsdbCloseMFile(pMFile); + +// if (code == 0) { +// // rename meta.tmp -> meta +// tsdbInfo("vgId:%d meta file rename %s -> %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf), +// TSDB_FILE_FULL_NAME(pMFile)); taosRename(mf.f.aname,pMFile->f.aname); tstrncpy(mf.f.aname, pMFile->f.aname, +// TSDB_FILENAME_LEN); tstrncpy(mf.f.rname, pMFile->f.rname, TSDB_FILENAME_LEN); +// // update current meta file info +// pfs->nstatus->pmf = NULL; +// tsdbUpdateMFile(pfs, &mf); + +// taosHashCleanup(pfs->metaCache); +// pfs->metaCache = pfs->metaCacheComp; +// pfs->metaCacheComp = NULL; +// } else { +// // remove meta.tmp file +// remove(mf.f.aname); +// taosHashCleanup(pfs->metaCacheComp); +// pfs->metaCacheComp = NULL; +// } + +// tfree(pBuf); + +// ASSERT(mf.info.nDels == 0); +// ASSERT(mf.info.tombSize == 0); + +// tsdbInfo("end compact tsdb meta file,code:%d,nRecords:%" PRId64 ",size:%" PRId64, +// code,mf.info.nRecords,mf.info.size); +// return code; +// } + +// // =================== Commit Time-Series Data +// #if 0 +// static bool tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) { +// for (int i = 0; i < nIters; i++) { +// TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter); +// if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return true; +// } +// return false; +// } +// #endif + +static int tsdbCommitToTable(SCommitH *pCommith, int tid) { + SCommitIter *pIter = pCommith->iters + tid; + TSKEY nextKey = tsdbNextIterKey(pIter->pIter); + + tsdbResetCommitTable(pCommith); + + // Set commit table + if (tsdbSetCommitTable(pCommith, pIter->pTable) < 0) { + return -1; + } + + // No disk data and no memory data, just return + if (pCommith->readh.pBlkIdx == NULL && (nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey)) { + return 0; + } + + // Must has disk data or has memory data + int nBlocks; + int bidx = 0; + SBlock *pBlock; + + if (pCommith->readh.pBlkIdx) { + if (tsdbLoadBlockInfo(&(pCommith->readh), NULL) < 0) { + return -1; + } + + nBlocks = pCommith->readh.pBlkIdx->numOfBlocks; + } else { + nBlocks = 0; + } + + if (bidx < nBlocks) { + pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + } else { + pBlock = NULL; + } + + while (true) { + if (pBlock == NULL && (nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey)) break; + + if ((nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey) || + (pBlock && (!pBlock->last) && tsdbComparKeyBlock((void *)(&nextKey), pBlock) > 0)) { + if (tsdbMoveBlock(pCommith, bidx) < 0) { + return -1; + } + + bidx++; + if (bidx < nBlocks) { + pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + } else { + pBlock = NULL; + } + } else if (pBlock && (pBlock->last || tsdbComparKeyBlock((void *)(&nextKey), pBlock) == 0)) { + // merge pBlock data and memory data + if (tsdbMergeMemData(pCommith, pIter, bidx) < 0) { + return -1; + } + + bidx++; + if (bidx < nBlocks) { + pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + } else { + pBlock = NULL; + } + nextKey = tsdbNextIterKey(pIter->pIter); + } else { + // Only commit memory data + if (pBlock == NULL) { + if (tsdbCommitMemData(pCommith, pIter, pCommith->maxKey, false) < 0) { + return -1; + } + } else { + if (tsdbCommitMemData(pCommith, pIter, pBlock->keyFirst - 1, true) < 0) { + return -1; + } + } + nextKey = tsdbNextIterKey(pIter->pIter); + } + } + + if (tsdbWriteBlockInfo(pCommith) < 0) { + tsdbError("vgId:%d failed to write SBlockInfo part into file %s since %s", TSDB_COMMIT_REPO_ID(pCommith), + TSDB_FILE_FULL_NAME(TSDB_COMMIT_HEAD_FILE(pCommith)), tstrerror(terrno)); + return -1; + } + + return 0; +} + +static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable) { + STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); + + pCommith->pTable = pTable; + + if (tdInitDataCols(pCommith->pDataCols, pSchema) < 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + if (pCommith->isRFileSet) { + if (tsdbSetReadTable(&(pCommith->readh), pTable) < 0) { + return -1; + } + } else { + pCommith->readh.pBlkIdx = NULL; + } + return 0; +} + +static int tsdbComparKeyBlock(const void *arg1, const void *arg2) { + TSKEY key = *(TSKEY *)arg1; + SBlock *pBlock = (SBlock *)arg2; + + if (key < pBlock->keyFirst) { + return -1; + } else if (key > pBlock->keyLast) { + return 1; + } else { + return 0; + } +} + +int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock, bool isLast, + bool isSuper, void **ppBuf, void **ppCBuf) { + STsdbCfg *pCfg = REPO_CFG(pRepo); + SBlockData *pBlockData; + int64_t offset = 0; + int rowsToWrite = pDataCols->numOfRows; + + ASSERT(rowsToWrite > 0 && rowsToWrite <= pCfg->maxRowsPerFileBlock); + ASSERT((!isLast) || rowsToWrite < pCfg->minRowsPerFileBlock); + + // Make buffer space + if (tsdbMakeRoom(ppBuf, TSDB_BLOCK_STATIS_SIZE(pDataCols->numOfCols)) < 0) { + return -1; + } + pBlockData = (SBlockData *)(*ppBuf); + + // Get # of cols not all NULL(not including key column) + int nColsNotAllNull = 0; + for (int ncol = 1; ncol < pDataCols->numOfCols; ncol++) { // ncol from 1, we skip the timestamp column + SDataCol *pDataCol = pDataCols->cols + ncol; + SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull; + + if (isAllRowsNull(pDataCol)) { // all data to commit are NULL, just ignore it + continue; + } + + memset(pBlockCol, 0, sizeof(*pBlockCol)); + + pBlockCol->colId = pDataCol->colId; + pBlockCol->type = pDataCol->type; + if (tDataTypes[pDataCol->type].statisFunc) { + (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pBlockCol->min), &(pBlockCol->max), + &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex), + &(pBlockCol->numOfNull)); + if (pBlockCol->numOfNull == 0) { + TD_SET_COL_ROWS_NORM(pBlockCol); + } else { + TD_SET_COL_ROWS_MISC(pBlockCol); + } + } else { + TD_SET_COL_ROWS_MISC(pBlockCol); + } + nColsNotAllNull++; + } + + ASSERT(nColsNotAllNull >= 0 && nColsNotAllNull <= pDataCols->numOfCols); + + // Compress the data if neccessary + int tcol = 0; // counter of not all NULL and written columns + uint32_t toffset = 0; + int32_t tsize = TSDB_BLOCK_STATIS_SIZE(nColsNotAllNull); + int32_t lsize = tsize; + int32_t keyLen = 0; + int32_t nBitmaps = (int32_t)TD_BITMAP_BYTES(rowsToWrite); + int32_t tBitmaps = 0; + + for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) { + // All not NULL columns finish + if (ncol != 0 && tcol >= nColsNotAllNull) break; + + SDataCol *pDataCol = pDataCols->cols + ncol; + SBlockCol *pBlockCol = pBlockData->cols + tcol; + + if (ncol != 0 && (pDataCol->colId != pBlockCol->colId)) continue; + + int32_t flen; // final length + int32_t tlen = dataColGetNEleLen(pDataCol, rowsToWrite); + +#ifdef TD_SUPPORT_BITMAP + int32_t tBitmaps = 0; + if ((ncol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) { + if (IS_VAR_DATA_TYPE(pDataCol->type)) { + tBitmaps = nBitmaps; + tlen += tBitmaps; + } else { + tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]); + tlen += tBitmaps * TYPE_BYTES[pDataCol->type]; + } + // move bitmap parts ahead + // TODO: put bitmap part to the 1st location(pBitmap points to pData) to avoid the memmove + memcpy(POINTER_SHIFT(pDataCol->pData, pDataCol->len), pDataCol->pBitmap, nBitmaps); + } +#endif + + void *tptr; + + // Make room + if (tsdbMakeRoom(ppBuf, lsize + tlen + COMP_OVERFLOW_BYTES + sizeof(TSCKSUM)) < 0) { + return -1; + } + pBlockData = (SBlockData *)(*ppBuf); + pBlockCol = pBlockData->cols + tcol; + tptr = POINTER_SHIFT(pBlockData, lsize); + + if (pCfg->compression == TWO_STAGE_COMP && tsdbMakeRoom(ppCBuf, tlen + COMP_OVERFLOW_BYTES) < 0) { + return -1; + } + + // Compress or just copy + if (pCfg->compression) { + flen = (*(tDataTypes[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite + tBitmaps, tptr, + tlen + COMP_OVERFLOW_BYTES, pCfg->compression, *ppCBuf, + tlen + COMP_OVERFLOW_BYTES); + } else { + flen = tlen; + memcpy(tptr, pDataCol->pData, flen); + } + + // Add checksum + ASSERT(flen > 0); + flen += sizeof(TSCKSUM); + taosCalcChecksumAppend(0, (uint8_t *)tptr, flen); + tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM))); + + if (ncol != 0) { + tsdbSetBlockColOffset(pBlockCol, toffset); + pBlockCol->len = flen; + tcol++; + } else { + keyLen = flen; + } + + toffset += flen; + lsize += flen; + } + + pBlockData->delimiter = TSDB_FILE_DELIMITER; + pBlockData->uid = TABLE_UID(pTable); + pBlockData->numOfCols = nColsNotAllNull; + + taosCalcChecksumAppend(0, (uint8_t *)pBlockData, tsize); + tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(pBlockData, tsize - sizeof(TSCKSUM))); + + // Write the whole block to file + if (tsdbAppendDFile(pDFile, (void *)pBlockData, lsize, &offset) < lsize) { + return -1; + } + + // Update pBlock membership vairables + pBlock->last = isLast; + pBlock->offset = offset; + pBlock->algorithm = pCfg->compression; + pBlock->numOfRows = rowsToWrite; + pBlock->len = lsize; + pBlock->keyLen = keyLen; + pBlock->numOfSubBlocks = isSuper ? 1 : 0; + pBlock->numOfCols = nColsNotAllNull; + pBlock->keyFirst = dataColsKeyFirst(pDataCols); + pBlock->keyLast = dataColsKeyLast(pDataCols); + + tsdbDebug("vgId:%d uid:%" PRId64 " a block of data is written to file %s, offset %" PRId64 + " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, + REPO_ID(pRepo), TABLE_TID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len, + pBlock->numOfCols, pBlock->keyFirst, pBlock->keyLast); + + return 0; +} + +static int tsdbWriteBlock(SCommitH *pCommith, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock, bool isLast, + bool isSuper) { + return tsdbWriteBlockImpl(TSDB_COMMIT_REPO(pCommith), TSDB_COMMIT_TABLE(pCommith), pDFile, pDataCols, pBlock, isLast, + isSuper, (void **)(&(TSDB_COMMIT_BUF(pCommith))), + (void **)(&(TSDB_COMMIT_COMP_BUF(pCommith)))); +} + +static int tsdbWriteBlockInfo(SCommitH *pCommih) { + SDFile *pHeadf = TSDB_COMMIT_HEAD_FILE(pCommih); + SBlockIdx blkIdx; + STable *pTable = TSDB_COMMIT_TABLE(pCommih); + + if (tsdbWriteBlockInfoImpl(pHeadf, pTable, pCommih->aSupBlk, pCommih->aSubBlk, (void **)(&(TSDB_COMMIT_BUF(pCommih))), + &blkIdx) < 0) { + return -1; + } + + if (blkIdx.numOfBlocks == 0) { + return 0; + } + + if (taosArrayPush(pCommih->aBlkIdx, (void *)(&blkIdx)) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData) { + STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg *pCfg = REPO_CFG(pRepo); + SMergeInfo mInfo; + int32_t defaultRows = TSDB_COMMIT_DEFAULT_ROWS(pCommith); + SDFile *pDFile; + bool isLast; + SBlock block; + + while (true) { + tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, keyLimit, defaultRows, pCommith->pDataCols, NULL, 0, + pCfg->update, &mInfo); + + if (pCommith->pDataCols->numOfRows <= 0) break; + + if (toData || pCommith->pDataCols->numOfRows >= pCfg->minRowsPerFileBlock) { + pDFile = TSDB_COMMIT_DATA_FILE(pCommith); + isLast = false; + } else { + pDFile = TSDB_COMMIT_LAST_FILE(pCommith); + isLast = true; + } + + if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1; + + if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) { + return -1; + } + } + + return 0; +} + +static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx) { + STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg *pCfg = REPO_CFG(pRepo); + int nBlocks = pCommith->readh.pBlkIdx->numOfBlocks; + SBlock *pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + TSKEY keyLimit; + int16_t colId = PRIMARYKEY_TIMESTAMP_COL_ID; + SMergeInfo mInfo; + SBlock subBlocks[TSDB_MAX_SUBBLOCKS]; + SBlock block, supBlock; + SDFile *pDFile; + + if (bidx == nBlocks - 1) { + keyLimit = pCommith->maxKey; + } else { + keyLimit = pBlock[1].keyFirst - 1; + } + + SSkipListIterator titer = *(pIter->pIter); + if (tsdbLoadBlockDataCols(&(pCommith->readh), pBlock, NULL, &colId, 1) < 0) return -1; + + tsdbLoadDataFromCache(pIter->pTable, &titer, keyLimit, INT32_MAX, NULL, pCommith->readh.pDCols[0]->cols[0].pData, + pCommith->readh.pDCols[0]->numOfRows, pCfg->update, &mInfo); + + if (mInfo.nOperations == 0) { + // no new data to insert (all updates denied) + if (tsdbMoveBlock(pCommith, bidx) < 0) { + return -1; + } + *(pIter->pIter) = titer; + } else if (pBlock->numOfRows + mInfo.rowsInserted - mInfo.rowsDeleteSucceed == 0) { + // Ignore the block + ASSERT(0); + *(pIter->pIter) = titer; + } else if (tsdbCanAddSubBlock(pCommith, pBlock, &mInfo)) { + // Add a sub-block + tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, keyLimit, INT32_MAX, pCommith->pDataCols, + pCommith->readh.pDCols[0]->cols[0].pData, pCommith->readh.pDCols[0]->numOfRows, pCfg->update, + &mInfo); + if (pBlock->last) { + pDFile = TSDB_COMMIT_LAST_FILE(pCommith); + } else { + pDFile = TSDB_COMMIT_DATA_FILE(pCommith); + } + + if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, pBlock->last, false) < 0) return -1; + + if (pBlock->numOfSubBlocks == 1) { + subBlocks[0] = *pBlock; + subBlocks[0].numOfSubBlocks = 0; + } else { + memcpy(subBlocks, POINTER_SHIFT(pCommith->readh.pBlkInfo, pBlock->offset), + sizeof(SBlock) * pBlock->numOfSubBlocks); + } + subBlocks[pBlock->numOfSubBlocks] = block; + supBlock = *pBlock; + supBlock.keyFirst = mInfo.keyFirst; + supBlock.keyLast = mInfo.keyLast; + supBlock.numOfSubBlocks++; + supBlock.numOfRows = pBlock->numOfRows + mInfo.rowsInserted - mInfo.rowsDeleteSucceed; + supBlock.offset = taosArrayGetSize(pCommith->aSubBlk) * sizeof(SBlock); + + if (tsdbCommitAddBlock(pCommith, &supBlock, subBlocks, supBlock.numOfSubBlocks) < 0) return -1; + } else { + if (tsdbLoadBlockData(&(pCommith->readh), pBlock, NULL) < 0) return -1; + if (tsdbMergeBlockData(pCommith, pIter, pCommith->readh.pDCols[0], keyLimit, bidx == (nBlocks - 1)) < 0) return -1; + } + + return 0; +} + +static int tsdbMoveBlock(SCommitH *pCommith, int bidx) { + SBlock *pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + SDFile *pDFile; + SBlock block; + bool isSameFile; + + ASSERT(pBlock->numOfSubBlocks > 0); + + if (pBlock->last) { + pDFile = TSDB_COMMIT_LAST_FILE(pCommith); + isSameFile = pCommith->isLFileSame; + } else { + pDFile = TSDB_COMMIT_DATA_FILE(pCommith); + isSameFile = pCommith->isDFileSame; + } + + if (isSameFile) { + if (pBlock->numOfSubBlocks == 1) { + if (tsdbCommitAddBlock(pCommith, pBlock, NULL, 0) < 0) { + return -1; + } + } else { + block = *pBlock; + block.offset = sizeof(SBlock) * taosArrayGetSize(pCommith->aSubBlk); + + if (tsdbCommitAddBlock(pCommith, &block, POINTER_SHIFT(pCommith->readh.pBlkInfo, pBlock->offset), + pBlock->numOfSubBlocks) < 0) { + return -1; + } + } + } else { + if (tsdbLoadBlockData(&(pCommith->readh), pBlock, NULL) < 0) return -1; + if (tsdbWriteBlock(pCommith, pDFile, pCommith->readh.pDCols[0], &block, pBlock->last, true) < 0) return -1; + if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) return -1; + } + + return 0; +} + +static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const SBlock *pSubBlocks, int nSubBlocks) { + if (taosArrayPush(pCommith->aSupBlk, pSupBlock) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + if (pSubBlocks && taosArrayAddBatch(pCommith->aSubBlk, pSubBlocks, nSubBlocks) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit, + bool isLastOneBlock) { + STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg *pCfg = REPO_CFG(pRepo); + SBlock block; + SDFile *pDFile; + bool isLast; + int32_t defaultRows = TSDB_COMMIT_DEFAULT_ROWS(pCommith); + + int biter = 0; + while (true) { + tsdbLoadAndMergeFromCache(pCommith->readh.pDCols[0], &biter, pIter, pCommith->pDataCols, keyLimit, defaultRows, + pCfg->update); + + if (pCommith->pDataCols->numOfRows == 0) break; + + if (isLastOneBlock) { + if (pCommith->pDataCols->numOfRows < pCfg->minRowsPerFileBlock) { + pDFile = TSDB_COMMIT_LAST_FILE(pCommith); + isLast = true; + } else { + pDFile = TSDB_COMMIT_DATA_FILE(pCommith); + isLast = false; + } + } else { + pDFile = TSDB_COMMIT_DATA_FILE(pCommith); + isLast = false; + } + + if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1; + if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) return -1; + } + + return 0; +} + +static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, + TSKEY maxKey, int maxRows, int8_t update) { + TSKEY key1 = INT64_MAX; + TSKEY key2 = INT64_MAX; + STSchema *pSchema = NULL; + + ASSERT(maxRows > 0 && dataColsKeyLast(pDataCols) <= maxKey); + tdResetDataCols(pTarget); + + while (true) { + key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter); + STSRow *row = tsdbNextIterRow(pCommitIter->pIter); + if (row == NULL || TD_ROW_KEY(row) > maxKey) { + key2 = INT64_MAX; + } else { + key2 = TD_ROW_KEY(row); + } + + if (key1 == INT64_MAX && key2 == INT64_MAX) break; + + if (key1 < key2) { + for (int i = 0; i < pDataCols->numOfCols; i++) { + // TODO: dataColAppendVal may fail + SCellVal sVal = {0}; + if (tdGetColDataOfRow(&sVal, pDataCols->cols + i, *iter) < 0) { + TASSERT(0); + } + tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints); + } + + pTarget->numOfRows++; + (*iter)++; + } else if (key1 > key2) { + if (pSchema == NULL || schemaVersion(pSchema) != TD_ROW_SVER(row)) { + pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, TD_ROW_SVER(row)); + ASSERT(pSchema != NULL); + } + + tdAppendSTSRowToDataCol(row, pSchema, pTarget, true); + + tSkipListIterNext(pCommitIter->pIter); + } else { + if (update != TD_ROW_OVERWRITE_UPDATE) { + // copy disk data + for (int i = 0; i < pDataCols->numOfCols; i++) { + // TODO: dataColAppendVal may fail + SCellVal sVal = {0}; + if (tdGetColDataOfRow(&sVal, pDataCols->cols + i, *iter) < 0) { + TASSERT(0); + } + tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints); + } + + if (update == TD_ROW_DISCARD_UPDATE) pTarget->numOfRows++; + } + if (update != TD_ROW_DISCARD_UPDATE) { + // copy mem data + if (pSchema == NULL || schemaVersion(pSchema) != TD_ROW_SVER(row)) { + pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, TD_ROW_SVER(row)); + ASSERT(pSchema != NULL); + } + + tdAppendSTSRowToDataCol(row, pSchema, pTarget, update == TD_ROW_OVERWRITE_UPDATE); + } + (*iter)++; + tSkipListIterNext(pCommitIter->pIter); + } + + if (pTarget->numOfRows >= maxRows) break; + } +} + +static void tsdbResetCommitTable(SCommitH *pCommith) { + taosArrayClear(pCommith->aSubBlk); + taosArrayClear(pCommith->aSupBlk); + pCommith->pTable = NULL; +} + +static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError) { + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + } + + if (!hasError) { + TSDB_FSET_FSYNC(TSDB_COMMIT_WRITE_FSET(pCommith)); + } + tsdbCloseDFileSet(TSDB_COMMIT_WRITE_FSET(pCommith)); +} + +static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo) { + STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg *pCfg = REPO_CFG(pRepo); + int mergeRows = pBlock->numOfRows + pInfo->rowsInserted - pInfo->rowsDeleteSucceed; + + ASSERT(mergeRows > 0); + + if (pBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && pInfo->nOperations <= pCfg->maxRowsPerFileBlock) { + if (pBlock->last) { + if (pCommith->isLFileSame && mergeRows < pCfg->minRowsPerFileBlock) return true; + } else { + if (pCommith->isDFileSame && mergeRows <= pCfg->maxRowsPerFileBlock) return true; + } + } + + return false; +} + +// int tsdbApplyRtn(STsdbRepo *pRepo) { +// SRtn rtn; +// SFSIter fsiter; +// STsdbFS * pfs = REPO_FS(pRepo); +// SDFileSet *pSet; + +// // Get retention snapshot +// tsdbGetRtnSnap(pRepo, &rtn); + +// tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); +// while ((pSet = tsdbFSIterNext(&fsiter))) { +// if (pSet->fid < rtn.minFid) { +// tsdbInfo("vgId:%d FSET %d at level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, +// TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); +// continue; +// } + +// if (tsdbApplyRtnOnFSet(pRepo, pSet, &rtn) < 0) { +// return -1; +// } +// } + +// return 0; +// } +#endif \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 78067f8f83..ba8eea809e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -15,6 +15,14 @@ #include "tsdbDef.h" +/** + * @brief insert TS data + * + * @param pTsdb + * @param pMsg + * @param pRsp + * @return int + */ int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp) { // Check if mem is there. If not, create one. if (pTsdb->mem == NULL) { @@ -24,4 +32,37 @@ int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp) { } } return tsdbMemTableInsert(pTsdb, pTsdb->mem, pMsg, NULL); +} + +/** + * @brief Insert/Update tSma(Time-range-wise SMA) data from stream computing engine + * + * @param pTsdb + * @param param + * @param pData + * @return int32_t + * TODO: Who is responsible for resource allocate and release? + */ +int32_t tsdbInsertTSmaData(STsdb *pTsdb, STSma *param, STSmaData *pData) { + int32_t code = TSDB_CODE_SUCCESS; + if ((code = tsdbInsertTSmaDataImpl(pTsdb, param, pData)) < 0) { + tsdbWarn("vgId:%d insert tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); + } + return code; +} + +/** + * @brief Insert Time-range-wise Rollup Sma(RSma) data + * + * @param pTsdb + * @param param + * @param pData + * @return int32_t + */ +int32_t tsdbInsertRSmaData(STsdb *pTsdb, SRSma *param, STSmaData *pData) { + int32_t code = TSDB_CODE_SUCCESS; + if ((code = tsdbInsertRSmaDataImpl(pTsdb, param, pData)) < 0) { + tsdbWarn("vgId:%d insert rSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); + } + return code; } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index a2616307ff..f3f21dc9c0 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -132,6 +132,15 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { if (tqProcessRebReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { } } break; + case TDMT_VND_CREATE_SMA: { // timeRangeSMA + // 1. tdCreateSmaMeta(pVnode->pMeta,...); + // 2. tdCreateSmaDataInit(); + // 3. tdCreateSmaData + } break; + case TDMT_VND_CANCEL_SMA: { // timeRangeSMA + } break; + case TDMT_VND_DROP_SMA: { // timeRangeSMA + } break; default: ASSERT(0); break; diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index 986986aa70..97157fc49c 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -95,7 +95,7 @@ TEST(testCase, tSmaEncodeDecodeTest) { // resource release tdDestroyTSma(&tSma, false); - tdDestroyTSmaWrapper(&dstTSmaWrapper); + tdDestroyTSmaWrapper(&dstTSmaWrapper, false); } TEST(testCase, tSma_DB_Put_Get_Del_Test) { @@ -161,7 +161,7 @@ TEST(testCase, tSma_DB_Put_Get_Del_Test) { EXPECT_EQ(qSmaCfg->interval, tSma.interval); tdDestroyTSma(qSmaCfg, true); - // get value by table uid + // get index name by table uid SMSmaCursor *pSmaCur = metaOpenSmaCursor(pMeta, tbUid); assert(pSmaCur != NULL); uint32_t indexCnt = 0; @@ -176,6 +176,15 @@ TEST(testCase, tSma_DB_Put_Get_Del_Test) { EXPECT_EQ(indexCnt, 2); metaCloseSmaCurosr(pSmaCur); + // get wrapper by table uid + STSmaWrapper *pSW = metaGetSmaInfoByUid(pMeta, tbUid); + assert(pSW != NULL); + EXPECT_EQ(pSW->number, 2); + EXPECT_STRCASEEQ(pSW->tSma->indexName, smaIndexName1); + EXPECT_EQ(pSW->tSma->tableUid, tSma.tableUid); + EXPECT_STRCASEEQ((pSW->tSma + 1)->indexName, smaIndexName2); + EXPECT_EQ((pSW->tSma + 1)->tableUid, tSma.tableUid); + // resource release metaRemoveSmaFromDb(pMeta, smaIndexName1); metaRemoveSmaFromDb(pMeta, smaIndexName2); @@ -197,15 +206,15 @@ TEST(testCase, tSmaInsertTest) { int32_t blockSize = tSma.numOfFuncIds * sizeof(int64_t); int32_t numOfColIds = 3; - int32_t numOfSmaBlocks = 10; + int32_t numOfBlocks = 10; - int32_t dataLen = numOfColIds * numOfSmaBlocks * blockSize; + int32_t dataLen = numOfColIds * numOfBlocks * blockSize; pSmaData = (STSmaData*)malloc(sizeof(STSmaData) + dataLen); ASSERT_EQ(pSmaData != NULL, true); pSmaData->tableUid = 3232329230; pSmaData->numOfColIds = numOfColIds; - pSmaData->numOfSmaBlocks = numOfSmaBlocks; + pSmaData->numOfBlocks = numOfBlocks; pSmaData->dataLen = dataLen; pSmaData->tsWindow.skey = 1640000000; pSmaData->tsWindow.ekey = 1645788649; From 8eb339bbd6b3ad045a62f749166e9c9938cdf9fa Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 6 Mar 2022 19:48:23 +0800 Subject: [PATCH 15/17] code optimization --- source/dnode/vnode/src/tsdb/tsdbSma.c | 1628 ------------------------- 1 file changed, 1628 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 297c2783ec..751438c12d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -549,1632 +549,4 @@ int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin) { return TSDB_CODE_SUCCESS; } -#if 0 -#define TSDB_MAX_SUBBLOCKS 8 - -typedef struct { - STable *pTable; - SSkipListIterator *pIter; -} SCommitIter; - -typedef struct { - SRtn rtn; // retention snapshot - SFSIter fsIter; // tsdb file iterator - int niters; // memory iterators - SCommitIter *iters; - bool isRFileSet; // read and commit FSET - SReadH readh; - SDFileSet wSet; - bool isDFileSame; - bool isLFileSame; - TSKEY minKey; - TSKEY maxKey; - SArray *aBlkIdx; // SBlockIdx array - STable *pTable; - SArray *aSupBlk; // Table super-block array - SArray *aSubBlk; // table sub-block array - SDataCols *pDataCols; -} STSmaWriteH; - -#define TSDB_DEFAULT_BLOCK_ROWS(maxRows) ((maxRows)*4 / 5) - -#define TSDB_COMMIT_REPO(ch) TSDB_READ_REPO(&(ch->readh)) -#define TSDB_COMMIT_REPO_ID(ch) REPO_ID(TSDB_READ_REPO(&(ch->readh))) -#define TSDB_COMMIT_WRITE_FSET(ch) (&((ch)->wSet)) -#define TSDB_COMMIT_TABLE(ch) ((ch)->pTable) -#define TSDB_COMMIT_HEAD_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_HEAD) -#define TSDB_COMMIT_DATA_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_DATA) -#define TSDB_COMMIT_LAST_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_LAST) -#define TSDB_COMMIT_SMAD_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_SMAD) -#define TSDB_COMMIT_SMAL_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_SMAL) -#define TSDB_COMMIT_BUF(ch) TSDB_READ_BUF(&((ch)->readh)) -#define TSDB_COMMIT_COMP_BUF(ch) TSDB_READ_COMP_BUF(&((ch)->readh)) -#define TSDB_COMMIT_DEFAULT_ROWS(ch) TSDB_DEFAULT_BLOCK_ROWS(TSDB_COMMIT_REPO(ch)->config.maxRowsPerFileBlock) -#define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch))) - -static void tsdbStartCommit(STsdb *pRepo); -static void tsdbEndCommit(STsdb *pTsdb, int eno); -static int tsdbInitCommitH(SCommitH *pCommith, STsdb *pRepo); -static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key); -static int tsdbNextCommitFid(SCommitH *pCommith); -static void tsdbDestroyCommitH(SCommitH *pCommith); -static int tsdbCreateCommitIters(SCommitH *pCommith); -static void tsdbDestroyCommitIters(SCommitH *pCommith); -static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid); -static void tsdbResetCommitFile(SCommitH *pCommith); -static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid); -// static int tsdbCommitMeta(STsdbRepo *pRepo); -// static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool compact); -// static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid); -// static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile); -static int tsdbCommitToTable(SCommitH *pCommith, int tid); -static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable); -static int tsdbComparKeyBlock(const void *arg1, const void *arg2); -static int tsdbWriteBlockInfo(SCommitH *pCommih); -static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData); -static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx); -static int tsdbMoveBlock(SCommitH *pCommith, int bidx); -static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const SBlock *pSubBlocks, int nSubBlocks); -static int tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit, - bool isLastOneBlock); -static void tsdbResetCommitTable(SCommitH *pCommith); -static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError); -static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo); -static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, - TSKEY maxKey, int maxRows, int8_t update); -int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf); - -int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) { - SDiskID did; - SDFileSet nSet = {0}; - STsdbFS *pfs = REPO_FS(pRepo); - int level; - - ASSERT(pSet->fid >= pRtn->minFid); - - level = tsdbGetFidLevel(pSet->fid, pRtn); - - if (tfsAllocDisk(pRepo->pTfs, level, &did) < 0) { - terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; - return -1; - } - - if (did.level > TSDB_FSET_LEVEL(pSet)) { - // Need to move the FSET to higher level - tsdbInitDFileSet(pRepo, &nSet, did, pSet->fid, FS_TXN_VERSION(pfs)); - - if (tsdbCopyDFileSet(pSet, &nSet) < 0) { - tsdbError("vgId:%d failed to copy FSET %d from level %d to level %d since %s", REPO_ID(pRepo), pSet->fid, - TSDB_FSET_LEVEL(pSet), did.level, tstrerror(terrno)); - return -1; - } - - if (tsdbUpdateDFileSet(pfs, &nSet) < 0) { - return -1; - } - - tsdbInfo("vgId:%d FSET %d is copied from level %d disk id %d to level %d disk id %d", REPO_ID(pRepo), pSet->fid, - TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet), did.level, did.id); - } else { - // On a correct level - if (tsdbUpdateDFileSet(pfs, pSet) < 0) { - return -1; - } - } - - return 0; -} - -int tsdbPrepareCommit(STsdb *pTsdb) { - if (pTsdb->mem == NULL) return 0; - - ASSERT(pTsdb->imem == NULL); - - pTsdb->imem = pTsdb->mem; - pTsdb->mem = NULL; - return 0; -} - -int tsdbCommit(STsdb *pRepo) { - STsdbMemTable *pMem = pRepo->imem; - SCommitH commith = {0}; - SDFileSet *pSet = NULL; - int fid; - - if (pRepo->imem == NULL) return 0; - - tsdbStartCommit(pRepo); - // Resource initialization - if (tsdbInitCommitH(&commith, pRepo) < 0) { - return -1; - } - - // Skip expired memory data and expired FSET - tsdbSeekCommitIter(&commith, commith.rtn.minKey); - while ((pSet = tsdbFSIterNext(&(commith.fsIter)))) { - if (pSet->fid < commith.rtn.minFid) { - tsdbInfo("vgId:%d FSET %d on level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, - TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); - } else { - break; - } - } - - // Loop to commit to each file - fid = tsdbNextCommitFid(&(commith)); - while (true) { - // Loop over both on disk and memory - if (pSet == NULL && fid == TSDB_IVLD_FID) break; - - if (pSet && (fid == TSDB_IVLD_FID || pSet->fid < fid)) { - // Only has existing FSET but no memory data to commit in this - // existing FSET, only check if file in correct retention - if (tsdbApplyRtnOnFSet(pRepo, pSet, &(commith.rtn)) < 0) { - tsdbDestroyCommitH(&commith); - return -1; - } - - pSet = tsdbFSIterNext(&(commith.fsIter)); - } else { - // Has memory data to commit - SDFileSet *pCSet; - int cfid; - - if (pSet == NULL || pSet->fid > fid) { - // Commit to a new FSET with fid: fid - pCSet = NULL; - cfid = fid; - } else { - // Commit to an existing FSET - pCSet = pSet; - cfid = pSet->fid; - pSet = tsdbFSIterNext(&(commith.fsIter)); - } - - if (tsdbCommitToFile(&commith, pCSet, cfid) < 0) { - tsdbDestroyCommitH(&commith); - return -1; - } - - fid = tsdbNextCommitFid(&commith); - } - } - - tsdbDestroyCommitH(&commith); - tsdbEndCommit(pRepo, TSDB_CODE_SUCCESS); - - return 0; -} - -void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) { - STsdbCfg *pCfg = REPO_CFG(pRepo); - TSKEY minKey, midKey, maxKey, now; - - now = taosGetTimestamp(pCfg->precision); - minKey = now - pCfg->keep * tsTickPerDay[pCfg->precision]; - midKey = now - pCfg->keep2 * tsTickPerDay[pCfg->precision]; - maxKey = now - pCfg->keep1 * tsTickPerDay[pCfg->precision]; - - pRtn->minKey = minKey; - pRtn->minFid = (int)(TSDB_KEY_FID(minKey, pCfg->daysPerFile, pCfg->precision)); - pRtn->midFid = (int)(TSDB_KEY_FID(midKey, pCfg->daysPerFile, pCfg->precision)); - pRtn->maxFid = (int)(TSDB_KEY_FID(maxKey, pCfg->daysPerFile, pCfg->precision)); - tsdbDebug("vgId:%d now:%" PRId64 " minKey:%" PRId64 " minFid:%d, midFid:%d, maxFid:%d", REPO_ID(pRepo), now, minKey, - pRtn->minFid, pRtn->midFid, pRtn->maxFid); -} - -static void tsdbStartCommit(STsdb *pRepo) { - STsdbMemTable *pMem = pRepo->imem; - - tsdbInfo("vgId:%d start to commit", REPO_ID(pRepo)); - - tsdbStartFSTxn(pRepo, 0, 0); -} - -static void tsdbEndCommit(STsdb *pTsdb, int eno) { - tsdbEndFSTxn(pTsdb); - tsdbFreeMemTable(pTsdb, pTsdb->imem); - pTsdb->imem = NULL; - tsdbInfo("vgId:%d commit over, %s", REPO_ID(pTsdb), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed"); -} - -static int tsdbInitCommitH(SCommitH *pCommith, STsdb *pRepo) { - STsdbCfg *pCfg = REPO_CFG(pRepo); - - memset(pCommith, 0, sizeof(*pCommith)); - tsdbGetRtnSnap(pRepo, &(pCommith->rtn)); - - TSDB_FSET_SET_CLOSED(TSDB_COMMIT_WRITE_FSET(pCommith)); - - // Init read handle - if (tsdbInitReadH(&(pCommith->readh), pRepo) < 0) { - return -1; - } - - // Init file iterator - tsdbFSIterInit(&(pCommith->fsIter), REPO_FS(pRepo), TSDB_FS_ITER_FORWARD); - - if (tsdbCreateCommitIters(pCommith) < 0) { - tsdbDestroyCommitH(pCommith); - return -1; - } - - pCommith->aBlkIdx = taosArrayInit(1024, sizeof(SBlockIdx)); - if (pCommith->aBlkIdx == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbDestroyCommitH(pCommith); - return -1; - } - - pCommith->aSupBlk = taosArrayInit(1024, sizeof(SBlock)); - if (pCommith->aSupBlk == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbDestroyCommitH(pCommith); - return -1; - } - - pCommith->aSubBlk = taosArrayInit(1024, sizeof(SBlock)); - if (pCommith->aSubBlk == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbDestroyCommitH(pCommith); - return -1; - } - - pCommith->pDataCols = tdNewDataCols(0, pCfg->maxRowsPerFileBlock); - if (pCommith->pDataCols == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbDestroyCommitH(pCommith); - return -1; - } - - return 0; -} - -// Skip all keys until key (not included) -static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key) { - for (int i = 0; i < pCommith->niters; i++) { - SCommitIter *pIter = pCommith->iters + i; - if (pIter->pTable == NULL || pIter->pIter == NULL) continue; - - tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, key - 1, INT32_MAX, NULL, NULL, 0, true, NULL); - } -} - -static int tsdbNextCommitFid(SCommitH *pCommith) { - STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); - STsdbCfg *pCfg = REPO_CFG(pRepo); - int fid = TSDB_IVLD_FID; - - for (int i = 0; i < pCommith->niters; i++) { - SCommitIter *pIter = pCommith->iters + i; - // if (pIter->pTable == NULL || pIter->pIter == NULL) continue; - - TSKEY nextKey = tsdbNextIterKey(pIter->pIter); - if (nextKey == TSDB_DATA_TIMESTAMP_NULL) { - continue; - } else { - int tfid = (int)(TSDB_KEY_FID(nextKey, pCfg->daysPerFile, pCfg->precision)); - if (fid == TSDB_IVLD_FID || fid > tfid) { - fid = tfid; - } - } - } - - return fid; -} - -static void tsdbDestroyCommitH(SCommitH *pCommith) { - pCommith->pDataCols = tdFreeDataCols(pCommith->pDataCols); - pCommith->aSubBlk = taosArrayDestroy(pCommith->aSubBlk); - pCommith->aSupBlk = taosArrayDestroy(pCommith->aSupBlk); - pCommith->aBlkIdx = taosArrayDestroy(pCommith->aBlkIdx); - tsdbDestroyCommitIters(pCommith); - tsdbDestroyReadH(&(pCommith->readh)); - tsdbCloseDFileSet(TSDB_COMMIT_WRITE_FSET(pCommith)); -} - -static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { - STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); - STsdbCfg *pCfg = REPO_CFG(pRepo); - - ASSERT(pSet == NULL || pSet->fid == fid); - - tsdbResetCommitFile(pCommith); - tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, fid, &(pCommith->minKey), &(pCommith->maxKey)); - - // Set and open files - if (tsdbSetAndOpenCommitFile(pCommith, pSet, fid) < 0) { - return -1; - } - - // Loop to commit each table data - for (int tid = 0; tid < pCommith->niters; tid++) { - SCommitIter *pIter = pCommith->iters + tid; - - if (pIter->pTable == NULL) continue; - - if (tsdbCommitToTable(pCommith, tid) < 0) { - tsdbCloseCommitFile(pCommith, true); - // revert the file change - tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); - return -1; - } - } - - if (tsdbWriteBlockIdx(TSDB_COMMIT_HEAD_FILE(pCommith), pCommith->aBlkIdx, (void **)(&(TSDB_COMMIT_BUF(pCommith)))) < - 0) { - tsdbError("vgId:%d failed to write SBlockIdx part to FSET %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); - tsdbCloseCommitFile(pCommith, true); - // revert the file change - tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); - return -1; - } - - if (tsdbUpdateDFileSetHeader(&(pCommith->wSet)) < 0) { - tsdbError("vgId:%d failed to update FSET %d header since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); - tsdbCloseCommitFile(pCommith, true); - // revert the file change - tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); - return -1; - } - - // Close commit file - tsdbCloseCommitFile(pCommith, false); - - if (tsdbUpdateDFileSet(REPO_FS(pRepo), &(pCommith->wSet)) < 0) { - return -1; - } - - return 0; -} - -static int tsdbCreateCommitIters(SCommitH *pCommith) { - STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); - STsdbMemTable *pMem = pRepo->imem; - SSkipListIterator *pSlIter; - SCommitIter *pCommitIter; - SSkipListNode *pNode; - STbData *pTbData; - - pCommith->niters = SL_SIZE(pMem->pSlIdx); - pCommith->iters = (SCommitIter *)calloc(pCommith->niters, sizeof(SCommitIter)); - if (pCommith->iters == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - // Loop to create iters for each skiplist - pSlIter = tSkipListCreateIter(pMem->pSlIdx); - if (pSlIter == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - for (int i = 0; i < pCommith->niters; i++) { - tSkipListIterNext(pSlIter); - pNode = tSkipListIterGet(pSlIter); - pTbData = (STbData *)pNode->pData; - - pCommitIter = pCommith->iters + i; - pCommitIter->pIter = tSkipListCreateIter(pTbData->pData); - tSkipListIterNext(pCommitIter->pIter); - - pCommitIter->pTable = (STable *)malloc(sizeof(STable)); - pCommitIter->pTable->uid = pTbData->uid; - pCommitIter->pTable->tid = pTbData->uid; - pCommitIter->pTable->pSchema = metaGetTbTSchema(pRepo->pMeta, pTbData->uid, 0); - } - - return 0; -} - -static void tsdbDestroyCommitIters(SCommitH *pCommith) { - if (pCommith->iters == NULL) return; - - for (int i = 1; i < pCommith->niters; i++) { - tSkipListDestroyIter(pCommith->iters[i].pIter); - tdFreeSchema(pCommith->iters[i].pTable->pSchema); - free(pCommith->iters[i].pTable); - } - - free(pCommith->iters); - pCommith->iters = NULL; - pCommith->niters = 0; -} - -static void tsdbResetCommitFile(SCommitH *pCommith) { - pCommith->isRFileSet = false; - pCommith->isDFileSame = false; - pCommith->isLFileSame = false; - taosArrayClear(pCommith->aBlkIdx); -} - -static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { - SDiskID did; - STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); - SDFileSet *pWSet = TSDB_COMMIT_WRITE_FSET(pCommith); - - if (tfsAllocDisk(pRepo->pTfs, tsdbGetFidLevel(fid, &(pCommith->rtn)), &did) < 0) { - terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; - return -1; - } - - // Open read FSET - if (pSet) { - if (tsdbSetAndOpenReadFSet(&(pCommith->readh), pSet) < 0) { - return -1; - } - - pCommith->isRFileSet = true; - - if (tsdbLoadBlockIdx(&(pCommith->readh)) < 0) { - tsdbCloseAndUnsetFSet(&(pCommith->readh)); - return -1; - } - - tsdbDebug("vgId:%d FSET %d at level %d disk id %d is opened to read to commit", REPO_ID(pRepo), TSDB_FSET_FID(pSet), - TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); - } else { - pCommith->isRFileSet = false; - } - - // Set and open commit FSET - if (pSet == NULL || did.level > TSDB_FSET_LEVEL(pSet)) { - // Create a new FSET to write data - tsdbInitDFileSet(pRepo, pWSet, did, fid, FS_TXN_VERSION(REPO_FS(pRepo))); - - if (tsdbCreateDFileSet(pRepo, pWSet, true) < 0) { - tsdbError("vgId:%d failed to create FSET %d at level %d disk id %d since %s", REPO_ID(pRepo), - TSDB_FSET_FID(pWSet), TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet), tstrerror(terrno)); - if (pCommith->isRFileSet) { - tsdbCloseAndUnsetFSet(&(pCommith->readh)); - } - return -1; - } - - pCommith->isDFileSame = false; - pCommith->isLFileSame = false; - - tsdbDebug("vgId:%d FSET %d at level %d disk id %d is created to commit", REPO_ID(pRepo), TSDB_FSET_FID(pWSet), - TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet)); - } else { - did.level = TSDB_FSET_LEVEL(pSet); - did.id = TSDB_FSET_ID(pSet); - - pCommith->wSet.fid = fid; - pCommith->wSet.state = 0; - - // TSDB_FILE_HEAD - SDFile *pWHeadf = TSDB_COMMIT_HEAD_FILE(pCommith); - tsdbInitDFile(pRepo, pWHeadf, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_HEAD); - if (tsdbCreateDFile(pRepo, pWHeadf, true) < 0) { - tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWHeadf), - tstrerror(terrno)); - - if (pCommith->isRFileSet) { - tsdbCloseAndUnsetFSet(&(pCommith->readh)); - return -1; - } - } - - // TSDB_FILE_DATA - SDFile *pRDataf = TSDB_READ_DATA_FILE(&(pCommith->readh)); - SDFile *pWDataf = TSDB_COMMIT_DATA_FILE(pCommith); - tsdbInitDFileEx(pWDataf, pRDataf); - if (tsdbOpenDFile(pWDataf, O_WRONLY) < 0) { - tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWDataf), - tstrerror(terrno)); - - tsdbCloseDFileSet(pWSet); - tsdbRemoveDFile(pWHeadf); - if (pCommith->isRFileSet) { - tsdbCloseAndUnsetFSet(&(pCommith->readh)); - return -1; - } - } - pCommith->isDFileSame = true; - - // TSDB_FILE_LAST - SDFile *pRLastf = TSDB_READ_LAST_FILE(&(pCommith->readh)); - SDFile *pWLastf = TSDB_COMMIT_LAST_FILE(pCommith); - if (pRLastf->info.size < 32 * 1024) { - tsdbInitDFileEx(pWLastf, pRLastf); - pCommith->isLFileSame = true; - - if (tsdbOpenDFile(pWLastf, O_WRONLY) < 0) { - tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), - tstrerror(terrno)); - - tsdbCloseDFileSet(pWSet); - tsdbRemoveDFile(pWHeadf); - if (pCommith->isRFileSet) { - tsdbCloseAndUnsetFSet(&(pCommith->readh)); - return -1; - } - } - } else { - tsdbInitDFile(pRepo, pWLastf, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_LAST); - pCommith->isLFileSame = false; - - if (tsdbCreateDFile(pRepo, pWLastf, true) < 0) { - tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), - tstrerror(terrno)); - - tsdbCloseDFileSet(pWSet); - (void)tsdbRemoveDFile(pWHeadf); - if (pCommith->isRFileSet) { - tsdbCloseAndUnsetFSet(&(pCommith->readh)); - return -1; - } - } - } - } - - return 0; -} - -// extern int32_t tsTsdbMetaCompactRatio; - -int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray *pSubA, void **ppBuf, - SBlockIdx *pIdx) { - size_t nSupBlocks; - size_t nSubBlocks; - uint32_t tlen; - SBlockInfo *pBlkInfo; - int64_t offset; - SBlock *pBlock; - - memset(pIdx, 0, sizeof(*pIdx)); - - nSupBlocks = taosArrayGetSize(pSupA); - nSubBlocks = (pSubA == NULL) ? 0 : taosArrayGetSize(pSubA); - - if (nSupBlocks <= 0) { - // No data (data all deleted) - return 0; - } - - tlen = (uint32_t)(sizeof(SBlockInfo) + sizeof(SBlock) * (nSupBlocks + nSubBlocks) + sizeof(TSCKSUM)); - if (tsdbMakeRoom(ppBuf, tlen) < 0) return -1; - pBlkInfo = *ppBuf; - - pBlkInfo->delimiter = TSDB_FILE_DELIMITER; - pBlkInfo->tid = TABLE_TID(pTable); - pBlkInfo->uid = TABLE_UID(pTable); - - memcpy((void *)(pBlkInfo->blocks), taosArrayGet(pSupA, 0), nSupBlocks * sizeof(SBlock)); - if (nSubBlocks > 0) { - memcpy((void *)(pBlkInfo->blocks + nSupBlocks), taosArrayGet(pSubA, 0), nSubBlocks * sizeof(SBlock)); - - for (int i = 0; i < nSupBlocks; i++) { - pBlock = pBlkInfo->blocks + i; - - if (pBlock->numOfSubBlocks > 1) { - pBlock->offset += (sizeof(SBlockInfo) + sizeof(SBlock) * nSupBlocks); - } - } - } - - taosCalcChecksumAppend(0, (uint8_t *)pBlkInfo, tlen); - - if (tsdbAppendDFile(pHeadf, (void *)pBlkInfo, tlen, &offset) < 0) { - return -1; - } - - tsdbUpdateDFileMagic(pHeadf, POINTER_SHIFT(pBlkInfo, tlen - sizeof(TSCKSUM))); - - // Set pIdx - pBlock = taosArrayGetLast(pSupA); - - pIdx->tid = TABLE_TID(pTable); - pIdx->uid = TABLE_UID(pTable); - pIdx->hasLast = pBlock->last ? 1 : 0; - pIdx->maxKey = pBlock->keyLast; - pIdx->numOfBlocks = (uint32_t)nSupBlocks; - pIdx->len = tlen; - pIdx->offset = (uint32_t)offset; - - return 0; -} - -int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { - SBlockIdx *pBlkIdx; - size_t nidx = taosArrayGetSize(pIdxA); - int tlen = 0, size; - int64_t offset; - - if (nidx <= 0) { - // All data are deleted - pHeadf->info.offset = 0; - pHeadf->info.len = 0; - return 0; - } - - for (size_t i = 0; i < nidx; i++) { - pBlkIdx = (SBlockIdx *)taosArrayGet(pIdxA, i); - - size = tsdbEncodeSBlockIdx(NULL, pBlkIdx); - if (tsdbMakeRoom(ppBuf, tlen + size) < 0) return -1; - - void *ptr = POINTER_SHIFT(*ppBuf, tlen); - tsdbEncodeSBlockIdx(&ptr, pBlkIdx); - - tlen += size; - } - - tlen += sizeof(TSCKSUM); - if (tsdbMakeRoom(ppBuf, tlen) < 0) return -1; - taosCalcChecksumAppend(0, (uint8_t *)(*ppBuf), tlen); - - if (tsdbAppendDFile(pHeadf, *ppBuf, tlen, &offset) < tlen) { - return -1; - } - - tsdbUpdateDFileMagic(pHeadf, POINTER_SHIFT(*ppBuf, tlen - sizeof(TSCKSUM))); - pHeadf->info.offset = (uint32_t)offset; - pHeadf->info.len = tlen; - - return 0; -} - -// // =================== Commit Meta Data -// static int tsdbInitCommitMetaFile(STsdbRepo *pRepo, SMFile* pMf, bool open) { -// STsdbFS * pfs = REPO_FS(pRepo); -// SMFile * pOMFile = pfs->cstatus->pmf; -// SDiskID did; - -// // Create/Open a meta file or open the existing file -// if (pOMFile == NULL) { -// // Create a new meta file -// did.level = TFS_PRIMARY_LEVEL; -// did.id = TFS_PRIMARY_ID; -// tsdbInitMFile(pMf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo))); - -// if (open && tsdbCreateMFile(pMf, true) < 0) { -// tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); -// return -1; -// } - -// tsdbInfo("vgId:%d meta file %s is created to commit", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMf)); -// } else { -// tsdbInitMFileEx(pMf, pOMFile); -// if (open && tsdbOpenMFile(pMf, O_WRONLY) < 0) { -// tsdbError("vgId:%d failed to open META file since %s", REPO_ID(pRepo), tstrerror(terrno)); -// return -1; -// } -// } - -// return 0; -// } - -// static int tsdbCommitMeta(STsdbRepo *pRepo) { -// STsdbFS * pfs = REPO_FS(pRepo); -// SMemTable *pMem = pRepo->imem; -// SMFile * pOMFile = pfs->cstatus->pmf; -// SMFile mf; -// SActObj * pAct = NULL; -// SActCont * pCont = NULL; -// SListNode *pNode = NULL; - -// ASSERT(pOMFile != NULL || listNEles(pMem->actList) > 0); - -// if (listNEles(pMem->actList) <= 0) { -// // no meta data to commit, just keep the old meta file -// tsdbUpdateMFile(pfs, pOMFile); -// if (tsTsdbMetaCompactRatio > 0) { -// if (tsdbInitCommitMetaFile(pRepo, &mf, false) < 0) { -// return -1; -// } -// int ret = tsdbCompactMetaFile(pRepo, pfs, &mf); -// if (ret < 0) tsdbError("compact meta file error"); - -// return ret; -// } -// return 0; -// } else { -// if (tsdbInitCommitMetaFile(pRepo, &mf, true) < 0) { -// return -1; -// } -// } - -// // Loop to write -// while ((pNode = tdListPopHead(pMem->actList)) != NULL) { -// pAct = (SActObj *)pNode->data; -// if (pAct->act == TSDB_UPDATE_META) { -// pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj)); -// if (tsdbUpdateMetaRecord(pfs, &mf, pAct->uid, (void *)(pCont->cont), pCont->len, false) < 0) { -// tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, -// tstrerror(terrno)); -// tsdbCloseMFile(&mf); -// (void)tsdbApplyMFileChange(&mf, pOMFile); -// // TODO: need to reload metaCache -// return -1; -// } -// } else if (pAct->act == TSDB_DROP_META) { -// if (tsdbDropMetaRecord(pfs, &mf, pAct->uid) < 0) { -// tsdbError("vgId:%d failed to drop META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, -// tstrerror(terrno)); -// tsdbCloseMFile(&mf); -// tsdbApplyMFileChange(&mf, pOMFile); -// // TODO: need to reload metaCache -// return -1; -// } -// } else { -// ASSERT(false); -// } -// } - -// if (tsdbUpdateMFileHeader(&mf) < 0) { -// tsdbError("vgId:%d failed to update META file header since %s, revert it", REPO_ID(pRepo), tstrerror(terrno)); -// tsdbApplyMFileChange(&mf, pOMFile); -// // TODO: need to reload metaCache -// return -1; -// } - -// TSDB_FILE_FSYNC(&mf); -// tsdbCloseMFile(&mf); -// tsdbUpdateMFile(pfs, &mf); - -// if (tsTsdbMetaCompactRatio > 0 && tsdbCompactMetaFile(pRepo, pfs, &mf) < 0) { -// tsdbError("compact meta file error"); -// } - -// return 0; -// } - -// int tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord) { -// int tlen = 0; -// tlen += taosEncodeFixedU64(buf, pRecord->uid); -// tlen += taosEncodeFixedI64(buf, pRecord->offset); -// tlen += taosEncodeFixedI64(buf, pRecord->size); - -// return tlen; -// } - -// void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord) { -// buf = taosDecodeFixedU64(buf, &(pRecord->uid)); -// buf = taosDecodeFixedI64(buf, &(pRecord->offset)); -// buf = taosDecodeFixedI64(buf, &(pRecord->size)); - -// return buf; -// } - -// static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool compact) { -// char buf[64] = "\0"; -// void * pBuf = buf; -// SKVRecord rInfo; -// int64_t offset; - -// // Seek to end of meta file -// offset = tsdbSeekMFile(pMFile, 0, SEEK_END); -// if (offset < 0) { -// return -1; -// } - -// rInfo.offset = offset; -// rInfo.uid = uid; -// rInfo.size = contLen; - -// int tlen = tsdbEncodeKVRecord((void **)(&pBuf), &rInfo); -// if (tsdbAppendMFile(pMFile, buf, tlen, NULL) < tlen) { -// return -1; -// } - -// if (tsdbAppendMFile(pMFile, cont, contLen, NULL) < contLen) { -// return -1; -// } - -// tsdbUpdateMFileMagic(pMFile, POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM))); - -// SHashObj* cache = compact ? pfs->metaCacheComp : pfs->metaCache; - -// pMFile->info.nRecords++; - -// SKVRecord *pRecord = taosHashGet(cache, (void *)&uid, sizeof(uid)); -// if (pRecord != NULL) { -// pMFile->info.tombSize += (pRecord->size + sizeof(SKVRecord)); -// } else { -// pMFile->info.nRecords++; -// } -// taosHashPut(cache, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo)); - -// return 0; -// } - -// static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) { -// SKVRecord rInfo = {0}; -// char buf[128] = "\0"; - -// SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)(&uid), sizeof(uid)); -// if (pRecord == NULL) { -// tsdbError("failed to drop META record with key %" PRIu64 " since not find", uid); -// return -1; -// } - -// rInfo.offset = -pRecord->offset; -// rInfo.uid = pRecord->uid; -// rInfo.size = pRecord->size; - -// void *pBuf = buf; -// tsdbEncodeKVRecord(&pBuf, &rInfo); - -// if (tsdbAppendMFile(pMFile, buf, sizeof(SKVRecord), NULL) < 0) { -// return -1; -// } - -// pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t *)buf, sizeof(SKVRecord)); -// pMFile->info.nDels++; -// pMFile->info.nRecords--; -// pMFile->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2); - -// taosHashRemove(pfs->metaCache, (void *)(&uid), sizeof(uid)); -// return 0; -// } - -// static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) { -// float delPercent = (float)(pMFile->info.nDels) / (float)(pMFile->info.nRecords); -// float tombPercent = (float)(pMFile->info.tombSize) / (float)(pMFile->info.size); -// float compactRatio = (float)(tsTsdbMetaCompactRatio)/100; - -// if (delPercent < compactRatio && tombPercent < compactRatio) { -// return 0; -// } - -// if (tsdbOpenMFile(pMFile, O_RDONLY) < 0) { -// tsdbError("open meta file %s compact fail", pMFile->f.rname); -// return -1; -// } - -// tsdbInfo("begin compact tsdb meta file, ratio:%d, nDels:%" PRId64 ",nRecords:%" PRId64 ",tombSize:%" PRId64 -// ",size:%" PRId64, -// tsTsdbMetaCompactRatio, pMFile->info.nDels,pMFile->info.nRecords,pMFile->info.tombSize,pMFile->info.size); - -// SMFile mf; -// SDiskID did; - -// // first create tmp meta file -// did.level = TFS_PRIMARY_LEVEL; -// did.id = TFS_PRIMARY_ID; -// tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)) + 1); - -// if (tsdbCreateMFile(&mf, true) < 0) { -// tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); -// return -1; -// } - -// tsdbInfo("vgId:%d meta file %s is created to compact meta data", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf)); - -// // second iterator metaCache -// int code = -1; -// int64_t maxBufSize = 1024; -// SKVRecord *pRecord; -// void *pBuf = NULL; - -// pBuf = malloc((size_t)maxBufSize); -// if (pBuf == NULL) { -// goto _err; -// } - -// // init Comp -// assert(pfs->metaCacheComp == NULL); -// pfs->metaCacheComp = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); -// if (pfs->metaCacheComp == NULL) { -// goto _err; -// } - -// pRecord = taosHashIterate(pfs->metaCache, NULL); -// while (pRecord) { -// if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) { -// tsdbError("vgId:%d failed to seek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), -// tstrerror(terrno)); -// goto _err; -// } -// if (pRecord->size > maxBufSize) { -// maxBufSize = pRecord->size; -// void* tmp = realloc(pBuf, (size_t)maxBufSize); -// if (tmp == NULL) { -// goto _err; -// } -// pBuf = tmp; -// } -// int nread = (int)tsdbReadMFile(pMFile, pBuf, pRecord->size); -// if (nread < 0) { -// tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), -// tstrerror(terrno)); -// goto _err; -// } - -// if (nread < pRecord->size) { -// tsdbError("vgId:%d failed to read file %s since file corrupted, expected read:%" PRId64 " actual read:%d", -// REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), pRecord->size, nread); -// goto _err; -// } - -// if (tsdbUpdateMetaRecord(pfs, &mf, pRecord->uid, pBuf, (int)pRecord->size, true) < 0) { -// tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pRecord->uid, -// tstrerror(terrno)); -// goto _err; -// } - -// pRecord = taosHashIterate(pfs->metaCache, pRecord); -// } -// code = 0; - -// _err: -// if (code == 0) TSDB_FILE_FSYNC(&mf); -// tsdbCloseMFile(&mf); -// tsdbCloseMFile(pMFile); - -// if (code == 0) { -// // rename meta.tmp -> meta -// tsdbInfo("vgId:%d meta file rename %s -> %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf), -// TSDB_FILE_FULL_NAME(pMFile)); taosRename(mf.f.aname,pMFile->f.aname); tstrncpy(mf.f.aname, pMFile->f.aname, -// TSDB_FILENAME_LEN); tstrncpy(mf.f.rname, pMFile->f.rname, TSDB_FILENAME_LEN); -// // update current meta file info -// pfs->nstatus->pmf = NULL; -// tsdbUpdateMFile(pfs, &mf); - -// taosHashCleanup(pfs->metaCache); -// pfs->metaCache = pfs->metaCacheComp; -// pfs->metaCacheComp = NULL; -// } else { -// // remove meta.tmp file -// remove(mf.f.aname); -// taosHashCleanup(pfs->metaCacheComp); -// pfs->metaCacheComp = NULL; -// } - -// tfree(pBuf); - -// ASSERT(mf.info.nDels == 0); -// ASSERT(mf.info.tombSize == 0); - -// tsdbInfo("end compact tsdb meta file,code:%d,nRecords:%" PRId64 ",size:%" PRId64, -// code,mf.info.nRecords,mf.info.size); -// return code; -// } - -// // =================== Commit Time-Series Data -// #if 0 -// static bool tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) { -// for (int i = 0; i < nIters; i++) { -// TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter); -// if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return true; -// } -// return false; -// } -// #endif - -static int tsdbCommitToTable(SCommitH *pCommith, int tid) { - SCommitIter *pIter = pCommith->iters + tid; - TSKEY nextKey = tsdbNextIterKey(pIter->pIter); - - tsdbResetCommitTable(pCommith); - - // Set commit table - if (tsdbSetCommitTable(pCommith, pIter->pTable) < 0) { - return -1; - } - - // No disk data and no memory data, just return - if (pCommith->readh.pBlkIdx == NULL && (nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey)) { - return 0; - } - - // Must has disk data or has memory data - int nBlocks; - int bidx = 0; - SBlock *pBlock; - - if (pCommith->readh.pBlkIdx) { - if (tsdbLoadBlockInfo(&(pCommith->readh), NULL) < 0) { - return -1; - } - - nBlocks = pCommith->readh.pBlkIdx->numOfBlocks; - } else { - nBlocks = 0; - } - - if (bidx < nBlocks) { - pBlock = pCommith->readh.pBlkInfo->blocks + bidx; - } else { - pBlock = NULL; - } - - while (true) { - if (pBlock == NULL && (nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey)) break; - - if ((nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey) || - (pBlock && (!pBlock->last) && tsdbComparKeyBlock((void *)(&nextKey), pBlock) > 0)) { - if (tsdbMoveBlock(pCommith, bidx) < 0) { - return -1; - } - - bidx++; - if (bidx < nBlocks) { - pBlock = pCommith->readh.pBlkInfo->blocks + bidx; - } else { - pBlock = NULL; - } - } else if (pBlock && (pBlock->last || tsdbComparKeyBlock((void *)(&nextKey), pBlock) == 0)) { - // merge pBlock data and memory data - if (tsdbMergeMemData(pCommith, pIter, bidx) < 0) { - return -1; - } - - bidx++; - if (bidx < nBlocks) { - pBlock = pCommith->readh.pBlkInfo->blocks + bidx; - } else { - pBlock = NULL; - } - nextKey = tsdbNextIterKey(pIter->pIter); - } else { - // Only commit memory data - if (pBlock == NULL) { - if (tsdbCommitMemData(pCommith, pIter, pCommith->maxKey, false) < 0) { - return -1; - } - } else { - if (tsdbCommitMemData(pCommith, pIter, pBlock->keyFirst - 1, true) < 0) { - return -1; - } - } - nextKey = tsdbNextIterKey(pIter->pIter); - } - } - - if (tsdbWriteBlockInfo(pCommith) < 0) { - tsdbError("vgId:%d failed to write SBlockInfo part into file %s since %s", TSDB_COMMIT_REPO_ID(pCommith), - TSDB_FILE_FULL_NAME(TSDB_COMMIT_HEAD_FILE(pCommith)), tstrerror(terrno)); - return -1; - } - - return 0; -} - -static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable) { - STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); - - pCommith->pTable = pTable; - - if (tdInitDataCols(pCommith->pDataCols, pSchema) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - if (pCommith->isRFileSet) { - if (tsdbSetReadTable(&(pCommith->readh), pTable) < 0) { - return -1; - } - } else { - pCommith->readh.pBlkIdx = NULL; - } - return 0; -} - -static int tsdbComparKeyBlock(const void *arg1, const void *arg2) { - TSKEY key = *(TSKEY *)arg1; - SBlock *pBlock = (SBlock *)arg2; - - if (key < pBlock->keyFirst) { - return -1; - } else if (key > pBlock->keyLast) { - return 1; - } else { - return 0; - } -} - -int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock, bool isLast, - bool isSuper, void **ppBuf, void **ppCBuf) { - STsdbCfg *pCfg = REPO_CFG(pRepo); - SBlockData *pBlockData; - int64_t offset = 0; - int rowsToWrite = pDataCols->numOfRows; - - ASSERT(rowsToWrite > 0 && rowsToWrite <= pCfg->maxRowsPerFileBlock); - ASSERT((!isLast) || rowsToWrite < pCfg->minRowsPerFileBlock); - - // Make buffer space - if (tsdbMakeRoom(ppBuf, TSDB_BLOCK_STATIS_SIZE(pDataCols->numOfCols)) < 0) { - return -1; - } - pBlockData = (SBlockData *)(*ppBuf); - - // Get # of cols not all NULL(not including key column) - int nColsNotAllNull = 0; - for (int ncol = 1; ncol < pDataCols->numOfCols; ncol++) { // ncol from 1, we skip the timestamp column - SDataCol *pDataCol = pDataCols->cols + ncol; - SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull; - - if (isAllRowsNull(pDataCol)) { // all data to commit are NULL, just ignore it - continue; - } - - memset(pBlockCol, 0, sizeof(*pBlockCol)); - - pBlockCol->colId = pDataCol->colId; - pBlockCol->type = pDataCol->type; - if (tDataTypes[pDataCol->type].statisFunc) { - (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pBlockCol->min), &(pBlockCol->max), - &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex), - &(pBlockCol->numOfNull)); - if (pBlockCol->numOfNull == 0) { - TD_SET_COL_ROWS_NORM(pBlockCol); - } else { - TD_SET_COL_ROWS_MISC(pBlockCol); - } - } else { - TD_SET_COL_ROWS_MISC(pBlockCol); - } - nColsNotAllNull++; - } - - ASSERT(nColsNotAllNull >= 0 && nColsNotAllNull <= pDataCols->numOfCols); - - // Compress the data if neccessary - int tcol = 0; // counter of not all NULL and written columns - uint32_t toffset = 0; - int32_t tsize = TSDB_BLOCK_STATIS_SIZE(nColsNotAllNull); - int32_t lsize = tsize; - int32_t keyLen = 0; - int32_t nBitmaps = (int32_t)TD_BITMAP_BYTES(rowsToWrite); - int32_t tBitmaps = 0; - - for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) { - // All not NULL columns finish - if (ncol != 0 && tcol >= nColsNotAllNull) break; - - SDataCol *pDataCol = pDataCols->cols + ncol; - SBlockCol *pBlockCol = pBlockData->cols + tcol; - - if (ncol != 0 && (pDataCol->colId != pBlockCol->colId)) continue; - - int32_t flen; // final length - int32_t tlen = dataColGetNEleLen(pDataCol, rowsToWrite); - -#ifdef TD_SUPPORT_BITMAP - int32_t tBitmaps = 0; - if ((ncol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) { - if (IS_VAR_DATA_TYPE(pDataCol->type)) { - tBitmaps = nBitmaps; - tlen += tBitmaps; - } else { - tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]); - tlen += tBitmaps * TYPE_BYTES[pDataCol->type]; - } - // move bitmap parts ahead - // TODO: put bitmap part to the 1st location(pBitmap points to pData) to avoid the memmove - memcpy(POINTER_SHIFT(pDataCol->pData, pDataCol->len), pDataCol->pBitmap, nBitmaps); - } -#endif - - void *tptr; - - // Make room - if (tsdbMakeRoom(ppBuf, lsize + tlen + COMP_OVERFLOW_BYTES + sizeof(TSCKSUM)) < 0) { - return -1; - } - pBlockData = (SBlockData *)(*ppBuf); - pBlockCol = pBlockData->cols + tcol; - tptr = POINTER_SHIFT(pBlockData, lsize); - - if (pCfg->compression == TWO_STAGE_COMP && tsdbMakeRoom(ppCBuf, tlen + COMP_OVERFLOW_BYTES) < 0) { - return -1; - } - - // Compress or just copy - if (pCfg->compression) { - flen = (*(tDataTypes[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite + tBitmaps, tptr, - tlen + COMP_OVERFLOW_BYTES, pCfg->compression, *ppCBuf, - tlen + COMP_OVERFLOW_BYTES); - } else { - flen = tlen; - memcpy(tptr, pDataCol->pData, flen); - } - - // Add checksum - ASSERT(flen > 0); - flen += sizeof(TSCKSUM); - taosCalcChecksumAppend(0, (uint8_t *)tptr, flen); - tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM))); - - if (ncol != 0) { - tsdbSetBlockColOffset(pBlockCol, toffset); - pBlockCol->len = flen; - tcol++; - } else { - keyLen = flen; - } - - toffset += flen; - lsize += flen; - } - - pBlockData->delimiter = TSDB_FILE_DELIMITER; - pBlockData->uid = TABLE_UID(pTable); - pBlockData->numOfCols = nColsNotAllNull; - - taosCalcChecksumAppend(0, (uint8_t *)pBlockData, tsize); - tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(pBlockData, tsize - sizeof(TSCKSUM))); - - // Write the whole block to file - if (tsdbAppendDFile(pDFile, (void *)pBlockData, lsize, &offset) < lsize) { - return -1; - } - - // Update pBlock membership vairables - pBlock->last = isLast; - pBlock->offset = offset; - pBlock->algorithm = pCfg->compression; - pBlock->numOfRows = rowsToWrite; - pBlock->len = lsize; - pBlock->keyLen = keyLen; - pBlock->numOfSubBlocks = isSuper ? 1 : 0; - pBlock->numOfCols = nColsNotAllNull; - pBlock->keyFirst = dataColsKeyFirst(pDataCols); - pBlock->keyLast = dataColsKeyLast(pDataCols); - - tsdbDebug("vgId:%d uid:%" PRId64 " a block of data is written to file %s, offset %" PRId64 - " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, - REPO_ID(pRepo), TABLE_TID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len, - pBlock->numOfCols, pBlock->keyFirst, pBlock->keyLast); - - return 0; -} - -static int tsdbWriteBlock(SCommitH *pCommith, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock, bool isLast, - bool isSuper) { - return tsdbWriteBlockImpl(TSDB_COMMIT_REPO(pCommith), TSDB_COMMIT_TABLE(pCommith), pDFile, pDataCols, pBlock, isLast, - isSuper, (void **)(&(TSDB_COMMIT_BUF(pCommith))), - (void **)(&(TSDB_COMMIT_COMP_BUF(pCommith)))); -} - -static int tsdbWriteBlockInfo(SCommitH *pCommih) { - SDFile *pHeadf = TSDB_COMMIT_HEAD_FILE(pCommih); - SBlockIdx blkIdx; - STable *pTable = TSDB_COMMIT_TABLE(pCommih); - - if (tsdbWriteBlockInfoImpl(pHeadf, pTable, pCommih->aSupBlk, pCommih->aSubBlk, (void **)(&(TSDB_COMMIT_BUF(pCommih))), - &blkIdx) < 0) { - return -1; - } - - if (blkIdx.numOfBlocks == 0) { - return 0; - } - - if (taosArrayPush(pCommih->aBlkIdx, (void *)(&blkIdx)) == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData) { - STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); - STsdbCfg *pCfg = REPO_CFG(pRepo); - SMergeInfo mInfo; - int32_t defaultRows = TSDB_COMMIT_DEFAULT_ROWS(pCommith); - SDFile *pDFile; - bool isLast; - SBlock block; - - while (true) { - tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, keyLimit, defaultRows, pCommith->pDataCols, NULL, 0, - pCfg->update, &mInfo); - - if (pCommith->pDataCols->numOfRows <= 0) break; - - if (toData || pCommith->pDataCols->numOfRows >= pCfg->minRowsPerFileBlock) { - pDFile = TSDB_COMMIT_DATA_FILE(pCommith); - isLast = false; - } else { - pDFile = TSDB_COMMIT_LAST_FILE(pCommith); - isLast = true; - } - - if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1; - - if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) { - return -1; - } - } - - return 0; -} - -static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx) { - STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); - STsdbCfg *pCfg = REPO_CFG(pRepo); - int nBlocks = pCommith->readh.pBlkIdx->numOfBlocks; - SBlock *pBlock = pCommith->readh.pBlkInfo->blocks + bidx; - TSKEY keyLimit; - int16_t colId = PRIMARYKEY_TIMESTAMP_COL_ID; - SMergeInfo mInfo; - SBlock subBlocks[TSDB_MAX_SUBBLOCKS]; - SBlock block, supBlock; - SDFile *pDFile; - - if (bidx == nBlocks - 1) { - keyLimit = pCommith->maxKey; - } else { - keyLimit = pBlock[1].keyFirst - 1; - } - - SSkipListIterator titer = *(pIter->pIter); - if (tsdbLoadBlockDataCols(&(pCommith->readh), pBlock, NULL, &colId, 1) < 0) return -1; - - tsdbLoadDataFromCache(pIter->pTable, &titer, keyLimit, INT32_MAX, NULL, pCommith->readh.pDCols[0]->cols[0].pData, - pCommith->readh.pDCols[0]->numOfRows, pCfg->update, &mInfo); - - if (mInfo.nOperations == 0) { - // no new data to insert (all updates denied) - if (tsdbMoveBlock(pCommith, bidx) < 0) { - return -1; - } - *(pIter->pIter) = titer; - } else if (pBlock->numOfRows + mInfo.rowsInserted - mInfo.rowsDeleteSucceed == 0) { - // Ignore the block - ASSERT(0); - *(pIter->pIter) = titer; - } else if (tsdbCanAddSubBlock(pCommith, pBlock, &mInfo)) { - // Add a sub-block - tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, keyLimit, INT32_MAX, pCommith->pDataCols, - pCommith->readh.pDCols[0]->cols[0].pData, pCommith->readh.pDCols[0]->numOfRows, pCfg->update, - &mInfo); - if (pBlock->last) { - pDFile = TSDB_COMMIT_LAST_FILE(pCommith); - } else { - pDFile = TSDB_COMMIT_DATA_FILE(pCommith); - } - - if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, pBlock->last, false) < 0) return -1; - - if (pBlock->numOfSubBlocks == 1) { - subBlocks[0] = *pBlock; - subBlocks[0].numOfSubBlocks = 0; - } else { - memcpy(subBlocks, POINTER_SHIFT(pCommith->readh.pBlkInfo, pBlock->offset), - sizeof(SBlock) * pBlock->numOfSubBlocks); - } - subBlocks[pBlock->numOfSubBlocks] = block; - supBlock = *pBlock; - supBlock.keyFirst = mInfo.keyFirst; - supBlock.keyLast = mInfo.keyLast; - supBlock.numOfSubBlocks++; - supBlock.numOfRows = pBlock->numOfRows + mInfo.rowsInserted - mInfo.rowsDeleteSucceed; - supBlock.offset = taosArrayGetSize(pCommith->aSubBlk) * sizeof(SBlock); - - if (tsdbCommitAddBlock(pCommith, &supBlock, subBlocks, supBlock.numOfSubBlocks) < 0) return -1; - } else { - if (tsdbLoadBlockData(&(pCommith->readh), pBlock, NULL) < 0) return -1; - if (tsdbMergeBlockData(pCommith, pIter, pCommith->readh.pDCols[0], keyLimit, bidx == (nBlocks - 1)) < 0) return -1; - } - - return 0; -} - -static int tsdbMoveBlock(SCommitH *pCommith, int bidx) { - SBlock *pBlock = pCommith->readh.pBlkInfo->blocks + bidx; - SDFile *pDFile; - SBlock block; - bool isSameFile; - - ASSERT(pBlock->numOfSubBlocks > 0); - - if (pBlock->last) { - pDFile = TSDB_COMMIT_LAST_FILE(pCommith); - isSameFile = pCommith->isLFileSame; - } else { - pDFile = TSDB_COMMIT_DATA_FILE(pCommith); - isSameFile = pCommith->isDFileSame; - } - - if (isSameFile) { - if (pBlock->numOfSubBlocks == 1) { - if (tsdbCommitAddBlock(pCommith, pBlock, NULL, 0) < 0) { - return -1; - } - } else { - block = *pBlock; - block.offset = sizeof(SBlock) * taosArrayGetSize(pCommith->aSubBlk); - - if (tsdbCommitAddBlock(pCommith, &block, POINTER_SHIFT(pCommith->readh.pBlkInfo, pBlock->offset), - pBlock->numOfSubBlocks) < 0) { - return -1; - } - } - } else { - if (tsdbLoadBlockData(&(pCommith->readh), pBlock, NULL) < 0) return -1; - if (tsdbWriteBlock(pCommith, pDFile, pCommith->readh.pDCols[0], &block, pBlock->last, true) < 0) return -1; - if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) return -1; - } - - return 0; -} - -static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const SBlock *pSubBlocks, int nSubBlocks) { - if (taosArrayPush(pCommith->aSupBlk, pSupBlock) == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - if (pSubBlocks && taosArrayAddBatch(pCommith->aSubBlk, pSubBlocks, nSubBlocks) == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -static int tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit, - bool isLastOneBlock) { - STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); - STsdbCfg *pCfg = REPO_CFG(pRepo); - SBlock block; - SDFile *pDFile; - bool isLast; - int32_t defaultRows = TSDB_COMMIT_DEFAULT_ROWS(pCommith); - - int biter = 0; - while (true) { - tsdbLoadAndMergeFromCache(pCommith->readh.pDCols[0], &biter, pIter, pCommith->pDataCols, keyLimit, defaultRows, - pCfg->update); - - if (pCommith->pDataCols->numOfRows == 0) break; - - if (isLastOneBlock) { - if (pCommith->pDataCols->numOfRows < pCfg->minRowsPerFileBlock) { - pDFile = TSDB_COMMIT_LAST_FILE(pCommith); - isLast = true; - } else { - pDFile = TSDB_COMMIT_DATA_FILE(pCommith); - isLast = false; - } - } else { - pDFile = TSDB_COMMIT_DATA_FILE(pCommith); - isLast = false; - } - - if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1; - if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) return -1; - } - - return 0; -} - -static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, - TSKEY maxKey, int maxRows, int8_t update) { - TSKEY key1 = INT64_MAX; - TSKEY key2 = INT64_MAX; - STSchema *pSchema = NULL; - - ASSERT(maxRows > 0 && dataColsKeyLast(pDataCols) <= maxKey); - tdResetDataCols(pTarget); - - while (true) { - key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter); - STSRow *row = tsdbNextIterRow(pCommitIter->pIter); - if (row == NULL || TD_ROW_KEY(row) > maxKey) { - key2 = INT64_MAX; - } else { - key2 = TD_ROW_KEY(row); - } - - if (key1 == INT64_MAX && key2 == INT64_MAX) break; - - if (key1 < key2) { - for (int i = 0; i < pDataCols->numOfCols; i++) { - // TODO: dataColAppendVal may fail - SCellVal sVal = {0}; - if (tdGetColDataOfRow(&sVal, pDataCols->cols + i, *iter) < 0) { - TASSERT(0); - } - tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints); - } - - pTarget->numOfRows++; - (*iter)++; - } else if (key1 > key2) { - if (pSchema == NULL || schemaVersion(pSchema) != TD_ROW_SVER(row)) { - pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, TD_ROW_SVER(row)); - ASSERT(pSchema != NULL); - } - - tdAppendSTSRowToDataCol(row, pSchema, pTarget, true); - - tSkipListIterNext(pCommitIter->pIter); - } else { - if (update != TD_ROW_OVERWRITE_UPDATE) { - // copy disk data - for (int i = 0; i < pDataCols->numOfCols; i++) { - // TODO: dataColAppendVal may fail - SCellVal sVal = {0}; - if (tdGetColDataOfRow(&sVal, pDataCols->cols + i, *iter) < 0) { - TASSERT(0); - } - tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints); - } - - if (update == TD_ROW_DISCARD_UPDATE) pTarget->numOfRows++; - } - if (update != TD_ROW_DISCARD_UPDATE) { - // copy mem data - if (pSchema == NULL || schemaVersion(pSchema) != TD_ROW_SVER(row)) { - pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, TD_ROW_SVER(row)); - ASSERT(pSchema != NULL); - } - - tdAppendSTSRowToDataCol(row, pSchema, pTarget, update == TD_ROW_OVERWRITE_UPDATE); - } - (*iter)++; - tSkipListIterNext(pCommitIter->pIter); - } - - if (pTarget->numOfRows >= maxRows) break; - } -} - -static void tsdbResetCommitTable(SCommitH *pCommith) { - taosArrayClear(pCommith->aSubBlk); - taosArrayClear(pCommith->aSupBlk); - pCommith->pTable = NULL; -} - -static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError) { - if (pCommith->isRFileSet) { - tsdbCloseAndUnsetFSet(&(pCommith->readh)); - } - - if (!hasError) { - TSDB_FSET_FSYNC(TSDB_COMMIT_WRITE_FSET(pCommith)); - } - tsdbCloseDFileSet(TSDB_COMMIT_WRITE_FSET(pCommith)); -} - -static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo) { - STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); - STsdbCfg *pCfg = REPO_CFG(pRepo); - int mergeRows = pBlock->numOfRows + pInfo->rowsInserted - pInfo->rowsDeleteSucceed; - - ASSERT(mergeRows > 0); - - if (pBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && pInfo->nOperations <= pCfg->maxRowsPerFileBlock) { - if (pBlock->last) { - if (pCommith->isLFileSame && mergeRows < pCfg->minRowsPerFileBlock) return true; - } else { - if (pCommith->isDFileSame && mergeRows <= pCfg->maxRowsPerFileBlock) return true; - } - } - - return false; -} - -// int tsdbApplyRtn(STsdbRepo *pRepo) { -// SRtn rtn; -// SFSIter fsiter; -// STsdbFS * pfs = REPO_FS(pRepo); -// SDFileSet *pSet; - -// // Get retention snapshot -// tsdbGetRtnSnap(pRepo, &rtn); - -// tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); -// while ((pSet = tsdbFSIterNext(&fsiter))) { -// if (pSet->fid < rtn.minFid) { -// tsdbInfo("vgId:%d FSET %d at level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, -// TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); -// continue; -// } - -// if (tsdbApplyRtnOnFSet(pRepo, pSet, &rtn) < 0) { -// return -1; -// } -// } - -// return 0; -// } #endif \ No newline at end of file From 3245b73e27175bc1122e943eafde35927ce68a49 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 6 Mar 2022 19:54:09 +0800 Subject: [PATCH 16/17] code optimization --- source/dnode/vnode/src/tsdb/tsdbSma.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 751438c12d..b465dc3a88 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -547,6 +547,4 @@ int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin) { // // remove the tSmaFile // } return TSDB_CODE_SUCCESS; -} - -#endif \ No newline at end of file +} \ No newline at end of file From 07227d7ad20fc5c18ebb9884cb0f593a95a166d6 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 7 Mar 2022 18:27:31 +0800 Subject: [PATCH 17/17] code optimization --- include/common/tmsg.h | 14 ++++---------- source/dnode/vnode/src/meta/metaBDBImpl.c | 6 ++++-- source/dnode/vnode/test/tsdbSmaTest.cpp | 12 +++++++----- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 65860d4959..f0e1944e25 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1924,27 +1924,21 @@ typedef struct { STSma* tSma; } STSmaWrapper; -static FORCE_INLINE void tdDestroyTSma(STSma* pSma, bool releaseSelf) { +static FORCE_INLINE void tdDestroyTSma(STSma* pSma) { if (pSma) { tfree(pSma->colIds); tfree(pSma->funcIds); - if (releaseSelf) { - free(pSma); - } } } -static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW, bool releaseSelf) { +static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW) { if (pSW) { if (pSW->tSma) { for (uint32_t i = 0; i < pSW->number; ++i) { - tdDestroyTSma(pSW->tSma + i, false); + tdDestroyTSma(pSW->tSma + i); } tfree(pSW->tSma); } - if (releaseSelf) { - free(pSW); - } } } @@ -2031,7 +2025,7 @@ static FORCE_INLINE void* tDecodeTSmaWrapper(void* buf, STSmaWrapper* pSW) { for (uint32_t i = 0; i < pSW->number; ++i) { if ((buf = tDecodeTSma(buf, pSW->tSma + i)) == NULL) { for (uint32_t j = i; j >= 0; --i) { - tdDestroyTSma(pSW->tSma + j, false); + tdDestroyTSma(pSW->tSma + j); } free(pSW->tSma); return NULL; diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index 03a2937679..c2f147bcaa 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -896,14 +896,16 @@ STSmaWrapper *metaGetSmaInfoByUid(SMeta *pMeta, tb_uid_t uid) { STSma *tptr = (STSma *)realloc(pSW->tSma, pSW->number * sizeof(STSma)); if (tptr == NULL) { metaCloseSmaCurosr(pCur); - tdDestroyTSmaWrapper(pSW, true); + tdDestroyTSmaWrapper(pSW); + tfree(pSW); return NULL; } pSW->tSma = tptr; pBuf = pval.data; if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) { metaCloseSmaCurosr(pCur); - tdDestroyTSmaWrapper(pSW, true); + tdDestroyTSmaWrapper(pSW); + tfree(pSW); return NULL; } continue; diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index 97157fc49c..d97dca8469 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -94,8 +94,8 @@ TEST(testCase, tSmaEncodeDecodeTest) { } // resource release - tdDestroyTSma(&tSma, false); - tdDestroyTSmaWrapper(&dstTSmaWrapper, false); + tdDestroyTSma(&tSma); + tdDestroyTSmaWrapper(&dstTSmaWrapper); } TEST(testCase, tSma_DB_Put_Get_Del_Test) { @@ -152,14 +152,16 @@ TEST(testCase, tSma_DB_Put_Get_Del_Test) { printf("name1 = %s\n", qSmaCfg->indexName); EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName1); EXPECT_EQ(qSmaCfg->tableUid, tSma.tableUid); - tdDestroyTSma(qSmaCfg, true); + tdDestroyTSma(qSmaCfg); + free(qSmaCfg); qSmaCfg = metaGetSmaInfoByName(pMeta, smaIndexName2); assert(qSmaCfg != NULL); printf("name2 = %s\n", qSmaCfg->indexName); EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName2); EXPECT_EQ(qSmaCfg->interval, tSma.interval); - tdDestroyTSma(qSmaCfg, true); + tdDestroyTSma(qSmaCfg); + free(qSmaCfg); // get index name by table uid SMSmaCursor *pSmaCur = metaOpenSmaCursor(pMeta, tbUid); @@ -189,7 +191,7 @@ TEST(testCase, tSma_DB_Put_Get_Del_Test) { metaRemoveSmaFromDb(pMeta, smaIndexName1); metaRemoveSmaFromDb(pMeta, smaIndexName2); - tdDestroyTSma(&tSma, false); + tdDestroyTSma(&tSma); metaClose(pMeta); }